diff --git a/.circleci/config.yml b/.circleci/config.yml index a518628afb9..0adfd5be529 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3496,8 +3496,13 @@ jobs: command: | npx playwright test e2e_ui_tests/ --reporter=html --output=test-results no_output_timeout: 120m - - store_test_results: + - store_artifacts: path: test-results + destination: playwright-results + + - store_artifacts: + path: playwright-report + destination: playwright-report test_nonroot_image: machine: diff --git a/Dockerfile b/Dockerfile index f75706805e0..d8397ec4811 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base # Builder stage FROM $LITELLM_BUILD_IMAGE AS builder @@ -12,11 +12,9 @@ WORKDIR /app USER root # Install build dependencies -RUN apk add --no-cache gcc python3-dev openssl openssl-dev +RUN apk add --no-cache bash gcc py3-pip python3 python3-dev openssl openssl-dev - -RUN pip install --upgrade pip>=24.3.1 && \ - pip install build +RUN python -m pip install build # Copy the current directory contents into the container at /app COPY . . @@ -48,10 +46,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root # Install runtime dependencies -RUN apk add --no-cache openssl tzdata nodejs npm - -# Upgrade pip to fix CVE-2025-8869 -RUN pip install --upgrade pip>=24.3.1 +RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip WORKDIR /app # Copy the current directory contents into the container at /app diff --git a/README.md b/README.md index 20483ce70c0..9fed1c6dbc7 100644 --- a/README.md +++ b/README.md @@ -274,8 +274,6 @@ echo 'LITELLM_MASTER_KEY="sk-1234"' > .env # password generator to get a random hash for litellm salt key echo 'LITELLM_SALT_KEY="sk-1234"' >> .env -source .env - # Start docker compose up ``` @@ -348,7 +346,7 @@ curl 'http://0.0.0.0:4000/key/generate' \ | [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | | | [FriendliAI (`friendliai`)](https://docs.litellm.ai/docs/providers/friendliai) | ✅ | ✅ | ✅ | | | | | | | | | [Galadriel (`galadriel`)](https://docs.litellm.ai/docs/providers/galadriel) | ✅ | ✅ | ✅ | | | | | | | | -| [GitHub Copilot (`github_copilot`)](https://docs.litellm.ai/docs/providers/github_copilot) | ✅ | ✅ | ✅ | | | | | | | | +| [GitHub Copilot (`github_copilot`)](https://docs.litellm.ai/docs/providers/github_copilot) | ✅ | ✅ | ✅ | ✅ | | | | | | | | [GitHub Models (`github`)](https://docs.litellm.ai/docs/providers/github) | ✅ | ✅ | ✅ | | | | | | | | | [Google - PaLM](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ | | | | | | | | | [Google - Vertex AI (`vertex_ai`)](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh index 7b2a76a85a6..6950880320b 100755 --- a/ci_cd/security_scans.sh +++ b/ci_cd/security_scans.sh @@ -76,6 +76,8 @@ run_grype_scans() { "GHSA-4xh5-x5gv-qwph" "CVE-2025-8291" # no fix available as of Oct 11, 2025 "GHSA-5j98-mcp5-4vw2" + "CVE-2025-13836" # Python 3.13 HTTP response reading OOM/DoS - no fix available in base image + "CVE-2025-12084" # Python 3.13 xml.dom.minidom quadratic algorithm - no fix available in base image ) # Build JSON array of allowlisted CVE IDs for jq diff --git a/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md b/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md index a12da32f1d0..ab2cf334459 100644 --- a/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md +++ b/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md @@ -404,6 +404,93 @@ This release has a known issue... - **New Providers** - Provider name, supported endpoints, description - **New LLM API Endpoints** (optional) - Endpoint, method, description, documentation link - Only include major new provider integrations, not minor provider updates +- **IMPORTANT**: When adding new providers, also update `provider_endpoints_support.json` in the repository root (see Section 13) + +### 12. Section Header Counts + +**Always include counts in section headers for:** +- **New Providers** - Add count in parentheses: `### New Providers (X new providers)` +- **New LLM API Endpoints** - Add count in parentheses: `### New LLM API Endpoints (X new endpoints)` +- **New Model Support** - Add count in parentheses: `#### New Model Support (X new models)` + +**Format:** +```markdown +### New Providers (4 new providers) + +| Provider | Supported LiteLLM Endpoints | Description | +| -------- | --------------------------- | ----------- | +... + +### New LLM API Endpoints (2 new endpoints) + +| Endpoint | Method | Description | Documentation | +| -------- | ------ | ----------- | ------------- | +... + +#### New Model Support (32 new models) + +| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features | +| -------- | ----- | -------------- | ------------------- | -------------------- | -------- | +... +``` + +**Counting Rules:** +- Count each row in the table (excluding the header row) +- For models, count each model entry in the pricing table +- For providers, count each new provider added +- For endpoints, count each new API endpoint added + +### 13. Update provider_endpoints_support.json + +**When adding new providers or endpoints, you MUST also update `provider_endpoints_support.json` in the repository root.** + +This file tracks which endpoints are supported by each LiteLLM provider and is used to generate documentation. + +**Required Steps:** +1. For each new provider added to the release notes, add a corresponding entry to `provider_endpoints_support.json` +2. For each new endpoint type added, update the schema comment and add the endpoint to relevant providers + +**Provider Entry Format:** +```json +"provider_slug": { + "display_name": "Provider Name (`provider_slug`)", + "url": "https://docs.litellm.ai/docs/providers/provider_slug", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true + } +} +``` + +**Available Endpoint Types:** +- `chat_completions` - `/chat/completions` endpoint +- `messages` - `/messages` endpoint (Anthropic format) +- `responses` - `/responses` endpoint (OpenAI/Anthropic unified) +- `embeddings` - `/embeddings` endpoint +- `image_generations` - `/image/generations` endpoint +- `audio_transcriptions` - `/audio/transcriptions` endpoint +- `audio_speech` - `/audio/speech` endpoint +- `moderations` - `/moderations` endpoint +- `batches` - `/batches` endpoint +- `rerank` - `/rerank` endpoint +- `ocr` - `/ocr` endpoint +- `search` - `/search` endpoint +- `vector_stores` - `/vector_stores` endpoint +- `a2a` - `/a2a/{agent}/message/send` endpoint (A2A Protocol) + +**Checklist:** +- [ ] All new providers from release notes are added to `provider_endpoints_support.json` +- [ ] Endpoint support flags accurately reflect provider capabilities +- [ ] Documentation URL points to correct provider docs page ## Example Command Workflow diff --git a/cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py b/cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py new file mode 100644 index 00000000000..7bf9cc32484 --- /dev/null +++ b/cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py @@ -0,0 +1,540 @@ +#!/usr/bin/env python3 +""" +Mock Bedrock Guardrail API Server + +This is a FastAPI server that mimics the AWS Bedrock Guardrail API for testing purposes. +It follows the same API spec as the real Bedrock guardrail endpoint. + +Usage: + python mock_bedrock_guardrail_server.py + +The server will start on http://localhost:8080 +""" + +import os +import re +from typing import Any, Dict, List, Literal, Optional + +from fastapi import Depends, FastAPI, Header, HTTPException, status +from fastapi.responses import JSONResponse +from pydantic import BaseModel, Field + +# ============================================================================ +# Request/Response Models (matching Bedrock API spec) +# ============================================================================ + + +class BedrockTextContent(BaseModel): + text: str + + +class BedrockContentItem(BaseModel): + text: BedrockTextContent + + +class BedrockRequest(BaseModel): + source: Literal["INPUT", "OUTPUT"] + content: List[BedrockContentItem] = Field(default_factory=list) + + +class BedrockGuardrailOutput(BaseModel): + text: Optional[str] = None + + +class TopicPolicyItem(BaseModel): + name: str + type: str + action: Literal["BLOCKED", "NONE"] + + +class TopicPolicy(BaseModel): + topics: List[TopicPolicyItem] = Field(default_factory=list) + + +class ContentFilterItem(BaseModel): + type: str + confidence: str + action: Literal["BLOCKED", "NONE"] + + +class ContentPolicy(BaseModel): + filters: List[ContentFilterItem] = Field(default_factory=list) + + +class CustomWord(BaseModel): + match: str + action: Literal["BLOCKED", "NONE"] + + +class WordPolicy(BaseModel): + customWords: List[CustomWord] = Field(default_factory=list) + managedWordLists: List[Dict[str, Any]] = Field(default_factory=list) + + +class PiiEntity(BaseModel): + type: str + match: str + action: Literal["BLOCKED", "ANONYMIZED", "NONE"] + + +class RegexMatch(BaseModel): + name: str + match: str + regex: str + action: Literal["BLOCKED", "ANONYMIZED", "NONE"] + + +class SensitiveInformationPolicy(BaseModel): + piiEntities: List[PiiEntity] = Field(default_factory=list) + regexes: List[RegexMatch] = Field(default_factory=list) + + +class ContextualGroundingFilter(BaseModel): + type: str + threshold: float + score: float + action: Literal["BLOCKED", "NONE"] + + +class ContextualGroundingPolicy(BaseModel): + filters: List[ContextualGroundingFilter] = Field(default_factory=list) + + +class Assessment(BaseModel): + topicPolicy: Optional[TopicPolicy] = None + contentPolicy: Optional[ContentPolicy] = None + wordPolicy: Optional[WordPolicy] = None + sensitiveInformationPolicy: Optional[SensitiveInformationPolicy] = None + contextualGroundingPolicy: Optional[ContextualGroundingPolicy] = None + + +class BedrockGuardrailResponse(BaseModel): + usage: Dict[str, int] = Field( + default_factory=lambda: {"topicPolicyUnits": 1, "contentPolicyUnits": 1} + ) + action: Literal["NONE", "GUARDRAIL_INTERVENED"] = "NONE" + outputs: List[BedrockGuardrailOutput] = Field(default_factory=list) + assessments: List[Assessment] = Field(default_factory=list) + + +# ============================================================================ +# Mock Guardrail Configuration +# ============================================================================ + + +class GuardrailConfig(BaseModel): + """Configuration for mock guardrail behavior""" + + blocked_words: List[str] = Field( + default_factory=lambda: ["offensive", "inappropriate", "badword"] + ) + blocked_topics: List[str] = Field(default_factory=lambda: ["violence", "illegal"]) + pii_patterns: Dict[str, str] = Field( + default_factory=lambda: { + "EMAIL": r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", + "PHONE": r"\b\d{3}[-.]?\d{3}[-.]?\d{4}\b", + "SSN": r"\b\d{3}-\d{2}-\d{4}\b", + "CREDIT_CARD": r"\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b", + } + ) + anonymize_pii: bool = True # If True, ANONYMIZE PII; if False, BLOCK it + bearer_token: str = "mock-bedrock-token-12345" + + +# Global config +GUARDRAIL_CONFIG = GuardrailConfig() + +# ============================================================================ +# FastAPI App Setup +# ============================================================================ + +app = FastAPI( + title="Mock Bedrock Guardrail API", + description="Mock server mimicking AWS Bedrock Guardrail API", + version="1.0.0", +) + + +# ============================================================================ +# Authentication +# ============================================================================ + + +async def verify_bearer_token(authorization: Optional[str] = Header(None)) -> str: + """ + Verify the Bearer token from the Authorization header. + + Args: + authorization: The Authorization header value + + Returns: + The token if valid + + Raises: + HTTPException: If token is missing or invalid + """ + if authorization is None: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Missing Authorization header", + headers={"WWW-Authenticate": "Bearer"}, + ) + + # Check if it's a Bearer token + parts = authorization.split() + print(f"parts: {parts}") + if len(parts) != 2 or parts[0].lower() != "bearer": + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid Authorization header format. Expected: Bearer ", + headers={"WWW-Authenticate": "Bearer"}, + ) + + token = parts[1] + + # Verify token + if token != GUARDRAIL_CONFIG.bearer_token: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid bearer token", + ) + + return token + + +# ============================================================================ +# Guardrail Logic +# ============================================================================ + + +def check_blocked_words(text: str) -> Optional[WordPolicy]: + """Check if text contains blocked words""" + found_words = [] + text_lower = text.lower() + + for word in GUARDRAIL_CONFIG.blocked_words: + if word.lower() in text_lower: + found_words.append(CustomWord(match=word, action="BLOCKED")) + + if found_words: + return WordPolicy(customWords=found_words) + return None + + +def check_blocked_topics(text: str) -> Optional[TopicPolicy]: + """Check if text contains blocked topics""" + found_topics = [] + text_lower = text.lower() + + for topic in GUARDRAIL_CONFIG.blocked_topics: + if topic.lower() in text_lower: + found_topics.append( + TopicPolicyItem(name=topic, type=topic.upper(), action="BLOCKED") + ) + + if found_topics: + return TopicPolicy(topics=found_topics) + return None + + +def check_pii(text: str) -> tuple[Optional[SensitiveInformationPolicy], str]: + """ + Check for PII in text and return policy + anonymized text + + Returns: + Tuple of (SensitiveInformationPolicy or None, anonymized_text) + """ + pii_entities = [] + anonymized_text = text + action = "ANONYMIZED" if GUARDRAIL_CONFIG.anonymize_pii else "BLOCKED" + + for pii_type, pattern in GUARDRAIL_CONFIG.pii_patterns.items(): + try: + # Compile the regex pattern with a timeout to prevent ReDoS attacks + compiled_pattern = re.compile(pattern) + matches = compiled_pattern.finditer(text) + for match in matches: + matched_text = match.group() + pii_entities.append( + PiiEntity(type=pii_type, match=matched_text, action=action) + ) + + # Anonymize the text if configured + if GUARDRAIL_CONFIG.anonymize_pii: + anonymized_text = anonymized_text.replace( + matched_text, f"[{pii_type}_REDACTED]" + ) + except re.error: + # Invalid regex pattern - skip it and log a warning + print(f"Warning: Invalid regex pattern for PII type {pii_type}: {pattern}") + continue + + if pii_entities: + return SensitiveInformationPolicy(piiEntities=pii_entities), anonymized_text + + return None, text + + +def process_guardrail_request( + request: BedrockRequest, +) -> tuple[BedrockGuardrailResponse, List[str]]: + """ + Process a guardrail request and return the response. + + Returns: + Tuple of (response, list of output texts) + """ + all_text_content = [] + output_texts = [] + + # Extract all text from content items + for content_item in request.content: + if content_item.text and content_item.text.text: + all_text_content.append(content_item.text.text) + + # Combine all text for analysis + combined_text = " ".join(all_text_content) + + # Initialize response + response = BedrockGuardrailResponse() + assessment = Assessment() + has_intervention = False + + # Check for blocked words + word_policy = check_blocked_words(combined_text) + if word_policy: + assessment.wordPolicy = word_policy + has_intervention = True + + # Check for blocked topics + topic_policy = check_blocked_topics(combined_text) + if topic_policy: + assessment.topicPolicy = topic_policy + has_intervention = True + + # Check for PII + for text in all_text_content: + pii_policy, anonymized_text = check_pii(text) + if pii_policy: + assessment.sensitiveInformationPolicy = pii_policy + if GUARDRAIL_CONFIG.anonymize_pii: + # If anonymizing, we don't block, we modify the text + output_texts.append(anonymized_text) + has_intervention = True + else: + # If not anonymizing PII, we block it + output_texts.append(text) + has_intervention = True + else: + output_texts.append(text) + + # Build response + if has_intervention: + response.action = "GUARDRAIL_INTERVENED" + # Only add assessment if there were interventions + response.assessments = [assessment] + + # Add outputs (modified or original text) + response.outputs = [BedrockGuardrailOutput(text=txt) for txt in output_texts] + + return response, output_texts + + +# ============================================================================ +# API Endpoints +# ============================================================================ + + +@app.get("/") +async def root(): + """Health check endpoint""" + return { + "service": "Mock Bedrock Guardrail API", + "status": "running", + "endpoint_format": "/guardrail/{guardrailIdentifier}/version/{guardrailVersion}/apply", + } + + +@app.get("/health") +async def health(): + """Health check endpoint""" + return {"status": "healthy"} + + +""" +LiteLLM exposes a basic guardrail API with the text extracted from the request and sent to the guardrail API, as well as the received request body for any further processing. + +This works across all LiteLLM endpoints (completion, anthropic /v1/messages, responses api, image generation, embedding, etc.) + +This makes it easy to support your own guardrail API without having to make a PR to LiteLLM. + +LiteLLM supports passing any provider specific params from LiteLLM config.yaml to the guardrail API. + +Example: + +```yaml +guardrails: + - guardrail_name: "bedrock-content-guard" + litellm_params: + guardrail: generic_guardrail_api + mode: "pre_call" + api_key: os.environ/GUARDRAIL_API_KEY + api_base: os.environ/GUARDRAIL_API_BASE + additional_provider_specific_params: + api_version: os.environ/GUARDRAIL_API_VERSION # additional provider specific params +``` + +This is a beta API. Please help us improve it. +""" + + +class LitellmBasicGuardrailRequest(BaseModel): + texts: List[str] + images: Optional[List[str]] = None + tools: Optional[List[dict]] = None + tool_calls: Optional[List[dict]] = None + request_data: Dict[str, Any] = Field(default_factory=dict) + additional_provider_specific_params: Dict[str, Any] = Field(default_factory=dict) + input_type: Literal["request", "response"] + litellm_call_id: Optional[str] = None + litellm_trace_id: Optional[str] = None + structured_messages: Optional[List[Dict[str, Any]]] = None + + +class LitellmBasicGuardrailResponse(BaseModel): + action: Literal[ + "BLOCKED", "NONE", "GUARDRAIL_INTERVENED" + ] # BLOCKED = litellm will raise an error, NONE = litellm will continue, GUARDRAIL_INTERVENED = litellm will continue, but the text was modified by the guardrail + blocked_reason: Optional[str] = None # only if action is BLOCKED, otherwise None + texts: Optional[List[str]] = None + images: Optional[List[str]] = None + + +@app.post( + "/beta/litellm_basic_guardrail_api", + response_model=LitellmBasicGuardrailResponse, +) +async def beta_litellm_basic_guardrail_api( + request: LitellmBasicGuardrailRequest, +) -> LitellmBasicGuardrailResponse: + """ + Apply guardrail to input or output content. + + This endpoint mimics the AWS Bedrock ApplyGuardrail API. + + Args: + request: The guardrail request containing content to analyze + token: Bearer token (verified by dependency) + + Returns: + LitellmBasicGuardrailResponse with analysis results + """ + print(f"request: {request}") + if any("ishaan" in text.lower() for text in request.texts): + return LitellmBasicGuardrailResponse( + action="BLOCKED", blocked_reason="Ishaan is not allowed" + ) + elif any("pii_value" in text for text in request.texts): + return LitellmBasicGuardrailResponse( + action="GUARDRAIL_INTERVENED", + texts=[ + text.replace("pii_value", "pii_value_redacted") + for text in request.texts + ], + ) + return LitellmBasicGuardrailResponse(action="NONE") + + +@app.post("/config/update") +async def update_config( + config: GuardrailConfig, token: str = Depends(verify_bearer_token) +): + """ + Update the guardrail configuration. + + This is a testing endpoint to modify the mock guardrail behavior. + + Args: + config: New guardrail configuration + token: Bearer token (verified by dependency) + + Returns: + Updated configuration + """ + global GUARDRAIL_CONFIG + GUARDRAIL_CONFIG = config + return {"status": "updated", "config": GUARDRAIL_CONFIG} + + +@app.get("/config") +async def get_config(token: str = Depends(verify_bearer_token)): + """ + Get the current guardrail configuration. + + Args: + token: Bearer token (verified by dependency) + + Returns: + Current configuration + """ + return GUARDRAIL_CONFIG + + +# ============================================================================ +# Error Handlers +# ============================================================================ + + +@app.exception_handler(HTTPException) +async def http_exception_handler(request, exc: HTTPException): + """Custom error handler for HTTP exceptions""" + return JSONResponse( + status_code=exc.status_code, + content={"error": exc.detail}, + headers=exc.headers, + ) + + +# ============================================================================ +# Main +# ============================================================================ + +if __name__ == "__main__": + import uvicorn + + # Get configuration from environment + host = os.getenv("MOCK_BEDROCK_HOST", "0.0.0.0") + port = int(os.getenv("MOCK_BEDROCK_PORT", "8080")) + bearer_token = os.getenv("MOCK_BEDROCK_TOKEN", "mock-bedrock-token-12345") + + # Update config with environment token + GUARDRAIL_CONFIG.bearer_token = bearer_token + + print("=" * 80) + print("Mock Bedrock Guardrail API Server") + print("=" * 80) + print(f"Server starting on: http://{host}:{port}") + print(f"Bearer Token: {bearer_token}") + print(f"Endpoint: POST /guardrail/{{id}}/version/{{version}}/apply") + print("=" * 80) + print("\nExample curl command:") + print( + f""" +curl -X POST "http://{host}:{port}/guardrail/test-guardrail/version/1/apply" \\ + -H "Authorization: Bearer {bearer_token}" \\ + -H "Content-Type: application/json" \\ + -d '{{ + "source": "INPUT", + "content": [ + {{ + "text": {{ + "text": "Hello, my email is test@example.com" + }} + }} + ] + }}' + """ + ) + print("=" * 80) + + uvicorn.run(app, host=host, port=port) diff --git a/deploy/charts/litellm-helm/Chart.yaml b/deploy/charts/litellm-helm/Chart.yaml index eedadebaa8e..b77693ba8d5 100644 --- a/deploy/charts/litellm-helm/Chart.yaml +++ b/deploy/charts/litellm-helm/Chart.yaml @@ -18,7 +18,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.8 +version: 0.4.10 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -33,5 +33,5 @@ dependencies: condition: db.deployStandalone - name: redis version: ">=18.0.0" - repository: oci://registry-1.docker.io/bitnamicharts + repository: oci://registry-1.docker.io/bitnamicharts condition: redis.enabled diff --git a/deploy/charts/litellm-helm/README.md b/deploy/charts/litellm-helm/README.md index 352c3e9ddff..6fdc423a177 100644 --- a/deploy/charts/litellm-helm/README.md +++ b/deploy/charts/litellm-helm/README.md @@ -10,46 +10,48 @@ - Helm 3.8.0+ If `db.deployStandalone` is used: + - PV provisioner support in the underlying infrastructure If `db.useStackgresOperator` is used (not yet implemented): -- The Stackgres Operator must already be installed in the Kubernetes Cluster. This chart will **not** install the operator if it is missing. + +- The Stackgres Operator must already be installed in the Kubernetes Cluster. This chart will **not** install the operator if it is missing. ## Parameters ### LiteLLM Proxy Deployment Settings -| Name | Description | Value | -| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | -| `replicaCount` | The number of LiteLLM Proxy pods to be deployed | `1` | -| `masterkeySecretName` | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name. | N/A | -| `masterkeySecretKey` | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key. | N/A | -| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated. | N/A | -| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | -| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | -| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` | -| `image.pullPolicy` | LiteLLM Proxy image pull policy | `IfNotPresent` | -| `image.tag` | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published. | `""` | -| `imagePullSecrets` | Registry credentials for the LiteLLM and initContainer images. | `[]` | -| `serviceAccount.create` | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API. | `false` | -| `service.type` | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.) | `ClusterIP` | -| `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` | -| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` | -| `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A | -| `proxyConfigMap.create` | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it. | `true` | -| `proxyConfigMap.name` | When `create=false`, name of the existing ConfigMap to mount. | `""` | -| `proxyConfigMap.key` | Key in the ConfigMap that contains the proxy config file. | `"config.yaml"` | -| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` | -| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy. -| `pdb.enabled` | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment | `false` | -| `pdb.minAvailable` | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` | -| `pdb.maxUnavailable` | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` | -| `pdb.annotations` | Extra metadata annotations to add to the PDB | `{}` | -| `pdb.labels` | Extra metadata labels to add to the PDB | `{}` | +| Name | Description | Value | +| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | +| `replicaCount` | The number of LiteLLM Proxy pods to be deployed | `1` | +| `masterkeySecretName` | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name. | N/A | +| `masterkeySecretKey` | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key. | N/A | +| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated. | N/A | +| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | +| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | +| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` | +| `image.pullPolicy` | LiteLLM Proxy image pull policy | `IfNotPresent` | +| `image.tag` | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published. | `""` | +| `imagePullSecrets` | Registry credentials for the LiteLLM and initContainer images. | `[]` | +| `serviceAccount.create` | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API. | `false` | +| `service.type` | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.) | `ClusterIP` | +| `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` | +| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` | +| `ingress.labels` | Additional labels for the Ingress resource | `{}` | +| `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A | +| `proxyConfigMap.create` | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it. | `true` | +| `proxyConfigMap.name` | When `create=false`, name of the existing ConfigMap to mount. | `""` | +| `proxyConfigMap.key` | Key in the ConfigMap that contains the proxy config file. | `"config.yaml"` | +| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` | +| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy. | +| `pdb.enabled` | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment | `false` | +| `pdb.minAvailable` | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` | +| `pdb.maxUnavailable` | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` | +| `pdb.annotations` | Extra metadata annotations to add to the PDB | `{}` | +| `pdb.labels` | Extra metadata labels to add to the PDB | `{}` | #### Example `proxy_config` ConfigMap from values (default): - ``` proxyConfigMap: create: true @@ -67,7 +69,6 @@ proxy_config: #### Example using existing `proxyConfigMap` instead of creating it: - ``` proxyConfigMap: create: false @@ -77,8 +78,7 @@ proxyConfigMap: # proxy_config is ignored in this mode ``` -#### Example `environmentSecrets` Secret - +#### Example `environmentSecrets` Secret ``` apiVersion: v1 @@ -91,21 +91,23 @@ type: Opaque ``` ### Database Settings -| Name | Description | Value | -| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | -| `db.useExisting` | Use an existing Postgres database. A Kubernetes Secret object must exist that contains credentials for connecting to the database. An example secret object definition is provided below. | `false` | -| `db.endpoint` | If `db.useExisting` is `true`, this is the IP, Hostname or Service Name of the Postgres server to connect to. | `localhost` | -| `db.database` | If `db.useExisting` is `true`, the name of the existing database to connect to. | `litellm` | -| `db.url` | If `db.useExisting` is `true`, the connection url of the existing database to connect to can be overwritten with this value. | `postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)` | -| `db.secret.name` | If `db.useExisting` is `true`, the name of the Kubernetes Secret that contains credentials. | `postgres` | -| `db.secret.usernameKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the username for authenticating with the Postgres instance. | `username` | -| `db.secret.passwordKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the password associates with the above user. | `password` | -| `db.useStackgresOperator` | Not yet implemented. | `false` | -| `db.deployStandalone` | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart. This is useful for getting started but doesn't provide HA or (by default) data backups. | `true` | -| `postgresql.*` | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart. See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details. See [values.yaml](./values.yaml) for the default configuration. | See [values.yaml](./values.yaml) | -| `postgresql.auth.*` | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used. | `NoTaGrEaTpAsSwOrD` | + +| Name | Description | Value | +| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | +| `db.useExisting` | Use an existing Postgres database. A Kubernetes Secret object must exist that contains credentials for connecting to the database. An example secret object definition is provided below. | `false` | +| `db.endpoint` | If `db.useExisting` is `true`, this is the IP, Hostname or Service Name of the Postgres server to connect to. | `localhost` | +| `db.database` | If `db.useExisting` is `true`, the name of the existing database to connect to. | `litellm` | +| `db.url` | If `db.useExisting` is `true`, the connection url of the existing database to connect to can be overwritten with this value. | `postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)` | +| `db.secret.name` | If `db.useExisting` is `true`, the name of the Kubernetes Secret that contains credentials. | `postgres` | +| `db.secret.usernameKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the username for authenticating with the Postgres instance. | `username` | +| `db.secret.passwordKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the password associates with the above user. | `password` | +| `db.useStackgresOperator` | Not yet implemented. | `false` | +| `db.deployStandalone` | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart. This is useful for getting started but doesn't provide HA or (by default) data backups. | `true` | +| `postgresql.*` | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart. See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details. See [values.yaml](./values.yaml) for the default configuration. | See [values.yaml](./values.yaml) | +| `postgresql.auth.*` | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used. | `NoTaGrEaTpAsSwOrD` | #### Example Postgres `db.useExisting` Secret + ```yaml apiVersion: v1 kind: Secret @@ -143,7 +145,7 @@ metadata: name: litellm-env-secret type: Opaque data: - SOME_PASSWORD: cDZbUGVXeU5e0ZW # base64 encoded + SOME_PASSWORD: cDZbUGVXeU5e0ZW # base64 encoded ANOTHER_PASSWORD: AAZbUGVXeU5e0ZB # base64 encoded ``` @@ -153,23 +155,23 @@ Source: [GitHub Gist from troyharvey](https://gist.github.com/troyharvey/4506472 The migration job supports both ArgoCD and Helm hooks to ensure database migrations run at the appropriate time during deployments. -| Name | Description | Value | -| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | -| `migrationJob.enabled` | Enable or disable the schema migration Job | `true` | -| `migrationJob.backoffLimit` | Backoff limit for Job restarts | `4` | -| `migrationJob.ttlSecondsAfterFinished` | TTL for completed migration jobs | `120` | -| `migrationJob.annotations` | Additional annotations for the migration job pod | `{}` | -| `migrationJob.extraContainers` | Additional containers to run alongside the migration job | `[]` | -| `migrationJob.hooks.argocd.enabled` | Enable ArgoCD hooks for the migration job (uses PreSync hook with BeforeHookCreation delete policy) | `true` | -| `migrationJob.hooks.helm.enabled` | Enable Helm hooks for the migration job (uses pre-install,pre-upgrade hooks with before-hook-creation delete policy) | `false` | -| `migrationJob.hooks.helm.weight` | Helm hook execution order (lower weights executed first). Optional - defaults to "1" if not specified. | N/A | - +| Name | Description | Value | +| -------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------- | +| `migrationJob.enabled` | Enable or disable the schema migration Job | `true` | +| `migrationJob.backoffLimit` | Backoff limit for Job restarts | `4` | +| `migrationJob.ttlSecondsAfterFinished` | TTL for completed migration jobs | `120` | +| `migrationJob.annotations` | Additional annotations for the migration job pod | `{}` | +| `migrationJob.extraContainers` | Additional containers to run alongside the migration job | `[]` | +| `migrationJob.hooks.argocd.enabled` | Enable ArgoCD hooks for the migration job (uses PreSync hook with BeforeHookCreation delete policy) | `true` | +| `migrationJob.hooks.helm.enabled` | Enable Helm hooks for the migration job (uses pre-install,pre-upgrade hooks with before-hook-creation delete policy) | `false` | +| `migrationJob.hooks.helm.weight` | Helm hook execution order (lower weights executed first). Optional - defaults to "1" if not specified. | N/A | ## Accessing the Admin UI + When browsing to the URL published per the settings in `ingress.*`, you will -be prompted for **Admin Configuration**. The **Proxy Endpoint** is the internal +be prompted for **Admin Configuration**. The **Proxy Endpoint** is the internal (from the `litellm` pod's perspective) URL published by the `-litellm` -Kubernetes Service. If the deployment uses the default settings for this +Kubernetes Service. If the deployment uses the default settings for this service, the **Proxy Endpoint** should be set to `http://-litellm:4000`. The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey` @@ -181,7 +183,8 @@ kubectl -n litellm get secret -litellm-masterkey -o jsonpath="{.data.ma ``` ## Admin UI Limitations -At the time of writing, the Admin UI is unable to add models. This is because + +At the time of writing, the Admin UI is unable to add models. This is because it would need to update the `config.yaml` file which is a exposed ConfigMap, and -therefore, read-only. This is a limitation of this helm chart, not the Admin UI +therefore, read-only. This is a limitation of this helm chart, not the Admin UI itself. diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index 316323be99a..0dab2ec40e0 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -6,6 +6,9 @@ metadata: name: {{ include "litellm.fullname" . }} labels: {{- include "litellm.labels" . | nindent 4 }} + {{- if .Values.deploymentLabels }} + {{- toYaml .Values.deploymentLabels | nindent 4 }} + {{- end }} spec: {{- if not .Values.autoscaling.enabled }} replicas: {{ .Values.replicaCount }} @@ -126,6 +129,12 @@ spec: - configMapRef: name: {{ . }} {{- end }} + {{- if .Values.command }} + command: {{ toYaml .Values.command | nindent 12 }} + {{- end }} + {{- if .Values.args }} + args: {{ toYaml .Values.args | nindent 12 }} + {{- else }} args: - --config - /etc/litellm/config.yaml @@ -133,6 +142,7 @@ spec: - --num_workers - {{ .Values.numWorkers | quote }} {{- end }} + {{- end }} ports: - name: http containerPort: {{ .Values.service.port }} diff --git a/deploy/charts/litellm-helm/templates/ingress.yaml b/deploy/charts/litellm-helm/templates/ingress.yaml index 09e8d715ab8..ea9ffcbb54c 100644 --- a/deploy/charts/litellm-helm/templates/ingress.yaml +++ b/deploy/charts/litellm-helm/templates/ingress.yaml @@ -18,6 +18,9 @@ metadata: name: {{ $fullName }} labels: {{- include "litellm.labels" . | nindent 4 }} + {{- with .Values.ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .Values.ingress.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/deploy/charts/litellm-helm/tests/deployment_command_args_labels_tests.yaml b/deploy/charts/litellm-helm/tests/deployment_command_args_labels_tests.yaml new file mode 100644 index 00000000000..6b0d45ebf48 --- /dev/null +++ b/deploy/charts/litellm-helm/tests/deployment_command_args_labels_tests.yaml @@ -0,0 +1,68 @@ +suite: test deployment command, args, and deploymentLabels +templates: + - deployment.yaml + - configmap-litellm.yaml +tests: + - it: should override args when custom args specified + template: deployment.yaml + set: + args: + - --custom-arg1 + - value1 + - --custom-arg2 + asserts: + - equal: + path: spec.template.spec.containers[0].args + value: + - --custom-arg1 + - value1 + - --custom-arg2 + - it: should set custom command when specified + template: deployment.yaml + set: + command: + - /bin/sh + - -c + asserts: + - equal: + path: spec.template.spec.containers[0].command + value: + - /bin/sh + - -c + - it: should set custom command and args together + template: deployment.yaml + set: + command: + - python + - -u + args: + - my_script.py + - --verbose + asserts: + - equal: + path: spec.template.spec.containers[0].command + value: + - python + - -u + - equal: + path: spec.template.spec.containers[0].args + value: + - my_script.py + - --verbose + - it: should add deploymentLabels to deployment metadata + template: deployment.yaml + set: + deploymentLabels: + environment: production + team: platform + version: v1.2.3 + asserts: + - equal: + path: metadata.labels.environment + value: production + - equal: + path: metadata.labels.team + value: platform + - equal: + path: metadata.labels.version + value: v1.2.3 diff --git a/deploy/charts/litellm-helm/tests/ingress_tests.yaml b/deploy/charts/litellm-helm/tests/ingress_tests.yaml new file mode 100644 index 00000000000..aad6ecfcee8 --- /dev/null +++ b/deploy/charts/litellm-helm/tests/ingress_tests.yaml @@ -0,0 +1,45 @@ +suite: Ingress Configuration Tests +templates: + - ingress.yaml +tests: + - it: should not create Ingress by default + asserts: + - hasDocuments: + count: 0 + + - it: should create Ingress when enabled + set: + ingress.enabled: true + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Ingress + + - it: should add custom labels + set: + ingress.enabled: true + ingress.labels: + custom-label: "true" + another-label: "value" + asserts: + - isKind: + of: Ingress + - equal: + path: metadata.labels.custom-label + value: "true" + - equal: + path: metadata.labels.another-label + value: "value" + + - it: should add annotations + set: + ingress.enabled: true + ingress.annotations: + kubernetes.io/ingress.class: "nginx" + asserts: + - isKind: + of: Ingress + - equal: + path: metadata.annotations["kubernetes.io/ingress.class"] + value: "nginx" diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index acb8c9ca32f..3a351d7b862 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -30,12 +30,14 @@ serviceAccount: # annotations for litellm deployment deploymentAnnotations: {} +deploymentLabels: {} # annotations for litellm pods podAnnotations: {} podLabels: {} terminationGracePeriodSeconds: 90 -topologySpreadConstraints: [] +topologySpreadConstraints: + [] # - maxSkew: 1 # topologyKey: kubernetes.io/hostname # whenUnsatisfiable: DoNotSchedule @@ -46,7 +48,8 @@ topologySpreadConstraints: [] # At the time of writing, the litellm docker image requires write access to the # filesystem on startup so that prisma can install some dependencies. podSecurityContext: {} -securityContext: {} +securityContext: + {} # capabilities: # drop: # - ALL @@ -57,13 +60,15 @@ securityContext: {} # A list of Kubernetes Secret objects that will be exported to the LiteLLM proxy # pod as environment variables. These secrets can then be referenced in the # configuration file (or "litellm" ConfigMap) with `os.environ/` -environmentSecrets: [] +environmentSecrets: + [] # - litellm-env-secret # A list of Kubernetes ConfigMap objects that will be exported to the LiteLLM proxy # pod as environment variables. The ConfigMap kv-pairs can then be referenced in the # configuration file (or "litellm" ConfigMap) with `os.environ/` -environmentConfigMaps: [] +environmentConfigMaps: + [] # - litellm-env-configmap service: @@ -82,7 +87,9 @@ separateHealthPort: 8081 ingress: enabled: false className: "nginx" - annotations: {} + labels: {} + annotations: + {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" hosts: @@ -129,7 +136,8 @@ proxy_config: general_settings: master_key: os.environ/PROXY_MASTER_KEY -resources: {} +resources: + {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following @@ -231,7 +239,7 @@ migrationJob: # cpu: 100m # memory: 100Mi extraContainers: [] - + # Hook configuration hooks: argocd: @@ -240,30 +248,35 @@ migrationJob: enabled: false # Additional environment variables to be added to the deployment as a map of key-value pairs -envVars: { - # USE_DDTRACE: "true" -} +envVars: {} +# USE_DDTRACE: "true" # Additional environment variables to be added to the deployment as a list of k8s env vars -extraEnvVars: { - # - name: EXTRA_ENV_VAR - # value: EXTRA_ENV_VAR_VALUE -} +extraEnvVars: {} +# if you want to override the container command, you can do so here +command: {} +# if you want to override the container args, you can do so here +args: {} + +# - name: EXTRA_ENV_VAR +# value: EXTRA_ENV_VAR_VALUE # Pod Disruption Budget pdb: enabled: false # Set exactly one of the following. If both are set, minAvailable takes precedence. - minAvailable: null # e.g. "50%" or 1 - maxUnavailable: null # e.g. 1 or "20%" + minAvailable: null # e.g. "50%" or 1 + maxUnavailable: null # e.g. 1 or "20%" annotations: {} labels: {} serviceMonitor: enabled: false - labels: {} + labels: + {} # test: test - annotations: {} + annotations: + {} # kubernetes.io/test: test interval: 15s scrapeTimeout: 10s @@ -273,4 +286,4 @@ serviceMonitor: # action: replace namespaceSelector: matchNames: [] - # - test-namespace \ No newline at end of file + # - test-namespace diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 09b5265191b..0e804cbfd12 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -1,8 +1,8 @@ # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base # Builder stage FROM $LITELLM_BUILD_IMAGE AS builder @@ -13,13 +13,15 @@ USER root # Install build dependencies RUN apk add --no-cache \ - build-base \ + bash \ + gcc \ + py3-pip \ + python3 \ python3-dev \ + openssl \ openssl-dev - -RUN pip install --upgrade pip && \ - pip install build +RUN python -m pip install build # Copy the current directory contents into the container at /app COPY . . @@ -46,7 +48,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root # Install runtime dependencies -RUN apk add --no-cache openssl +RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip WORKDIR /app # Copy the current directory contents into the container at /app diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index bb656e04e5b..9fc8acf2a18 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -1,6 +1,6 @@ # Base images -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base # ----------------- # Builder Stage @@ -10,7 +10,20 @@ WORKDIR /app # Install build dependencies including Node.js for UI build USER root -RUN apk add --no-cache build-base bash nodejs npm \ +RUN for i in 1 2 3; do \ + apk add --no-cache \ + python3 \ + py3-pip \ + clang \ + llvm \ + lld \ + gcc \ + linux-headers \ + build-base \ + bash \ + nodejs \ + npm && break || sleep 5; \ + done \ && pip install --no-cache-dir --upgrade pip build # Copy project files @@ -26,7 +39,7 @@ RUN npm install -g npm@latest && npm cache clean --force RUN cd /app/ui/litellm-dashboard && \ if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \ - cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \ + cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \ fi RUN cd /app/ui/litellm-dashboard && rm -f package-lock.json @@ -40,11 +53,11 @@ RUN mkdir -p /tmp/litellm_assets && cp /app/litellm/proxy/logo.jpg /tmp/litellm_ RUN cd /tmp/litellm_ui && \ for html_file in *.html; do \ - if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \ - folder_name="${html_file%.html}" && \ - mkdir -p "$folder_name" && \ - mv "$html_file" "$folder_name/index.html"; \ - fi; \ + if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \ + folder_name="${html_file%.html}" && \ + mkdir -p "$folder_name" && \ + mv "$html_file" "$folder_name/index.html"; \ + fi; \ done RUN cd /app/ui/litellm-dashboard && rm -rf ./out @@ -62,8 +75,12 @@ WORKDIR /app # Install runtime dependencies USER root -RUN apk upgrade --no-cache && \ - apk add --no-cache bash libstdc++ ca-certificates openssl supervisor +RUN for i in 1 2 3; do \ + apk upgrade --no-cache && break || sleep 5; \ + done \ + && for i in 1 2 3; do \ + apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \ + done # Copy only necessary artifacts from builder stage for runtime COPY . . @@ -82,7 +99,7 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \ # Remove test files and keys from dependencies RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \ - find /usr/lib -type d -path "*/tornado/test" -delete + find /usr/lib -type d -path "*/tornado/test" -delete # Install semantic_router and aurelio-sdk using script RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh diff --git a/docker/build_from_pip/Dockerfile.build_from_pip b/docker/build_from_pip/Dockerfile.build_from_pip index aeb19bce21f..05236008ded 100644 --- a/docker/build_from_pip/Dockerfile.build_from_pip +++ b/docker/build_from_pip/Dockerfile.build_from_pip @@ -1,14 +1,16 @@ -FROM cgr.dev/chainguard/python:latest-dev +FROM python:3.13-alpine -USER root WORKDIR /app ENV HOME=/home/litellm ENV PATH="${HOME}/venv/bin:$PATH" # Install runtime dependencies +# Note: Using Python 3.13 for compatibility with ddtrace and other packages +# rust and cargo are required for building ddtrace from source +# musl-dev and libffi-dev are needed for some Python packages on Alpine RUN apk update && \ - apk add --no-cache gcc python3-dev openssl openssl-dev + apk add --no-cache gcc musl-dev libffi-dev openssl openssl-dev rust cargo RUN python -m venv ${HOME}/venv RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip diff --git a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md index b545e936186..1e5f968b2ca 100644 --- a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md +++ b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md @@ -33,7 +33,7 @@ This guide covers Anthropic's latest model (Claude Opus 4.5) and its advanced fe | Input Examples | Claude Opus 4.5, Sonnet 4.5 | | Effort Parameter | Claude Opus 4.5 only | -Supported Providers: [Anthropic](../../docs/providers/anthropic), [Bedrock](../../docs/providers/bedrock), [Vertex AI](../../docs/providers/vertex_partner#vertex-ai---anthropic-claude). +Supported Providers: [Anthropic](../../docs/providers/anthropic), [Bedrock](../../docs/providers/bedrock), [Vertex AI](../../docs/providers/vertex_partner#vertex-ai---anthropic-claude), [Azure AI](../../docs/providers/azure_ai). ## Usage @@ -327,6 +327,81 @@ curl --location 'http://0.0.0.0:4000/v1/messages' \ +## Usage - Azure Anthropic (Azure Foundry Claude) + +LiteLLM funnels Azure Claude deployments through the `azure_ai/` provider so Claude Opus models on Azure Foundry keep working with Tool Search, Effort, streaming, and the rest of the advanced feature set. Point `AZURE_AI_API_BASE` to `https://.services.ai.azure.com/anthropic` (LiteLLM appends `/v1/messages` automatically) and authenticate with `AZURE_AI_API_KEY` or an Azure AD token. + + + + +```python +import os +from litellm import completion + +# Configure Azure credentials +os.environ["AZURE_AI_API_KEY"] = "your-azure-ai-api-key" +os.environ["AZURE_AI_API_BASE"] = "https://my-resource.services.ai.azure.com/anthropic" + +response = completion( + model="azure_ai/claude-opus-4-1", + messages=[{"role": "user", "content": "Explain how Azure Anthropic hosts Claude Opus differently from the public Anthropic API."}], + max_tokens=1200, + temperature=0.7, + stream=True, +) + +for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="", flush=True) +``` + + + + +**1. Set environment variables** + +```bash +export AZURE_AI_API_KEY="your-azure-ai-api-key" +export AZURE_AI_API_BASE="https://my-resource.services.ai.azure.com/anthropic" +``` + +**2. Configure the proxy** + +```yaml +model_list: + - model_name: claude-4-azure + litellm_params: + model: azure_ai/claude-opus-4-1 + api_key: os.environ/AZURE_AI_API_KEY + api_base: os.environ/AZURE_AI_API_BASE +``` + +**3. Start LiteLLM** + +```bash +litellm --config /path/to/config.yaml +``` + +**4. Test the Azure Claude route** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer $LITELLM_KEY' \ + --data '{ + "model": "claude-4-azure", + "messages": [ + { + "role": "user", + "content": "How do I use Claude Opus 4 via Azure Anthropic in LiteLLM?" + } + ], + "max_tokens": 1024 + }' +``` + + + ## Tool Search {#tool-search} @@ -897,14 +972,13 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ## Effort Parameter: Control Token Usage {#effort-parameter} -Controls aspects like how much effort the model puts into its response, via `output_config={"effort": ..}`. +Control how much effort Claude puts into its response using the `reasoning_effort` parameter. This allows you to trade off between response thoroughness and token efficiency. :::info - -Soon, we will map OpenAI's `reasoning_effort` parameter to this. +LiteLLM automatically maps `reasoning_effort` to Anthropic's `output_config` format and adds the required `effort-2025-11-24` beta header for Claude Opus 4.5. ::: -Potential Values for `effort` parameter: `"high"`, `"medium"`, `"low"`. +Potential values for `reasoning_effort` parameter: `"high"`, `"medium"`, `"low"`. ### Usage Example @@ -920,7 +994,7 @@ message = "Analyze the trade-offs between microservices and monolithic architect response_high = litellm.completion( model="anthropic/claude-opus-4-5-20251101", messages=[{"role": "user", "content": message}], - output_config={"effort": "high"} + reasoning_effort="high" ) print("High effort response:") @@ -931,7 +1005,7 @@ print(f"Tokens used: {response_high.usage.completion_tokens}\n") response_medium = litellm.completion( model="anthropic/claude-opus-4-5-20251101", messages=[{"role": "user", "content": message}], - output_config={"effort": "medium"} + reasoning_effort="medium" ) print("Medium effort response:") @@ -942,7 +1016,7 @@ print(f"Tokens used: {response_medium.usage.completion_tokens}\n") response_low = litellm.completion( model="anthropic/claude-opus-4-5-20251101", messages=[{"role": "user", "content": message}], - output_config={"effort": "low"} + reasoning_effort="low" ) print("Low effort response:") @@ -987,295 +1061,9 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ "role": "user", "content": "Analyze the trade-offs between microservices and monolithic architectures" }], - "output_config": { - "effort": "high" - } + "reasoning_effort": "high" } ' ``` - - -## Cost Tracking: Monitor Tool Search Usage {#cost-tracking} - -### Understanding Tool Search Costs - -Tool search operations are tracked separately in the usage object, allowing you to monitor and optimize costs. - -It is available in the `usage` object, under `server_tool_use.tool_search_requests`. - -Anthropic charges $0.0001 per tool search request. - -### Tracking Example - - - - -```python -import litellm - -tools = [ - { - "type": "tool_search_tool_regex_20251119", - "name": "tool_search_tool_regex" - }, - # ... 100 deferred tools -] - -response = litellm.completion( - model="anthropic/claude-sonnet-4-5-20250929", - messages=[{ - "role": "user", - "content": "Find and use the weather tool for San Francisco" - }], - tools=tools -) - -# Standard token usage -print("Token Usage:") -print(f" Input tokens: {response.usage.prompt_tokens}") -print(f" Output tokens: {response.usage.completion_tokens}") -print(f" Total tokens: {response.usage.total_tokens}") - -# Tool search specific usage -if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use: - print(f"\nTool Search Usage:") - print(f" Search requests: {response.usage.server_tool_use.tool_search_requests}") - - # Calculate cost (example pricing) - input_cost = response.usage.prompt_tokens * 0.000003 # $3 per 1M tokens - output_cost = response.usage.completion_tokens * 0.000015 # $15 per 1M tokens - search_cost = response.usage.server_tool_use.tool_search_requests * 0.0001 # Example - - total_cost = input_cost + output_cost + search_cost - - print(f"\nCost Breakdown:") - print(f" Input tokens: ${input_cost:.6f}") - print(f" Output tokens: ${output_cost:.6f}") - print(f" Tool searches: ${search_cost:.6f}") - print(f" Total: ${total_cost:.6f}") -``` - - - - -1. Setup config.yaml - -```yaml -model_list: - - model_name: claude-4 - litellm_params: - model: anthropic/claude-opus-4-5-20251101 - api_key: os.environ/ANTHROPIC_API_KEY -``` - -2. Start the proxy - -```bash -litellm --config /path/to/config.yaml -``` - -3. Test it! - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ ---header 'Content-Type: application/json' \ ---header 'Authorization: Bearer $LITELLM_KEY' \ ---data ' { - "model": "claude-4", - "messages": [{ - "role": "user", - "content": "Find and use the weather tool for San Francisco" - }], - "tools": [ - { - "type": "tool_search_tool_regex_20251119", - "name": "tool_search_tool_regex" - }, - # ... 100 deferred tools - ] - } -' -``` - -Expected Response: - -```json -{ - ..., - "usage": { - ..., - "server_tool_use": { - "tool_search_requests": 1 - } - } -} -``` - - - - -### Cost Optimization Tips - -1. **Keep frequently used tools non-deferred** (3-5 tools) -2. **Use tool search for large catalogs** (10+ tools) -3. **Monitor search requests** to identify optimization opportunities -4. **Combine with effort parameter** for maximum efficiency - - ---- - -## Combining Features {#combining-features} - -### The Power of Integration - -These features work together seamlessly. Here's a real-world example combining all of them: - - - - -```python -import litellm -import json - -# Large tool catalog with search, programmatic calling, and examples -tools = [ - # Enable tool search - { - "type": "tool_search_tool_regex_20251119", - "name": "tool_search_tool_regex" - }, - # Enable programmatic calling - { - "type": "code_execution_20250825", - "name": "code_execution" - }, - # Database tool with all features - { - "type": "function", - "function": { - "name": "query_database", - "description": "Execute SQL queries against the analytics database. Returns JSON array of results.", - "parameters": { - "type": "object", - "properties": { - "sql": { - "type": "string", - "description": "SQL SELECT statement" - }, - "limit": { - "type": "integer", - "description": "Maximum rows to return" - } - }, - "required": ["sql"] - } - }, - "defer_loading": True, # Tool search - "allowed_callers": ["code_execution_20250825"], # Programmatic calling - "input_examples": [ # Input examples - { - "sql": "SELECT region, SUM(revenue) as total FROM sales GROUP BY region", - "limit": 100 - } - ] - }, - # ... 50 more tools with defer_loading -] - -# Make request with effort control -response = litellm.completion( - model="anthropic/claude-opus-4-5-20251101", - messages=[{ - "role": "user", - "content": "Analyze sales by region for the last quarter and identify top performers" - }], - tools=tools, - output_config={"effort": "medium"} # Balanced efficiency -) - -# Track comprehensive usage -print("Complete Usage Metrics:") -print(f" Input tokens: {response.usage.prompt_tokens}") -print(f" Output tokens: {response.usage.completion_tokens}") -print(f" Total tokens: {response.usage.total_tokens}") - -if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use: - print(f" Tool searches: {response.usage.server_tool_use.tool_search_requests}") - -print(f"\nResponse: {response.choices[0].message.content}") -``` - - - - -1. Setup config.yaml - -```yaml -model_list: - - model_name: claude-4 - litellm_params: - model: anthropic/claude-opus-4-5-20251101 - api_key: os.environ/ANTHROPIC_API_KEY -``` - -2. Start the proxy - -```bash -litellm --config /path/to/config.yaml -``` - -3. Test it! - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ ---header 'Content-Type: application/json' \ ---header 'Authorization: Bearer $LITELLM_KEY' \ ---data ' { - "model": "claude-4", - "messages": [{ - "role": "user", - "content": "Analyze sales by region for the last quarter and identify top performers" - }], - "tools": [ - { - "type": "tool_search_tool_regex_20251119", - "name": "tool_search_tool_regex" - }, - # ... 100 deferred tools - ], - "output_config": { - "effort": "medium" - } - } -' -``` - -Expected Response: - -```json -{ - ..., - "usage": { - ..., - "server_tool_use": { - "tool_search_requests": 1 - } - } -} -``` - - - - -### Real-World Benefits - -This combination enables: - -1. **Massive scale** - Handle 1000+ tools efficiently -2. **Low latency** - Programmatic calling reduces round trips -3. **High accuracy** - Input examples ensure correct tool usage -4. **Cost control** - Effort parameter optimizes token spend -5. **Full visibility** - Track all usage metrics - diff --git a/docs/my-website/docs/a2a.md b/docs/my-website/docs/a2a.md new file mode 100644 index 00000000000..b4aa4ed03ac --- /dev/null +++ b/docs/my-website/docs/a2a.md @@ -0,0 +1,232 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import Image from '@theme/IdealImage'; + +# Agent Gateway (A2A Protocol) - Overview + +Add A2A Agents on LiteLLM AI Gateway, Invoke agents in A2A Protocol, track request/response logs in LiteLLM Logs. Manage which Teams, Keys can access which Agents onboarded. + + + +
+
+ +| Feature | Supported | +|---------|-----------| +| Logging | ✅ | +| Load Balancing | ✅ | +| Streaming | ✅ | + +:::tip + +LiteLLM follows the [A2A (Agent-to-Agent) Protocol](https://github.com/google/A2A) for invoking agents. + +::: + +## Adding your Agent + +You can add A2A-compatible agents through the LiteLLM Admin UI. + +1. Navigate to the **Agents** tab +2. Click **Add Agent** +3. Enter the agent name (e.g., `ij-local`) and the URL of your A2A agent + + + +The URL should be the invocation URL for your A2A agent (e.g., `http://localhost:10001`). + +## Invoking your Agents + +Use the [A2A Python SDK](https://pypi.org/project/a2a/) to invoke agents through LiteLLM. + +This example shows how to: +1. **List available agents** - Query `/v1/agents` to see which agents your key can access +2. **Select an agent** - Pick an agent from the list +3. **Invoke via A2A** - Use the A2A protocol to send messages to the agent + +```python showLineNumbers title="invoke_a2a_agent.py" +from uuid import uuid4 +import httpx +import asyncio +from a2a.client import A2ACardResolver, A2AClient +from a2a.types import MessageSendParams, SendMessageRequest + +# === CONFIGURE THESE === +LITELLM_BASE_URL = "http://localhost:4000" # Your LiteLLM proxy URL +LITELLM_VIRTUAL_KEY = "sk-1234" # Your LiteLLM Virtual Key +# ======================= + +async def main(): + headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"} + + async with httpx.AsyncClient(headers=headers) as client: + # Step 1: List available agents + response = await client.get(f"{LITELLM_BASE_URL}/v1/agents") + agents = response.json() + + print("Available agents:") + for agent in agents: + print(f" - {agent['agent_name']} (ID: {agent['agent_id']})") + + if not agents: + print("No agents available for this key") + return + + # Step 2: Select an agent and invoke it + selected_agent = agents[0] + agent_id = selected_agent["agent_id"] + agent_name = selected_agent["agent_name"] + print(f"\nInvoking: {agent_name}") + + # Step 3: Use A2A protocol to invoke the agent + base_url = f"{LITELLM_BASE_URL}/a2a/{agent_id}" + resolver = A2ACardResolver(httpx_client=client, base_url=base_url) + agent_card = await resolver.get_agent_card() + a2a_client = A2AClient(httpx_client=client, agent_card=agent_card) + + request = SendMessageRequest( + id=str(uuid4()), + params=MessageSendParams( + message={ + "role": "user", + "parts": [{"kind": "text", "text": "Hello, what can you do?"}], + "messageId": uuid4().hex, + } + ), + ) + response = await a2a_client.send_message(request) + print(f"Response: {response.model_dump(mode='json', exclude_none=True, indent=4)}") + +if __name__ == "__main__": + asyncio.run(main()) +``` + +### Streaming Responses + +For streaming responses, use `send_message_streaming`: + +```python showLineNumbers title="invoke_a2a_agent_streaming.py" +from uuid import uuid4 +import httpx +import asyncio +from a2a.client import A2ACardResolver, A2AClient +from a2a.types import MessageSendParams, SendStreamingMessageRequest + +# === CONFIGURE THESE === +LITELLM_BASE_URL = "http://localhost:4000" # Your LiteLLM proxy URL +LITELLM_VIRTUAL_KEY = "sk-1234" # Your LiteLLM Virtual Key +LITELLM_AGENT_NAME = "ij-local" # Agent name registered in LiteLLM +# ======================= + +async def main(): + base_url = f"{LITELLM_BASE_URL}/a2a/{LITELLM_AGENT_NAME}" + headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"} + + async with httpx.AsyncClient(headers=headers) as httpx_client: + # Resolve agent card and create client + resolver = A2ACardResolver(httpx_client=httpx_client, base_url=base_url) + agent_card = await resolver.get_agent_card() + client = A2AClient(httpx_client=httpx_client, agent_card=agent_card) + + # Send a streaming message + request = SendStreamingMessageRequest( + id=str(uuid4()), + params=MessageSendParams( + message={ + "role": "user", + "parts": [{"kind": "text", "text": "Hello, what can you do?"}], + "messageId": uuid4().hex, + } + ), + ) + + # Stream the response + async for chunk in client.send_message_streaming(request): + print(chunk.model_dump(mode="json", exclude_none=True)) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +## Tracking Agent Logs + +After invoking an agent, you can view the request logs in the LiteLLM **Logs** tab. + +The logs show: +- **Request/Response content** sent to and received from the agent +- **User, Key, Team** information for tracking who made the request +- **Latency and cost** metrics + + + +## API Reference + +### Endpoint + +``` +POST /a2a/{agent_name}/message/send +``` + +### Authentication + +Include your LiteLLM Virtual Key in the `Authorization` header: + +``` +Authorization: Bearer sk-your-litellm-key +``` + +### Request Format + +LiteLLM follows the [A2A JSON-RPC 2.0 specification](https://github.com/google/A2A): + +```json title="Request Body" +{ + "jsonrpc": "2.0", + "id": "unique-request-id", + "method": "message/send", + "params": { + "message": { + "role": "user", + "parts": [{"kind": "text", "text": "Your message here"}], + "messageId": "unique-message-id" + } + } +} +``` + +### Response Format + +```json title="Response" +{ + "jsonrpc": "2.0", + "id": "unique-request-id", + "result": { + "kind": "task", + "id": "task-id", + "contextId": "context-id", + "status": {"state": "completed", "timestamp": "2025-01-01T00:00:00Z"}, + "artifacts": [ + { + "artifactId": "artifact-id", + "name": "response", + "parts": [{"kind": "text", "text": "Agent response here"}] + } + ] + } +} +``` + +## Agent Registry + +Want to create a central registry so your team can discover what agents are available within your company? + +Use the [AI Hub](./proxy/ai_hub) to make agents public and discoverable across your organization. This allows developers to browse available agents without needing to rebuild them. diff --git a/docs/my-website/docs/a2a_agent_permissions.md b/docs/my-website/docs/a2a_agent_permissions.md new file mode 100644 index 00000000000..93f367f43e7 --- /dev/null +++ b/docs/my-website/docs/a2a_agent_permissions.md @@ -0,0 +1,259 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import Image from '@theme/IdealImage'; + +# Agent Permission Management + +Control which A2A agents can be accessed by specific keys or teams in LiteLLM. + +## Overview + +Agent Permission Management lets you restrict which agents a LiteLLM Virtual Key or Team can access. This is useful for: + +- **Multi-tenant environments**: Give different teams access to different agents +- **Security**: Prevent keys from invoking agents they shouldn't have access to +- **Compliance**: Enforce access policies for sensitive agent workflows + +When permissions are configured: +- `GET /v1/agents` only returns agents the key/team can access +- `POST /a2a/{agent_id}` (Invoking an agent) returns `403 Forbidden` if access is denied + +## Setting Permissions on a Key + +This example shows how to create a key with agent permissions and test access. + +### 1. Get Your Agent ID + + + + +1. Go to **Agents** in the sidebar +2. Click into the agent you want +3. Copy the **Agent ID** + + + + + + +```bash title="List all agents" showLineNumbers +curl "http://localhost:4000/v1/agents" \ + -H "Authorization: Bearer sk-master-key" +``` + +Response: +```json title="Response" showLineNumbers +{ + "agents": [ + {"agent_id": "agent-123", "name": "Support Agent"}, + {"agent_id": "agent-456", "name": "Sales Agent"} + ] +} +``` + + + + +### 2. Create a Key with Agent Permissions + + + + +1. Go to **Keys** → **Create Key** +2. Expand **Agent Settings** +3. Select the agents you want to allow + + + + + + +```bash title="Create key with agent permissions" showLineNumbers +curl -X POST "http://localhost:4000/key/generate" \ + -H "Authorization: Bearer sk-master-key" \ + -H "Content-Type: application/json" \ + -d '{ + "object_permission": { + "agents": ["agent-123"] + } + }' +``` + + + + +### 3. Test Access + +**Allowed agent (succeeds):** +```bash title="Invoke allowed agent" showLineNumbers +curl -X POST "http://localhost:4000/a2a/agent-123" \ + -H "Authorization: Bearer sk-your-new-key" \ + -H "Content-Type: application/json" \ + -d '{"message": {"role": "user", "parts": [{"type": "text", "text": "Hello"}]}}' +``` + +**Blocked agent (fails with 403):** +```bash title="Invoke blocked agent" showLineNumbers +curl -X POST "http://localhost:4000/a2a/agent-456" \ + -H "Authorization: Bearer sk-your-new-key" \ + -H "Content-Type: application/json" \ + -d '{"message": {"role": "user", "parts": [{"type": "text", "text": "Hello"}]}}' +``` + +Response: +```json title="403 Forbidden Response" showLineNumbers +{ + "error": { + "message": "Access denied to agent: agent-456", + "code": 403 + } +} +``` + +## Setting Permissions on a Team + +Restrict all keys belonging to a team to only access specific agents. + +### 1. Create a Team with Agent Permissions + + + + +1. Go to **Teams** → **Create Team** +2. Expand **Agent Settings** +3. Select the agents you want to allow for this team + + + + + + +```bash title="Create team with agent permissions" showLineNumbers +curl -X POST "http://localhost:4000/team/new" \ + -H "Authorization: Bearer sk-master-key" \ + -H "Content-Type: application/json" \ + -d '{ + "team_alias": "support-team", + "object_permission": { + "agents": ["agent-123"] + } + }' +``` + +Response: +```json title="Response" showLineNumbers +{ + "team_id": "team-abc-123", + "team_alias": "support-team" +} +``` + + + + +### 2. Create a Key for the Team + + + + +1. Go to **Keys** → **Create Key** +2. Select the **Team** from the dropdown + + + + + + +```bash title="Create key for team" showLineNumbers +curl -X POST "http://localhost:4000/key/generate" \ + -H "Authorization: Bearer sk-master-key" \ + -H "Content-Type: application/json" \ + -d '{ + "team_id": "team-abc-123" + }' +``` + + + + +### 3. Test Access + +The key inherits agent permissions from the team. + +**Allowed agent (succeeds):** +```bash title="Invoke allowed agent" showLineNumbers +curl -X POST "http://localhost:4000/a2a/agent-123" \ + -H "Authorization: Bearer sk-team-key" \ + -H "Content-Type: application/json" \ + -d '{"message": {"role": "user", "parts": [{"type": "text", "text": "Hello"}]}}' +``` + +**Blocked agent (fails with 403):** +```bash title="Invoke blocked agent" showLineNumbers +curl -X POST "http://localhost:4000/a2a/agent-456" \ + -H "Authorization: Bearer sk-team-key" \ + -H "Content-Type: application/json" \ + -d '{"message": {"role": "user", "parts": [{"type": "text", "text": "Hello"}]}}' +``` + +## How It Works + +```mermaid +flowchart TD + A[Request to invoke agent] --> B{LiteLLM Virtual Key has agent restrictions?} + B -->|Yes| C{LiteLLM Team has agent restrictions?} + B -->|No| D{LiteLLM Team has agent restrictions?} + + C -->|Yes| E[Use intersection of key + team permissions] + C -->|No| F[Use key permissions only] + + D -->|Yes| G[Inherit team permissions] + D -->|No| H[Allow ALL agents] + + E --> I{Agent in allowed list?} + F --> I + G --> I + H --> J[Allow request] + + I -->|Yes| J + I -->|No| K[Return 403 Forbidden] +``` + +| Key Permissions | Team Permissions | Result | Notes | +|-----------------|------------------|--------|-------| +| None | None | Key can access **all** agents | Open access by default when no restrictions are set | +| `["agent-1", "agent-2"]` | None | Key can access `agent-1` and `agent-2` | Key uses its own permissions | +| None | `["agent-1", "agent-3"]` | Key can access `agent-1` and `agent-3` | Key inherits team's permissions | +| `["agent-1", "agent-2"]` | `["agent-1", "agent-3"]` | Key can access `agent-1` only | Intersection of both lists (most restrictive wins) | + +## Viewing Permissions + + + + +1. Go to **Keys** or **Teams** +2. Click into the key/team you want to view +3. Agent permissions are displayed in the info view + + + + +```bash title="Get key info" showLineNumbers +curl "http://localhost:4000/key/info?key=sk-your-key" \ + -H "Authorization: Bearer sk-master-key" +``` + + + diff --git a/docs/my-website/docs/adding_provider/generic_guardrail_api.md b/docs/my-website/docs/adding_provider/generic_guardrail_api.md new file mode 100644 index 00000000000..cd2b25d125b --- /dev/null +++ b/docs/my-website/docs/adding_provider/generic_guardrail_api.md @@ -0,0 +1,373 @@ +# [BETA] Generic Guardrail API - Integrate Without a PR + +## The Problem + +As a guardrail provider, integrating with LiteLLM traditionally requires: +- Making a PR to the LiteLLM repository +- Waiting for review and merge +- Maintaining provider-specific code in LiteLLM's codebase +- Updating the integration for changes to your API + +## The Solution + +The **Generic Guardrail API** lets you integrate with LiteLLM **instantly** by implementing a simple API endpoint. No PR required. + +### Key Benefits + +1. **No PR Needed** - Deploy and integrate immediately +2. **Universal Support** - Works across ALL LiteLLM endpoints (chat, embeddings, image generation, etc.) +3. **Simple Contract** - One endpoint, three response types +4. **Multi-Modal Support** - Handle both text and images in requests/responses +5. **Custom Parameters** - Pass provider-specific params via config +6. **Full Control** - You own and maintain your guardrail API + +## Supported Endpoints + +The Generic Guardrail API works with the following LiteLLM endpoints: + +- `/v1/chat/completions` - OpenAI Chat Completions +- `/v1/completions` - OpenAI Text Completions +- `/v1/responses` - OpenAI Responses API +- `/v1/images/generations` - OpenAI Image Generation +- `/v1/audio/transcriptions` - OpenAI Audio Transcriptions +- `/v1/audio/speech` - OpenAI Text-to-Speech +- `/v1/messages` - Anthropic Messages +- `/v1/rerank` - Cohere Rerank +- Pass-through endpoints + +## How It Works + +1. LiteLLM extracts text and images from any request (chat messages, embeddings, image prompts, etc.) +2. Sends extracted content + metadata to your API endpoint +3. Your API responds with: `BLOCKED`, `NONE`, or `GUARDRAIL_INTERVENED` +4. LiteLLM enforces the decision and applies any modifications + +## API Contract + +### Endpoint + +Implement `POST /beta/litellm_basic_guardrail_api` + +### Request Format + +```json +{ + "texts": ["extracted text from the request"], // array of text strings + "images": ["base64_encoded_image_data"], // optional array of images + "tools": [ // tool calls sent to the LLM (in the OpenAI Chat Completions spec) + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + } + } + } + } + ], + "tool_calls": [ // tool calls received from the LLM (in the OpenAI Chat Completions spec) + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"location\": \"San Francisco\"}" + } + } + ], + "structured_messages": [ // optional, full messages in OpenAI format (for chat endpoints) + {"role": "system", "content": "You are a helpful assistant"}, + {"role": "user", "content": "Hello"} + ], + "request_data": { + "user_api_key_hash": "hash of the litellm virtual key used", + "user_api_key_alias": "alias of the litellm virtual key used", + "user_api_key_user_id": "user id associated with the litellm virtual key used", + "user_api_key_user_email": "user email associated with the litellm virtual key used", + "user_api_key_team_id": "team id associated with the litellm virtual key used", + "user_api_key_team_alias": "team alias associated with the litellm virtual key used", + "user_api_key_end_user_id": "end user id associated with the litellm virtual key used", + "user_api_key_org_id": "org id associated with the litellm virtual key used" + }, + "input_type": "request", // "request" or "response" + "litellm_call_id": "unique_call_id", // the call id of the individual LLM call + "litellm_trace_id": "trace_id", // the trace id of the LLM call - useful if there are multiple LLM calls for the same conversation + "additional_provider_specific_params": { + // your custom params from config + } +} +``` + +### Response Format + +```json +{ + "action": "BLOCKED" | "NONE" | "GUARDRAIL_INTERVENED", + "blocked_reason": "why content was blocked", // required if action=BLOCKED + "texts": ["modified text"], // optional array of modified text strings + "images": ["modified_base64_image"] // optional array of modified images +} +``` + +**Actions:** +- `BLOCKED` - LiteLLM raises error and blocks request +- `NONE` - Request proceeds unchanged +- `GUARDRAIL_INTERVENED` - Request proceeds with modified texts/images (provide `texts` and/or `images` fields) + +## Parameters + +### `tools` Parameter + +The `tools` parameter provides information about available function/tool definitions in the request. + +**Format:** OpenAI `ChatCompletionToolParam` format (see [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools)) + +**Example:** +```json +{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"] + } + }, + "required": ["location"] + } + } +} +``` + +**Availability:** +- **Input only:** Tools are only passed for `input_type="request"` (pre-call guardrails). Output/response guardrails do not currently receive tool definitions. +- **Supported endpoints:** The `tools` parameter is supported on: `/v1/chat/completions`, `/v1/responses`, and `/v1/messages`. Other endpoints do not have tool support. + +**Use cases:** +- Enforce tool permission policies (e.g., only allow certain users/teams to access specific tools) +- Validate tool schemas before sending to LLM +- Log tool usage for audit purposes +- Block sensitive tools based on user context + +### `tool_calls` Parameter + +The `tool_calls` parameter contains actual function/tool invocations being made in the request or response. + +**Format:** OpenAI `ChatCompletionMessageToolCall` format (see [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/object#chat/object-tool_calls)) + +**Example:** +```json +{ + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"location\": \"San Francisco\", \"unit\": \"celsius\"}" + } +} +``` + +**Key Difference from `tools`:** +- **`tools`** = Tool definitions/schemas (what tools are *available*) +- **`tool_calls`** = Tool invocations/executions (what tools are *being called* with what arguments) + +**Availability:** +- **Both input and output:** Tool calls can be present in both `input_type="request"` (assistant messages requesting tool calls) and `input_type="response"` (LLM responses with tool calls). +- **Supported endpoints:** The `tool_calls` parameter is supported on: `/v1/chat/completions`, `/v1/responses`, and `/v1/messages`. + +**Use cases:** +- Validate tool call arguments before execution +- Redact sensitive data from tool call arguments (e.g., PII) +- Log tool invocations for audit/debugging +- Block tool calls with dangerous parameters +- Modify tool call arguments (e.g., enforce constraints, sanitize inputs) +- Monitor tool usage patterns across users/teams + +### `structured_messages` Parameter + +The `structured_messages` parameter provides the full input in OpenAI chat completion spec format, useful for distinguishing between system and user messages. + +**Format:** Array of OpenAI chat completion messages (see [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat-create-messages)) + +**Example:** +```json +[ + {"role": "system", "content": "You are a helpful assistant"}, + {"role": "user", "content": "Hello"} +] +``` + +**Availability:** +- **Supported endpoints:** `/v1/chat/completions`, `/v1/messages`, `/v1/responses` +- **Input only:** Only passed for `input_type="request"` (pre-call guardrails) + +**Use cases:** +- Apply different policies for system vs user messages +- Enforce role-based content restrictions +- Log structured conversation context + +## LiteLLM Configuration + +Add to `config.yaml`: + +```yaml +litellm_settings: + guardrails: + - guardrail_name: "my-guardrail" + litellm_params: + guardrail: generic_guardrail_api + mode: pre_call # or post_call, during_call + api_base: https://your-guardrail-api.com + api_key: os.environ/YOUR_GUARDRAIL_API_KEY # optional + additional_provider_specific_params: + # your custom parameters + threshold: 0.8 + language: "en" +``` + +## Usage + +Users apply your guardrail by name: + +```python +response = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": "hello"}], + guardrails=["my-guardrail"] +) +``` + +Or with dynamic parameters: + +```python +response = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": "hello"}], + guardrails=[{ + "my-guardrail": { + "extra_body": { + "custom_threshold": 0.9 + } + } + }] +) +``` + +## Implementation Example + +See [mock_bedrock_guardrail_server.py](https://github.com/BerriAI/litellm/blob/main/cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py) for a complete reference implementation. + +**Minimal FastAPI example:** + +```python +from fastapi import FastAPI +from pydantic import BaseModel +from typing import List, Optional, Dict, Any + +app = FastAPI() + +class GuardrailRequest(BaseModel): + texts: List[str] + images: Optional[List[str]] = None + tools: Optional[List[Dict[str, Any]]] = None # OpenAI ChatCompletionToolParam format (tool definitions) + tool_calls: Optional[List[Dict[str, Any]]] = None # OpenAI ChatCompletionMessageToolCall format (tool invocations) + structured_messages: Optional[List[Dict[str, Any]]] = None # OpenAI messages format (for chat endpoints) + request_data: Dict[str, Any] + input_type: str # "request" or "response" + litellm_call_id: Optional[str] = None + litellm_trace_id: Optional[str] = None + additional_provider_specific_params: Dict[str, Any] + +class GuardrailResponse(BaseModel): + action: str # BLOCKED, NONE, or GUARDRAIL_INTERVENED + blocked_reason: Optional[str] = None + texts: Optional[List[str]] = None + images: Optional[List[str]] = None + +@app.post("/beta/litellm_basic_guardrail_api") +async def apply_guardrail(request: GuardrailRequest): + # Your guardrail logic here + + # Example: Check text content + for text in request.texts: + if "badword" in text.lower(): + return GuardrailResponse( + action="BLOCKED", + blocked_reason="Content contains prohibited terms" + ) + + # Example: Check tool definitions (if present in request) + if request.tools: + for tool in request.tools: + if tool.get("type") == "function": + function_name = tool.get("function", {}).get("name", "") + # Block sensitive tool definitions + if function_name in ["delete_data", "access_admin_panel"]: + return GuardrailResponse( + action="BLOCKED", + blocked_reason=f"Tool '{function_name}' is not allowed" + ) + + # Example: Check tool calls (if present in request or response) + if request.tool_calls: + for tool_call in request.tool_calls: + if tool_call.get("type") == "function": + function_name = tool_call.get("function", {}).get("name", "") + arguments_str = tool_call.get("function", {}).get("arguments", "{}") + + # Parse arguments and validate + import json + try: + arguments = json.loads(arguments_str) + # Block dangerous arguments + if "file_path" in arguments and ".." in str(arguments["file_path"]): + return GuardrailResponse( + action="BLOCKED", + blocked_reason="Tool call contains path traversal attempt" + ) + except json.JSONDecodeError: + pass + + # Example: Check structured messages (if present in request) + if request.structured_messages: + for message in request.structured_messages: + if message.get("role") == "system": + # Apply stricter policies to system messages + if "admin" in message.get("content", "").lower(): + return GuardrailResponse( + action="BLOCKED", + blocked_reason="System message contains restricted terms" + ) + + return GuardrailResponse(action="NONE") +``` + +## When to Use This + +✅ **Use Generic Guardrail API when:** +- You want instant integration without waiting for PRs +- You maintain your own guardrail service +- You need full control over updates and features +- You want to support all LiteLLM endpoints automatically + +❌ **Make a PR when:** +- You want deeper integration with LiteLLM internals +- Your guardrail requires complex LiteLLM-specific logic +- You want to be featured as a built-in provider + +## Questions? + +This is a **beta API**. We're actively improving it based on feedback. Open an issue or PR if you need additional capabilities. + diff --git a/docs/my-website/docs/audio_transcription.md b/docs/my-website/docs/audio_transcription.md index fd55cc66e92..5853b5c1872 100644 --- a/docs/my-website/docs/audio_transcription.md +++ b/docs/my-website/docs/audio_transcription.md @@ -13,7 +13,7 @@ import TabItem from '@theme/TabItem'; | Fallbacks | ✅ | Works between supported models | | Loadbalancing | ✅ | Works between supported models | | Guardrails | ✅ | Applies to output transcribed text (non-streaming only) | -| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai` | | +| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai`, `ovhcloud` | | ## Quick Start @@ -126,6 +126,7 @@ transcript = client.audio.transcriptions.create( - [Fireworks AI](./providers/fireworks_ai.md#audio-transcription) - [Groq](./providers/groq.md#speech-to-text---whisper) - [Deepgram](./providers/deepgram.md) +- [OVHcloud AI Endpoints](./providers/ovhcloud.md) --- diff --git a/docs/my-website/docs/completion/knowledgebase.md b/docs/my-website/docs/completion/knowledgebase.md index 759f7912d87..fd6ef7a9982 100644 --- a/docs/my-website/docs/completion/knowledgebase.md +++ b/docs/my-website/docs/completion/knowledgebase.md @@ -21,6 +21,7 @@ LiteLLM integrates with vector stores, allowing your models to access your organ - [Azure Vector Stores](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/file-search?tabs=python#vector-stores) (Cannot be directly queried. Only available for calling in Assistants messages. We will be adding Azure AI Search Vector Store API support soon.) - [Vertex AI RAG API](https://cloud.google.com/vertex-ai/generative-ai/docs/rag-overview) - [Gemini File Search](https://ai.google.dev/gemini-api/docs/file-search) +- [RAGFlow Datasets](/docs/providers/ragflow_vector_store.md) (Dataset management only, search not supported) ## Quick Start diff --git a/docs/my-website/docs/completion/web_search.md b/docs/my-website/docs/completion/web_search.md index b0d8fcdf4c0..db50c7b5bc5 100644 --- a/docs/my-website/docs/completion/web_search.md +++ b/docs/my-website/docs/completion/web_search.md @@ -371,6 +371,22 @@ model_list: web_search_options: {} # Enables web search with default settings ``` +### Advanced +You can configure LiteLLM's router to optionally drop models that do not support WebSearch, for example +```yaml + - model_name: gpt-4.1 + litellm_params: + model: openai/gpt-4.1 + - model_name: gpt-4.1 + litellm_params: + model: azure/gpt-4.1 + api_base: "x.openai.azure.com/" + api_version: 2025-03-01-preview + model_info: + supports_web_search: False <---- KEY CHANGE! +``` +In this example, LiteLLM will still route LLM requests to both deployments, but for WebSearch, will solely route to OpenAI. + diff --git a/docs/my-website/docs/contributing/adding_openai_compatible_providers.md b/docs/my-website/docs/contributing/adding_openai_compatible_providers.md new file mode 100644 index 00000000000..bb89eea35bf --- /dev/null +++ b/docs/my-website/docs/contributing/adding_openai_compatible_providers.md @@ -0,0 +1,130 @@ +# Adding OpenAI-Compatible Providers + +For simple OpenAI-compatible providers (like Hyperbolic, Nscale, etc.), you can add support by editing a single JSON file. + +## Quick Start + +1. Edit `litellm/llms/openai_like/providers.json` +2. Add your provider configuration +3. Test with: `litellm.completion(model="your_provider/model-name", ...)` + +## Basic Configuration + +For a fully OpenAI-compatible provider: + +```json +{ + "your_provider": { + "base_url": "https://api.yourprovider.com/v1", + "api_key_env": "YOUR_PROVIDER_API_KEY" + } +} +``` + +That's it! The provider is now available. + +## Configuration Options + +### Required Fields + +- `base_url` - API endpoint (e.g., `https://api.provider.com/v1`) +- `api_key_env` - Environment variable name for API key (e.g., `PROVIDER_API_KEY`) + +### Optional Fields + +- `api_base_env` - Environment variable to override `base_url` +- `base_class` - Use `"openai_gpt"` (default) or `"openai_like"` +- `param_mappings` - Map OpenAI parameter names to provider-specific names +- `constraints` - Parameter value constraints (min/max) +- `special_handling` - Special behaviors like content format conversion + +## Examples + +### Simple Provider (Fully Compatible) + +```json +{ + "hyperbolic": { + "base_url": "https://api.hyperbolic.xyz/v1", + "api_key_env": "HYPERBOLIC_API_KEY" + } +} +``` + +### Provider with Parameter Mapping + +```json +{ + "publicai": { + "base_url": "https://api.publicai.co/v1", + "api_key_env": "PUBLICAI_API_KEY", + "param_mappings": { + "max_completion_tokens": "max_tokens" + } + } +} +``` + +### Provider with Constraints + +```json +{ + "custom_provider": { + "base_url": "https://api.custom.com/v1", + "api_key_env": "CUSTOM_API_KEY", + "constraints": { + "temperature_max": 1.0, + "temperature_min": 0.0 + } + } +} +``` + +## Usage + +```python +import litellm +import os + +# Set your API key +os.environ["YOUR_PROVIDER_API_KEY"] = "your-key-here" + +# Use the provider +response = litellm.completion( + model="your_provider/model-name", + messages=[{"role": "user", "content": "Hello"}], +) +``` + +## When to Use Python Instead + +Use a Python config class if you need: + +- Custom authentication flows (OAuth, JWT, etc.) +- Complex request/response transformations +- Provider-specific streaming logic +- Advanced tool calling modifications + +For these cases, create a config class in `litellm/llms/your_provider/chat/transformation.py` that inherits from `OpenAIGPTConfig` or `OpenAILikeChatConfig`. + +## Testing + +Test your provider: + +```bash +# Quick test +python -c " +import litellm +import os +os.environ['PROVIDER_API_KEY'] = 'your-key' +response = litellm.completion( + model='provider/model-name', + messages=[{'role': 'user', 'content': 'test'}] +) +print(response.choices[0].message.content) +" +``` + +## Reference + +See existing providers in `litellm/llms/openai_like/providers.json` for examples. diff --git a/docs/my-website/docs/embedding/supported_embedding.md b/docs/my-website/docs/embedding/supported_embedding.md index e63d9403665..0e8252b409b 100644 --- a/docs/my-website/docs/embedding/supported_embedding.md +++ b/docs/my-website/docs/embedding/supported_embedding.md @@ -263,6 +263,8 @@ print(response) | Model Name | Function Call | |----------------------|---------------------------------------------| +| Amazon Nova Multimodal Embeddings | `embedding(model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0", input=input)` | [Nova Docs](../providers/bedrock_embedding#amazon-nova-multimodal-embeddings) | +| Amazon Nova (Async) | `embedding(model="bedrock/async_invoke/amazon.nova-2-multimodal-embeddings-v1:0", input=input, input_type="text", output_s3_uri="s3://bucket/")` | [Nova Async Docs](../providers/bedrock_embedding#asynchronous-embeddings-with-segmentation) | | Titan Embeddings - G1 | `embedding(model="amazon.titan-embed-text-v1", input=input)` | | Cohere Embeddings - English | `embedding(model="cohere.embed-english-v3", input=input)` | | Cohere Embeddings - Multilingual | `embedding(model="cohere.embed-multilingual-v3", input=input)` | diff --git a/docs/my-website/docs/files_endpoints.md b/docs/my-website/docs/files_endpoints.md index fc0484e9219..30677c748a9 100644 --- a/docs/my-website/docs/files_endpoints.md +++ b/docs/my-website/docs/files_endpoints.md @@ -301,6 +301,17 @@ content = await litellm.afile_content( print("file content=", content) ``` +**Get File Content (Bedrock)** +```python +# For Bedrock batch output files stored in S3 +content = await litellm.afile_content( + file_id="s3://bucket-name/path/to/file.jsonl", # S3 URI or unified file ID + custom_llm_provider="bedrock", + aws_region_name="us-west-2" +) +print("file content=", content.text) +``` + @@ -313,4 +324,6 @@ print("file content=", content) ### [Vertex AI](./providers/vertex#batch-apis) +### [Bedrock](./providers/bedrock_batches#4-retrieve-batch-results) + ## [Swagger API Reference](https://litellm-api.up.railway.app/#/files) diff --git a/docs/my-website/docs/getting_started.md b/docs/my-website/docs/getting_started.md deleted file mode 100644 index 6b2c1fd531e..00000000000 --- a/docs/my-website/docs/getting_started.md +++ /dev/null @@ -1,108 +0,0 @@ -# Getting Started - -import QuickStart from '../src/components/QuickStart.js' - -LiteLLM simplifies LLM API calls by mapping them all to the [OpenAI ChatCompletion format](https://platform.openai.com/docs/api-reference/chat). - -## basic usage - -By default we provide a free $10 community-key to try all providers supported on LiteLLM. - -```python -from litellm import completion - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "your-api-key" -os.environ["COHERE_API_KEY"] = "your-api-key" - -messages = [{ "content": "Hello, how are you?","role": "user"}] - -# openai call -response = completion(model="gpt-3.5-turbo", messages=messages) - -# cohere call -response = completion("command-nightly", messages) -``` - -**Need a dedicated key?** -Email us @ krrish@berri.ai - -Next Steps 👉 [Call all supported models - e.g. Claude-2, Llama2-70b, etc.](./proxy_api.md#supported-models) - -More details 👉 - -- [Completion() function details](./completion/) -- [Overview of supported models / providers on LiteLLM](./providers/) -- [Search all models / providers](https://models.litellm.ai/) -- [Build your own OpenAI proxy](https://github.com/BerriAI/liteLLM-proxy/tree/main) - -## streaming - -Same example from before. Just pass in `stream=True` in the completion args. - -```python -from litellm import completion - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "openai key" -os.environ["COHERE_API_KEY"] = "cohere key" - -messages = [{ "content": "Hello, how are you?","role": "user"}] - -# openai call -response = completion(model="gpt-3.5-turbo", messages=messages, stream=True) - -# cohere call -response = completion("command-nightly", messages, stream=True) - -print(response) -``` - -More details 👉 - -- [streaming + async](./completion/stream.md) -- [tutorial for streaming Llama2 on TogetherAI](./tutorials/TogetherAI_liteLLM.md) - -## exception handling - -LiteLLM maps exceptions across all supported providers to the OpenAI exceptions. All our exceptions inherit from OpenAI's exception types, so any error-handling you have for that, should work out of the box with LiteLLM. - -```python -from openai.error import OpenAIError -from litellm import completion - -os.environ["ANTHROPIC_API_KEY"] = "bad-key" -try: - # some code - completion(model="claude-instant-1", messages=[{"role": "user", "content": "Hey, how's it going?"}]) -except OpenAIError as e: - print(e) -``` - -## Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks)) - -LiteLLM exposes pre defined callbacks to send data to MLflow, Lunary, Langfuse, Helicone, Promptlayer, Traceloop, Slack - -```python -from litellm import completion - -## set env variables for logging tools (API key set up is not required when using MLflow) -os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # get your public key at https://app.lunary.ai/settings -os.environ["HELICONE_API_KEY"] = "your-helicone-key" -os.environ["LANGFUSE_PUBLIC_KEY"] = "" -os.environ["LANGFUSE_SECRET_KEY"] = "" - -os.environ["OPENAI_API_KEY"] - -# set callbacks -litellm.success_callback = ["lunary", "mlflow", "langfuse", "helicone"] # log input/output to MLflow, langfuse, lunary, helicone - -#openai call -response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}]) -``` - -More details 👉 - -- [exception mapping](./exception_mapping.md) -- [retries + model fallbacks for completion()](./completion/reliable_completions.md) -- [tutorial for model fallbacks with completion()](./tutorials/fallbacks.md) diff --git a/docs/my-website/docs/observability/datadog.md b/docs/my-website/docs/observability/datadog.md index 5cb5ab3af2d..b2901650ea6 100644 --- a/docs/my-website/docs/observability/datadog.md +++ b/docs/my-website/docs/observability/datadog.md @@ -71,17 +71,19 @@ DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source. use to different Send logs through a local DataDog agent (useful for containerized environments): ```shell -DD_AGENT_HOST="localhost" # hostname or IP of DataDog agent -DD_AGENT_PORT="10518" # [OPTIONAL] port of DataDog agent (default: 10518) -DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (agent handles auth) -DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source +LITELLM_DD_AGENT_HOST="localhost" # hostname or IP of DataDog agent +LITELLM_DD_AGENT_PORT="10518" # [OPTIONAL] port of DataDog agent (default: 10518) +DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (agent handles auth) +DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source ``` -When `DD_AGENT_HOST` is set, logs are sent to the agent instead of directly to DataDog API. This is useful for: +When `LITELLM_DD_AGENT_HOST` is set, logs are sent to the agent instead of directly to DataDog API. This is useful for: - Centralized log shipping in containerized environments - Reducing direct API calls from multiple services - Leveraging agent-side processing and filtering +**Note:** We use `LITELLM_DD_AGENT_HOST` instead of `DD_AGENT_HOST` to avoid conflicts with `ddtrace` which automatically sets `DD_AGENT_HOST` for APM tracing. + **Step 3**: Start the proxy, make a test request Start proxy @@ -191,8 +193,8 @@ LiteLLM supports customizing the following Datadog environment variables |---------------------|-------------|---------------|----------| | `DD_API_KEY` | Your Datadog API key for authentication (required for direct API, optional for agent) | None | Conditional* | | `DD_SITE` | Your Datadog site (e.g., "us5.datadoghq.com") (required for direct API) | None | Conditional* | -| `DD_AGENT_HOST` | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API | None | ❌ No | -| `DD_AGENT_PORT` | Port of DataDog agent for log intake | "10518" | ❌ No | +| `LITELLM_DD_AGENT_HOST` | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API | None | ❌ No | +| `LITELLM_DD_AGENT_PORT` | Port of DataDog agent for log intake | "10518" | ❌ No | | `DD_ENV` | Environment tag for your logs (e.g., "production", "staging") | "unknown" | ❌ No | | `DD_SERVICE` | Service name for your logs | "litellm-server" | ❌ No | | `DD_SOURCE` | Source name for your logs | "litellm" | ❌ No | @@ -201,5 +203,5 @@ LiteLLM supports customizing the following Datadog environment variables | `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No | \* **Required when using Direct API** (default): `DD_API_KEY` and `DD_SITE` are required -\* **Optional when using DataDog Agent**: Set `DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required +\* **Optional when using DataDog Agent**: Set `LITELLM_DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required diff --git a/docs/my-website/docs/observability/phoenix_integration.md b/docs/my-website/docs/observability/phoenix_integration.md index ad337439934..898d780668d 100644 --- a/docs/my-website/docs/observability/phoenix_integration.md +++ b/docs/my-website/docs/observability/phoenix_integration.md @@ -6,7 +6,7 @@ Open source tracing and evaluation platform :::tip -This is community maintained, Please make an issue if you run into a bug +This is community maintained. Please make an issue if you run into a bug: https://github.com/BerriAI/litellm ::: @@ -31,19 +31,16 @@ litellm.callbacks = ["arize_phoenix"] import litellm import os -os.environ["PHOENIX_API_KEY"] = "" # Necessary only using Phoenix Cloud -os.environ["PHOENIX_COLLECTOR_HTTP_ENDPOINT"] = "" # The URL of your Phoenix OSS instance e.g. http://localhost:6006/v1/traces -os.environ["PHOENIX_PROJECT_NAME"]="litellm" # OPTIONAL: you can configure project names, otherwise traces would go to "default" project +# Set env variables +os.environ["PHOENIX_API_KEY"] = "d0*****" # Set the Phoenix API key here. It is necessary only when using Phoenix Cloud. +os.environ["PHOENIX_COLLECTOR_HTTP_ENDPOINT"] = "https://app.phoenix.arize.com/s//v1/traces" # Set the URL of your Phoenix OSS instance, otherwise tracer would use https://app.phoenix.arize.com/v1/traces for Phoenix Cloud. +os.environ["PHOENIX_PROJECT_NAME"] = "litellm" # Configure the project name, otherwise traces would go to "default" project. +os.environ['OPENAI_API_KEY'] = "fake-key" # Set the OpenAI API key here. -# This defaults to https://app.phoenix.arize.com/v1/traces for Phoenix Cloud - -# LLM API Keys -os.environ['OPENAI_API_KEY']="" - -# set arize as a callback, litellm will send the data to arize +# Set arize_phoenix as a callback & LiteLLM will send the data to Phoenix. litellm.callbacks = ["arize_phoenix"] - -# openai call + +# OpenAI call response = litellm.completion( model="gpt-3.5-turbo", messages=[ @@ -52,8 +49,9 @@ response = litellm.completion( ) ``` -### Using with LiteLLM Proxy +## Using with LiteLLM Proxy +1. Setup config.yaml ```yaml model_list: @@ -66,12 +64,63 @@ model_list: litellm_settings: callbacks: ["arize_phoenix"] +general_settings: + master_key: "sk-1234" + environment_variables: PHOENIX_API_KEY: "d0*****" - PHOENIX_COLLECTOR_ENDPOINT: "https://app.phoenix.arize.com/v1/traces" # OPTIONAL, for setting the GRPC endpoint - PHOENIX_COLLECTOR_HTTP_ENDPOINT: "https://app.phoenix.arize.com/v1/traces" # OPTIONAL, for setting the HTTP endpoint + PHOENIX_COLLECTOR_ENDPOINT: "https://app.phoenix.arize.com/s//v1/traces" # OPTIONAL - For setting the gRPC endpoint + PHOENIX_COLLECTOR_HTTP_ENDPOINT: "https://app.phoenix.arize.com/s//v1/traces" # OPTIONAL - For setting the HTTP endpoint ``` +2. Start the proxy + +```bash +litellm --config config.yaml +``` + +3. Test it! + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ "model": "gpt-4o", "messages": [{"role": "user", "content": "Hi 👋 - i'm openai"}]}' +``` + +## Supported Phoenix Endpoints +Phoenix now supports multiple deployment types. The correct endpoint depends on which version of Phoenix Cloud you are using. + +**Phoenix Cloud (With Spaces - New Version)** +Use this if your Phoenix URL contains `/s/` path. + +```bash +https://app.phoenix.arize.com/s//v1/traces +``` + +**Phoenix Cloud (Legacy - Deprecated)** +Use this only if your deployment still shows the `/legacy` pattern. + +```bash +https://app.phoenix.arize.com/legacy/v1/traces +``` + +**Phoenix Cloud (Without Spaces - Old Version)** +Use this if your Phoenix Cloud URL does not contain `/s/` or `/legacy` path. + +```bash +https://app.phoenix.arize.com/v1/traces +``` + +**Self-Hosted Phoenix (Local Instance)** +Use this when running Phoenix on your machine or a private server. + +```bash +http://localhost:6006/v1/traces +``` + +Depending on which Phoenix Cloud version or deployment you are using, you should set the corresponding endpoint in `PHOENIX_COLLECTOR_HTTP_ENDPOINT` or `PHOENIX_COLLECTOR_ENDPOINT`. + ## Support & Talk to Founders - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) diff --git a/docs/my-website/docs/projects/Agent Lightning.md b/docs/my-website/docs/projects/Agent Lightning.md new file mode 100644 index 00000000000..28e5546e398 --- /dev/null +++ b/docs/my-website/docs/projects/Agent Lightning.md @@ -0,0 +1,10 @@ + +# Agent Lightning + +[Agent Lightning](https://github.com/microsoft/agent-lightning) is Microsoft's open-source framework for training and optimizing AI agents with Reinforcement Learning, Automatic Prompt Optimization, and Supervised Fine-tuning — with almost zero code changes. + +It works with any agent framework including LangChain, OpenAI Agents SDK, AutoGen, and CrewAI. Agent Lightning uses LiteLLM Proxy under the hood to route LLM requests and collect traces that power its training algorithms. + +- [GitHub](https://github.com/microsoft/agent-lightning) +- [Docs](https://microsoft.github.io/agent-lightning/) +- [arXiv Paper](https://arxiv.org/abs/2508.03680) diff --git a/docs/my-website/docs/projects/Google ADK.md b/docs/my-website/docs/projects/Google ADK.md new file mode 100644 index 00000000000..25e910dcbad --- /dev/null +++ b/docs/my-website/docs/projects/Google ADK.md @@ -0,0 +1,21 @@ + +# Google ADK (Agent Development Kit) + +[Google ADK](https://github.com/google/adk-python) is an open-source, code-first Python framework for building, evaluating, and deploying sophisticated AI agents. While optimized for Gemini, ADK is model-agnostic and supports LiteLLM for using 100+ providers. + +```python +from google.adk.agents.llm_agent import Agent +from google.adk.models.lite_llm import LiteLlm + +root_agent = Agent( + model=LiteLlm(model="openai/gpt-4o"), # Or any LiteLLM-supported model + name="my_agent", + description="An agent using LiteLLM", + instruction="You are a helpful assistant.", + tools=[your_tools], +) +``` + +- [GitHub](https://github.com/google/adk-python) +- [Documentation](https://google.github.io/adk-docs) +- [LiteLLM Samples](https://github.com/google/adk-python/tree/main/contributing/samples/hello_world_litellm) diff --git a/docs/my-website/docs/projects/GraphRAG.md b/docs/my-website/docs/projects/GraphRAG.md new file mode 100644 index 00000000000..6c5e3dea334 --- /dev/null +++ b/docs/my-website/docs/projects/GraphRAG.md @@ -0,0 +1,8 @@ + +# Microsoft GraphRAG + +GraphRAG is a data pipeline and transformation suite that extracts meaningful, structured data from unstructured text using the power of LLMs. It uses a graph-based approach to RAG (Retrieval-Augmented Generation) that leverages knowledge graphs to improve reasoning over private datasets. + +- [Github](https://github.com/microsoft/graphrag) +- [Docs](https://microsoft.github.io/graphrag/) +- [Paper](https://arxiv.org/pdf/2404.16130) diff --git a/docs/my-website/docs/projects/Harbor.md b/docs/my-website/docs/projects/Harbor.md new file mode 100644 index 00000000000..684dfa93720 --- /dev/null +++ b/docs/my-website/docs/projects/Harbor.md @@ -0,0 +1,24 @@ + +# Harbor + +[Harbor](https://github.com/laude-institute/harbor) is a framework from the creators of Terminal-Bench for evaluating and optimizing agents and language models. It uses LiteLLM to call 100+ LLM providers. + +```bash +# Install +pip install harbor + +# Run a benchmark with any LiteLLM-supported model +harbor run --dataset terminal-bench@2.0 \ + --agent claude-code \ + --model anthropic/claude-opus-4-1 \ + --n-concurrent 4 +``` + +Key features: +- Evaluate agents like Claude Code, OpenHands, Codex CLI +- Build and share benchmarks and environments +- Run experiments in parallel across cloud providers (Daytona, Modal) +- Generate rollouts for RL optimization + +- [GitHub](https://github.com/laude-institute/harbor) +- [Documentation](https://harborframework.com/docs) diff --git a/docs/my-website/docs/provider_registration/index.md b/docs/my-website/docs/provider_registration/index.md index 66f61554783..60570dee7b7 100644 --- a/docs/my-website/docs/provider_registration/index.md +++ b/docs/my-website/docs/provider_registration/index.md @@ -2,6 +2,12 @@ title: "Integrate as a Model Provider" --- +## Quick Start for OpenAI-Compatible Providers + +If your API is OpenAI-compatible, you can add support by editing a single JSON file. See [Adding OpenAI-Compatible Providers](/docs/contributing/adding_openai_compatible_providers) for the simple approach. + +--- + This guide focuses on how to setup the classes and configuration necessary to act as a chat provider. Please see this guide first and look at the existing code in the codebase to understand how to act as a different provider, e.g. handling embeddings or image-generation. diff --git a/docs/my-website/docs/providers/amazon_nova.md b/docs/my-website/docs/providers/amazon_nova.md new file mode 100644 index 00000000000..509127036df --- /dev/null +++ b/docs/my-website/docs/providers/amazon_nova.md @@ -0,0 +1,291 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Amazon Nova + +| Property | Details | +|-------|-------| +| Description | Amazon Nova is a family of foundation models built by Amazon that deliver frontier intelligence and industry-leading price performance. | +| Provider Route on LiteLLM | `amazon_nova/` | +| Provider Doc | [Amazon Nova ↗](https://docs.aws.amazon.com/nova/latest/userguide/what-is-nova.html) | +| Supported OpenAI Endpoints | `/chat/completions`, `v1/responses` | +| Other Supported Endpoints | `v1/messages`, `/generateContent` | + +## Authentication + +Amazon Nova uses API key authentication. You can obtain your API key from the [Amazon Nova developer console ↗](https://nova.amazon.com/dev/documentation). + +```bash +export AMAZON_NOVA_API_KEY="your-api-key" +``` + +## Usage + + + + +```python +import os +from litellm import completion + +# Set your API key +os.environ["AMAZON_NOVA_API_KEY"] = "your-api-key" + +response = completion( + model="amazon_nova/nova-micro-v1", + messages=[ + {"role": "system", "content": "You are a helpful assistant"}, + {"role": "user", "content": "Hello, how are you?"} + ] +) + +print(response) +``` + + + + +### 1. Setup config.yaml + +```yaml +model_list: + - model_name: amazon-nova-micro + litellm_params: + model: amazon_nova/nova-micro-v1 + api_key: os.environ/AMAZON_NOVA_API_KEY +``` +### 2. Start the proxy +```bash +litellm --config /path/to/config.yaml +``` + +### 3. Test it + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data '{ + "model": "amazon-nova-micro", + "messages": [ + { + "role": "user", + "content": "Hello, how are you?" + } + ] +}' +``` + + + + +## Supported Models + +| Model Name | Usage | Context Window | +|------------|-------|----------------| +| Nova Micro | `completion(model="amazon_nova/nova-micro-v1", messages=messages)` | 128K tokens | +| Nova Lite | `completion(model="amazon_nova/nova-lite-v1", messages=messages)` | 300K tokens | +| Nova Pro | `completion(model="amazon_nova/nova-pro-v1", messages=messages)` | 300K tokens | +| Nova Premier | `completion(model="amazon_nova/nova-premier-v1", messages=messages)` | 1M tokens | + +## Usage - Streaming + + + + +```python +import os +from litellm import completion + +os.environ["AMAZON_NOVA_API_KEY"] = "your-api-key" + +response = completion( + model="amazon_nova/nova-micro-v1", + messages=[ + {"role": "system", "content": "You are a helpful assistant"}, + {"role": "user", "content": "Tell me about machine learning"} + ], + stream=True +) + +for chunk in response: + print(chunk.choices[0].delta.content or "", end="") +``` + + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data '{ + "model": "amazon-nova-micro", + "messages": [ + { + "role": "user", + "content": "Tell me about machine learning" + } + ], + "stream": true +}' +``` + + + + +## Usage - Function Calling / Tool Usage + + + + +```python +import os +from litellm import completion + +os.environ["AMAZON_NOVA_API_KEY"] = "your-api-key" + +tools = [ + { + "type": "function", + "function": { + "name": "getCurrentWeather", + "description": "Get the current weather in a given city", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City and country e.g. San Francisco, CA" + } + }, + "required": ["location"] + } + } + } +] + +response = completion( + model="amazon_nova/nova-micro-v1", + messages=[ + {"role": "user", "content": "What's the weather like in San Francisco?"} + ], + tools=tools +) + +print(response) +``` + + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data '{ + "model": "amazon-nova-micro", + "messages": [ + { + "role": "user", + "content": "What'\''s the weather like in San Francisco?" + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "getCurrentWeather", + "description": "Get the current weather in a given city", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City and country e.g. San Francisco, CA" + } + }, + "required": ["location"] + } + } + } + ] +}' +``` + + + + +## Set temperature, top_p, etc. + + + + +```python +import os +from litellm import completion + +os.environ["AMAZON_NOVA_API_KEY"] = "your-api-key" + +response = completion( + model="amazon_nova/nova-pro-v1", + messages=[ + {"role": "user", "content": "Write a creative story"} + ], + temperature=0.8, + max_tokens=500, + top_p=0.9 +) + +print(response) +``` + + + + +**Set on yaml** + +```yaml +model_list: + - model_name: amazon-nova-pro + litellm_params: + model: amazon_nova/nova-pro-v1 + temperature: 0.8 + max_tokens: 500 + top_p: 0.9 +``` +**Set on request** +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data '{ + "model": "amazon-nova-pro", + "messages": [ + { + "role": "user", + "content": "Write a creative story" + } + ], + "temperature": 0.8, + "max_tokens": 500, + "top_p": 0.9 +}' +``` + + + + +## Model Comparison + +| Model | Best For | Speed | Cost | Context | +|-------|----------|-------|------|---------| +| **Nova Micro** | Simple tasks, high throughput | Fastest | Lowest | 128K | +| **Nova Lite** | Balanced performance | Fast | Low | 300K | +| **Nova Pro** | Complex reasoning | Medium | Medium | 300K | +| **Nova Premier** | Most advanced tasks | Slower | Higher | 1M | + +## Error Handling + +Common error codes and their meanings: + +- `401 Unauthorized`: Invalid API key +- `429 Too Many Requests`: Rate limit exceeded +- `400 Bad Request`: Invalid request format +- `500 Internal Server Error`: Service temporarily unavailable \ No newline at end of file diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index 24365f0cc47..f78af51bd90 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -41,7 +41,8 @@ Check this in code, [here](../completion/input.md#translated-openai-params) "extra_headers", "parallel_tool_calls", "response_format", -"user" +"user", +"reasoning_effort", ``` :::info @@ -49,6 +50,7 @@ Check this in code, [here](../completion/input.md#translated-openai-params) **Notes:** - Anthropic API fails requests when `max_tokens` are not passed. Due to this litellm passes `max_tokens=4096` when no `max_tokens` are passed. - `response_format` is fully supported for Claude Sonnet 4.5 and Opus 4.1 models (see [Structured Outputs](#structured-outputs) section) +- `reasoning_effort` is automatically mapped to `output_config={"effort": ...}` for Claude Opus 4.5 models (see [Effort Parameter](./anthropic_effort.md)) ::: @@ -199,6 +201,30 @@ Without `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX`: With `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX=true`: - Base URL `https://my-proxy.com/custom/path` → `https://my-proxy.com/custom/path` (unchanged) +### Azure AI Foundry (Alternative Method) + +:::tip Recommended Method +For full Azure support including Azure AD authentication, use the dedicated [Azure Anthropic provider](./azure/azure_anthropic) with `azure_ai/` prefix. +::: + +As an alternative, you can use the `anthropic/` provider directly with your Azure endpoint since Azure exposes Claude using Anthropic's native API. + +```python +from litellm import completion + +response = completion( + model="anthropic/claude-sonnet-4-5", + api_base="https://.services.ai.azure.com/anthropic", + api_key="", + messages=[{"role": "user", "content": "Hello!"}], +) +print(response) +``` + +:::info +**Finding your Azure endpoint:** Go to Azure AI Foundry → Your deployment → Overview. Your base URL will be `https://.services.ai.azure.com/anthropic` +::: + ## Usage ```python diff --git a/docs/my-website/docs/providers/anthropic_effort.md b/docs/my-website/docs/providers/anthropic_effort.md index 0015162a95b..e4bfd50e6c2 100644 --- a/docs/my-website/docs/providers/anthropic_effort.md +++ b/docs/my-website/docs/providers/anthropic_effort.md @@ -9,7 +9,10 @@ Control how many tokens Claude uses when responding with the `effort` parameter, The `effort` parameter allows you to control how eager Claude is about spending tokens when responding to requests. This gives you the ability to trade off between response thoroughness and token efficiency, all with a single model. -**Note**: The effort parameter is currently in beta and only supported by Claude Opus 4.5. You must include the beta header `effort-2025-11-24` when using this feature (LiteLLM automatically adds this header when `output_config` with `effort` is detected). +**Note**: The effort parameter is currently in beta and only supported by Claude Opus 4.5. LiteLLM automatically adds the `effort-2025-11-24` beta header when: +- `reasoning_effort` parameter is provided (for Claude Opus 4.5 only) + +For Claude Opus 4.5, `reasoning_effort="medium"`—both are automatically mapped to the correct format. ## How Effort Works @@ -52,9 +55,7 @@ response = litellm.completion( "role": "user", "content": "Analyze the trade-offs between microservices and monolithic architectures" }], - output_config={ - "effort": "medium" - } + reasoning_effort="medium" # Automatically mapped to output_config for Opus 4.5 ) print(response.choices[0].message.content) @@ -217,11 +218,14 @@ response = litellm.completion( The effort parameter is supported across all Anthropic-compatible providers: -- **Standard Anthropic**: ✅ Supported (Claude Opus 4.5) -- **Azure Anthropic**: ✅ Supported (Claude Opus 4.5) -- **Vertex AI Anthropic**: ✅ Supported (Claude Opus 4.5) +- **Standard Anthropic API**: ✅ Supported (Claude Opus 4.5) +- **Azure Anthropic / Microsoft Foundry**: ✅ Supported (Claude Opus 4.5) +- **Amazon Bedrock**: ✅ Supported (Claude Opus 4.5) +- **Google Cloud Vertex AI**: ✅ Supported (Claude Opus 4.5) -LiteLLM automatically handles the beta header injection for all providers. +LiteLLM automatically handles: +- Beta header injection (`effort-2025-11-24`) for all providers +- Parameter mapping: `reasoning_effort` → `output_config={"effort": ...}` for Claude Opus 4.5 ## Usage and Pricing @@ -242,9 +246,12 @@ print(f"Total tokens: {response.usage.total_tokens}") ### Beta header not being added -LiteLLM automatically adds the `effort-2025-11-24` beta header when `output_config` with `effort` is detected. If you're not seeing the header: +LiteLLM automatically adds the `effort-2025-11-24` beta header when: +- `reasoning_effort` parameter is provided (for Claude Opus 4.5 only) -1. Ensure you're using `output_config` with an `effort` field +If you're not seeing the header: + +1. Ensure you're using `reasoning_effort` parameter 2. Verify the model is Claude Opus 4.5 3. Check that LiteLLM version supports this feature diff --git a/docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md b/docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md index 6d3e15785e5..574dd7b0935 100644 --- a/docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md +++ b/docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md @@ -3,7 +3,11 @@ Programmatic tool calling allows Claude to write code that calls your tools programmatically within a code execution container, rather than requiring round trips through the model for each tool invocation. This reduces latency for multi-tool workflows and decreases token consumption by allowing Claude to filter or process data before it reaches the model's context window. :::info -Programmatic tool calling is currently in public beta. LiteLLM automatically adds the required `advanced-tool-use-2025-11-20` beta header when it detects tools with the `allowed_callers` field. +Programmatic tool calling is currently in public beta. LiteLLM automatically detects tools with the `allowed_callers` field and adds the appropriate beta header based on your provider: + +- **Anthropic API & Microsoft Foundry**: `advanced-tool-use-2025-11-20` +- **Amazon Bedrock**: `advanced-tool-use-2025-11-20` +- **Google Cloud Vertex AI**: Not supported This feature requires the code execution tool to be enabled. ::: @@ -380,13 +384,14 @@ For example, calling 10 tools directly uses ~10x the tokens of calling them prog ## Provider Support -LiteLLM supports programmatic tool calling across all Anthropic-compatible providers: +LiteLLM supports programmatic tool calling across the following Anthropic-compatible providers: -- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`) -- **Azure Anthropic** (`azure/claude-sonnet-4-5-20250929`) -- **Vertex AI Anthropic** (`vertex_ai/claude-sonnet-4-5-20250929`) +- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`) ✅ +- **Azure Anthropic / Microsoft Foundry** (`azure/claude-sonnet-4-5-20250929`) ✅ +- **Amazon Bedrock** (`bedrock/invoke/anthropic.claude-sonnet-4-5-20250929-v1:0`) ✅ +- **Google Cloud Vertex AI** (`vertex_ai/claude-sonnet-4-5-20250929`) ❌ Not supported -The beta header is automatically added when LiteLLM detects tools with `allowed_callers` field. +The beta header (`advanced-tool-use-2025-11-20`) is automatically added when LiteLLM detects tools with the `allowed_callers` field. ## Limitations diff --git a/docs/my-website/docs/providers/anthropic_tool_input_examples.md b/docs/my-website/docs/providers/anthropic_tool_input_examples.md index d0b7cc1762c..39f4d8555f4 100644 --- a/docs/my-website/docs/providers/anthropic_tool_input_examples.md +++ b/docs/my-website/docs/providers/anthropic_tool_input_examples.md @@ -3,7 +3,13 @@ Provide concrete examples of valid tool inputs to help Claude understand how to use your tools more effectively. This is particularly useful for complex tools with nested objects, optional parameters, or format-sensitive inputs. :::info -Tool input examples is a beta feature. LiteLLM automatically adds the required `advanced-tool-use-2025-11-20` beta header when it detects tools with the `input_examples` field. +Tool input examples is a beta feature. LiteLLM automatically detects tools with the `input_examples` field and adds the appropriate beta header based on your provider: + +- **Anthropic API & Microsoft Foundry**: `advanced-tool-use-2025-11-20` +- **Amazon Bedrock**: `advanced-tool-use-2025-11-20` (Claude Opus 4.5 only) +- **Google Cloud Vertex AI**: Not supported + +You don't need to manually specify beta headers—LiteLLM handles this automatically. ::: ## When to Use Input Examples @@ -378,13 +384,14 @@ Input examples work seamlessly with other Anthropic tool features: ## Provider Support -LiteLLM supports input examples across all Anthropic-compatible providers: +LiteLLM supports input examples across the following Anthropic-compatible providers: -- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`) -- **Azure Anthropic** (`azure/claude-sonnet-4-5-20250929`) -- **Vertex AI Anthropic** (`vertex_ai/claude-sonnet-4-5-20250929`) +- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`) ✅ +- **Azure Anthropic / Microsoft Foundry** (`azure/claude-sonnet-4-5-20250929`) ✅ +- **Amazon Bedrock** (`bedrock/invoke/anthropic.claude-opus-4-5-20251101-v1:0`) ✅ (Opus 4.5 only) +- **Google Cloud Vertex AI** (`vertex_ai/claude-sonnet-4-5-20250929`) ❌ Not supported -The beta header is automatically added when LiteLLM detects tools with `input_examples` field. +The beta header (`advanced-tool-use-2025-11-20`) is automatically added when LiteLLM detects tools with the `input_examples` field. ## Troubleshooting diff --git a/docs/my-website/docs/providers/anthropic_tool_search.md b/docs/my-website/docs/providers/anthropic_tool_search.md index 7b9e7cfaa72..28ce5688eeb 100644 --- a/docs/my-website/docs/providers/anthropic_tool_search.md +++ b/docs/my-website/docs/providers/anthropic_tool_search.md @@ -290,7 +290,13 @@ response = client.chat.completions.create( ### Beta Header -LiteLLM automatically adds the `advanced-tool-use-2025-11-20` beta header when tool search tools are detected. You don't need to manually specify it. +LiteLLM automatically detects tool search tools and adds the appropriate beta header based on your provider: + +- **Anthropic API & Microsoft Foundry**: `advanced-tool-use-2025-11-20` +- **Google Cloud Vertex AI**: `tool-search-tool-2025-10-19` +- **Amazon Bedrock** (Invoke API, Opus 4.5 only): `tool-search-tool-2025-10-19` + +You don't need to manually specify beta headers—LiteLLM handles this automatically. ### Deferred Loading @@ -387,9 +393,18 @@ If Claude references a tool that isn't in your deferred tools list, you'll get a - Not compatible with tool use examples - Requires Claude Opus 4.5 or Sonnet 4.5 - On Bedrock, only available via invoke API (not converse API) +- On Bedrock, only supported for Claude Opus 4.5 (not Sonnet 4.5) +- BM25 variant (`tool_search_tool_bm25_20251119`) is not supported on Bedrock - Maximum 10,000 tools in catalog - Returns 3-5 most relevant tools per search +### Bedrock-Specific Notes + +When using Bedrock's Invoke API: +- The regex variant (`tool_search_tool_regex_20251119`) is automatically normalized to `tool_search_tool_regex` +- The BM25 variant (`tool_search_tool_bm25_20251119`) is automatically filtered out as it's not supported +- Tool search is only available for Claude Opus 4.5 models + ## Additional Resources - [Anthropic Tool Search Documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/tool-search) diff --git a/docs/my-website/docs/providers/azure/azure_anthropic.md b/docs/my-website/docs/providers/azure/azure_anthropic.md index 771912646b5..4c722b30397 100644 --- a/docs/my-website/docs/providers/azure/azure_anthropic.md +++ b/docs/my-website/docs/providers/azure/azure_anthropic.md @@ -16,7 +16,7 @@ Azure Foundry supports the following Claude models: | Property | Details | |-------|-------| | Description | Claude models deployed via Microsoft Azure Foundry. Uses the same API as Anthropic's Messages API but with Azure authentication. | -| Provider Route on LiteLLM | `azure/` (add this prefix to Claude model names - e.g. `azure/claude-sonnet-4-5`) | +| Provider Route on LiteLLM | `azure_ai/` (add this prefix to Claude model names - e.g. `azure_ai/claude-sonnet-4-5`) | | Provider Doc | [Azure Foundry Claude Models ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude) | | API Endpoint | `https://.services.ai.azure.com/anthropic/v1/messages` | | Supported Endpoints | `/chat/completions`, `/anthropic/v1/messages`| @@ -68,7 +68,7 @@ os.environ["AZURE_API_BASE"] = "https://.services.ai.azure.com/an # Make a completion request response = completion( - model="azure/claude-sonnet-4-5", + model="azure_ai/claude-sonnet-4-5", messages=[ {"role": "user", "content": "What are 3 things to visit in Seattle?"} ], @@ -85,7 +85,7 @@ print(response) import litellm response = litellm.completion( - model="azure/claude-sonnet-4-5", + model="azure_ai/claude-sonnet-4-5", api_base="https://.services.ai.azure.com/anthropic", api_key="your-azure-api-key", messages=[ @@ -101,7 +101,7 @@ response = litellm.completion( import litellm response = litellm.completion( - model="azure/claude-sonnet-4-5", + model="azure_ai/claude-sonnet-4-5", api_base="https://.services.ai.azure.com/anthropic", azure_ad_token="your-azure-ad-token", messages=[ @@ -117,7 +117,7 @@ response = litellm.completion( from litellm import completion response = completion( - model="azure/claude-sonnet-4-5", + model="azure_ai/claude-sonnet-4-5", messages=[ {"role": "user", "content": "Write a short story"} ], @@ -136,7 +136,7 @@ for chunk in response: from litellm import completion response = completion( - model="azure/claude-sonnet-4-5", + model="azure_ai/claude-sonnet-4-5", messages=[ {"role": "user", "content": "What's the weather in Seattle?"} ], @@ -181,7 +181,7 @@ export AZURE_API_BASE="https://.services.ai.azure.com/anthropic" model_list: - model_name: claude-sonnet-4-5 litellm_params: - model: azure/claude-sonnet-4-5 + model: azure_ai/claude-sonnet-4-5 api_base: https://.services.ai.azure.com/anthropic api_key: os.environ/AZURE_API_KEY ``` @@ -331,7 +331,7 @@ os.environ["AZURE_API_BASE"] = "https://my-resource.services.ai.azure.com/anthro # Make a request response = completion( - model="azure/claude-sonnet-4-5", + model="azure_ai/claude-sonnet-4-5", messages=[ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Explain quantum computing in simple terms."} @@ -358,7 +358,7 @@ Or pass it directly: ```python response = completion( - model="azure/claude-sonnet-4-5", + model="azure_ai/claude-sonnet-4-5", api_base="https://.services.ai.azure.com/anthropic", # ... ) diff --git a/docs/my-website/docs/providers/azure_ai.md b/docs/my-website/docs/providers/azure_ai.md index b1b5de5bb34..68e2df676e6 100644 --- a/docs/my-website/docs/providers/azure_ai.md +++ b/docs/my-website/docs/providers/azure_ai.md @@ -312,6 +312,82 @@ LiteLLM supports **ALL** azure ai models. Here's a few examples: | mistral-large-latest | `completion(model="azure_ai/mistral-large-latest", messages)` | | AI21-Jamba-Instruct | `completion(model="azure_ai/ai21-jamba-instruct", messages)` | +## Usage - Azure Anthropic (Azure Foundry Claude) + +LiteLLM funnels Azure Claude deployments through the `azure_ai/` provider so Claude Opus models on Azure Foundry keep working with Tool Search, Effort, streaming, and the rest of the advanced feature set. Point `AZURE_AI_API_BASE` to `https://.services.ai.azure.com/anthropic` (LiteLLM appends `/v1/messages` automatically) and authenticate with `AZURE_AI_API_KEY` or an Azure AD token. + + + + +```python +import os +from litellm import completion + +# Configure Azure credentials +os.environ["AZURE_AI_API_KEY"] = "your-azure-ai-api-key" +os.environ["AZURE_AI_API_BASE"] = "https://my-resource.services.ai.azure.com/anthropic" + +response = completion( + model="azure_ai/claude-opus-4-1", + messages=[{"role": "user", "content": "Explain how Azure Anthropic hosts Claude Opus differently from the public Anthropic API."}], + max_tokens=1200, + temperature=0.7, + stream=True, +) + +for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="", flush=True) +``` + + + + +**1. Set environment variables** + +```bash +export AZURE_AI_API_KEY="your-azure-ai-api-key" +export AZURE_AI_API_BASE="https://my-resource.services.ai.azure.com/anthropic" +``` + +**2. Configure the proxy** + +```yaml +model_list: + - model_name: claude-4-azure + litellm_params: + model: azure_ai/claude-opus-4-1 + api_key: os.environ/AZURE_AI_API_KEY + api_base: os.environ/AZURE_AI_API_BASE +``` + +**3. Start LiteLLM** + +```bash +litellm --config /path/to/config.yaml +``` + +**4. Test the Azure Claude route** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer $LITELLM_KEY' \ + --data '{ + "model": "claude-4-azure", + "messages": [ + { + "role": "user", + "content": "How do I use Claude Opus 4 via Azure Anthropic in LiteLLM?" + } + ], + "max_tokens": 1024 + }' +``` + + + + ## Rerank Endpoint @@ -397,4 +473,5 @@ curl http://0.0.0.0:4000/rerank \ ``` - \ No newline at end of file + + diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index 9e22f67527e..17c0d38111d 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -7,7 +7,7 @@ ALL Bedrock models (Anthropic, Meta, Deepseek, Mistral, Amazon, etc.) are Suppor | Property | Details | |-------|-------| | Description | Amazon Bedrock is a fully managed service that offers a choice of high-performing foundation models (FMs). | -| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models), [`bedrock/openai/`](./bedrock_imported.md#openai-compatible-imported-models-qwen-25-vl-etc) | +| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models), [`bedrock/qwen2/`](./bedrock_imported.md#qwen2-imported-models), [`bedrock/openai/`](./bedrock_imported.md#openai-compatible-imported-models-qwen-25-vl-etc) | | Provider Doc | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) | | Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings`, `/images/generations` | | Rerank Endpoint | `/rerank` | @@ -43,6 +43,8 @@ export AWS_BEARER_TOKEN_BEDROCK="your-api-key" Option 2: use the api_key parameter to pass in API key for completion, embedding, image_generation API calls. + + ```python response = completion( model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0", @@ -50,7 +52,17 @@ response = completion( api_key="your-api-key" ) ``` - + + +```yaml +model_list: + - model_name: bedrock-claude-3-sonnet + litellm_params: + model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0 + api_key: os.environ/AWS_BEARER_TOKEN_BEDROCK +``` + + ## Usage @@ -1683,6 +1695,131 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ +## TwelveLabs Pegasus - Video Understanding + +TwelveLabs Pegasus 1.2 is a video understanding model that can analyze and describe video content. LiteLLM supports this model through Bedrock's `/invoke` endpoint. + +| Property | Details | +|----------|---------| +| Provider Route | `bedrock/us.twelvelabs.pegasus-1-2-v1:0`, `bedrock/eu.twelvelabs.pegasus-1-2-v1:0` | +| Provider Documentation | [TwelveLabs Pegasus Docs ↗](https://docs.twelvelabs.io/docs/models/pegasus) | +| Supported Parameters | `max_tokens`, `temperature`, `response_format` | +| Media Input | S3 URI or base64-encoded video | + +### Supported Features + +- **Video Analysis**: Analyze video content from S3 or base64 input +- **Structured Output**: Support for JSON schema response format +- **S3 Integration**: Support for S3 video URLs with bucket owner specification + +### Usage with S3 Video + + + + +```python title="TwelveLabs Pegasus SDK Usage" showLineNumbers +from litellm import completion +import os + +# Set AWS credentials +os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key" +os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key" +os.environ["AWS_REGION_NAME"] = "us-east-1" + +response = completion( + model="bedrock/us.twelvelabs.pegasus-1-2-v1:0", + messages=[{"role": "user", "content": "Describe what happens in this video."}], + mediaSource={ + "s3Location": { + "uri": "s3://your-bucket/video.mp4", + "bucketOwner": "123456789012", # 12-digit AWS account ID + } + }, + temperature=0.2 +) + +print(response.choices[0].message.content) +``` + + + + + +**1. Add to config** + +```yaml title="config.yaml" showLineNumbers +model_list: + - model_name: pegasus-video + litellm_params: + model: bedrock/us.twelvelabs.pegasus-1-2-v1:0 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: os.environ/AWS_REGION_NAME +``` + +**2. Start proxy** + +```bash title="Start LiteLLM Proxy" showLineNumbers +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash title="Test Pegasus via Proxy" showLineNumbers +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "pegasus-video", + "messages": [ + { + "role": "user", + "content": "Describe what happens in this video." + } + ], + "mediaSource": { + "s3Location": { + "uri": "s3://your-bucket/video.mp4", + "bucketOwner": "123456789012" + } + }, + "temperature": 0.2 + }' +``` + + + + +### Usage with Base64 Video + +You can also pass video content directly as base64: + +```python title="Base64 Video Input" showLineNumbers +from litellm import completion +import base64 + +# Read video file and encode to base64 +with open("video.mp4", "rb") as video_file: + video_base64 = base64.b64encode(video_file.read()).decode("utf-8") + +response = completion( + model="bedrock/us.twelvelabs.pegasus-1-2-v1:0", + messages=[{"role": "user", "content": "What is happening in this video?"}], + mediaSource={ + "base64String": video_base64 + }, + temperature=0.2, +) + +print(response.choices[0].message.content) +``` + +### Important Notes + +- **Response Format**: The model supports structured output via `response_format` with JSON schema + ## Provisioned throughput models To use provisioned throughput Bedrock models pass - `model=bedrock/`, example `model=bedrock/anthropic.claude-v2`. Set `model` to any of the [Supported AWS models](#supported-aws-bedrock-models) @@ -1743,6 +1880,8 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re | Meta Llama 2 Chat 70b | `completion(model='bedrock/meta.llama2-70b-chat-v1', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` | | Mistral 7B Instruct | `completion(model='bedrock/mistral.mistral-7b-instruct-v0:2', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` | | Mixtral 8x7B Instruct | `completion(model='bedrock/mistral.mixtral-8x7b-instruct-v0:1', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` | +| TwelveLabs Pegasus 1.2 (US) | `completion(model='bedrock/us.twelvelabs.pegasus-1-2-v1:0', messages=messages, mediaSource={...})` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` | +| TwelveLabs Pegasus 1.2 (EU) | `completion(model='bedrock/eu.twelvelabs.pegasus-1-2-v1:0', messages=messages, mediaSource={...})` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` | ## Bedrock Embedding diff --git a/docs/my-website/docs/providers/bedrock_batches.md b/docs/my-website/docs/providers/bedrock_batches.md index a1116f41076..19446fda837 100644 --- a/docs/my-website/docs/providers/bedrock_batches.md +++ b/docs/my-website/docs/providers/bedrock_batches.md @@ -172,6 +172,97 @@ curl http://localhost:4000/v1/batches \ +### 4. Retrieve batch results + +Once the batch job is completed, download the results from S3: + + + + +```python showLineNumbers title="bedrock_batch.py" +... +# Wait for batch completion (check status periodically) +batch_status = client.batches.retrieve(batch_id=batch.id) + +if batch_status.status == "completed": + # Download the output file + result = client.files.content( + file_id=batch_status.output_file_id, + extra_headers={"custom-llm-provider": "bedrock"} + ) + + # Save or process the results + with open("batch_output.jsonl", "wb") as f: + f.write(result.content) + + # Parse JSONL results + for line in result.text.strip().split('\n'): + record = json.loads(line) + print(f"Record ID: {record['recordId']}") + print(f"Output: {record.get('modelOutput', {})}") +``` + + + + +```bash showLineNumbers title="Download Batch Results" +# First retrieve batch to get output_file_id +curl http://localhost:4000/v1/batches/batch_abc123 \ + -H "Authorization: Bearer sk-1234" + +# Then download the output file +curl http://localhost:4000/v1/files/{output_file_id}/content \ + -H "Authorization: Bearer sk-1234" \ + -H "custom-llm-provider: bedrock" \ + -o batch_output.jsonl +``` + + + + +```python showLineNumbers title="bedrock_batch.py" +import litellm +from litellm import file_content + +# Download using litellm directly (bypasses proxy managed files) +result = file_content( + file_id=batch_status.output_file_id, # Can be S3 URI or unified file ID + custom_llm_provider="bedrock", + aws_region_name="us-west-2", +) + +# Process results +print(result.text) +``` + + + + +**Output Format:** + +The batch output file is in JSONL format with each line containing: + +```json +{ + "recordId": "request-1", + "modelInput": { + "messages": [...], + "max_tokens": 1000 + }, + "modelOutput": { + "content": [...], + "id": "msg_abc123", + "model": "claude-3-5-sonnet-20240620-v1:0", + "role": "assistant", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 15, + "output_tokens": 10 + } + } +} +``` + ## FAQ ### Where are my files written? diff --git a/docs/my-website/docs/providers/bedrock_embedding.md b/docs/my-website/docs/providers/bedrock_embedding.md index 76c9606533e..e2e7c0dcedd 100644 --- a/docs/my-website/docs/providers/bedrock_embedding.md +++ b/docs/my-website/docs/providers/bedrock_embedding.md @@ -4,7 +4,8 @@ | Provider | LiteLLM Route | AWS Documentation | Cost Tracking | |----------|---------------|-------------------|---------------| -| Amazon Titan | `bedrock/amazon.*` | [Amazon Titan Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/titan-embedding-models.html) | ✅ | +| Amazon Titan | `bedrock/amazon.titan-*` | [Amazon Titan Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/titan-embedding-models.html) | ✅ | +| Amazon Nova | `bedrock/amazon.nova-*` | [Amazon Nova Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/nova-embed.html) | ✅ | | Cohere | `bedrock/cohere.*` | [Cohere Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-embed.html) | ✅ | | TwelveLabs | `bedrock/us.twelvelabs.*` | [TwelveLabs](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-twelvelabs.html) | ✅ | @@ -16,6 +17,7 @@ LiteLLM supports AWS Bedrock's async-invoke feature for embedding models that re | Provider | Async Invoke Route | Use Case | |----------|-------------------|----------| +| Amazon Nova | `bedrock/async_invoke/amazon.nova-2-multimodal-embeddings-v1:0` | Multimodal embeddings with segmentation for long text, video, and audio | | TwelveLabs Marengo | `bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0` | Video, audio, image, and text embeddings | ### Required Parameters @@ -116,7 +118,7 @@ def check_async_job_status(invocation_arn, aws_region_name="us-east-1"): """Check the status of an async invoke job using LiteLLM batch API""" try: response = retrieve_batch( - batch_id=invocation_arn, + batch_id=invocation_arn, # Pass the invocation ARN here custom_llm_provider="bedrock", aws_region_name=aws_region_name ) @@ -128,11 +130,47 @@ def check_async_job_status(invocation_arn, aws_region_name="us-east-1"): # Check status status = check_async_job_status(invocation_arn, "us-east-1") if status: - print(f"Job Status: {status.status}") - print(f"Output Location: {status.output_file_id}") + print(f"Job Status: {status.status}") # "in_progress", "completed", or "failed" + print(f"Output Location: {status.metadata['output_file_id']}") # S3 URI where results are stored ``` -**Note:** The actual embedding results are stored in S3. The `output_file_id` from the batch status can be used to locate the results file in your S3 bucket. +#### Polling Until Complete + +Here's a complete example of polling for job completion: + +```python +def wait_for_async_job(invocation_arn, aws_region_name="us-east-1", max_wait=3600): + """Poll job status until completion""" + start_time = time.time() + + while True: + status = retrieve_batch( + batch_id=invocation_arn, + custom_llm_provider="bedrock", + aws_region_name=aws_region_name, + ) + + if status.status == "completed": + print("✅ Job completed!") + return status + elif status.status == "failed": + error_msg = status.metadata.get('failure_message', 'Unknown error') + raise Exception(f"❌ Job failed: {error_msg}") + else: + elapsed = time.time() - start_time + if elapsed > max_wait: + raise TimeoutError(f"Job timed out after {max_wait} seconds") + + print(f"⏳ Job still processing... (elapsed: {elapsed:.0f}s)") + time.sleep(10) # Wait 10 seconds before checking again + +# Wait for completion +completed_status = wait_for_async_job(invocation_arn) +output_s3_uri = completed_status.metadata['output_file_id'] +print(f"Results available at: {output_s3_uri}") +``` + +**Note:** The actual embedding results are stored in S3. When the job is completed, download the results from the S3 location specified in `status.metadata['output_file_id']`. The results will be in JSON/JSONL format containing the embedding vectors. ### Error Handling @@ -179,7 +217,7 @@ except Exception as e: ### Limitations -- Async-invoke is currently only supported for TwelveLabs Marengo models +- Async-invoke is supported for TwelveLabs Marengo and Amazon Nova models - Results are stored in S3 and must be retrieved separately using the output file ID - Job status checking requires using LiteLLM's `retrieve_batch()` function - No built-in polling mechanism in LiteLLM (must implement your own status checking loop) @@ -259,6 +297,7 @@ print(response) | Model Name | Usage | Supported Additional OpenAI params | |----------------------|---------------------------------------------|-----| +| **Amazon Nova Multimodal Embeddings** | `embedding(model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0", input=input)` | Supports multimodal input (text, image, video, audio), multiple purposes, dimensions (256, 384, 1024, 3072) | | Titan Embeddings V2 | `embedding(model="bedrock/amazon.titan-embed-text-v2:0", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py#L59) | | Titan Embeddings - V1 | `embedding(model="bedrock/amazon.titan-embed-text-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py#L53) | Titan Multimodal Embeddings | `embedding(model="bedrock/amazon.titan-embed-image-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py#L28) | diff --git a/docs/my-website/docs/providers/bedrock_imported.md b/docs/my-website/docs/providers/bedrock_imported.md index 8b0dd721c3c..0784f716925 100644 --- a/docs/my-website/docs/providers/bedrock_imported.md +++ b/docs/my-website/docs/providers/bedrock_imported.md @@ -203,6 +203,71 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ +### Qwen2 Imported Models + +| Property | Details | +|----------|---------| +| Provider Route | `bedrock/qwen2/{model_arn}` | +| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html) | +| Note | Qwen2 and Qwen3 architectures are mostly similar. The main difference is in the response format: Qwen2 uses "text" field while Qwen3 uses "generation" field. | + + + + +```python +from litellm import completion +import os + +response = completion( + model="bedrock/qwen2/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen2-model", # bedrock/qwen2/{your-model-arn} + messages=[{"role": "user", "content": "Tell me a joke"}], + max_tokens=100, + temperature=0.7 +) +``` + + + + + +**1. Add to config** + +```yaml +model_list: + - model_name: Qwen2-72B + litellm_params: + model: bedrock/qwen2/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen2-model + +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "Qwen2-72B", # 👈 the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + ### OpenAI-Compatible Imported Models (Qwen 2.5 VL, etc.) Use this route for Bedrock imported models that follow the **OpenAI Chat Completions API spec**. This includes models like Qwen 2.5 VL that accept OpenAI-formatted messages with support for vision (images), tool calling, and other OpenAI features. diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md index 1b21ed8d03c..f33c492f182 100644 --- a/docs/my-website/docs/providers/gemini.md +++ b/docs/my-website/docs/providers/gemini.md @@ -2006,3 +2006,34 @@ curl -L -X POST 'http://localhost:4000/v1/chat/completions' \ +### Image Generation Pricing + +Gemini image generation models (like `gemini-3-pro-image-preview`) return `image_tokens` in the response usage. These tokens are priced differently from text tokens: + +| Token Type | Price per 1M tokens | Price per token | +|------------|---------------------|-----------------| +| Text output | $12 | $0.000012 | +| Image output | $120 | $0.00012 | + +The number of image tokens depends on the output resolution: + +| Resolution | Tokens per image | Cost per image | +|------------|------------------|----------------| +| 1K-2K (1024x1024 to 2048x2048) | 1,120 | $0.134 | +| 4K (4096x4096) | 2,000 | $0.24 | + +LiteLLM automatically calculates costs using `output_cost_per_image_token` from the model pricing configuration. + +**Example response usage:** +```json +{ + "completion_tokens_details": { + "reasoning_tokens": 225, + "text_tokens": 0, + "image_tokens": 1120 + } +} +``` + +For more details, see [Google's Gemini pricing documentation](https://ai.google.dev/gemini-api/docs/pricing). + diff --git a/docs/my-website/docs/providers/github_copilot.md b/docs/my-website/docs/providers/github_copilot.md index 2ebe6eacb1c..306c9f949ec 100644 --- a/docs/my-website/docs/providers/github_copilot.md +++ b/docs/my-website/docs/providers/github_copilot.md @@ -15,7 +15,7 @@ https://docs.github.com/en/copilot |-------|-------| | Description | GitHub Copilot Chat API provides access to GitHub's AI-powered coding assistant. | | Provider Route on LiteLLM | `github_copilot/` | -| Supported Endpoints | `/chat/completions` | +| Supported Endpoints | `/chat/completions`, `/embeddings` | | API Reference | [GitHub Copilot docs](https://docs.github.com/en/copilot) | ## Authentication @@ -62,6 +62,34 @@ for chunk in stream: print(chunk.choices[0].delta.content, end="") ``` +### Responses + +For GPT Codex models, only responses API is supported. + +```python showLineNumbers title="GitHub Copilot Responses" +import litellm + +response = await litellm.aresponses( + model="github_copilot/gpt-5.1-codex", + input="Write a Python hello world", + max_output_tokens=500 +) + +print(response) +``` + +### Embedding + +```python showLineNumbers title="GitHub Copilot Embedding" +import litellm + +response = litellm.embedding( + model="github_copilot/text-embedding-3-small", + input=["good morning from litellm"] +) +print(response) +``` + ## Usage - LiteLLM Proxy Add the following to your LiteLLM Proxy configuration file: @@ -71,6 +99,16 @@ model_list: - model_name: github_copilot/gpt-4 litellm_params: model: github_copilot/gpt-4 + - model_name: github_copilot/gpt-5.1-codex + model_info: + mode: responses + litellm_params: + model: github_copilot/gpt-5.1-codex + - model_name: github_copilot/text-embedding-ada-002 + model_info: + mode: embedding + litellm_params: + model: github_copilot/text-embedding-ada-002 ``` Start your LiteLLM Proxy server: @@ -180,7 +218,7 @@ extra_headers = { "editor-version": "vscode/1.85.1", # Editor version "editor-plugin-version": "copilot/1.155.0", # Plugin version "Copilot-Integration-Id": "vscode-chat", # Integration ID - "user-agent": "GithubCopilot/1.155.0" # User agent + "user-agent": "GithubCopilot/1.155.0" # User agent } ``` diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md index 6f46807c89a..f1f88999d83 100644 --- a/docs/my-website/docs/providers/openai.md +++ b/docs/my-website/docs/providers/openai.md @@ -191,6 +191,7 @@ os.environ["OPENAI_BASE_URL"] = "https://your_host/v1" # OPTIONAL | gpt-5.1 | `response = completion(model="gpt-5.1", messages=messages)` | | gpt-5.1-codex | `response = completion(model="gpt-5.1-codex", messages=messages)` | | gpt-5.1-codex-mini | `response = completion(model="gpt-5.1-codex-mini", messages=messages)` | +| gpt-5.1-codex-max | `response = completion(model="gpt-5.1-codex-max", messages=messages)` | | gpt-4.1 | `response = completion(model="gpt-4.1", messages=messages)` | | gpt-4.1-mini | `response = completion(model="gpt-4.1-mini", messages=messages)` | | gpt-4.1-nano | `response = completion(model="gpt-4.1-nano", messages=messages)` | @@ -427,7 +428,7 @@ Expected Response: ### Advanced: Using `reasoning_effort` with `summary` field -By default, `reasoning_effort` accepts a string value (`"none"`, `"minimal"`, `"low"`, `"medium"`, `"high"`) and only sets the effort level without including a reasoning summary. +By default, `reasoning_effort` accepts a string value (`"none"`, `"minimal"`, `"low"`, `"medium"`, `"high"`, `"xhigh"`—`"xhigh"` is only supported on `gpt-5.1-codex-max`) and only sets the effort level without including a reasoning summary. To opt-in to the `summary` feature, you can pass `reasoning_effort` as a dictionary. **Note:** The `summary` field requires your OpenAI organization to have verification status. Using `summary` without verification will result in a 400 error from OpenAI. @@ -494,10 +495,12 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ | `gpt-5-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) | | `gpt-5.1-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) | | `gpt-5.1-codex-mini` | `adaptive` | `low`, `medium`, `high` (no `minimal`) | +| `gpt-5.1-codex-max` | `adaptive` | `low`, `medium`, `high`, `xhigh` (no `minimal`) | | `gpt-5-pro` | `high` | `high` only | **Note:** - GPT-5.1 introduced a new `reasoning_effort="none"` setting for faster, lower-latency responses. This replaces the `"minimal"` setting from GPT-5. +- `gpt-5.1-codex-max` is the only model that supports `reasoning_effort="xhigh"`. All other models will reject this value. - `gpt-5-pro` only accepts `reasoning_effort="high"`. Other values will return an error. - When `reasoning_effort` is not set (None), OpenAI defaults to the value shown in the "Default" column. @@ -509,7 +512,7 @@ The `verbosity` parameter controls the length and detail of responses from GPT-5 **Supported models:** `gpt-5`, `gpt-5.1`, `gpt-5-mini`, `gpt-5-nano`, `gpt-5-pro` -**Note:** GPT-5-Codex models (`gpt-5-codex`, `gpt-5.1-codex`, `gpt-5.1-codex-mini`) do **not** support the `verbosity` parameter. +**Note:** GPT-5-Codex models (`gpt-5-codex`, `gpt-5.1-codex`, `gpt-5.1-codex-mini`, `gpt-5.1-codex-max`) do **not** support the `verbosity` parameter. **Use cases:** - **`"low"`**: Best for concise answers or simple code generation (e.g., SQL queries) @@ -988,4 +991,4 @@ response = completion( LiteLLM supports OpenAI's video generation models including Sora. -For detailed documentation on video generation, see [OpenAI Video Generation →](./openai/video_generation.md) \ No newline at end of file +For detailed documentation on video generation, see [OpenAI Video Generation →](./openai/video_generation.md) diff --git a/docs/my-website/docs/providers/openai_compatible.md b/docs/my-website/docs/providers/openai_compatible.md index 2f11379a8db..f67500f2b10 100644 --- a/docs/my-website/docs/providers/openai_compatible.md +++ b/docs/my-website/docs/providers/openai_compatible.md @@ -11,7 +11,7 @@ Selecting `openai` as the provider routes your request to an OpenAI-compatible e This library **requires** an API key for all requests, either through the `api_key` parameter or the `OPENAI_API_KEY` environment variable. -If you don’t want to provide a fake API key in each request, consider using a provider that directly matches your +If you don't want to provide a fake API key in each request, consider using a provider that directly matches your OpenAI-compatible endpoint, such as [`hosted_vllm`](/docs/providers/vllm) or [`llamafile`](/docs/providers/llamafile). ::: @@ -150,4 +150,4 @@ model_list: api_base: http://my-custom-base api_key: "" supports_system_message: False # 👈 KEY CHANGE -``` \ No newline at end of file +``` diff --git a/docs/my-website/docs/providers/ovhcloud.md b/docs/my-website/docs/providers/ovhcloud.md index 6c42208f2cc..94625b0f2ed 100644 --- a/docs/my-website/docs/providers/ovhcloud.md +++ b/docs/my-website/docs/providers/ovhcloud.md @@ -311,6 +311,21 @@ response = embedding( print(response.data) ``` +### Audio Transcription + +```python +from litellm import transcription + +audio_file = open("path/to/your/audio.wav", "rb") + +response = transcription( + model="ovhcloud/whisper-large-v3-turbo", + file=audio_file +) + +print(response.text) +``` + ## Usage with LiteLLM Proxy Server Here's how to call a OVHCloud AI Endpoints model with the LiteLLM Proxy Server diff --git a/docs/my-website/docs/providers/ragflow.md b/docs/my-website/docs/providers/ragflow.md new file mode 100644 index 00000000000..73223bd07b5 --- /dev/null +++ b/docs/my-website/docs/providers/ragflow.md @@ -0,0 +1,244 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# RAGFlow + +Litellm supports Ragflow's chat completions APIs + +## Supported Features + +- ✅ Chat completions +- ✅ Streaming responses +- ✅ Both chat and agent endpoints +- ✅ Multiple credential sources (params, env vars, litellm_params) +- ✅ OpenAI-compatible API format + + +## API Key + +```python +# env variable +os.environ['RAGFLOW_API_KEY'] +``` + +## API Base + +```python +# env variable +os.environ['RAGFLOW_API_BASE'] +``` + +## Overview + +RAGFlow provides OpenAI-compatible APIs with unique path structures that include chat and agent IDs: + +- **Chat endpoint**: `/api/v1/chats_openai/{chat_id}/chat/completions` +- **Agent endpoint**: `/api/v1/agents_openai/{agent_id}/chat/completions` + +The model name format embeds the endpoint type and ID: +- Chat: `ragflow/chat/{chat_id}/{model_name}` +- Agent: `ragflow/agent/{agent_id}/{model_name}` + + +## Sample Usage - Chat Endpoint + +```python +from litellm import completion +import os + +os.environ['RAGFLOW_API_KEY'] = "your-ragflow-api-key" +os.environ['RAGFLOW_API_BASE'] = "http://localhost:9380" # or your hosted URL + +response = completion( + model="ragflow/chat/my-chat-id/gpt-4o-mini", + messages=[{"role": "user", "content": "How does the deep doc understanding work?"}] +) +print(response) +``` + +## Sample Usage - Agent Endpoint + +```python +from litellm import completion +import os + +os.environ['RAGFLOW_API_KEY'] = "your-ragflow-api-key" +os.environ['RAGFLOW_API_BASE'] = "http://localhost:9380" # or your hosted URL + +response = completion( + model="ragflow/agent/my-agent-id/gpt-4o-mini", + messages=[{"role": "user", "content": "What are the key features?"}] +) +print(response) +``` + +## Sample Usage - With Parameters + +You can also pass `api_key` and `api_base` directly as parameters: + +```python +from litellm import completion + +response = completion( + model="ragflow/chat/my-chat-id/gpt-4o-mini", + messages=[{"role": "user", "content": "Hello!"}], + api_key="your-ragflow-api-key", + api_base="http://localhost:9380" +) +print(response) +``` + +## Sample Usage - Streaming + +```python +from litellm import completion +import os + +os.environ['RAGFLOW_API_KEY'] = "your-ragflow-api-key" +os.environ['RAGFLOW_API_BASE'] = "http://localhost:9380" + +response = completion( + model="ragflow/agent/my-agent-id/gpt-4o-mini", + messages=[{"role": "user", "content": "Explain RAGFlow"}], + stream=True +) + +for chunk in response: + print(chunk) +``` + +## Model Name Format + +The model name must follow one of these formats: + +### Chat Endpoint +``` +ragflow/chat/{chat_id}/{model_name} +``` + +Example: `ragflow/chat/my-chat-id/gpt-4o-mini` + +### Agent Endpoint +``` +ragflow/agent/{agent_id}/{model_name} +``` + +Example: `ragflow/agent/my-agent-id/gpt-4o-mini` + +Where: +- `{chat_id}` or `{agent_id}` is the ID of your chat or agent in RAGFlow +- `{model_name}` is the actual model name (e.g., `gpt-4o-mini`, `gpt-4o`, etc.) + +## Configuration Sources + +LiteLLM supports multiple ways to provide credentials, checked in this order: + +1. **Function parameters**: `api_key="..."`, `api_base="..."` +2. **litellm_params**: `litellm_params={"api_key": "...", "api_base": "..."}` +3. **Environment variables**: `RAGFLOW_API_KEY`, `RAGFLOW_API_BASE` +4. **Global litellm settings**: `litellm.api_key`, `litellm.api_base` + +## Usage - LiteLLM Proxy Server + +### 1. Save key in your environment + +```bash +export RAGFLOW_API_KEY="your-ragflow-api-key" +export RAGFLOW_API_BASE="http://localhost:9380" +``` + +### 2. Start the proxy + + + + +```yaml +model_list: + - model_name: ragflow-chat-gpt4 + litellm_params: + model: ragflow/chat/my-chat-id/gpt-4o-mini + api_key: os.environ/RAGFLOW_API_KEY + api_base: os.environ/RAGFLOW_API_BASE + - model_name: ragflow-agent-gpt4 + litellm_params: + model: ragflow/agent/my-agent-id/gpt-4o-mini + api_key: os.environ/RAGFLOW_API_KEY + api_base: os.environ/RAGFLOW_API_BASE +``` + + + + +```bash +$ litellm --config /path/to/config.yaml + +# Server running on http://0.0.0.0:4000 +``` + + + + +### 3. Test it + + + + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "ragflow-chat-gpt4", + "messages": [ + {"role": "user", "content": "How does RAGFlow work?"} + ] + }' +``` + + + + +```python +from openai import OpenAI + +client = OpenAI( + api_key="sk-1234", # Your LiteLLM proxy key + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="ragflow-chat-gpt4", + messages=[ + {"role": "user", "content": "How does RAGFlow work?"} + ] +) +print(response) +``` + + + + +## API Base URL Handling + +The `api_base` parameter can be provided with or without `/v1` suffix. LiteLLM will automatically handle it: + +- `http://localhost:9380` → `http://localhost:9380/api/v1/chats_openai/{chat_id}/chat/completions` +- `http://localhost:9380/v1` → `http://localhost:9380/api/v1/chats_openai/{chat_id}/chat/completions` +- `http://localhost:9380/api/v1` → `http://localhost:9380/api/v1/chats_openai/{chat_id}/chat/completions` + +All three formats will work correctly. + +## Error Handling + +If you encounter errors: + +1. **Invalid model format**: Ensure your model name follows `ragflow/{chat|agent}/{id}/{model_name}` format +2. **Missing api_base**: Provide `api_base` via parameter, environment variable, or litellm_params +3. **Connection errors**: Verify your RAGFlow server is running and accessible at the provided `api_base` + +:::info + +For more information about passing provider-specific parameters, [go here](../completion/provider_specific_params.md) + +::: + diff --git a/docs/my-website/docs/providers/ragflow_vector_store.md b/docs/my-website/docs/providers/ragflow_vector_store.md new file mode 100644 index 00000000000..bc014cacbe6 --- /dev/null +++ b/docs/my-website/docs/providers/ragflow_vector_store.md @@ -0,0 +1,349 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import Image from '@theme/IdealImage'; + +# RAGFlow Vector Stores + +Litellm support creation and management of datasets for document processing and knowledge base management in Ragflow. + +| Property | Details | +|----------|---------| +| Description | RAGFlow datasets enable document processing, chunking, and knowledge base management for RAG applications. | +| Provider Route on LiteLLM | `ragflow` in the litellm vector_store_registry | +| Provider Doc | [RAGFlow API Documentation ↗](https://ragflow.io/docs) | +| Supported Operations | Dataset Management (Create, List, Update, Delete) | +| Search/Retrieval | ❌ Not supported (management only) | + +## Quick Start + +### LiteLLM Python SDK + +```python showLineNumbers title="Example using LiteLLM Python SDK" +import os +import litellm + +# Set RAGFlow credentials +os.environ["RAGFLOW_API_KEY"] = "your-ragflow-api-key" +os.environ["RAGFLOW_API_BASE"] = "http://localhost:9380" # Optional, defaults to localhost:9380 + +# Create a RAGFlow dataset +response = litellm.vector_stores.create( + name="my-dataset", + custom_llm_provider="ragflow", + metadata={ + "description": "My knowledge base dataset", + "embedding_model": "BAAI/bge-large-zh-v1.5@BAAI", + "chunk_method": "naive" + } +) + +print(f"Created dataset ID: {response.id}") +print(f"Dataset name: {response.name}") +``` + +### LiteLLM Proxy + +#### 1. Configure your vector_store_registry + + + + +```yaml +model_list: + - model_name: gpt-4o-mini + litellm_params: + model: gpt-4o-mini + api_key: os.environ/OPENAI_API_KEY + +vector_store_registry: + - vector_store_name: "ragflow-knowledge-base" + litellm_params: + vector_store_id: "your-dataset-id" + custom_llm_provider: "ragflow" + api_key: os.environ/RAGFLOW_API_KEY + api_base: os.environ/RAGFLOW_API_BASE # Optional + vector_store_description: "RAGFlow dataset for knowledge base" + vector_store_metadata: + source: "Company documentation" +``` + + + + + +On the LiteLLM UI, Navigate to Experimental > Vector Stores > Create Vector Store. On this page you can create a vector store with a name, vector store id and credentials. + + + + + + +#### 2. Create a dataset via Proxy + + + + +```bash +curl http://localhost:4000/v1/vector_stores \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -d '{ + "name": "my-ragflow-dataset", + "custom_llm_provider": "ragflow", + "metadata": { + "description": "Test dataset", + "chunk_method": "naive" + } + }' +``` + + + + + +```python +from openai import OpenAI + +# Initialize client with your LiteLLM proxy URL +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-litellm-api-key" +) + +# Create a RAGFlow dataset +response = client.vector_stores.create( + name="my-ragflow-dataset", + custom_llm_provider="ragflow", + metadata={ + "description": "Test dataset", + "chunk_method": "naive" + } +) + +print(f"Created dataset: {response.id}") +``` + + + + +## Configuration + +### Environment Variables + +RAGFlow vector stores support configuration via environment variables: + +- `RAGFLOW_API_KEY` - Your RAGFlow API key (required) +- `RAGFLOW_API_BASE` - RAGFlow API base URL (optional, defaults to `http://localhost:9380`) + +### Parameters + +You can also pass these via `litellm_params`: + +- `api_key` - RAGFlow API key (overrides `RAGFLOW_API_KEY` env var) +- `api_base` - RAGFlow API base URL (overrides `RAGFLOW_API_BASE` env var) + +## Dataset Creation Options + +### Basic Dataset Creation + +```python +response = litellm.vector_stores.create( + name="basic-dataset", + custom_llm_provider="ragflow" +) +``` + +### Dataset with Chunk Method + +RAGFlow supports various chunk methods for different document types: + + + + +```python +response = litellm.vector_stores.create( + name="general-dataset", + custom_llm_provider="ragflow", + metadata={ + "chunk_method": "naive", + "parser_config": { + "chunk_token_num": 512, + "delimiter": "\n", + "html4excel": False, + "layout_recognize": "DeepDOC" + } + } +) +``` + + + + + +```python +response = litellm.vector_stores.create( + name="book-dataset", + custom_llm_provider="ragflow", + metadata={ + "chunk_method": "book", + "parser_config": { + "raptor": { + "use_raptor": False + } + } + } +) +``` + + + + + +```python +response = litellm.vector_stores.create( + name="qa-dataset", + custom_llm_provider="ragflow", + metadata={ + "chunk_method": "qa", + "parser_config": { + "raptor": { + "use_raptor": False + } + } + } +) +``` + + + + + +```python +response = litellm.vector_stores.create( + name="paper-dataset", + custom_llm_provider="ragflow", + metadata={ + "chunk_method": "paper", + "parser_config": { + "raptor": { + "use_raptor": False + } + } + } +) +``` + + + + +### Dataset with Ingestion Pipeline + +Instead of using a chunk method, you can use an ingestion pipeline: + +```python +response = litellm.vector_stores.create( + name="pipeline-dataset", + custom_llm_provider="ragflow", + metadata={ + "parse_type": 2, # Number of parsers in your pipeline + "pipeline_id": "d0bebe30ae2211f0970942010a8e0005" # 32-character hex ID + } +) +``` + +**Note**: `chunk_method` and `pipeline_id` are mutually exclusive. Use one or the other. + +### Advanced Parser Configuration + +```python +response = litellm.vector_stores.create( + name="advanced-dataset", + custom_llm_provider="ragflow", + metadata={ + "chunk_method": "naive", + "description": "Advanced dataset with custom parser config", + "embedding_model": "BAAI/bge-large-zh-v1.5@BAAI", + "permission": "me", # or "team" + "parser_config": { + "chunk_token_num": 1024, + "delimiter": "\n!?;。；！？", + "html4excel": True, + "layout_recognize": "DeepDOC", + "auto_keywords": 5, + "auto_questions": 3, + "task_page_size": 12, + "raptor": { + "use_raptor": True + }, + "graphrag": { + "use_graphrag": False + } + } + } +) +``` + +## Supported Chunk Methods + +RAGFlow supports the following chunk methods: + +- `naive` - General purpose (default) +- `book` - For book documents +- `email` - For email documents +- `laws` - For legal documents +- `manual` - Manual chunking +- `one` - Single chunk +- `paper` - For academic papers +- `picture` - For image documents +- `presentation` - For presentation documents +- `qa` - Q&A format +- `table` - For table documents +- `tag` - Tag-based chunking + +## RAGFlow-Specific Parameters + +All RAGFlow-specific parameters should be passed via the `metadata` field: + +| Parameter | Type | Description | +|-----------|------|-------------| +| `avatar` | string | Base64 encoding of the avatar (max 65535 chars) | +| `description` | string | Brief description of the dataset (max 65535 chars) | +| `embedding_model` | string | Embedding model name (e.g., "BAAI/bge-large-zh-v1.5@BAAI") | +| `permission` | string | Access permission: "me" (default) or "team" | +| `chunk_method` | string | Chunking method (see supported methods above) | +| `parser_config` | object | Parser configuration (varies by chunk_method) | +| `parse_type` | int | Number of parsers in pipeline (required with pipeline_id) | +| `pipeline_id` | string | 32-character hex pipeline ID (required with parse_type) | + +## Error Handling + +RAGFlow returns error responses in the following format: + +```json +{ + "code": 101, + "message": "Dataset name 'my-dataset' already exists" +} +``` + +LiteLLM automatically maps these to appropriate exceptions: + +- `code != 0` → Raises exception with the error message +- Missing required fields → Raises `ValueError` +- Mutually exclusive parameters → Raises `ValueError` + +## Limitations + +- **Search/Retrieval**: RAGFlow vector stores support dataset management only. Search operations are not supported and will raise `NotImplementedError`. +- **List/Update/Delete**: These operations are not yet implemented through the standard vector store API. Use RAGFlow's native API endpoints directly. + +## Further Reading + +Vector Stores: +- [Vector Store Creation](../vector_stores/create.md) +- [Using Vector Stores with Completions](../completion/knowledgebase.md) +- [Vector Store Registry](../completion/knowledgebase.md#vectorstoreregistry) + diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 70babea3814..33ebf535d29 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -1604,6 +1604,56 @@ litellm.vertex_location = "us-central1 # Your Location | gemini-2.5-flash-preview-09-2025 | `completion('gemini-2.5-flash-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-preview-09-2025', messages)` | | gemini-2.5-flash-lite-preview-09-2025 | `completion('gemini-2.5-flash-lite-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-lite-preview-09-2025', messages)` | +## Private Service Connect (PSC) Endpoints + +LiteLLM supports Vertex AI models deployed to Private Service Connect (PSC) endpoints, allowing you to use custom `api_base` URLs for private deployments. + +### Usage + +```python +from litellm import completion + +# Use PSC endpoint with custom api_base +response = completion( + model="vertex_ai/1234567890", # Numeric endpoint ID + messages=[{"role": "user", "content": "Hello!"}], + api_base="http://10.96.32.8", # Your PSC endpoint + vertex_project="my-project-id", + vertex_location="us-central1", + use_psc_endpoint_format=True +) +``` + +**Key Features:** +- Supports both numeric endpoint IDs and custom model names +- Works with both completion and embedding endpoints +- Automatically constructs full PSC URL: `{api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint}` +- Compatible with streaming requests + +### Configuration + +Add PSC endpoints to your `config.yaml`: + +```yaml +model_list: + - model_name: psc-gemini + litellm_params: + model: vertex_ai/1234567890 # Numeric endpoint ID + api_base: "http://10.96.32.8" # Your PSC endpoint + vertex_project: "my-project-id" + vertex_location: "us-central1" + vertex_credentials: "/path/to/service_account.json" + use_psc_endpoint_format: True + - model_name: psc-embedding + litellm_params: + model: vertex_ai/text-embedding-004 + api_base: "http://10.96.32.8" # Your PSC endpoint + vertex_project: "my-project-id" + vertex_location: "us-central1" + vertex_credentials: "/path/to/service_account.json" + use_psc_endpoint_format: True +``` + ## Fine-tuned Models You can call fine-tuned Vertex AI Gemini models through LiteLLM @@ -2550,355 +2600,6 @@ print(response) - -## **Gemini TTS (Text-to-Speech) Audio Output** - -:::info - -LiteLLM supports Gemini TTS models on Vertex AI that can generate audio responses using the OpenAI-compatible `audio` parameter format. - -::: - -### Supported Models - -LiteLLM supports Gemini TTS models with audio capabilities on Vertex AI (e.g. `vertex_ai/gemini-2.5-flash-preview-tts` and `vertex_ai/gemini-2.5-pro-preview-tts`). For the complete list of available TTS models and voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation). - -### Limitations - -:::warning - -**Important Limitations**: -- Gemini TTS models only support the `pcm16` audio format -- **Streaming support has not been added** to TTS models yet -- The `modalities` parameter must be set to `['audio']` for TTS requests - -::: - -### Quick Start - - - - -```python -from litellm import completion -import json - -## GET CREDENTIALS -file_path = 'path/to/vertex_ai_service_account.json' - -# Load the JSON file -with open(file_path, 'r') as file: - vertex_credentials = json.load(file) - -# Convert to JSON string -vertex_credentials_json = json.dumps(vertex_credentials) - -response = completion( - model="vertex_ai/gemini-2.5-flash-preview-tts", - messages=[{"role": "user", "content": "Say hello in a friendly voice"}], - modalities=["audio"], # Required for TTS models - audio={ - "voice": "Kore", - "format": "pcm16" # Required: must be "pcm16" - }, - vertex_credentials=vertex_credentials_json -) - -print(response) -``` - - - - -1. Setup config.yaml - -```yaml -model_list: - - model_name: gemini-tts-flash - litellm_params: - model: vertex_ai/gemini-2.5-flash-preview-tts - vertex_project: "your-project-id" - vertex_location: "us-central1" - vertex_credentials: "/path/to/service_account.json" - - model_name: gemini-tts-pro - litellm_params: - model: vertex_ai/gemini-2.5-pro-preview-tts - vertex_project: "your-project-id" - vertex_location: "us-central1" - vertex_credentials: "/path/to/service_account.json" -``` - -2. Start proxy - -```bash -litellm --config /path/to/config.yaml -``` - -3. Make TTS request - -```bash -curl http://0.0.0.0:4000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer " \ - -d '{ - "model": "gemini-tts-flash", - "messages": [{"role": "user", "content": "Say hello in a friendly voice"}], - "modalities": ["audio"], - "audio": { - "voice": "Kore", - "format": "pcm16" - } - }' -``` - - - - -### Advanced Usage - -You can combine TTS with other Gemini features: - -```python -response = completion( - model="vertex_ai/gemini-2.5-pro-preview-tts", - messages=[ - {"role": "system", "content": "You are a helpful assistant that speaks clearly."}, - {"role": "user", "content": "Explain quantum computing in simple terms"} - ], - modalities=["audio"], - audio={ - "voice": "Charon", - "format": "pcm16" - }, - temperature=0.7, - max_tokens=150, - vertex_credentials=vertex_credentials_json -) -``` - -For more information about Gemini's TTS capabilities and available voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation). - -## **Text to Speech APIs** - -:::info - -LiteLLM supports calling [Vertex AI Text to Speech API](https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech) in the OpenAI text to speech API format - -::: - - - -### Usage - Basic - - - - -Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param - -**Sync Usage** - -```python -speech_file_path = Path(__file__).parent / "speech_vertex.mp3" -response = litellm.speech( - model="vertex_ai/", - input="hello what llm guardrail do you have", -) -response.stream_to_file(speech_file_path) -``` - -**Async Usage** -```python -speech_file_path = Path(__file__).parent / "speech_vertex.mp3" -response = litellm.aspeech( - model="vertex_ai/", - input="hello what llm guardrail do you have", -) -response.stream_to_file(speech_file_path) -``` - - - - -1. Add model to config.yaml -```yaml -model_list: - - model_name: vertex-tts - litellm_params: - model: vertex_ai/ # Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param - vertex_project: "adroit-crow-413218" - vertex_location: "us-central1" - vertex_credentials: adroit-crow-413218-a956eef1a2a8.json - -litellm_settings: - drop_params: True -``` - -2. Start Proxy - -``` -$ litellm --config /path/to/config.yaml -``` - -3. Make Request use OpenAI Python SDK - - -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -# see supported values for "voice" on vertex here: -# https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech -response = client.audio.speech.create( - model = "vertex-tts", - input="the quick brown fox jumped over the lazy dogs", - voice={'languageCode': 'en-US', 'name': 'en-US-Studio-O'} -) -print("response from proxy", response) -``` - - - - - -### Usage - `ssml` as input - -Pass your `ssml` as input to the `input` param, if it contains ``, it will be automatically detected and passed as `ssml` to the Vertex AI API - -If you need to force your `input` to be passed as `ssml`, set `use_ssml=True` - - - - -Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param - - -```python -speech_file_path = Path(__file__).parent / "speech_vertex.mp3" - - -ssml = """ - -

Hello, world!

-

This is a test of the text-to-speech API.

- -""" - -response = litellm.speech( - input=ssml, - model="vertex_ai/test", - voice={ - "languageCode": "en-UK", - "name": "en-UK-Studio-O", - }, - audioConfig={ - "audioEncoding": "LINEAR22", - "speakingRate": "10", - }, -) -response.stream_to_file(speech_file_path) -``` - - - - - -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -ssml = """ - -

Hello, world!

-

This is a test of the text-to-speech API.

- -""" - -# see supported values for "voice" on vertex here: -# https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech -response = client.audio.speech.create( - model = "vertex-tts", - input=ssml, - voice={'languageCode': 'en-US', 'name': 'en-US-Studio-O'}, -) -print("response from proxy", response) -``` - - - - - -### Forcing SSML Usage - -You can force the use of SSML by setting the `use_ssml` parameter to `True`. This is useful when you want to ensure that your input is treated as SSML, even if it doesn't contain the `` tags. - -Here are examples of how to force SSML usage: - - - - - -Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param - - -```python -speech_file_path = Path(__file__).parent / "speech_vertex.mp3" - - -ssml = """ - -

Hello, world!

-

This is a test of the text-to-speech API.

- -""" - -response = litellm.speech( - input=ssml, - use_ssml=True, - model="vertex_ai/test", - voice={ - "languageCode": "en-UK", - "name": "en-UK-Studio-O", - }, - audioConfig={ - "audioEncoding": "LINEAR22", - "speakingRate": "10", - }, -) -response.stream_to_file(speech_file_path) -``` - - - - - -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -ssml = """ - -

Hello, world!

-

This is a test of the text-to-speech API.

- -""" - -# see supported values for "voice" on vertex here: -# https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech -response = client.audio.speech.create( - model = "vertex-tts", - input=ssml, # pass as None since OpenAI SDK requires this param - voice={'languageCode': 'en-US', 'name': 'en-US-Studio-O'}, - extra_body={"use_ssml": True}, -) -print("response from proxy", response) -``` - - - - ## **Fine Tuning APIs** diff --git a/docs/my-website/docs/providers/vertex_embedding.md b/docs/my-website/docs/providers/vertex_embedding.md new file mode 100644 index 00000000000..5656ade337b --- /dev/null +++ b/docs/my-website/docs/providers/vertex_embedding.md @@ -0,0 +1,587 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Vertex AI Embedding + +## Usage - Embedding + + + + +```python +import litellm +from litellm import embedding +litellm.vertex_project = "hardy-device-38811" # Your Project ID +litellm.vertex_location = "us-central1" # proj location + +response = embedding( + model="vertex_ai/textembedding-gecko", + input=["good morning from litellm"], +) +print(response) +``` + + + + + +1. Add model to config.yaml +```yaml +model_list: + - model_name: snowflake-arctic-embed-m-long-1731622468876 + litellm_params: + model: vertex_ai/ + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: adroit-crow-413218-a956eef1a2a8.json + +litellm_settings: + drop_params: True +``` + +2. Start Proxy + +``` +$ litellm --config /path/to/config.yaml +``` + +3. Make Request using OpenAI Python SDK, Langchain Python SDK + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +response = client.embeddings.create( + model="snowflake-arctic-embed-m-long-1731622468876", + input = ["good morning from litellm", "this is another item"], +) + +print(response) +``` + + + + + +#### Supported Embedding Models +All models listed [here](https://github.com/BerriAI/litellm/blob/57f37f743886a0249f630a6792d49dffc2c5d9b7/model_prices_and_context_window.json#L835) are supported + +| Model Name | Function Call | +|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| text-embedding-004 | `embedding(model="vertex_ai/text-embedding-004", input)` | +| text-multilingual-embedding-002 | `embedding(model="vertex_ai/text-multilingual-embedding-002", input)` | +| textembedding-gecko | `embedding(model="vertex_ai/textembedding-gecko", input)` | +| textembedding-gecko-multilingual | `embedding(model="vertex_ai/textembedding-gecko-multilingual", input)` | +| textembedding-gecko-multilingual@001 | `embedding(model="vertex_ai/textembedding-gecko-multilingual@001", input)` | +| textembedding-gecko@001 | `embedding(model="vertex_ai/textembedding-gecko@001", input)` | +| textembedding-gecko@003 | `embedding(model="vertex_ai/textembedding-gecko@003", input)` | +| text-embedding-preview-0409 | `embedding(model="vertex_ai/text-embedding-preview-0409", input)` | +| text-multilingual-embedding-preview-0409 | `embedding(model="vertex_ai/text-multilingual-embedding-preview-0409", input)` | +| Fine-tuned OR Custom Embedding models | `embedding(model="vertex_ai/", input)` | + +### Supported OpenAI (Unified) Params + +| [param](../embedding/supported_embedding.md#input-params-for-litellmembedding) | type | [vertex equivalent](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api) | +|-------|-------------|--------------------| +| `input` | **string or List[string]** | `instances` | +| `dimensions` | **int** | `output_dimensionality` | +| `input_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** | `task_type` | + +#### Usage with OpenAI (Unified) Params + + + + + +```python +response = litellm.embedding( + model="vertex_ai/text-embedding-004", + input=["good morning from litellm", "gm"] + input_type = "RETRIEVAL_DOCUMENT", + dimensions=1, +) +``` + + + + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +response = client.embeddings.create( + model="text-embedding-004", + input = ["good morning from litellm", "gm"], + dimensions=1, + extra_body = { + "input_type": "RETRIEVAL_QUERY", + } +) + +print(response) +``` + + + + +### Supported Vertex Specific Params + +| param | type | +|-------|-------------| +| `auto_truncate` | **bool** | +| `task_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** | +| `title` | **str** | + +#### Usage with Vertex Specific Params (Use `task_type` and `title`) + +You can pass any vertex specific params to the embedding model. Just pass them to the embedding function like this: + +[Relevant Vertex AI doc with all embedding params](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#request_body) + + + + +```python +response = litellm.embedding( + model="vertex_ai/text-embedding-004", + input=["good morning from litellm", "gm"] + task_type = "RETRIEVAL_DOCUMENT", + title = "test", + dimensions=1, + auto_truncate=True, +) +``` + + + + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +response = client.embeddings.create( + model="text-embedding-004", + input = ["good morning from litellm", "gm"], + dimensions=1, + extra_body = { + "task_type": "RETRIEVAL_QUERY", + "auto_truncate": True, + "title": "test", + } +) + +print(response) +``` + + + +## **BGE Embeddings** + +Use BGE (Baidu General Embedding) models deployed on Vertex AI. + +### Usage + + + + +```python showLineNumbers title="Using BGE on Vertex AI" +import litellm + +response = litellm.embedding( + model="vertex_ai/bge/", + input=["Hello", "World"], + vertex_project="your-project-id", + vertex_location="your-location" +) + +print(response) +``` + + + + + +1. Add model to config.yaml +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: bge-embedding + litellm_params: + model: vertex_ai/bge/ + vertex_project: "your-project-id" + vertex_location: "us-central1" + vertex_credentials: your-credentials.json + +litellm_settings: + drop_params: True +``` + +2. Start Proxy + +```bash +$ litellm --config /path/to/config.yaml +``` + +3. Make Request using OpenAI Python SDK + +```python showLineNumbers title="Making requests to BGE" +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +response = client.embeddings.create( + model="bge-embedding", + input=["good morning from litellm", "this is another item"] +) + +print(response) +``` + +Using a Private Service Connect (PSC) endpoint + +```yaml showLineNumbers title="config.yaml (PSC)" +model_list: + - model_name: bge-small-en-v1.5 + litellm_params: + model: vertex_ai/bge/1234567890 + api_base: http://10.96.32.8 # Your PSC IP + vertex_project: my-project-id #optional + vertex_location: us-central1 #optional +``` + + + + +## **Multi-Modal Embeddings** + + +Known Limitations: +- Only supports 1 image / video / image per request +- Only supports GCS or base64 encoded images / videos + +### Usage + + + + +Using GCS Images + +```python +response = await litellm.aembedding( + model="vertex_ai/multimodalembedding@001", + input="gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" # will be sent as a gcs image +) +``` + +Using base 64 encoded images + +```python +response = await litellm.aembedding( + model="vertex_ai/multimodalembedding@001", + input="data:image/jpeg;base64,..." # will be sent as a base64 encoded image +) +``` + + + + +1. Add model to config.yaml +```yaml +model_list: + - model_name: multimodalembedding@001 + litellm_params: + model: vertex_ai/multimodalembedding@001 + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: adroit-crow-413218-a956eef1a2a8.json + +litellm_settings: + drop_params: True +``` + +2. Start Proxy + +``` +$ litellm --config /path/to/config.yaml +``` + +3. Make Request use OpenAI Python SDK, Langchain Python SDK + + + + + + +Requests with GCS Image / Video URI + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +# # request sent to model set on litellm proxy, `litellm --model` +response = client.embeddings.create( + model="multimodalembedding@001", + input = "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", +) + +print(response) +``` + +Requests with base64 encoded images + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +# # request sent to model set on litellm proxy, `litellm --model` +response = client.embeddings.create( + model="multimodalembedding@001", + input = "data:image/jpeg;base64,...", +) + +print(response) +``` + + + + + +Requests with GCS Image / Video URI +```python +from langchain_openai import OpenAIEmbeddings + +embeddings_models = "multimodalembedding@001" + +embeddings = OpenAIEmbeddings( + model="multimodalembedding@001", + base_url="http://0.0.0.0:4000", + api_key="sk-1234", # type: ignore +) + + +query_result = embeddings.embed_query( + "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" +) +print(query_result) + +``` + +Requests with base64 encoded images + +```python +from langchain_openai import OpenAIEmbeddings + +embeddings_models = "multimodalembedding@001" + +embeddings = OpenAIEmbeddings( + model="multimodalembedding@001", + base_url="http://0.0.0.0:4000", + api_key="sk-1234", # type: ignore +) + + +query_result = embeddings.embed_query( + "data:image/jpeg;base64,..." +) +print(query_result) + +``` + + + + + + + + + +1. Add model to config.yaml +```yaml +default_vertex_config: + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: adroit-crow-413218-a956eef1a2a8.json +``` + +2. Start Proxy + +``` +$ litellm --config /path/to/config.yaml +``` + +3. Make Request use OpenAI Python SDK + +```python +import vertexai + +from vertexai.vision_models import Image, MultiModalEmbeddingModel, Video +from vertexai.vision_models import VideoSegmentConfig +from google.auth.credentials import Credentials + + +LITELLM_PROXY_API_KEY = "sk-1234" +LITELLM_PROXY_BASE = "http://0.0.0.0:4000/vertex-ai" + +import datetime + +class CredentialsWrapper(Credentials): + def __init__(self, token=None): + super().__init__() + self.token = token + self.expiry = None # or set to a future date if needed + + def refresh(self, request): + pass + + def apply(self, headers, token=None): + headers['Authorization'] = f'Bearer {self.token}' + + @property + def expired(self): + return False # Always consider the token as non-expired + + @property + def valid(self): + return True # Always consider the credentials as valid + +credentials = CredentialsWrapper(token=LITELLM_PROXY_API_KEY) + +vertexai.init( + project="adroit-crow-413218", + location="us-central1", + api_endpoint=LITELLM_PROXY_BASE, + credentials = credentials, + api_transport="rest", + +) + +model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding") +image = Image.load_from_file( + "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" +) + +embeddings = model.get_embeddings( + image=image, + contextual_text="Colosseum", + dimension=1408, +) +print(f"Image Embedding: {embeddings.image_embedding}") +print(f"Text Embedding: {embeddings.text_embedding}") +``` + + + + + +### Text + Image + Video Embeddings + + + + +Text + Image + +```python +response = await litellm.aembedding( + model="vertex_ai/multimodalembedding@001", + input=["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"] # will be sent as a gcs image +) +``` + +Text + Video + +```python +response = await litellm.aembedding( + model="vertex_ai/multimodalembedding@001", + input=["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image +) +``` + +Image + Video + +```python +response = await litellm.aembedding( + model="vertex_ai/multimodalembedding@001", + input=["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image +) +``` + + + + + +1. Add model to config.yaml +```yaml +model_list: + - model_name: multimodalembedding@001 + litellm_params: + model: vertex_ai/multimodalembedding@001 + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: adroit-crow-413218-a956eef1a2a8.json + +litellm_settings: + drop_params: True +``` + +2. Start Proxy + +``` +$ litellm --config /path/to/config.yaml +``` + +3. Make Request use OpenAI Python SDK, Langchain Python SDK + + +Text + Image + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +# # request sent to model set on litellm proxy, `litellm --model` +response = client.embeddings.create( + model="multimodalembedding@001", + input = ["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"], +) + +print(response) +``` + +Text + Video +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +# # request sent to model set on litellm proxy, `litellm --model` +response = client.embeddings.create( + model="multimodalembedding@001", + input = ["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"], +) + +print(response) +``` + +Image + Video +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +# # request sent to model set on litellm proxy, `litellm --model` +response = client.embeddings.create( + model="multimodalembedding@001", + input = ["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"], +) + +print(response) +``` + + + \ No newline at end of file diff --git a/docs/my-website/docs/providers/vertex_speech.md b/docs/my-website/docs/providers/vertex_speech.md new file mode 100644 index 00000000000..d0acacb5aec --- /dev/null +++ b/docs/my-website/docs/providers/vertex_speech.md @@ -0,0 +1,423 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Vertex AI Text to Speech + +| Property | Details | +|-------|-------| +| Description | Google Cloud Text-to-Speech with Chirp3 HD voices and Gemini TTS | +| Provider Route on LiteLLM | `vertex_ai/chirp` (Chirp), `vertex_ai/gemini-*-tts` (Gemini) | + +## Chirp3 HD Voices + +Google Cloud Text-to-Speech API with high-quality Chirp3 HD voices. + +### Quick Start + +#### LiteLLM Python SDK + +```python showLineNumbers title="Chirp3 Quick Start" +from litellm import speech +from pathlib import Path + +speech_file_path = Path(__file__).parent / "speech.mp3" +response = speech( + model="vertex_ai/chirp", + voice="alloy", # OpenAI voice name - automatically mapped + input="Hello, this is Vertex AI Text to Speech", + vertex_project="your-project-id", + vertex_location="us-central1", +) +response.stream_to_file(speech_file_path) +``` + +#### LiteLLM AI Gateway + +**1. Setup config.yaml** + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: vertex-tts + litellm_params: + model: vertex_ai/chirp + vertex_project: "your-project-id" + vertex_location: "us-central1" + vertex_credentials: "/path/to/service_account.json" +``` + +**2. Start the proxy** + +```bash title="Start LiteLLM Proxy" +litellm --config /path/to/config.yaml +``` + +**3. Make requests** + + + + +```bash showLineNumbers title="Chirp3 Quick Start" +curl http://0.0.0.0:4000/v1/audio/speech \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "vertex-tts", + "voice": "alloy", + "input": "Hello, this is Vertex AI Text to Speech" + }' \ + --output speech.mp3 +``` + + + + +```python showLineNumbers title="Chirp3 Quick Start" +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +response = client.audio.speech.create( + model="vertex-tts", + voice="alloy", + input="Hello, this is Vertex AI Text to Speech", +) +response.stream_to_file("speech.mp3") +``` + + + + +### Voice Mapping + +LiteLLM maps OpenAI voice names to Google Cloud voices. You can use either OpenAI voices or Google Cloud voices directly. + +| OpenAI Voice | Google Cloud Voice | +|-------------|-------------------| +| `alloy` | en-US-Studio-O | +| `echo` | en-US-Studio-M | +| `fable` | en-GB-Studio-B | +| `onyx` | en-US-Wavenet-D | +| `nova` | en-US-Studio-O | +| `shimmer` | en-US-Wavenet-F | + +### Using Google Cloud Voices Directly + +#### LiteLLM Python SDK + +```python showLineNumbers title="Chirp3 HD Voice" +from litellm import speech + +# Pass Chirp3 HD voice name directly +response = speech( + model="vertex_ai/chirp", + voice="en-US-Chirp3-HD-Charon", + input="Hello with a Chirp3 HD voice", + vertex_project="your-project-id", +) +response.stream_to_file("speech.mp3") +``` + +```python showLineNumbers title="Voice as Dict (Multilingual)" +from litellm import speech + +# Pass as dict for full control over language and voice +response = speech( + model="vertex_ai/chirp", + voice={ + "languageCode": "de-DE", + "name": "de-DE-Chirp3-HD-Charon", + }, + input="Hallo, dies ist ein Test", + vertex_project="your-project-id", +) +response.stream_to_file("speech.mp3") +``` + +#### LiteLLM AI Gateway + + + + +```bash showLineNumbers title="Chirp3 HD Voice" +curl http://0.0.0.0:4000/v1/audio/speech \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "vertex-tts", + "voice": "en-US-Chirp3-HD-Charon", + "input": "Hello with a Chirp3 HD voice" + }' \ + --output speech.mp3 +``` + +```bash showLineNumbers title="Voice as Dict (Multilingual)" +curl http://0.0.0.0:4000/v1/audio/speech \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "vertex-tts", + "voice": {"languageCode": "de-DE", "name": "de-DE-Chirp3-HD-Charon"}, + "input": "Hallo, dies ist ein Test" + }' \ + --output speech.mp3 +``` + + + + +```python showLineNumbers title="Chirp3 HD Voice" +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +response = client.audio.speech.create( + model="vertex-tts", + voice="en-US-Chirp3-HD-Charon", + input="Hello with a Chirp3 HD voice", +) +response.stream_to_file("speech.mp3") +``` + +```python showLineNumbers title="Voice as Dict (Multilingual)" +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +response = client.audio.speech.create( + model="vertex-tts", + voice={"languageCode": "de-DE", "name": "de-DE-Chirp3-HD-Charon"}, + input="Hallo, dies ist ein Test", +) +response.stream_to_file("speech.mp3") +``` + + + + +Browse available voices: [Google Cloud Text-to-Speech Console](https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech) + +### Passing Raw SSML + +LiteLLM auto-detects SSML when your input contains `` tags and passes it through unchanged. + +#### LiteLLM Python SDK + +```python showLineNumbers title="SSML Input" +from litellm import speech + +ssml = """ + +

Hello, world!

+

This is a test of the text-to-speech API.

+ +""" + +response = speech( + model="vertex_ai/chirp", + voice="en-US-Studio-O", + input=ssml, # Auto-detected as SSML + vertex_project="your-project-id", +) +response.stream_to_file("speech.mp3") +``` + +```python showLineNumbers title="Force SSML Mode" +from litellm import speech + +# Force SSML mode with use_ssml=True +response = speech( + model="vertex_ai/chirp", + voice="en-US-Studio-O", + input="Speaking slowly", + use_ssml=True, + vertex_project="your-project-id", +) +response.stream_to_file("speech.mp3") +``` + +#### LiteLLM AI Gateway + + + + +```bash showLineNumbers title="SSML Input" +curl http://0.0.0.0:4000/v1/audio/speech \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "vertex-tts", + "voice": "en-US-Studio-O", + "input": "

Hello!

How are you?

" + }' \ + --output speech.mp3 +``` + + + + +```python showLineNumbers title="SSML Input" +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +ssml = """

Hello!

How are you?

""" + +response = client.audio.speech.create( + model="vertex-tts", + voice="en-US-Studio-O", + input=ssml, +) +response.stream_to_file("speech.mp3") +``` + + + + +### Supported Parameters + +| Parameter | Description | Values | +|-----------|-------------|--------| +| `voice` | Voice selection | OpenAI voice, Google Cloud voice name, or dict | +| `input` | Text to convert | Plain text or SSML | +| `speed` | Speaking rate | 0.25 to 4.0 (default: 1.0) | +| `response_format` | Audio format | `mp3`, `opus`, `wav`, `pcm`, `flac` | +| `use_ssml` | Force SSML mode | `True` / `False` | + +### Async Usage + +```python showLineNumbers title="Async Speech Generation" +import asyncio +from litellm import aspeech + +async def main(): + response = await aspeech( + model="vertex_ai/chirp", + voice="alloy", + input="Hello from async", + vertex_project="your-project-id", + ) + response.stream_to_file("speech.mp3") + +asyncio.run(main()) +``` + +--- + +## Gemini TTS + +Gemini models with audio output capabilities using the chat completions API. + +:::warning +**Limitations:** +- Only supports `pcm16` audio format +- Streaming not yet supported +- Must set `modalities: ["audio"]` +::: + +### Quick Start + +#### LiteLLM Python SDK + +```python showLineNumbers title="Gemini TTS Quick Start" +from litellm import completion +import json + +# Load credentials +with open('path/to/service_account.json', 'r') as file: + vertex_credentials = json.dumps(json.load(file)) + +response = completion( + model="vertex_ai/gemini-2.5-flash-preview-tts", + messages=[{"role": "user", "content": "Say hello in a friendly voice"}], + modalities=["audio"], + audio={ + "voice": "Kore", + "format": "pcm16" + }, + vertex_credentials=vertex_credentials +) +print(response) +``` + +#### LiteLLM AI Gateway + +**1. Setup config.yaml** + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: gemini-tts + litellm_params: + model: vertex_ai/gemini-2.5-flash-preview-tts + vertex_project: "your-project-id" + vertex_location: "us-central1" + vertex_credentials: "/path/to/service_account.json" +``` + +**2. Start the proxy** + +```bash title="Start LiteLLM Proxy" +litellm --config /path/to/config.yaml +``` + +**3. Make requests** + + + + +```bash showLineNumbers title="Gemini TTS Request" +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gemini-tts", + "messages": [{"role": "user", "content": "Say hello in a friendly voice"}], + "modalities": ["audio"], + "audio": {"voice": "Kore", "format": "pcm16"} + }' +``` + + + + +```python showLineNumbers title="Gemini TTS Request" +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +response = client.chat.completions.create( + model="gemini-tts", + messages=[{"role": "user", "content": "Say hello in a friendly voice"}], + modalities=["audio"], + audio={"voice": "Kore", "format": "pcm16"}, +) +print(response) +``` + + + + +### Supported Models + +- `vertex_ai/gemini-2.5-flash-preview-tts` +- `vertex_ai/gemini-2.5-pro-preview-tts` + +See [Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation) for available voices. + +### Advanced Usage + +```python showLineNumbers title="Gemini TTS with System Prompt" +from litellm import completion + +response = completion( + model="vertex_ai/gemini-2.5-pro-preview-tts", + messages=[ + {"role": "system", "content": "You are a helpful assistant that speaks clearly."}, + {"role": "user", "content": "Explain quantum computing in simple terms"} + ], + modalities=["audio"], + audio={"voice": "Charon", "format": "pcm16"}, + temperature=0.7, + max_tokens=150, + vertex_credentials=vertex_credentials +) +``` diff --git a/docs/my-website/docs/providers/watsonx/index.md b/docs/my-website/docs/providers/watsonx/index.md index 279d2d1024e..14e0c07c081 100644 --- a/docs/my-website/docs/providers/watsonx/index.md +++ b/docs/my-website/docs/providers/watsonx/index.md @@ -175,3 +175,56 @@ For all available models, see [watsonx.ai documentation](https://dataplatform.cl For all available embedding models, see [watsonx.ai embedding documentation](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx). + +## Advanced + +### Using Zen API Key + +You can use a Zen API key for long-term authentication instead of generating IAM tokens. Pass it either as an environment variable or as a parameter: + +```python +import os +from litellm import completion + +# Option 1: Set as environment variable +os.environ["WATSONX_ZENAPIKEY"] = "your-zen-api-key" + +response = completion( + model="watsonx/ibm/granite-13b-chat-v2", + messages=[{"content": "What is your favorite color?", "role": "user"}], + project_id="your-project-id" +) + +# Option 2: Pass as parameter +response = completion( + model="watsonx/ibm/granite-13b-chat-v2", + messages=[{"content": "What is your favorite color?", "role": "user"}], + zen_api_key="your-zen-api-key", + project_id="your-project-id" +) +``` + +**Using with LiteLLM Proxy via OpenAI client:** + +```python +import openai + +client = openai.OpenAI( + api_key="sk-1234", # LiteLLM proxy key + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="watsonx/ibm/granite-3-3-8b-instruct", + messages=[{"role": "user", "content": "What is your favorite color?"}], + max_tokens=2048, + extra_body={ + "project_id": "your-project-id", + "zen_api_key": "your-zen-api-key" + } +) +``` + +See [IBM documentation](https://www.ibm.com/docs/en/watsonx/w-and-w/2.2.0?topic=keys-generating-zenapikey-authorization-tokens) for more information on generating Zen API keys. + + diff --git a/docs/my-website/docs/providers/zai.md b/docs/my-website/docs/providers/zai.md new file mode 100644 index 00000000000..5055d0c1cdd --- /dev/null +++ b/docs/my-website/docs/providers/zai.md @@ -0,0 +1,135 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Z.AI (Zhipu AI) +https://z.ai/ + +**We support Z.AI GLM text/chat models, just set `zai/` as a prefix when sending completion requests** + +## API Key +```python +# env variable +os.environ['ZAI_API_KEY'] +``` + +## Sample Usage +```python +from litellm import completion +import os + +os.environ['ZAI_API_KEY'] = "" +response = completion( + model="zai/glm-4.6", + messages=[ + {"role": "user", "content": "hello from litellm"} + ], +) +print(response) +``` + +## Sample Usage - Streaming +```python +from litellm import completion +import os + +os.environ['ZAI_API_KEY'] = "" +response = completion( + model="zai/glm-4.6", + messages=[ + {"role": "user", "content": "hello from litellm"} + ], + stream=True +) + +for chunk in response: + print(chunk) +``` + +## Supported Models + +We support ALL Z.AI GLM models, just set `zai/` as a prefix when sending completion requests. + +| Model Name | Function Call | Notes | +|------------|---------------|-------| +| glm-4.6 | `completion(model="zai/glm-4.6", messages)` | Latest flagship model, 200K context | +| glm-4.5 | `completion(model="zai/glm-4.5", messages)` | 128K context | +| glm-4.5v | `completion(model="zai/glm-4.5v", messages)` | Vision model | +| glm-4.5-x | `completion(model="zai/glm-4.5-x", messages)` | Premium tier | +| glm-4.5-air | `completion(model="zai/glm-4.5-air", messages)` | Lightweight | +| glm-4.5-airx | `completion(model="zai/glm-4.5-airx", messages)` | Fast lightweight | +| glm-4-32b-0414-128k | `completion(model="zai/glm-4-32b-0414-128k", messages)` | 32B parameter model | +| glm-4.5-flash | `completion(model="zai/glm-4.5-flash", messages)` | **FREE tier** | + +## Model Pricing + +| Model | Input ($/1M tokens) | Output ($/1M tokens) | Context Window | +|-------|---------------------|----------------------|----------------| +| glm-4.6 | $0.60 | $2.20 | 200K | +| glm-4.5 | $0.60 | $2.20 | 128K | +| glm-4.5v | $0.60 | $1.80 | 128K | +| glm-4.5-x | $2.20 | $8.90 | 128K | +| glm-4.5-air | $0.20 | $1.10 | 128K | +| glm-4.5-airx | $1.10 | $4.50 | 128K | +| glm-4-32b-0414-128k | $0.10 | $0.10 | 128K | +| glm-4.5-flash | **FREE** | **FREE** | 128K | + +## Using with LiteLLM Proxy + + + + +```python +from litellm import completion +import os + +os.environ['ZAI_API_KEY'] = "" +response = completion( + model="zai/glm-4.6", + messages=[{"role": "user", "content": "Hello, how are you?"}], +) + +print(response.choices[0].message.content) +``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: glm-4.6 + litellm_params: + model: zai/glm-4.6 + api_key: os.environ/ZAI_API_KEY + - model_name: glm-4.5-flash # Free tier + litellm_params: + model: zai/glm-4.5-flash + api_key: os.environ/ZAI_API_KEY +``` + +2. Run proxy + +```bash +litellm --config config.yaml +``` + +3. Test it! + +```bash +curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "glm-4.6", + "messages": [ + { + "role": "user", + "content": "Hello, how are you?" + } + ] +}' +``` + + + diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 7140c99e6fb..65b1c4afdbc 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -29,7 +29,8 @@ litellm_settings: request_timeout: 10 # (int) llm requesttimeout in seconds. Raise Timeout error if call takes longer than 10s. Sets litellm.request_timeout force_ipv4: boolean # If true, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6 + Anthropic API - set_verbose: boolean # sets litellm.set_verbose=True to view verbose debug logs. DO NOT LEAVE THIS ON IN PRODUCTION + # Debugging - see debugging docs for more options + # Use `--debug` or `--detailed_debug` CLI flags, or set LITELLM_LOG env var to "INFO", "DEBUG", or "ERROR" json_logs: boolean # if true, logs will be in json format # Fallbacks, reliability @@ -113,7 +114,7 @@ general_settings: # Database Settings database_url: string - database_connection_pool_limit: 0 # default 100 + database_connection_pool_limit: 0 # default 10 database_connection_timeout: 0 # default 60s allow_requests_on_db_unavailable: boolean # if true, will allow requests that can not connect to the DB to verify Virtual Key to still work @@ -171,7 +172,7 @@ router_settings: | redact_user_api_key_info | boolean | If true, redacts information about the user api key from logs [Proxy Logging](logging#redacting-userapikeyinfo) | | mcp_aliases | object | Maps friendly aliases to MCP server names for easier tool access. Only the first alias for each server is used. [MCP Aliases](../mcp#mcp-aliases) | | langfuse_default_tags | array of strings | Default tags for Langfuse Logging. Use this if you want to control which LiteLLM-specific fields are logged as tags by the LiteLLM proxy. By default LiteLLM Proxy logs no LiteLLM-specific fields as tags. [Further docs](./logging#litellm-specific-tags-on-langfuse---cache_hit-cache_key) | -| set_verbose | boolean | If true, sets litellm.set_verbose=True to view verbose debug logs. DO NOT LEAVE THIS ON IN PRODUCTION | +| set_verbose | boolean | [DEPRECATED - see debugging docs](./debugging) Use `--debug` or `--detailed_debug` CLI flags, or set `LITELLM_LOG` env var to "INFO", "DEBUG", or "ERROR" instead. | | json_logs | boolean | If true, logs will be in json format. If you need to store the logs as JSON, just set the `litellm.json_logs = True`. We currently just log the raw POST request from litellm as a JSON [Further docs](./debugging) | | default_fallbacks | array of strings | List of fallback models to use if a specific model group is misconfigured / bad. [Further docs](./reliability#default-fallbacks) | | request_timeout | integer | The timeout for requests in seconds. If not set, the default value is `6000 seconds`. [For reference OpenAI Python SDK defaults to `600 seconds`.](https://github.com/openai/openai-python/blob/main/src/openai/_constants.py) | @@ -234,7 +235,7 @@ router_settings: | max_response_size_mb | int | The maximum size for responses in MB. LLM Responses above this size will not be sent. | | proxy_budget_rescheduler_min_time | int | The minimum time (in seconds) to wait before checking db for budget resets. **Default is 597 seconds** | | proxy_budget_rescheduler_max_time | int | The maximum time (in seconds) to wait before checking db for budget resets. **Default is 605 seconds** | -| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 30 seconds** | +| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 10 seconds** | | proxy_batch_polling_interval | int | Time (in seconds) to wait before polling a batch, to check if it's completed. **Default is 6000 seconds (1 hour)** | | alerting_args | dict | Args for Slack Alerting [Doc on Slack Alerting](./alerting.md) | | custom_key_generate | str | Custom function for key generation [Doc on custom key generation](./virtual_keys.md#custom--key-generate) | @@ -333,7 +334,7 @@ router_settings: | caching_groups | Optional[List[tuple]] | List of model groups for caching across model groups. Defaults to None. - e.g. caching_groups=[("openai-gpt-3.5-turbo", "azure-gpt-3.5-turbo")]| | alerting_config | AlertingConfig | [SDK-only arg] Slack alerting configuration. Defaults to None. [Further Docs](../routing.md#alerting-) | | assistants_config | AssistantsConfig | Set on proxy via `assistant_settings`. [Further docs](../assistants.md) | -| set_verbose | boolean | [DEPRECATED PARAM - see debug docs](./debugging.md) If true, sets the logging level to verbose. | +| set_verbose | boolean | [DEPRECATED PARAM - see debug docs](./debugging) If true, sets the logging level to verbose. | | retry_after | int | Time to wait before retrying a request in seconds. Defaults to 0. If `x-retry-after` is received from LLM API, this value is overridden. | | provider_budget_config | ProviderBudgetConfig | Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None. [Further Docs](./provider_budget_routing.md) | | enable_pre_call_checks | boolean | If true, checks if a call is within the model's context window before making the call. [More information here](reliability) | @@ -359,6 +360,7 @@ router_settings: | AISPEND_ACCOUNT_ID | Account ID for AI Spend | AISPEND_API_KEY | API Key for AI Spend | AIOHTTP_CONNECTOR_LIMIT | Connection limit for aiohttp connector. When set to 0, no limit is applied. **Default is 0** +| AIOHTTP_CONNECTOR_LIMIT_PER_HOST | Connection limit per host for aiohttp connector. When set to 0, no limit is applied. **Default is 0** | AIOHTTP_KEEPALIVE_TIMEOUT | Keep-alive timeout for aiohttp connections in seconds. **Default is 120** | AIOHTTP_TRUST_ENV | Flag to enable aiohttp trust environment. When this is set to True, aiohttp will respect HTTP(S)_PROXY env vars. **Default is False** | AIOHTTP_TTL_DNS_CACHE | DNS cache time-to-live for aiohttp in seconds. **Default is 300** @@ -377,6 +379,8 @@ router_settings: | ATHINA_API_KEY | API key for Athina service | ATHINA_BASE_URL | Base URL for Athina service (defaults to `https://log.athina.ai`) | AUTH_STRATEGY | Strategy used for authentication (e.g., OAuth, API key) +| AUTO_REDIRECT_UI_LOGIN_TO_SSO | Flag to enable automatic redirect of UI login page to SSO when SSO is configured. Default is **true** +| AUDIO_SPEECH_CHUNK_SIZE | Chunk size for audio speech processing. Default is 1024 | ANTHROPIC_API_KEY | API key for Anthropic service | ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com | AWS_ACCESS_KEY_ID | Access Key ID for AWS services @@ -439,6 +443,7 @@ router_settings: | CYBERARK_CLIENT_CERT | Path to client certificate for CyberArk authentication | CYBERARK_CLIENT_KEY | Path to client key for CyberArk authentication | CYBERARK_USERNAME | Username for CyberArk authentication +| CYBERARK_SSL_VERIFY | Flag to enable or disable SSL certificate verification for CyberArk. Default is True | CONFIDENT_API_KEY | API key for DeepEval integration | CUSTOM_TIKTOKEN_CACHE_DIR | Custom directory for Tiktoken cache | CONFIDENT_API_KEY | API key for Confident AI (Deepeval) Logging service @@ -653,6 +658,8 @@ router_settings: | LITERAL_API_URL | API URL for Literal service | LITERAL_BATCH_SIZE | Batch size for Literal operations | LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints +| LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API +| LITELLM_DD_AGENT_PORT | Port of DataDog agent for LiteLLM-specific log intake. Default is 10518 | LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI | LITELLM_DROP_PARAMS | Parameters to drop in LiteLLM requests | LITELLM_MODIFY_PARAMS | Parameters to modify in LiteLLM requests @@ -763,7 +770,7 @@ router_settings: | PROMPTLAYER_API_KEY | API key for PromptLayer integration | PROXY_ADMIN_ID | Admin identifier for proxy server | PROXY_BASE_URL | Base URL for proxy service -| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 30 +| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 10 | PROXY_BATCH_POLLING_INTERVAL | Time in seconds to wait before polling a batch, to check if it's completed. Default is 6000s (1 hour) | PROXY_BUDGET_RESCHEDULER_MAX_TIME | Maximum time in seconds to wait before checking database for budget resets. Default is 605 | PROXY_BUDGET_RESCHEDULER_MIN_TIME | Minimum time in seconds to wait before checking database for budget resets. Default is 597 @@ -797,7 +804,7 @@ router_settings: | SEND_USER_API_KEY_ALIAS | Flag to send user API key alias to Zscaler AI Guard. Default is False | SEND_USER_API_KEY_TEAM_ID | Flag to send user API key team ID to Zscaler AI Guard. Default is False | SEND_USER_API_KEY_USER_ID | Flag to send user API key user ID to Zscaler AI Guard. Default is False -| SET_VERBOSE | Flag to enable verbose logging +| SET_VERBOSE | [DEPRECATED] Use `LITELLM_LOG` instead with values "INFO", "DEBUG", or "ERROR". See [debugging docs](./debugging) | SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD | Minimum number of requests to consider "reasonable traffic" for single-deployment cooldown logic. Default is 1000 | SLACK_DAILY_REPORT_FREQUENCY | Frequency of daily Slack reports (e.g., daily, weekly) | SLACK_WEBHOOK_URL | Webhook URL for Slack integration @@ -839,6 +846,9 @@ router_settings: | UPSTREAM_LANGFUSE_SECRET_KEY | Secret key for upstream Langfuse authentication | USE_AWS_KMS | Flag to enable AWS Key Management Service for encryption | USE_PRISMA_MIGRATE | Flag to use prisma migrate instead of prisma db push. Recommended for production environments. +| WANDB_API_KEY | API key for Weights & Biases (W&B) logging integration +| WANDB_HOST | Host URL for Weights & Biases (W&B) service +| WANDB_PROJECT_ID | Project ID for Weights & Biases (W&B) logging integration | WEBHOOK_URL | URL for receiving webhooks from external services | SPEND_LOG_RUN_LOOPS | Constant for setting how many runs of 1000 batch deletes should spend_log_cleanup task run | SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000 diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 18177b7c4d2..77ab3158f74 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -576,7 +576,7 @@ custom_tokenizer: ```yaml general_settings: - database_connection_pool_limit: 100 # sets connection pool for prisma client to postgres db at 100 + database_connection_pool_limit: 10 # sets connection pool for prisma client to postgres db (default: 10, recommended: 10-20) database_connection_timeout: 60 # sets a 60s timeout for any connection call to the db ``` diff --git a/docs/my-website/docs/proxy/cursor.md b/docs/my-website/docs/proxy/cursor.md new file mode 100644 index 00000000000..d01c1e62036 --- /dev/null +++ b/docs/my-website/docs/proxy/cursor.md @@ -0,0 +1,108 @@ +--- +id: cursor +title: /cursor/chat/completions - Cursor Endpoint +description: Accept Responses API input from Cursor and return OpenAI Chat Completions output +--- + +LiteLLM provides a Cursor-specific endpoint to make Cursor IDE work seamlessly with the LiteLLM Proxy when using BYOK + custom `base_url`. + +- Accepts Requests in OpenAI Responses API input format (Cursor sends this) +- Returns Responses in OpenAI Chat Completions format (Cursor expects this) +- Supports streaming and non‑streaming + +## Endpoint + +- Path: `/cursor/chat/completions` +- Auth: Standard LiteLLM Proxy auth (`Authorization: Bearer `) +- Behavior: Internally routes to LiteLLM `/responses` flow and transforms output to Chat Completions + +## Why this exists + +When setting up Cursor with BYOK against a custom `base_url`, Cursor sends requests to the Chat Completions endpoint but in the OpenAI Responses API input shape. Without translation, Cursor won’t display streamed output. This endpoint bridges the formats: + +- Input: Responses API (`input`, tool calls, etc.) +- Output: Chat Completions (`choices`, `delta`, `finish_reason`, etc.) + +## Usage + +### Non-streaming + +```bash +curl -X POST https://litellm-internal/cursor/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-4o", + "input": [{"role": "user", "content": "Hello"}] + }' +``` + +Example response (shape): + +```json +{ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1733333333, + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! How can I help you?" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 8, + "total_tokens": 18 + } +} +``` + +### Streaming + +```bash +curl -N -X POST https://litellm-internal/cursor/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-4o", + "input": [{"role": "user", "content": "Hello"}], + "stream": true + }' +``` + +- Server-Sent Events (SSE) +- Emits `chat.completion.chunk` deltas (`choices[].delta`) and ends with `data: [DONE]` + +## Configuration + +### Base URL Setup + +**Important**: When configuring Cursor IDE to use this endpoint, you must include `/cursor` in the base URL. + +Cursor automatically appends `/chat/completions` to the base URL you provide. To ensure requests go to `/cursor/chat/completions`, configure your base URL in Cursor as: + +``` +Base URL: https://litellm-internal/cursor +``` + +This way, when Cursor appends `/chat/completions`, the full path becomes `/cursor/chat/completions`, which is the correct endpoint. + +**Example**: If your LiteLLM Proxy is running at `https://litellm-internal`, set the base URL in Cursor to `https://litellm-internal/cursor` (not just `https://litellm-internal`). + +### General Setup + +No special configuration is required beyond your normal LiteLLM Proxy setup. Ensure that: + +- Your `config.yaml` includes the models you want to call via this endpoint +- Your Cursor project uses your LiteLLM Proxy `base_url` (with `/cursor` included) and a valid API key + +## Notes +- This endpoint is intended specifically for Cursor’s request/response expectations. Other clients should continue to use `/v1/chat/completions` or `/v1/responses` as appropriate. + + diff --git a/docs/my-website/docs/proxy/customer_usage.md b/docs/my-website/docs/proxy/customer_usage.md new file mode 100644 index 00000000000..8e366586b15 --- /dev/null +++ b/docs/my-website/docs/proxy/customer_usage.md @@ -0,0 +1,110 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Customer Usage + +Track and visualize end-user spend directly in the dashboard. Monitor customer-level usage analytics, spend logs, and activity metrics to understand how your customers are using your LLM services. + +This feature is **available in v1.80.8-stable and above**. + +## Overview + +Customer Usage enables you to track spend and usage for individual customers (end users) by passing an ID in your API requests. This allows you to: + +- Track spend per customer automatically +- View customer-level usage analytics in the Admin UI +- Filter spend logs and activity metrics by customer ID +- Set budgets and rate limits per customer +- Monitor customer usage patterns and trends + + + +## How to Track Spend + +Track customer spend by including a `user` field in your API requests. The customer ID will be automatically tracked and associated with all spend from that request. + +### Example using cURL + +Make a `/chat/completions` call with the `user` field containing your customer ID: + +```bash showLineNumbers title="Track spend with customer ID" +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer sk-1234' \ # 👈 YOUR PROXY KEY + --data '{ + "model": "gpt-3.5-turbo", + "user": "customer-123", # 👈 CUSTOMER ID + "messages": [ + { + "role": "user", + "content": "What is the capital of France?" + } + ] + }' +``` + +The customer ID (`customer-123`) will be automatically upserted into the database with the new spend. If the customer ID already exists, spend will be incremented. + +### Example using OpenWebUI + +See the [Open WebUI tutorial](../tutorials/openweb_ui.md) for detailed instructions on connecting Open WebUI to LiteLLM and tracking customer usage. + +## How to View Spend + +### View Spend in Admin UI + +Navigate to the Customer Usage tab in the Admin UI to view customer-level spend analytics: + +#### 1. Access Customer Usage + +Go to the Usage page in the Admin UI (`PROXY_BASE_URL/ui/?login=success&page=new_usage`) and click on the **Customer Usage** tab. + + + +#### 2. View Customer Analytics + +The Customer Usage dashboard provides: + +- **Total spend per customer**: View aggregated spend across all customers +- **Daily spend trends**: See how customer spend changes over time +- **Model usage breakdown**: Understand which models each customer uses +- **Activity metrics**: Track requests, tokens, and success rates per customer + + + +#### 3. Filter by Customer + +Use the customer filter dropdown to view spend for specific customers: + +- Select one or more customer IDs from the dropdown +- View filtered analytics, spend logs, and activity metrics +- Compare spend across different customers + + + +## Use Cases + +### Customer Billing + +Track spend per customer to accurately bill your end users: + +- Monitor individual customer usage +- Generate invoices based on actual spend +- Set spending limits per customer + +### Usage Analytics + +Understand how different customers use your service: + +- Identify high-value customers +- Analyze usage patterns +- Optimize resource allocation + +--- + +## Related Features + +- [Customers / End-User Budgets](./customers.md) - Set budgets and rate limits for customers +- [Cost Tracking](./cost_tracking.md) - Comprehensive cost tracking and analytics +- [Billing](./billing.md) - Bill customers based on their usage diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index e40d7acc7c8..0f0e5f678d3 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -26,8 +26,6 @@ echo 'LITELLM_MASTER_KEY="sk-1234"' > .env # password generator to get a random hash for litellm salt key echo 'LITELLM_SALT_KEY="sk-1234"' >> .env -source .env - # Start docker compose up ``` @@ -1072,4 +1070,4 @@ A: We explored MySQL but that was hard to maintain and led to bugs for customers **Q: If there is Postgres downtime, how does LiteLLM react? Does it fail-open or is there API downtime?** -A: You can gracefully handle DB unavailability if it's on your VPC. See our production guide for more details: [Gracefully Handle DB Unavailability](https://docs.litellm.ai/docs/proxy/prod#6-if-running-litellm-on-vpc-gracefully-handle-db-unavailability) \ No newline at end of file +A: You can gracefully handle DB unavailability if it's on your VPC. See our production guide for more details: [Gracefully Handle DB Unavailability](https://docs.litellm.ai/docs/proxy/prod#6-if-running-litellm-on-vpc-gracefully-handle-db-unavailability) diff --git a/docs/my-website/docs/proxy/docker_quick_start.md b/docs/my-website/docs/proxy/docker_quick_start.md index d82a0b01d1d..35d9923e92c 100644 --- a/docs/my-website/docs/proxy/docker_quick_start.md +++ b/docs/my-website/docs/proxy/docker_quick_start.md @@ -52,8 +52,6 @@ echo 'LITELLM_MASTER_KEY="sk-1234"' > .env # password generator to get a random hash for litellm salt key echo 'LITELLM_SALT_KEY="sk-1234"' >> .env -source .env - # Start docker compose up ``` diff --git a/docs/my-website/docs/proxy/dynamic_rate_limit.md b/docs/my-website/docs/proxy/dynamic_rate_limit.md index 9c875a51eba..f5438b5a6f5 100644 --- a/docs/my-website/docs/proxy/dynamic_rate_limit.md +++ b/docs/my-website/docs/proxy/dynamic_rate_limit.md @@ -175,7 +175,37 @@ general_settings: litellm --config /path/to/config.yaml ``` -#### 2. Create Keys with Priority Levels +### Set priority on either a team or a key + +Priority can be set at either the **team level** or **key level**. Team-level priority takes precedence over key-level priority. + +**Option A: Set Priority on Team (Recommended)** + +All keys within a team will inherit the team's priority. This is useful when you want all keys for a specific environment or project to have the same priority. + +```bash +curl -X POST 'http://0.0.0.0:4000/team/new' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "team_alias": "production-team", + "metadata": {"priority": "prod"} +}' +``` + +Create a key for this team: +```bash +curl -X POST 'http://0.0.0.0:4000/key/generate' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "team_id": "team-id-from-previous-response" +}' +``` + +**Option B: Set Priority on Individual Keys** + +Set priority directly on the key. This is useful when you need fine-grained control per key. **Production Key:** ```bash @@ -205,7 +235,7 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \ -d '{}' ``` -**Expected Response for both:** +**Expected Response:** ```json { "key": "sk-...", @@ -214,6 +244,11 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \ } ``` +**Priority Resolution Order:** +1. If key belongs to a team with `metadata.priority` set → use team priority +2. Else if key has `metadata.priority` set → use key priority +3. Else → use `default_priority` from config + #### 3. Test Priority Allocation **Test Production Key (should get 9 RPM):** diff --git a/docs/my-website/docs/proxy/error_diagnosis.md b/docs/my-website/docs/proxy/error_diagnosis.md new file mode 100644 index 00000000000..9629fc52b0c --- /dev/null +++ b/docs/my-website/docs/proxy/error_diagnosis.md @@ -0,0 +1,90 @@ +# Diagnosing Errors - Provider vs Gateway + +Having trouble diagnosing if an error is from the **LLM Provider** (OpenAI, Anthropic, etc.) or from the **LiteLLM AI Gateway** itself? Here's how to tell. + +## Quick Rule + +**If the error contains `Exception`, it's from the provider.** + +| Error Contains | Error Source | +|----------------|--------------| +| `AnthropicException` | Anthropic | +| `OpenAIException` | OpenAI | +| `AzureException` | Azure | +| `BedrockException` | AWS Bedrock | +| `VertexAIException` | Google Vertex AI | +| No provider name | LiteLLM AI Gateway | + +## Examples + +### Provider Error (from AWS Bedrock) + +``` +{ + "error": { + "message": "litellm.BadRequestError: BedrockException - {\"message\":\"The model returned the following errors: messages.1.content.0.type: Expected `thinking` or `redacted_thinking`, but found `text`.\"}", + "type": "invalid_request_error", + "param": null, + "code": "400" + } +} +``` + +This error is from **AWS Bedrock** (notice `BedrockException`). The Bedrock API is rejecting the request due to invalid message format - this is not a LiteLLM issue. + +### Provider Error (from OpenAI) + +``` +{ + "error": { + "message": "litellm.AuthenticationError: OpenAIException - Incorrect API key provided: . You can find your API key at https://platform.openai.com/account/api-keys.", + "type": "invalid_request_error", + "param": null, + "code": "invalid_api_key" + } +} +``` + +This error is from **OpenAI** (notice `OpenAIException`). The OpenAI API key configured in LiteLLM is invalid. + +### Provider Error (from Anthropic) + +``` +{ + "error": { + "message": "litellm.InternalServerError: AnthropicException - Overloaded. Handle with `litellm.InternalServerError`.", + "type": "internal_server_error", + "param": null, + "code": "500" + } +} +``` + +This error is from **Anthropic** (notice `AnthropicException`). The Anthropic API is overloaded - this is not a LiteLLM issue. + +### Gateway Error (from LiteLLM) + +``` +{ + "error": { + "message": "Invalid API Key. Please check your LiteLLM API key.", + "type": "auth_error", + "param": null, + "code": "401" + } +} +``` + +This error is from the **LiteLLM AI Gateway** (no provider name). Your LiteLLM virtual key is invalid. + +## What to do? + +| Error Source | Action | +|--------------|--------| +| Provider Error | Check the provider's status page, adjust rate limits, or retry later | +| Gateway Error | Check your LiteLLM configuration, API keys, or [open an issue](https://github.com/BerriAI/litellm/issues) | + +## See Also + +- [Debugging](/docs/proxy/debugging) - Enable debug logs to see detailed request/response info +- [Exception Mapping](/docs/exception_mapping) - Full list of LiteLLM exception types diff --git a/docs/my-website/docs/proxy/guardrails/bedrock.md b/docs/my-website/docs/proxy/guardrails/bedrock.md index 4a1a0a246f8..8c71508fd23 100644 --- a/docs/my-website/docs/proxy/guardrails/bedrock.md +++ b/docs/my-website/docs/proxy/guardrails/bedrock.md @@ -188,6 +188,28 @@ My email is [EMAIL] and my phone number is [PHONE_NUMBER] This helps protect sensitive information while still allowing the model to understand the context of the request. +## Experimental: Only Send Latest User Message + +When you're chaining long conversations through Bedrock guardrails, you can opt into a lighter, experimental behavior by setting `experimental_use_latest_role_message_only: true` in the guardrail's `litellm_params`. When enabled, LiteLLM only sends the most recent `user` message (or assistant output during post-call checks) to Bedrock, which: + +- prevents unintended blocks on older system/dev messages +- keeps Bedrock payloads smaller, reducing latency and cost +- applies to proxy hooks (`pre_call`, `during_call`) and the `/guardrails/apply_guardrail` testing endpoint + +```yaml showLineNumbers title="litellm proxy config.yaml" +guardrails: + - guardrail_name: "bedrock-pre-guard" + litellm_params: + guardrail: bedrock + mode: "pre_call" + guardrailIdentifier: wf0hkdb5x07f + guardrailVersion: "DRAFT" + aws_region_name: os.environ/AWS_REGION + experimental_use_latest_role_message_only: true # NEW +``` + +> ⚠️ This flag is currently experimental and defaults to `false` to preserve the legacy behavior (entire message history). We'll be listening to user feedback to decide if this becomes the default or rolls out more broadly. + ## Disabling Exceptions on Bedrock BLOCK By default, when Bedrock guardrails block content, LiteLLM raises an HTTP 400 exception. However, you can disable this behavior by setting `disable_exception_on_block: true`. This is particularly useful when integrating with **OpenWebUI**, where exceptions can interrupt the chat flow and break the user experience. diff --git a/docs/my-website/docs/proxy/guardrails/lasso_security.md b/docs/my-website/docs/proxy/guardrails/lasso_security.md index 21528790afe..113e3f8974a 100644 --- a/docs/my-website/docs/proxy/guardrails/lasso_security.md +++ b/docs/my-website/docs/proxy/guardrails/lasso_security.md @@ -35,7 +35,7 @@ guardrails: guardrail: lasso mode: "pre_call" api_key: os.environ/LASSO_API_KEY - api_base: "https://server.lasso.security" + api_base: "https://server.lasso.security/gateway/v3" - guardrail_name: "lasso-post-guard" litellm_params: guardrail: lasso @@ -228,7 +228,7 @@ Expected response: ## PII Masking with Lasso -Lasso supports automatic PII detection and masking using the `/gateway/v1/classifix` endpoint. When enabled, sensitive information like emails, phone numbers, and other PII will be automatically masked with appropriate placeholders. +Lasso supports automatic PII detection and masking using the `/classifix` endpoint. When enabled, sensitive information like emails, phone numbers, and other PII will be automatically masked with appropriate placeholders. ### Enabling PII Masking diff --git a/docs/my-website/docs/proxy/guardrails/tool_permission.md b/docs/my-website/docs/proxy/guardrails/tool_permission.md index 19b674c9e55..1827333654f 100644 --- a/docs/my-website/docs/proxy/guardrails/tool_permission.md +++ b/docs/my-website/docs/proxy/guardrails/tool_permission.md @@ -1,4 +1,3 @@ -import Image from '@theme/IdealImage'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; @@ -7,9 +6,34 @@ import TabItem from '@theme/TabItem'; LiteLLM provides the LiteLLM Tool Permission Guardrail that lets you control which **tool calls** a model is allowed to invoke, using configurable allow/deny rules. This offers fine-grained, provider-agnostic control over tool execution (e.g., OpenAI Chat Completions `tool_calls`, Anthropic Messages `tool_use`, MCP tools). ## Quick Start -### 1. Define Guardrails on your LiteLLM config.yaml -Define your guardrails under the `guardrails` section +### LiteLLM UI + +#### Step 1: Select Tool Permission Guardrail + +Open the LiteLLM Dashboard, click **Add New Guardrail**, and choose **LiteLLM Tool Permission Guardrail**. This loads the rule builder UI. + +#### Step 2: Define Regex Rules + +1. Click **Add Rule**. +2. Enter a unique Rule ID. +3. Provide a regex for the tool name (e.g., `^mcp__github_.*$`). +4. Optionally add a regex for tool type (e.g., `^function$`). +5. Pick **Allow** or **Deny**. + +#### Step 3: Restrict Tool Arguments (Optional) + +Select **+ Restrict tool arguments** to attach regex validations to nested paths (dot + `[]` notation). This enforces that sensitive parameters (such as `arguments.to[]`) conform to pre-approved formats. + +#### Step 4: Choose Defaults & Actions + +- Set the fallback decision (`default_action`) for tools that do not hit any rule. +- Decide how disallowed tools behave: **Block** halts the request, **Rewrite** strips forbidden tools and returns an error message inside the response. +- Customize `violation_message_template` if you want branded error copy. +- Save the guardrail. + +### LiteLLM Config.yaml Setup + ```yaml guardrails: - guardrail_name: "tool-permission-guardrail" @@ -21,16 +45,17 @@ guardrails: tool_name: "Bash" decision: "allow" - id: "allow_github_mcp" - tool_name: "mcp__github_*" + tool_name: "^mcp__github_.*$" decision: "allow" - id: "allow_aws_documentation" - tool_name: "mcp__aws-documentation_*_documentation" + tool_name: "^mcp__aws-documentation_.*_documentation$" decision: "allow" - id: "deny_read_commands" tool_name: "Read" - decision: "Deny" + decision: "deny" - id: "mail-domain" - tool_name: "send_email" + tool_name: "^send_email$" + tool_type: "^function$" decision: "allow" allowed_param_patterns: "to[]": "^.+@berri\\.ai$" @@ -44,7 +69,8 @@ guardrails: ```yaml - id: "unique_rule_id" # Unique identifier for the rule - tool_name: "pattern" # Tool name or pattern to match + tool_name: "^regex$" # Regex for tool name (optional, at least one of name/type required) + tool_type: "^function$" # Regex for tool type (optional) decision: "allow" # "allow" or "deny" allowed_param_patterns: # Optional - regex map for argument paths (dot + [] notation) "path.to[].field": "^regex$" diff --git a/docs/my-website/docs/proxy/multi_tenant_architecture.md b/docs/my-website/docs/proxy/multi_tenant_architecture.md new file mode 100644 index 00000000000..9e71530f165 --- /dev/null +++ b/docs/my-website/docs/proxy/multi_tenant_architecture.md @@ -0,0 +1,710 @@ +import Image from '@theme/IdealImage'; + +# Multi-Tenant Architecture with LiteLLM + +## Overview + +LiteLLM provides a centralized solution that scales across multiple tenants, enabling organizations to: + +- **Centrally manage** LLM access for multiple tenants (organizations, teams, departments) +- **Isolate spend and usage** across different organizational units +- **Delegate administration** without compromising security +- **Track costs** at granular levels (organization → team → user → key) +- **Scale seamlessly** as new teams and users are added + +:::info Open Source vs. Enterprise +- **Teams + Virtual Keys**: ✅ Available in open source +- **Organizations + Org Admins**: ✨ Enterprise feature ([Get a 7 day trial](https://www.litellm.ai/#trial)) + +You can implement multi-tenancy using **Teams** alone in the open source version, or add **Organizations** on top for additional hierarchy in the enterprise version. +::: + +## The Multi-Tenant Challenge + +Organizations with multi-tenant architectures face several challenges when deploying LLM solutions: + +1. **Centralized vs. Decentralized**: Need a single unified gateway while maintaining tenant isolation +2. **Cost Attribution**: Tracking spend across different business units, departments, or customers +3. **Access Control**: Different teams need different models, budgets, and rate limits +4. **Delegation**: Team leads should manage their teams without platform-wide admin access +5. **Scalability**: Solution must scale from 10 to 10,000+ users without architectural changes + +## How LiteLLM Solves Multi-Tenancy + + + +LiteLLM implements a hierarchical multi-tenant architecture with four levels: + +### 1. Organizations (Top-Level Tenants) ✨ Enterprise Feature + +**Organizations** represent the highest level of tenant isolation - typically different business units, departments, or customers. + +- Each organization has its own: + - Budget limits + - Allowed models + - Admin users (org admins) + - Teams + - Spend tracking + +**Use Cases:** +- **Enterprise Departments**: Separate organizations for Engineering, Marketing, Sales +- **Multi-Customer SaaS**: Each customer is an organization with full isolation +- **Geographic Regions**: EMEA, APAC, Americas as separate organizations + +**Key Features:** +- Organizations cannot see each other's data +- Each organization can have multiple teams +- Organization admins manage teams within their organization only +- Spend and usage tracked at organization level + +[API Reference for Organizations](https://litellm-api.up.railway.app/#/organization%20management) + +--- + +### 2. Teams (Mid-Level Grouping) ✅ Open Source + +**Teams** can work independently or sit within organizations, representing logical groupings of users working together. + +:::tip +Teams are available in **open source** and can be used as your primary multi-tenant boundary without needing Organizations. Organizations provide an additional layer of hierarchy for enterprise deployments. +::: + +- Each team has: + - Team-specific budgets and rate limits + - Team admins who manage members + - Service account keys for shared resources + - Model access controls + - Granular team member permissions + +**Use Cases:** +- **Project Teams**: ML Research team, Product team, Data Science team +- **Customer Sub-Groups**: Different divisions within a customer organization +- **Environment Separation**: Development, Staging, Production teams + +**Key Features:** +- Teams inherit organization constraints (can't exceed org budget/models) +- Team admins can manage their team without affecting others +- Service account keys survive team member changes +- Per-team spend tracking and billing + +[API Reference for Teams](https://litellm-api.up.railway.app/#/team%20management) + +--- + +### 3. Users (Individual Members) ✅ Open Source + +**Users** are individuals who belong to teams and create/use API keys. + +- Each user can: + - Belong to multiple teams + - Have their own budget limits + - Create personal API keys + - Track individual spend + +**User Types:** +- **Internal Users**: Employees, developers, data scientists +- **Team Admins**: Lead their teams, manage members +- **Org Admins**: Manage multiple teams within their organization +- **Proxy Admins**: Platform-wide administrators + +**Key Features:** +- User spend tracked individually +- Users can be on multiple teams simultaneously +- Role-based permissions control what users can do +- User keys deleted when user is removed + +[API Reference for Users](https://litellm-api.up.railway.app/#/user%20management) + +--- + +### 4. Virtual Keys (Authentication Layer) ✅ Open Source + +**Virtual Keys** are the API keys used to authenticate requests and track spend. + +Each key can be one of three types: + +| Key Type | Configuration | Use Case | Spend Tracking | Lifecycle | +|----------|---------------|----------|----------------|-----------| +| **User-only** | `user_id` only | Developer personal keys | User level | Deleted with user | +| **Team Service Account** | `team_id` only | Production apps, CI/CD | Team level | Survives member changes | +| **User + Team** | Both `user_id` and `team_id` | User within team context | User AND Team | Deleted with user | + +**Example Scenarios:** +- Use **user-only keys** for developers testing locally +- Use **team service account keys** for your production application that shouldn't break when employees leave +- Use **user + team keys** when you want individual accountability within a team budget + +[API Reference for Keys](https://litellm-api.up.railway.app/#/key%20management) + +--- + +## Role-Based Access Control (RBAC) + +LiteLLM provides granular RBAC across the hierarchy: + +### Global Proxy Roles (Platform-Wide) + +| Role | Scope | Permissions | +|------|-------|-------------| +| **Proxy Admin** | Entire platform | Create orgs, teams, users. View all spend. Full control. | +| **Proxy Admin Viewer** | Entire platform | View-only access to all data. Cannot make changes. | +| **Internal User** | Own resources | Create/delete own keys. View own spend. | + +### Organization/Team Roles (Scoped) + +| Role | Scope | Permissions | +|------|-------|-------------| +| **Org Admin** ✨ | Specific organization | Create teams, add users, view org spend within their org only. | +| **Team Admin** ✨ | Specific team | Manage team members, budgets, keys within their team only. | + +✨ = Premium Feature + +### Team Member Permissions + +Team admins can configure granular permissions for regular team members: + +**Read-only** (default): +```json +["/key/info", "/key/health"] +``` + +**Allow key creation**: +```json +["/key/info", "/key/health", "/key/generate", "/key/update"] +``` + +**Full key management**: +```json +["/key/info", "/key/health", "/key/generate", "/key/update", "/key/delete", "/key/regenerate", "/key/block", "/key/unblock"] +``` + +[Learn more about RBAC](./access_control) + +--- + +## Spend Tracking & Cost Attribution + +LiteLLM provides multi-level spend tracking that flows through the hierarchy: + +### Hierarchical Spend Flow + +``` +Organization Spend + ├── Team 1 Spend + │ ├── User A Spend + │ │ ├── Key 1 Spend + │ │ └── Key 2 Spend + │ └── Service Account Spend + │ └── Key 3 Spend + └── Team 2 Spend + └── User B Spend + └── Key 4 Spend +``` + +### Budget Enforcement + +Budgets can be set at every level with inheritance: + +1. **Organization Budget**: `$10,000/month` + - Team 1: `$6,000/month` (within org limit) + - User A: `$3,000/month` (within team limit) + - User B: `$3,000/month` (within team limit) + - Team 2: `$4,000/month` (within org limit) + +**Enforcement Rules:** +- Team budgets cannot exceed organization budget +- User budgets cannot exceed team budget +- Requests blocked when any level exceeds budget +- Real-time tracking prevents overruns + +[Learn more about Budgets](./team_budgets) + +--- + +## Common Multi-Tenant Patterns + +### Pattern 1: Enterprise Departments + +**Scenario**: Large enterprise with multiple departments needing centralized LLM access + +**Enterprise Setup** (with Organizations): +``` +Platform (LiteLLM Instance) +├── Engineering Organization ✨ +│ ├── Backend Team +│ ├── Frontend Team +│ └── ML Team +├── Marketing Organization ✨ +│ ├── Content Team +│ └── Analytics Team +└── Sales Organization ✨ + ├── Sales Ops Team + └── Customer Success Team +``` + +**Open Source Alternative** (Teams only): +``` +Platform (LiteLLM Instance) +├── Engineering Backend Team +├── Engineering Frontend Team +├── Engineering ML Team +├── Marketing Content Team +├── Marketing Analytics Team +├── Sales Ops Team +└── Customer Success Team +``` + +**Benefits:** +- Each department/team manages their own budget +- Department leads (org/team admins) control their teams +- Centralized billing and model access +- Cross-department cost visibility for finance + +--- + +### Pattern 2: Multi-Customer SaaS + +**Scenario**: SaaS provider offering LLM-powered features to multiple customers + +**Enterprise Setup** (with Organizations): +``` +Platform (LiteLLM Instance) +├── Customer A Organization ✨ +│ ├── Production Team (Service Accounts) +│ ├── Development Team +│ └── QA Team +├── Customer B Organization ✨ +│ ├── Production Team (Service Accounts) +│ └── Development Team +└── Customer C Organization ✨ + └── Production Team (Service Accounts) +``` + +**Open Source Alternative** (Teams only): +``` +Platform (LiteLLM Instance) +├── Customer A Production Team (Service Accounts) +├── Customer A Development Team +├── Customer A QA Team +├── Customer B Production Team (Service Accounts) +├── Customer B Development Team +└── Customer C Production Team (Service Accounts) +``` + +**Benefits:** +- Complete isolation between customers/teams +- Per-customer/team billing and usage tracking +- Customer/team admins can self-serve +- Production service account keys survive employee turnover + +--- + +### Pattern 3: Environment Separation + +**Scenario**: Single organization with multiple environments + +``` +Platform (LiteLLM Instance) +└── Company Organization + ├── Production Team + │ └── Service Account Keys (strict rate limits) + ├── Staging Team + │ └── Service Account Keys (moderate limits) + └── Development Team + └── User Keys (generous limits for testing) +``` + +**Benefits:** +- Separate budgets for each environment +- Different model access (production vs. development) +- Prevent development usage from affecting production budget +- Easy cost attribution by environment + +--- + +## Delegation & Self-Service + +One of LiteLLM's key advantages is delegated administration: + +### Without LiteLLM +``` +Every team → Requests platform admin → Admin makes changes +``` +❌ Bottleneck on platform team +❌ Slow onboarding +❌ Poor scalability + +### With LiteLLM +``` +Proxy Admin → Creates org + org admin +Org Admin → Creates teams + team admins +Team Admin → Manages their team independently +``` +✅ Decentralized management +✅ Fast onboarding +✅ Scales to thousands of users + +### Self-Service Capabilities + +**Team Admins Can:** +- Add/remove team members +- Create API keys for team members +- Update team budgets (within org limits) +- Configure team member permissions +- View team usage and spend + +**Org Admins Can:** +- Create new teams within their organization +- Assign team admins +- View organization-wide spend +- Manage users across their teams + +**Platform Admins Can:** +- Create organizations +- Assign org admins +- Set organization-level policies +- View platform-wide analytics + +--- + +## Scalability + +LiteLLM's architecture scales from small teams to enterprise deployments: + +### Small Team (10-100 users) +- Single organization +- Few teams (5-10) +- Proxy admins manage everything + +### Mid-Size (100-1,000 users) +- Multiple organizations +- Many teams (50+) +- Org admins delegate to team admins + +### Enterprise (1,000+ users) +- Many organizations (departments/regions) +- Hundreds of teams +- Fully delegated admin structure +- Centralized observability and billing + +**Key Scalability Features:** +- No architectural changes needed as you grow +- Database-backed (PostgreSQL) for reliability +- Horizontal scaling support +- Efficient spend tracking and logging + +--- + +## Security & Isolation + +### Tenant Isolation + +Each tenant (organization) is isolated: +- ✅ Cannot view other organizations' data +- ✅ Cannot access other organizations' keys +- ✅ Cannot exceed their budget limits +- ✅ Cannot access models not in their allowed list + +### Authentication Security + +- Master key for platform admins +- Virtual keys with scoped permissions +- SSO integration support +- JWT authentication +- IP allowlisting + +### Audit & Compliance + +- All API calls logged with user/team/org context +- Spend tracking for chargeback/showback +- Admin actions audited +- Integration with observability tools + +[Learn more about Security](../data_security) + +--- + +## Getting Started + +:::info Enterprise vs. Open Source Setup +The steps below show the **full enterprise hierarchy** with Organizations. + +For **open source**, skip Steps 1-2 and start directly with **Step 3** (creating teams). Teams can function as your top-level tenant boundary without Organizations. +::: + +### Step 1: Set Up Organizations ✨ Enterprise + +Create your first organization: + +```bash +curl --location 'http://0.0.0.0:4000/organization/new' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "organization_alias": "engineering_department", + "models": ["gpt-4", "gpt-4o", "claude-3-5-sonnet"], + "max_budget": 10000 + }' +``` + +### Step 2: Add an Organization Admin ✨ Enterprise + +```bash +curl -X POST 'http://0.0.0.0:4000/organization/member_add' \ + -H 'Authorization: Bearer sk-1234' \ + -H 'Content-Type: application/json' \ + -d '{ + "organization_id": "org-123", + "member": { + "role": "org_admin", + "user_id": "admin@company.com" + } + }' +``` + +### Step 3: Create Teams ✅ Open Source + +**For Enterprise:** Organization admin creates team within their organization +**For Open Source:** Proxy admin creates team directly (no `organization_id` needed) + +```bash +# Enterprise: Org admin creates team in their organization +curl --location 'http://0.0.0.0:4000/team/new' \ + --header 'Authorization: Bearer sk-org-admin-key' \ + --header 'Content-Type: application/json' \ + --data '{ + "team_alias": "ml_team", + "organization_id": "org-123", + "max_budget": 5000 + }' + +# Open Source: Proxy admin creates team directly +curl --location 'http://0.0.0.0:4000/team/new' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "team_alias": "ml_team", + "max_budget": 5000 + }' +``` + +### Step 4: Add Team Admin + +```bash +curl -X POST 'http://0.0.0.0:4000/team/member_add' \ + -H 'Authorization: Bearer sk-org-admin-key' \ + -H 'Content-Type: application/json' \ + -d '{ + "team_id": "team-456", + "member": { + "role": "admin", + "user_id": "team-lead@company.com" + } + }' +``` + +### Step 5: Team Admin Manages Their Team + +```bash +# Team admin adds members +curl -X POST 'http://0.0.0.0:4000/team/member_add' \ + -H 'Authorization: Bearer sk-team-admin-key' \ + -H 'Content-Type: application/json' \ + -d '{ + "team_id": "team-456", + "member": { + "role": "user", + "user_id": "developer@company.com" + } + }' + +# Team admin creates keys for members +curl --location 'http://0.0.0.0:4000/key/generate' \ + --header 'Authorization: Bearer sk-team-admin-key' \ + --header 'Content-Type: application/json' \ + --data '{ + "user_id": "developer@company.com", + "team_id": "team-456" + }' +``` + +--- + +## Use Case Examples + +### Example 1: Chargeback Model + +**Goal**: Each business unit pays for their own LLM usage + +**Setup:** +1. Create organization per business unit +2. Set budgets based on allocated budgets +3. Track spend per organization +4. Generate monthly reports for finance + +**Result**: Finance can charge back costs to respective departments with accurate attribution. + +--- + +### Example 2: Customer-Facing AI Product + +**Goal**: Provide LLM capabilities to customers with isolation and cost tracking + +**Setup:** +1. Create organization per customer +2. Use service account keys for production workloads +3. Track spend per customer organization +4. Set rate limits per customer tier + +**Result**: Bill customers accurately, prevent noisy neighbors, maintain isolation. + +--- + +### Example 3: Development vs. Production + +**Goal**: Separate development and production environments with different policies + +**Setup:** +1. Create "Development" and "Production" teams +2. Development: Generous budgets, all models, user keys +3. Production: Strict budgets, approved models only, service account keys +4. Different rate limits per environment + +**Result**: Developers can experiment freely without impacting production budget or reliability. + +--- + +## Best Practices + +### 1. Organization Design + +- ✅ Map organizations to cost centers or customers +- ✅ Set realistic budgets with buffer for growth +- ✅ Assign 1-2 org admins per organization +- ❌ Don't create too many organizations (adds management overhead) + +### 2. Team Structure + +- ✅ Keep teams aligned with actual working groups +- ✅ Use service account keys for production +- ✅ Give team admins enough permissions to self-serve +- ❌ Don't create single-user teams (use user-only keys instead) + +### 3. Key Management + +- ✅ Use descriptive key names +- ✅ Rotate keys regularly +- ✅ Delete unused keys +- ✅ Use appropriate key type for use case +- ❌ Don't share keys across users/teams + +### 4. Budget Management + +- ✅ Set budgets at multiple levels (org → team → user) +- ✅ Monitor spend regularly +- ✅ Alert before budget exhaustion +- ❌ Don't set budgets too tight (may block legitimate usage) + +### 5. Delegation + +- ✅ Assign org admins for large organizations +- ✅ Assign team admins for active teams +- ✅ Configure team member permissions appropriately +- ❌ Don't make everyone a proxy admin + +--- + +## Monitoring & Observability + +LiteLLM provides comprehensive monitoring: + +- **Spend Tracking**: Real-time spend by org/team/user/key +- **Usage Analytics**: Request counts, token usage, model usage +- **Admin UI**: Visual dashboard for all metrics +- **Logging**: Detailed logs with tenant context +- **Alerting**: Budget alerts, rate limit alerts, error alerts + +[Learn more about Logging](./logging) + +--- + +## Comparison with Other Approaches + +| Approach | Pros | Cons | LiteLLM Advantage | +|----------|------|------|-------------------| +| **Separate instances per tenant** | Strong isolation | High operational overhead, cost inefficient | Single instance, same isolation, 90% cost reduction | +| **Single shared pool** | Simple setup | No cost attribution, no access control | Full attribution, granular access control | +| **API key prefixes** | Basic separation | Manual tracking, no hierarchy, no RBAC | Automatic tracking, hierarchical, full RBAC | +| **External auth layer** | Flexible | Complex integration, no built-in budgets | Native integration, built-in budgets | + +--- + +## FAQ + +**Q: Can users belong to multiple teams?** +A: Yes, users can be members of multiple teams and have different keys for each team. + +**Q: What happens when a user leaves?** +A: User-specific keys are deleted, but team service account keys remain active. + +**Q: Can team budgets exceed organization budget?** +A: No, the system enforces that team budgets cannot exceed their organization's budget. + +**Q: How granular is the cost tracking?** +A: Every API call is tracked with organization, team, user, and key context. + +**Q: Can I have teams without organizations?** +A: Yes! Teams work independently in **open source** without needing Organizations. Organizations are an **enterprise feature** that adds an additional hierarchy layer on top of teams. + +**Q: Is there a limit to hierarchy depth?** +A: The hierarchy is: Organization → Team → User → Key (4 levels). This covers most use cases. + +**Q: How do I migrate from flat structure to hierarchical?** +A: You can gradually create organizations and teams, then move existing users/keys into them. + +--- + +## Related Documentation + +- [User Management Hierarchy](./user_management_heirarchy) - Visual hierarchy overview +- [Access Control (RBAC)](./access_control) - Detailed role permissions +- [Team Budgets](./team_budgets) - Budget management guide +- [Virtual Keys](./virtual_keys) - API key management +- [Admin UI](./ui) - Visual dashboard for management + +--- + +## Summary + +LiteLLM solves multi-tenant architecture challenges through: + +1. **Hierarchical Structure**: Organizations → Teams → Users → Keys +2. **Granular RBAC**: Platform-wide and tenant-scoped roles +3. **Cost Attribution**: Spend tracking at every level +4. **Delegation**: Org admins and team admins self-manage +5. **Isolation**: Strong tenant boundaries +6. **Scalability**: Handles 10 to 10,000+ users with same architecture + +### Open Source vs. Enterprise + +**Open Source** (Teams + Users + Keys): +- ✅ Teams as primary tenant boundary +- ✅ Team admins manage their teams +- ✅ Virtual keys with team/user tracking +- ✅ Budget and rate limits per team +- ✅ Spend tracking and logging + +**Enterprise** (Adds Organizations layer): +- ✨ Organizations for top-level tenant isolation +- ✨ Organization admins manage multiple teams +- ✨ Organization-level budgets and model access +- ✨ Hierarchical delegation and reporting + +This makes LiteLLM ideal for: +- ✅ Enterprises with multiple departments +- ✅ SaaS providers with multiple customers +- ✅ Organizations needing cost chargeback/showback +- ✅ Teams requiring self-service LLM access +- ✅ Any multi-tenant LLM deployment + +[Start with LiteLLM Proxy →](./quick_start) diff --git a/docs/my-website/docs/proxy/pass_through.md b/docs/my-website/docs/proxy/pass_through.md index 7309cdeda26..03454004b8c 100644 --- a/docs/my-website/docs/proxy/pass_through.md +++ b/docs/my-website/docs/proxy/pass_through.md @@ -275,6 +275,20 @@ In this video, we'll add the Azure OpenAI Assistants API as a pass through endpo - Check LiteLLM proxy logs for error details - Verify the target API's expected request format +### Allowing Team JWTs to use pass-through routes + +If you are using pass-through provider routes (e.g., `/anthropic/*`) and want your JWT team tokens to access these routes, add `mapped_pass_through_routes` to the `team_allowed_routes` in `litellm_jwtauth` or explicitly add the relevant route(s). + +Example (`proxy_server_config.yaml`): + +```yaml +general_settings: + enable_jwt_auth: True + litellm_jwtauth: + team_ids_jwt_field: "team_ids" + team_allowed_routes: ["openai_routes","info_routes","mapped_pass_through_routes"] +``` + ### Getting Help [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) diff --git a/docs/my-website/docs/proxy/token_auth.md b/docs/my-website/docs/proxy/token_auth.md index c2a88010d79..1db1b2a8965 100644 --- a/docs/my-website/docs/proxy/token_auth.md +++ b/docs/my-website/docs/proxy/token_auth.md @@ -338,6 +338,58 @@ general_settings: team_allowed_routes: ["/v1/chat/completions"] # 👈 Set accepted routes ``` +### Allowing other provider routes for Teams + +To enable team JWT tokens to access Anthropic-style endpoints such as `/v1/messages`, update `team_allowed_routes` in your `litellm_jwtauth` configuration. `team_allowed_routes` supports the following values: + +- Named route groups from `LiteLLMRoutes` (e.g., `openai_routes`, `anthropic_routes`, `info_routes`, `mapped_pass_through_routes`). + +Below is a quick reference for the route groups you can use and example representative routes from each group. If you need the exhaustive list, see the `LiteLLMRoutes` enum in `litellm/proxy/_types.py` for the authoritative list. + +| Route Group | What it contains | Representative routes | +|-------------|------------------|-----------------------| +| `openai_routes` | OpenAI-compatible REST endpoints (chat, completion, embeddings, images, responses, models, etc.) | `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/images/generations`, `/v1/models` | +| `anthropic_routes` | Anthropic-style endpoints (`/v1/messages` and related) | `/v1/messages`, `/v1/messages/count_tokens`, `/v1/skills` | +| `mapped_pass_through_routes` | Provider-specific pass-through route prefixes (e.g., Anthropic when proxied via `/anthropic`). Use with `mapped_pass_through_routes` for provider wildcard mapping | `/anthropic/*`, `/vertex-ai/*`, `/bedrock/*` | +| `passthrough_routes_wildcard` | Wildcard mapping for providers (e.g., `/anthropic/*`) - precomputed wildcard list used by the proxy | `/anthropic/*`, `/vllm/*` | +| `google_routes` | Google-specific (e.g., Vertex / Batching endpoints) | `/v1beta/models/{model_name}:generateContent` | +| `mcp_routes` | Internal MCP management endpoints | `/mcp/tools`, `/mcp/tools/call` | +| `info_routes` | Read-only & info endpoints used by the UI | `/key/info`, `/team/info`, `/v1/models` | +| `management_routes` | Admin-only management endpoints (create/update/delete user/team/model) | `/team/new`, `/key/generate`, `/model/new` | +| `spend_tracking_routes` | Budget/spend related endpoints | `/spend/logs`, `/spend/keys` | +| `public_routes` | Public and unauthenticated endpoints | `/`, `/routes`, `/.well-known/litellm-ui-config` | + +Note: `llm_api_routes` is the union of OpenAI, Anthropic, Google, pass-through and other LLM routes (`openai_routes + anthropic_routes + google_routes + mapped_pass_through_routes + passthrough_routes_wildcard + apply_guardrail_routes + mcp_routes + litellm_native_routes`). + +Defaults (what the proxy uses if you don't override them in `litellm_jwtauth`): + +- `admin_jwt_scope`: `litellm_proxy_admin` +- `admin_allowed_routes` (default): `management_routes`, `spend_tracking_routes`, `global_spend_tracking_routes`, `info_routes` +- `team_allowed_routes` (default): `openai_routes`, `info_routes` +- `public_allowed_routes` (default): `public_routes` + + +Example: Allow team JWTs to call Anthropic `/v1/messages` (either by route group or by explicit route string): + +```yaml +general_settings: + enable_jwt_auth: True + litellm_jwtauth: + team_ids_jwt_field: "team_ids" + team_allowed_routes: ["openai_routes", "info_routes", "anthropic_routes"] +``` + +Or selectively allow the exact Anthropic message endpoint only: + +```yaml +general_settings: + enable_jwt_auth: True + litellm_jwtauth: + team_ids_jwt_field: "team_ids" + team_allowed_routes: ["/v1/messages", "info_routes"] +``` + + ### Caching Public Keys Control how long public keys are cached for (in seconds). @@ -407,6 +459,72 @@ general_settings: user_id_upsert: true # 👈 upserts the user to db, if valid email but not in db ``` +## OIDC UserInfo Endpoint + +Use this when your JWT/access token doesn't contain user-identifying information. LiteLLM will call your identity provider's UserInfo endpoint to fetch user details. + +### When to Use + +- Your JWT is opaque (not self-contained) or lacks user claims +- You need to fetch fresh user information from your identity provider +- Your access tokens don't include email, roles, or other identifying data + +### Configuration + +```yaml title="config.yaml" showLineNumbers +general_settings: + enable_jwt_auth: True + litellm_jwtauth: + # Enable OIDC UserInfo endpoint + oidc_userinfo_enabled: true + oidc_userinfo_endpoint: "https://your-idp.com/oauth2/userinfo" + oidc_userinfo_cache_ttl: 300 # Cache for 5 minutes (default: 300) + + # Map fields from UserInfo response + user_id_jwt_field: "sub" + user_email_jwt_field: "email" + user_roles_jwt_field: "roles" +``` + +### Flow Diagram + +```mermaid +sequenceDiagram + participant Client + participant LiteLLM + participant IdP as Identity Provider + + Client->>LiteLLM: Request with Bearer token + Note over LiteLLM: Check cache for UserInfo + + LiteLLM->>IdP: GET /userinfo (if not cached)
Authorization: Bearer {token} + IdP-->>LiteLLM: User data (sub, email, roles) + + Note over LiteLLM: Cache response (TTL: 5min)
Extract user_id, email, roles
Perform RBAC checks + + LiteLLM-->>Client: Authorized/Denied +``` + +### Example: Azure AD + +```yaml title="config.yaml" showLineNumbers +litellm_jwtauth: + oidc_userinfo_enabled: true + oidc_userinfo_endpoint: "https://graph.microsoft.com/oidc/userinfo" + user_id_jwt_field: "sub" + user_email_jwt_field: "email" +``` + +### Example: Keycloak + +```yaml title="config.yaml" showLineNumbers +litellm_jwtauth: + oidc_userinfo_enabled: true + oidc_userinfo_endpoint: "https://keycloak.example.com/realms/your-realm/protocol/openid-connect/userinfo" + user_id_jwt_field: "sub" + user_roles_jwt_field: "resource_access.your-client.roles" +``` + ## [BETA] Control Access with OIDC Roles Allow JWT tokens with supported roles to access the proxy. diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md index 96bfc196d0e..52e4c1e26f5 100644 --- a/docs/my-website/docs/response_api.md +++ b/docs/my-website/docs/response_api.md @@ -81,6 +81,85 @@ for event in stream: f.write(image_bytes) ``` +#### Image Generation (Non-streaming) + +Image generation is supported for models that generate images. Generated images are returned in the `output` array with `type: "image_generation_call"`. + +**Gemini (Google AI Studio):** +```python showLineNumbers title="Gemini Image Generation" +import litellm +import base64 + +# Gemini image generation models don't require tools parameter +response = litellm.responses( + model="gemini/gemini-2.5-flash-image", + input="Generate a cute cat playing with yarn" +) + +# Access generated images from output +for item in response.output: + if item.type == "image_generation_call": + # item.result contains pure base64 (no data: prefix) + image_bytes = base64.b64decode(item.result) + + # Save the image + with open(f"generated_{item.id}.png", "wb") as f: + f.write(image_bytes) + +print(f"Image saved: generated_{response.output[0].id}.png") +``` + +**OpenAI:** +```python showLineNumbers title="OpenAI Image Generation" +import litellm +import base64 + +# OpenAI models require tools parameter for image generation +response = litellm.responses( + model="openai/gpt-4o", + input="Generate a futuristic city at sunset", + tools=[{"type": "image_generation"}] +) + +# Access generated images from output +for item in response.output: + if item.type == "image_generation_call": + image_bytes = base64.b64decode(item.result) + with open(f"generated_{item.id}.png", "wb") as f: + f.write(image_bytes) +``` + +**Response Format:** + +When image generation is successful, the response contains: + +```json +{ + "id": "resp_abc123", + "status": "completed", + "output": [ + { + "type": "image_generation_call", + "id": "resp_abc123_img_0", + "status": "completed", + "result": "iVBORw0KGgo..." // Pure base64 string (no data: prefix) + } + ] +} +``` + +**Supported Models:** + +| Provider | Models | Requires `tools` Parameter | +|----------|--------|---------------------------| +| Google AI Studio | `gemini/gemini-2.5-flash-image` | ❌ No | +| Vertex AI | `vertex_ai/gemini-2.5-flash-image-preview` | ❌ No | +| OpenAI | `gpt-4o`, `gpt-4o-mini`, `gpt-4.1`, `gpt-4.1-mini`, `gpt-4.1-nano`, `o3` | ✅ Yes | +| AWS Bedrock | Stability AI, Amazon Nova Canvas models | Model-specific | +| Fal AI | Various image generation models | Check model docs | + +**Note:** The `result` field contains pure base64-encoded image data without the `data:image/png;base64,` prefix. You must decode it with `base64.b64decode()` before saving. + #### GET a Response ```python showLineNumbers title="Get Response by ID" import litellm diff --git a/docs/my-website/docs/secret_managers/cyberark.md b/docs/my-website/docs/secret_managers/cyberark.md index 37aa1086691..c33aa286703 100644 --- a/docs/my-website/docs/secret_managers/cyberark.md +++ b/docs/my-website/docs/secret_managers/cyberark.md @@ -41,6 +41,7 @@ CYBERARK_CLIENT_KEY="path/to/client.key" # OPTIONAL CYBERARK_REFRESH_INTERVAL="300" # defaults to 300 seconds (5 minutes), frequency of token refresh +CYBERARK_SSL_VERIFY="true" # defaults to true, set to "false" to disable SSL verification (for self-signed certificates) ``` **Step 2.** Add to proxy config.yaml @@ -172,6 +173,24 @@ If these commands work successfully against your CyberArk instance, then CyberAr - The `CYBERARK_API_BASE` URL is accessible from your LiteLLM instance - Your API key or certificates have the necessary permissions in CyberArk +### SSL Certificate Errors + +If you encounter SSL certificate verification errors like: + +``` +RuntimeError: Could not authenticate to CyberArk Conjur: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain +``` + +This typically occurs when your CyberArk Conjur instance uses a self-signed certificate. You can disable SSL verification by setting: + +```bash +CYBERARK_SSL_VERIFY="false" +``` + +:::warning +Disabling SSL verification is insecure and should only be used for testing or development environments with self-signed certificates. For production, configure your certificate chain properly or use certificate-based authentication with `CYBERARK_CLIENT_CERT` and `CYBERARK_CLIENT_KEY`. +::: + ## Video Walkthrough This video walks through using CyberArk Conjur as a secret manager with LiteLLM. We create a virtual key in the LiteLLM Admin UI and verify it exists in CyberArk. Then we rotate the secret key and verify it exists in CyberArk. diff --git a/docs/my-website/docs/tutorials/cursor_integration.md b/docs/my-website/docs/tutorials/cursor_integration.md new file mode 100644 index 00000000000..f0d87b050cf --- /dev/null +++ b/docs/my-website/docs/tutorials/cursor_integration.md @@ -0,0 +1,226 @@ +--- +sidebar_label: "Cursor IDE" +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Cursor IDE Integration with LiteLLM + +This tutorial shows you how to integrate Cursor IDE with LiteLLM Proxy, allowing you to use any LiteLLM-supported model through Cursor's interface with BYOK (Bring Your Own Key) and custom base URL. + +## Benefits of using Cursor with LiteLLM + +When you use Cursor IDE with LiteLLM you get the following benefits: + +**Developer Benefits:** +- Universal Model Access: Use any LiteLLM supported model (Anthropic, OpenAI, Vertex AI, Bedrock, etc.) through the Cursor IDE interface. +- Higher Rate Limits & Reliability: Load balance across multiple models and providers to avoid hitting individual provider limits, with fallbacks to ensure you get responses even if one provider fails. +- Streaming Support: Full streaming support with proper response transformation for Cursor's expected format. + +**Proxy Admin Benefits:** +- Centralized Management: Control access to all models through a single LiteLLM proxy instance without giving your developers API Keys to each provider. +- Budget Controls: Set spending limits and track costs across all Cursor usage. +- Request Logging: Track all requests made through Cursor for debugging and monitoring. + +## Prerequisites + +Before you begin, ensure you have: +- Cursor IDE installed +- A running LiteLLM Proxy instance with **HTTPS enabled** (HTTP is not supported) +- A valid LiteLLM Proxy API key +- An HTTPS domain for your LiteLLM Proxy (required by Cursor) + +## Quick Start Guide + +### Step 1: Install LiteLLM + +Install LiteLLM with proxy support: + +```bash +pip install litellm[proxy] +``` + +### Step 2: Configure LiteLLM Proxy + +Create a `config.yaml` file with your model configurations: + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: gpt-4o + litellm_params: + model: gpt-4o + api_key: os.environ/OPENAI_API_KEY + + - model_name: claude-3-5-sonnet + litellm_params: + model: anthropic/claude-3-5-sonnet-20241022 + api_key: os.environ/ANTHROPIC_API_KEY + +general_settings: + master_key: sk-1234567890 # Change this to a secure key +``` + +### Step 3: Start LiteLLM Proxy + +Start the proxy server with HTTPS enabled: + +```bash +litellm --config config.yaml --port 4000 +``` + +:::warning HTTPS Required + +**Important**: Cursor IDE requires HTTPS connections. HTTP (`http://`) will not work. You must: +- Deploy your LiteLLM Proxy with HTTPS enabled +- Use a valid SSL certificate +- Access the proxy via an HTTPS domain (e.g., `https://your-proxy-domain.com`) + +For local development, you'll need to set up HTTPS (e.g., using a reverse proxy like nginx with SSL, or deploying to a cloud service with HTTPS). + +::: + +### Step 4: Configure Cursor IDE + +Configure Cursor IDE to use your LiteLLM proxy with the `/cursor/chat/completions` endpoint: + +1. Open Cursor IDE +2. Go to **Settings** → **Features** → **AI** +3. Enable **"Use Custom API"** or **"Bring Your Own Key"** +4. Set the following: + - **Base URL**: `https://your-proxy-domain.com/cursor` (⚠️ **Important**: Must use HTTPS and include `/cursor`) + - **API Key**: Your LiteLLM Proxy API key (e.g., `sk-1234567890`) + +:::warning HTTPS Required + +Cursor IDE **requires HTTPS** connections. HTTP (`http://`) will not work. You must: +- Use an HTTPS URL for your base URL (e.g., `https://your-proxy-domain.com/cursor`) +- Ensure your LiteLLM Proxy is accessible via HTTPS +- Have a valid SSL certificate configured + +::: + +**Example Configuration:** + +``` +Base URL: https://your-proxy-domain.com/cursor +API Key: sk-1234567890 +``` + +Replace `your-proxy-domain.com` with your actual HTTPS domain where LiteLLM Proxy is running. + +:::info Why `/cursor` in the base URL? + +Cursor automatically appends `/chat/completions` to the base URL you provide. By setting the base URL to `https://your-proxy-domain.com/cursor`, Cursor will send requests to `/cursor/chat/completions`, which is the special endpoint that handles Cursor's Responses API input format and transforms it to Chat Completions output format. + +If you set the base URL to just `https://your-proxy-domain.com`, Cursor would send requests to `/chat/completions`, which won't work correctly with Cursor's request format. + + +::: + +### Step 5: Test the Integration + +1. Restart Cursor IDE to apply the settings +2. Open a code file and try using Cursor's AI features (completions, chat, etc.) +3. Your requests will now be routed through LiteLLM Proxy + +You can verify it's working by: +- Checking the LiteLLM Proxy logs for incoming requests +- Using Cursor's chat feature and seeing responses stream correctly +- Checking your LiteLLM dashboard for request logs and cost tracking + +## How It Works + +The `/cursor/chat/completions` endpoint is specifically designed to handle Cursor's unique request format: + +1. **Input**: Cursor sends requests in OpenAI Responses API format (with `input` field) +2. **Processing**: LiteLLM processes the request through its internal `/responses` flow +3. **Output**: The response is transformed to OpenAI Chat Completions format (with `choices` field) that Cursor expects + +This transformation happens automatically for both streaming and non-streaming responses. + +## Advanced Configuration + +### Using Different Models + +You can configure Cursor to use different models by updating your `config.yaml`: + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: gpt-4o + litellm_params: + model: gpt-4o + api_key: os.environ/OPENAI_API_KEY + + - model_name: claude-3-5-sonnet + litellm_params: + model: anthropic/claude-3-5-sonnet-20241022 + api_key: os.environ/ANTHROPIC_API_KEY + + - model_name: gemini-pro + litellm_params: + model: gemini/gemini-1.5-pro + api_key: os.environ/GEMINI_API_KEY +``` + +Then in Cursor, you can specify which model to use in your requests. + +### Rate Limiting and Budgets + +Set up rate limits and budgets in your `config.yaml`: + +```yaml showLineNumbers title="config.yaml" +general_settings: + master_key: sk-1234567890 + +litellm_settings: + # Set max budget per user + max_budget: 100.0 + + # Set rate limits + rate_limit: 100 # requests per minute +``` + +### Request Logging + +All requests from Cursor will be logged by LiteLLM Proxy. You can: +- View logs in the LiteLLM Admin UI +- Export logs to your preferred logging service +- Track costs per user/team + +## Troubleshooting + +### Cursor shows no output + +- **Check base URL**: Ensure it uses HTTPS and includes `/cursor` (e.g., `https://your-proxy-domain.com/cursor`, not `http://` or without `/cursor`) +- **Verify HTTPS**: Cursor requires HTTPS - HTTP connections will not work +- **Check API key**: Verify your LiteLLM Proxy API key is correct +- **Check proxy logs**: Look for errors in the LiteLLM Proxy logs + +### Requests failing + +- **Verify HTTPS is enabled**: Cursor requires HTTPS connections. Ensure your LiteLLM Proxy is accessible via HTTPS with a valid SSL certificate +- **Verify proxy is running**: Check that LiteLLM Proxy is accessible at your HTTPS base URL +- **Check SSL certificate**: Ensure your SSL certificate is valid and not expired +- **Check model configuration**: Ensure the model you're trying to use is configured in `config.yaml` +- **Check API keys**: Verify provider API keys are set correctly in environment variables + +### HTTP not working + +If you're trying to use HTTP (`http://`) and it's not working: +- **This is expected**: Cursor IDE requires HTTPS connections +- **Solution**: Deploy your LiteLLM Proxy with HTTPS enabled (use a reverse proxy like nginx, or deploy to a cloud service that provides HTTPS) + +### Streaming not working + +The `/cursor/chat/completions` endpoint automatically handles streaming. If streaming isn't working: +- Check that your model supports streaming +- Verify the proxy logs for any transformation errors +- Ensure Cursor IDE is up to date + +## Related Documentation + +- [Cursor Endpoint Documentation](/docs/proxy/cursor) - Detailed endpoint documentation +- [LiteLLM Proxy Setup](/docs/proxy/quick_start) - General proxy setup guide +- [Model Configuration](/docs/proxy/configs) - How to configure models + diff --git a/docs/my-website/docs/vector_stores/create.md b/docs/my-website/docs/vector_stores/create.md index 19b4f39cd9e..7025c490a32 100644 --- a/docs/my-website/docs/vector_stores/create.md +++ b/docs/my-website/docs/vector_stores/create.md @@ -14,6 +14,7 @@ Create a vector store which can be used to store and search document chunks for | End-user Tracking | ✅ | | | Support LLM Providers (OpenAI `/vector_stores` API) | **OpenAI** | Full vector stores API support across providers | | Support LLM Providers (Passthrough API) | [**Azure AI**](/docs/providers/azure_ai/azure_ai_vector_stores_passthrough) | Full vector stores API support across providers | +| Support LLM Providers (Dataset Management) | [**RAGFlow**](/docs/providers/ragflow_vector_store.md) | Dataset creation and management (search not supported) | ## Usage diff --git a/docs/my-website/img/a2a_gateway.png b/docs/my-website/img/a2a_gateway.png new file mode 100644 index 00000000000..c53a9910d58 Binary files /dev/null and b/docs/my-website/img/a2a_gateway.png differ diff --git a/docs/my-website/img/add_agent1.png b/docs/my-website/img/add_agent1.png new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docs/my-website/img/add_agent_1.png b/docs/my-website/img/add_agent_1.png new file mode 100644 index 00000000000..e60435996a9 Binary files /dev/null and b/docs/my-website/img/add_agent_1.png differ diff --git a/docs/my-website/img/agent2.png b/docs/my-website/img/agent2.png new file mode 100644 index 00000000000..412047a6aa3 Binary files /dev/null and b/docs/my-website/img/agent2.png differ diff --git a/docs/my-website/img/agent_id.png b/docs/my-website/img/agent_id.png new file mode 100644 index 00000000000..d3b11907f25 Binary files /dev/null and b/docs/my-website/img/agent_id.png differ diff --git a/docs/my-website/img/agent_key.png b/docs/my-website/img/agent_key.png new file mode 100644 index 00000000000..7769e0edba9 Binary files /dev/null and b/docs/my-website/img/agent_key.png differ diff --git a/docs/my-website/img/agent_team.png b/docs/my-website/img/agent_team.png new file mode 100644 index 00000000000..0439e772028 Binary files /dev/null and b/docs/my-website/img/agent_team.png differ diff --git a/docs/my-website/img/create_guard_tool_permission.png b/docs/my-website/img/create_guard_tool_permission.png new file mode 100644 index 00000000000..f6e0e77b1aa Binary files /dev/null and b/docs/my-website/img/create_guard_tool_permission.png differ diff --git a/docs/my-website/img/create_rule_tool_permission.png b/docs/my-website/img/create_rule_tool_permission.png new file mode 100644 index 00000000000..2944136e3ed Binary files /dev/null and b/docs/my-website/img/create_rule_tool_permission.png differ diff --git a/docs/my-website/img/customer_usage.png b/docs/my-website/img/customer_usage.png new file mode 100644 index 00000000000..8e601c1f331 Binary files /dev/null and b/docs/my-website/img/customer_usage.png differ diff --git a/docs/my-website/img/customer_usage_analytics.png b/docs/my-website/img/customer_usage_analytics.png new file mode 100644 index 00000000000..443337d3839 Binary files /dev/null and b/docs/my-website/img/customer_usage_analytics.png differ diff --git a/docs/my-website/img/customer_usage_filter.png b/docs/my-website/img/customer_usage_filter.png new file mode 100644 index 00000000000..d544cd9b25b Binary files /dev/null and b/docs/my-website/img/customer_usage_filter.png differ diff --git a/docs/my-website/img/customer_usage_ui_navigation.png b/docs/my-website/img/customer_usage_ui_navigation.png new file mode 100644 index 00000000000..2c92f7303b4 Binary files /dev/null and b/docs/my-website/img/customer_usage_ui_navigation.png differ diff --git a/docs/my-website/package-lock.json b/docs/my-website/package-lock.json index 9c7ee7741cd..a48056491f4 100644 --- a/docs/my-website/package-lock.json +++ b/docs/my-website/package-lock.json @@ -14619,9 +14619,9 @@ } }, "node_modules/mdast-util-to-hast": { - "version": "13.2.0", - "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.0.tgz", - "integrity": "sha512-QGYKEuUsYT9ykKBCMOEDLsU5JRObWQusAolFMeko/tYPufNkRffBAQjIE+99jbA87xv6FgmjLtwjh9wBWajwAA==", + "version": "13.2.1", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.1.tgz", + "integrity": "sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA==", "license": "MIT", "dependencies": { "@types/hast": "^3.0.0", diff --git a/docs/my-website/package.json b/docs/my-website/package.json index f217cef7fcc..e532f7c2cb5 100644 --- a/docs/my-website/package.json +++ b/docs/my-website/package.json @@ -61,6 +61,7 @@ "mermaid": ">=11.10.0", "gray-matter": "4.0.3", "glob": ">=11.1.0", - "node-forge": ">=1.3.2" + "node-forge": ">=1.3.2", + "mdast-util-to-hast": ">=13.2.1" } -} +} \ No newline at end of file diff --git a/docs/my-website/release_notes/v1.80.5-stable/index.md b/docs/my-website/release_notes/v1.80.5-stable/index.md index 4324fdef776..7c3283ce349 100644 --- a/docs/my-website/release_notes/v1.80.5-stable/index.md +++ b/docs/my-website/release_notes/v1.80.5-stable/index.md @@ -1,5 +1,5 @@ --- -title: "[PREVIEW] v1.80.5.rc.2 - Gemini 3.0 Support" +title: "v1.80.5-stable - Gemini 3.0 Support" slug: "v1-80-5" date: 2025-11-22T10:00:00 authors: @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.80.5.rc.2 +ghcr.io/berriai/litellm:v1.80.5-stable ``` diff --git a/docs/my-website/release_notes/v1.80.8-stable/index.md b/docs/my-website/release_notes/v1.80.8-stable/index.md new file mode 100644 index 00000000000..4d94024e0cd --- /dev/null +++ b/docs/my-website/release_notes/v1.80.8-stable/index.md @@ -0,0 +1,607 @@ +--- +title: "[Preview] v1.80.8.rc.1 - Introducing A2A Agent Gateway" +slug: "v1-80-8" +date: 2025-12-06T10:00:00 +authors: + - name: Krrish Dholakia + title: CEO, LiteLLM + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: CTO, LiteLLM + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +hide_table_of_contents: false +--- + +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +## Deploy this version + + + + +``` showLineNumbers title="docker run litellm" +docker run \ +-e STORE_MODEL_IN_DB=True \ +-p 4000:4000 \ +ghcr.io/berriai/litellm:v1.80.8.rc.1 +``` + + + + + +``` showLineNumbers title="pip install litellm" +pip install litellm==1.80.8 +``` + + + + +--- + +## Key Highlights + +- **Agent Gateway (A2A)** - [Invoke agents through the AI Gateway with request/response logging and access controls](../../docs/a2a) +- **Guardrails API v2** - [Generic Guardrail API with streaming support, structured messages, and tool call checks](../../docs/adding_provider/generic_guardrail_api) +- **Customer (End User) Usage UI** - [Track and visualize end-user spend directly in the dashboard](../../docs/proxy/customer_usage) +- **vLLM Batch + Files API** - [Support for batch and files API with vLLM deployments](../../docs/batches) +- **Dynamic Rate Limiting on Teams** - [Enable dynamic rate limits and priority reservation on team-level](../../docs/proxy/team_budgets) +- **Google Cloud Chirp3 HD** - [New text-to-speech provider with Chirp3 HD voices](../../docs/text_to_speech) + +--- + +### Agent Gateway (A2A) + + + +
+ +This release introduces **A2A Agent Gateway** for LiteLLM, allowing you to invoke and manage A2A agents with the same controls you have for LLM APIs. + +As a **LiteLLM Gateway Admin**, you can now do the following: + - **Request/Response Logging** - Every agent invocation is logged to the Logs page with full request and response tracking. + - **Access Control** - Control which Team/Key can access which agents. + +As a developer, you can continue using the A2A SDK, all you need to do is point you `A2AClient` to the LiteLLM proxy URL and your API key. + +**Works with the A2A SDK:** + +```python +from a2a.client import A2AClient + +client = A2AClient( + base_url="http://localhost:4000", # Your LiteLLM proxy + api_key="sk-1234" # LiteLLM API key +) + +response = client.send_message( + agent_id="my-agent", + message="What's the status of my order?" +) +``` + +Get started with Agent Gateway here: [Agent Gateway Documentation](../../docs/a2a) + +--- + +### Customer (End User) Usage UI + + + +Users can now filter usage statistics by customers, providing the same granular filtering capabilities available for teams and organizations. + +**Details:** + +- Filter usage analytics, spend logs, and activity metrics by customer ID +- View customer-level breakdowns alongside existing team and user-level filters +- Consistent filtering experience across all usage and analytics views + +--- + +## New Providers and Endpoints + +### New Providers (5 new providers) + +| Provider | Supported LiteLLM Endpoints | Description | +| -------- | ------------------- | ----------- | +| **[Z.AI (Zhipu AI)](../../docs/providers/zai)** | `/v1/chat/completions`, `/v1/responses`, `/v1/messages` | Built-in support for Zhipu AI GLM models | +| **[RAGFlow](../../docs/providers/ragflow)** | `/v1/chat/completions`, `/v1/responses`, `/v1/messages`, `/v1/vector_stores` | RAG-based chat completions with vector store support | +| **[PublicAI](../../docs/providers/publicai)** | `/v1/chat/completions`, `/v1/responses`, `/v1/messages` | OpenAI-compatible provider via JSON config | +| **[Google Cloud Chirp3 HD](../../docs/text_to_speech)** | `/v1/audio/speech`, `/v1/audio/speech/stream` | Text-to-speech with Google Cloud Chirp3 HD voices | + +### New LLM API Endpoints (2 new endpoints) + +| Endpoint | Method | Description | Documentation | +| -------- | ------ | ----------- | ------------- | +| `/v1/agents/invoke` | POST | Invoke A2A agents through the AI Gateway | [Agent Gateway](../../docs/a2a) | +| `/cursor/chat/completions` | POST | Cursor BYOK endpoint - accepts Responses API input, returns Chat Completions output | [Cursor Integration](../../docs/tutorials/cursor_integration) | + +--- + +## New Models / Updated Models + +#### New Model Support (33 new models) + +| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features | +| -------- | ----- | -------------- | ------------------- | -------------------- | -------- | +| OpenAI | `gpt-5.1-codex-max` | 400K | $1.25 | $10.00 | Reasoning, vision, PDF input, responses API | +| Azure | `azure/gpt-5.1-codex-max` | 400K | $1.25 | $10.00 | Reasoning, vision, PDF input, responses API | +| Anthropic | `claude-opus-4-5` | 200K | $5.00 | $25.00 | Computer use, reasoning, vision | +| Bedrock | `global.anthropic.claude-opus-4-5-20251101-v1:0` | 200K | $5.00 | $25.00 | Computer use, reasoning, vision | +| Bedrock | `amazon.nova-2-lite-v1:0` | 1M | $0.30 | $2.50 | Reasoning, vision, video, PDF input | +| Bedrock | `amazon.titan-image-generator-v2:0` | - | - | $0.008/image | Image generation | +| Fireworks | `fireworks_ai/deepseek-v3p2` | 164K | $1.20 | $1.20 | Function calling, response schema | +| Fireworks | `fireworks_ai/kimi-k2-instruct-0905` | 262K | $0.60 | $2.50 | Function calling, response schema | +| DeepSeek | `deepseek/deepseek-v3.2` | 164K | $0.28 | $0.40 | Reasoning, function calling | +| Mistral | `mistral/mistral-large-3` | 256K | $0.50 | $1.50 | Function calling, vision | +| Azure AI | `azure_ai/mistral-large-3` | 256K | $0.50 | $1.50 | Function calling, vision | +| Moonshot | `moonshot/kimi-k2-0905-preview` | 262K | $0.60 | $2.50 | Function calling, web search | +| Moonshot | `moonshot/kimi-k2-turbo-preview` | 262K | $1.15 | $8.00 | Function calling, web search | +| Moonshot | `moonshot/kimi-k2-thinking-turbo` | 262K | $1.15 | $8.00 | Function calling, web search | +| OpenRouter | `openrouter/deepseek/deepseek-v3.2` | 164K | $0.28 | $0.40 | Reasoning, function calling | +| Databricks | `databricks/databricks-claude-haiku-4-5` | 200K | $1.00 | $5.00 | Reasoning, function calling | +| Databricks | `databricks/databricks-claude-opus-4` | 200K | $15.00 | $75.00 | Reasoning, function calling | +| Databricks | `databricks/databricks-claude-opus-4-1` | 200K | $15.00 | $75.00 | Reasoning, function calling | +| Databricks | `databricks/databricks-claude-opus-4-5` | 200K | $5.00 | $25.00 | Reasoning, function calling | +| Databricks | `databricks/databricks-claude-sonnet-4` | 200K | $3.00 | $15.00 | Reasoning, function calling | +| Databricks | `databricks/databricks-claude-sonnet-4-1` | 200K | $3.00 | $15.00 | Reasoning, function calling | +| Databricks | `databricks/databricks-gemini-2-5-flash` | 1M | $0.30 | $2.50 | Function calling | +| Databricks | `databricks/databricks-gemini-2-5-pro` | 1M | $1.25 | $10.00 | Function calling | +| Databricks | `databricks/databricks-gpt-5` | 400K | $1.25 | $10.00 | Function calling | +| Databricks | `databricks/databricks-gpt-5-1` | 400K | $1.25 | $10.00 | Function calling | +| Databricks | `databricks/databricks-gpt-5-mini` | 400K | $0.25 | $2.00 | Function calling | +| Databricks | `databricks/databricks-gpt-5-nano` | 400K | $0.05 | $0.40 | Function calling | +| Vertex AI | `vertex_ai/chirp` | - | $30.00/1M chars | - | Text-to-speech (Chirp3 HD) | +| Z.AI | `zai/glm-4.6` | 200K | $0.60 | $2.20 | Function calling | +| Z.AI | `zai/glm-4.5` | 128K | $0.60 | $2.20 | Function calling | +| Z.AI | `zai/glm-4.5v` | 128K | $0.60 | $1.80 | Function calling, vision | +| Z.AI | `zai/glm-4.5-flash` | 128K | Free | Free | Function calling | +| Vertex AI | `vertex_ai/bge-large-en-v1.5` | - | - | - | BGE Embeddings | + +#### Features + +- **[OpenAI](../../docs/providers/openai)** + - Add `gpt-5.1-codex-max` model pricing and configuration - [PR #17541](https://github.com/BerriAI/litellm/pull/17541) + - Add xhigh reasoning effort for gpt-5.1-codex-max - [PR #17585](https://github.com/BerriAI/litellm/pull/17585) + - Add clear error message for empty LLM endpoint responses - [PR #17445](https://github.com/BerriAI/litellm/pull/17445) + +- **[Azure OpenAI](../../docs/providers/azure/azure)** + - Allow reasoning_effort='none' for Azure gpt-5.1 models - [PR #17311](https://github.com/BerriAI/litellm/pull/17311) + +- **[Anthropic](../../docs/providers/anthropic)** + - Add `claude-opus-4-5` alias to pricing data - [PR #17313](https://github.com/BerriAI/litellm/pull/17313) + - Parse `` blocks for opus 4.5 - [PR #17534](https://github.com/BerriAI/litellm/pull/17534) + - Update new Anthropic features as reviewed - [PR #17142](https://github.com/BerriAI/litellm/pull/17142) + - Skip empty text blocks in Anthropic system messages - [PR #17442](https://github.com/BerriAI/litellm/pull/17442) + +- **[Bedrock](../../docs/providers/bedrock)** + - Add Nova embedding support - [PR #17253](https://github.com/BerriAI/litellm/pull/17253) + - Add support for Bedrock Qwen 2 imported model - [PR #17461](https://github.com/BerriAI/litellm/pull/17461) + - Bedrock OpenAI model support - [PR #17368](https://github.com/BerriAI/litellm/pull/17368) + - Add support for file content download for Bedrock batches - [PR #17470](https://github.com/BerriAI/litellm/pull/17470) + - Make streaming chunk size configurable in Bedrock API - [PR #17357](https://github.com/BerriAI/litellm/pull/17357) + - Add experimental latest-user filtering for Bedrock - [PR #17282](https://github.com/BerriAI/litellm/pull/17282) + - Handle Cohere v4 embed response dictionary format - [PR #17220](https://github.com/BerriAI/litellm/pull/17220) + - Remove not compatible beta header from Bedrock - [PR #17301](https://github.com/BerriAI/litellm/pull/17301) + - Add model price and details for Global Opus 4.5 Bedrock endpoint - [PR #17380](https://github.com/BerriAI/litellm/pull/17380) + +- **[Gemini (Google AI Studio + Vertex AI)](../../docs/providers/gemini)** + - Add better handling in image generation for Gemini models - [PR #17292](https://github.com/BerriAI/litellm/pull/17292) + - Fix reasoning_content showing duplicate content in streaming responses - [PR #17266](https://github.com/BerriAI/litellm/pull/17266) + - Handle partial JSON chunks after first valid chunk - [PR #17496](https://github.com/BerriAI/litellm/pull/17496) + - Fix Gemini 3 last chunk thinking block - [PR #17403](https://github.com/BerriAI/litellm/pull/17403) + - Fix Gemini image_tokens treated as text tokens in cost calculation - [PR #17554](https://github.com/BerriAI/litellm/pull/17554) + - Make sure that media resolution is only for Gemini 3 model - [PR #17137](https://github.com/BerriAI/litellm/pull/17137) + +- **[Vertex AI](../../docs/providers/vertex)** + - Add Google Cloud Chirp3 HD support on /speech - [PR #17391](https://github.com/BerriAI/litellm/pull/17391) + - Add BGE Embeddings support - [PR #17362](https://github.com/BerriAI/litellm/pull/17362) + - Handle global location for Vertex AI image generation endpoint - [PR #17255](https://github.com/BerriAI/litellm/pull/17255) + - Add Google Private API Endpoint to Vertex AI fields - [PR #17382](https://github.com/BerriAI/litellm/pull/17382) + +- **[Z.AI (Zhipu AI)](../../docs/providers/zai)** + - Add Z.AI as built-in provider - [PR #17307](https://github.com/BerriAI/litellm/pull/17307) + +- **[GitHub Copilot](../../docs/providers/github_copilot)** + - Add Embedding API support - [PR #17278](https://github.com/BerriAI/litellm/pull/17278) + - Preserve encrypted_content in reasoning items for multi-turn conversations - [PR #17130](https://github.com/BerriAI/litellm/pull/17130) + +- **[Databricks](../../docs/providers/databricks)** + - Update Databricks model pricing and add new models - [PR #17277](https://github.com/BerriAI/litellm/pull/17277) + +- **[OVHcloud](../../docs/providers/ovhcloud)** + - Add support of audio transcription for OVHcloud - [PR #17305](https://github.com/BerriAI/litellm/pull/17305) + +- **[Mistral](../../docs/providers/mistral)** + - Add Mistral Large 3 model support - [PR #17547](https://github.com/BerriAI/litellm/pull/17547) + +- **[Moonshot](../../docs/providers/moonshot)** + - Fix missing Moonshot turbo models and fix incorrect pricing - [PR #17432](https://github.com/BerriAI/litellm/pull/17432) + +- **[Together AI](../../docs/providers/togetherai)** + - Add context window exception mapping for Together AI - [PR #17284](https://github.com/BerriAI/litellm/pull/17284) + +- **[WatsonX](../../docs/providers/watsonx/index)** + - Allow passing zen_api_key dynamically - [PR #16655](https://github.com/BerriAI/litellm/pull/16655) + - Fix Watsonx Audio Transcription API - [PR #17326](https://github.com/BerriAI/litellm/pull/17326) + - Fix audio transcriptions, don't force content type in request headers - [PR #17546](https://github.com/BerriAI/litellm/pull/17546) + +- **[Fireworks AI](../../docs/providers/fireworks_ai)** + - Add new model `fireworks_ai/kimi-k2-instruct-0905` - [PR #17328](https://github.com/BerriAI/litellm/pull/17328) + - Add `fireworks/deepseek-v3p2` - [PR #17395](https://github.com/BerriAI/litellm/pull/17395) + +- **[DeepSeek](../../docs/providers/deepseek)** + - Support Deepseek 3.2 with Reasoning - [PR #17384](https://github.com/BerriAI/litellm/pull/17384) + +- **[Nova Lite 2](../../docs/providers/bedrock)** + - Add Nova Lite 2 reasoning support with reasoningConfig - [PR #17371](https://github.com/BerriAI/litellm/pull/17371) + +- **[Ollama](../../docs/providers/ollama)** + - Fix auth not working with ollama.com - [PR #17191](https://github.com/BerriAI/litellm/pull/17191) + +- **[Groq](../../docs/providers/groq)** + - Fix supports_response_schema before using json_tool_call workaround - [PR #17438](https://github.com/BerriAI/litellm/pull/17438) + +- **[vLLM](../../docs/providers/vllm)** + - Fix empty response + vLLM streaming - [PR #17516](https://github.com/BerriAI/litellm/pull/17516) + +- **[Azure AI](../../docs/providers/azure_ai)** + - Migrate Anthropic provider to Azure AI - [PR #17202](https://github.com/BerriAI/litellm/pull/17202) + - Fix GA path for Azure OpenAI realtime models - [PR #17260](https://github.com/BerriAI/litellm/pull/17260) + +- **[Bedrock TwelveLabs](../../docs/providers/bedrock#twelvelabs-pegasus---video-understanding)** + - Add support for TwelveLabs Pegasus video understanding - [PR #17193](https://github.com/BerriAI/litellm/pull/17193) + +### Bug Fixes + +- **[Bedrock](../../docs/providers/bedrock)** + - Fix extra_headers in messages API bedrock invoke - [PR #17271](https://github.com/BerriAI/litellm/pull/17271) + - Fix Bedrock models in model map - [PR #17419](https://github.com/BerriAI/litellm/pull/17419) + - Make Bedrock converse messages respect modify_params as expected - [PR #17427](https://github.com/BerriAI/litellm/pull/17427) + - Fix Anthropic beta headers for Bedrock imported Qwen models - [PR #17467](https://github.com/BerriAI/litellm/pull/17467) + - Preserve usage from JSON response for OpenAI provider in Bedrock - [PR #17589](https://github.com/BerriAI/litellm/pull/17589) + +- **[SambaNova](../../docs/providers/sambanova)** + - Fix acompletion throws error with SambaNova models - [PR #17217](https://github.com/BerriAI/litellm/pull/17217) + +- **General** + - Fix AttributeError when metadata is null in request body - [PR #17306](https://github.com/BerriAI/litellm/pull/17306) + - Fix 500 error for malformed request - [PR #17291](https://github.com/BerriAI/litellm/pull/17291) + - Respect custom LLM provider in header - [PR #17290](https://github.com/BerriAI/litellm/pull/17290) + - Replace deprecated .dict() with .model_dump() in streaming_handler - [PR #17359](https://github.com/BerriAI/litellm/pull/17359) + +--- + +## LLM API Endpoints + +#### Features + +- **[Responses API](../../docs/response_api)** + - Add cost tracking for responses API - [PR #17258](https://github.com/BerriAI/litellm/pull/17258) + - Map output_tokens_details of responses API to completion_tokens_details - [PR #17458](https://github.com/BerriAI/litellm/pull/17458) + - Add image generation support for Responses API - [PR #16586](https://github.com/BerriAI/litellm/pull/16586) + +- **[Batch API](../../docs/batches)** + - Add vLLM batch+files API support - [PR #15823](https://github.com/BerriAI/litellm/pull/15823) + - Fix optional parameter default value - [PR #17434](https://github.com/BerriAI/litellm/pull/17434) + - Add status parameter as optional for FileObject - [PR #17431](https://github.com/BerriAI/litellm/pull/17431) + +- **[Video Generation API](../../docs/videos)** + - Add passthrough cost tracking for Veo - [PR #17296](https://github.com/BerriAI/litellm/pull/17296) + +- **[OCR API](../../docs/ocr)** + - Add missing OCR and aOCR to CallTypes enum - [PR #17435](https://github.com/BerriAI/litellm/pull/17435) + +- **General** + - Support routing to only websearch supported deployments - [PR #17500](https://github.com/BerriAI/litellm/pull/17500) + +#### Bugs + +- **General** + - Fix streaming error validation - [PR #17242](https://github.com/BerriAI/litellm/pull/17242) + - Add length validation for empty tool_calls in delta - [PR #17523](https://github.com/BerriAI/litellm/pull/17523) + +--- + +## Management Endpoints / UI + +#### Features + +- **New Login Page** + - New Login Page UI - [PR #17443](https://github.com/BerriAI/litellm/pull/17443) + - Refactor /login route - [PR #17379](https://github.com/BerriAI/litellm/pull/17379) + - Add auto_redirect_to_sso to UI Config - [PR #17399](https://github.com/BerriAI/litellm/pull/17399) + - Add Auto Redirect to SSO to New Login Page - [PR #17451](https://github.com/BerriAI/litellm/pull/17451) + +- **Customer (End User) Usage** + - Customer (end user) Usage feature - [PR #17498](https://github.com/BerriAI/litellm/pull/17498) + - Customer Usage UI - [PR #17506](https://github.com/BerriAI/litellm/pull/17506) + - Add Info Banner for Customer Usage - [PR #17598](https://github.com/BerriAI/litellm/pull/17598) + +- **Virtual Keys** + - Standardize API Key vs Virtual Key in UI - [PR #17325](https://github.com/BerriAI/litellm/pull/17325) + - Add User Alias Column to Internal User Table - [PR #17321](https://github.com/BerriAI/litellm/pull/17321) + - Delete Credential Enhancements - [PR #17317](https://github.com/BerriAI/litellm/pull/17317) + +- **Models + Endpoints** + - Show all credential values on Edit Credential Modal - [PR #17397](https://github.com/BerriAI/litellm/pull/17397) + - Change Edit Team Models Shown to Match Create Team - [PR #17394](https://github.com/BerriAI/litellm/pull/17394) + - Support Images in Compare UI - [PR #17562](https://github.com/BerriAI/litellm/pull/17562) + +- **Callbacks** + - Show all callbacks on UI - [PR #16335](https://github.com/BerriAI/litellm/pull/16335) + - Credentials to use React Query - [PR #17465](https://github.com/BerriAI/litellm/pull/17465) + +- **Management Routes** + - Allow admin viewer to access global tag usage - [PR #17501](https://github.com/BerriAI/litellm/pull/17501) + - Allow wildcard routes for nonproxy admin (SCIM) - [PR #17178](https://github.com/BerriAI/litellm/pull/17178) + - Return 404 when a user is not found on /user/info - [PR #16850](https://github.com/BerriAI/litellm/pull/16850) + +- **OCI Configuration** + - Enable Oracle Cloud Infrastructure configuration via UI - [PR #17159](https://github.com/BerriAI/litellm/pull/17159) + +#### Bugs + +- **UI Fixes** + - Fix Request and Response Panel JSONViewer - [PR #17233](https://github.com/BerriAI/litellm/pull/17233) + - Adding Button Loading States to Edit Settings - [PR #17236](https://github.com/BerriAI/litellm/pull/17236) + - Fix Various Text, button state, and test changes - [PR #17237](https://github.com/BerriAI/litellm/pull/17237) + - Fix Fallbacks Immediately Deleting before API resolves - [PR #17238](https://github.com/BerriAI/litellm/pull/17238) + - Remove Feature Flags - [PR #17240](https://github.com/BerriAI/litellm/pull/17240) + - Fix metadata tags and model name display in UI for Azure passthrough - [PR #17258](https://github.com/BerriAI/litellm/pull/17258) + - Change labeling around Vertex Fields - [PR #17383](https://github.com/BerriAI/litellm/pull/17383) + - Remove second scrollbar when sidebar is expanded + tooltip z index - [PR #17436](https://github.com/BerriAI/litellm/pull/17436) + - Fix Select in Edit Membership Modal - [PR #17524](https://github.com/BerriAI/litellm/pull/17524) + - Change useAuthorized Hook to redirect to new Login Page - [PR #17553](https://github.com/BerriAI/litellm/pull/17553) + +- **SSO** + - Fix the generic SSO provider - [PR #17227](https://github.com/BerriAI/litellm/pull/17227) + - Clear SSO integration for all users - [PR #17287](https://github.com/BerriAI/litellm/pull/17287) + - Fix SSO users not added to Entra synced team - [PR #17331](https://github.com/BerriAI/litellm/pull/17331) + +- **Auth / JWT** + - JWT Auth - Allow using regular OIDC flow with user info endpoints - [PR #17324](https://github.com/BerriAI/litellm/pull/17324) + - Fix litellm user auth not passing issue - [PR #17342](https://github.com/BerriAI/litellm/pull/17342) + - Add other routes in JWT auth - [PR #17345](https://github.com/BerriAI/litellm/pull/17345) + - Fix new org team validate against org - [PR #17333](https://github.com/BerriAI/litellm/pull/17333) + - Fix litellm_enterprise ensure imported routes exist - [PR #17337](https://github.com/BerriAI/litellm/pull/17337) + - Use organization.members instead of deprecated organization field - [PR #17557](https://github.com/BerriAI/litellm/pull/17557) + +- **Organizations/Teams** + - Fix organization max budget not enforced - [PR #17334](https://github.com/BerriAI/litellm/pull/17334) + - Fix budget update to allow null max_budget - [PR #17545](https://github.com/BerriAI/litellm/pull/17545) + +--- + +## AI Integrations (2 new integrations) + +### Logging (1 new integration) + +#### New Integration + +- **[Weave](../../docs/proxy/logging)** + - Basic Weave OTEL integration - [PR #17439](https://github.com/BerriAI/litellm/pull/17439) + +#### Improvements & Fixes + +- **[DataDog](../../docs/proxy/logging#datadog)** + - Fix Datadog callback regression when ddtrace is installed - [PR #17393](https://github.com/BerriAI/litellm/pull/17393) + +- **[Arize Phoenix](../../docs/observability/arize_integration)** + - Fix clean arize-phoenix traces - [PR #16611](https://github.com/BerriAI/litellm/pull/16611) + +- **[MLflow](../../docs/proxy/logging#mlflow)** + - Fix MLflow streaming spans for Anthropic passthrough - [PR #17288](https://github.com/BerriAI/litellm/pull/17288) + +- **[Langfuse](../../docs/proxy/logging#langfuse)** + - Fix Langfuse logger test mock setup - [PR #17591](https://github.com/BerriAI/litellm/pull/17591) + +- **General** + - Improve PII anonymization handling in logging callbacks - [PR #17207](https://github.com/BerriAI/litellm/pull/17207) + +### Guardrails (1 new integration) + +#### New Integration + +- **[Generic Guardrail API](../../docs/adding_provider/generic_guardrail_api)** + - Generic Guardrail API - allows guardrail providers to add INSTANT support for LiteLLM w/out PR to repo - [PR #17175](https://github.com/BerriAI/litellm/pull/17175) + - Guardrails API V2 - user api key metadata, session id, specify input type (request/response), image support - [PR #17338](https://github.com/BerriAI/litellm/pull/17338) + - Guardrails API - add streaming support - [PR #17400](https://github.com/BerriAI/litellm/pull/17400) + - Guardrails API - support tool call checks on OpenAI `/chat/completions`, OpenAI `/responses`, Anthropic `/v1/messages` - [PR #17459](https://github.com/BerriAI/litellm/pull/17459) + - Guardrails API - new `structured_messages` param - [PR #17518](https://github.com/BerriAI/litellm/pull/17518) + - Correctly map a v1/messages call to the anthropic unified guardrail - [PR #17424](https://github.com/BerriAI/litellm/pull/17424) + - Support during_call event type for unified guardrails - [PR #17514](https://github.com/BerriAI/litellm/pull/17514) + +#### Improvements & Fixes + +- **[Noma Guardrail](../../docs/proxy/guardrails/noma_security)** + - Refactor Noma guardrail to use shared Responses transformation and include system instructions - [PR #17315](https://github.com/BerriAI/litellm/pull/17315) + +- **[Presidio](../../docs/proxy/guardrails/pii_masking_v2)** + - Handle empty content and error dict responses in guardrails - [PR #17489](https://github.com/BerriAI/litellm/pull/17489) + - Fix Presidio guardrail test TypeError and license base64 decoding error - [PR #17538](https://github.com/BerriAI/litellm/pull/17538) + +- **[Tool Permissions](../../docs/proxy/guardrails/tool_permission)** + - Add regex-based tool_name/tool_type matching for tool-permission - [PR #17164](https://github.com/BerriAI/litellm/pull/17164) + - Add images for tool permission guardrail documentation - [PR #17322](https://github.com/BerriAI/litellm/pull/17322) + +- **[AIM Guardrails](../../docs/proxy/guardrails/aim_security)** + - Fix AIM guardrail tests - [PR #17499](https://github.com/BerriAI/litellm/pull/17499) + +- **[Bedrock Guardrails](../../docs/proxy/guardrails/bedrock)** + - Fix Bedrock Guardrail indent and import - [PR #17378](https://github.com/BerriAI/litellm/pull/17378) + +- **General Guardrails** + - Mask all matching keywords in content filter - [PR #17521](https://github.com/BerriAI/litellm/pull/17521) + - Ensure guardrail metadata is preserved in request_data - [PR #17593](https://github.com/BerriAI/litellm/pull/17593) + - Fix apply_guardrail method and improve test isolation - [PR #17555](https://github.com/BerriAI/litellm/pull/17555) + +### Secret Managers + +- **[CyberArk](../../docs/secret_managers/cyberark)** + - Allow setting SSL verify to false - [PR #17433](https://github.com/BerriAI/litellm/pull/17433) + +- **General** + - Make email and secret manager operations independent in key management hooks - [PR #17551](https://github.com/BerriAI/litellm/pull/17551) + +--- + +## Spend Tracking, Budgets and Rate Limiting + +- **Rate Limiting** + - Parallel Request Limiter with /messages - [PR #17426](https://github.com/BerriAI/litellm/pull/17426) + - Allow using dynamic rate limit/priority reservation on teams - [PR #17061](https://github.com/BerriAI/litellm/pull/17061) + - Dynamic Rate Limiter - Fix token count increases/decreases by 1 instead of actual count + Redis TTL - [PR #17558](https://github.com/BerriAI/litellm/pull/17558) + +- **Spend Logs** + - Deprecate `spend/logs` & add `spend/logs/v2` - [PR #17167](https://github.com/BerriAI/litellm/pull/17167) + - Optimize SpendLogs queries to use timestamp filtering for index usage - [PR #17504](https://github.com/BerriAI/litellm/pull/17504) + +- **Enforce User Param** + - Enforce support of enforce_user_param to OpenAI post endpoints - [PR #17407](https://github.com/BerriAI/litellm/pull/17407) + +--- + +## MCP Gateway + +- **MCP Configuration** + - Remove URL format validation for MCP server endpoints - [PR #17270](https://github.com/BerriAI/litellm/pull/17270) + - Add stack trace to MCP error message - [PR #17269](https://github.com/BerriAI/litellm/pull/17269) + +- **MCP Tool Results** + - Preserve tool metadata in CallToolResult - [PR #17561](https://github.com/BerriAI/litellm/pull/17561) + +--- + +## Agent Gateway (A2A) + +- **Agent Invocation** + - Allow invoking agents through AI Gateway - [PR #17440](https://github.com/BerriAI/litellm/pull/17440) + - Allow tracking request/response in "Logs" Page - [PR #17449](https://github.com/BerriAI/litellm/pull/17449) + +- **Agent Access Control** + - Enforce Allowed agents by key, team + add agent access groups on backend - [PR #17502](https://github.com/BerriAI/litellm/pull/17502) + +- **Agent Gateway UI** + - Allow testing agents on UI - [PR #17455](https://github.com/BerriAI/litellm/pull/17455) + - Set allowed agents by key, team - [PR #17511](https://github.com/BerriAI/litellm/pull/17511) + +--- + +## Performance / Loadbalancing / Reliability improvements + +- **Audio/Speech Performance** + - Fix `/audio/speech` performance by using `shared_sessions` - [PR #16739](https://github.com/BerriAI/litellm/pull/16739) + +- **Memory Optimization** + - Prevent memory leak in aiohttp connection pooling - [PR #17388](https://github.com/BerriAI/litellm/pull/17388) + - Lazy-load utils to reduce memory + import time - [PR #17171](https://github.com/BerriAI/litellm/pull/17171) + +- **Database** + - Update default database connection number - [PR #17353](https://github.com/BerriAI/litellm/pull/17353) + - Update default proxy_batch_write_at number - [PR #17355](https://github.com/BerriAI/litellm/pull/17355) + - Add background health checks to db - [PR #17528](https://github.com/BerriAI/litellm/pull/17528) + +- **Proxy Caching** + - Fix proxy caching between requests in aiohttp transport - [PR #17122](https://github.com/BerriAI/litellm/pull/17122) + +- **Session Management** + - Fix session consistency, move Lasso API version away from source code - [PR #17316](https://github.com/BerriAI/litellm/pull/17316) + - Conditionally pass enable_cleanup_closed to aiohttp TCPConnector - [PR #17367](https://github.com/BerriAI/litellm/pull/17367) + +- **Vector Store** + - Fix vector store configuration synchronization failure - [PR #17525](https://github.com/BerriAI/litellm/pull/17525) + +--- + +## Documentation Updates + +- **Provider Documentation** + - Add Azure AI Foundry documentation for Claude models - [PR #17104](https://github.com/BerriAI/litellm/pull/17104) + - Document responses and embedding API for GitHub Copilot - [PR #17456](https://github.com/BerriAI/litellm/pull/17456) + - Add gpt-5.1-codex-max to OpenAI provider documentation - [PR #17602](https://github.com/BerriAI/litellm/pull/17602) + - Update Instructions For Phoenix Integration - [PR #17373](https://github.com/BerriAI/litellm/pull/17373) + +- **Guides** + - Add guide on how to debug gateway error vs provider error - [PR #17387](https://github.com/BerriAI/litellm/pull/17387) + - Agent Gateway documentation - [PR #17454](https://github.com/BerriAI/litellm/pull/17454) + - A2A Permission management documentation - [PR #17515](https://github.com/BerriAI/litellm/pull/17515) + - Update docs to link agent hub - [PR #17462](https://github.com/BerriAI/litellm/pull/17462) + +- **Projects** + - Add Google ADK and Harbor to projects - [PR #17352](https://github.com/BerriAI/litellm/pull/17352) + - Add Microsoft Agent Lightning to projects - [PR #17422](https://github.com/BerriAI/litellm/pull/17422) + +- **Cleanup** + - Cleanup: Remove orphan docs pages and Docusaurus template files - [PR #17356](https://github.com/BerriAI/litellm/pull/17356) + - Remove `source .env` from docs - [PR #17466](https://github.com/BerriAI/litellm/pull/17466) + +--- + +## Infrastructure / CI/CD + +- **Helm Chart** + - Add ingress-only labels - [PR #17348](https://github.com/BerriAI/litellm/pull/17348) + +- **Docker** + - Add retry logic to apk package installation in Dockerfile.non_root - [PR #17596](https://github.com/BerriAI/litellm/pull/17596) + - Chainguard fixes - [PR #17406](https://github.com/BerriAI/litellm/pull/17406) + +- **OpenAPI Schema** + - Refactor add_schema_to_components to move definitions to components/schemas - [PR #17389](https://github.com/BerriAI/litellm/pull/17389) + +- **Security** + - Fix security vulnerability: update mdast-util-to-hast to 13.2.1 - [PR #17601](https://github.com/BerriAI/litellm/pull/17601) + - Bump jws from 3.2.2 to 3.2.3 - [PR #17494](https://github.com/BerriAI/litellm/pull/17494) + +--- + +## New Contributors + +* @weichiet made their first contribution in [PR #17242](https://github.com/BerriAI/litellm/pull/17242) +* @AndyForest made their first contribution in [PR #17220](https://github.com/BerriAI/litellm/pull/17220) +* @omkar806 made their first contribution in [PR #17217](https://github.com/BerriAI/litellm/pull/17217) +* @v0rtex20k made their first contribution in [PR #17178](https://github.com/BerriAI/litellm/pull/17178) +* @hxomer made their first contribution in [PR #17207](https://github.com/BerriAI/litellm/pull/17207) +* @orgersh92 made their first contribution in [PR #17316](https://github.com/BerriAI/litellm/pull/17316) +* @dannykopping made their first contribution in [PR #17313](https://github.com/BerriAI/litellm/pull/17313) +* @rioiart made their first contribution in [PR #17333](https://github.com/BerriAI/litellm/pull/17333) +* @codgician made their first contribution in [PR #17278](https://github.com/BerriAI/litellm/pull/17278) +* @epistoteles made their first contribution in [PR #17277](https://github.com/BerriAI/litellm/pull/17277) +* @kothamah made their first contribution in [PR #17368](https://github.com/BerriAI/litellm/pull/17368) +* @flozonn made their first contribution in [PR #17371](https://github.com/BerriAI/litellm/pull/17371) +* @richardmcsong made their first contribution in [PR #17389](https://github.com/BerriAI/litellm/pull/17389) +* @matt-greathouse made their first contribution in [PR #17384](https://github.com/BerriAI/litellm/pull/17384) +* @mossbanay made their first contribution in [PR #17380](https://github.com/BerriAI/litellm/pull/17380) +* @mhielpos-asapp made their first contribution in [PR #17376](https://github.com/BerriAI/litellm/pull/17376) +* @Joilence made their first contribution in [PR #17367](https://github.com/BerriAI/litellm/pull/17367) +* @deepaktammali made their first contribution in [PR #17357](https://github.com/BerriAI/litellm/pull/17357) +* @axiomofjoy made their first contribution in [PR #16611](https://github.com/BerriAI/litellm/pull/16611) +* @DevajMody made their first contribution in [PR #17445](https://github.com/BerriAI/litellm/pull/17445) +* @andrewtruong made their first contribution in [PR #17439](https://github.com/BerriAI/litellm/pull/17439) +* @AnasAbdelR made their first contribution in [PR #17490](https://github.com/BerriAI/litellm/pull/17490) +* @dominicfeliton made their first contribution in [PR #17516](https://github.com/BerriAI/litellm/pull/17516) +* @kristianmitk made their first contribution in [PR #17504](https://github.com/BerriAI/litellm/pull/17504) +* @rgshr made their first contribution in [PR #17130](https://github.com/BerriAI/litellm/pull/17130) +* @dominicfallows made their first contribution in [PR #17489](https://github.com/BerriAI/litellm/pull/17489) +* @irfansofyana made their first contribution in [PR #17467](https://github.com/BerriAI/litellm/pull/17467) +* @GusBricker made their first contribution in [PR #17191](https://github.com/BerriAI/litellm/pull/17191) +* @OlivverX made their first contribution in [PR #17255](https://github.com/BerriAI/litellm/pull/17255) +* @withsmilo made their first contribution in [PR #17585](https://github.com/BerriAI/litellm/pull/17585) + +--- + +## Full Changelog + +**[View complete changelog on GitHub](https://github.com/BerriAI/litellm/compare/v1.80.7-nightly...v1.80.8)** + diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 802ffdd5bb1..04afb0ba776 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -45,6 +45,7 @@ const sidebars = { type: "category", "label": "Contributing to Guardrails", items: [ + "adding_provider/generic_guardrail_api", "adding_provider/simple_guardrail_tutorial", "adding_provider/adding_guardrail_support", ] @@ -104,6 +105,7 @@ const sidebars = { items: [ "tutorials/claude_responses_api", "tutorials/cost_tracking_coding", + "tutorials/cursor_integration", "tutorials/github_copilot_integration", "tutorials/litellm_gemini_cli", "tutorials/litellm_qwen_code_cli", @@ -128,6 +130,16 @@ const sidebars = { }, items: [ "proxy/docker_quick_start", + { + type: "link", + label: "A2A Agent Gateway", + href: "https://docs.litellm.ai/docs/a2a", + }, + { + type: "link", + label: "MCP Gateway", + href: "https://docs.litellm.ai/docs/mcp", + }, { "type": "category", "label": "Config.yaml", @@ -140,6 +152,7 @@ const sidebars = { "proxy/quick_start", "proxy/cli", "proxy/debugging", + "proxy/error_diagnosis", "proxy/deploy", "proxy/health", "proxy/master_key_rotations", @@ -183,6 +196,7 @@ const sidebars = { label: "Architecture", items: [ "proxy/architecture", + "proxy/multi_tenant_architecture", "proxy/control_plane_and_data_plane", "proxy/db_deadlocks", "proxy/db_info", @@ -221,6 +235,7 @@ const sidebars = { "proxy/team_budgets", "proxy/tag_budgets", "proxy/customers", + "proxy/customer_usage", "proxy/dynamic_rate_limit", "proxy/rate_limit_tiers", "proxy/temporary_budget_increase", @@ -314,6 +329,14 @@ const sidebars = { slug: "/supported_endpoints", }, items: [ + { + type: "category", + label: "/a2a - A2A Agent Gateway", + items: [ + "a2a", + "a2a_agent_permissions", + ], + }, "assistants", { type: "category", @@ -469,6 +492,11 @@ const sidebars = { id: "provider_registration/index", label: "Integrate as a Model Provider", }, + { + type: "doc", + id: "contributing/adding_openai_compatible_providers", + label: "Add OpenAI-Compatible Provider (JSON)", + }, { type: "doc", id: "provider_registration/add_model_pricing", @@ -518,7 +546,9 @@ const sidebars = { "providers/vertex_ai/videos", "providers/vertex_partner", "providers/vertex_self_deployed", + "providers/vertex_embedding", "providers/vertex_image", + "providers/vertex_speech", "providers/vertex_batch", "providers/vertex_ocr", ] @@ -624,6 +654,7 @@ const sidebars = { "providers/petals", "providers/publicai", "providers/predibase", + "providers/ragflow", "providers/recraft", "providers/replicate", { @@ -655,6 +686,7 @@ const sidebars = { }, "providers/xai", "providers/xinference", + "providers/zai", ], }, { @@ -790,6 +822,7 @@ const sidebars = { type: "category", label: "Adding Providers", items: [ + "contributing/adding_openai_compatible_providers", "adding_provider/directory_structure", "adding_provider/new_rerank_provider", ] @@ -816,10 +849,14 @@ const sidebars = { "Learn how to deploy + call models from different providers on LiteLLM", slug: "/project", }, - items: [ + items: [ "projects/smolagents", "projects/mini-swe-agent", "projects/openai-agents", + "projects/Google ADK", + "projects/Agent Lightning", + "projects/Harbor", + "projects/GraphRAG", "projects/Docq.AI", "projects/PDL", "projects/OpenInterpreter", diff --git a/docs/my-website/src/pages/intro.md b/docs/my-website/src/pages/intro.md deleted file mode 100644 index 8a2e69d95f9..00000000000 --- a/docs/my-website/src/pages/intro.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -sidebar_position: 1 ---- - -# Tutorial Intro - -Let's discover **Docusaurus in less than 5 minutes**. - -## Getting Started - -Get started by **creating a new site**. - -Or **try Docusaurus immediately** with **[docusaurus.new](https://docusaurus.new)**. - -### What you'll need - -- [Node.js](https://nodejs.org/en/download/) version 16.14 or above: - - When installing Node.js, you are recommended to check all checkboxes related to dependencies. - -## Generate a new site - -Generate a new Docusaurus site using the **classic template**. - -The classic template will automatically be added to your project after you run the command: - -```bash -npm init docusaurus@latest my-website classic -``` - -You can type this command into Command Prompt, Powershell, Terminal, or any other integrated terminal of your code editor. - -The command also installs all necessary dependencies you need to run Docusaurus. - -## Start your site - -Run the development server: - -```bash -cd my-website -npm run start -``` - -The `cd` command changes the directory you're working with. In order to work with your newly created Docusaurus site, you'll need to navigate the terminal there. - -The `npm run start` command builds your website locally and serves it through a development server, ready for you to view at http://localhost:3000/. - -Open `docs/intro.md` (this page) and edit some lines: the site **reloads automatically** and displays your changes. diff --git a/docs/my-website/src/pages/tutorial-basics/_category_.json b/docs/my-website/src/pages/tutorial-basics/_category_.json deleted file mode 100644 index 2e6db55b1eb..00000000000 --- a/docs/my-website/src/pages/tutorial-basics/_category_.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "label": "Tutorial - Basics", - "position": 2, - "link": { - "type": "generated-index", - "description": "5 minutes to learn the most important Docusaurus concepts." - } -} diff --git a/docs/my-website/src/pages/tutorial-basics/congratulations.md b/docs/my-website/src/pages/tutorial-basics/congratulations.md deleted file mode 100644 index 04771a00b72..00000000000 --- a/docs/my-website/src/pages/tutorial-basics/congratulations.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -sidebar_position: 6 ---- - -# Congratulations! - -You have just learned the **basics of Docusaurus** and made some changes to the **initial template**. - -Docusaurus has **much more to offer**! - -Have **5 more minutes**? Take a look at **[versioning](../tutorial-extras/manage-docs-versions.md)** and **[i18n](../tutorial-extras/translate-your-site.md)**. - -Anything **unclear** or **buggy** in this tutorial? [Please report it!](https://github.com/facebook/docusaurus/discussions/4610) - -## What's next? - -- Read the [official documentation](https://docusaurus.io/) -- Modify your site configuration with [`docusaurus.config.js`](https://docusaurus.io/docs/api/docusaurus-config) -- Add navbar and footer items with [`themeConfig`](https://docusaurus.io/docs/api/themes/configuration) -- Add a custom [Design and Layout](https://docusaurus.io/docs/styling-layout) -- Add a [search bar](https://docusaurus.io/docs/search) -- Find inspirations in the [Docusaurus showcase](https://docusaurus.io/showcase) -- Get involved in the [Docusaurus Community](https://docusaurus.io/community/support) diff --git a/docs/my-website/src/pages/tutorial-basics/create-a-blog-post.md b/docs/my-website/src/pages/tutorial-basics/create-a-blog-post.md deleted file mode 100644 index ea472bbaf87..00000000000 --- a/docs/my-website/src/pages/tutorial-basics/create-a-blog-post.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -sidebar_position: 3 ---- - -# Create a Blog Post - -Docusaurus creates a **page for each blog post**, but also a **blog index page**, a **tag system**, an **RSS** feed... - -## Create your first Post - -Create a file at `blog/2021-02-28-greetings.md`: - -```md title="blog/2021-02-28-greetings.md" ---- -slug: greetings -title: Greetings! -authors: - - name: Joel Marcey - title: Co-creator of Docusaurus 1 - url: https://github.com/JoelMarcey - image_url: https://github.com/JoelMarcey.png - - name: Sébastien Lorber - title: Docusaurus maintainer - url: https://sebastienlorber.com - image_url: https://github.com/slorber.png -tags: [greetings] ---- - -Congratulations, you have made your first post! - -Feel free to play around and edit this post as much you like. -``` - -A new blog post is now available at [http://localhost:3000/blog/greetings](http://localhost:3000/blog/greetings). diff --git a/docs/my-website/src/pages/tutorial-basics/create-a-document.md b/docs/my-website/src/pages/tutorial-basics/create-a-document.md deleted file mode 100644 index ffddfa8eb8a..00000000000 --- a/docs/my-website/src/pages/tutorial-basics/create-a-document.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -sidebar_position: 2 ---- - -# Create a Document - -Documents are **groups of pages** connected through: - -- a **sidebar** -- **previous/next navigation** -- **versioning** - -## Create your first Doc - -Create a Markdown file at `docs/hello.md`: - -```md title="docs/hello.md" -# Hello - -This is my **first Docusaurus document**! -``` - -A new document is now available at [http://localhost:3000/docs/hello](http://localhost:3000/docs/hello). - -## Configure the Sidebar - -Docusaurus automatically **creates a sidebar** from the `docs` folder. - -Add metadata to customize the sidebar label and position: - -```md title="docs/hello.md" {1-4} ---- -sidebar_label: 'Hi!' -sidebar_position: 3 ---- - -# Hello - -This is my **first Docusaurus document**! -``` - -It is also possible to create your sidebar explicitly in `sidebars.js`: - -```js title="sidebars.js" -module.exports = { - tutorialSidebar: [ - 'intro', - // highlight-next-line - 'hello', - { - type: 'category', - label: 'Tutorial', - items: ['tutorial-basics/create-a-document'], - }, - ], -}; -``` diff --git a/docs/my-website/src/pages/tutorial-basics/create-a-page.md b/docs/my-website/src/pages/tutorial-basics/create-a-page.md deleted file mode 100644 index 20e2ac30055..00000000000 --- a/docs/my-website/src/pages/tutorial-basics/create-a-page.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -sidebar_position: 1 ---- - -# Create a Page - -Add **Markdown or React** files to `src/pages` to create a **standalone page**: - -- `src/pages/index.js` → `localhost:3000/` -- `src/pages/foo.md` → `localhost:3000/foo` -- `src/pages/foo/bar.js` → `localhost:3000/foo/bar` - -## Create your first React Page - -Create a file at `src/pages/my-react-page.js`: - -```jsx title="src/pages/my-react-page.js" -import React from 'react'; -import Layout from '@theme/Layout'; - -export default function MyReactPage() { - return ( - -

My React page

-

This is a React page

- - ); -} -``` - -A new page is now available at [http://localhost:3000/my-react-page](http://localhost:3000/my-react-page). - -## Create your first Markdown Page - -Create a file at `src/pages/my-markdown-page.md`: - -```mdx title="src/pages/my-markdown-page.md" -# My Markdown page - -This is a Markdown page -``` - -A new page is now available at [http://localhost:3000/my-markdown-page](http://localhost:3000/my-markdown-page). diff --git a/docs/my-website/src/pages/tutorial-basics/deploy-your-site.md b/docs/my-website/src/pages/tutorial-basics/deploy-your-site.md deleted file mode 100644 index 1c50ee063ef..00000000000 --- a/docs/my-website/src/pages/tutorial-basics/deploy-your-site.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -sidebar_position: 5 ---- - -# Deploy your site - -Docusaurus is a **static-site-generator** (also called **[Jamstack](https://jamstack.org/)**). - -It builds your site as simple **static HTML, JavaScript and CSS files**. - -## Build your site - -Build your site **for production**: - -```bash -npm run build -``` - -The static files are generated in the `build` folder. - -## Deploy your site - -Test your production build locally: - -```bash -npm run serve -``` - -The `build` folder is now served at [http://localhost:3000/](http://localhost:3000/). - -You can now deploy the `build` folder **almost anywhere** easily, **for free** or very small cost (read the **[Deployment Guide](https://docusaurus.io/docs/deployment)**). diff --git a/docs/my-website/src/pages/tutorial-basics/markdown-features.mdx b/docs/my-website/src/pages/tutorial-basics/markdown-features.mdx deleted file mode 100644 index 0337f34d6a5..00000000000 --- a/docs/my-website/src/pages/tutorial-basics/markdown-features.mdx +++ /dev/null @@ -1,150 +0,0 @@ ---- -sidebar_position: 4 ---- - -# Markdown Features - -Docusaurus supports **[Markdown](https://daringfireball.net/projects/markdown/syntax)** and a few **additional features**. - -## Front Matter - -Markdown documents have metadata at the top called [Front Matter](https://jekyllrb.com/docs/front-matter/): - -```text title="my-doc.md" -// highlight-start ---- -id: my-doc-id -title: My document title -description: My document description -slug: /my-custom-url ---- -// highlight-end - -## Markdown heading - -Markdown text with [links](./hello.md) -``` - -## Links - -Regular Markdown links are supported, using url paths or relative file paths. - -```md -Let's see how to [Create a page](/create-a-page). -``` - -```md -Let's see how to [Create a page](./create-a-page.md). -``` - -**Result:** Let's see how to [Create a page](./create-a-page.md). - -## Images - -Regular Markdown images are supported. - -You can use absolute paths to reference images in the static directory (`static/img/docusaurus.png`): - -```md -![Docusaurus logo](/img/docusaurus.png) -``` - -![Docusaurus logo](/img/docusaurus.png) - -You can reference images relative to the current file as well. This is particularly useful to colocate images close to the Markdown files using them: - -```md -![Docusaurus logo](./img/docusaurus.png) -``` - -## Code Blocks - -Markdown code blocks are supported with Syntax highlighting. - - ```jsx title="src/components/HelloDocusaurus.js" - function HelloDocusaurus() { - return ( -

Hello, Docusaurus!

- ) - } - ``` - -```jsx title="src/components/HelloDocusaurus.js" -function HelloDocusaurus() { - return

Hello, Docusaurus!

; -} -``` - -## Admonitions - -Docusaurus has a special syntax to create admonitions and callouts: - - :::tip My tip - - Use this awesome feature option - - ::: - - :::danger Take care - - This action is dangerous - - ::: - -:::tip My tip - -Use this awesome feature option - -::: - -:::danger Take care - -This action is dangerous - -::: - -## MDX and React Components - -[MDX](https://mdxjs.com/) can make your documentation more **interactive** and allows using any **React components inside Markdown**: - -```jsx -export const Highlight = ({children, color}) => ( - { - alert(`You clicked the color ${color} with label ${children}`) - }}> - {children} - -); - -This is Docusaurus green ! - -This is Facebook blue ! -``` - -export const Highlight = ({children, color}) => ( - { - alert(`You clicked the color ${color} with label ${children}`); - }}> - {children} - -); - -This is Docusaurus green ! - -This is Facebook blue ! diff --git a/docs/my-website/src/pages/tutorial-extras/_category_.json b/docs/my-website/src/pages/tutorial-extras/_category_.json deleted file mode 100644 index a8ffcc19300..00000000000 --- a/docs/my-website/src/pages/tutorial-extras/_category_.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "label": "Tutorial - Extras", - "position": 3, - "link": { - "type": "generated-index" - } -} diff --git a/docs/my-website/src/pages/tutorial-extras/img/docsVersionDropdown.png b/docs/my-website/src/pages/tutorial-extras/img/docsVersionDropdown.png deleted file mode 100644 index 97e4164618b..00000000000 Binary files a/docs/my-website/src/pages/tutorial-extras/img/docsVersionDropdown.png and /dev/null differ diff --git a/docs/my-website/src/pages/tutorial-extras/img/localeDropdown.png b/docs/my-website/src/pages/tutorial-extras/img/localeDropdown.png deleted file mode 100644 index e257edc1f93..00000000000 Binary files a/docs/my-website/src/pages/tutorial-extras/img/localeDropdown.png and /dev/null differ diff --git a/docs/my-website/src/pages/tutorial-extras/manage-docs-versions.md b/docs/my-website/src/pages/tutorial-extras/manage-docs-versions.md deleted file mode 100644 index e12c3f3444f..00000000000 --- a/docs/my-website/src/pages/tutorial-extras/manage-docs-versions.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -sidebar_position: 1 ---- - -# Manage Docs Versions - -Docusaurus can manage multiple versions of your docs. - -## Create a docs version - -Release a version 1.0 of your project: - -```bash -npm run docusaurus docs:version 1.0 -``` - -The `docs` folder is copied into `versioned_docs/version-1.0` and `versions.json` is created. - -Your docs now have 2 versions: - -- `1.0` at `http://localhost:3000/docs/` for the version 1.0 docs -- `current` at `http://localhost:3000/docs/next/` for the **upcoming, unreleased docs** - -## Add a Version Dropdown - -To navigate seamlessly across versions, add a version dropdown. - -Modify the `docusaurus.config.js` file: - -```js title="docusaurus.config.js" -module.exports = { - themeConfig: { - navbar: { - items: [ - // highlight-start - { - type: 'docsVersionDropdown', - }, - // highlight-end - ], - }, - }, -}; -``` - -The docs version dropdown appears in your navbar: - -![Docs Version Dropdown](./img/docsVersionDropdown.png) - -## Update an existing version - -It is possible to edit versioned docs in their respective folder: - -- `versioned_docs/version-1.0/hello.md` updates `http://localhost:3000/docs/hello` -- `docs/hello.md` updates `http://localhost:3000/docs/next/hello` diff --git a/docs/my-website/src/pages/tutorial-extras/translate-your-site.md b/docs/my-website/src/pages/tutorial-extras/translate-your-site.md deleted file mode 100644 index caeaffb0554..00000000000 --- a/docs/my-website/src/pages/tutorial-extras/translate-your-site.md +++ /dev/null @@ -1,88 +0,0 @@ ---- -sidebar_position: 2 ---- - -# Translate your site - -Let's translate `docs/intro.md` to French. - -## Configure i18n - -Modify `docusaurus.config.js` to add support for the `fr` locale: - -```js title="docusaurus.config.js" -module.exports = { - i18n: { - defaultLocale: 'en', - locales: ['en', 'fr'], - }, -}; -``` - -## Translate a doc - -Copy the `docs/intro.md` file to the `i18n/fr` folder: - -```bash -mkdir -p i18n/fr/docusaurus-plugin-content-docs/current/ - -cp docs/intro.md i18n/fr/docusaurus-plugin-content-docs/current/intro.md -``` - -Translate `i18n/fr/docusaurus-plugin-content-docs/current/intro.md` in French. - -## Start your localized site - -Start your site on the French locale: - -```bash -npm run start -- --locale fr -``` - -Your localized site is accessible at [http://localhost:3000/fr/](http://localhost:3000/fr/) and the `Getting Started` page is translated. - -:::caution - -In development, you can only use one locale at a same time. - -::: - -## Add a Locale Dropdown - -To navigate seamlessly across languages, add a locale dropdown. - -Modify the `docusaurus.config.js` file: - -```js title="docusaurus.config.js" -module.exports = { - themeConfig: { - navbar: { - items: [ - // highlight-start - { - type: 'localeDropdown', - }, - // highlight-end - ], - }, - }, -}; -``` - -The locale dropdown now appears in your navbar: - -![Locale Dropdown](./img/localeDropdown.png) - -## Build your localized site - -Build your site for a specific locale: - -```bash -npm run build -- --locale fr -``` - -Or build your site to include all the locales at once: - -```bash -npm run build -``` diff --git a/enterprise/dist/litellm_enterprise-0.1.23-py3-none-any.whl b/enterprise/dist/litellm_enterprise-0.1.23-py3-none-any.whl new file mode 100644 index 00000000000..c061e793bc2 Binary files /dev/null and b/enterprise/dist/litellm_enterprise-0.1.23-py3-none-any.whl differ diff --git a/enterprise/dist/litellm_enterprise-0.1.23.tar.gz b/enterprise/dist/litellm_enterprise-0.1.23.tar.gz new file mode 100644 index 00000000000..b84c2ba0f21 Binary files /dev/null and b/enterprise/dist/litellm_enterprise-0.1.23.tar.gz differ diff --git a/enterprise/litellm_enterprise/proxy/enterprise_routes.py b/enterprise/litellm_enterprise/proxy/enterprise_routes.py index f3227892bbd..e28d8b8a4c6 100644 --- a/enterprise/litellm_enterprise/proxy/enterprise_routes.py +++ b/enterprise/litellm_enterprise/proxy/enterprise_routes.py @@ -5,14 +5,10 @@ from litellm_enterprise.enterprise_callbacks.send_emails.endpoints import ( ) from .audit_logging_endpoints import router as audit_logging_router -from .guardrails.endpoints import router as guardrails_router from .management_endpoints import management_endpoints_router from .utils import _should_block_robots -from .vector_stores.endpoints import router as vector_stores_router router = APIRouter() -router.include_router(vector_stores_router) -router.include_router(guardrails_router) router.include_router(email_events_router) router.include_router(audit_logging_router) router.include_router(management_endpoints_router) diff --git a/enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py b/enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py new file mode 100644 index 00000000000..21933165217 --- /dev/null +++ b/enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py @@ -0,0 +1,345 @@ +""" +VECTOR STORE MANAGEMENT + +All /vector_store management endpoints + +/vector_store/new +/vector_store/delete +/vector_store/list +""" + +import copy +import json +from typing import List, Optional + +from fastapi import APIRouter, Depends, HTTPException + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.proxy._types import ( + LiteLLM_ManagedVectorStoresTable, + ResponseLiteLLM_ManagedVectorStore, + UserAPIKeyAuth, +) +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.types.vector_stores import ( + LiteLLM_ManagedVectorStore, + LiteLLM_ManagedVectorStoreListResponse, + VectorStoreDeleteRequest, + VectorStoreInfoRequest, + VectorStoreUpdateRequest, +) +from litellm.vector_stores.vector_store_registry import VectorStoreRegistry + +router = APIRouter() + + +######################################################## +# Management Endpoints +######################################################## +@router.post( + "/vector_store/new", + tags=["vector store management"], + dependencies=[Depends(user_api_key_auth)], +) +async def new_vector_store( + vector_store: LiteLLM_ManagedVectorStore, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Create a new vector store. + + Parameters: + - vector_store_id: str - Unique identifier for the vector store + - custom_llm_provider: str - Provider of the vector store + - vector_store_name: Optional[str] - Name of the vector store + - vector_store_description: Optional[str] - Description of the vector store + - vector_store_metadata: Optional[Dict] - Additional metadata for the vector store + """ + from litellm.proxy.proxy_server import prisma_client + from litellm.types.router import GenericLiteLLMParams + + if prisma_client is None: + raise HTTPException(status_code=500, detail="Database not connected") + + try: + # Check if vector store already exists + existing_vector_store = ( + await prisma_client.db.litellm_managedvectorstorestable.find_unique( + where={"vector_store_id": vector_store.get("vector_store_id")} + ) + ) + if existing_vector_store is not None: + raise HTTPException( + status_code=400, + detail=f"Vector store with ID {vector_store.get('vector_store_id')} already exists", + ) + + if vector_store.get("vector_store_metadata") is not None: + vector_store["vector_store_metadata"] = safe_dumps( + vector_store.get("vector_store_metadata") + ) + + # Safely handle JSON serialization of litellm_params + litellm_params_json: Optional[str] = None + _input_litellm_params: dict = vector_store.get("litellm_params", {}) or {} + if _input_litellm_params is not None: + litellm_params_dict = GenericLiteLLMParams( + **_input_litellm_params + ).model_dump(exclude_none=True) + litellm_params_json = safe_dumps(litellm_params_dict) + del vector_store["litellm_params"] + + _new_vector_store = ( + await prisma_client.db.litellm_managedvectorstorestable.create( + data={ + **vector_store, + "litellm_params": litellm_params_json, + } + ) + ) + + new_vector_store: LiteLLM_ManagedVectorStore = LiteLLM_ManagedVectorStore( + **_new_vector_store.model_dump() + ) + + # Add vector store to registry + if litellm.vector_store_registry is not None: + litellm.vector_store_registry.add_vector_store_to_registry( + vector_store=new_vector_store + ) + + return { + "status": "success", + "message": f"Vector store {vector_store.get('vector_store_id')} created successfully", + "vector_store": new_vector_store, + } + except Exception as e: + verbose_proxy_logger.exception(f"Error creating vector store: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.get( + "/vector_store/list", + tags=["vector store management"], + dependencies=[Depends(user_api_key_auth)], + response_model=LiteLLM_ManagedVectorStoreListResponse, +) +async def list_vector_stores( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + page: int = 1, + page_size: int = 100, +): + """ + List all available vector stores with optional filtering and pagination. + Combines both in-memory vector stores and those stored in the database. + + Parameters: + - page: int - Page number for pagination (default: 1) + - page_size: int - Number of items per page (default: 100) + """ + from litellm.proxy.proxy_server import prisma_client + + try: + # Get vector stores from database (source of truth) + # Only return what's in the database to ensure consistency across instances + vector_stores_from_db = await VectorStoreRegistry._get_vector_stores_from_db( + prisma_client=prisma_client + ) + + # Also clean up in-memory registry to remove any deleted vector stores + if litellm.vector_store_registry is not None: + db_vector_store_ids = { + vs.get("vector_store_id") + for vs in vector_stores_from_db + if vs.get("vector_store_id") + } + # Remove any in-memory vector stores that no longer exist in database + vector_stores_to_remove = [] + for vs in litellm.vector_store_registry.vector_stores: + vs_id = vs.get("vector_store_id") + if vs_id and vs_id not in db_vector_store_ids: + vector_stores_to_remove.append(vs_id) + for vs_id in vector_stores_to_remove: + litellm.vector_store_registry.delete_vector_store_from_registry( + vector_store_id=vs_id + ) + verbose_proxy_logger.debug( + f"Removed deleted vector store {vs_id} from in-memory registry" + ) + + # Use database as single source of truth for listing + combined_vector_stores: List[LiteLLM_ManagedVectorStore] = vector_stores_from_db + + total_count = len(combined_vector_stores) + total_pages = (total_count + page_size - 1) // page_size + + # Format response using LiteLLM_ManagedVectorStoreListResponse + response = LiteLLM_ManagedVectorStoreListResponse( + object="list", + data=combined_vector_stores, + total_count=total_count, + current_page=page, + total_pages=total_pages, + ) + + return response + except Exception as e: + verbose_proxy_logger.exception(f"Error listing vector stores: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post( + "/vector_store/delete", + tags=["vector store management"], + dependencies=[Depends(user_api_key_auth)], +) +async def delete_vector_store( + data: VectorStoreDeleteRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Delete a vector store. + + Parameters: + - vector_store_id: str - ID of the vector store to delete + """ + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException(status_code=500, detail="Database not connected") + + try: + # Check if vector store exists + existing_vector_store = ( + await prisma_client.db.litellm_managedvectorstorestable.find_unique( + where={"vector_store_id": data.vector_store_id} + ) + ) + if existing_vector_store is None: + raise HTTPException( + status_code=404, + detail=f"Vector store with ID {data.vector_store_id} not found", + ) + + # Delete vector store + await prisma_client.db.litellm_managedvectorstorestable.delete( + where={"vector_store_id": data.vector_store_id} + ) + + # Delete vector store from registry + if litellm.vector_store_registry is not None: + litellm.vector_store_registry.delete_vector_store_from_registry( + vector_store_id=data.vector_store_id + ) + + return {"message": f"Vector store {data.vector_store_id} deleted successfully"} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post( + "/vector_store/info", + tags=["vector store management"], + dependencies=[Depends(user_api_key_auth)], + response_model=ResponseLiteLLM_ManagedVectorStore, +) +async def get_vector_store_info( + data: VectorStoreInfoRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """Return a single vector store's details""" + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException(status_code=500, detail="Database not connected") + + try: + if litellm.vector_store_registry is not None: + vector_store = litellm.vector_store_registry.get_litellm_managed_vector_store_from_registry( + vector_store_id=data.vector_store_id + ) + if vector_store is not None: + vector_store_metadata = vector_store.get("vector_store_metadata") + # Parse metadata if it's a JSON string + parsed_metadata: Optional[dict] = None + if isinstance(vector_store_metadata, str): + parsed_metadata = json.loads(vector_store_metadata) + elif isinstance(vector_store_metadata, dict): + parsed_metadata = vector_store_metadata + + vector_store_pydantic_obj = LiteLLM_ManagedVectorStoresTable( + vector_store_id=vector_store.get("vector_store_id") or "", + custom_llm_provider=vector_store.get("custom_llm_provider") or "", + vector_store_name=vector_store.get("vector_store_name") or None, + vector_store_description=vector_store.get( + "vector_store_description" + ) + or None, + vector_store_metadata=parsed_metadata, + created_at=vector_store.get("created_at") or None, + updated_at=vector_store.get("updated_at") or None, + litellm_credential_name=vector_store.get("litellm_credential_name"), + litellm_params=vector_store.get("litellm_params") or None, + ) + return {"vector_store": vector_store_pydantic_obj} + + vector_store = ( + await prisma_client.db.litellm_managedvectorstorestable.find_unique( + where={"vector_store_id": data.vector_store_id} + ) + ) + if vector_store is None: + raise HTTPException( + status_code=404, + detail=f"Vector store with ID {data.vector_store_id} not found", + ) + + vector_store_dict = vector_store.model_dump() # type: ignore[attr-defined] + return {"vector_store": vector_store_dict} + except Exception as e: + verbose_proxy_logger.exception(f"Error getting vector store info: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post( + "/vector_store/update", + tags=["vector store management"], + dependencies=[Depends(user_api_key_auth)], +) +async def update_vector_store( + data: VectorStoreUpdateRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """Update vector store details""" + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException(status_code=500, detail="Database not connected") + + try: + update_data = data.model_dump(exclude_unset=True) + vector_store_id = update_data.pop("vector_store_id") + if update_data.get("vector_store_metadata") is not None: + update_data["vector_store_metadata"] = safe_dumps( + update_data["vector_store_metadata"] + ) + + updated = await prisma_client.db.litellm_managedvectorstorestable.update( + where={"vector_store_id": vector_store_id}, + data=update_data, + ) + + updated_vs = LiteLLM_ManagedVectorStore(**updated.model_dump()) + + if litellm.vector_store_registry is not None: + litellm.vector_store_registry.update_vector_store_in_registry( + vector_store_id=vector_store_id, + updated_data=updated_vs, + ) + + return {"vector_store": updated_vs} + except Exception as e: + verbose_proxy_logger.exception(f"Error updating vector store: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index 2c1fa9945bb..2305a5e635c 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-enterprise" -version = "0.1.22" +version = "0.1.23" description = "Package for LiteLLM Enterprise features" authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.1.22" +version = "0.1.23" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-enterprise==", diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.10-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.10-py3-none-any.whl new file mode 100644 index 00000000000..ce4e805663a Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.10-py3-none-any.whl differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.10.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.10.tar.gz new file mode 100644 index 00000000000..a4e218ee2fa Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.10.tar.gz differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.11-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.11-py3-none-any.whl new file mode 100644 index 00000000000..39f05a5418e Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.11-py3-none-any.whl differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.11.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.11.tar.gz new file mode 100644 index 00000000000..82e6be80ea2 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.11.tar.gz differ diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251204124859_add_end_user_spend_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251204124859_add_end_user_spend_table/migration.sql new file mode 100644 index 00000000000..c4234785c54 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251204124859_add_end_user_spend_table/migration.sql @@ -0,0 +1,42 @@ +-- CreateTable +CREATE TABLE "LiteLLM_DailyEndUserSpend" ( + "id" TEXT NOT NULL, + "end_user_id" TEXT, + "date" TEXT NOT NULL, + "api_key" TEXT NOT NULL, + "model" TEXT, + "model_group" TEXT, + "custom_llm_provider" TEXT, + "mcp_namespaced_tool_name" TEXT, + "prompt_tokens" BIGINT NOT NULL DEFAULT 0, + "completion_tokens" BIGINT NOT NULL DEFAULT 0, + "cache_read_input_tokens" BIGINT NOT NULL DEFAULT 0, + "cache_creation_input_tokens" BIGINT NOT NULL DEFAULT 0, + "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "api_requests" BIGINT NOT NULL DEFAULT 0, + "successful_requests" BIGINT NOT NULL DEFAULT 0, + "failed_requests" BIGINT NOT NULL DEFAULT 0, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "LiteLLM_DailyEndUserSpend_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyEndUserSpend_date_idx" ON "LiteLLM_DailyEndUserSpend"("date"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyEndUserSpend_end_user_id_idx" ON "LiteLLM_DailyEndUserSpend"("end_user_id"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyEndUserSpend_api_key_idx" ON "LiteLLM_DailyEndUserSpend"("api_key"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyEndUserSpend_model_idx" ON "LiteLLM_DailyEndUserSpend"("model"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyEndUserSpend_mcp_namespaced_tool_name_idx" ON "LiteLLM_DailyEndUserSpend"("mcp_namespaced_tool_name"); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_DailyEndUserSpend_end_user_id_date_api_key_model_cu_key" ON "LiteLLM_DailyEndUserSpend"("end_user_id", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name"); + diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251204142718_add_agent_permissions/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251204142718_add_agent_permissions/migration.sql new file mode 100644 index 00000000000..c1b3384a69d --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251204142718_add_agent_permissions/migration.sql @@ -0,0 +1,7 @@ +-- Add agent permission fields to LiteLLM_ObjectPermissionTable +ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN IF NOT EXISTS "agents" TEXT[] DEFAULT ARRAY[]::TEXT[]; +ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN IF NOT EXISTS "agent_access_groups" TEXT[] DEFAULT ARRAY[]::TEXT[]; + +-- Add agent_access_groups field to LiteLLM_AgentsTable +ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN IF NOT EXISTS "agent_access_groups" TEXT[] DEFAULT ARRAY[]::TEXT[]; + diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 2883dfc4b82..40f00437e58 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -61,6 +61,7 @@ model LiteLLM_AgentsTable { agent_name String @unique litellm_params Json? agent_card_params Json + agent_access_groups String[] @default([]) created_at DateTime @default(now()) @map("created_at") created_by String updated_at DateTime @default(now()) @updatedAt @map("updated_at") @@ -172,6 +173,8 @@ model LiteLLM_ObjectPermissionTable { mcp_access_groups String[] @default([]) mcp_tool_permissions Json? // Tool-level permissions for MCP servers. Format: {"server_id": ["tool_name_1", "tool_name_2"]} vector_stores String[] @default([]) + agents String[] @default([]) + agent_access_groups String[] @default([]) teams LiteLLM_TeamTable[] verification_tokens LiteLLM_VerificationToken[] organizations LiteLLM_OrganizationTable[] @@ -462,6 +465,34 @@ model LiteLLM_DailyOrganizationSpend { @@index([mcp_namespaced_tool_name]) } +// Track daily end user (customer) spend metrics per model and key +model LiteLLM_DailyEndUserSpend { + id String @id @default(uuid()) + end_user_id String? + date String + api_key String + model String? + model_group String? + custom_llm_provider String? + mcp_namespaced_tool_name String? + prompt_tokens BigInt @default(0) + completion_tokens BigInt @default(0) + cache_read_input_tokens BigInt @default(0) + cache_creation_input_tokens BigInt @default(0) + spend Float @default(0.0) + api_requests BigInt @default(0) + successful_requests BigInt @default(0) + failed_requests BigInt @default(0) + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + @@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@index([date]) + @@index([end_user_id]) + @@index([api_key]) + @@index([model]) + @@index([mcp_namespaced_tool_name]) +} + // Track daily team spend metrics per model and key model LiteLLM_DailyTeamSpend { id String @id @default(uuid()) diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 6bc576e4f3d..8d29a4220d1 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.4.9" +version = "0.4.11" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.4.9" +version = "0.4.11" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/litellm/__init__.py b/litellm/__init__.py index bbe17c97b7d..a4ade3bccae 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -151,6 +151,7 @@ _custom_logger_compatible_callbacks_literal = Literal[ "mlflow", "langfuse", "langfuse_otel", + "weave_otel", "pagerduty", "humanloop", "gcs_pubsub", @@ -264,6 +265,7 @@ heroku_key: Optional[str] = None cometapi_key: Optional[str] = None ovhcloud_key: Optional[str] = None lemonade_key: Optional[str] = None +amazon_nova_api_key: Optional[str] = None common_cloud_provider_auth_params: dict = { "params": ["project", "region_name", "token"], "providers": ["vertex_ai", "bedrock", "watsonx", "azure", "vertex_ai_beta"], @@ -519,6 +521,7 @@ perplexity_models: Set = set() watsonx_models: Set = set() gemini_models: Set = set() xai_models: Set = set() +zai_models: Set = set() deepseek_models: Set = set() runwayml_models: Set = set() azure_ai_models: Set = set() @@ -570,6 +573,7 @@ ovhcloud_models: Set = set() ovhcloud_embedding_models: Set = set() lemonade_models: Set = set() docker_model_runner_models: Set = set() +amazon_nova_models: Set = set() def is_bedrock_pricing_only_model(key: str) -> bool: @@ -710,6 +714,8 @@ def add_known_models(): text_completion_codestral_models.add(key) elif value.get("litellm_provider") == "xai": xai_models.add(key) + elif value.get("litellm_provider") == "zai": + zai_models.add(key) elif value.get("litellm_provider") == "fal_ai": fal_ai_models.add(key) elif value.get("litellm_provider") == "deepseek": @@ -810,6 +816,8 @@ def add_known_models(): lemonade_models.add(key) elif value.get("litellm_provider") == "docker_model_runner": docker_model_runner_models.add(key) + elif value.get("litellm_provider") == "amazon_nova": + amazon_nova_models.add(key) add_known_models() @@ -871,6 +879,7 @@ model_list = list( | gemini_models | text_completion_codestral_models | xai_models + | zai_models | fal_ai_models | deepseek_models | azure_ai_models @@ -959,6 +968,7 @@ models_by_provider: dict = { "aleph_alpha": aleph_alpha_models, "text-completion-codestral": text_completion_codestral_models, "xai": xai_models, + "zai": zai_models, "fal_ai": fal_ai_models, "deepseek": deepseek_models, "runwayml": runwayml_models, @@ -1009,6 +1019,7 @@ models_by_provider: dict = { "ovhcloud": ovhcloud_models | ovhcloud_embedding_models, "lemonade": lemonade_models, "clarifai": clarifai_models, + "amazon_nova": amazon_nova_models, } # mapping for those models which have larger equivalents @@ -1056,57 +1067,10 @@ from .timeout import timeout from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.core_helpers import remove_index_from_tool_calls from litellm.litellm_core_utils.token_counter import get_modified_max_tokens -from .utils import ( - client, - exception_type, - get_optional_params, - get_response_string, - token_counter, - create_pretrained_tokenizer, - create_tokenizer, - supports_function_calling, - supports_web_search, - supports_url_context, - supports_response_schema, - supports_parallel_function_calling, - supports_vision, - supports_audio_input, - supports_audio_output, - supports_system_messages, - supports_reasoning, - get_litellm_params, - acreate, - get_max_tokens, - get_model_info, - register_prompt_template, - validate_environment, - check_valid_key, - register_model, - encode, - decode, - _calculate_retry_after, - _should_retry, - get_supported_openai_params, - get_api_base, - get_first_chars_messages, - ModelResponse, - ModelResponseStream, - EmbeddingResponse, - ImageResponse, - TranscriptionResponse, - TextCompletionResponse, - get_provider_fields, - ModelResponseListIterator, - get_valid_models, -) - -ALL_LITELLM_RESPONSE_TYPES = [ - ModelResponse, - EmbeddingResponse, - ImageResponse, - TranscriptionResponse, - TextCompletionResponse, -] +# client must be imported immediately as it's used as a decorator at function definition time +from .utils import client +# Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py +# (which imports tiktoken) at import time from .llms.bytez.chat.transformation import BytezChatConfig from .llms.custom_llm import CustomLLM @@ -1125,7 +1089,7 @@ from .llms.openrouter.chat.transformation import OpenrouterConfig from .llms.datarobot.chat.transformation import DataRobotConfig from .llms.anthropic.chat.transformation import AnthropicConfig from .llms.anthropic.common_utils import AnthropicModelInfo -from .llms.azure.anthropic.transformation import AzureAnthropicConfig +from .llms.azure_ai.anthropic.transformation import AzureAnthropicConfig from .llms.groq.stt.transformation import GroqSTTConfig from .llms.anthropic.completion.transformation import AnthropicTextConfig from .llms.triton.completion.transformation import TritonConfig @@ -1210,6 +1174,9 @@ from .llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation import from .llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import ( AmazonInvokeNovaConfig, ) +from .llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation import ( + AmazonQwen2Config, +) from .llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import ( AmazonQwen3Config, ) @@ -1234,6 +1201,9 @@ from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation imp from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import ( AmazonTitanConfig, ) +from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import ( + AmazonTwelveLabsPegasusConfig, +) from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, ) @@ -1257,6 +1227,9 @@ from .llms.bedrock.embed.cohere_transformation import BedrockCohereEmbeddingConf from .llms.bedrock.embed.twelvelabs_marengo_transformation import ( TwelveLabsMarengoEmbeddingConfig, ) +from .llms.bedrock.embed.amazon_nova_transformation import ( + AmazonNovaEmbeddingConfig, +) from .llms.openai.openai import OpenAIConfig, MistralEmbeddingConfig from .llms.openai.image_variations.transformation import OpenAIImageVariationConfig from .llms.deepinfra.chat.transformation import DeepInfraConfig @@ -1337,6 +1310,7 @@ from .llms.friendliai.chat.transformation import FriendliaiChatConfig from .llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig from .llms.xai.chat.transformation import XAIChatConfig from .llms.xai.common_utils import XAIModelInfo +from .llms.zai.chat.transformation import ZAIChatConfig from .llms.aiml.chat.transformation import AIMLChatConfig from .llms.volcengine.chat.transformation import ( VolcEngineChatConfig as VolcEngineConfig, @@ -1371,15 +1345,17 @@ from .llms.github_copilot.chat.transformation import GithubCopilotConfig from .llms.github_copilot.responses.transformation import ( GithubCopilotResponsesAPIConfig, ) +from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig from .llms.nebius.chat.transformation import NebiusConfig from .llms.wandb.chat.transformation import WandbConfig from .llms.dashscope.chat.transformation import DashScopeChatConfig from .llms.moonshot.chat.transformation import MoonshotChatConfig -from .llms.publicai.chat.transformation import PublicAIChatConfig +# PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json) from .llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig from .llms.v0.chat.transformation import V0ChatConfig from .llms.oci.chat.transformation import OCIChatConfig from .llms.morph.chat.transformation import MorphChatConfig +from .llms.ragflow.chat.transformation import RAGFlowConfig from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig @@ -1388,6 +1364,7 @@ from .llms.ovhcloud.embedding.transformation import OVHCloudEmbeddingConfig from .llms.cometapi.embed.transformation import CometAPIEmbeddingConfig from .llms.lemonade.chat.transformation import LemonadeChatConfig from .llms.snowflake.embedding.transformation import SnowflakeEmbeddingConfig +from .llms.amazon_nova.chat.transformation import AmazonNovaChatConfig from .main import * # type: ignore # Skills API @@ -1530,67 +1507,91 @@ def set_global_gitlab_config(config: Dict[str, Any]) -> None: # Lazy loading system for heavy modules to reduce initial import time and memory usage -def _lazy_import_cost_calculator(name: str) -> Any: - """Lazy import for cost_calculator functions.""" - from .cost_calculator import ( - completion_cost as _completion_cost, - cost_per_token as _cost_per_token, - response_cost_calculator as _response_cost_calculator, - ) - - _cost_functions = { - "completion_cost": _completion_cost, - "cost_per_token": _cost_per_token, - "response_cost_calculator": _response_cost_calculator, - } - - func = _cost_functions[name] - globals()[name] = func - return func - - -def _lazy_import_litellm_logging(name: str) -> Any: - """Lazy import for litellm_logging module.""" - try: - from litellm.litellm_core_utils.litellm_logging import ( - Logging as _Logging, - modify_integration as _modify_integration, - ) - - _logging_objects = { - "Logging": _Logging, - "modify_integration": _modify_integration, - } - - obj = _logging_objects[name] - globals()[name] = obj - return obj - except Exception as e: - raise AttributeError( - f"module {__name__!r} has no attribute {name!r}. " - f"Lazy import failed: {e}" - ) from e - - -_LAZY_LOAD_REGISTRY: Dict[str, Callable[[str], Any]] = { - "completion_cost": _lazy_import_cost_calculator, - "cost_per_token": _lazy_import_cost_calculator, - "response_cost_calculator": _lazy_import_cost_calculator, - "Logging": _lazy_import_litellm_logging, - "modify_integration": _lazy_import_litellm_logging, -} - if TYPE_CHECKING: + from litellm.types.utils import ModelInfo as _ModelInfoType + + # Cost calculator functions cost_per_token: Callable[..., Tuple[float, float]] completion_cost: Callable[..., float] response_cost_calculator: Any modify_integration: Any + + # Utils functions - type stubs for truly lazy loaded functions only + # (functions NOT imported via "from .main import *") + get_response_string: Callable[..., str] + supports_function_calling: Callable[..., bool] + supports_web_search: Callable[..., bool] + supports_url_context: Callable[..., bool] + supports_response_schema: Callable[..., bool] + supports_parallel_function_calling: Callable[..., bool] + supports_vision: Callable[..., bool] + supports_audio_input: Callable[..., bool] + supports_audio_output: Callable[..., bool] + supports_system_messages: Callable[..., bool] + supports_reasoning: Callable[..., bool] + acreate: Callable[..., Any] + get_max_tokens: Callable[..., int] + get_model_info: Callable[..., _ModelInfoType] + register_prompt_template: Callable[..., None] + validate_environment: Callable[..., dict] + check_valid_key: Callable[..., bool] + register_model: Callable[..., None] + encode: Callable[..., list] + decode: Callable[..., str] + _calculate_retry_after: Callable[..., float] + _should_retry: Callable[..., bool] + get_supported_openai_params: Callable[..., Optional[list]] + get_api_base: Callable[..., Optional[str]] + get_first_chars_messages: Callable[..., str] + get_provider_fields: Callable[..., List] + get_valid_models: Callable[..., list] + + # Response types - truly lazy loaded only (not in main.py or elsewhere) + ModelResponseListIterator: Type[Any] def __getattr__(name: str) -> Any: """Lazy import handler for cost_calculator and litellm_logging functions.""" - if name in _LAZY_LOAD_REGISTRY: - return _LAZY_LOAD_REGISTRY[name](name) + # Lazy load cost_calculator functions + _cost_calculator_names = ( + "completion_cost", + "cost_per_token", + "response_cost_calculator", + ) + if name in _cost_calculator_names: + from ._lazy_imports import _lazy_import_cost_calculator + return _lazy_import_cost_calculator(name) + + # Lazy load litellm_logging functions + _litellm_logging_names = ( + "Logging", + "modify_integration", + ) + if name in _litellm_logging_names: + from ._lazy_imports import _lazy_import_litellm_logging + return _lazy_import_litellm_logging(name) + + # Lazy load utils functions + _utils_names = ( + "exception_type", "get_optional_params", "get_response_string", "token_counter", + "create_pretrained_tokenizer", "create_tokenizer", "supports_function_calling", + "supports_web_search", "supports_url_context", "supports_response_schema", + "supports_parallel_function_calling", "supports_vision", "supports_audio_input", + "supports_audio_output", "supports_system_messages", "supports_reasoning", + "get_litellm_params", "acreate", "get_max_tokens", "get_model_info", + "register_prompt_template", "validate_environment", "check_valid_key", + "register_model", "encode", "decode", "_calculate_retry_after", "_should_retry", + "get_supported_openai_params", "get_api_base", "get_first_chars_messages", + "ModelResponse", "ModelResponseStream", "EmbeddingResponse", "ImageResponse", + "TranscriptionResponse", "TextCompletionResponse", "get_provider_fields", + "ModelResponseListIterator", "get_valid_models", + ) + if name in _utils_names: + from ._lazy_imports import _lazy_import_utils + return _lazy_import_utils(name) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +# ALL_LITELLM_RESPONSE_TYPES is lazy-loaded via __getattr__ to avoid loading utils at import time diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py new file mode 100644 index 00000000000..91b16864de1 --- /dev/null +++ b/litellm/_lazy_imports.py @@ -0,0 +1,259 @@ +from typing import Any +import sys + +def _get_litellm_globals() -> dict: + """Helper to get the globals dictionary of the litellm module.""" + return sys.modules["litellm"].__dict__ + +# Lazy import for utils module - imports only the requested item by name. +# Note: PLR0915 (too many statements) is suppressed because the many if statements +# are intentional - each attribute is imported individually only when requested, +# ensuring true lazy imports rather than importing the entire utils module. +def _lazy_import_utils(name: str) -> Any: # noqa: PLR0915 + """Lazy import for utils module - imports only the requested item by name.""" + _globals = _get_litellm_globals() + if name == "exception_type": + from .utils import exception_type as _exception_type + _globals["exception_type"] = _exception_type + return _exception_type + + if name == "get_optional_params": + from .utils import get_optional_params as _get_optional_params + _globals["get_optional_params"] = _get_optional_params + return _get_optional_params + + if name == "get_response_string": + from .utils import get_response_string as _get_response_string + _globals["get_response_string"] = _get_response_string + return _get_response_string + + if name == "token_counter": + from .utils import token_counter as _token_counter + _globals["token_counter"] = _token_counter + return _token_counter + + if name == "create_pretrained_tokenizer": + from .utils import create_pretrained_tokenizer as _create_pretrained_tokenizer + _globals["create_pretrained_tokenizer"] = _create_pretrained_tokenizer + return _create_pretrained_tokenizer + + if name == "create_tokenizer": + from .utils import create_tokenizer as _create_tokenizer + _globals["create_tokenizer"] = _create_tokenizer + return _create_tokenizer + + if name == "supports_function_calling": + from .utils import supports_function_calling as _supports_function_calling + _globals["supports_function_calling"] = _supports_function_calling + return _supports_function_calling + + if name == "supports_web_search": + from .utils import supports_web_search as _supports_web_search + _globals["supports_web_search"] = _supports_web_search + return _supports_web_search + + if name == "supports_url_context": + from .utils import supports_url_context as _supports_url_context + _globals["supports_url_context"] = _supports_url_context + return _supports_url_context + + if name == "supports_response_schema": + from .utils import supports_response_schema as _supports_response_schema + _globals["supports_response_schema"] = _supports_response_schema + return _supports_response_schema + + if name == "supports_parallel_function_calling": + from .utils import supports_parallel_function_calling as _supports_parallel_function_calling + _globals["supports_parallel_function_calling"] = _supports_parallel_function_calling + return _supports_parallel_function_calling + + if name == "supports_vision": + from .utils import supports_vision as _supports_vision + _globals["supports_vision"] = _supports_vision + return _supports_vision + + if name == "supports_audio_input": + from .utils import supports_audio_input as _supports_audio_input + _globals["supports_audio_input"] = _supports_audio_input + return _supports_audio_input + + if name == "supports_audio_output": + from .utils import supports_audio_output as _supports_audio_output + _globals["supports_audio_output"] = _supports_audio_output + return _supports_audio_output + + if name == "supports_system_messages": + from .utils import supports_system_messages as _supports_system_messages + _globals["supports_system_messages"] = _supports_system_messages + return _supports_system_messages + + if name == "supports_reasoning": + from .utils import supports_reasoning as _supports_reasoning + _globals["supports_reasoning"] = _supports_reasoning + return _supports_reasoning + + if name == "get_litellm_params": + from .utils import get_litellm_params as _get_litellm_params + _globals["get_litellm_params"] = _get_litellm_params + return _get_litellm_params + + if name == "acreate": + from .utils import acreate as _acreate + _globals["acreate"] = _acreate + return _acreate + + if name == "get_max_tokens": + from .utils import get_max_tokens as _get_max_tokens + _globals["get_max_tokens"] = _get_max_tokens + return _get_max_tokens + + if name == "get_model_info": + from .utils import get_model_info as _get_model_info + _globals["get_model_info"] = _get_model_info + return _get_model_info + + if name == "register_prompt_template": + from .utils import register_prompt_template as _register_prompt_template + _globals["register_prompt_template"] = _register_prompt_template + return _register_prompt_template + + if name == "validate_environment": + from .utils import validate_environment as _validate_environment + _globals["validate_environment"] = _validate_environment + return _validate_environment + + if name == "check_valid_key": + from .utils import check_valid_key as _check_valid_key + _globals["check_valid_key"] = _check_valid_key + return _check_valid_key + + if name == "register_model": + from .utils import register_model as _register_model + _globals["register_model"] = _register_model + return _register_model + + if name == "encode": + from .utils import encode as _encode + _globals["encode"] = _encode + return _encode + + if name == "decode": + from .utils import decode as _decode + _globals["decode"] = _decode + return _decode + + if name == "_calculate_retry_after": + from .utils import _calculate_retry_after as __calculate_retry_after + _globals["_calculate_retry_after"] = __calculate_retry_after + return __calculate_retry_after + + if name == "_should_retry": + from .utils import _should_retry as __should_retry + _globals["_should_retry"] = __should_retry + return __should_retry + + if name == "get_supported_openai_params": + from .utils import get_supported_openai_params as _get_supported_openai_params + _globals["get_supported_openai_params"] = _get_supported_openai_params + return _get_supported_openai_params + + if name == "get_api_base": + from .utils import get_api_base as _get_api_base + _globals["get_api_base"] = _get_api_base + return _get_api_base + + if name == "get_first_chars_messages": + from .utils import get_first_chars_messages as _get_first_chars_messages + _globals["get_first_chars_messages"] = _get_first_chars_messages + return _get_first_chars_messages + + if name == "ModelResponse": + from .utils import ModelResponse as _ModelResponse + _globals["ModelResponse"] = _ModelResponse + return _ModelResponse + + if name == "ModelResponseStream": + from .utils import ModelResponseStream as _ModelResponseStream + _globals["ModelResponseStream"] = _ModelResponseStream + return _ModelResponseStream + + if name == "EmbeddingResponse": + from .utils import EmbeddingResponse as _EmbeddingResponse + _globals["EmbeddingResponse"] = _EmbeddingResponse + return _EmbeddingResponse + + if name == "ImageResponse": + from .utils import ImageResponse as _ImageResponse + _globals["ImageResponse"] = _ImageResponse + return _ImageResponse + + if name == "TranscriptionResponse": + from .utils import TranscriptionResponse as _TranscriptionResponse + _globals["TranscriptionResponse"] = _TranscriptionResponse + return _TranscriptionResponse + + if name == "TextCompletionResponse": + from .utils import TextCompletionResponse as _TextCompletionResponse + _globals["TextCompletionResponse"] = _TextCompletionResponse + return _TextCompletionResponse + + if name == "get_provider_fields": + from .utils import get_provider_fields as _get_provider_fields + _globals["get_provider_fields"] = _get_provider_fields + return _get_provider_fields + + if name == "ModelResponseListIterator": + from .utils import ModelResponseListIterator as _ModelResponseListIterator + _globals["ModelResponseListIterator"] = _ModelResponseListIterator + return _ModelResponseListIterator + + if name == "get_valid_models": + from .utils import get_valid_models as _get_valid_models + _globals["get_valid_models"] = _get_valid_models + return _get_valid_models + + raise AttributeError(f"Utils lazy import: unknown attribute {name!r}") + + +def _lazy_import_cost_calculator(name: str) -> Any: + """Lazy import for cost_calculator functions.""" + _globals = _get_litellm_globals() + from .cost_calculator import ( + completion_cost as _completion_cost, + cost_per_token as _cost_per_token, + response_cost_calculator as _response_cost_calculator, + ) + + _cost_functions = { + "completion_cost": _completion_cost, + "cost_per_token": _cost_per_token, + "response_cost_calculator": _response_cost_calculator, + } + + func = _cost_functions[name] + _globals[name] = func + return func + + +def _lazy_import_litellm_logging(name: str) -> Any: + """Lazy import for litellm_logging module.""" + _globals = _get_litellm_globals() + try: + from litellm.litellm_core_utils.litellm_logging import ( + Logging as _Logging, + modify_integration as _modify_integration, + ) + + _logging_objects = { + "Logging": _Logging, + "modify_integration": _modify_integration, + } + + obj = _logging_objects[name] + _globals[name] = obj + return obj + except Exception as e: + raise AttributeError( + f"module 'litellm' has no attribute {name!r}. " + f"Lazy import failed: {e}" + ) from e \ No newline at end of file diff --git a/litellm/a2a_protocol/__init__.py b/litellm/a2a_protocol/__init__.py new file mode 100644 index 00000000000..d8d349bb98a --- /dev/null +++ b/litellm/a2a_protocol/__init__.py @@ -0,0 +1,59 @@ +""" +LiteLLM A2A - Wrapper for invoking A2A protocol agents. + +This module provides a thin wrapper around the official `a2a` SDK that: +- Handles httpx client creation and agent card resolution +- Adds LiteLLM logging via @client decorator +- Matches the A2A SDK interface (SendMessageRequest, SendMessageResponse, etc.) + +Example usage (standalone functions with @client decorator): + ```python + from litellm.a2a_protocol import asend_message + from a2a.types import SendMessageRequest, MessageSendParams + from uuid import uuid4 + + request = SendMessageRequest( + id=str(uuid4()), + params=MessageSendParams( + message={ + "role": "user", + "parts": [{"kind": "text", "text": "Hello!"}], + "messageId": uuid4().hex, + } + ) + ) + response = await asend_message( + base_url="http://localhost:10001", + request=request, + ) + print(response.model_dump(mode='json', exclude_none=True)) + ``` + +Example usage (class-based): + ```python + from litellm.a2a_protocol import A2AClient + + client = A2AClient(base_url="http://localhost:10001") + response = await client.send_message(request) + ``` +""" + +from litellm.a2a_protocol.client import A2AClient +from litellm.a2a_protocol.main import ( + aget_agent_card, + asend_message, + asend_message_streaming, + create_a2a_client, + send_message, +) +from litellm.types.agents import LiteLLMSendMessageResponse + +__all__ = [ + "A2AClient", + "asend_message", + "send_message", + "asend_message_streaming", + "aget_agent_card", + "create_a2a_client", + "LiteLLMSendMessageResponse", +] diff --git a/litellm/a2a_protocol/client.py b/litellm/a2a_protocol/client.py new file mode 100644 index 00000000000..31f7c3b6a90 --- /dev/null +++ b/litellm/a2a_protocol/client.py @@ -0,0 +1,107 @@ +""" +LiteLLM A2A Client class. + +Provides a class-based interface for A2A agent invocation. +""" + +from typing import TYPE_CHECKING, AsyncIterator, Dict, Optional + +from litellm.types.agents import LiteLLMSendMessageResponse + +if TYPE_CHECKING: + from a2a.client import A2AClient as A2AClientType + from a2a.types import ( + AgentCard, + SendMessageRequest, + SendStreamingMessageRequest, + SendStreamingMessageResponse, + ) + + +class A2AClient: + """ + LiteLLM wrapper for A2A agent invocation. + + Creates the underlying A2A client once on first use and reuses it. + + Example: + ```python + from litellm.a2a_protocol import A2AClient + from a2a.types import SendMessageRequest, MessageSendParams + from uuid import uuid4 + + client = A2AClient(base_url="http://localhost:10001") + + request = SendMessageRequest( + id=str(uuid4()), + params=MessageSendParams( + message={ + "role": "user", + "parts": [{"kind": "text", "text": "Hello!"}], + "messageId": uuid4().hex, + } + ) + ) + response = await client.send_message(request) + ``` + """ + + def __init__( + self, + base_url: str, + timeout: float = 60.0, + extra_headers: Optional[Dict[str, str]] = None, + ): + """ + Initialize the A2A client wrapper. + + Args: + base_url: The base URL of the A2A agent (e.g., "http://localhost:10001") + timeout: Request timeout in seconds (default: 60.0) + extra_headers: Optional additional headers to include in requests + """ + self.base_url = base_url + self.timeout = timeout + self.extra_headers = extra_headers + self._a2a_client: Optional["A2AClientType"] = None + + async def _get_client(self) -> "A2AClientType": + """Get or create the underlying A2A client.""" + if self._a2a_client is None: + from litellm.a2a_protocol.main import create_a2a_client + + self._a2a_client = await create_a2a_client( + base_url=self.base_url, + timeout=self.timeout, + extra_headers=self.extra_headers, + ) + return self._a2a_client + + async def get_agent_card(self) -> "AgentCard": + """Fetch the agent card from the server.""" + from litellm.a2a_protocol.main import aget_agent_card + + return await aget_agent_card( + base_url=self.base_url, + timeout=self.timeout, + extra_headers=self.extra_headers, + ) + + async def send_message( + self, request: "SendMessageRequest" + ) -> LiteLLMSendMessageResponse: + """Send a message to the A2A agent.""" + from litellm.a2a_protocol.main import asend_message + + a2a_client = await self._get_client() + return await asend_message(a2a_client=a2a_client, request=request) + + async def send_message_streaming( + self, request: "SendStreamingMessageRequest" + ) -> AsyncIterator["SendStreamingMessageResponse"]: + """Send a streaming message to the A2A agent.""" + from litellm.a2a_protocol.main import asend_message_streaming + + a2a_client = await self._get_client() + async for chunk in asend_message_streaming(a2a_client=a2a_client, request=request): + yield chunk diff --git a/litellm/a2a_protocol/cost_calculator.py b/litellm/a2a_protocol/cost_calculator.py new file mode 100644 index 00000000000..2d821bb9c3b --- /dev/null +++ b/litellm/a2a_protocol/cost_calculator.py @@ -0,0 +1,36 @@ +""" +Cost calculator for A2A (Agent-to-Agent) calls. +""" + +from typing import TYPE_CHECKING, Any, Optional + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import ( + Logging as LitellmLoggingObject, + ) +else: + LitellmLoggingObject = Any + + +class A2ACostCalculator: + @staticmethod + def calculate_a2a_cost( + litellm_logging_obj: Optional[LitellmLoggingObject], + ) -> float: + """ + Calculate the cost of an A2A send_message call. + + Default is 0.0. In the future, users can configure cost per agent call. + """ + if litellm_logging_obj is None: + return 0.0 + + # Check if user set a custom response cost + response_cost = litellm_logging_obj.model_call_details.get( + "response_cost", None + ) + if response_cost is not None: + return response_cost + + # Default to 0.0 for A2A calls + return 0.0 diff --git a/litellm/a2a_protocol/main.py b/litellm/a2a_protocol/main.py new file mode 100644 index 00000000000..5cb238904f0 --- /dev/null +++ b/litellm/a2a_protocol/main.py @@ -0,0 +1,298 @@ +""" +LiteLLM A2A SDK functions. + +Provides standalone functions with @client decorator for LiteLLM logging integration. +""" + +import asyncio +from typing import TYPE_CHECKING, Any, AsyncIterator, Coroutine, Dict, Optional, Union + +from litellm._logging import verbose_logger +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, +) +from litellm.types.agents import LiteLLMSendMessageResponse +from litellm.utils import client + +if TYPE_CHECKING: + from a2a.client import A2AClient as A2AClientType + from a2a.types import ( + AgentCard, + SendMessageRequest, + SendStreamingMessageRequest, + SendStreamingMessageResponse, + ) + +# Runtime imports with availability check +A2A_SDK_AVAILABLE = False +A2ACardResolver: Any = None +_A2AClient: Any = None + +try: + from a2a.client import A2ACardResolver # type: ignore[no-redef] + from a2a.client import A2AClient as _A2AClient # type: ignore[no-redef] + + A2A_SDK_AVAILABLE = True +except ImportError: + pass + + +def _get_a2a_model_info(a2a_client: Any, kwargs: Dict[str, Any]) -> str: + """ + Extract agent info and set model/custom_llm_provider for cost tracking. + + Sets model info on the litellm_logging_obj if available. + Returns the agent name for logging. + """ + agent_name = "unknown" + + # Try to get agent card from our stored attribute first, then fallback to SDK attribute + agent_card = getattr(a2a_client, "_litellm_agent_card", None) + if agent_card is None: + agent_card = getattr(a2a_client, "agent_card", None) + + if agent_card is not None: + agent_name = getattr(agent_card, "name", "unknown") or "unknown" + + # Build model string + model = f"a2a_agent/{agent_name}" + custom_llm_provider = "a2a_agent" + + # Set on litellm_logging_obj if available (for standard logging payload) + litellm_logging_obj = kwargs.get("litellm_logging_obj") + if litellm_logging_obj is not None: + litellm_logging_obj.model = model + litellm_logging_obj.custom_llm_provider = custom_llm_provider + litellm_logging_obj.model_call_details["model"] = model + litellm_logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider + + return agent_name + + +@client +async def asend_message( + a2a_client: "A2AClientType", + request: "SendMessageRequest", + **kwargs: Any, +) -> LiteLLMSendMessageResponse: + """ + Async: Send a message to an A2A agent. + + Uses the @client decorator for LiteLLM logging and tracking. + + Args: + a2a_client: An initialized a2a.client.A2AClient instance + request: SendMessageRequest from a2a.types + **kwargs: Additional arguments passed to the client decorator + + Returns: + LiteLLMSendMessageResponse (wraps a2a SendMessageResponse with _hidden_params) + + Example: + ```python + from litellm.a2a_protocol import asend_message, create_a2a_client + from a2a.types import SendMessageRequest, MessageSendParams + from uuid import uuid4 + + # Create client once + a2a_client = await create_a2a_client(base_url="http://localhost:10001") + + # Use it for multiple requests + request = SendMessageRequest( + id=str(uuid4()), + params=MessageSendParams( + message={ + "role": "user", + "parts": [{"kind": "text", "text": "Hello!"}], + "messageId": uuid4().hex, + } + ) + ) + response = await asend_message(a2a_client=a2a_client, request=request) + ``` + """ + agent_name = _get_a2a_model_info(a2a_client, kwargs) + + verbose_logger.info(f"A2A send_message request_id={request.id}, agent={agent_name}") + + a2a_response = await a2a_client.send_message(request) + + verbose_logger.info(f"A2A send_message completed, request_id={request.id}") + + # Wrap in LiteLLM response type for _hidden_params support + response = LiteLLMSendMessageResponse.from_a2a_response(a2a_response) + + return response + + +@client +def send_message( + a2a_client: "A2AClientType", + request: "SendMessageRequest", + **kwargs: Any, +) -> Union[LiteLLMSendMessageResponse, Coroutine[Any, Any, LiteLLMSendMessageResponse]]: + """ + Sync: Send a message to an A2A agent. + + Uses the @client decorator for LiteLLM logging and tracking. + + Args: + a2a_client: An initialized a2a.client.A2AClient instance + request: SendMessageRequest from a2a.types + **kwargs: Additional arguments passed to the client decorator + + Returns: + LiteLLMSendMessageResponse (wraps a2a SendMessageResponse with _hidden_params) + """ + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = None + + if loop is not None: + return asend_message(a2a_client=a2a_client, request=request, **kwargs) + else: + return asyncio.run(asend_message(a2a_client=a2a_client, request=request, **kwargs)) + + +async def asend_message_streaming( + a2a_client: "A2AClientType", + request: "SendStreamingMessageRequest", +) -> AsyncIterator["SendStreamingMessageResponse"]: + """ + Async: Send a streaming message to an A2A agent. + + Args: + a2a_client: An initialized a2a.client.A2AClient instance + request: SendStreamingMessageRequest from a2a.types + + Yields: + SendStreamingMessageResponse chunks from the agent + """ + verbose_logger.info(f"A2A send_message_streaming request_id={request.id}") + + stream = a2a_client.send_message_streaming(request) + + chunk_count = 0 + async for chunk in stream: + chunk_count += 1 + yield chunk + + verbose_logger.info( + f"A2A send_message_streaming completed, request_id={request.id}, chunks={chunk_count}" + ) + + +async def create_a2a_client( + base_url: str, + timeout: float = 60.0, + extra_headers: Optional[Dict[str, str]] = None, +) -> "A2AClientType": + """ + Create an A2A client for the given agent URL. + + This resolves the agent card and returns a ready-to-use A2A client. + The client can be reused for multiple requests. + + Args: + base_url: The base URL of the A2A agent (e.g., "http://localhost:10001") + timeout: Request timeout in seconds (default: 60.0) + extra_headers: Optional additional headers to include in requests + + Returns: + An initialized a2a.client.A2AClient instance + + Example: + ```python + from litellm.a2a_protocol import create_a2a_client, asend_message + + # Create client once + client = await create_a2a_client(base_url="http://localhost:10001") + + # Reuse for multiple requests + response1 = await asend_message(a2a_client=client, request=request1) + response2 = await asend_message(a2a_client=client, request=request2) + ``` + """ + if not A2A_SDK_AVAILABLE: + raise ImportError( + "The 'a2a' package is required for A2A agent invocation. " + "Install it with: pip install a2a" + ) + + verbose_logger.info(f"Creating A2A client for {base_url}") + + # Use LiteLLM's cached httpx client + http_handler = get_async_httpx_client( + llm_provider=httpxSpecialProvider.A2A, + params={"timeout": timeout}, + ) + httpx_client = http_handler.client + + # Resolve agent card + resolver = A2ACardResolver( + httpx_client=httpx_client, + base_url=base_url, + ) + agent_card = await resolver.get_agent_card() + + verbose_logger.debug( + f"Resolved agent card: {agent_card.name if hasattr(agent_card, 'name') else 'unknown'}" + ) + + # Create A2A client + a2a_client = _A2AClient( + httpx_client=httpx_client, + agent_card=agent_card, + ) + + # Store agent_card on client for later retrieval (SDK doesn't expose it) + a2a_client._litellm_agent_card = agent_card # type: ignore[attr-defined] + + verbose_logger.info(f"A2A client created for {base_url}") + + return a2a_client + + +async def aget_agent_card( + base_url: str, + timeout: float = 60.0, + extra_headers: Optional[Dict[str, str]] = None, +) -> "AgentCard": + """ + Fetch the agent card from an A2A agent. + + Args: + base_url: The base URL of the A2A agent (e.g., "http://localhost:10001") + timeout: Request timeout in seconds (default: 60.0) + extra_headers: Optional additional headers to include in requests + + Returns: + AgentCard from the A2A agent + """ + if not A2A_SDK_AVAILABLE: + raise ImportError( + "The 'a2a' package is required for A2A agent invocation. " + "Install it with: pip install a2a" + ) + + verbose_logger.info(f"Fetching agent card from {base_url}") + + # Use LiteLLM's cached httpx client + http_handler = get_async_httpx_client( + llm_provider=httpxSpecialProvider.A2A, + params={"timeout": timeout}, + ) + httpx_client = http_handler.client + + resolver = A2ACardResolver( + httpx_client=httpx_client, + base_url=base_url, + ) + agent_card = await resolver.get_agent_card() + + verbose_logger.info( + f"Fetched agent card: {agent_card.name if hasattr(agent_card, 'name') else 'unknown'}" + ) + return agent_card diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index 8289801ee30..42ff534c289 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -14,7 +14,7 @@ from litellm.utils import token_counter async def calculate_batch_cost_and_usage( file_content_dictionary: List[dict], - custom_llm_provider: Literal["openai", "azure", "vertex_ai"], + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"], model_name: Optional[str] = None, ) -> Tuple[float, Usage, List[str]]: """ @@ -37,7 +37,7 @@ async def calculate_batch_cost_and_usage( async def _handle_completed_batch( batch: Batch, - custom_llm_provider: Literal["openai", "azure", "vertex_ai"], + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"], model_name: Optional[str] = None, ) -> Tuple[float, Usage, List[str]]: """Helper function to process a completed batch and handle logging""" @@ -84,7 +84,7 @@ def _get_batch_models_from_file_content( def _batch_cost_calculator( file_content_dictionary: List[dict], - custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai", model_name: Optional[str] = None, ) -> float: """ @@ -186,7 +186,7 @@ def calculate_vertex_ai_batch_cost_and_usage( async def _get_batch_output_file_content_as_dictionary( batch: Batch, - custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai", ) -> List[dict]: """ Get the batch output file content as a list of dictionaries @@ -225,7 +225,7 @@ def _get_file_content_as_dictionary(file_content: bytes) -> List[dict]: def _get_batch_job_cost_from_file_content( file_content_dictionary: List[dict], - custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai", ) -> float: """ Get the cost of a batch job from the file content @@ -253,7 +253,7 @@ def _get_batch_job_cost_from_file_content( def _get_batch_job_total_usage_from_file_content( file_content_dictionary: List[dict], - custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai", model_name: Optional[str] = None, ) -> Usage: """ diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 995c45b925a..b99f4a628dc 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -23,6 +23,7 @@ import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.azure.batches.handler import AzureBatchesAPI +from litellm.llms.bedrock.batches.handler import BedrockBatchesHandler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.openai.openai import OpenAIBatchesAPI @@ -35,7 +36,11 @@ from litellm.types.llms.openai import ( RetrieveBatchRequest, ) from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import LiteLLMBatch, LlmProviders +from litellm.types.utils import ( + OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS, + LiteLLMBatch, + LlmProviders, +) from litellm.utils import ( ProviderConfigManager, client, @@ -100,7 +105,7 @@ async def acreate_batch( completion_window: Literal["24h"], endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"], input_file_id: str, - custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai", + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai", metadata: Optional[Dict[str, str]] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, @@ -148,7 +153,7 @@ def create_batch( completion_window: Literal["24h"], endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"], input_file_id: str, - custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai", + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai", metadata: Optional[Dict[str, str]] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, @@ -235,7 +240,7 @@ def create_batch( ) return response api_base: Optional[str] = None - if custom_llm_provider == "openai": + if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS: # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there api_base = ( optional_params.api_base @@ -350,7 +355,7 @@ def create_batch( @client async def aretrieve_batch( batch_id: str, - custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai", + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai", metadata: Optional[Dict[str, str]] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, @@ -396,10 +401,10 @@ def _handle_retrieve_batch_providers_without_provider_config( litellm_params: dict, _retrieve_batch_request: RetrieveBatchRequest, _is_async: bool, - custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai", + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai", ): api_base: Optional[str] = None - if custom_llm_provider == "openai": + if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS: # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there api_base = ( optional_params.api_base @@ -512,7 +517,7 @@ def _handle_retrieve_batch_providers_without_provider_config( @client def retrieve_batch( batch_id: str, - custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai", + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai", metadata: Optional[Dict[str, str]] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, @@ -576,7 +581,7 @@ def retrieve_batch( async_kwargs = kwargs.copy() async_kwargs.pop("aws_region_name", None) - return _handle_async_invoke_status( + return BedrockBatchesHandler._handle_async_invoke_status( batch_id=batch_id, aws_region_name=kwargs.get("aws_region_name", "us-east-1"), logging_obj=litellm_logging_obj, @@ -644,7 +649,7 @@ def retrieve_batch( async def alist_batches( after: Optional[str] = None, limit: Optional[int] = None, - custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", + custom_llm_provider: Literal["openai", "azure", "hosted_vllm", "vertex_ai"] = "openai", metadata: Optional[Dict[str, str]] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, @@ -687,7 +692,7 @@ async def alist_batches( def list_batches( after: Optional[str] = None, limit: Optional[int] = None, - custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", + custom_llm_provider: Literal["openai", "azure", "hosted_vllm", "vertex_ai"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -727,7 +732,7 @@ def list_batches( timeout = 600.0 _is_async = kwargs.pop("alist_batches", False) is True - if custom_llm_provider == "openai": + if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS: # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there api_base = ( optional_params.api_base @@ -928,7 +933,7 @@ def cancel_batch( _is_async = kwargs.pop("acancel_batch", False) is True api_base: Optional[str] = None - if custom_llm_provider == "openai": + if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS: api_base = ( optional_params.api_base or litellm.api_base @@ -1043,30 +1048,49 @@ def _handle_async_invoke_status( ) # Transform response to a LiteLLMBatch object + from litellm.types.llms.openai import BatchJobStatus from litellm.types.utils import LiteLLMBatch + # Normalize status to lowercase (AWS returns 'Completed', 'Failed', etc.) + aws_status_raw = status_response.get("status", "") + aws_status_lower = aws_status_raw.lower() + # Map AWS status values to LiteLLM expected values + status_mapping: dict[str, BatchJobStatus] = { + "completed": "completed", + "failed": "failed", + "inprogress": "in_progress", + "in_progress": "in_progress", + } + normalized_status: BatchJobStatus = status_mapping.get(aws_status_lower, "failed") # Default to "failed" if unknown status + + # Get output S3 URI safely + output_s3_uri = "" + try: + output_s3_uri = status_response["outputDataConfig"]["s3OutputDataConfig"]["s3Uri"] + except (KeyError, TypeError): + pass + + # Use BedrockBatchesConfig's timestamp parsing method (expects raw AWS status string) + import time + + from litellm.llms.bedrock.batches.transformation import BedrockBatchesConfig + created_at, in_progress_at, completed_at, failed_at, _, _ = BedrockBatchesConfig()._parse_timestamps_and_status(status_response, aws_status_raw) result = LiteLLMBatch( id=status_response["invocationArn"], object="batch", - status=status_response["status"], - created_at=status_response["submitTime"], - in_progress_at=status_response["lastModifiedTime"], - completed_at=status_response.get("endTime"), - failed_at=( - status_response.get("endTime") - if status_response["status"] == "failed" - else None - ), + status=normalized_status, + created_at=created_at or int(time.time()), # Provide default timestamp if None + in_progress_at=in_progress_at, + completed_at=completed_at, + failed_at=failed_at, request_counts=BatchRequestCounts( total=1, - completed=1 if status_response["status"] == "completed" else 0, - failed=1 if status_response["status"] == "failed" else 0, + completed=1 if normalized_status == "completed" else 0, + failed=1 if normalized_status == "failed" else 0, ), metadata=dict( **{ - "output_file_id": status_response["outputDataConfig"][ - "s3OutputDataConfig" - ]["s3Uri"], + "output_file_id": output_s3_uri, "failure_message": status_response.get("failureMessage") or "", "model_arn": status_response["modelArn"], } diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 07d9de5a016..8ee730defb9 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -148,7 +148,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if role == "system": # Extract system message as instructions if isinstance(content, str): - instructions = content + if instructions: + # Concatenate multiple system prompts with a space + instructions = f"{instructions} {content}" + else: + instructions = content else: input_items.append( { @@ -363,49 +367,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): reasoning_content = None # flush reasoning content index += 1 elif isinstance(item, ResponseFunctionToolCall): - - provider_specific_fields = getattr( - item, "provider_specific_fields", None + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, ) - if provider_specific_fields and not isinstance( - provider_specific_fields, dict - ): - provider_specific_fields = ( - dict(provider_specific_fields) - if hasattr(provider_specific_fields, "__dict__") - else {} - ) - elif hasattr(item, "get") and callable(item.get): # type: ignore - provider_fields = item.get("provider_specific_fields") # type: ignore - if provider_fields: - provider_specific_fields = ( - provider_fields - if isinstance(provider_fields, dict) - else ( - dict(provider_fields) # type: ignore - if hasattr(provider_fields, "__dict__") - else {} - ) - ) - function_dict: Dict[str, Any] = { - "name": item.name, - "arguments": item.arguments, - } - - if provider_specific_fields: - function_dict["provider_specific_fields"] = provider_specific_fields - - tool_call_dict: Dict[str, Any] = { - "id": item.call_id, - "function": function_dict, - "type": "function", - } - - if provider_specific_fields: - tool_call_dict["provider_specific_fields"] = ( - provider_specific_fields - ) + tool_call_dict = LiteLLMCompletionResponsesConfig.convert_response_function_tool_call_to_chat_completion_tool_call( + tool_call_item=item, + index=index, + ) msg = Message( content=None, @@ -663,6 +632,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return Reasoning(effort="none") # type: ignore elif reasoning_effort == "high": return Reasoning(effort="high") + elif reasoning_effort == "xhigh": + return Reasoning(effort="xhigh") # type: ignore[typeddict-item] elif reasoning_effort == "medium": return Reasoning(effort="medium") elif reasoning_effort == "low": @@ -714,17 +685,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): } } elif format_type == "json_object": - return { - "format": { - "type": "json_object" - } - } + return {"format": {"type": "json_object"}} elif format_type == "text": - return { - "format": { - "type": "text" - } - } + return {"format": {"type": "text"}} return None diff --git a/litellm/constants.py b/litellm/constants.py index f1126b19090..1c0f800eccd 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1,4 +1,5 @@ import os +import sys from typing import List, Literal DEFAULT_HEALTH_CHECK_PROMPT = str( @@ -99,10 +100,18 @@ RUNWAYML_POLLING_TIMEOUT = int( ########## Networking constants ############################################################## _DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600 # 1 hour, re-use the same httpx client for 1 hour -# Aiohttp connection pooling constants -AIOHTTP_CONNECTOR_LIMIT = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 0)) +# Aiohttp connection pooling - prevents memory leaks from unbounded connection growth +# Set to 0 for unlimited (not recommended for production) +AIOHTTP_CONNECTOR_LIMIT = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 300)) +AIOHTTP_CONNECTOR_LIMIT_PER_HOST = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 50)) AIOHTTP_KEEPALIVE_TIMEOUT = int(os.getenv("AIOHTTP_KEEPALIVE_TIMEOUT", 120)) AIOHTTP_TTL_DNS_CACHE = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300)) +# enable_cleanup_closed is only needed for Python versions with the SSL leak bug +# Fixed in Python 3.12.7+ and 3.13.1+ (see https://github.com/python/cpython/pull/118960) +# Reference: https://github.com/aio-libs/aiohttp/blob/master/aiohttp/connector.py#L74-L78 +AIOHTTP_NEEDS_CLEANUP_CLOSED = ( + (3, 13, 0) <= sys.version_info < (3, 13, 1) or sys.version_info < (3, 12, 7) +) # WebSocket constants # Default to None (unlimited) to match OpenAI's official agents SDK behavior @@ -140,6 +149,7 @@ REDIS_UPDATE_BUFFER_KEY = "litellm_spend_update_buffer" REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_spend_update_buffer" REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_team_spend_update_buffer" REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_org_spend_update_buffer" +REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_end_user_spend_update_buffer" REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_tag_spend_update_buffer" MAX_REDIS_BUFFER_DEQUEUE_COUNT = int(os.getenv("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100)) MAX_SIZE_IN_MEMORY_QUEUE = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", 10000)) @@ -255,6 +265,9 @@ TOGETHER_AI_EMBEDDING_350_M = int(os.getenv("TOGETHER_AI_EMBEDDING_350_M", 350)) QDRANT_SCALAR_QUANTILE = float(os.getenv("QDRANT_SCALAR_QUANTILE", 0.99)) QDRANT_VECTOR_SIZE = int(os.getenv("QDRANT_VECTOR_SIZE", 1536)) CACHED_STREAMING_CHUNK_DELAY = float(os.getenv("CACHED_STREAMING_CHUNK_DELAY", 0.02)) +AUDIO_SPEECH_CHUNK_SIZE = int( + os.getenv("AUDIO_SPEECH_CHUNK_SIZE", 8192) +) # chunk_size for audio speech streaming. Balance between latency and memory usage MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int( os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 512) ) @@ -277,10 +290,16 @@ REDACTED_BY_LITELM_STRING = "REDACTED_BY_LITELM" MAX_LANGFUSE_INITIALIZED_CLIENTS = int( os.getenv("MAX_LANGFUSE_INITIALIZED_CLIENTS", 50) ) -LOGGING_WORKER_CONCURRENCY = int(os.getenv("LOGGING_WORKER_CONCURRENCY", 100)) # Must be above 0 +LOGGING_WORKER_CONCURRENCY = int( + os.getenv("LOGGING_WORKER_CONCURRENCY", 100) +) # Must be above 0 LOGGING_WORKER_MAX_QUEUE_SIZE = int(os.getenv("LOGGING_WORKER_MAX_QUEUE_SIZE", 50_000)) -LOGGING_WORKER_MAX_TIME_PER_COROUTINE = float(os.getenv("LOGGING_WORKER_MAX_TIME_PER_COROUTINE", 20.0)) -LOGGING_WORKER_CLEAR_PERCENTAGE = int(os.getenv("LOGGING_WORKER_CLEAR_PERCENTAGE", 50)) # Percentage of queue to clear (default: 50%) +LOGGING_WORKER_MAX_TIME_PER_COROUTINE = float( + os.getenv("LOGGING_WORKER_MAX_TIME_PER_COROUTINE", 20.0) +) +LOGGING_WORKER_CLEAR_PERCENTAGE = int( + os.getenv("LOGGING_WORKER_CLEAR_PERCENTAGE", 50) +) # Percentage of queue to clear (default: 50%) MAX_ITERATIONS_TO_CLEAR_QUEUE = int(os.getenv("MAX_ITERATIONS_TO_CLEAR_QUEUE", 200)) MAX_TIME_TO_CLEAR_QUEUE = float(os.getenv("MAX_TIME_TO_CLEAR_QUEUE", 5.0)) LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS = float( @@ -395,6 +414,7 @@ LITELLM_CHAT_PROVIDERS = [ "ovhcloud", "lemonade", "docker_model_runner", + "amazon_nova", ] LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [ @@ -520,6 +540,7 @@ openai_compatible_endpoints: List = [ "https://api.friendli.ai/serverless/v1", "api.sambanova.ai/v1", "api.x.ai/v1", + "ollama.com", "api.galadriel.ai/v1", "api.llama.com/compat/v1/", "api.featherless.ai/v1", @@ -527,7 +548,7 @@ openai_compatible_endpoints: List = [ "api.studio.nebius.ai/v1", "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", "https://api.moonshot.ai/v1", - "https://platform.publicai.co/v1", + "https://api.publicai.co/v1", "https://api.v0.dev/v1", "https://api.morphllm.com/v1", "https://api.lambda.ai/v1", @@ -554,6 +575,7 @@ openai_compatible_providers: List = [ "perplexity", "xinference", "xai", + "zai", "together_ai", "fireworks_ai", "empower", @@ -568,6 +590,7 @@ openai_compatible_providers: List = [ "github_copilot", # GitHub Copilot Chat API "novita", "meta_llama", + "publicai", # PublicAI - JSON-configured provider "featherless_ai", "nscale", "nebius", @@ -584,6 +607,7 @@ openai_compatible_providers: List = [ "cometapi", "clarifai", "docker_model_runner", + "ragflow", ] openai_text_completion_compatible_providers: List = ( [ # providers that support `/v1/completions` @@ -856,12 +880,16 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[ "nova", "deepseek_r1", "qwen3", + "qwen2", + "twelvelabs", + "openai", ] BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[ "cohere", "amazon", "twelvelabs", + "nova", ] BEDROCK_CONVERSE_MODELS = [ @@ -904,6 +932,9 @@ BEDROCK_CONVERSE_MODELS = [ "meta.llama3-2-3b-instruct-v1:0", "meta.llama3-2-11b-instruct-v1:0", "meta.llama3-2-90b-instruct-v1:0", + "amazon.nova-lite-v1:0", + "amazon.nova-2-lite-v1:0", + "amazon.nova-pro-v1:0", ] @@ -922,6 +953,7 @@ cohere_embedding_models: set = set( bedrock_embedding_models: set = set( [ "amazon.titan-embed-text-v1", + "amazon.nova-2-multimodal-embeddings-v1:0", "cohere.embed-english-v3", "cohere.embed-multilingual-v3", "cohere.embed-v4:0", diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 9ef26d23ce2..4407134fa92 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -95,6 +95,7 @@ from litellm.utils import ( EmbeddingResponse, ImageResponse, ModelResponse, + ModelResponseStream, ProviderConfigManager, TextCompletionResponse, TranscriptionResponse, @@ -654,7 +655,9 @@ def _infer_call_type( if completion_response is None: return None - if isinstance(completion_response, ModelResponse): + if isinstance(completion_response, ModelResponse) or isinstance( + completion_response, ModelResponseStream + ): return "completion" elif isinstance(completion_response, EmbeddingResponse): return "embedding" @@ -857,9 +860,9 @@ def completion_cost( # noqa: PLR0915 or isinstance(completion_response, dict) ): # tts returns a custom class if isinstance(completion_response, dict): - usage_obj: Optional[Union[dict, Usage]] = ( - completion_response.get("usage", {}) - ) + usage_obj: Optional[ + Union[dict, Usage] + ] = completion_response.get("usage", {}) else: usage_obj = getattr(completion_response, "usage", {}) if isinstance(usage_obj, BaseModel) and not _is_known_usage_objects( @@ -934,6 +937,17 @@ def completion_cost( # noqa: PLR0915 prompt_tokens = token_counter(model=model, text=prompt) completion_tokens = token_counter(model=model, text=completion) + # Handle A2A calls before model check - A2A doesn't require a model + if call_type in ( + CallTypes.asend_message.value, + CallTypes.send_message.value, + ): + from litellm.a2a_protocol.cost_calculator import A2ACostCalculator + + return A2ACostCalculator.calculate_a2a_cost( + litellm_logging_obj=litellm_logging_obj + ) + if model is None: raise ValueError( f"Model is None and does not exist in passed completion_response. Passed completion_response={completion_response}, model={model}" @@ -1046,29 +1060,36 @@ def completion_cost( # noqa: PLR0915 number_of_queries = len(query) elif query is not None: number_of_queries = 1 - + search_model = model or "" if custom_llm_provider and "/" not in search_model: # If model is like "tavily-search", construct "tavily/search" for cost lookup search_model = f"{custom_llm_provider}/search" - - prompt_cost, completion_cost_result = search_provider_cost_per_query( + + ( + prompt_cost, + completion_cost_result, + ) = search_provider_cost_per_query( model=search_model, custom_llm_provider=custom_llm_provider, number_of_queries=number_of_queries, optional_params=optional_params, ) - + # Return the total cost (prompt_cost + completion_cost, but for search it's just prompt_cost) _final_cost = prompt_cost + completion_cost_result - + # Apply discount original_cost = _final_cost - _final_cost, discount_percent, discount_amount = _apply_cost_discount( + ( + _final_cost, + discount_percent, + discount_amount, + ) = _apply_cost_discount( base_cost=_final_cost, custom_llm_provider=custom_llm_provider, ) - + # Store cost breakdown in logging object if available _store_cost_breakdown_in_logging_obj( litellm_logging_obj=litellm_logging_obj, @@ -1080,7 +1101,7 @@ def completion_cost( # noqa: PLR0915 discount_percent=discount_percent, discount_amount=discount_amount, ) - + return _final_cost elif call_type == CallTypes.arealtime.value and isinstance( completion_response, LiteLLMRealtimeStreamLoggingObject @@ -1311,9 +1332,8 @@ def response_cost_calculator( response_cost = 0.0 else: if isinstance(response_object, BaseModel): - response_object._hidden_params["optional_params"] = optional_params - if hasattr(response_object, "_hidden_params"): + response_object._hidden_params["optional_params"] = optional_params provider_response_cost = get_response_cost_from_hidden_params( response_object._hidden_params ) diff --git a/litellm/files/main.py b/litellm/files/main.py index 535772fa42c..9378715a472 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -18,6 +18,7 @@ from litellm import get_secret_str from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.azure.files.handler import AzureOpenAIFilesAPI +from litellm.llms.bedrock.files.handler import BedrockFilesHandler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.openai.openai import FileDeleted, FileObject, OpenAIFilesAPI @@ -30,7 +31,10 @@ from litellm.types.llms.openai import ( OpenAIFileObject, ) from litellm.types.router import * -from litellm.types.utils import LlmProviders +from litellm.types.utils import ( + OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS, + LlmProviders, +) from litellm.utils import ( ProviderConfigManager, client, @@ -44,6 +48,7 @@ base_llm_http_handler = BaseLLMHTTPHandler() openai_files_instance = OpenAIFilesAPI() azure_files_instance = AzureOpenAIFilesAPI() vertex_ai_files_instance = VertexAIFilesHandler() +bedrock_files_instance = BedrockFilesHandler() ################################################# @@ -51,7 +56,7 @@ vertex_ai_files_instance = VertexAIFilesHandler() async def acreate_file( file: FileTypes, purpose: Literal["assistants", "batch", "fine-tune"], - custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai", + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -95,9 +100,7 @@ async def acreate_file( def create_file( file: FileTypes, purpose: Literal["assistants", "batch", "fine-tune"], - custom_llm_provider: Optional[ - Literal["openai", "azure", "vertex_ai", "bedrock"] - ] = None, + custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"]] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -165,7 +168,7 @@ def create_file( ), timeout=timeout, ) - elif custom_llm_provider == "openai": + elif custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS: # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there api_base = ( optional_params.api_base @@ -276,7 +279,7 @@ def create_file( @client async def afile_retrieve( file_id: str, - custom_llm_provider: Literal["openai", "azure"] = "openai", + custom_llm_provider: Literal["openai", "azure", "hosted_vllm"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -317,7 +320,7 @@ async def afile_retrieve( @client def file_retrieve( file_id: str, - custom_llm_provider: Literal["openai", "azure"] = "openai", + custom_llm_provider: Literal["openai", "azure", "hosted_vllm"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -347,7 +350,7 @@ def file_retrieve( _is_async = kwargs.pop("is_async", False) is True - if custom_llm_provider == "openai": + if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS: # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there api_base = ( optional_params.api_base @@ -514,7 +517,7 @@ def file_delete( elif timeout is None: timeout = 600.0 _is_async = kwargs.pop("is_async", False) is True - if custom_llm_provider == "openai": + if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS: # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there api_base = ( optional_params.api_base @@ -670,7 +673,7 @@ def file_list( timeout = 600.0 _is_async = kwargs.pop("is_async", False) is True - if custom_llm_provider == "openai": + if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS: # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there api_base = ( optional_params.api_base @@ -754,7 +757,7 @@ def file_list( @client async def afile_content( file_id: str, - custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -799,7 +802,7 @@ def file_content( file_id: str, model: Optional[str] = None, custom_llm_provider: Optional[ - Union[Literal["openai", "azure", "vertex_ai"], str] + Union[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"], str] ] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, @@ -846,7 +849,7 @@ def file_content( _is_async = kwargs.pop("afile_content", False) is True - if custom_llm_provider == "openai": + if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS: # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there api_base = ( optional_params.api_base @@ -937,9 +940,18 @@ def file_content( timeout=timeout, max_retries=optional_params.max_retries, ) + elif custom_llm_provider == "bedrock": + response = bedrock_files_instance.file_content( + _is_async=_is_async, + file_content_request=_file_content_request, + api_base=optional_params.api_base, + optional_params=litellm_params_dict, + timeout=timeout, + max_retries=optional_params.max_retries, + ) else: raise litellm.exceptions.BadRequestError( - message="LiteLLM doesn't support {} for 'custom_llm_provider'. Supported providers are 'openai', 'azure', 'vertex_ai'.".format( + message="LiteLLM doesn't support {} for 'custom_llm_provider'. Supported providers are 'openai', 'azure', 'vertex_ai', 'bedrock'.".format( custom_llm_provider ), model="n/a", diff --git a/litellm/images/main.py b/litellm/images/main.py index eacd4778299..770b16c1ed2 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -6,7 +6,9 @@ from typing import Any, Coroutine, Dict, List, Literal, Optional, Union, cast, o import httpx import litellm -from litellm import client, exception_type, get_litellm_params +from litellm.utils import exception_type, get_litellm_params +# client is imported from litellm as it's a decorator +from litellm import client from litellm.constants import DEFAULT_IMAGE_ENDPOINT_MODEL from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT from litellm.exceptions import LiteLLMUnknownProvider diff --git a/litellm/integrations/SlackAlerting/budget_alert_types.py b/litellm/integrations/SlackAlerting/budget_alert_types.py index 1e9ad286e37..dadfef3fc40 100644 --- a/litellm/integrations/SlackAlerting/budget_alert_types.py +++ b/litellm/integrations/SlackAlerting/budget_alert_types.py @@ -50,6 +50,14 @@ class TeamBudgetAlert(BaseBudgetAlertType): return user_info.team_id or "default_id" +class OrganizationBudgetAlert(BaseBudgetAlertType): + def get_event_message(self) -> str: + return "Organization Budget: " + + def get_id(self, user_info: CallInfo) -> str: + return user_info.organization_id or "default_id" + + class TokenBudgetAlert(BaseBudgetAlertType): def get_event_message(self) -> str: return "Key Budget: " @@ -72,6 +80,7 @@ def get_budget_alert_type( "soft_budget", "user_budget", "team_budget", + "organization_budget", "proxy_budget", "projected_limit_exceeded", ], @@ -83,6 +92,7 @@ def get_budget_alert_type( "soft_budget": SoftBudgetAlert(), "user_budget": UserBudgetAlert(), "team_budget": TeamBudgetAlert(), + "organization_budget": OrganizationBudgetAlert(), "token_budget": TokenBudgetAlert(), "projected_limit_exceeded": ProjectedLimitExceededAlert(), } diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 3efe5873786..0e691e2c43f 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -134,19 +134,25 @@ class SlackAlerting(CustomBatchLogger): if llm_router is not None: self.llm_router = llm_router - def _prepare_outage_value_for_cache(self, outage_value: Union[dict, ProviderRegionOutageModel, OutageModel]) -> dict: + def _prepare_outage_value_for_cache( + self, outage_value: Union[dict, ProviderRegionOutageModel, OutageModel] + ) -> dict: """ Helper method to prepare outage value for Redis caching. Converts set objects to lists for JSON serialization. """ # Convert to dict for processing cache_value = dict(outage_value) - - if "deployment_ids" in cache_value and isinstance(cache_value["deployment_ids"], set): + + if "deployment_ids" in cache_value and isinstance( + cache_value["deployment_ids"], set + ): cache_value["deployment_ids"] = list(cache_value["deployment_ids"]) return cache_value - def _restore_outage_value_from_cache(self, outage_value: Optional[dict]) -> Optional[dict]: + def _restore_outage_value_from_cache( + self, outage_value: Optional[dict] + ) -> Optional[dict]: """ Helper method to restore outage value after retrieving from cache. Converts list objects back to sets for proper handling. @@ -528,6 +534,7 @@ class SlackAlerting(CustomBatchLogger): "soft_budget", "user_budget", "team_budget", + "organization_budget", "proxy_budget", "projected_limit_exceeded", ], @@ -1338,7 +1345,7 @@ Model Info: subject=email_event["subject"], html=email_event["html"], ) - if webhook_event.event_group == "team": + if webhook_event.event_group == Litellm_EntityType.TEAM: from litellm.integrations.email_alerting import send_team_budget_alert await send_team_budget_alert(webhook_event=webhook_event) @@ -1399,7 +1406,7 @@ Model Info: current_time = datetime.now().strftime("%H:%M:%S") _proxy_base_url = os.getenv("PROXY_BASE_URL", None) # Use .name if it's an enum, otherwise use as is - alert_type_name = getattr(alert_type, 'name', alert_type) + alert_type_name = getattr(alert_type, "name", alert_type) alert_type_formatted = f"Alert type: `{alert_type_name}`" if alert_type == "daily_reports" or alert_type == "new_model_added": formatted_message = alert_type_formatted + message diff --git a/litellm/integrations/arize/arize_phoenix.py b/litellm/integrations/arize/arize_phoenix.py index ab70dd9d0e2..4a6e0cec8ca 100644 --- a/litellm/integrations/arize/arize_phoenix.py +++ b/litellm/integrations/arize/arize_phoenix.py @@ -1,18 +1,20 @@ import os -from typing import TYPE_CHECKING, Any, Union +from typing import TYPE_CHECKING, Any, Optional, Union +from datetime import datetime from litellm._logging import verbose_logger from litellm.integrations.arize import _utils from litellm.integrations.arize._utils import ArizeOTELAttributes from litellm.types.integrations.arize_phoenix import ArizePhoenixConfig +from litellm.types.services import ServiceLoggerPayload +from litellm.integrations.opentelemetry import OpenTelemetry if TYPE_CHECKING: from opentelemetry.trace import Span as _Span + from litellm.integrations.opentelemetry import OpenTelemetryConfig as _OpenTelemetryConfig from litellm.types.integrations.arize import Protocol as _Protocol - from .opentelemetry import OpenTelemetryConfig as _OpenTelemetryConfig - Protocol = _Protocol OpenTelemetryConfig = _OpenTelemetryConfig Span = Union[_Span, Any] @@ -25,7 +27,11 @@ else: ARIZE_HOSTED_PHOENIX_ENDPOINT = "https://otlp.arize.com/v1/traces" -class ArizePhoenixLogger: +class ArizePhoenixLogger(OpenTelemetry): + def set_attributes(self, span: Span, kwargs, response_obj: Optional[Any]): + ArizePhoenixLogger.set_arize_phoenix_attributes(span, kwargs, response_obj) + return + @staticmethod def set_arize_phoenix_attributes(span: Span, kwargs, response_obj): _utils.set_attributes(span, kwargs, response_obj, ArizeOTELAttributes) @@ -97,3 +103,46 @@ class ArizePhoenixLogger: endpoint=endpoint, project_name=project_name, ) + + async def async_service_success_hook( + self, + payload: ServiceLoggerPayload, + parent_otel_span: Optional[Span] = None, + start_time: Optional[Union[datetime, float]] = None, + end_time: Optional[Union[datetime, float]] = None, + event_metadata: Optional[dict] = None, + ): + pass # suppress additional spans + + async def async_service_failure_hook( + self, + payload: ServiceLoggerPayload, + error: Optional[str] = "", + parent_otel_span: Optional[Span] = None, + start_time: Optional[Union[datetime, float]] = None, + end_time: Optional[Union[float, datetime]] = None, + event_metadata: Optional[dict] = None, + ): + pass # suppress additional spans + + def create_litellm_proxy_request_started_span( + self, + start_time: datetime, + headers: dict, + ): + pass # suppress additional spans + + async def async_health_check(self): + + config = self.get_arize_phoenix_config() + + if not config.otlp_auth_headers: + return { + "status": "unhealthy", + "error_message": "PHOENIX_API_KEY environment variable not set", + } + + return { + "status": "healthy", + "message": "Arize-Phoenix credentials are configured properly", + } \ No newline at end of file diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 42555841401..bb5883dd475 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -1,15 +1,25 @@ from datetime import datetime -from typing import Any, Dict, List, Optional, Type, Union, get_args +from typing import ( + TYPE_CHECKING, + Any, + Dict, + List, + Literal, + Optional, + Type, + Union, + get_args, +) from litellm._logging import verbose_logger from litellm.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.types.guardrails import ( DynamicGuardrailParams, + GenericGuardrailAPIInputs, GuardrailEventHooks, LitellmParams, Mode, - PiiEntityType, ) from litellm.types.llms.openai import AllMessageValues from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel @@ -20,6 +30,8 @@ from litellm.types.utils import ( StandardLoggingGuardrailInformation, ) +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj dc = DualCache() @@ -437,30 +449,33 @@ class CustomGuardrail(CustomLogger): async def apply_guardrail( self, - text: str, - language: Optional[str] = None, - entities: Optional[List[PiiEntityType]] = None, - request_data: Optional[dict] = None, - ) -> str: + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional["LiteLLMLoggingObj"] = None, + ) -> GenericGuardrailAPIInputs: """ - Apply your guardrail logic to the given text + Apply your guardrail logic to the given inputs Args: - text: The text to apply the guardrail to - language: The language of the text - entities: The entities to mask, optional - request_data: The request data dictionary to store guardrail metadata + inputs: Dictionary containing: + - texts: List of texts to apply the guardrail to + - images: Optional list of images to apply the guardrail to + - tool_calls: Optional list of tool calls to apply the guardrail to + request_data: The request data dictionary - containing user api key metadata (e.g. user_id, team_id, etc.) + input_type: The type of input to apply the guardrail to - "request" or "response" + logging_obj: Optional logging object for tracking the guardrail execution Any of the custom guardrails can override this method to provide custom guardrail logic - Returns the text with the guardrail applied + Returns the texts with the guardrail applied and the images with the guardrail applied (if any) Raises: Exception: - If the guardrail raises an exception """ - return text + return inputs def _process_response( self, diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 481a2a3ecb7..a3e67d8a73f 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -80,6 +80,44 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac self.turn_off_message_logging = turn_off_message_logging pass + @staticmethod + def get_callback_env_vars(callback_name: Optional[str] = None) -> List[str]: + """ + Return the environment variables associated with a given callback + name as defined in the proxy callback registry. + + Args: + callback_name: The name of the callback to look up. + + Returns: + List[str]: A list of required environment variable names. + """ + if callback_name is None: + return [] + + normalized_name = callback_name.lower() + + alias_map = { + "langfuse_otel": "langfuse", + } + lookup_name = alias_map.get(normalized_name, normalized_name) + + try: + from litellm.proxy._types import AllCallbacks + except Exception: + return [] + + callbacks = AllCallbacks() + callback_info = getattr(callbacks, lookup_name, None) + if callback_info is None: + return [] + + params = getattr(callback_info, "litellm_callback_params", None) + if not params: + return [] + + return list(params) + def log_pre_api_call(self, model, messages, kwargs): pass diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index 46e1a2c201f..21e1d562224 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -65,11 +65,11 @@ class DataDogLogger( `DD_SITE` - your datadog site, example = `"us5.datadoghq.com"` Optional environment variables (DataDog Agent): - `DD_AGENT_HOST` - hostname or IP of DataDog agent, example = `"localhost"` - `DD_AGENT_PORT` - port of DataDog agent (default: 10518 for logs) + `LITELLM_DD_AGENT_HOST` - hostname or IP of DataDog agent, example = `"localhost"` + `LITELLM_DD_AGENT_PORT` - port of DataDog agent (default: 10518 for logs) - Note: If DD_AGENT_HOST is set, logs will be sent to the agent instead of directly to DataDog API. - In this case, DD_API_KEY and DD_SITE are not required (agent handles authentication). + Note: We use LITELLM_DD_AGENT_HOST instead of DD_AGENT_HOST to avoid conflicts + with ddtrace which automatically sets DD_AGENT_HOST for APM tracing. """ try: verbose_logger.debug("Datadog: in init datadog logger") @@ -85,7 +85,8 @@ class DataDogLogger( ) # Configure DataDog endpoint (Agent or Direct API) - dd_agent_host = os.getenv("DD_AGENT_HOST") + # Use LITELLM_DD_AGENT_HOST to avoid conflicts with ddtrace's DD_AGENT_HOST + dd_agent_host = os.getenv("LITELLM_DD_AGENT_HOST") if dd_agent_host: self._configure_dd_agent(dd_agent_host=dd_agent_host) else: @@ -127,7 +128,7 @@ class DataDogLogger( Args: dd_agent_host: Hostname or IP of DataDog agent """ - dd_agent_port = os.getenv("DD_AGENT_PORT", "10518") # default port for logs + dd_agent_port = os.getenv("LITELLM_DD_AGENT_PORT", "10518") # default port for logs self.intake_url = f"http://{dd_agent_host}:{dd_agent_port}/api/v2/logs" self.DD_API_KEY = os.getenv("DD_API_KEY") # Optional when using agent verbose_logger.debug(f"Datadog: Using DD Agent at {self.intake_url}") diff --git a/litellm/integrations/mlflow.py b/litellm/integrations/mlflow.py index b348737868d..6378e55f7e1 100644 --- a/litellm/integrations/mlflow.py +++ b/litellm/integrations/mlflow.py @@ -129,8 +129,11 @@ class MlflowLogger(CustomLogger): self._add_chunk_events(span, response_obj) # If this is the final chunk, end the span. The final chunk - # has complete_streaming_response that gathers the full response. - if final_response := kwargs.get("complete_streaming_response"): + # has the assembled streaming response (key differs between sync/async paths). + final_response = kwargs.get("complete_streaming_response") or kwargs.get( + "async_complete_streaming_response" + ) + if final_response: end_time_ns = int(end_time.timestamp() * 1e9) self._extract_and_set_chat_attributes(span, kwargs, final_response) @@ -153,7 +156,9 @@ class MlflowLogger(CustomLogger): span.add_event( SpanEvent( name="streaming_chunk", - attributes={"delta": json.dumps(choice.delta.model_dump())}, + attributes={ + "delta": json.dumps(choice.delta.model_dump, default=str) + }, ) ) except Exception: diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 90bf19b21fe..9f9d45d0e7d 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -1065,14 +1065,7 @@ class OpenTelemetry(CustomLogger): self, span: Span, kwargs, response_obj: Optional[Any] ): try: - if self.callback_name == "arize_phoenix": - from litellm.integrations.arize.arize_phoenix import ArizePhoenixLogger - - ArizePhoenixLogger.set_arize_phoenix_attributes( - span, kwargs, response_obj - ) - return - elif self.callback_name == "langtrace": + if self.callback_name == "langtrace": from litellm.integrations.langtrace import LangtraceAttributes LangtraceAttributes().set_langtrace_attributes( @@ -1088,6 +1081,11 @@ class OpenTelemetry(CustomLogger): span, kwargs, response_obj ) return + elif self.callback_name == "weave_otel": + from litellm.integrations.weave.weave_otel import set_weave_otel_attributes + + set_weave_otel_attributes(span, kwargs, response_obj) + return from litellm.proxy._types import SpanAttributes optional_params = kwargs.get("optional_params", {}) diff --git a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py index 236935778d6..218581a41ad 100644 --- a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py +++ b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py @@ -74,9 +74,20 @@ class VectorStorePreCallHook(CustomLogger): if litellm.vector_store_registry is None: return model, messages, non_default_params + # Get prisma_client for database fallback + prisma_client = None + try: + from litellm.proxy.proxy_server import prisma_client as _prisma_client + prisma_client = _prisma_client + except ImportError: + pass + + # Use database fallback to ensure synchronization across instances vector_stores_to_run: List[LiteLLM_ManagedVectorStore] = ( - litellm.vector_store_registry.pop_vector_stores_to_run( - non_default_params=non_default_params, tools=tools + await litellm.vector_store_registry.pop_vector_stores_to_run_with_db_fallback( + non_default_params=non_default_params, + tools=tools, + prisma_client=prisma_client ) ) diff --git a/litellm/integrations/weave/__init__.py b/litellm/integrations/weave/__init__.py new file mode 100644 index 00000000000..49af77b55e8 --- /dev/null +++ b/litellm/integrations/weave/__init__.py @@ -0,0 +1,7 @@ +""" +Weave (W&B) integration for LiteLLM via OpenTelemetry. +""" + +from litellm.integrations.weave.weave_otel import WeaveOtelLogger + +__all__ = ["WeaveOtelLogger"] diff --git a/litellm/integrations/weave/weave_otel.py b/litellm/integrations/weave/weave_otel.py new file mode 100644 index 00000000000..167deaf2cdc --- /dev/null +++ b/litellm/integrations/weave/weave_otel.py @@ -0,0 +1,329 @@ +from __future__ import annotations + +import base64 +import json +import os +from typing import TYPE_CHECKING, Any, Optional + +from opentelemetry.trace import Status, StatusCode +from typing_extensions import override + +from litellm._logging import verbose_logger +from litellm.integrations._types.open_inference import SpanAttributes as OpenInferenceSpanAttributes +from litellm.integrations.arize import _utils +from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig +from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes import ( + BaseLLMObsOTELAttributes, + safe_set_attribute, +) +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.types.integrations.weave_otel import WeaveOtelConfig, WeaveSpanAttributes +from litellm.types.utils import StandardCallbackDynamicParams + +if TYPE_CHECKING: + from opentelemetry.trace import Span + + +# Weave OTEL endpoint +# Multi-tenant cloud: https://trace.wandb.ai/otel/v1/traces +# Dedicated cloud: https://.wandb.io/traces/otel/v1/traces +WEAVE_BASE_URL = "https://trace.wandb.ai" +WEAVE_OTEL_ENDPOINT = "/otel/v1/traces" + + +class WeaveLLMObsOTELAttributes(BaseLLMObsOTELAttributes): + """ + Weave-specific LLM observability OTEL attributes. + + Weave automatically maps attributes from multiple frameworks including + GenAI, OpenInference, Langfuse, and others. + """ + + @staticmethod + @override + def set_messages(span: "Span", kwargs: dict[str, Any]): + """Set input messages as span attributes using OpenInference conventions.""" + + messages = kwargs.get("messages") or [] + optional_params = kwargs.get("optional_params") or {} + + prompt = {"messages": messages} + functions = optional_params.get("functions") + tools = optional_params.get("tools") + if functions is not None: + prompt["functions"] = functions + if tools is not None: + prompt["tools"] = tools + safe_set_attribute(span, OpenInferenceSpanAttributes.INPUT_VALUE, json.dumps(prompt)) + + +def _set_weave_specific_attributes(span: Span, kwargs: dict[str, Any], response_obj: Any): + """ + Sets Weave-specific metadata attributes onto the OTEL span. + + Based on Weave's OTEL attribute mappings from: + https://github.com/wandb/weave/blob/master/weave/trace_server/opentelemetry/constants.py + """ + + # Extract all needed data upfront + litellm_params = kwargs.get("litellm_params") or {} + # optional_params = kwargs.get("optional_params") or {} + metadata = kwargs.get("metadata") or {} + model = kwargs.get("model") or "" + custom_llm_provider = litellm_params.get("custom_llm_provider") or "" + + # Weave supports a custom display name and will default to the model name if not provided. + display_name = metadata.get("display_name") + if not display_name and model: + if custom_llm_provider: + display_name = f"{custom_llm_provider}/{model}" + else: + display_name = model + if display_name: + display_name = display_name.replace("/", "__") + safe_set_attribute(span, WeaveSpanAttributes.DISPLAY_NAME.value, display_name) + + # Weave threads are OpenInference sessions. + if (session_id := metadata.get("session_id")) is not None: + if isinstance(session_id, (list, dict)): + session_id = safe_dumps(session_id) + safe_set_attribute(span, WeaveSpanAttributes.THREAD_ID.value, session_id) + safe_set_attribute(span, WeaveSpanAttributes.IS_TURN.value, True) + + # Response attributes are already set by _utils.set_attributes, + # but we override them here to better match Weave's expectations + if response_obj: + output_dict = None + if hasattr(response_obj, "model_dump"): + output_dict = response_obj.model_dump() + elif hasattr(response_obj, "get"): + output_dict = response_obj + + if output_dict: + safe_set_attribute(span, OpenInferenceSpanAttributes.OUTPUT_VALUE, safe_dumps(output_dict)) + + +def _get_weave_authorization_header(api_key: str) -> str: + """ + Get the authorization header for Weave OpenTelemetry. + + Weave uses Basic auth with format: api: + """ + auth_string = f"api:{api_key}" + auth_header = base64.b64encode(auth_string.encode()).decode() + return f"Basic {auth_header}" + + +def get_weave_otel_config() -> WeaveOtelConfig: + """ + Retrieves the Weave OpenTelemetry configuration based on environment variables. + + Environment Variables: + WANDB_API_KEY: Required. W&B API key for authentication. + WANDB_PROJECT_ID: Required. Project ID in format /. + WANDB_HOST: Optional. Custom Weave host URL. Defaults to cloud endpoint. + + Returns: + WeaveOtelConfig: A Pydantic model containing Weave OTEL configuration. + + Raises: + ValueError: If required environment variables are missing. + """ + api_key = os.getenv("WANDB_API_KEY") + project_id = os.getenv("WANDB_PROJECT_ID") + host = os.getenv("WANDB_HOST") + + if not api_key: + raise ValueError("WANDB_API_KEY must be set for Weave OpenTelemetry integration.") + + if not project_id: + raise ValueError( + "WANDB_PROJECT_ID must be set for Weave OpenTelemetry integration. Format: /" + ) + + if host: + if not host.startswith("http"): + host = "https://" + host + # Self-managed instances use a different path + endpoint = host.rstrip("/") + WEAVE_OTEL_ENDPOINT + verbose_logger.debug(f"Using Weave OTEL endpoint from host: {endpoint}") + else: + endpoint = WEAVE_BASE_URL + WEAVE_OTEL_ENDPOINT + verbose_logger.debug(f"Using Weave cloud endpoint: {endpoint}") + + # Weave uses Basic auth with format: api: + auth_header = _get_weave_authorization_header(api_key=api_key) + otlp_auth_headers = f"Authorization={auth_header},project_id={project_id}" + + # Set standard OTEL environment variables + os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = endpoint + os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = otlp_auth_headers + + return WeaveOtelConfig( + otlp_auth_headers=otlp_auth_headers, + endpoint=endpoint, + project_id=project_id, + protocol="otlp_http", + ) + + +def set_weave_otel_attributes(span: Span, kwargs: dict[str, Any], response_obj: Any): + """ + Sets OpenTelemetry span attributes for Weave observability. + Uses the same attribute setting logic as other OTEL integrations for consistency. + """ + _utils.set_attributes(span, kwargs, response_obj, WeaveLLMObsOTELAttributes) + _set_weave_specific_attributes(span=span, kwargs=kwargs, response_obj=response_obj) + + +class WeaveOtelLogger(OpenTelemetry): + """ + Weave (W&B) OpenTelemetry Logger for LiteLLM. + + Sends LLM traces to Weave via the OpenTelemetry Protocol (OTLP). + + Environment Variables: + WANDB_API_KEY: Required. Weights & Biases API key for authentication. + WANDB_PROJECT_ID: Required. Project ID in format /. + WANDB_HOST: Optional. Custom Weave host URL. Defaults to cloud endpoint. + + Usage: + litellm.callbacks = ["weave_otel"] + + Or manually: + from litellm.integrations.weave.weave_otel import WeaveOtelLogger + weave_logger = WeaveOtelLogger(callback_name="weave_otel") + litellm.callbacks = [weave_logger] + + Reference: + https://docs.wandb.ai/weave/guides/tracking/otel + """ + + def __init__( + self, + config: Optional[OpenTelemetryConfig] = None, + callback_name: Optional[str] = "weave_otel", + **kwargs, + ): + """ + Initialize WeaveOtelLogger. + + If config is not provided, automatically configures from environment variables + (WANDB_API_KEY, WANDB_PROJECT_ID, WANDB_HOST) via get_weave_otel_config(). + """ + if config is None: + # Auto-configure from Weave environment variables + weave_config = get_weave_otel_config() + + config = OpenTelemetryConfig( + exporter=weave_config.protocol, + endpoint=weave_config.endpoint, + headers=weave_config.otlp_auth_headers, + ) + + super().__init__(config=config, callback_name=callback_name, **kwargs) + + def _maybe_log_raw_request(self, kwargs, response_obj, start_time, end_time, parent_span): + """ + Override to skip creating the raw_gen_ai_request child span. + + For Weave, we only want a single span per LLM call. The parent span + already contains all the necessary attributes, so the child span + is redundant. + """ + pass + + def _start_primary_span( + self, + kwargs, + response_obj, + start_time, + end_time, + context, + parent_span=None, + ): + """ + Override to always create a child span instead of reusing the parent span. + + This ensures that wrapper spans (like "B", "C", "D", "E") remain separate + from the LiteLLM LLM call spans, creating proper nesting in Weave. + """ + + otel_tracer = self.get_tracer_to_use_for_request(kwargs) + # Always create a new child span, even if parent_span is provided + # This ensures wrapper spans remain separate from LLM call spans + span = otel_tracer.start_span( + name=self._get_span_name(kwargs), + start_time=self._to_ns(start_time), + context=context, + ) + span.set_status(Status(StatusCode.OK)) + self.set_attributes(span, kwargs, response_obj) + span.end(end_time=self._to_ns(end_time)) + return span + + def _handle_success(self, kwargs, response_obj, start_time, end_time): + """ + Override to prevent ending externally created parent spans. + + When wrapper spans (like "B", "C", "D", "E") are provided as parent spans, + they should be managed by the user code, not ended by LiteLLM. + """ + + verbose_logger.debug( + "Weave OpenTelemetry Logger: Logging kwargs: %s, OTEL config settings=%s", + kwargs, + self.config, + ) + ctx, parent_span = self._get_span_context(kwargs) + + # Always create a child span (handled by _start_primary_span override) + primary_span_parent = None + + # 1. Primary span + span = self._start_primary_span(kwargs, response_obj, start_time, end_time, ctx, primary_span_parent) + + # 2. Raw-request sub-span (skipped for Weave via _maybe_log_raw_request override) + self._maybe_log_raw_request(kwargs, response_obj, start_time, end_time, span) + + # 3. Guardrail span + self._create_guardrail_span(kwargs=kwargs, context=ctx) + + # 4. Metrics & cost recording + self._record_metrics(kwargs, response_obj, start_time, end_time) + + # 5. Semantic logs. + if self.config.enable_events: + self._emit_semantic_logs(kwargs, response_obj, span) + + # 6. Don't end parent span - it's managed by user code + # Since we always create a child span (never reuse parent), the parent span + # lifecycle is owned by the user. This prevents double-ending of wrapper spans + # like "B", "C", "D", "E" that users create and manage themselves. + + def construct_dynamic_otel_headers( + self, standard_callback_dynamic_params: StandardCallbackDynamicParams + ) -> dict | None: + """ + Construct dynamic Weave headers from standard callback dynamic params. + + This is used for team/key based logging. + + Returns: + dict: A dictionary of dynamic Weave headers + """ + dynamic_headers = {} + + dynamic_wandb_api_key = standard_callback_dynamic_params.get("wandb_api_key") + dynamic_weave_project_id = standard_callback_dynamic_params.get("weave_project_id") + + if dynamic_wandb_api_key: + auth_header = _get_weave_authorization_header( + api_key=dynamic_wandb_api_key, + ) + dynamic_headers["Authorization"] = auth_header + + if dynamic_weave_project_id: + dynamic_headers["project_id"] = dynamic_weave_project_id + + return dynamic_headers if dynamic_headers else None diff --git a/litellm/litellm_core_utils/README.md b/litellm/litellm_core_utils/README.md index 6494041291b..b61c8982762 100644 --- a/litellm/litellm_core_utils/README.md +++ b/litellm/litellm_core_utils/README.md @@ -9,4 +9,5 @@ Core files: - `default_encoding.py`: code for loading the default encoding (tiktoken) - `get_llm_provider_logic.py`: code for inferring the LLM provider from a given model name. - `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s" +- `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion]) diff --git a/litellm/litellm_core_utils/api_route_to_call_types.py b/litellm/litellm_core_utils/api_route_to_call_types.py new file mode 100644 index 00000000000..35f83de1dd7 --- /dev/null +++ b/litellm/litellm_core_utils/api_route_to_call_types.py @@ -0,0 +1,38 @@ +""" +Dictionary mapping API routes to their corresponding CallTypes in LiteLLM. + +This dictionary maps each API endpoint to the CallTypes that can be used for that route. +Each route can have both async (prefixed with 'a') and sync call types. +""" + +from litellm.types.utils import API_ROUTE_TO_CALL_TYPES, CallTypes + + +def get_call_types_for_route(route: str) -> list: + """ + Get the list of CallTypes for a given API route. + + Args: + route: API route path (e.g., "/chat/completions") + + Returns: + List of CallTypes for that route, or empty list if route not found + """ + return API_ROUTE_TO_CALL_TYPES.get(route, []) + + +def get_routes_for_call_type(call_type: CallTypes) -> list: + """ + Get all routes that use a specific CallType. + + Args: + call_type: The CallType to search for + + Returns: + List of routes that use this CallType + """ + routes = [] + for route, types in API_ROUTE_TO_CALL_TYPES.items(): + if call_type in types: + routes.append(route) + return routes diff --git a/litellm/litellm_core_utils/custom_logger_registry.py b/litellm/litellm_core_utils/custom_logger_registry.py index 538ef6be283..fdc9f374553 100644 --- a/litellm/litellm_core_utils/custom_logger_registry.py +++ b/litellm/litellm_core_utils/custom_logger_registry.py @@ -75,6 +75,7 @@ class CustomLoggerRegistry: "langfuse_otel": OpenTelemetry, "arize_phoenix": OpenTelemetry, "langtrace": OpenTelemetry, + "weave_otel": OpenTelemetry, "mlflow": MlflowLogger, "langfuse": LangfusePromptManagement, "otel": OpenTelemetry, diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index f8c786daef3..056522e2a64 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -77,6 +77,7 @@ class ExceptionCheckers: "model's maximum context limit", "is longer than the model's context length", "input tokens exceed the configured limit", + "`inputs` tokens + `max_new_tokens` must be", ] for substring in known_exception_substrings: if substring in _error_str_lowercase: diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 3bb1e4afb92..168c837f1e5 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -22,17 +22,16 @@ def _is_non_openai_azure_model(model: str) -> bool: return False -def _is_azure_anthropic_model(model: str) -> Optional[str]: +def _is_azure_claude_model(model: str) -> bool: + """ + Check if a model name contains 'claude' (case-insensitive). + Used to detect Claude models that need Anthropic-specific handling. + """ try: - model_parts = model.split("/", 1) - if len(model_parts) > 1: - model_name = model_parts[1].lower() - # Check if model name contains claude - if "claude" in model_name or model_name.startswith("claude"): - return model_parts[1] # Return model name without "azure/" prefix + model_lower = model.lower() + return "claude" in model_lower or model_lower.startswith("claude") except Exception: - pass - return None + return False def handle_cohere_chat_model_custom_llm_provider( @@ -136,11 +135,6 @@ def get_llm_provider( # noqa: PLR0915 # AZURE AI-Studio Logic - Azure AI Studio supports AZURE/Cohere # If User passes azure/command-r-plus -> we should send it to cohere_chat/command-r-plus if model.split("/", 1)[0] == "azure": - # Check if it's an Azure Anthropic model (claude models) - azure_anthropic_model = _is_azure_anthropic_model(model) - if azure_anthropic_model: - custom_llm_provider = "azure_anthropic" - return azure_anthropic_model, custom_llm_provider, dynamic_api_key, api_base if _is_non_openai_azure_model(model): custom_llm_provider = "openai" return model, custom_llm_provider, dynamic_api_key, api_base @@ -235,6 +229,9 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "api.deepseek.com/v1": custom_llm_provider = "deepseek" dynamic_api_key = get_secret_str("DEEPSEEK_API_KEY") + elif endpoint == "ollama.com": + custom_llm_provider = "ollama" + dynamic_api_key = get_secret_str("OLLAMA_API_KEY") elif endpoint == "https://api.friendli.ai/serverless/v1": custom_llm_provider = "friendliai" dynamic_api_key = get_secret_str( @@ -407,6 +404,8 @@ def get_llm_provider( # noqa: PLR0915 custom_llm_provider = "lemonade" elif model.startswith("clarifai/"): custom_llm_provider = "clarifai" + elif model.startswith("amazon_nova"): + custom_llm_provider = "amazon_nova" if not custom_llm_provider: if litellm.suppress_debug_info is False: print() # noqa @@ -474,6 +473,20 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 custom_llm_provider = model.split("/", 1)[0] model = model.split("/", 1)[1] + # Check JSON providers FIRST (before hardcoded ones) + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + if JSONProviderRegistry.exists(custom_llm_provider): + provider_config = JSONProviderRegistry.get(custom_llm_provider) + if provider_config is None: + raise ValueError(f"Provider {custom_llm_provider} not found") + config_class = create_config_class(provider_config) + api_base, dynamic_api_key = config_class()._get_openai_compatible_provider_info( + api_base, api_key + ) + return model, custom_llm_provider, dynamic_api_key, api_base + if custom_llm_provider == "perplexity": # perplexity is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.perplexity.ai ( @@ -550,6 +563,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 or "https://api.studio.nebius.ai/v1" ) # type: ignore dynamic_api_key = api_key or get_secret_str("NEBIUS_API_KEY") + elif custom_llm_provider == "ollama": + api_base = ( + api_base + or get_secret("OLLAMA_API_BASE") + or "http://localhost:11434" + ) # type: ignore + dynamic_api_key = api_key or get_secret_str("OLLAMA_API_KEY") elif (custom_llm_provider == "ai21_chat") or ( custom_llm_provider == "ai21" and model in litellm.ai21_chat_models ): @@ -668,6 +688,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.XAIChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "zai": + ( + api_base, + dynamic_api_key, + ) = litellm.ZAIChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) elif custom_llm_provider == "together_ai": api_base = ( api_base @@ -762,13 +789,7 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.MoonshotChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) - elif custom_llm_provider == "publicai": - ( - api_base, - dynamic_api_key, - ) = litellm.PublicAIChatConfig()._get_openai_compatible_provider_info( - api_base, api_key - ) + # publicai is now handled by JSON config (see litellm/llms/openai_like/providers.json) elif custom_llm_provider == "docker_model_runner": ( api_base, @@ -839,6 +860,16 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.ClarifaiConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "ragflow": + full_model = f"ragflow/{model}" + ( + api_base, + dynamic_api_key, + _, + ) = litellm.RAGFlowConfig()._get_openai_compatible_provider_info( + full_model, api_base, api_key, "ragflow" + ) + model = full_model if api_base is not None and not isinstance(api_base, str): raise Exception("api base needs to be a string. api_base={}".format(api_base)) diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py index 06e650f938d..19b52d2dace 100644 --- a/litellm/litellm_core_utils/get_supported_openai_params.py +++ b/litellm/litellm_core_utils/get_supported_openai_params.py @@ -266,6 +266,15 @@ def get_supported_openai_params( # noqa: PLR0915 model=model ) ) + elif custom_llm_provider == "ovhcloud": + if request_type == "transcription": + from litellm.llms.ovhcloud.audio_transcription.transformation import ( + OVHCloudAudioTranscriptionConfig, + ) + + return OVHCloudAudioTranscriptionConfig().get_supported_openai_params( + model=model + ) elif custom_llm_provider == "elevenlabs": if request_type == "transcription": from litellm.llms.elevenlabs.audio_transcription.transformation import ( diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 0b0f483ff74..2b52d58e29d 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -71,6 +71,7 @@ from litellm.litellm_core_utils.redact_messages import ( from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.responses.utils import ResponseAPILoggingUtils +from litellm.types.agents import LiteLLMSendMessageResponse from litellm.types.containers.main import ContainerObject from litellm.types.llms.openai import ( AllMessageValues, @@ -1738,6 +1739,7 @@ class Logging(LiteLLMLoggingBaseClass): and logging_result.get("object") == "search" # Search API (dict format) or isinstance(logging_result, VideoObject) or isinstance(logging_result, ContainerObject) + or isinstance(logging_result, LiteLLMSendMessageResponse) # A2A or (self.call_type == CallTypes.call_mcp_tool.value) ): return True @@ -3617,15 +3619,15 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 for callback in _in_memory_loggers: if ( - isinstance(callback, OpenTelemetry) + isinstance(callback, ArizePhoenixLogger) and callback.callback_name == "arize_phoenix" ): return callback # type: ignore - _otel_logger = OpenTelemetry( + _arize_phoenix_otel_logger = ArizePhoenixLogger( config=otel_config, callback_name="arize_phoenix" ) - _in_memory_loggers.append(_otel_logger) - return _otel_logger # type: ignore + _in_memory_loggers.append(_arize_phoenix_otel_logger) + return _arize_phoenix_otel_logger # type: ignore elif logging_integration == "otel": from litellm.integrations.opentelemetry import OpenTelemetry @@ -3800,6 +3802,31 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 ) _in_memory_loggers.append(_otel_logger) return _otel_logger # type: ignore + elif logging_integration == "weave_otel": + from litellm.integrations.opentelemetry import ( + OpenTelemetryConfig, + ) + from litellm.integrations.weave.weave_otel import WeaveOtelLogger, get_weave_otel_config + + weave_otel_config = get_weave_otel_config() + + otel_config = OpenTelemetryConfig( + exporter=weave_otel_config.protocol, + endpoint=weave_otel_config.endpoint, + headers=weave_otel_config.otlp_auth_headers, + ) + + for callback in _in_memory_loggers: + if ( + isinstance(callback, WeaveOtelLogger) + and callback.callback_name == "weave_otel" + ): + return callback # type: ignore + _otel_logger = WeaveOtelLogger( + config=otel_config, callback_name="weave_otel" + ) + _in_memory_loggers.append(_otel_logger) + return _otel_logger # type: ignore elif logging_integration == "pagerduty": for callback in _in_memory_loggers: if isinstance(callback, PagerDutyAlerting): diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 9717f442b82..ef2183a4556 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -583,9 +583,11 @@ def generic_cost_per_token( reasoning_tokens = completion_tokens_details["reasoning_tokens"] image_tokens = completion_tokens_details["image_tokens"] - if text_tokens == 0: + # Only assume all tokens are text if there's NO breakdown at all + # If image_tokens, audio_tokens, or reasoning_tokens exist, respect text_tokens=0 + has_token_breakdown = image_tokens > 0 or audio_tokens > 0 or reasoning_tokens > 0 + if text_tokens == 0 and not has_token_breakdown: text_tokens = usage.completion_tokens - if text_tokens == usage.completion_tokens: is_text_tokens_total = True ## TEXT COST completion_cost = float(text_tokens) * completion_base_cost diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index c50ceeabdb2..d2c91f4a841 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1071,7 +1071,7 @@ def _parse_content_for_reasoning( return None, message_text reasoning_match = re.match( - r"<(?:think|thinking)>(.*?)(.*)", message_text, re.DOTALL + r"<(?:think|thinking|budget:thinking)>(.*?)(.*)", message_text, re.DOTALL ) if reasoning_match: diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 262692d6d1a..04c8c235557 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -3446,8 +3446,25 @@ class BedrockConverseMessagesProcessor: @staticmethod def _initial_message_setup( messages: List, + model: str, + llm_provider: str, user_continue_message: Optional[ChatCompletionUserMessage] = None, ) -> List: + # gracefully handle base case of no messages at all + if len(messages) == 0: + if user_continue_message is not None: + messages.append(user_continue_message) + elif litellm.modify_params: + messages.append(DEFAULT_USER_CONTINUE_MESSAGE) + else: + raise litellm.BadRequestError( + message=BAD_MESSAGE_ERROR_STR + + "bedrock requires at least one non-system message", + model=model, + llm_provider=llm_provider, + ) + + # if initial message is assistant message if messages[0].get("role") is not None and messages[0]["role"] == "assistant": if user_continue_message is not None: messages.insert(0, user_continue_message) @@ -3475,18 +3492,8 @@ class BedrockConverseMessagesProcessor: contents: List[BedrockMessageBlock] = [] msg_i = 0 - ## BASE CASE ## - if len(messages) == 0: - raise litellm.BadRequestError( - message=BAD_MESSAGE_ERROR_STR - + "bedrock requires at least one non-system message", - model=model, - llm_provider=llm_provider, - ) - - # if initial message is assistant message messages = BedrockConverseMessagesProcessor._initial_message_setup( - messages, user_continue_message + messages, model, llm_provider, user_continue_message ) while msg_i < len(messages): @@ -3847,28 +3854,9 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 contents: List[BedrockMessageBlock] = [] msg_i = 0 - ## BASE CASE ## - if len(messages) == 0: - raise litellm.BadRequestError( - message=BAD_MESSAGE_ERROR_STR - + "bedrock requires at least one non-system message", - model=model, - llm_provider=llm_provider, - ) - - # if initial message is assistant message - if messages[0].get("role") is not None and messages[0]["role"] == "assistant": - if user_continue_message is not None: - messages.insert(0, user_continue_message) - elif litellm.modify_params: - messages.insert(0, DEFAULT_USER_CONTINUE_MESSAGE) - - # if final message is assistant message - if messages[-1].get("role") is not None and messages[-1]["role"] == "assistant": - if user_continue_message is not None: - messages.append(user_continue_message) - elif litellm.modify_params: - messages.append(DEFAULT_USER_CONTINUE_MESSAGE) + messages = BedrockConverseMessagesProcessor._initial_message_setup( + messages, model, llm_provider, user_continue_message + ) while msg_i < len(messages): user_content: List[BedrockContentBlock] = [] diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 4d8e109d882..4ffb7ace5b0 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -96,9 +96,9 @@ class CustomStreamWrapper: self.system_fingerprint: Optional[str] = None self.received_finish_reason: Optional[str] = None - self.intermittent_finish_reason: Optional[str] = ( - None # finish reasons that show up mid-stream - ) + self.intermittent_finish_reason: Optional[ + str + ] = None # finish reasons that show up mid-stream self.special_tokens = [ "<|assistant|>", "<|system|>", @@ -735,8 +735,9 @@ class CustomStreamWrapper: and completion_obj["function_call"] is not None ) or ( - "tool_calls" in model_response.choices[0].delta + "tool_calls" in model_response.choices[0].delta and model_response.choices[0].delta["tool_calls"] is not None + and len(model_response.choices[0].delta["tool_calls"]) > 0 ) or ( "function_call" in model_response.choices[0].delta @@ -889,7 +890,6 @@ class CustomStreamWrapper: ## check if openai/azure chunk original_chunk = response_obj.get("original_chunk", None) if original_chunk: - if len(original_chunk.choices) > 0: choices = [] for choice in original_chunk.choices: @@ -906,7 +906,6 @@ class CustomStreamWrapper: print_verbose(f"choices in streaming: {choices}") setattr(model_response, "choices", choices) else: - return model_response.system_fingerprint = ( original_chunk.system_fingerprint @@ -1435,9 +1434,9 @@ class CustomStreamWrapper: _json_delta = delta.model_dump() print_verbose(f"_json_delta: {_json_delta}") if "role" not in _json_delta or _json_delta["role"] is None: - _json_delta["role"] = ( - "assistant" # mistral's api returns role as None - ) + _json_delta[ + "role" + ] = "assistant" # mistral's api returns role as None if "tool_calls" in _json_delta and isinstance( _json_delta["tool_calls"], list ): @@ -1533,7 +1532,7 @@ class CustomStreamWrapper: async def _call_post_streaming_deployment_hook(self, chunk): """ Call the post-call streaming deployment hook for callbacks. - + This allows callbacks to modify streaming chunks before they're returned. """ try: @@ -1544,15 +1543,17 @@ class CustomStreamWrapper: # Get request kwargs from logging object request_data = self.logging_obj.model_call_details call_type_str = self.logging_obj.call_type - + try: typed_call_type = CallTypes(call_type_str) except ValueError: typed_call_type = None - + # Call hooks for all callbacks for callback in litellm.callbacks: - if isinstance(callback, CustomLogger) and hasattr(callback, "async_post_call_streaming_deployment_hook"): + if isinstance(callback, CustomLogger) and hasattr( + callback, "async_post_call_streaming_deployment_hook" + ): result = await callback.async_post_call_streaming_deployment_hook( request_data=request_data, response_chunk=chunk, @@ -1560,11 +1561,14 @@ class CustomStreamWrapper: ) if result is not None: chunk = result - + return chunk except Exception as e: from litellm._logging import verbose_logger - verbose_logger.exception(f"Error in post-call streaming deployment hook: {str(e)}") + + verbose_logger.exception( + f"Error in post-call streaming deployment hook: {str(e)}" + ) return chunk def cache_streaming_response(self, processed_chunk, cache_hit: bool): @@ -1687,7 +1691,7 @@ class CustomStreamWrapper: response, "usage" ): # remove usage from chunk, only send on final chunk # Convert the object to a dictionary - obj_dict = response.dict() + obj_dict = response.model_dump() # Remove an attribute (e.g., 'attr2') if "usage" in obj_dict: @@ -1852,7 +1856,7 @@ class CustomStreamWrapper: processed_chunk, "usage" ): # remove usage from chunk, only send on final chunk # Convert the object to a dictionary - obj_dict = processed_chunk.dict() + obj_dict = processed_chunk.model_dump() # Remove an attribute (e.g., 'attr2') if "usage" in obj_dict: @@ -1872,11 +1876,15 @@ class CustomStreamWrapper: if self.sent_last_chunk is True and self.stream_options is None: usage = calculate_total_usage(chunks=self.chunks) processed_chunk._hidden_params["usage"] = usage - + # Call post-call streaming deployment hook for final chunk if self.sent_last_chunk is True: - processed_chunk = await self._call_post_streaming_deployment_hook(processed_chunk) - + processed_chunk = ( + await self._call_post_streaming_deployment_hook( + processed_chunk + ) + ) + return processed_chunk raise StopAsyncIteration else: # temporary patch for non-aiohttp async calls @@ -1890,9 +1898,9 @@ class CustomStreamWrapper: chunk = next(self.completion_stream) if chunk is not None and chunk != b"": print_verbose(f"PROCESSED CHUNK PRE CHUNK CREATOR: {chunk}") - processed_chunk: Optional[ModelResponseStream] = ( - self.chunk_creator(chunk=chunk) - ) + processed_chunk: Optional[ + ModelResponseStream + ] = self.chunk_creator(chunk=chunk) print_verbose( f"PROCESSED CHUNK POST CHUNK CREATOR: {processed_chunk}" ) diff --git a/litellm/llms/amazon_nova/chat/transformation.py b/litellm/llms/amazon_nova/chat/transformation.py new file mode 100644 index 00000000000..6d321e298b8 --- /dev/null +++ b/litellm/llms/amazon_nova/chat/transformation.py @@ -0,0 +1,115 @@ +""" +Translate from OpenAI's `/v1/chat/completions` to Amazon Nova's `/v1/chat/completions` +""" +from typing import Any, List, Optional, Tuple + +import httpx + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + AllMessageValues, +) +from litellm.types.utils import ModelResponse + +from ...openai_like.chat.transformation import OpenAILikeChatConfig + + +class AmazonNovaChatConfig(OpenAILikeChatConfig): + max_completion_tokens: Optional[int] = None + max_tokens: Optional[int] = None + metadata: Optional[int] = None + temperature: Optional[int] = None + top_p: Optional[int] = None + tools: Optional[list] = None + reasoning_effort: Optional[list] = None + + def __init__( + self, + max_completion_tokens: Optional[int] = None, + max_tokens: Optional[int] = None, + temperature: Optional[int] = None, + top_p: Optional[int] = None, + tools: Optional[list] = None, + reasoning_effort: Optional[list] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @property + def custom_llm_provider(self) -> Optional[str]: + return "amazon_nova" + + @classmethod + def get_config(cls): + return super().get_config() + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + # Amazon Nova is openai compatible, we just need to set this to custom_openai and have the api_base be Nova's endpoint + api_base = ( + api_base + or get_secret_str("AMAZON_NOVA_API_BASE") + or "https://api.nova.amazon.com/v1" + ) # type: ignore + + # Get API key from multiple sources + key = ( + api_key + or litellm.amazon_nova_api_key + or get_secret_str("AMAZON_NOVA_API_KEY") + or litellm.api_key + ) + return api_base, key + + def get_supported_openai_params(self, model: str) -> List: + return [ + "top_p", + "temperature", + "max_tokens", + "max_completion_tokens", + "metadata", + "stop", + "stream", + "stream_options", + "tools", + "tool_choice", + "reasoning_effort" + ] + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + model_response = super().transform_response( + model=model, + model_response=model_response, + raw_response=raw_response, + messages=messages, + logging_obj=logging_obj, + request_data=request_data, + encoding=encoding, + optional_params=optional_params, + json_mode=json_mode, + litellm_params=litellm_params, + api_key=api_key, + ) + + # Storing amazon_nova in the model response for easier cost calculation later + setattr(model_response, "model", "amazon-nova/" + model) + + return model_response \ No newline at end of file diff --git a/litellm/llms/amazon_nova/cost_calculation.py b/litellm/llms/amazon_nova/cost_calculation.py new file mode 100644 index 00000000000..9d9cedde875 --- /dev/null +++ b/litellm/llms/amazon_nova/cost_calculation.py @@ -0,0 +1,21 @@ +""" +Helper util for handling amazon nova cost calculation +- e.g.: prompt caching +""" + +from typing import TYPE_CHECKING, Tuple + +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + +if TYPE_CHECKING: + from litellm.types.utils import Usage + + +def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]: + """ + Calculates the cost per token for a given model, prompt tokens, and completion tokens. + Follows the same logic as Anthropic's cost per token calculation. + """ + return generic_cost_per_token( + model=model, usage=usage, custom_llm_provider="amazon_nova" + ) \ No newline at end of file diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 6aba2947d3b..b1c4b1484da 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -12,11 +12,24 @@ Pattern Overview: 4. Apply guardrail responses back to the original structure """ -import asyncio -from typing import TYPE_CHECKING, Any, Coroutine, Dict, List, Optional, Tuple, cast +import json +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast from litellm._logging import verbose_proxy_logger +from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, +) from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation +from litellm.types.guardrails import GenericGuardrailAPIInputs +from litellm.types.llms.anthropic import ( + AllAnthropicToolsValues, + AnthropicMessagesRequest, +) +from litellm.types.llms.openai import ( + ChatCompletionToolCallChunk, + ChatCompletionToolParam, +) if TYPE_CHECKING: from litellm.integrations.custom_guardrail import CustomGuardrail @@ -37,6 +50,10 @@ class AnthropicMessagesHandler(BaseTranslation): Methods can be overridden to customize behavior for different message formats. """ + def __init__(self): + super().__init__() + self.adapter = LiteLLMAnthropicMessagesAdapter() + async def process_input_messages( self, data: dict, @@ -50,30 +67,57 @@ class AnthropicMessagesHandler(BaseTranslation): if messages is None: return data - tasks: List[Coroutine[Any, Any, str]] = [] + chat_completion_compatible_request = ( + LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request=cast(AnthropicMessagesRequest, data) + ) + ) + + structured_messages = chat_completion_compatible_request.get("messages", []) + + texts_to_check: List[str] = [] + images_to_check: List[str] = [] + tools_to_check: List[ChatCompletionToolParam] = ( + chat_completion_compatible_request.get("tools", []) + ) task_mappings: List[Tuple[int, Optional[int]]] = [] - # Track (message_index, content_index) for each task + # Track (message_index, content_index) for each text # content_index is None for string content, int for list content - # Step 1: Extract all text content and create guardrail tasks + # Step 1: Extract all text content and images for msg_idx, message in enumerate(messages): - await self._extract_input_text_and_create_tasks( + self._extract_input_text_and_images( message=message, msg_idx=msg_idx, - tasks=tasks, + texts_to_check=texts_to_check, + images_to_check=images_to_check, task_mappings=task_mappings, - guardrail_to_apply=guardrail_to_apply, ) - # Step 2: Run all guardrail tasks in parallel - responses = await asyncio.gather(*tasks) + # Step 2: Apply guardrail to all texts in batch + if texts_to_check: + inputs = GenericGuardrailAPIInputs(texts=texts_to_check) + if images_to_check: + inputs["images"] = images_to_check + if tools_to_check: + inputs["tools"] = tools_to_check + if structured_messages: + inputs["structured_messages"] = structured_messages + guardrailed_inputs = await guardrail_to_apply.apply_guardrail( + inputs=inputs, + request_data=data, + input_type="request", + logging_obj=litellm_logging_obj, + ) - # Step 3: Map guardrail responses back to original message structure - await self._apply_guardrail_responses_to_input( - messages=messages, - responses=responses, - task_mappings=task_mappings, - ) + guardrailed_texts = guardrailed_inputs.get("texts", []) + + # Step 3: Map guardrail responses back to original message structure + await self._apply_guardrail_responses_to_input( + messages=messages, + responses=guardrailed_texts, + task_mappings=task_mappings, + ) verbose_proxy_logger.debug( "Anthropic Messages: Processed input messages: %s", messages @@ -81,36 +125,63 @@ class AnthropicMessagesHandler(BaseTranslation): return data - async def _extract_input_text_and_create_tasks( + def _extract_input_text_and_images( self, message: Dict[str, Any], msg_idx: int, - tasks: List, + texts_to_check: List[str], + images_to_check: List[str], task_mappings: List[Tuple[int, Optional[int]]], - guardrail_to_apply: "CustomGuardrail", ) -> None: """ - Extract text content from a message and create guardrail tasks. + Extract text content and images from a message. - Override this method to customize text extraction logic. + Override this method to customize text/image extraction logic. """ content = message.get("content", None) - if content is None: + tools = message.get("tools", None) + if content is None and tools is None: return - if isinstance(content, str): + ## CHECK FOR TEXT + IMAGES + if content is not None and isinstance(content, str): # Simple string content - tasks.append(guardrail_to_apply.apply_guardrail(text=content)) + texts_to_check.append(content) task_mappings.append((msg_idx, None)) - elif isinstance(content, list): + elif content is not None and isinstance(content, list): # List content (e.g., multimodal with text and images) for content_idx, content_item in enumerate(content): + # Extract text text_str = content_item.get("text", None) - if text_str is None: - continue - tasks.append(guardrail_to_apply.apply_guardrail(text=text_str)) - task_mappings.append((msg_idx, int(content_idx))) + if text_str is not None: + texts_to_check.append(text_str) + task_mappings.append((msg_idx, int(content_idx))) + + # Extract images + if content_item.get("type") == "image": + source = content_item.get("source", {}) + if isinstance(source, dict): + # Could be base64 or url + data = source.get("data") + if data: + images_to_check.append(data) + + def _extract_input_tools( + self, + tools: List[Dict[str, Any]], + tools_to_check: List[ChatCompletionToolParam], + ) -> None: + """ + Extract tools from a message. + """ + ## CHECK FOR TOOLS + if tools is not None and isinstance(tools, list): + # TRANSFORM ANTHROPIC TOOLS TO OPENAI TOOLS + openai_tools = self.adapter.translate_anthropic_tools_to_openai( + tools=cast(List[AllAnthropicToolsValues], tools) + ) + tools_to_check.extend(openai_tools) async def _apply_guardrail_responses_to_input( self, @@ -147,56 +218,87 @@ class AnthropicMessagesHandler(BaseTranslation): response: "AnthropicMessagesResponse", guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: Optional[Any] = None, + user_api_key_dict: Optional[Any] = None, ) -> Any: """ - Process output response by applying guardrails to text content. + Process output response by applying guardrails to text content and tool calls. Args: response: Anthropic MessagesResponse object guardrail_to_apply: The guardrail instance to apply + litellm_logging_obj: Optional logging object + user_api_key_dict: User API key metadata to pass to guardrails Returns: Modified response with guardrail applied to content Response Format Support: - - List content: response.content = [{"type": "text", "text": "text here"}, ...] + - List content: response.content = [ + {"type": "text", "text": "text here"}, + {"type": "tool_use", "id": "...", "name": "...", "input": {...}}, + ... + ] """ - # Step 0: Check if response has any text content to process - if not self._has_text_content(response): - verbose_proxy_logger.warning( - "Anthropic Messages: No text content in response, skipping guardrail" - ) - return response - - tasks: List[Coroutine[Any, Any, str]] = [] + texts_to_check: List[str] = [] + images_to_check: List[str] = [] + tool_calls_to_check: List[ChatCompletionToolCallChunk] = [] task_mappings: List[Tuple[int, Optional[int]]] = [] - # Track (choice_index, content_index) for each task + # Track (content_index, None) for each text response_content = response.get("content", []) if not response_content: return response - # Step 1: Extract all text content from response choices + + # Step 1: Extract all text content and tool calls from response for content_idx, content_block in enumerate(response_content): - # Check if this is a text block by checking the 'type' field - if isinstance(content_block, dict) and content_block.get("type") == "text": + # Check if this is a text or tool_use block by checking the 'type' field + if isinstance(content_block, dict) and content_block.get("type") in [ + "text", + "tool_use", + ]: # Cast to dict to handle the union type properly - await self._extract_output_text_and_create_tasks( + self._extract_output_text_and_images( content_block=cast(Dict[str, Any], content_block), content_idx=content_idx, - tasks=tasks, + texts_to_check=texts_to_check, + images_to_check=images_to_check, task_mappings=task_mappings, - guardrail_to_apply=guardrail_to_apply, + tool_calls_to_check=tool_calls_to_check, ) - # Step 2: Run all guardrail tasks in parallel - responses = await asyncio.gather(*tasks) + # Step 2: Apply guardrail to all texts in batch + if texts_to_check or tool_calls_to_check: + # Create a request_data dict with response info and user API key metadata + request_data: dict = {"response": response} - # Step 3: Map guardrail responses back to original response structure - await self._apply_guardrail_responses_to_output( - response=response, - responses=responses, - task_mappings=task_mappings, - ) + # Add user API key metadata with prefixed keys + user_metadata = self.transform_user_api_key_dict_to_metadata( + user_api_key_dict + ) + if user_metadata: + request_data["litellm_metadata"] = user_metadata + + inputs = GenericGuardrailAPIInputs(texts=texts_to_check) + if images_to_check: + inputs["images"] = images_to_check + if tool_calls_to_check: + inputs["tool_calls"] = tool_calls_to_check + + guardrailed_inputs = await guardrail_to_apply.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="response", + logging_obj=litellm_logging_obj, + ) + + guardrailed_texts = guardrailed_inputs.get("texts", []) + + # Step 3: Map guardrail responses back to original response structure + await self._apply_guardrail_responses_to_output( + response=response, + responses=guardrailed_texts, + task_mappings=task_mappings, + ) verbose_proxy_logger.debug( "Anthropic Messages: Processed output response: %s", response @@ -204,6 +306,112 @@ class AnthropicMessagesHandler(BaseTranslation): return response + async def process_output_streaming_response( + self, + responses_so_far: List[Any], + guardrail_to_apply: "CustomGuardrail", + litellm_logging_obj: Optional[Any] = None, + user_api_key_dict: Optional[Any] = None, + ) -> List[Any]: + """ + Process output streaming response by applying guardrails to text content. + + Get the string so far, check the apply guardrail to the string so far, and return the list of responses so far. + """ + string_so_far = self.get_streaming_string_so_far(responses_so_far) + _guardrailed_inputs = await guardrail_to_apply.apply_guardrail( # allow rejecting the response, if invalid + inputs={"texts": [string_so_far]}, + request_data={}, + input_type="response", + logging_obj=litellm_logging_obj, + ) + return responses_so_far + + def get_streaming_string_so_far(self, responses_so_far: List[Any]) -> str: + """ + Parse streaming responses and extract accumulated text content. + + Handles two formats: + 1. Raw bytes in SSE (Server-Sent Events) format from Anthropic API + 2. Parsed dict objects (for backwards compatibility) + + SSE format example: + b'event: content_block_delta\\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" curious"}}\\n\\n' + + Dict format example: + { + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "text_delta", + "text": " curious" + } + } + """ + text_so_far = "" + for response in responses_so_far: + # Handle raw bytes in SSE format + if isinstance(response, bytes): + text_so_far += self._extract_text_from_sse(response) + # Handle already-parsed dict format + elif isinstance(response, dict): + delta = response.get("delta") if response.get("delta") else None + if delta and delta.get("type") == "text_delta": + text = delta.get("text", "") + if text: + text_so_far += text + return text_so_far + + def _extract_text_from_sse(self, sse_bytes: bytes) -> str: + """ + Extract text content from Server-Sent Events (SSE) format. + + Args: + sse_bytes: Raw bytes in SSE format + + Returns: + Accumulated text from all content_block_delta events + """ + text = "" + try: + # Decode bytes to string + sse_string = sse_bytes.decode("utf-8") + + # Split by double newline to get individual events + events = sse_string.split("\n\n") + + for event in events: + if not event.strip(): + continue + + # Parse event lines + lines = event.strip().split("\n") + event_type = None + data_line = None + + for line in lines: + if line.startswith("event:"): + event_type = line[6:].strip() + elif line.startswith("data:"): + data_line = line[5:].strip() + + # Only process content_block_delta events + if event_type == "content_block_delta" and data_line: + try: + data = json.loads(data_line) + delta = data.get("delta", {}) + if delta.get("type") == "text_delta": + text += delta.get("text", "") + except json.JSONDecodeError: + verbose_proxy_logger.warning( + f"Failed to parse JSON from SSE data: {data_line}" + ) + + except Exception as e: + verbose_proxy_logger.error(f"Error extracting text from SSE: {e}") + + return text + def _has_text_content(self, response: "AnthropicMessagesResponse") -> bool: """ Check if response has any text content to process. @@ -221,24 +429,39 @@ class AnthropicMessagesHandler(BaseTranslation): return True return False - async def _extract_output_text_and_create_tasks( + def _extract_output_text_and_images( self, content_block: Dict[str, Any], content_idx: int, - tasks: List, + texts_to_check: List[str], + images_to_check: List[str], task_mappings: List[Tuple[int, Optional[int]]], - guardrail_to_apply: "CustomGuardrail", + tool_calls_to_check: Optional[List[ChatCompletionToolCallChunk]] = None, ) -> None: """ - Extract text content from a response choice and create guardrail tasks. + Extract text content, images, and tool calls from a response content block. - Override this method to customize text extraction logic. + Override this method to customize text/image/tool extraction logic. """ - content_text = content_block.get("text") - if content_text and isinstance(content_text, str): - # Simple string content - tasks.append(guardrail_to_apply.apply_guardrail(text=content_text)) - task_mappings.append((content_idx, None)) + content_type = content_block.get("type") + + # Extract text content + if content_type == "text": + content_text = content_block.get("text") + if content_text and isinstance(content_text, str): + # Simple string content + texts_to_check.append(content_text) + task_mappings.append((content_idx, None)) + + # Extract tool calls + elif content_type == "tool_use": + tool_call = AnthropicConfig.convert_tool_use_to_openai_format( + anthropic_tool_content=content_block, + index=content_idx, + ) + if tool_calls_to_check is None: + tool_calls_to_check = [] + tool_calls_to_check.append(tool_call) async def _apply_guardrail_responses_to_output( self, diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index b363b747de5..36156d56a59 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -436,9 +436,7 @@ class AnthropicChatCompletion(BaseLLM): else: if client is None or not isinstance(client, HTTPHandler): - client = _get_httpx_client( - params={"timeout": timeout} - ) + client = _get_httpx_client(params={"timeout": timeout}) else: client = client @@ -528,9 +526,7 @@ class ModelResponseIterator: usage_object=cast(dict, anthropic_usage_chunk), reasoning_content=None ) - def _content_block_delta_helper( - self, chunk: dict - ) -> Tuple[ + def _content_block_delta_helper(self, chunk: dict) -> Tuple[ str, Optional[ChatCompletionToolCallChunk], List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]], diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index ac1c9b1e000..b477dbd457e 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -54,10 +54,7 @@ from litellm.types.utils import ( CompletionTokensDetailsWrapper, ) from litellm.types.utils import Message as LitellmMessage -from litellm.types.utils import ( - PromptTokensDetailsWrapper, - ServerToolUse, -) +from litellm.types.utils import PromptTokensDetailsWrapper, ServerToolUse from litellm.utils import ( ModelResponse, Usage, @@ -119,6 +116,40 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): def get_config(cls): return super().get_config() + @staticmethod + def convert_tool_use_to_openai_format( + anthropic_tool_content: Dict[str, Any], + index: int, + ) -> ChatCompletionToolCallChunk: + """ + Convert Anthropic tool_use format to OpenAI ChatCompletionToolCallChunk format. + + Args: + anthropic_tool_content: Anthropic tool_use content block with format: + {"type": "tool_use", "id": "...", "name": "...", "input": {...}} + index: The index of this tool call + + Returns: + ChatCompletionToolCallChunk in OpenAI format + """ + tool_call = ChatCompletionToolCallChunk( + id=anthropic_tool_content["id"], + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=anthropic_tool_content["name"], + arguments=json.dumps(anthropic_tool_content["input"]), + ), + index=index, + ) + # Include caller information if present (for programmatic tool calling) + if "caller" in anthropic_tool_content: + tool_call["caller"] = cast(Dict[str, Any], anthropic_tool_content["caller"]) # type: ignore[typeddict-item] + return tool_call + + def _is_claude_opus_4_5(self, model: str) -> bool: + """Check if the model is Claude Opus 4.5.""" + return "opus-4-5" in model.lower() or "opus_4_5" in model.lower() + def get_supported_openai_params(self, model: str): params = [ "stream", @@ -275,7 +306,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): elif tool["type"] == "tool_search_tool_regex_20251119": # Tool search tool using regex from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex - + tool_name_obj = tool.get("name", "tool_search_tool_regex") if not isinstance(tool_name_obj, str): raise ValueError("Tool search tool must have a valid name") @@ -287,7 +318,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): elif tool["type"] == "tool_search_tool_bm25_20251119": # Tool search tool using BM25 from litellm.types.llms.anthropic import AnthropicToolSearchToolBM25 - + tool_name_obj = tool.get("name", "tool_search_tool_bm25") if not isinstance(tool_name_obj, str): raise ValueError("Tool search tool must have a valid name") @@ -305,7 +336,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if returned_tool is not None: # Only set cache_control on tools that support it (not tool search tools) tool_type = returned_tool.get("type", "") - if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"): + if tool_type not in ( + "tool_search_tool_regex_20251119", + "tool_search_tool_bm25_20251119", + ): if _cache_control is not None: returned_tool["cache_control"] = _cache_control # type: ignore[typeddict-item] elif _cache_control_function is not None and isinstance( @@ -314,14 +348,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): returned_tool["cache_control"] = ChatCompletionCachedContent( # type: ignore[typeddict-item] **_cache_control_function # type: ignore ) - + ## check if defer_loading is set in the tool _defer_loading = tool.get("defer_loading", None) _defer_loading_function = tool.get("function", {}).get("defer_loading", None) if returned_tool is not None: # Only set defer_loading on tools that support it (not tool search tools or computer tools) tool_type = returned_tool.get("type", "") - if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119", "computer_20241022", "computer_20250124"): + if tool_type not in ( + "tool_search_tool_regex_20251119", + "tool_search_tool_bm25_20251119", + "computer_20241022", + "computer_20250124", + ): if _defer_loading is not None: if not isinstance(_defer_loading, bool): raise ValueError("defer_loading must be a boolean") @@ -330,14 +369,21 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if not isinstance(_defer_loading_function, bool): raise ValueError("defer_loading must be a boolean") returned_tool["defer_loading"] = _defer_loading_function # type: ignore[typeddict-item] - + ## check if allowed_callers is set in the tool _allowed_callers = tool.get("allowed_callers", None) - _allowed_callers_function = tool.get("function", {}).get("allowed_callers", None) + _allowed_callers_function = tool.get("function", {}).get( + "allowed_callers", None + ) if returned_tool is not None: # Only set allowed_callers on tools that support it (not tool search tools or computer tools) tool_type = returned_tool.get("type", "") - if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119", "computer_20241022", "computer_20250124"): + if tool_type not in ( + "tool_search_tool_regex_20251119", + "tool_search_tool_bm25_20251119", + "computer_20241022", + "computer_20250124", + ): if _allowed_callers is not None: if not isinstance(_allowed_callers, list) or not all( isinstance(item, str) for item in _allowed_callers @@ -350,7 +396,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ): raise ValueError("allowed_callers must be a list of strings") returned_tool["allowed_callers"] = _allowed_callers_function # type: ignore[typeddict-item] - + ## check if input_examples is set in the tool _input_examples = tool.get("input_examples", None) _input_examples_function = tool.get("function", {}).get("input_examples", None) @@ -419,31 +465,32 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): """Check if tool search tools are present in the tools list.""" if not tools: return False - + for tool in tools: tool_type = tool.get("type", "") - if tool_type in ["tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"]: + if tool_type in [ + "tool_search_tool_regex_20251119", + "tool_search_tool_bm25_20251119", + ]: return True return False - def _separate_deferred_tools( - self, tools: List - ) -> Tuple[List, List]: + def _separate_deferred_tools(self, tools: List) -> Tuple[List, List]: """ Separate tools into deferred and non-deferred lists. - + Returns: Tuple of (non_deferred_tools, deferred_tools) """ non_deferred = [] deferred = [] - + for tool in tools: if tool.get("defer_loading", False): deferred.append(tool) else: non_deferred.append(tool) - + return non_deferred, deferred def _expand_tool_references( @@ -453,28 +500,28 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) -> List: """ Expand tool_reference blocks to full tool definitions. - + When Anthropic's tool search returns results, it includes tool_reference blocks that reference tools by name. This method expands those references to full tool definitions from the deferred_tools catalog. - + Args: content: Response content that may contain tool_reference blocks deferred_tools: List of deferred tools that can be referenced - + Returns: Content with tool_reference blocks expanded to full tool definitions """ if not deferred_tools: return content - + # Create a mapping of tool names to tool definitions tool_map = {} for tool in deferred_tools: tool_name = tool.get("name") or tool.get("function", {}).get("name") if tool_name: tool_map[tool_name] = tool - + # Expand tool references in content expanded_content = [] for item in content: @@ -488,7 +535,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): expanded_content.append(item) else: expanded_content.append(item) - + return expanded_content def _map_stop_sequences( @@ -626,7 +673,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return hosted_web_search_tool - def map_openai_params( + def map_openai_params( # noqa: PLR0915 self, non_default_params: dict, optional_params: dict, @@ -712,9 +759,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if param == "thinking": optional_params["thinking"] = value elif param == "reasoning_effort" and isinstance(value, str): - optional_params["thinking"] = AnthropicConfig._map_reasoning_effort( - value - ) + # For Claude Opus 4.5, map reasoning_effort to output_config + if self._is_claude_opus_4_5(model): + optional_params["output_config"] = {"effort": value} + else: + # For other models, map to thinking parameter + optional_params["thinking"] = AnthropicConfig._map_reasoning_effort( + value + ) elif param == "web_search_options" and isinstance(value, dict): hosted_web_search_tool = self.map_web_search_tool( cast(OpenAIWebSearchOptions, value) @@ -777,6 +829,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): valid_content: bool = False system_message_block = ChatCompletionSystemMessage(**message) if isinstance(system_message_block["content"], str): + # Skip empty text blocks - Anthropic API raises errors for empty text + if not system_message_block["content"]: + continue anthropic_system_message_content = AnthropicSystemMessageContent( type="text", text=system_message_block["content"], @@ -791,10 +846,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): valid_content = True elif isinstance(message["content"], list): for _content in message["content"]: + # Skip empty text blocks - Anthropic API raises errors for empty text + text_value = _content.get("text") + if _content.get("type") == "text" and not text_value: + continue anthropic_system_message_content = ( AnthropicSystemMessageContent( type=_content.get("type"), - text=_content.get("text"), + text=text_value, ) ) if "cache_control" in _content: @@ -979,7 +1038,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "messages": anthropic_messages, **optional_params, } - + ## Handle output_config (Anthropic-specific parameter) if "output_config" in optional_params: output_config = optional_params.get("output_config") @@ -1038,34 +1097,20 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): text_content += content["text"] ## TOOL CALLING elif content["type"] == "tool_use": - tool_call = ChatCompletionToolCallChunk( - id=content["id"], - type="function", - function=ChatCompletionToolCallFunctionChunk( - name=content["name"], - arguments=json.dumps(content["input"]), - ), + tool_call = AnthropicConfig.convert_tool_use_to_openai_format( + anthropic_tool_content=content, index=idx, ) - # Include caller information if present (for programmatic tool calling) - if "caller" in content: - tool_call["caller"] = cast(Dict[str, Any], content["caller"]) # type: ignore[typeddict-item] tool_calls.append(tool_call) ## SERVER TOOL USE (for tool search) elif content["type"] == "server_tool_use": # Server tool use blocks are for tool search - treat as tool calls - tool_call = ChatCompletionToolCallChunk( - id=content["id"], - type="function", - function=ChatCompletionToolCallFunctionChunk( - name=content["name"], - arguments=json.dumps(content.get("input", {})), - ), + # Note: using .get("input", {}) for server_tool_use as input may not be present + content_with_input = {**content, "input": content.get("input", {})} + tool_call = AnthropicConfig.convert_tool_use_to_openai_format( + anthropic_tool_content=content_with_input, index=idx, ) - # Include caller information if present (for programmatic tool calling) - if "caller" in content: - tool_call["caller"] = cast(Dict[str, Any], content["caller"]) # type: ignore[typeddict-item] tool_calls.append(tool_call) ## TOOL SEARCH TOOL RESULT (skip - this is metadata about tool discovery) elif content["type"] == "tool_search_tool_result": @@ -1106,7 +1151,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return text_content, citations, thinking_blocks, reasoning_content, tool_calls def calculate_usage( - self, usage_object: dict, reasoning_content: Optional[str], completion_response: Optional[dict] = None + self, + usage_object: dict, + reasoning_content: Optional[str], + completion_response: Optional[dict] = None, ) -> Usage: # NOTE: Sometimes the usage object has None set explicitly for token counts, meaning .get() & key access returns None, and we need to account for this prompt_tokens = usage_object.get("input_tokens", 0) or 0 @@ -1144,7 +1192,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): tool_search_requests = cast( int, _usage["server_tool_use"]["tool_search_requests"] ) - + # Count tool_search_requests from content blocks if not in usage # Anthropic doesn't always include tool_search_requests in the usage object if tool_search_requests is None and completion_response is not None: diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 9f5688f9e01..7ca3c555542 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -12,7 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.types.llms.anthropic import AllAnthropicToolsValues, AnthropicMcpServerTool +from litellm.types.llms.anthropic import AllAnthropicToolsValues, AnthropicMcpServerTool, ANTHROPIC_HOSTED_TOOLS from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import TokenCountResponse @@ -72,6 +72,17 @@ class AnthropicModelInfo(BaseLLMModelInfo): return tool["type"] return None + def is_web_search_tool_used( + self, tools: Optional[List[AllAnthropicToolsValues]] + ) -> bool: + """Returns True if web_search tool is used""" + if tools is None: + return False + for tool in tools: + if "type" in tool and tool["type"].startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value): + return True + return False + def is_pdf_used(self, messages: List[AllMessageValues]) -> bool: """ Set to true if media passed into messages. @@ -151,15 +162,22 @@ class AnthropicModelInfo(BaseLLMModelInfo): return False - def is_effort_used(self, optional_params: Optional[dict]) -> bool: + def is_effort_used(self, optional_params: Optional[dict], model: Optional[str] = None) -> bool: """ - Check if effort parameter is being used via output_config. + Check if effort parameter is being used. - Returns True if output_config with effort field is present. + Returns True if effort-related parameters are present. """ if not optional_params: return False + # Check if reasoning_effort is provided for Claude Opus 4.5 + if model and ("opus-4-5" in model.lower() or "opus_4_5" in model.lower()): + reasoning_effort = optional_params.get("reasoning_effort") + if reasoning_effort and isinstance(reasoning_effort, str): + return True + + # Check if output_config is directly provided output_config = optional_params.get("output_config") if output_config and isinstance(output_config, dict): effort = output_config.get("effort") @@ -193,6 +211,49 @@ class AnthropicModelInfo(BaseLLMModelInfo): computer_tool_version, "computer-use-2024-10-22" # Default fallback ) + def get_anthropic_beta_list( + self, + model: str, + optional_params: Optional[dict] = None, + computer_tool_used: Optional[str] = None, + prompt_caching_set: bool = False, + file_id_used: bool = False, + mcp_server_used: bool = False, + ) -> List[str]: + """ + Get list of common beta headers based on the features that are active. + + Returns: + List of beta header strings + """ + from litellm.types.llms.anthropic import ( + ANTHROPIC_EFFORT_BETA_HEADER, + ) + + betas = [] + + # Detect features + effort_used = self.is_effort_used(optional_params, model) + + if effort_used: + betas.append(ANTHROPIC_EFFORT_BETA_HEADER) # effort-2025-11-24 + + if computer_tool_used: + beta_header = self.get_computer_tool_beta_header(computer_tool_used) + betas.append(beta_header) + + if prompt_caching_set: + betas.append("prompt-caching-2024-07-31") + + if file_id_used: + betas.append("files-api-2025-04-14") + betas.append("code-execution-2025-05-22") + + if mcp_server_used: + betas.append("mcp-client-2025-04-04") + + return list(set(betas)) + def get_anthropic_headers( self, api_key: str, @@ -202,6 +263,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): pdf_used: bool = False, file_id_used: bool = False, mcp_server_used: bool = False, + web_search_tool_used: bool = False, tool_search_used: bool = False, programmatic_tool_calling_used: bool = False, input_examples_used: bool = False, @@ -242,9 +304,12 @@ class AnthropicModelInfo(BaseLLMModelInfo): if user_anthropic_beta_headers is not None: betas.update(user_anthropic_beta_headers) - # Don't send any beta headers to Vertex, Vertex has failed requests when they are sent + # Don't send any beta headers to Vertex, except web search which is required if is_vertex_request is True: - pass + # Vertex AI requires web search beta header for web search to work + if web_search_tool_used: + from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES + headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value elif len(betas) > 0: headers["anthropic-beta"] = ",".join(betas) @@ -275,10 +340,11 @@ class AnthropicModelInfo(BaseLLMModelInfo): ) pdf_used = self.is_pdf_used(messages=messages) file_id_used = self.is_file_id_used(messages=messages) + web_search_tool_used = self.is_web_search_tool_used(tools=tools) tool_search_used = self.is_tool_search_used(tools=tools) programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools=tools) input_examples_used = self.is_input_examples_used(tools=tools) - effort_used = self.is_effort_used(optional_params=optional_params) + effort_used = self.is_effort_used(optional_params=optional_params, model=model) user_anthropic_beta_headers = self._get_user_anthropic_beta_headers( anthropic_beta_header=headers.get("anthropic-beta") ) @@ -288,6 +354,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): pdf_used=pdf_used, api_key=api_key, file_id_used=file_id_used, + web_search_tool_used=web_search_tool_used, is_vertex_request=optional_params.get("is_vertex_request", False), user_anthropic_beta_headers=user_anthropic_beta_headers, mcp_server_used=mcp_server_used, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 99d19f0460c..790e7901960 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -103,7 +103,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): status_code=400, ) ####### get required params for all anthropic messages requests ###### - verbose_logger.debug(f"🔍 TRANSFORMATION DEBUG - Messages: {messages}") + verbose_logger.debug(f"TRANSFORMATION DEBUG - Messages: {messages}") anthropic_messages_request: AnthropicMessagesRequest = AnthropicMessagesRequest( messages=messages, max_tokens=max_tokens, diff --git a/litellm/llms/anthropic/skills/readme.md b/litellm/llms/anthropic/skills/readme.md index 898639cd44b..0602272256c 100644 --- a/litellm/llms/anthropic/skills/readme.md +++ b/litellm/llms/anthropic/skills/readme.md @@ -1,17 +1,279 @@ -# Anthropic Skills API +# Anthropic Skills API Integration -This folder maintains the integration for the Anthropic Skills API. +This module provides comprehensive support for the Anthropic Skills API through LiteLLM. -You can do the following with the Anthropic Skills API: +## Features -1. Create a new skill -2. List all skills -3. Get a skill -4. Delete a skill +The Skills API allows you to: +- **Create skills**: Define reusable AI capabilities +- **List skills**: Browse all available skills +- **Get skills**: Retrieve detailed information about a specific skill +- **Delete skills**: Remove skills that are no longer needed +## Quick Start -Versions: - - Create Skill Version - - List Skill Versions - - Get Skill Version - - Delete Skill Version \ No newline at end of file +### Prerequisites + +Set your Anthropic API key: +```python +import os +os.environ["ANTHROPIC_API_KEY"] = "your-api-key-here" +``` + +### Basic Usage + +#### Create a Skill + +```python +import litellm + +# Create a skill with files +# Note: All files must be in the same top-level directory +# and must include a SKILL.md file at the root +skill = litellm.create_skill( + files=[ + # List of file objects to upload + # Must include SKILL.md + ], + display_title="Python Code Generator", + custom_llm_provider="anthropic" +) +print(f"Created skill: {skill.id}") + +# Asynchronous version +skill = await litellm.acreate_skill( + files=[...], # Your files here + display_title="Python Code Generator", + custom_llm_provider="anthropic" +) +``` + +#### List Skills + +```python +# List all skills +skills = litellm.list_skills( + custom_llm_provider="anthropic" +) + +for skill in skills.data: + print(f"{skill.display_title}: {skill.id}") + +# With pagination and filtering +skills = litellm.list_skills( + limit=20, + source="custom", # Filter by 'custom' or 'anthropic' + custom_llm_provider="anthropic" +) + +# Get next page if available +if skills.has_more: + next_page = litellm.list_skills( + page=skills.next_page, + custom_llm_provider="anthropic" + ) +``` + +#### Get a Skill + +```python +skill = litellm.get_skill( + skill_id="skill_abc123", + custom_llm_provider="anthropic" +) + +print(f"Skill: {skill.display_title}") +print(f"Created: {skill.created_at}") +print(f"Latest version: {skill.latest_version}") +print(f"Source: {skill.source}") +``` + +#### Delete a Skill + +```python +result = litellm.delete_skill( + skill_id="skill_abc123", + custom_llm_provider="anthropic" +) + +print(f"Deleted skill {result.id}, type: {result.type}") +``` + +## API Reference + +### `create_skill()` + +Create a new skill. + +**Parameters:** +- `files` (List[Any], optional): Files to upload for the skill. All files must be in the same top-level directory and must include a SKILL.md file at the root. +- `display_title` (str, optional): Display title for the skill +- `custom_llm_provider` (str, optional): Provider name (default: "anthropic") +- `extra_headers` (dict, optional): Additional HTTP headers +- `timeout` (float, optional): Request timeout + +**Returns:** +- `Skill`: The created skill object + +**Async version:** `acreate_skill()` + +### `list_skills()` + +List all skills. + +**Parameters:** +- `limit` (int, optional): Number of results to return per page (max 100, default 20) +- `page` (str, optional): Pagination token for fetching a specific page of results +- `source` (str, optional): Filter skills by source ('custom' or 'anthropic') +- `custom_llm_provider` (str, optional): Provider name (default: "anthropic") +- `extra_headers` (dict, optional): Additional HTTP headers +- `timeout` (float, optional): Request timeout + +**Returns:** +- `ListSkillsResponse`: Object containing a list of skills and pagination info + +**Async version:** `alist_skills()` + +### `get_skill()` + +Get a specific skill by ID. + +**Parameters:** +- `skill_id` (str, required): The skill ID +- `custom_llm_provider` (str, optional): Provider name (default: "anthropic") +- `extra_headers` (dict, optional): Additional HTTP headers +- `timeout` (float, optional): Request timeout + +**Returns:** +- `Skill`: The requested skill object + +**Async version:** `aget_skill()` + +### `delete_skill()` + +Delete a skill. + +**Parameters:** +- `skill_id` (str, required): The skill ID to delete +- `custom_llm_provider` (str, optional): Provider name (default: "anthropic") +- `extra_headers` (dict, optional): Additional HTTP headers +- `timeout` (float, optional): Request timeout + +**Returns:** +- `DeleteSkillResponse`: Object with `id` and `type` fields + +**Async version:** `adelete_skill()` + +## Response Types + +### `Skill` + +Represents a skill from the Anthropic Skills API. + +**Fields:** +- `id` (str): Unique identifier +- `created_at` (str): ISO 8601 timestamp +- `display_title` (str, optional): Display title +- `latest_version` (str, optional): Latest version identifier +- `source` (str): Source ("custom" or "anthropic") +- `type` (str): Object type (always "skill") +- `updated_at` (str): ISO 8601 timestamp + +### `ListSkillsResponse` + +Response from listing skills. + +**Fields:** +- `data` (List[Skill]): List of skills +- `next_page` (str, optional): Pagination token for the next page +- `has_more` (bool): Whether more skills are available + +### `DeleteSkillResponse` + +Response from deleting a skill. + +**Fields:** +- `id` (str): The deleted skill ID +- `type` (str): Deleted object type (always "skill_deleted") + +## Architecture + +The Skills API implementation follows LiteLLM's standard patterns: + +1. **Type Definitions** (`litellm/types/llms/anthropic_skills.py`) + - Pydantic models for request/response types + - TypedDict definitions for request parameters + +2. **Base Configuration** (`litellm/llms/base_llm/skills/transformation.py`) + - Abstract base class `BaseSkillsAPIConfig` + - Defines transformation interface for provider-specific implementations + +3. **Provider Implementation** (`litellm/llms/anthropic/skills/transformation.py`) + - `AnthropicSkillsConfig` - Anthropic-specific transformations + - Handles API authentication, URL construction, and response mapping + +4. **Main Handler** (`litellm/skills/main.py`) + - Public API functions (sync and async) + - Request validation and routing + - Error handling + +5. **HTTP Handlers** (`litellm/llms/custom_httpx/llm_http_handler.py`) + - Low-level HTTP request/response handling + - Connection pooling and retry logic + +## Beta API Support + +The Skills API is in beta. The beta header (`skills-2025-10-02`) is automatically added by the Anthropic provider configuration. You can customize it if needed: + +```python +skill = litellm.create_skill( + display_title="My Skill", + extra_headers={ + "anthropic-beta": "skills-2025-10-02" # Or any other beta version + }, + custom_llm_provider="anthropic" +) +``` + +The default beta version is configured in `litellm.constants.ANTHROPIC_SKILLS_API_BETA_VERSION`. + +## Error Handling + +All Skills API functions follow LiteLLM's standard error handling: + +```python +import litellm + +try: + skill = litellm.create_skill( + display_title="My Skill", + custom_llm_provider="anthropic" + ) +except litellm.exceptions.AuthenticationError as e: + print(f"Authentication failed: {e}") +except litellm.exceptions.RateLimitError as e: + print(f"Rate limit exceeded: {e}") +except litellm.exceptions.APIError as e: + print(f"API error: {e}") +``` + +## Contributing + +To add support for Skills API to a new provider: + +1. Create provider-specific configuration class inheriting from `BaseSkillsAPIConfig` +2. Implement all abstract methods for request/response transformations +3. Register the config in `ProviderConfigManager.get_provider_skills_api_config()` +4. Add appropriate tests + +## Related Documentation + +- [Anthropic Skills API Documentation](https://platform.claude.com/docs/en/api/beta/skills/create) +- [LiteLLM Responses API](../../../responses/) +- [Provider Configuration System](../../base_llm/) + +## Support + +For issues or questions: +- GitHub Issues: https://github.com/BerriAI/litellm/issues +- Discord: https://discord.gg/wuPM9dRgDw diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index e7aa93ac882..994afa26e9c 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -1020,7 +1020,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): headers: dict, client=None, timeout=None, - ) -> litellm.ImageResponse: + ) -> ImageResponse: response: Optional[dict] = None try: diff --git a/litellm/llms/azure/chat/gpt_5_transformation.py b/litellm/llms/azure/chat/gpt_5_transformation.py index 209475730f8..87f81d117f0 100644 --- a/litellm/llms/azure/chat/gpt_5_transformation.py +++ b/litellm/llms/azure/chat/gpt_5_transformation.py @@ -40,7 +40,11 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): or optional_params.get("reasoning_effort") ) - if reasoning_effort_value == "none": + # gpt-5.1 supports reasoning_effort='none', but other gpt-5 models don't + # See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/reasoning + is_gpt_5_1 = self.is_model_gpt_5_1_model(model) + + if reasoning_effort_value == "none" and not is_gpt_5_1: if litellm.drop_params is True or ( drop_params is not None and drop_params is True ): @@ -54,7 +58,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): raise UnsupportedParamsError( status_code=400, message=( - "Azure OpenAI does not support reasoning_effort='none'. " + "Azure OpenAI does not support reasoning_effort='none' for this model. " "Supported values are: 'low', 'medium', and 'high'. " "To drop this parameter, set `litellm.drop_params=True` or for proxy:\n\n" "`litellm_settings:\n drop_params: true`\n" @@ -70,7 +74,8 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): drop_params=drop_params, ) - if result.get("reasoning_effort") == "none": + # Only drop reasoning_effort='none' for non-gpt-5.1 models + if result.get("reasoning_effort") == "none" and not is_gpt_5_1: result.pop("reasoning_effort") return result diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index 0dc42dad43e..217a05c83a4 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -12,6 +12,7 @@ from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from ....litellm_core_utils.realtime_streaming import RealTimeStreaming from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from ..azure import AzureChatCompletion +from litellm._logging import verbose_proxy_logger # BACKEND_WS_URL = "ws://localhost:8080/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01" @@ -51,18 +52,18 @@ class AzureOpenAIRealtime(AzureChatCompletion): Examples: beta/default: "wss://.../openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview" - GA/v1: "wss://.../openai/v1/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview" + GA/v1: "wss://.../openai/v1/realtime?model=gpt-realtime-deployment" """ api_base = api_base.replace("https://", "wss://") # Determine path based on realtime_protocol if realtime_protocol in ("GA", "v1"): - path = "/openai/v1/realtime" + path = "/openai/v1/realtime" + return f"{api_base}{path}?model={model}" else: # Default to beta path for backwards compatibility path = "/openai/realtime" - - return f"{api_base}{path}?api-version={api_version}&deployment={model}" + return f"{api_base}{path}?api-version={api_version}&deployment={model}" async def async_realtime( self, @@ -107,4 +108,5 @@ class AzureOpenAIRealtime(AzureChatCompletion): except websockets.exceptions.InvalidStatusCode as e: # type: ignore await websocket.close(code=e.status_code, reason=str(e)) except Exception: + verbose_proxy_logger.exception("Error in AzureOpenAIRealtime.async_realtime") pass diff --git a/litellm/llms/azure/anthropic/__init__.py b/litellm/llms/azure_ai/anthropic/__init__.py similarity index 100% rename from litellm/llms/azure/anthropic/__init__.py rename to litellm/llms/azure_ai/anthropic/__init__.py diff --git a/litellm/llms/azure/anthropic/handler.py b/litellm/llms/azure_ai/anthropic/handler.py similarity index 93% rename from litellm/llms/azure/anthropic/handler.py rename to litellm/llms/azure_ai/anthropic/handler.py index cf4765190c2..fe4524fd5be 100644 --- a/litellm/llms/azure/anthropic/handler.py +++ b/litellm/llms/azure_ai/anthropic/handler.py @@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Callable, Union import httpx -import litellm from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -55,7 +54,6 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion): Completion method that uses Azure authentication instead of Anthropic's x-api-key. All other logic is the same as AnthropicChatCompletion. """ - from litellm.utils import ProviderConfigManager optional_params = copy.deepcopy(optional_params) stream = optional_params.pop("stream", None) @@ -64,8 +62,10 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion): _is_function_call = False messages = copy.deepcopy(messages) - # Use AzureAnthropicConfig instead of AnthropicConfig - headers = AzureAnthropicConfig().validate_environment( + # Use AzureAnthropicConfig for both azure_anthropic and azure_ai Claude models + config = AzureAnthropicConfig() + + headers = config.validate_environment( api_key=api_key, headers=headers, model=model, @@ -74,15 +74,6 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion): litellm_params=litellm_params, ) - config = ProviderConfigManager.get_provider_chat_config( - model=model, - provider=litellm.types.utils.LlmProviders(custom_llm_provider), - ) - if config is None: - raise ValueError( - f"Provider config not found for model: {model} and provider: {custom_llm_provider}" - ) - data = config.transform_request( model=model, messages=messages, @@ -183,7 +174,7 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion): return CustomStreamWrapper( completion_stream=completion_stream, model=model, - custom_llm_provider="azure_anthropic", + custom_llm_provider="azure_ai", logging_obj=logging_obj, _response_headers=process_anthropic_headers(response_headers), ) diff --git a/litellm/llms/azure/anthropic/messages_transformation.py b/litellm/llms/azure_ai/anthropic/messages_transformation.py similarity index 100% rename from litellm/llms/azure/anthropic/messages_transformation.py rename to litellm/llms/azure_ai/anthropic/messages_transformation.py diff --git a/litellm/llms/azure/anthropic/transformation.py b/litellm/llms/azure_ai/anthropic/transformation.py similarity index 82% rename from litellm/llms/azure/anthropic/transformation.py rename to litellm/llms/azure_ai/anthropic/transformation.py index 9bc4f130563..150ad0a48b2 100644 --- a/litellm/llms/azure/anthropic/transformation.py +++ b/litellm/llms/azure_ai/anthropic/transformation.py @@ -21,7 +21,7 @@ class AzureAnthropicConfig(AnthropicConfig): @property def custom_llm_provider(self) -> Optional[str]: - return "azure_anthropic" + return "azure_ai" def validate_environment( self, @@ -94,3 +94,29 @@ class AzureAnthropicConfig(AnthropicConfig): return headers + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform request using parent AnthropicConfig, then remove extra_body if present. + Azure Anthropic doesn't support extra_body parameter. + """ + # Call parent transform_request + data = super().transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + # Remove extra_body if present (Azure Anthropic doesn't support it) + data.pop("extra_body", None) + + return data + diff --git a/litellm/llms/azure_ai/embed/handler.py b/litellm/llms/azure_ai/embed/handler.py index 13b8cc4cf29..67733d1ccb5 100644 --- a/litellm/llms/azure_ai/embed/handler.py +++ b/litellm/llms/azure_ai/embed/handler.py @@ -58,7 +58,7 @@ class AzureAIEmbedding(OpenAIChatCompletion): data: ImageEmbeddingRequest, timeout: float, logging_obj, - model_response: litellm.EmbeddingResponse, + model_response: EmbeddingResponse, optional_params: dict, api_key: Optional[str], api_base: Optional[str], @@ -138,7 +138,7 @@ class AzureAIEmbedding(OpenAIChatCompletion): input: List, timeout: float, logging_obj, - model_response: litellm.EmbeddingResponse, + model_response: EmbeddingResponse, optional_params: dict, api_key: Optional[str] = None, api_base: Optional[str] = None, diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index 926ad59cee4..7106c207bd6 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -1,12 +1,57 @@ from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional if TYPE_CHECKING: from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.proxy._types import UserAPIKeyAuth class BaseTranslation(ABC): + @staticmethod + def transform_user_api_key_dict_to_metadata( + user_api_key_dict: Optional[Any], + ) -> Dict[str, Any]: + """ + Transform user_api_key_dict to a metadata dict with prefixed keys. + + Converts keys like 'user_id' to 'user_api_key_user_id' to clearly indicate + the source of the metadata. + + Args: + user_api_key_dict: UserAPIKeyAuth object or dict with user information + + Returns: + Dict with keys prefixed with 'user_api_key_' + """ + if user_api_key_dict is None: + return {} + + # Convert to dict if it's a Pydantic object + user_dict = ( + user_api_key_dict.model_dump() + if hasattr(user_api_key_dict, "model_dump") + else user_api_key_dict + ) + + if not isinstance(user_dict, dict): + return {} + + # Transform keys to be prefixed with 'user_api_key_' + transformed = {} + for key, value in user_dict.items(): + # Skip None values and internal fields + if value is None or key.startswith("_"): + continue + + # If key already has the prefix, use as-is, otherwise add prefix + if key.startswith("user_api_key_"): + transformed[key] = value + else: + transformed[f"user_api_key_{key}"] = value + + return transformed + @abstractmethod async def process_input_messages( self, @@ -14,6 +59,11 @@ class BaseTranslation(ABC): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None, ) -> Any: + """ + Process input messages with guardrails. + + Note: user_api_key_dict metadata should be available in the data dict. + """ pass @abstractmethod @@ -22,5 +72,29 @@ class BaseTranslation(ABC): response: Any, guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None, + user_api_key_dict: Optional["UserAPIKeyAuth"] = None, ) -> Any: + """ + Process output response with guardrails. + + Args: + response: The response object from the LLM + guardrail_to_apply: The guardrail instance to apply + litellm_logging_obj: Optional logging object + user_api_key_dict: User API key metadata (passed separately since response doesn't contain it) + """ pass + + async def process_output_streaming_response( + self, + responses_so_far: List[Any], + guardrail_to_apply: "CustomGuardrail", + litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None, + user_api_key_dict: Optional["UserAPIKeyAuth"] = None, + ) -> Any: + """ + Process output streaming response with guardrails. + + Optional to override in subclasses. + """ + return responses_so_far diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index 72e270428ac..816b93edd20 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -353,6 +353,10 @@ class BaseAWSLLM: model_id = BaseAWSLLM._get_model_id_from_model_with_spec( model_id, spec="deepseek_r1" ) + elif provider == "openai" and "openai/" in model_id: + model_id = BaseAWSLLM._get_model_id_from_model_with_spec( + model_id, spec="openai" + ) return model_id @staticmethod @@ -387,9 +391,16 @@ class BaseAWSLLM: Handles scenarios like: 1. model=cohere.embed-english-v3:0 -> Returns `cohere` 2. model=amazon.titan-embed-text-v1 -> Returns `amazon` - 3. model=us.twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs` - 4. model=twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs` + 3. model=amazon.nova-2-multimodal-embeddings-v1:0 -> Returns `nova` + 4. model=us.twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs` + 5. model=twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs` """ + # Special case: Check for "nova" in model name first (before "amazon") + # This handles amazon.nova-* models + if "nova" in model.lower(): + if "nova" in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL): + return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, "nova") + # Handle regional models like us.twelvelabs.marengo-embed-2-7-v1:0 if "." in model: parts = model.split(".") diff --git a/litellm/llms/bedrock/batches/handler.py b/litellm/llms/bedrock/batches/handler.py new file mode 100644 index 00000000000..4a26bd43348 --- /dev/null +++ b/litellm/llms/bedrock/batches/handler.py @@ -0,0 +1,96 @@ +from openai.types.batch import BatchRequestCounts +from openai.types.batch import Metadata as OpenAIBatchMetadata + +from litellm.types.utils import LiteLLMBatch + + +class BedrockBatchesHandler: + """ + Handler for Bedrock Batches. + + Specific providers/models needed some special handling. + + E.g. Twelve Labs Embedding Async Invoke + """ + @staticmethod + def _handle_async_invoke_status( + batch_id: str, aws_region_name: str, logging_obj=None, **kwargs + ) -> "LiteLLMBatch": + """ + Handle async invoke status check for AWS Bedrock. + + This is for Twelve Labs Embedding Async Invoke. + + Args: + batch_id: The async invoke ARN + aws_region_name: AWS region name + **kwargs: Additional parameters + + Returns: + dict: Status information including status, output_file_id (S3 URL), etc. + """ + import asyncio + + from litellm.llms.bedrock.embed.embedding import BedrockEmbedding + + async def _async_get_status(): + # Create embedding handler instance + embedding_handler = BedrockEmbedding() + + # Get the status of the async invoke job + status_response = await embedding_handler._get_async_invoke_status( + invocation_arn=batch_id, + aws_region_name=aws_region_name, + logging_obj=logging_obj, + **kwargs, + ) + + # Transform response to a LiteLLMBatch object + from litellm.types.utils import LiteLLMBatch + + openai_batch_metadata: OpenAIBatchMetadata = { + "output_file_id": status_response["outputDataConfig"][ + "s3OutputDataConfig" + ]["s3Uri"], + "failure_message": status_response.get("failureMessage") or "", + "model_arn": status_response["modelArn"], + } + + result = LiteLLMBatch( + id=status_response["invocationArn"], + object="batch", + status=status_response["status"], + created_at=status_response["submitTime"], + in_progress_at=status_response["lastModifiedTime"], + completed_at=status_response.get("endTime"), + failed_at=status_response.get("endTime") + if status_response["status"] == "failed" + else None, + request_counts=BatchRequestCounts( + total=1, + completed=1 if status_response["status"] == "completed" else 0, + failed=1 if status_response["status"] == "failed" else 0, + ), + metadata=openai_batch_metadata, + completion_window="24h", + endpoint="/v1/embeddings", + input_file_id="", + ) + + return result + + # Since this function is called from within an async context via run_in_executor, + # we need to create a new event loop in a thread to avoid conflicts + import concurrent.futures + + def run_in_thread(): + new_loop = asyncio.new_event_loop() + asyncio.set_event_loop(new_loop) + try: + return new_loop.run_until_complete(_async_get_status()) + finally: + new_loop.close() + + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit(run_in_thread) + return future.result() diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index fd1f6f0c893..d5bd054118d 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -29,6 +29,7 @@ def make_sync_call( logging_obj: LiteLLMLoggingObject, json_mode: Optional[bool] = False, fake_stream: bool = False, + stream_chunk_size: int = 1024, ): if client is None: client = _get_httpx_client() # Create a new client if none provided @@ -66,7 +67,7 @@ def make_sync_call( ) else: decoder = AWSEventStreamDecoder(model=model) - completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=1024)) + completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size)) # LOGGING logging_obj.post_call( @@ -102,6 +103,7 @@ class BedrockConverseLLM(BaseAWSLLM): fake_stream: bool = False, json_mode: Optional[bool] = False, api_key: Optional[str] = None, + stream_chunk_size: int = 1024, ) -> CustomStreamWrapper: request_data = await litellm.AmazonConverseConfig()._async_transform_request( model=model, @@ -143,6 +145,7 @@ class BedrockConverseLLM(BaseAWSLLM): logging_obj=logging_obj, fake_stream=fake_stream, json_mode=json_mode, + stream_chunk_size=stream_chunk_size, ) streaming_response = CustomStreamWrapper( completion_stream=completion_stream, @@ -260,6 +263,7 @@ class BedrockConverseLLM(BaseAWSLLM): ): ## SETUP ## stream = optional_params.pop("stream", None) + stream_chunk_size = optional_params.pop("stream_chunk_size", 1024) unencoded_model_id = optional_params.pop("model_id", None) fake_stream = optional_params.pop("fake_stream", False) json_mode = optional_params.get("json_mode", False) @@ -356,7 +360,8 @@ class BedrockConverseLLM(BaseAWSLLM): json_mode=json_mode, fake_stream=fake_stream, credentials=credentials, - api_key=api_key + api_key=api_key, + stream_chunk_size=stream_chunk_size, ) # type: ignore ### ASYNC COMPLETION return self.async_completion( @@ -433,6 +438,7 @@ class BedrockConverseLLM(BaseAWSLLM): logging_obj=logging_obj, json_mode=json_mode, fake_stream=fake_stream, + stream_chunk_size=stream_chunk_size, ) streaming_response = CustomStreamWrapper( completion_stream=completion_stream, diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 9e001f533d1..2a1d7f2e3a3 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -246,6 +246,93 @@ class AmazonConverseConfig(BaseConfig): llm_provider="bedrock", ) + def _is_nova_lite_2_model(self, model: str) -> bool: + """ + Check if the model is a Nova Lite 2 model that supports reasoningConfig. + + Nova Lite 2 models use a different reasoning configuration structure compared to + Anthropic's thinking parameter and GPT-OSS's reasoning_effort parameter. + + Supported models: + - amazon.nova-2-lite-v1:0 + - us.amazon.nova-2-lite-v1:0 + - eu.amazon.nova-2-lite-v1:0 + - apac.amazon.nova-2-lite-v1:0 + + Args: + model: The model identifier + + Returns: + True if the model is a Nova Lite 2 model, False otherwise + + Examples: + >>> config = AmazonConverseConfig() + >>> config._is_nova_lite_2_model("amazon.nova-2-lite-v1:0") + True + >>> config._is_nova_lite_2_model("us.amazon.nova-2-lite-v1:0") + True + >>> config._is_nova_lite_2_model("amazon.nova-pro-1-5-v1:0") + False + >>> config._is_nova_lite_2_model("amazon.nova-pro-v1:0") + False + """ + # Remove regional prefix if present (us., eu., apac.) + model_without_region = model + for prefix in ["us.", "eu.", "apac."]: + if model.startswith(prefix): + model_without_region = model[len(prefix) :] + break + + # Check if the model is specifically Nova Lite 2 + return "nova-2-lite" in model_without_region + + def _transform_reasoning_effort_to_reasoning_config( + self, reasoning_effort: str + ) -> dict: + """ + Transform reasoning_effort parameter to Nova 2 reasoningConfig structure. + + Nova 2 models use a reasoningConfig structure in additionalModelRequestFields + that differs from both Anthropic's thinking parameter and GPT-OSS's reasoning_effort. + + Args: + reasoning_effort: The reasoning effort level, must be "low" or "high" + + Returns: + dict: A dictionary containing the reasoningConfig structure: + { + "reasoningConfig": { + "type": "enabled", + "maxReasoningEffort": "low" | "medium" |"high" + } + } + + Raises: + BadRequestError: If reasoning_effort is not "low", "medium" or "high" + + Examples: + >>> config = AmazonConverseConfig() + >>> config._transform_reasoning_effort_to_reasoning_config("high") + {'reasoningConfig': {'type': 'enabled', 'maxReasoningEffort': 'high'}} + >>> config._transform_reasoning_effort_to_reasoning_config("low") + {'reasoningConfig': {'type': 'enabled', 'maxReasoningEffort': 'low'}} + """ + valid_values = ["low", "medium", "high"] + if reasoning_effort not in valid_values: + raise litellm.exceptions.BadRequestError( + message=f"Invalid reasoning_effort value '{reasoning_effort}' for Nova 2 models. " + f"Supported values: {valid_values}", + model="amazon.nova-2-lite-v1:0", + llm_provider="bedrock_converse", + ) + + return { + "reasoningConfig": { + "type": "enabled", + "maxReasoningEffort": reasoning_effort, + } + } + def get_supported_openai_params(self, model: str) -> List[str]: from litellm.utils import supports_function_calling @@ -299,6 +386,10 @@ class AmazonConverseConfig(BaseConfig): if "gpt-oss" in model: supported_params.append("reasoning_effort") + elif self._is_nova_lite_2_model(model): + # Nova Lite 2 models support reasoning_effort (transformed to reasoningConfig) + # These models use a different reasoning structure than Anthropic's thinking parameter + supported_params.append("reasoning_effort") elif ( "claude-3-7" in model or "claude-sonnet-4" in model @@ -564,6 +655,12 @@ class AmazonConverseConfig(BaseConfig): # GPT-OSS models: keep reasoning_effort as-is # It will be passed through to additionalModelRequestFields optional_params["reasoning_effort"] = value + elif self._is_nova_lite_2_model(model): + # Nova Lite 2 models: transform to reasoningConfig + reasoning_config = ( + self._transform_reasoning_effort_to_reasoning_config(value) + ) + optional_params.update(reasoning_config) else: # Anthropic and other models: convert to thinking parameter optional_params["thinking"] = AnthropicConfig._map_reasoning_effort( @@ -574,8 +671,9 @@ class AmazonConverseConfig(BaseConfig): self._validate_request_metadata(value) # type: ignore optional_params["requestMetadata"] = value - # Only update thinking tokens for non-GPT-OSS models - if "gpt-oss" not in model: + # Only update thinking tokens for non-GPT-OSS models and non-Nova-Lite-2 models + # Nova Lite 2 handles token budgeting differently through reasoningConfig + if "gpt-oss" not in model and not self._is_nova_lite_2_model(model): self.update_optional_params_with_thinking_tokens( non_default_params=non_default_params, optional_params=optional_params ) @@ -829,11 +927,21 @@ class AmazonConverseConfig(BaseConfig): user_betas = get_anthropic_beta_from_headers(headers) anthropic_beta_list.extend(user_betas) + # Filter out tool search tools - Bedrock Converse API doesn't support them + filtered_tools = [] + if original_tools: + for tool in original_tools: + tool_type = tool.get("type", "") + if tool_type in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"): + # Tool search not supported in Converse API - skip it + continue + filtered_tools.append(tool) + # Only separate tools if computer use tools are actually present - if original_tools and self.is_computer_use_tool_used(original_tools, model): + if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model): # Separate computer use tools from regular function tools computer_use_tools, regular_tools = self._separate_computer_use_tools( - original_tools, model + filtered_tools, model ) # Process regular function tools using existing logic @@ -849,10 +957,13 @@ class AmazonConverseConfig(BaseConfig): additional_request_params["tools"] = transformed_computer_tools else: # No computer use tools, process all tools as regular tools - bedrock_tools = _bedrock_tools_pt(original_tools) + bedrock_tools = _bedrock_tools_pt(filtered_tools) # Set anthropic_beta in additional_request_params if we have any beta features - if anthropic_beta_list: + # ONLY apply to Anthropic/Claude models - other models (e.g., Qwen, Llama) don't support this field + # and will error with "unknown variant anthropic_beta" if included + base_model = BedrockModelInfo.get_base_model(model) + if anthropic_beta_list and base_model.startswith("anthropic"): # Remove duplicates while preserving order unique_betas = [] seen = set() diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index b35e86cabd2..49292545208 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -73,6 +73,9 @@ bedrock_tool_name_mappings: InMemoryCache = InMemoryCache( max_size_in_memory=50, default_ttl=600 ) from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig +from litellm.llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import ( + AmazonBedrockOpenAIConfig, +) converse_config = AmazonConverseConfig() @@ -189,6 +192,7 @@ async def make_call( fake_stream: bool = False, json_mode: Optional[bool] = False, bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None, + stream_chunk_size: int = 1024, ): try: if client is None: @@ -232,7 +236,7 @@ async def make_call( json_mode=json_mode, ) completion_stream = decoder.aiter_bytes( - response.aiter_bytes(chunk_size=1024) + response.aiter_bytes(chunk_size=stream_chunk_size) ) elif bedrock_invoke_provider == "deepseek_r1": decoder = AmazonDeepSeekR1StreamDecoder( @@ -240,12 +244,12 @@ async def make_call( sync_stream=False, ) completion_stream = decoder.aiter_bytes( - response.aiter_bytes(chunk_size=1024) + response.aiter_bytes(chunk_size=stream_chunk_size) ) else: decoder = AWSEventStreamDecoder(model=model) completion_stream = decoder.aiter_bytes( - response.aiter_bytes(chunk_size=1024) + response.aiter_bytes(chunk_size=stream_chunk_size) ) # LOGGING @@ -278,6 +282,7 @@ def make_sync_call( fake_stream: bool = False, json_mode: Optional[bool] = False, bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None, + stream_chunk_size: int = 1024, ): try: if client is None: @@ -318,16 +323,16 @@ def make_sync_call( sync_stream=True, json_mode=json_mode, ) - completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=1024)) + completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size)) elif bedrock_invoke_provider == "deepseek_r1": decoder = AmazonDeepSeekR1StreamDecoder( model=model, sync_stream=True, ) - completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=1024)) + completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size)) else: decoder = AWSEventStreamDecoder(model=model) - completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=1024)) + completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size)) # LOGGING logging_obj.post_call( @@ -401,6 +406,10 @@ class BedrockLLM(BaseAWSLLM): prompt = prompt_factory( model=model, messages=messages, custom_llm_provider="bedrock" ) + elif provider == "openai": + # OpenAI uses messages directly, no prompt conversion needed + # Return empty prompt as it won't be used + prompt = "" elif provider == "cohere": prompt, chat_history = cohere_message_pt(messages=messages) else: @@ -578,6 +587,30 @@ class BedrockLLM(BaseAWSLLM): ) elif provider == "meta" or provider == "llama": outputText = completion_response["generation"] + elif provider == "openai": + # OpenAI imported models use OpenAI Chat Completions format + if "choices" in completion_response and len(completion_response["choices"]) > 0: + choice = completion_response["choices"][0] + if "message" in choice: + outputText = choice["message"].get("content") + elif "text" in choice: # fallback for completion format + outputText = choice["text"] + + # Set finish reason + if "finish_reason" in choice: + model_response.choices[0].finish_reason = map_finish_reason( + choice["finish_reason"] + ) + + # Set usage if available + if "usage" in completion_response: + usage = completion_response["usage"] + _usage = litellm.Usage( + prompt_tokens=usage.get("prompt_tokens", 0), + completion_tokens=usage.get("completion_tokens", 0), + total_tokens=usage.get("total_tokens", 0), + ) + setattr(model_response, "usage", _usage) elif provider == "mistral": outputText = completion_response["outputs"][0]["text"] model_response.choices[0].finish_reason = completion_response[ @@ -641,33 +674,39 @@ class BedrockLLM(BaseAWSLLM): ) ## CALCULATING USAGE - bedrock returns usage in the headers - bedrock_input_tokens = response.headers.get( - "x-amzn-bedrock-input-token-count", None - ) - bedrock_output_tokens = response.headers.get( - "x-amzn-bedrock-output-token-count", None - ) - - prompt_tokens = int( - bedrock_input_tokens or litellm.token_counter(messages=messages) - ) - - completion_tokens = int( - bedrock_output_tokens - or litellm.token_counter( - text=model_response.choices[0].message.content, # type: ignore - count_response_tokens=True, + # Skip if usage was already set (e.g., from JSON response for OpenAI provider) + if not hasattr(model_response, "usage") or getattr(model_response, "usage", None) is None: + bedrock_input_tokens = response.headers.get( + "x-amzn-bedrock-input-token-count", None + ) + bedrock_output_tokens = response.headers.get( + "x-amzn-bedrock-output-token-count", None ) - ) - model_response.created = int(time.time()) - model_response.model = model - usage = Usage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - ) - setattr(model_response, "usage", usage) + prompt_tokens = int( + bedrock_input_tokens or litellm.token_counter(messages=messages) + ) + + completion_tokens = int( + bedrock_output_tokens + or litellm.token_counter( + text=model_response.choices[0].message.content, # type: ignore + count_response_tokens=True, + ) + ) + + model_response.created = int(time.time()) + model_response.model = model + usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + setattr(model_response, "usage", usage) + else: + # Ensure created and model are set even if usage was already set + model_response.created = int(time.time()) + model_response.model = model return model_response @@ -698,6 +737,7 @@ class BedrockLLM(BaseAWSLLM): ## SETUP ## stream = optional_params.pop("stream", None) + stream_chunk_size = optional_params.pop("stream_chunk_size", 1024) provider = self.get_bedrock_invoke_provider(model) modelId = self.get_bedrock_model_id( @@ -895,6 +935,20 @@ class BedrockLLM(BaseAWSLLM): ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in inference_params[k] = v data = json.dumps({"prompt": prompt, **inference_params}) + elif provider == "openai": + ## OpenAI imported models use OpenAI Chat Completions format (messages-based) + # Use AmazonBedrockOpenAIConfig for proper OpenAI transformation + openai_config = AmazonBedrockOpenAIConfig() + supported_params = openai_config.get_supported_openai_params(model=model) + + # Filter to only supported OpenAI params + filtered_params = { + k: v for k, v in inference_params.items() + if k in supported_params + } + + # OpenAI uses messages format, not prompt + data = json.dumps({"messages": messages, **filtered_params}) else: ## LOGGING logging_obj.pre_call( @@ -958,6 +1012,7 @@ class BedrockLLM(BaseAWSLLM): headers=prepped.headers, timeout=timeout, client=client, + stream_chunk_size=stream_chunk_size, ) # type: ignore ### ASYNC COMPLETION return self.async_completion( @@ -1003,7 +1058,7 @@ class BedrockLLM(BaseAWSLLM): decoder = AWSEventStreamDecoder(model=model) - completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=1024)) + completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size)) streaming_response = CustomStreamWrapper( completion_stream=completion_stream, model=model, @@ -1123,6 +1178,7 @@ class BedrockLLM(BaseAWSLLM): logger_fn=None, headers={}, client: Optional[AsyncHTTPHandler] = None, + stream_chunk_size: int = 1024, ) -> CustomStreamWrapper: # The call is not made here; instead, we prepare the necessary objects for the stream. @@ -1138,6 +1194,7 @@ class BedrockLLM(BaseAWSLLM): messages=messages, logging_obj=logging_obj, fake_stream=True if "ai21" in api_base else False, + stream_chunk_size=stream_chunk_size, ), model=model, custom_llm_provider="bedrock", diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py index a81d55f0ad2..3506c8f1cc0 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py @@ -10,7 +10,6 @@ from typing import Any, List, Optional import httpx -import litellm from litellm.litellm_core_utils.litellm_logging import Logging from litellm.types.llms.bedrock import BedrockInvokeNovaRequest from litellm.types.llms.openai import AllMessageValues @@ -80,7 +79,7 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig): encoding: Any, api_key: Optional[str] = None, json_mode: Optional[bool] = None, - ) -> litellm.ModelResponse: + ) -> ModelResponse: return AmazonConverseConfig.transform_response( self, model, diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py new file mode 100644 index 00000000000..c532d8ea27c --- /dev/null +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py @@ -0,0 +1,98 @@ +""" +Handles transforming requests for `bedrock/invoke/{qwen2} models` + +Inherits from `AmazonQwen3Config` since Qwen2 and Qwen3 architectures are mostly similar. +The main difference is in the response format: Qwen2 uses "text" field while Qwen3 uses "generation" field. + +Qwen2 + Invoke API Tutorial: https://docs.aws.amazon.com/bedrock/latest/userguide/invoke-imported-model.html +""" + +from typing import Any, List, Optional + +import httpx + +from litellm.llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import ( + AmazonQwen3Config, +) +from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + LiteLLMLoggingObj, +) +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ModelResponse + + +class AmazonQwen2Config(AmazonQwen3Config): + """ + Config for sending `qwen2` requests to `/bedrock/invoke/` + + Inherits from AmazonQwen3Config since Qwen2 and Qwen3 architectures are mostly similar. + The main difference is in the response format: Qwen2 uses "text" field while Qwen3 uses "generation" field. + + Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/invoke-imported-model.html + """ + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + """ + Transform Qwen2 Bedrock response to OpenAI format + + Qwen2 uses "text" field, but we also support "generation" field for compatibility. + """ + try: + if hasattr(raw_response, 'json'): + response_data = raw_response.json() + else: + response_data = raw_response + + # Extract the generated text - Qwen2 uses "text" field, but also support "generation" for compatibility + generated_text = response_data.get("generation", "") or response_data.get("text", "") + + # Clean up the response (remove assistant start token if present) + if generated_text.startswith("<|im_start|>assistant\n"): + generated_text = generated_text[len("<|im_start|>assistant\n"):] + if generated_text.endswith("<|im_end|>"): + generated_text = generated_text[:-len("<|im_end|>")] + + # Set the content in the existing model_response structure + if hasattr(model_response, 'choices') and len(model_response.choices) > 0: + choice = model_response.choices[0] + if hasattr(choice, 'message'): + choice.message.content = generated_text + choice.finish_reason = "stop" + else: + # Handle streaming choices + choice.delta.content = generated_text + choice.finish_reason = "stop" + + # Set usage information if available in response + if "usage" in response_data: + usage_data = response_data["usage"] + if hasattr(model_response, 'usage'): + model_response.usage.prompt_tokens = usage_data.get("prompt_tokens", 0) + model_response.usage.completion_tokens = usage_data.get("completion_tokens", 0) + model_response.usage.total_tokens = usage_data.get("total_tokens", 0) + + return model_response + + except Exception as e: + if logging_obj: + logging_obj.post_call( + input=messages, + api_key=api_key, + original_response=raw_response, + additional_args={"error": str(e)}, + ) + raise e + diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py new file mode 100644 index 00000000000..62e98f7472f --- /dev/null +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py @@ -0,0 +1,280 @@ +""" +Transforms OpenAI-style requests into TwelveLabs Pegasus 1.2 requests for Bedrock. + +Reference: +https://docs.twelvelabs.io/docs/models/pegasus +""" + +import json +import time +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +import httpx + +import litellm +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.llms.base_llm.base_utils import type_to_response_format_param +from litellm.llms.base_llm.chat.transformation import BaseConfig +from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + AmazonInvokeConfig, +) +from litellm.llms.bedrock.common_utils import BedrockError +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ModelResponse, Usage +from litellm.utils import get_base64_str + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): + """ + Handles transforming OpenAI-style requests into Bedrock InvokeModel requests for + `twelvelabs.pegasus-1-2-v1:0`. + + Pegasus 1.2 requires an `inputPrompt` and a `mediaSource` that either references + an S3 object or a base64-encoded clip. Optional OpenAI params (temperature, + response_format, max_tokens) are translated to the TwelveLabs schema. + """ + + def get_supported_openai_params(self, model: str) -> List[str]: + return [ + "max_tokens", + "max_completion_tokens", + "temperature", + "response_format", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + for param, value in non_default_params.items(): + if param in {"max_tokens", "max_completion_tokens"}: + optional_params["maxOutputTokens"] = value + if param == "temperature": + optional_params["temperature"] = value + if param == "response_format": + optional_params["responseFormat"] = self._normalize_response_format( + value + ) + return optional_params + + def _normalize_response_format(self, value: Any) -> Any: + """Normalize response_format to TwelveLabs format. + + TwelveLabs expects: + { + "jsonSchema": {...} + } + + But OpenAI format is: + { + "type": "json_schema", + "json_schema": { + "name": "...", + "schema": {...} + } + } + """ + if isinstance(value, dict): + # If it has json_schema field, extract and transform it + if "json_schema" in value: + json_schema = value["json_schema"] + # Extract the schema if nested + if isinstance(json_schema, dict) and "schema" in json_schema: + return {"jsonSchema": json_schema["schema"]} + # Otherwise use json_schema directly + return {"jsonSchema": json_schema} + # If it already has jsonSchema, return as is + if "jsonSchema" in value: + return value + # Otherwise return the dict as is + return value + return type_to_response_format_param(response_format=value) or value + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + input_prompt = self._convert_messages_to_prompt(messages=messages) + request_data: Dict[str, Any] = {"inputPrompt": input_prompt} + + media_source = self._build_media_source(optional_params) + if media_source is not None: + request_data["mediaSource"] = media_source + + # Handle temperature and maxOutputTokens + for key in ("temperature", "maxOutputTokens"): + if key in optional_params: + request_data[key] = optional_params.get(key) + + # Handle responseFormat - transform to TwelveLabs format + if "responseFormat" in optional_params: + response_format = optional_params["responseFormat"] + transformed_format = self._normalize_response_format(response_format) + if transformed_format: + request_data["responseFormat"] = transformed_format + + return request_data + + def _build_media_source(self, optional_params: dict) -> Optional[dict]: + direct_source = optional_params.get("mediaSource") or optional_params.get( + "media_source" + ) + if isinstance(direct_source, dict): + return direct_source + + base64_input = optional_params.get("video_base64") or optional_params.get( + "base64_string" + ) + if base64_input: + return {"base64String": get_base64_str(base64_input)} + + s3_uri = ( + optional_params.get("video_s3_uri") + or optional_params.get("s3_uri") + or optional_params.get("media_source_s3_uri") + ) + if s3_uri: + s3_location = {"uri": s3_uri} + bucket_owner = ( + optional_params.get("video_s3_bucket_owner") + or optional_params.get("s3_bucket_owner") + or optional_params.get("media_source_bucket_owner") + ) + if bucket_owner: + s3_location["bucketOwner"] = bucket_owner + return {"s3Location": s3_location} + return None + + def _convert_messages_to_prompt(self, messages: List[AllMessageValues]) -> str: + prompt_parts: List[str] = [] + for message in messages: + role = message.get("role", "user") + content = message.get("content", "") + if isinstance(content, list): + text_fragments = [] + for item in content: + if isinstance(item, dict): + item_type = item.get("type") + if item_type == "text": + text_fragments.append(item.get("text", "")) + elif item_type == "image_url": + text_fragments.append("") + elif item_type == "video_url": + text_fragments.append("