diff --git a/.circleci/config.yml b/.circleci/config.yml index 62c12c1cf92..6dd1177f79c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3339,7 +3339,7 @@ jobs: python -m build twine upload --verbose dist/* - e2e_ui_testing: + ui_build: machine: image: ubuntu-2204:2023.10.1 resource_class: xlarge @@ -3366,6 +3366,48 @@ jobs: # Now source the build script source ./build_ui.sh + - persist_to_workspace: + root: . + paths: + - litellm/proxy/_experimental/out + + ui_unit_tests: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Run UI unit tests (Vitest) + command: | + # Use Node 20 (several deps require >=20) + export NVM_DIR="/opt/circleci/.nvm" + source "$NVM_DIR/nvm.sh" + nvm install 20 + nvm use 20 + + cd ui/litellm-dashboard + npm ci || npm install + + # CI run, with both LCOV (Codecov) and HTML (artifact you can click) + CI=true npm run test -- --run --coverage \ + --coverage.provider=v8 \ + --coverage.reporter=lcov \ + --coverage.reporter=html \ + --coverage.reportsDirectory=coverage/html + + e2e_ui_testing: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - attach_workspace: + at: ~/project - run: name: Upgrade Docker to v24.x (API 1.44+) command: | @@ -3411,24 +3453,6 @@ jobs: name: Install Playwright Browsers command: | npx playwright install - - run: - name: Run UI unit tests (Vitest) - command: | - # Use Node 20 (several deps require >=20) - export NVM_DIR="/opt/circleci/.nvm" - source "$NVM_DIR/nvm.sh" - nvm install 20 - nvm use 20 - - cd ui/litellm-dashboard - npm ci || npm install - - # CI run, with both LCOV (Codecov) and HTML (artifact you can click) - CI=true npm run test -- --run --coverage \ - --coverage.provider=v8 \ - --coverage.reporter=lcov \ - --coverage.reporter=html \ - --coverage.reportsDirectory=coverage/html - run: name: Build Docker image @@ -3633,6 +3657,20 @@ workflows: only: - main - /litellm_.*/ + - ui_build: + filters: + branches: + only: + - main + - /litellm_.*/ + - ui_unit_tests: + requires: + - ui_build + filters: + branches: + only: + - main + - /litellm_.*/ - auth_ui_unit_tests: filters: branches: @@ -3640,6 +3678,8 @@ workflows: - main - /litellm_.*/ - e2e_ui_testing: + requires: + - ui_build filters: branches: only: diff --git a/AGENTS.md b/AGENTS.md index 8e7b5f2bd2e..d72b00f7e14 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -94,6 +94,10 @@ LiteLLM supports MCP for agent workflows: - Support for external MCP servers (Zapier, Jira, Linear, etc.) - See `litellm/experimental_mcp_client/` and `litellm/proxy/_experimental/mcp_server/` +## RUNNING SCRIPTS + +Use `poetry run python script.py` to run Python scripts in the project environment (for non-test files). + ## TESTING CONSIDERATIONS 1. **Provider Tests**: Test against real provider APIs when possible diff --git a/CLAUDE.md b/CLAUDE.md index 50bed6e43e2..15984323394 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -25,6 +25,9 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co - `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file - `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test +### Running Scripts +- `poetry run python script.py` - Run Python scripts (use for non-test files) + ## Architecture Overview LiteLLM is a unified interface for 100+ LLM providers with two main components: diff --git a/Makefile b/Makefile index a79a397f945..1614a58fc7d 100644 --- a/Makefile +++ b/Makefile @@ -34,13 +34,13 @@ install-proxy-dev: # CI-compatible installations (matches GitHub workflows exactly) install-dev-ci: - pip install openai==1.99.5 + pip install openai==2.8.0 poetry install --with dev - pip install openai==1.99.5 + pip install openai==2.8.0 install-proxy-dev-ci: poetry install --with dev,proxy-dev --extras proxy - pip install openai==1.99.5 + pip install openai==2.8.0 install-test-deps: install-proxy-dev poetry run pip install "pytest-retry==1.6.3" diff --git a/VERTEX_ENV_SETUP.md b/VERTEX_ENV_SETUP.md deleted file mode 100644 index 93a631c82f1..00000000000 --- a/VERTEX_ENV_SETUP.md +++ /dev/null @@ -1,261 +0,0 @@ -# Vertex AI Environment Variables Setup Guide - -## Overview - -LiteLLM can load Vertex AI credentials from environment variables instead of storing them in config files. This is more secure and easier to manage for local development. - -## Environment Variables - -LiteLLM looks for these environment variables (in order of precedence): - -### 1. **DEFAULT_VERTEXAI_PROJECT** (Required) -Your GCP project ID that has Vertex AI enabled. - -```bash -export DEFAULT_VERTEXAI_PROJECT="my-gcp-project-id" -``` - -### 2. **DEFAULT_VERTEXAI_LOCATION** (Required) -The region/location for Vertex AI services. - -```bash -export DEFAULT_VERTEXAI_LOCATION="global" -# or -export DEFAULT_VERTEXAI_LOCATION="us-central1" -``` - -Common locations: -- `global` - For Discovery Engine and global services -- `us-central1` - US Central region -- `us-east1` - US East region -- `europe-west1` - Europe West region -- `asia-southeast1` - Asia Southeast region - -### 3. **DEFAULT_GOOGLE_APPLICATION_CREDENTIALS** (Required) -Path to your service account JSON key file. - -```bash -export DEFAULT_GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json" -``` - -### 4. **GOOGLE_APPLICATION_CREDENTIALS** (Fallback) -Standard Google Cloud environment variable (used as fallback). - -```bash -export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json" -``` - -## Quick Setup - -### Option 1: Interactive Script - -```bash -chmod +x setup_vertex_env.sh -source setup_vertex_env.sh -``` - -### Option 2: Manual Setup - -1. **Set environment variables** (for current session): - -```bash -export DEFAULT_VERTEXAI_PROJECT="your-project-id" -export DEFAULT_VERTEXAI_LOCATION="global" -export DEFAULT_GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json" -export GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json" -``` - -2. **Make them persistent** (add to `~/.zshrc` or `~/.bashrc`): - -```bash -echo 'export DEFAULT_VERTEXAI_PROJECT="your-project-id"' >> ~/.zshrc -echo 'export DEFAULT_VERTEXAI_LOCATION="global"' >> ~/.zshrc -echo 'export DEFAULT_GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json"' >> ~/.zshrc -echo 'export GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json"' >> ~/.zshrc -``` - -3. **Reload your shell**: - -```bash -source ~/.zshrc -``` - -## Service Account Setup - -### 1. Create a Service Account - -```bash -gcloud iam service-accounts create litellm-vertex-sa \ - --display-name="LiteLLM Vertex AI Service Account" -``` - -### 2. Grant Necessary Permissions - -For Discovery Engine (vector stores): -```bash -gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \ - --member="serviceAccount:litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com" \ - --role="roles/discoveryengine.viewer" - -gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \ - --member="serviceAccount:litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com" \ - --role="roles/discoveryengine.dataStoreEditor" -``` - -For general Vertex AI: -```bash -gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \ - --member="serviceAccount:litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com" \ - --role="roles/aiplatform.user" -``` - -### 3. Create and Download Key - -```bash -gcloud iam service-accounts keys create ~/service-account-key.json \ - --iam-account=litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com -``` - -## Verify Setup - -### Check Environment Variables - -```bash -python3 << 'EOF' -import os -print("✓ Environment Variables:") -print(f" DEFAULT_VERTEXAI_PROJECT: {os.getenv('DEFAULT_VERTEXAI_PROJECT')}") -print(f" DEFAULT_VERTEXAI_LOCATION: {os.getenv('DEFAULT_VERTEXAI_LOCATION')}") -print(f" DEFAULT_GOOGLE_APPLICATION_CREDENTIALS: {os.getenv('DEFAULT_GOOGLE_APPLICATION_CREDENTIALS')}") -print(f" GOOGLE_APPLICATION_CREDENTIALS: {os.getenv('GOOGLE_APPLICATION_CREDENTIALS')}") - -# Check if credentials file exists -creds_path = os.getenv('DEFAULT_GOOGLE_APPLICATION_CREDENTIALS') -if creds_path and os.path.exists(creds_path): - print(f"\n✅ Credentials file found at: {creds_path}") -else: - print(f"\n❌ Credentials file NOT found at: {creds_path}") -EOF -``` - -### Test Authentication - -```bash -python3 << 'EOF' -import os -import json -from google.oauth2 import service_account -from google.auth.transport.requests import Request - -creds_path = os.getenv('DEFAULT_GOOGLE_APPLICATION_CREDENTIALS') -project = os.getenv('DEFAULT_VERTEXAI_PROJECT') - -try: - # Load credentials - credentials = service_account.Credentials.from_service_account_file( - creds_path, - scopes=['https://www.googleapis.com/auth/cloud-platform'] - ) - - # Get access token - credentials.refresh(Request()) - - print("✅ Authentication successful!") - print(f" Project: {project}") - print(f" Service Account: {credentials.service_account_email}") - print(f" Token expiry: {credentials.expiry}") - -except Exception as e: - print(f"❌ Authentication failed: {e}") -EOF -``` - -## Using with Vector Store Passthrough - -Once your environment is set up, the vector store passthrough will work in two ways: - -### 1. **With Vector Store Config** (Priority 1) -If you have a vector store configured with its own credentials in `litellm_params`, those will be used first: - -```yaml -vector_stores: - - vector_store_id: test-store-123 - custom_llm_provider: vertex_ai - litellm_params: - vertex_project: "specific-project" - vertex_location: "us-central1" - vertex_credentials: "{...}" # Inline credentials -``` - -### 2. **Environment Variables Fallback** (Priority 2) -If the vector store doesn't have explicit credentials, it falls back to your environment variables: - -```yaml -vector_stores: - - vector_store_id: test-store-123 - custom_llm_provider: vertex_ai - # No litellm_params - will use DEFAULT_VERTEXAI_PROJECT, DEFAULT_VERTEXAI_LOCATION, etc. -``` - -### 3. **Model Config Fallback** (Priority 3) -If neither above work, it looks for credentials in your model configuration. - -## Troubleshooting - -### "No credentials found" - -Check that all environment variables are set: -```bash -env | grep -E "(DEFAULT_VERTEXAI|GOOGLE_APPLICATION_CREDENTIALS)" -``` - -### "Authentication failed" - -Verify your service account key is valid: -```bash -cat $DEFAULT_GOOGLE_APPLICATION_CREDENTIALS | python3 -m json.tool -``` - -### "Permission denied" - -Ensure your service account has the necessary roles: -```bash -gcloud projects get-iam-policy YOUR_PROJECT_ID \ - --flatten="bindings[].members" \ - --filter="bindings.members:serviceAccount:litellm-vertex-sa@*" -``` - -### Different Credentials for Different Projects - -If you need to use different credentials for different vector stores, configure them explicitly in the vector store config rather than relying on environment variables. - -## Start LiteLLM Proxy - -Once your environment is configured: - -```bash -# Start the proxy (it will automatically load env vars) -litellm --config proxy_server_config.yaml - -# Or with debug logging -export LITELLM_LOG=DEBUG -litellm --config proxy_server_config.yaml -``` - -You should see logs like: -``` -Vertex: Loading vertex credentials from /path/to/service-account.json -Found credentials for vertex_ai_default -``` - -## Test the Endpoint - -```bash -curl -X POST http://0.0.0.0:4000/vertex_ai/discovery/v1/projects/fake-project/locations/global/dataStores/test-store-123/servingConfigs/default_config:search \ - -H 'Authorization: Bearer YOUR_LITELLM_API_KEY' \ - -H 'Content-Type: application/json' \ - -d '{"query": "test query"}' -``` - -The proxy will use your environment credentials to make the request to Vertex AI! - diff --git a/cookbook/LiteLLM_CometAPI.ipynb b/cookbook/LiteLLM_CometAPI.ipynb index bdd916c5bfe..0a7ab581ae3 100644 --- a/cookbook/LiteLLM_CometAPI.ipynb +++ b/cookbook/LiteLLM_CometAPI.ipynb @@ -28,7 +28,7 @@ "Requirement already satisfied: importlib-metadata>=6.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (8.6.1)\n", "Requirement already satisfied: jinja2<4.0.0,>=3.1.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (3.1.6)\n", "Requirement already satisfied: jsonschema<5.0.0,>=4.22.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (4.25.1)\n", - "Requirement already satisfied: openai>=1.99.5 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.109.1)\n", + "Requirement already satisfied: openai>=2.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.109.1)\n", "Requirement already satisfied: pydantic<3.0.0,>=2.5.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (2.11.10)\n", "Requirement already satisfied: python-dotenv>=0.2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.1.1)\n", "Requirement already satisfied: tiktoken>=0.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.12.0)\n", @@ -50,11 +50,11 @@ "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (2025.9.1)\n", "Requirement already satisfied: referencing>=0.28.4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.36.2)\n", "Requirement already satisfied: rpds-py>=0.7.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.27.1)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (1.9.0)\n", - "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (0.11.0)\n", - "Requirement already satisfied: sniffio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (1.3.1)\n", - "Requirement already satisfied: tqdm>4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (4.67.1)\n", - "Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (4.15.0)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.9.0)\n", + "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (0.11.0)\n", + "Requirement already satisfied: sniffio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.3.1)\n", + "Requirement already satisfied: tqdm>4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.67.1)\n", + "Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.15.0)\n", "Requirement already satisfied: annotated-types>=0.6.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.7.0)\n", "Requirement already satisfied: pydantic-core==2.33.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (2.33.2)\n", "Requirement already satisfied: typing-inspection>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.4.2)\n", diff --git a/cookbook/LiteLLM_HuggingFace.ipynb b/cookbook/LiteLLM_HuggingFace.ipynb index d608c2675a1..bf8482a5f11 100644 --- a/cookbook/LiteLLM_HuggingFace.ipynb +++ b/cookbook/LiteLLM_HuggingFace.ipynb @@ -131,7 +131,7 @@ " {\n", " \"type\": \"image_url\",\n", " \"image_url\": {\n", - " \"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\",\n", + " \"url\": \"https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png\",\n", " },\n", " },\n", " ],\n", diff --git a/deploy/charts/litellm-helm/Chart.yaml b/deploy/charts/litellm-helm/Chart.yaml index aa81e4efecc..eedadebaa8e 100644 --- a/deploy/charts/litellm-helm/Chart.yaml +++ b/deploy/charts/litellm-helm/Chart.yaml @@ -18,7 +18,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.7 +version: 0.4.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml index 243a4ba7d48..f8893a47afe 100644 --- a/deploy/charts/litellm-helm/templates/migrations-job.yaml +++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml @@ -22,6 +22,9 @@ spec: metadata: labels: {{- include "litellm.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} annotations: {{- with .Values.migrationJob.annotations }} {{- toYaml . | nindent 8 }} diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 351c4f6bc48..09b5265191b 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -12,7 +12,10 @@ WORKDIR /app USER root # Install build dependencies -RUN apk add --no-cache gcc python3-dev openssl openssl-dev +RUN apk add --no-cache \ + build-base \ + python3-dev \ + openssl-dev RUN pip install --upgrade pip && \ diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 0cbdf761fe8..3fa0ab69e3b 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -21,11 +21,14 @@ ENV LITELLM_NON_ROOT=true # Build Admin UI RUN mkdir -p /tmp/litellm_ui && \ + npm install -g npm@latest && \ + npm cache clean --force && \ cd ui/litellm-dashboard && \ if [ -f "../../enterprise/enterprise_ui/enterprise_colors.json" ]; then \ cp ../../enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \ fi && \ - npm install && \ + rm -f package-lock.json && \ + npm install --legacy-peer-deps && \ npm run build && \ cp -r ./out/* /tmp/litellm_ui/ && \ cd /tmp/litellm_ui && \ diff --git a/docs/my-website/blog/authors.yml b/docs/my-website/blog/authors.yml new file mode 100644 index 00000000000..2a49a736333 --- /dev/null +++ b/docs/my-website/blog/authors.yml @@ -0,0 +1,24 @@ +litellm: + name: LiteLLM Team + title: LiteLLM Core Team + url: https://github.com/BerriAI/litellm + image_url: https://github.com/BerriAI.png + +krrish: + name: Krrish Dholakia + title: CEO, LiteLLM + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + +ishaan: + name: Ishaan Jaffer + title: CTO, LiteLLM + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + +# Alias for typo in name +ishaan-alt: + name: Ishaan Jaff + title: CTO, LiteLLM + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg diff --git a/docs/my-website/blog/gemini_3/index.md b/docs/my-website/blog/gemini_3/index.md new file mode 100644 index 00000000000..1b9ff359f3a --- /dev/null +++ b/docs/my-website/blog/gemini_3/index.md @@ -0,0 +1,982 @@ +--- +slug: gemini_3 +title: "DAY 0 Support: Gemini 3 on LiteLLM" +date: 2025-11-19T10:00:00 +authors: + - name: Sameer Kankute + title: SWE @ LiteLLM (LLM Translation) + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1765411200&v=beta&t=c8396f--_lH6Fb_pVvx_jGholPfcl0bvwmNynbNdnII + - name: Krrish Dholakia + title: "CEO, LiteLLM" + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +tags: [gemini, day 0 support, llms] +hide_table_of_contents: false +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +:::info + +This guide covers common questions and best practices for using `gemini-3-pro-preview` with LiteLLM Proxy and SDK. + +::: + +## Quick Start + + + + +```python +from litellm import completion +import os + +os.environ["GEMINI_API_KEY"] = "your-api-key" + +response = completion( + model="gemini/gemini-3-pro-preview", + messages=[{"role": "user", "content": "Hello!"}], + reasoning_effort="low" +) + +print(response.choices[0].message.content) +``` + + + + +**1. Add to config.yaml:** + +```yaml +model_list: + - model_name: gemini-3-pro-preview + litellm_params: + model: gemini/gemini-3-pro-preview + api_key: os.environ/GEMINI_API_KEY +``` + +**2. Start proxy:** + +```bash +litellm --config /path/to/config.yaml +``` + +**3. Make request:** + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gemini-3-pro-preview", + "messages": [{"role": "user", "content": "Hello!"}], + "reasoning_effort": "low" + }' +``` + + + + +## Supported Endpoints + +LiteLLM provides **full end-to-end support** for Gemini 3 Pro Preview on: + +- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint +- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming) +- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint +- ✅ `/v1/generateContent` – [Google Gemini API](https://cloud.google.com/vertex-ai/docs/generative-ai/model-reference/gemini#rest) compatible endpoint (for code, see: `client.models.generate_content(...)`) + +All endpoints support: +- Streaming and non-streaming responses +- Function calling with thought signatures +- Multi-turn conversations +- All Gemini 3-specific features + +## Thought Signatures + +#### What are Thought Signatures? + +Thought signatures are encrypted representations of the model's internal reasoning process. They're essential for maintaining context across multi-turn conversations, especially with function calling. + +#### How Thought Signatures Work + +1. **Automatic Extraction**: When Gemini 3 returns a function call, LiteLLM automatically extracts the `thought_signature` from the response +2. **Storage**: Thought signatures are stored in `provider_specific_fields.thought_signature` of tool calls +3. **Automatic Preservation**: When you include the assistant's message in conversation history, LiteLLM automatically preserves and returns thought signatures to Gemini + +## Example: Multi-Turn Function Calling + +#### Streaming with Thought Signatures + +When using streaming mode with `stream_chunk_builder()`, thought signatures are now automatically preserved: + + + + +```python +import os +import litellm +from litellm import completion + +os.environ["GEMINI_API_KEY"] = "your-api-key" + +MODEL = "gemini/gemini-3-pro-preview" + +messages = [ + {"role": "system", "content": "You are a helpful assistant. Use the calculate tool."}, + {"role": "user", "content": "What is 2+2?"}, +] + +tools = [{ + "type": "function", + "function": { + "name": "calculate", + "description": "Calculate a mathematical expression", + "parameters": { + "type": "object", + "properties": {"expression": {"type": "string"}}, + "required": ["expression"], + }, + }, +}] + +print("Step 1: Sending request with stream=True...") +response = completion( + model=MODEL, + messages=messages, + stream=True, + tools=tools, + reasoning_effort="low" +) + +# Collect all chunks +chunks = [] +for part in response: + chunks.append(part) + +# Reconstruct message using stream_chunk_builder +# Thought signatures are now preserved automatically! +full_response = litellm.stream_chunk_builder(chunks, messages=messages) +print(f"Full response: {full_response}") + +assistant_msg = full_response.choices[0].message + +# ✅ Thought signature is now preserved in provider_specific_fields +if assistant_msg.tool_calls and assistant_msg.tool_calls[0].provider_specific_fields: + thought_sig = assistant_msg.tool_calls[0].provider_specific_fields.get("thought_signature") + print(f"Thought signature preserved: {thought_sig is not None}") + +# Append assistant message (includes thought signatures automatically) +messages.append(assistant_msg) + +# Mock tool execution +messages.append({ + "role": "tool", + "content": "4", + "tool_call_id": assistant_msg.tool_calls[0].id +}) + +print("\nStep 2: Sending tool result back to model...") +response_2 = completion( + model=MODEL, + messages=messages, + stream=True, + tools=tools, + reasoning_effort="low" +) + +for part in response_2: + if part.choices[0].delta.content: + print(part.choices[0].delta.content, end="") +print() # New line +``` + +**Key Points:** +- ✅ `stream_chunk_builder()` now preserves `provider_specific_fields` including thought signatures +- ✅ Thought signatures are automatically included when appending `assistant_msg` to conversation history +- ✅ Multi-turn conversations work seamlessly with streaming + + + + +```python +from openai import OpenAI +import json + +client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000") + +# Define tools +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + } + } +] + +# Step 1: Initial request +messages = [{"role": "user", "content": "What's the weather in Tokyo?"}] + +response = client.chat.completions.create( + model="gemini-3-pro-preview", + messages=messages, + tools=tools, + reasoning_effort="low" +) + +# Step 2: Append assistant message (thought signatures automatically preserved) +messages.append(response.choices[0].message) + +# Step 3: Execute tool and append result +for tool_call in response.choices[0].message.tool_calls: + if tool_call.function.name == "get_weather": + result = {"temperature": 30, "unit": "celsius"} + messages.append({ + "role": "tool", + "content": json.dumps(result), + "tool_call_id": tool_call.id + }) + +# Step 4: Follow-up request (thought signatures automatically included) +response2 = client.chat.completions.create( + model="gemini-3-pro-preview", + messages=messages, + tools=tools, + reasoning_effort="low" +) + +print(response2.choices[0].message.content) +``` + +**Key Points:** +- ✅ Thought signatures are automatically extracted from `response.choices[0].message.tool_calls[].provider_specific_fields.thought_signature` +- ✅ When you append `response.choices[0].message` to your conversation history, thought signatures are automatically preserved +- ✅ You don't need to manually extract or manage thought signatures + + + + +```bash +# Step 1: Initial request +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gemini-3-pro-preview", + "messages": [ + {"role": "user", "content": "What'\''s the weather in Tokyo?"} + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + } + } + ], + "reasoning_effort": "low" + }' +``` + +**Response includes thought signature:** + +```json +{ + "choices": [{ + "message": { + "role": "assistant", + "tool_calls": [{ + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"location\": \"Tokyo\"}" + }, + "provider_specific_fields": { + "thought_signature": "CpcHAdHtim9+q4rstcbvQC0ic4x1/vqQlCJWgE+UZ6dTLYGHMMBkF/AxqL5UmP6SY46uYC8t4BTFiXG5zkw6EMJ..." + } + }] + } + }] +} +``` + +```bash +# Step 2: Follow-up request (include assistant message with thought signature) +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gemini-3-pro-preview", + "messages": [ + {"role": "user", "content": "What'\''s the weather in Tokyo?"}, + { + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"location\": \"Tokyo\"}" + }, + "provider_specific_fields": { + "thought_signature": "CpcHAdHtim9+q4rstcbvQC0ic4x1/vqQlCJWgE+UZ6dTLYGHMMBkF/AxqL5UmP6SY46uYC8t4BTFiXG5zkw6EMJ..." + } + }] + }, + { + "role": "tool", + "content": "{\"temperature\": 30, \"unit\": \"celsius\"}", + "tool_call_id": "call_abc123" + } + ], + "tools": [...], + "reasoning_effort": "low" + }' +``` + + + + +#### Important Notes on Thought Signatures + +1. **Automatic Handling**: LiteLLM automatically extracts and preserves thought signatures. You don't need to manually manage them. + +2. **Parallel Function Calls**: When the model makes parallel function calls, only the **first function call** has a thought signature. + +3. **Sequential Function Calls**: In multi-step function calling, each step's first function call has its own thought signature that must be preserved. + +4. **Required for Context**: Thought signatures are essential for maintaining reasoning context. Without them, the model may lose context of its previous reasoning. + +## Conversation History: Switching from Non-Gemini-3 Models + +#### Common Question: Will switching from a non-Gemini-3 model to Gemini-3 break conversation history? + +**Answer: No!** LiteLLM automatically handles this by adding dummy thought signatures when needed. + +#### How It Works + +When you switch from a model that doesn't use thought signatures (e.g., `gemini-2.5-flash`) to Gemini 3, LiteLLM: + +1. **Detects missing signatures**: Identifies assistant messages with tool calls that lack thought signatures +2. **Adds dummy signature**: Automatically injects a dummy thought signature (`skip_thought_signature_validator`) for compatibility +3. **Maintains conversation flow**: Your conversation history continues to work seamlessly + +#### Example: Switching Models Mid-Conversation + + + + +```python +from openai import OpenAI + +client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000") + +# Step 1: Start with gemini-2.5-flash (no thought signatures) +messages = [{"role": "user", "content": "What's the weather?"}] + +response1 = client.chat.completions.create( + model="gemini-2.5-flash", + messages=messages, + tools=[...], + reasoning_effort="low" +) + +# Append assistant message (no tool call thought signature from gemini-2.5-flash) +messages.append(response1.choices[0].message) + +# Step 2: Switch to gemini-3-pro-preview +# LiteLLM automatically adds dummy thought signature to the previous assistant message +response2 = client.chat.completions.create( + model="gemini-3-pro-preview", # 👈 Switched model + messages=messages, # 👈 Same conversation history + tools=[...], + reasoning_effort="low" +) + +# ✅ Works seamlessly! No errors, no breaking changes +print(response2.choices[0].message.content) +``` + + + + +```bash +# Step 1: Start with gemini-2.5-flash +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gemini-2.5-flash", + "messages": [{"role": "user", "content": "What'\''s the weather?"}], + "tools": [...], + "reasoning_effort": "low" + }' + +# Step 2: Switch to gemini-3-pro-preview with same conversation history +# LiteLLM automatically handles the missing thought signature +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gemini-3-pro-preview", # 👈 Switched model + "messages": [ + {"role": "user", "content": "What'\''s the weather?"}, + { + "role": "assistant", + "tool_calls": [...] # 👈 No thought_signature from gemini-2.5-flash + } + ], + "tools": [...], + "reasoning_effort": "low" + }' +# ✅ Works! LiteLLM adds dummy signature automatically +``` + + + + +#### Dummy Signature Details + +The dummy signature used is: `base64("skip_thought_signature_validator")` + +This is the recommended approach by Google for handling conversation history from models that don't support thought signatures. It allows Gemini 3 to: +- Accept the conversation history without validation errors +- Continue the conversation seamlessly +- Maintain context across model switches + +## Thinking Level Parameter + +#### How `reasoning_effort` Maps to `thinking_level` + +For Gemini 3 Pro Preview, LiteLLM automatically maps `reasoning_effort` to the new `thinking_level` parameter: + +| `reasoning_effort` | `thinking_level` | Notes | +|-------------------|------------------|-------| +| `"minimal"` | `"low"` | Maps to low thinking level | +| `"low"` | `"low"` | Default for most use cases | +| `"medium"` | `"high"` | Medium not available yet, maps to high | +| `"high"` | `"high"` | Maximum reasoning depth | +| `"disable"` | `"low"` | Gemini 3 cannot fully disable thinking | +| `"none"` | `"low"` | Gemini 3 cannot fully disable thinking | + +#### Default Behavior + +If you don't specify `reasoning_effort`, LiteLLM automatically sets `thinking_level="low"` for Gemini 3 models, to avoid high costs. + +### Example Usage + + + + +```python +from litellm import completion + +# Low thinking level (faster, lower cost) +response = completion( + model="gemini/gemini-3-pro-preview", + messages=[{"role": "user", "content": "What's the weather?"}], + reasoning_effort="low" # Maps to thinking_level="low" +) + +# High thinking level (deeper reasoning, higher cost) +response = completion( + model="gemini/gemini-3-pro-preview", + messages=[{"role": "user", "content": "Solve this complex math problem step by step."}], + reasoning_effort="high" # Maps to thinking_level="high" +) +``` + + + + +```bash +# Low thinking level +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gemini-3-pro-preview", + "messages": [{"role": "user", "content": "What'\''s the weather?"}], + "reasoning_effort": "low" + }' + +# High thinking level +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gemini-3-pro-preview", + "messages": [{"role": "user", "content": "Solve this complex problem."}], + "reasoning_effort": "high" + }' +``` + + + + +## Important Notes + +1. **Gemini 3 Cannot Disable Thinking**: Unlike Gemini 2.5 models, Gemini 3 cannot fully disable thinking. Even when you set `reasoning_effort="none"` or `"disable"`, it maps to `thinking_level="low"`. + +2. **Temperature Recommendation**: For Gemini 3 models, LiteLLM defaults `temperature` to `1.0` and strongly recommends keeping it at this default. Setting `temperature < 1.0` can cause: + - Infinite loops + - Degraded reasoning performance + - Failure on complex tasks + +3. **Automatic Defaults**: If you don't specify `reasoning_effort`, LiteLLM automatically sets `thinking_level="low"` for optimal performance. + +## Cost Tracking: Prompt Caching & Context Window + +LiteLLM provides comprehensive cost tracking for Gemini 3 Pro Preview, including support for prompt caching and tiered pricing based on context window size. + +### Prompt Caching Cost Tracking + +Gemini 3 supports prompt caching, which allows you to cache frequently used prompt prefixes to reduce costs. LiteLLM automatically tracks and calculates costs for: + +- **Cache Hit Tokens**: Tokens that are read from cache (charged at a lower rate) +- **Cache Creation Tokens**: Tokens that are written to cache (one-time cost) +- **Text Tokens**: Regular prompt tokens that are processed normally + +#### How It Works + +LiteLLM extracts caching information from the `prompt_tokens_details` field in the usage object: + +```python +{ + "usage": { + "prompt_tokens": 50000, + "completion_tokens": 1000, + "total_tokens": 51000, + "prompt_tokens_details": { + "cached_tokens": 30000, # Cache hit tokens + "cache_creation_tokens": 5000, # Tokens written to cache + "text_tokens": 15000 # Regular processed tokens + } + } +} +``` + +### Context Window Tiered Pricing + +Gemini 3 Pro Preview supports up to 1M tokens of context, with tiered pricing that automatically applies when your prompt exceeds 200k tokens. + +#### Automatic Tier Detection + +LiteLLM automatically detects when your prompt exceeds the 200k token threshold and applies the appropriate tiered pricing: + +```python +from litellm import completion_cost + +# Example: Small prompt (< 200k tokens) +response_small = completion( + model="gemini/gemini-3-pro-preview", + messages=[{"role": "user", "content": "Hello!"}] +) +# Uses base pricing: $0.000002/input token, $0.000012/output token + +# Example: Large prompt (> 200k tokens) +response_large = completion( + model="gemini/gemini-3-pro-preview", + messages=[{"role": "user", "content": "..." * 250000}] # 250k tokens +) +# Automatically uses tiered pricing: $0.000004/input token, $0.000018/output token +``` + +#### Cost Breakdown + +The cost calculation includes: + +1. **Text Processing Cost**: Regular tokens processed at base or tiered rate +2. **Cache Read Cost**: Cached tokens read at discounted rate +3. **Cache Creation Cost**: One-time cost for writing tokens to cache (applies tiered rate if above 200k) +4. **Output Cost**: Generated tokens at base or tiered rate + +### Example: Viewing Cost Breakdown + +You can view the detailed cost breakdown using LiteLLM's cost tracking: + +```python +from litellm import completion, completion_cost + +response = completion( + model="gemini/gemini-3-pro-preview", + messages=[{"role": "user", "content": "Explain prompt caching"}], + caching=True # Enable prompt caching +) + +# Get total cost +total_cost = completion_cost(completion_response=response) +print(f"Total cost: ${total_cost:.6f}") + +# Access usage details +usage = response.usage +print(f"Prompt tokens: {usage.prompt_tokens}") +print(f"Completion tokens: {usage.completion_tokens}") + +# Access caching details +if usage.prompt_tokens_details: + print(f"Cache hit tokens: {usage.prompt_tokens_details.cached_tokens}") + print(f"Cache creation tokens: {usage.prompt_tokens_details.cache_creation_tokens}") + print(f"Text tokens: {usage.prompt_tokens_details.text_tokens}") +``` + +### Cost Optimization Tips + +1. **Use Prompt Caching**: For repeated prompt prefixes, enable caching to reduce costs by up to 90% for cached portions +2. **Monitor Context Size**: Be aware that prompts above 200k tokens use tiered pricing (2x for input, 1.5x for output) +3. **Cache Management**: Cache creation tokens are charged once when writing to cache, then subsequent reads are much cheaper +4. **Track Usage**: Use LiteLLM's built-in cost tracking to monitor spending across different token types + +### Integration with LiteLLM Proxy + +When using LiteLLM Proxy, all cost tracking is automatically logged and available through: + +- **Usage Logs**: Detailed token and cost breakdowns in proxy logs +- **Budget Management**: Set budgets and alerts based on actual usage +- **Analytics Dashboard**: View cost trends and breakdowns by token type + +```yaml +# config.yaml +model_list: + - model_name: gemini-3-pro-preview + litellm_params: + model: gemini/gemini-3-pro-preview + api_key: os.environ/GEMINI_API_KEY + +litellm_settings: + # Enable detailed cost tracking + success_callback: ["langfuse"] # or your preferred logging service +``` + +## Using with Claude Code CLI + +You can use `gemini-3-pro-preview` with **Claude Code CLI** - Anthropic's command-line interface. This allows you to use Gemini 3 Pro Preview with Claude Code's native syntax and workflows. + +### Setup + +**1. Add Gemini 3 Pro Preview to your `config.yaml`:** + +```yaml +model_list: + - model_name: gemini-3-pro-preview + litellm_params: + model: gemini/gemini-3-pro-preview + api_key: os.environ/GEMINI_API_KEY + +litellm_settings: + master_key: os.environ/LITELLM_MASTER_KEY +``` + +**2. Set environment variables:** + +```bash +export GEMINI_API_KEY="your-gemini-api-key" +export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key +``` + +**3. Start LiteLLM Proxy:** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +**4. Configure Claude Code to use LiteLLM Proxy:** + +```bash +export ANTHROPIC_BASE_URL="http://0.0.0.0:4000" +export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY" +``` + +**5. Use Gemini 3 Pro Preview with Claude Code:** + +```bash +# Claude Code will use gemini-3-pro-preview from your LiteLLM proxy +claude --model gemini-3-pro-preview + +``` + +### Example Usage + +Once configured, you can interact with Gemini 3 Pro Preview using Claude Code's native interface: + +```bash +$ claude --model gemini-3-pro-preview +> Explain how thought signatures work in multi-turn conversations. + +# Gemini 3 Pro Preview responds through Claude Code interface +``` + +### Benefits + +- ✅ **Native Claude Code Experience**: Use Gemini 3 Pro Preview with Claude Code's familiar CLI interface +- ✅ **Unified Authentication**: Single API key for all models through LiteLLM proxy +- ✅ **Cost Tracking**: All usage tracked through LiteLLM's centralized logging +- ✅ **Seamless Model Switching**: Easily switch between Claude and Gemini models +- ✅ **Full Feature Support**: All Gemini 3 features (thought signatures, function calling, etc.) work through Claude Code + +### Troubleshooting + +**Claude Code not finding the model:** +- Ensure the model name in Claude Code matches exactly: `gemini-3-pro-preview` +- Verify your proxy is running: `curl http://0.0.0.0:4000/health` +- Check that `ANTHROPIC_BASE_URL` points to your LiteLLM proxy + +**Authentication errors:** +- Verify `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key +- Ensure `GEMINI_API_KEY` is set correctly +- Check LiteLLM proxy logs for detailed error messages + +## Responses API Support + +LiteLLM fully supports the OpenAI Responses API for Gemini 3 Pro Preview, including both streaming and non-streaming modes. The Responses API provides a structured way to handle multi-turn conversations with function calling, and LiteLLM automatically preserves thought signatures throughout the conversation. + +### Example: Using Responses API with Gemini 3 + + + + +```python +from openai import OpenAI +import json + +client = OpenAI() + +# 1. Define a list of callable tools for the model +tools = [ + { + "type": "function", + "name": "get_horoscope", + "description": "Get today's horoscope for an astrological sign.", + "parameters": { + "type": "object", + "properties": { + "sign": { + "type": "string", + "description": "An astrological sign like Taurus or Aquarius", + }, + }, + "required": ["sign"], + }, + }, +] + +def get_horoscope(sign): + return f"{sign}: Next Tuesday you will befriend a baby otter." + +# Create a running input list we will add to over time +input_list = [ + {"role": "user", "content": "What is my horoscope? I am an Aquarius."} +] + +# 2. Prompt the model with tools defined +response = client.responses.create( + model="gemini-3-pro-preview", + tools=tools, + input=input_list, +) + +# Save function call outputs for subsequent requests +input_list += response.output + +for item in response.output: + if item.type == "function_call": + if item.name == "get_horoscope": + # 3. Execute the function logic for get_horoscope + horoscope = get_horoscope(json.loads(item.arguments)) + + # 4. Provide function call results to the model + input_list.append({ + "type": "function_call_output", + "call_id": item.call_id, + "output": json.dumps({ + "horoscope": horoscope + }) + }) + +print("Final input:") +print(input_list) + +response = client.responses.create( + model="gemini-3-pro-preview", + instructions="Respond only with a horoscope generated by a tool.", + tools=tools, + input=input_list, +) + +# 5. The model should be able to give a response! +print("Final output:") +print(response.model_dump_json(indent=2)) +print("\n" + response.output_text) +``` + +**Key Points:** +- ✅ Thought signatures are automatically preserved in function calls +- ✅ Works seamlessly with multi-turn conversations +- ✅ All Gemini 3-specific features are fully supported + + + + +```python +from openai import OpenAI +import json + +client = OpenAI() + +tools = [ + { + "type": "function", + "name": "get_horoscope", + "description": "Get today's horoscope for an astrological sign.", + "parameters": { + "type": "object", + "properties": { + "sign": { + "type": "string", + "description": "An astrological sign like Taurus or Aquarius", + }, + }, + "required": ["sign"], + }, + }, +] + +def get_horoscope(sign): + return f"{sign}: Next Tuesday you will befriend a baby otter." + +input_list = [ + {"role": "user", "content": "What is my horoscope? I am an Aquarius."} +] + +# Streaming mode +response = client.responses.create( + model="gemini-3-pro-preview", + tools=tools, + input=input_list, + stream=True, +) + +# Collect all chunks +chunks = [] +for chunk in response: + chunks.append(chunk) + # Process streaming chunks as they arrive + print(chunk) + +# Thought signatures are automatically preserved in streaming mode +``` + +**Key Points:** +- ✅ Streaming mode fully supported +- ✅ Thought signatures preserved across streaming chunks +- ✅ Real-time processing of function calls and responses + + + + +### Responses API Benefits + +- ✅ **Structured Output**: Responses API provides a clear structure for handling function calls and multi-turn conversations +- ✅ **Thought Signature Preservation**: LiteLLM automatically preserves thought signatures in both streaming and non-streaming modes +- ✅ **Seamless Integration**: Works with existing OpenAI SDK patterns +- ✅ **Full Feature Support**: All Gemini 3 features (thought signatures, function calling, reasoning) are fully supported + + +## Best Practices + +#### 1. Always Include Thought Signatures in Conversation History + +When building multi-turn conversations with function calling: + +✅ **Do:** +```python +# Append the full assistant message (includes thought signatures) +messages.append(response.choices[0].message) +``` + +❌ **Don't:** +```python +# Don't manually construct assistant messages without thought signatures +messages.append({ + "role": "assistant", + "tool_calls": [...] # Missing thought signatures! +}) +``` + +#### 2. Use Appropriate Thinking Levels + +- **`reasoning_effort="low"`**: For simple queries, quick responses, cost optimization +- **`reasoning_effort="high"`**: For complex problems requiring deep reasoning + +#### 3. Keep Temperature at Default + +For Gemini 3 models, always use `temperature=1.0` (default). Lower temperatures can cause issues. + +#### 4. Handle Model Switches Gracefully + +When switching from non-Gemini-3 to Gemini-3: +- ✅ LiteLLM automatically handles missing thought signatures +- ✅ No manual intervention needed +- ✅ Conversation history continues seamlessly + + +## Troubleshooting + +#### Issue: Missing Thought Signatures + +**Symptom**: Error when including assistant messages in conversation history + +**Solution**: Ensure you're appending the full assistant message from the response: +```python +messages.append(response.choices[0].message) # ✅ Includes thought signatures +``` + +#### Issue: Conversation Breaks When Switching Models + +**Symptom**: Errors when switching from gemini-2.5-flash to gemini-3-pro-preview + +**Solution**: This should work automatically! LiteLLM adds dummy signatures. If you see errors, ensure you're using the latest LiteLLM version. + +#### Issue: Infinite Loops or Poor Performance + +**Symptom**: Model gets stuck or produces poor results + +**Solution**: +- Ensure `temperature=1.0` (default for Gemini 3) +- Check that `reasoning_effort` is set appropriately +- Verify you're using the correct model name: `gemini/gemini-3-pro-preview` + +## Additional Resources + +- [Gemini Provider Documentation](../gemini.md) +- [Thought Signatures Guide](../gemini.md#thought-signatures) +- [Reasoning Content Documentation](../../reasoning_content.md) +- [Function Calling Guide](../../function_calling.md) + diff --git a/docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md b/docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md index 2722a4a024c..9c654cd1560 100644 --- a/docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md +++ b/docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md @@ -37,57 +37,7 @@ Two files: `my_guardrail.py` (main class) and `__init__.py` (initialization). `my_guardrail.py`: -```python -import os -from typing import Optional, List -from fastapi import HTTPException - -from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.types.guardrails import PiiEntityType -from litellm._logging import verbose_proxy_logger -from litellm.llms.custom_httpx.http_handler import ( - get_async_httpx_client, - httpxSpecialProvider, -) - -class MyGuardrail(CustomGuardrail): - def __init__(self, api_key: Optional[str] = None, api_base: Optional[str] = None, **kwargs): - self.api_key = api_key or os.getenv("MY_GUARDRAIL_API_KEY") - self.api_base = api_base or os.getenv("MY_GUARDRAIL_API_BASE", "https://api.myguardrail.com") - super().__init__(default_on=True) - - async def apply_guardrail( - self, - text: str, - language: Optional[str] = None, - entities: Optional[List[PiiEntityType]] = None, - request_data: Optional[dict] = None, - ) -> str: - result = await self._check_with_api(text, request_data) - - if result.get("action") == "BLOCK": - raise Exception(f"Content blocked: {result.get('reason', 'Policy violation')}") - - return text - - async def _check_with_api(self, text: str, request_data: Optional[dict]) -> dict: - async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback) - - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {self.api_key}", - } - - response = await async_client.post( - f"{self.api_base}/check", - headers=headers, - json={"text": text}, - timeout=5, - ) - - response.raise_for_status() - return response.json() -``` +Follow from [Custom Guardrail](../proxy/guardrails/custom_guardrail#custom-guardrail) tutorial. ### Create the Init File diff --git a/docs/my-website/docs/batches.md b/docs/my-website/docs/batches.md index 1bd4c700ae7..269fee03106 100644 --- a/docs/my-website/docs/batches.md +++ b/docs/my-website/docs/batches.md @@ -174,6 +174,257 @@ print("list_batches_response=", list_batches_response) +## Multi-Account / Model-Based Routing + +Route batch operations to different provider accounts using model-specific credentials from your `config.yaml`. This eliminates the need for environment variables and enables multi-tenant batch processing. + +### How It Works + +**Priority Order:** +1. **Encoded Batch/File ID** (highest) - Model info embedded in the ID +2. **Model Parameter** - Via header (`x-litellm-model`), query param, or request body +3. **Custom Provider** (fallback) - Uses environment variables + +### Configuration + +```yaml +model_list: + - model_name: gpt-4o-account-1 + litellm_params: + model: openai/gpt-4o + api_key: sk-account-1-key + api_base: https://api.openai.com/v1 + + - model_name: gpt-4o-account-2 + litellm_params: + model: openai/gpt-4o + api_key: sk-account-2-key + api_base: https://api.openai.com/v1 + + - model_name: azure-batches + litellm_params: + model: azure/gpt-4 + api_key: azure-key-123 + api_base: https://my-resource.openai.azure.com + api_version: "2024-02-01" +``` + +### Usage Examples + +#### Scenario 1: Encoded File ID with Model + +When you upload a file with a model parameter, LiteLLM encodes the model information in the file ID. All subsequent operations automatically use those credentials. + +```bash +# Step 1: Upload file with model +curl http://localhost:4000/v1/files \ + -H "Authorization: Bearer sk-1234" \ + -H "x-litellm-model: gpt-4o-account-1" \ + -F purpose="batch" \ + -F file="@batch.jsonl" + +# Response includes encoded file ID: +# { +# "id": "file-bGl0ZWxsbTpmaWxlLUxkaUwzaVYxNGZRVlpYcU5KVEdkSjk7bW9kZWwsZ3B0LTRvLWFjY291bnQtMQ", +# ... +# } + +# Step 2: Create batch - automatically routes to gpt-4o-account-1 +curl http://localhost:4000/v1/batches \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "input_file_id": "file-bGl0ZWxsbTpmaWxlLUxkaUwzaVYxNGZRVlpYcU5KVEdkSjk7bW9kZWwsZ3B0LTRvLWFjY291bnQtMQ", + "endpoint": "/v1/chat/completions", + "completion_window": "24h" + }' + +# Batch ID is also encoded with model: +# { +# "id": "batch_bGl0ZWxsbTpiYXRjaF82OTIwM2IzNjg0MDQ4MTkwYTA3ODQ5NDY3YTFjMDJkYTttb2RlbCxncHQtNG8tYWNjb3VudC0x", +# "input_file_id": "file-bGl0ZWxsbTpmaWxlLUxkaUwzaVYxNGZRVlpYcU5KVEdkSjk7bW9kZWwsZ3B0LTRvLWFjY291bnQtMQ", +# ... +# } + +# Step 3: Retrieve batch - automatically routes to gpt-4o-account-1 +curl http://localhost:4000/v1/batches/batch_bGl0ZWxsbTpiYXRjaF82OTIwM2IzNjg0MDQ4MTkwYTA3ODQ5NDY3YTFjMDJkYTttb2RlbCxncHQtNG8tYWNjb3VudC0x \ + -H "Authorization: Bearer sk-1234" +``` + +**✅ Benefits:** +- No need to specify model on every request +- File and batch IDs "remember" which account created them +- Automatic routing for retrieve, cancel, and file content operations + +#### Scenario 2: Model via Header/Query Parameter + +Specify the model for each request without encoding it in the ID. + +```bash +# Create batch with model header +curl http://localhost:4000/v1/batches \ + -H "Authorization: Bearer sk-1234" \ + -H "x-litellm-model: gpt-4o-account-2" \ + -H "Content-Type: application/json" \ + -d '{ + "input_file_id": "file-abc123", + "endpoint": "/v1/chat/completions", + "completion_window": "24h" + }' + +# Or use query parameter +curl "http://localhost:4000/v1/batches?model=gpt-4o-account-2" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "input_file_id": "file-abc123", + "endpoint": "/v1/chat/completions", + "completion_window": "24h" + }' + +# List batches for specific model +curl "http://localhost:4000/v1/batches?model=gpt-4o-account-2" \ + -H "Authorization: Bearer sk-1234" +``` + +**✅ Use Case:** +- One-off batch operations +- Different models for different operations +- Explicit control over routing + +#### Scenario 3: Environment Variables (Fallback) + +Traditional approach using environment variables when no model is specified. + +```bash +export OPENAI_API_KEY="sk-env-key" + +curl http://localhost:4000/v1/batches \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "input_file_id": "file-abc123", + "endpoint": "/v1/chat/completions", + "completion_window": "24h" + }' +``` + +**✅ Use Case:** +- Backward compatibility +- Simple single-account setups +- Quick prototyping + +### Complete Multi-Account Example + +```bash +# Upload file to Account 1 +FILE_1=$(curl -s http://localhost:4000/v1/files \ + -H "x-litellm-model: gpt-4o-account-1" \ + -F purpose="batch" \ + -F file="@batch1.jsonl" | jq -r '.id') + +# Upload file to Account 2 +FILE_2=$(curl -s http://localhost:4000/v1/files \ + -H "x-litellm-model: gpt-4o-account-2" \ + -F purpose="batch" \ + -F file="@batch2.jsonl" | jq -r '.id') + +# Create batch on Account 1 (auto-routed via encoded file ID) +BATCH_1=$(curl -s http://localhost:4000/v1/batches \ + -d "{\"input_file_id\": \"$FILE_1\", \"endpoint\": \"/v1/chat/completions\", \"completion_window\": \"24h\"}" | jq -r '.id') + +# Create batch on Account 2 (auto-routed via encoded file ID) +BATCH_2=$(curl -s http://localhost:4000/v1/batches \ + -d "{\"input_file_id\": \"$FILE_2\", \"endpoint\": \"/v1/chat/completions\", \"completion_window\": \"24h\"}" | jq -r '.id') + +# Retrieve both batches (auto-routed to correct accounts) +curl http://localhost:4000/v1/batches/$BATCH_1 +curl http://localhost:4000/v1/batches/$BATCH_2 + +# List batches per account +curl "http://localhost:4000/v1/batches?model=gpt-4o-account-1" +curl "http://localhost:4000/v1/batches?model=gpt-4o-account-2" +``` + +### SDK Usage with Model Routing + +```python +import litellm +import asyncio + +# Upload file with model routing +file_obj = await litellm.acreate_file( + file=open("batch.jsonl", "rb"), + purpose="batch", + model="gpt-4o-account-1", # Route to specific account +) + +print(f"File ID: {file_obj.id}") +# File ID is encoded with model info + +# Create batch - automatically uses gpt-4o-account-1 credentials +batch = await litellm.acreate_batch( + completion_window="24h", + endpoint="/v1/chat/completions", + input_file_id=file_obj.id, # Model info embedded in ID +) + +print(f"Batch ID: {batch.id}") +# Batch ID is also encoded + +# Retrieve batch - automatically routes to correct account +retrieved = await litellm.aretrieve_batch( + batch_id=batch.id, # Model info embedded in ID +) + +print(f"Batch status: {retrieved.status}") + +# Or explicitly specify model +batch2 = await litellm.acreate_batch( + completion_window="24h", + endpoint="/v1/chat/completions", + input_file_id="file-regular-id", + model="gpt-4o-account-2", # Explicit routing +) +``` + +### How ID Encoding Works + +LiteLLM encodes model information into file and batch IDs using base64: + +``` +Original: file-abc123 +Encoded: file-bGl0ZWxsbTpmaWxlLWFiYzEyMzttb2RlbCxncHQtNG8tdGVzdA + └─┬─┘ └──────────────────┬──────────────────────┘ + prefix base64(litellm:file-abc123;model,gpt-4o-test) + +Original: batch_xyz789 +Encoded: batch_bGl0ZWxsbTpiYXRjaF94eXo3ODk7bW9kZWwsZ3B0LTRvLXRlc3Q + └──┬──┘ └──────────────────┬──────────────────────┘ + prefix base64(litellm:batch_xyz789;model,gpt-4o-test) +``` + +The encoding: +- ✅ Preserves OpenAI-compatible prefixes (`file-`, `batch_`) +- ✅ Is transparent to clients +- ✅ Enables automatic routing without additional parameters +- ✅ Works across all batch and file endpoints + +### Supported Endpoints + +All batch and file endpoints support model-based routing: + +| Endpoint | Method | Model Routing | +|----------|--------|---------------| +| `/v1/files` | POST | ✅ Via header/query/body | +| `/v1/files/{file_id}` | GET | ✅ Auto from encoded ID + header/query | +| `/v1/files/{file_id}/content` | GET | ✅ Auto from encoded ID + header/query | +| `/v1/files/{file_id}` | DELETE | ✅ Auto from encoded ID | +| `/v1/batches` | POST | ✅ Auto from file ID + header/query/body | +| `/v1/batches` | GET | ✅ Via header/query | +| `/v1/batches/{batch_id}` | GET | ✅ Auto from encoded ID | +| `/v1/batches/{batch_id}/cancel` | POST | ✅ Auto from encoded ID | + ## **Supported Providers**: ### [Azure OpenAI](./providers/azure#azure-batches-api) ### [OpenAI](#quick-start) diff --git a/docs/my-website/docs/benchmarks.md b/docs/my-website/docs/benchmarks.md index f00732450d1..4e4234949f8 100644 --- a/docs/my-website/docs/benchmarks.md +++ b/docs/my-website/docs/benchmarks.md @@ -125,18 +125,23 @@ class MyUser(HttpUser): ## LiteLLM vs Portkey Performance Comparison **Test Configuration**: 4 CPUs, 8 GB RAM per instance | Load: 1k concurrent users, 500 ramp-up +**Versions:** Portkey **v1.14.0** | LiteLLM **v1.79.1-stable** +**Test Duration:** 5 minutes ### Multi-Instance (4×) Performance -| Metric | Portkey (no DB) | LiteLLM (with DB) | -| ------------------- | --------------- | ----------------- | -| **Total Requests** | 293,796 | 312,405 | -| **Failed Requests** | 0 | 0 | -| **Median Latency** | 100 ms | 100 ms | -| **p95 Latency** | 230 ms | 150 ms | -| **p99 Latency** | 500 ms | 240 ms | -| **Average Latency** | 123 ms | 111 ms | -| **Current RPS** | 1,170.9 | 1,170 | +| Metric | Portkey (no DB) | LiteLLM (with DB) | Comment | +| ------------------- | --------------- | ----------------- | -------------- | +| **Total Requests** | 293,796 | 312,405 | LiteLLM higher | +| **Failed Requests** | 0 | 0 | Same | +| **Median Latency** | 100 ms | 100 ms | Same | +| **p95 Latency** | 230 ms | 150 ms | LiteLLM lower | +| **p99 Latency** | 500 ms | 240 ms | LiteLLM lower | +| **Average Latency** | 123 ms | 111 ms | LiteLLM lower | +| **Current RPS** | 1,170.9 | 1,170 | Same | + + +*Lower is better for latency metrics; higher is better for requests and RPS.* ### Technical Insights diff --git a/docs/my-website/docs/completion/vision.md b/docs/my-website/docs/completion/vision.md index 76700084868..90d6b2393fb 100644 --- a/docs/my-website/docs/completion/vision.md +++ b/docs/my-website/docs/completion/vision.md @@ -31,7 +31,7 @@ response = completion( { "type": "image_url", "image_url": { - "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + "url": "https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png" } } ] @@ -92,7 +92,7 @@ response = client.chat.completions.create( { "type": "image_url", "image_url": { - "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + "url": "https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png" } } ] @@ -230,7 +230,7 @@ response = completion( { "type": "image_url", "image_url": { - "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", + "url": "https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png", "format": "image/jpeg" } } @@ -292,7 +292,7 @@ response = client.chat.completions.create( { "type": "image_url", "image_url": { - "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", + "url": "https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png", "format": "image/jpeg" } } diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md index 6bcfed24c94..2eed0f53e59 100644 --- a/docs/my-website/docs/enterprise.md +++ b/docs/my-website/docs/enterprise.md @@ -3,7 +3,8 @@ import Image from '@theme/IdealImage'; # Enterprise :::info -✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise) +- ✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise) +- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) to discuss your needs. ::: For companies that need SSO, user management and professional support for LiteLLM Proxy diff --git a/docs/my-website/docs/extras/contributing_code.md b/docs/my-website/docs/extras/contributing_code.md index f3a8271b14b..930a47eec7e 100644 --- a/docs/my-website/docs/extras/contributing_code.md +++ b/docs/my-website/docs/extras/contributing_code.md @@ -107,3 +107,18 @@ docker run \ litellm_test_image \ --config /app/config.yaml --detailed_debug ``` +### Running LiteLLM Proxy Locally + +1. cd into the `proxy/` directory + +``` +cd litellm/litellm/proxy +``` + +2. Run the proxy + +```shell +python3 proxy_cli.py --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` \ No newline at end of file diff --git a/docs/my-website/docs/files_endpoints.md b/docs/my-website/docs/files_endpoints.md index 88493fe0bbd..fc0484e9219 100644 --- a/docs/my-website/docs/files_endpoints.md +++ b/docs/my-website/docs/files_endpoints.md @@ -16,7 +16,137 @@ Use this to call the provider's `/files` endpoints directly, in the OpenAI forma - Delete File - Get File Content +## Multi-Account Support (Multiple OpenAI Keys) +Use different OpenAI API keys for files and batches by specifying a `model` parameter that references entries in your `model_list`. This approach works **without requiring a database** and allows you to route files/batches to different OpenAI accounts. + +### How It Works + +1. Define models in `model_list` with different API keys +2. Pass `model` parameter when creating files +3. LiteLLM returns encoded IDs that contain routing information +4. Use encoded IDs for all subsequent operations (retrieve, delete, batches) +5. No need to specify model again - routing info is in the ID + +### Setup + +```yaml +model_list: + # litellm OpenAI Account + - model_name: "gpt-4o-litellm" + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_LITELLM_API_KEY + + # Free OpenAI Account + - model_name: "gpt-4o-free" + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_FREE_API_KEY +``` + +### Usage Example + +```python +from openai import OpenAI + +client = OpenAI( + api_key="sk-1234", # Your LiteLLM proxy key + base_url="http://0.0.0.0:4000" +) + +# Create file using litellm account +file_response = client.files.create( + file=open("batch_data.jsonl", "rb"), + purpose="batch", + extra_body={"model": "gpt-4o-litellm"} # Routes to litellm key +) +print(f"File ID: {file_response.id}") +# Returns encoded ID like: file-bGl0ZWxsbTpmaWxlLWFiYzEyMzttb2RlbCxncHQtNG8taWZvb2Q + +# Create batch using the encoded file ID +# No need to specify model again - it's embedded in the file ID +batch_response = client.batches.create( + input_file_id=file_response.id, # Encoded ID + endpoint="/v1/chat/completions", + completion_window="24h" +) +print(f"Batch ID: {batch_response.id}") +# Returns encoded batch ID with routing info + +# Retrieve batch - routing happens automatically +batch_status = client.batches.retrieve(batch_response.id) +print(f"Status: {batch_status.status}") + +# List files for a specific account +files = client.files.list( + extra_body={"model": "gpt-4o-free"} # List free files +) + +# List batches for a specific account +batches = client.batches.list( + extra_query={"model": "gpt-4o-litellm"} # List litellm batches +) +``` + +### Parameter Options + +You can pass the `model` parameter via: +- **Request body**: `extra_body={"model": "gpt-4o-litellm"}` +- **Query parameter**: `?model=gpt-4o-litellm` +- **Header**: `x-litellm-model: gpt-4o-litellm` + +### How Encoded IDs Work + +- When you create a file/batch with a `model` parameter, LiteLLM encodes the model name into the returned ID +- The encoded ID is base64-encoded and looks like: `file-bGl0ZWxsbTpmaWxlLWFiYzEyMzttb2RlbCxncHQtNG8taWZvb2Q` +- When you use this ID in subsequent operations (retrieve, delete, batch create), LiteLLM automatically: + 1. Decodes the ID + 2. Extracts the model name + 3. Looks up the credentials + 4. Routes the request to the correct OpenAI account +- The original provider file/batch ID is preserved internally + +### Benefits + +✅ **No Database Required** - All routing info stored in the ID +✅ **Stateless** - Works across proxy restarts +✅ **Simple** - Just pass the ID around like normal +✅ **Backward Compatible** - Existing `custom_llm_provider` and `files_settings` still work +✅ **Future-Proof** - Aligns with managed batches approach + +### Migration from files_settings + +**Old approach (still works):** +```yaml +files_settings: + - custom_llm_provider: openai + api_key: os.environ/OPENAI_KEY +``` + +```python +# Had to specify provider on every call +client.files.create(..., extra_headers={"custom-llm-provider": "openai"}) +client.files.retrieve(file_id, extra_headers={"custom-llm-provider": "openai"}) +``` + +**New approach (recommended):** +```yaml +model_list: + - model_name: "gpt-4o-account1" + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_KEY +``` + +```python +# Specify model once on create +file = client.files.create(..., extra_body={"model": "gpt-4o-account1"}) + +# Then just use the ID - routing is automatic +client.files.retrieve(file.id) # No need to specify account +client.batches.create(input_file_id=file.id) # Routes correctly +``` diff --git a/docs/my-website/docs/image_edits.md b/docs/my-website/docs/image_edits.md index 9a53da510f7..5a108aabf3a 100644 --- a/docs/my-website/docs/image_edits.md +++ b/docs/my-website/docs/image_edits.md @@ -16,7 +16,7 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit | Supported operations | Create image edits | Single and multiple images supported | | Supported LiteLLM SDK Versions | 1.63.8+ | Gemini support requires 1.79.3+ | | Supported LiteLLM Proxy Versions | 1.71.1+ | Gemini support requires 1.79.3+ | -| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)** | Gemini supports the new `gemini-2.5-flash-image` family | +| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. | #### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/) @@ -197,6 +197,53 @@ for idx, image_obj in enumerate(response.data): f.write(base64.b64decode(image_obj.b64_json)) ``` + + + + +#### Basic Image Edit (Gemini) +```python showLineNumbers title="Vertex AI Gemini Image Edit" +import os +import litellm + +# Set Vertex AI credentials +os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id" +os.environ["VERTEXAI_LOCATION"] = "us-central1" +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/path/to/service-account.json" + +response = litellm.image_edit( + model="vertex_ai/gemini-2.5-flash", + image=open("original_image.png", "rb"), + prompt="Add neon lights in the background", + size="1024x1024", +) + +print(response) +``` + +#### Image Edit with Imagen (Supports Masks) +```python showLineNumbers title="Vertex AI Imagen Image Edit" +import os +import litellm + +# Set Vertex AI credentials +os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id" +os.environ["VERTEXAI_LOCATION"] = "us-central1" +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/path/to/service-account.json" + +# Imagen supports mask for inpainting +response = litellm.image_edit( + model="vertex_ai/imagen-3.0-capability-001", + image=open("original_image.png", "rb"), + mask=open("mask_image.png", "rb"), # Optional: for inpainting + prompt="Turn this into watercolor style scenery", + n=2, # Number of variations + size="1024x1024", +) + +print(response) +``` + @@ -302,6 +349,55 @@ curl -X POST "http://0.0.0.0:4000/v1/images/edits" \ -F "size=1024x1024" ``` + + + + +1. Add Vertex AI image edit models to your `config.yaml`: +```yaml showLineNumbers title="Vertex AI Proxy Configuration" +model_list: + - model_name: vertex-gemini-image-edit + litellm_params: + model: vertex_ai/gemini-2.5-flash + vertex_project: os.environ/VERTEXAI_PROJECT + vertex_location: os.environ/VERTEXAI_LOCATION + vertex_credentials: os.environ/GOOGLE_APPLICATION_CREDENTIALS + + - model_name: vertex-imagen-image-edit + litellm_params: + model: vertex_ai/imagen-3.0-capability-001 + vertex_project: os.environ/VERTEXAI_PROJECT + vertex_location: os.environ/VERTEXAI_LOCATION + vertex_credentials: os.environ/GOOGLE_APPLICATION_CREDENTIALS +``` + +2. Start the LiteLLM proxy server: +```bash showLineNumbers title="Start LiteLLM Proxy Server" +litellm --config /path/to/config.yaml +``` + +3. Make an image edit request: +```bash showLineNumbers title="Vertex AI Gemini Proxy Image Edit" +curl -X POST "http://0.0.0.0:4000/v1/images/edits" \ + -H "Authorization: Bearer " \ + -F "model=vertex-gemini-image-edit" \ + -F "image=@original_image.png" \ + -F "prompt=Add neon lights in the background" \ + -F "size=1024x1024" +``` + +4. Imagen image edit with mask: +```bash showLineNumbers title="Vertex AI Imagen Proxy Image Edit with Mask" +curl -X POST "http://0.0.0.0:4000/v1/images/edits" \ + -H "Authorization: Bearer " \ + -F "model=vertex-imagen-image-edit" \ + -F "image=@original_image.png" \ + -F "mask=@mask_image.png" \ + -F "prompt=Turn this into watercolor style scenery" \ + -F "n=2" \ + -F "size=1024x1024" +``` + diff --git a/docs/my-website/docs/mcp.md b/docs/my-website/docs/mcp.md index 9fd803f434f..9a1e25a516c 100644 --- a/docs/my-website/docs/mcp.md +++ b/docs/my-website/docs/mcp.md @@ -657,7 +657,7 @@ LiteLLM Proxy provides two methods for controlling access to specific MCP server ### Method 1: URL-based Namespacing -LiteLLM Proxy supports URL-based namespacing for MCP servers using the format `/mcp/`. This allows you to: +LiteLLM Proxy supports URL-based namespacing for MCP servers using the format `//mcp`. This allows you to: - **Direct URL Access**: Point MCP clients directly to specific servers or access groups via URL - **Simplified Configuration**: Use URLs instead of headers for server selection @@ -666,14 +666,14 @@ LiteLLM Proxy supports URL-based namespacing for MCP servers using the format `/ #### URL Format ``` -/mcp/ +//mcp ``` **Examples:** -- `/mcp/github` - Access tools from the "github" MCP server -- `/mcp/zapier` - Access tools from the "zapier" MCP server -- `/mcp/dev_group` - Access tools from all servers in the "dev_group" access group -- `/mcp/github,zapier` - Access tools from multiple specific servers +- `/github_mcp/mcp` - Access tools from the "github_mcp" MCP server +- `/zapier/mcp` - Access tools from the "zapier" MCP server +- `/dev_group/mcp` - Access tools from all servers in the "dev_group" access group +- `/github_mcp,zapier/mcp` - Access tools from multiple specific servers #### Usage Examples @@ -690,7 +690,7 @@ curl --location 'https://api.openai.com/v1/responses' \ { "type": "mcp", "server_label": "litellm", - "server_url": "/mcp/github", + "server_url": "/github_mcp/mcp", "require_approval": "never", "headers": { "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY" @@ -718,7 +718,7 @@ curl --location '/v1/responses' \ { "type": "mcp", "server_label": "litellm", - "server_url": "/mcp/dev_group", + "server_url": "/dev_group/mcp", "require_approval": "never", "headers": { "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY" @@ -740,7 +740,7 @@ This example uses URL namespacing to access all servers in the "dev_group" acces { "mcpServers": { "LiteLLM": { - "url": "/mcp/github,zapier", + "url": "/github_mcp,zapier/mcp", "headers": { "x-litellm-api-key": "Bearer $LITELLM_API_KEY" } @@ -862,8 +862,8 @@ This configuration in Cursor IDE settings will limit tool access to only the spe | Feature | Header Namespacing | URL Namespacing | |---------|-------------------|-----------------| -| **Method** | Uses `x-mcp-servers` header | Uses URL path `/mcp/` | -| **Endpoint** | Standard `litellm_proxy` endpoint | Custom `/mcp/` endpoint | +| **Method** | Uses `x-mcp-servers` header | Uses URL path `//mcp` | +| **Endpoint** | Standard `litellm_proxy` endpoint | Custom `//mcp` endpoint | | **Configuration** | Requires additional header | Self-contained in URL | | **Multiple Servers** | Comma-separated in header | Comma-separated in URL path | | **Access Groups** | Supported via header | Supported via URL path | @@ -1221,7 +1221,6 @@ curl --location 'http://localhost:4000/github_mcp/mcp' \ LiteLLM v 1.77.6 added support for OAuth 2.0 Client Credentials for MCP servers. - This configuration is currently available on the config.yaml, with UI support coming soon. ```yaml @@ -1235,6 +1234,71 @@ mcp_servers: [**See Claude Code Tutorial**](./tutorials/claude_responses_api#connecting-mcp-servers) +### How It Works + +```mermaid +sequenceDiagram + participant Browser as User-Agent (Browser) + participant Client as Client + participant LiteLLM as LiteLLM Proxy + participant MCP as MCP Server (Resource Server) + participant Auth as Authorization Server + + Note over Client,LiteLLM: Step 1 – Resource discovery + Client->>LiteLLM: GET /.well-known/oauth-protected-resource/{mcp_server_name}/mcp + LiteLLM->>Client: Return resource metadata + + Note over Client,LiteLLM: Step 2 – Authorization server discovery + Client->>LiteLLM: GET /.well-known/oauth-authorization-server/{mcp_server_name} + LiteLLM->>Client: Return authorization server metadata + + Note over Client,Auth: Step 3 – Dynamic client registration + Client->>LiteLLM: POST /{mcp_server_name}/register + LiteLLM->>Auth: Forward registration request + Auth->>LiteLLM: Issue client credentials + LiteLLM->>Client: Return client credentials + + Note over Client,Browser: Step 4 – User authorization (PKCE) + Client->>Browser: Open authorization URL + code_challenge + resource + Browser->>Auth: Authorization request + Note over Auth: User authorizes + Auth->>Browser: Redirect with authorization code + Browser->>LiteLLM: Callback to LiteLLM with code + LiteLLM->>Browser: Redirect back with authorization code + Browser->>Client: Callback with authorization code + + Note over Client,Auth: Step 5 – Token exchange + Client->>LiteLLM: Token request + code_verifier + resource + LiteLLM->>Auth: Forward token request + Auth->>LiteLLM: Access (and refresh) token + LiteLLM->>Client: Return tokens + + Note over Client,MCP: Step 6 – Authenticated MCP call + Client->>LiteLLM: MCP request with access token + LiteLLM API key + LiteLLM->>MCP: MCP request with Bearer token + MCP-->>LiteLLM: MCP response + LiteLLM-->>Client: Return MCP response +``` + +**Participants** + +- **Client** – The MCP-capable AI agent (e.g., Claude Code, Cursor, or another IDE/agent) that initiates OAuth discovery, authorization, and tool invocations on behalf of the user. +- **LiteLLM Proxy** – Mediates all OAuth discovery, registration, token exchange, and MCP traffic while protecting stored credentials. +- **Authorization Server** – Issues OAuth 2.0 tokens via dynamic client registration, PKCE authorization, and token endpoints. +- **MCP Server (Resource Server)** – The protected MCP endpoint that receives LiteLLM’s authenticated JSON-RPC requests. +- **User-Agent (Browser)** – Temporarily involved so the end user can grant consent during the authorization step. + +**Flow Steps** + +1. **Resource Discovery**: The client fetches MCP resource metadata from LiteLLM’s `.well-known/oauth-protected-resource` endpoint to understand scopes and capabilities. +2. **Authorization Server Discovery**: The client retrieves the OAuth server metadata (token endpoint, authorization endpoint, supported PKCE methods) through LiteLLM’s `.well-known/oauth-authorization-server` endpoint. +3. **Dynamic Client Registration**: The client registers through LiteLLM, which forwards the request to the authorization server (RFC 7591). If the provider doesn’t support dynamic registration, you can pre-store `client_id`/`client_secret` in LiteLLM (e.g., GitHub MCP) and the flow proceeds the same way. +4. **User Authorization**: The client launches a browser session (with code challenge and resource hints). The user approves access, the authorization server sends the code through LiteLLM back to the client. +5. **Token Exchange**: The client calls LiteLLM with the authorization code, code verifier, and resource. LiteLLM exchanges them with the authorization server and returns the issued access/refresh tokens. +6. **MCP Invocation**: With a valid token, the client sends the MCP JSON-RPC request (plus LiteLLM API key) to LiteLLM, which forwards it to the MCP server and relays the tool response. + +See the official [MCP Authorization Flow](https://modelcontextprotocol.io/specification/2025-06-18/basic/authorization#authorization-flow-steps) for additional reference. + ## Using your MCP with client side credentials Use this if you want to pass a client side authentication token to LiteLLM to then pass to your MCP to auth to your MCP. diff --git a/docs/my-website/docs/observability/phoenix_integration.md b/docs/my-website/docs/observability/phoenix_integration.md index d15eea9a834..ad337439934 100644 --- a/docs/my-website/docs/observability/phoenix_integration.md +++ b/docs/my-website/docs/observability/phoenix_integration.md @@ -33,6 +33,8 @@ import os os.environ["PHOENIX_API_KEY"] = "" # Necessary only using Phoenix Cloud os.environ["PHOENIX_COLLECTOR_HTTP_ENDPOINT"] = "" # The URL of your Phoenix OSS instance e.g. http://localhost:6006/v1/traces +os.environ["PHOENIX_PROJECT_NAME"]="litellm" # OPTIONAL: you can configure project names, otherwise traces would go to "default" project + # This defaults to https://app.phoenix.arize.com/v1/traces for Phoenix Cloud # LLM API Keys diff --git a/docs/my-website/docs/providers/azure/azure.md b/docs/my-website/docs/providers/azure/azure.md index 2f845357328..0ff5b2a5a77 100644 --- a/docs/my-website/docs/providers/azure/azure.md +++ b/docs/my-website/docs/providers/azure/azure.md @@ -251,7 +251,7 @@ response = completion( { "type": "image_url", "image_url": { - "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + "url": "https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png" } } ] diff --git a/docs/my-website/docs/providers/azure_ai_speech.md b/docs/my-website/docs/providers/azure_ai_speech.md index 434a796a2fb..22db98cfac5 100644 --- a/docs/my-website/docs/providers/azure_ai_speech.md +++ b/docs/my-website/docs/providers/azure_ai_speech.md @@ -136,6 +136,89 @@ response = speech( | `wav` | riff-24khz-16bit-mono-pcm | 24kHz | | `pcm` | raw-24khz-16bit-mono-pcm | 24kHz | +## Passing Raw SSML + +LiteLLM automatically detects when your `input` contains SSML (by checking for `` tags) and passes it through to Azure without any transformation. This gives you complete control over speech synthesis. + +**When to use raw SSML:** +- Using the `` element with multilingual voices to translate text (e.g., English text → Spanish speech) +- Complex SSML structures with multiple voices or prosody changes +- Fine-grained control over pronunciation, breaks, emphasis, and other speech features + +### LiteLLM SDK + +```python showLineNumbers title="Raw SSML for Multilingual Translation" +from litellm import speech + +# Use element to convert English text to Spanish speech +# The element forces the output language regardless of input text language +language_code = "es-ES" +text = "Hello, how are you today?" # English text +voice = "en-US-AvaMultilingualNeural" + +ssml = f""" + + {text} + +""" + +response = speech( + model="azure/speech/azure-tts", + voice=voice, + input=ssml, # LiteLLM auto-detects SSML and sends as-is + api_base="https://eastus.tts.speech.microsoft.com", + api_key=os.environ["AZURE_TTS_API_KEY"], +) +response.stream_to_file("speech.mp3") +``` + +```python showLineNumbers title="Raw SSML with Complex Features" +from litellm import speech + +# Complex SSML with multiple prosody adjustments +ssml = """ + + + + Welcome to our service! + + + + + How can I help you today? + + +""" + +response = speech( + model="azure/speech/azure-tts", + voice="en-US-JennyNeural", + input=ssml, # LiteLLM detects and passes through unchanged + api_base="https://eastus.tts.speech.microsoft.com", + api_key=os.environ["AZURE_TTS_API_KEY"], +) +response.stream_to_file("speech.mp3") +``` + +### LiteLLM Proxy + +```bash +curl http://0.0.0.0:4000/v1/audio/speech \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "azure-speech", + "voice": "en-US-AvaMultilingualNeural", + "input": "Hello, how are you today?" + }' \ + --output speech.mp3 +``` + + ## Sending Azure-Specific Params Azure AI Speech supports advanced SSML features through optional parameters: diff --git a/docs/my-website/docs/providers/docker_model_runner.md b/docs/my-website/docs/providers/docker_model_runner.md new file mode 100644 index 00000000000..fcd4c74f8f4 --- /dev/null +++ b/docs/my-website/docs/providers/docker_model_runner.md @@ -0,0 +1,277 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Docker Model Runner + +## Overview + +| Property | Details | +|-------|-------| +| Description | Docker Model Runner allows you to run large language models locally using Docker Desktop. | +| Provider Route on LiteLLM | `docker_model_runner/` | +| Link to Provider Doc | [Docker Model Runner ↗](https://docs.docker.com/ai/model-runner/) | +| Base URL | `http://localhost:22088` | +| Supported Operations | [`/chat/completions`](#sample-usage) | + +
+
+ +https://docs.docker.com/ai/model-runner/ + +**We support ALL Docker Model Runner models, just set `docker_model_runner/` as a prefix when sending completion requests** + +## Quick Start + +Docker Model Runner is a Docker Desktop feature that lets you run AI models locally. It provides better performance than other local solutions while maintaining OpenAI compatibility. + +### Installation + +1. Install [Docker Desktop](https://www.docker.com/products/docker-desktop/) +2. Enable Docker Model Runner in Docker Desktop settings +3. Download your preferred model through Docker Desktop + +## Environment Variables + +```python showLineNumbers title="Environment Variables" +os.environ["DOCKER_MODEL_RUNNER_API_BASE"] = "http://localhost:22088/engines/llama.cpp" # Optional - defaults to this +os.environ["DOCKER_MODEL_RUNNER_API_KEY"] = "dummy-key" # Optional - Docker Model Runner may not require auth for local instances +``` + +**Note:** +- Docker Model Runner typically runs locally and may not require authentication. LiteLLM will use a dummy key by default if no key is provided. +- The API base should include the engine path (e.g., `/engines/llama.cpp`) + +## API Base Structure + +Docker Model Runner uses a unique URL structure: + +``` +http://model-runner.docker.internal/engines/{engine}/v1/chat/completions +``` + +Where `{engine}` is the engine you want to use (typically `llama.cpp`). + +**Important:** Specify the engine in your `api_base` URL, not in the model name: +- ✅ Correct: `api_base="http://localhost:22088/engines/llama.cpp"`, `model="docker_model_runner/llama-3.1"` +- ❌ Incorrect: `api_base="http://localhost:22088"`, `model="docker_model_runner/llama.cpp/llama-3.1"` + +## Usage - LiteLLM Python SDK + +### Non-streaming + +```python showLineNumbers title="Docker Model Runner Non-streaming Completion" +import os +import litellm +from litellm import completion + +# Specify the engine in the api_base URL +os.environ["DOCKER_MODEL_RUNNER_API_BASE"] = "http://localhost:22088/engines/llama.cpp" + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Docker Model Runner call +response = completion( + model="docker_model_runner/llama-3.1", + messages=messages +) + +print(response) +``` + +### Streaming + +```python showLineNumbers title="Docker Model Runner Streaming Completion" +import os +import litellm +from litellm import completion + +# Specify the engine in the api_base URL +os.environ["DOCKER_MODEL_RUNNER_API_BASE"] = "http://localhost:22088/engines/llama.cpp" + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Docker Model Runner call with streaming +response = completion( + model="docker_model_runner/llama-3.1", + messages=messages, + stream=True +) + +for chunk in response: + print(chunk) +``` + +### Custom API Base and Engine + +```python showLineNumbers title="Custom API Base with Different Engine" +import litellm +from litellm import completion + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Specify the engine in the api_base URL +# Using a different host and engine +response = completion( + model="docker_model_runner/llama-3.1", + messages=messages, + api_base="http://model-runner.docker.internal/engines/llama.cpp" +) + +print(response) +``` + +### Using Different Engines + +```python showLineNumbers title="Using a Different Engine" +import litellm +from litellm import completion + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# To use a different engine, specify it in the api_base +# For example, if Docker Model Runner supports other engines: +response = completion( + model="docker_model_runner/mistral-7b", + messages=messages, + api_base="http://localhost:22088/engines/custom-engine" +) + +print(response) +``` + +## Usage - LiteLLM Proxy + +Add the following to your LiteLLM Proxy configuration file: + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: llama-3.1 + litellm_params: + model: docker_model_runner/llama-3.1 + api_base: http://localhost:22088/engines/llama.cpp + + - model_name: mistral-7b + litellm_params: + model: docker_model_runner/mistral-7b + api_base: http://localhost:22088/engines/llama.cpp +``` + +Start your LiteLLM Proxy server: + +```bash showLineNumbers title="Start LiteLLM Proxy" +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + + + + +```python showLineNumbers title="Docker Model Runner via Proxy - Non-streaming" +from openai import OpenAI + +# Initialize client with your proxy URL +client = OpenAI( + base_url="http://localhost:4000", # Your proxy URL + api_key="your-proxy-api-key" # Your proxy API key +) + +# Non-streaming response +response = client.chat.completions.create( + model="llama-3.1", + messages=[{"role": "user", "content": "hello from litellm"}] +) + +print(response.choices[0].message.content) +``` + +```python showLineNumbers title="Docker Model Runner via Proxy - Streaming" +from openai import OpenAI + +# Initialize client with your proxy URL +client = OpenAI( + base_url="http://localhost:4000", # Your proxy URL + api_key="your-proxy-api-key" # Your proxy API key +) + +# Streaming response +response = client.chat.completions.create( + model="llama-3.1", + messages=[{"role": "user", "content": "hello from litellm"}], + stream=True +) + +for chunk in response: + if chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="") +``` + + + + + +```python showLineNumbers title="Docker Model Runner via Proxy - LiteLLM SDK" +import litellm + +# Configure LiteLLM to use your proxy +response = litellm.completion( + model="litellm_proxy/llama-3.1", + messages=[{"role": "user", "content": "hello from litellm"}], + api_base="http://localhost:4000", + api_key="your-proxy-api-key" +) + +print(response.choices[0].message.content) +``` + +```python showLineNumbers title="Docker Model Runner via Proxy - LiteLLM SDK Streaming" +import litellm + +# Configure LiteLLM to use your proxy with streaming +response = litellm.completion( + model="litellm_proxy/llama-3.1", + messages=[{"role": "user", "content": "hello from litellm"}], + api_base="http://localhost:4000", + api_key="your-proxy-api-key", + stream=True +) + +for chunk in response: + if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="") +``` + + + + + +```bash showLineNumbers title="Docker Model Runner via Proxy - cURL" +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "llama-3.1", + "messages": [{"role": "user", "content": "hello from litellm"}] + }' +``` + +```bash showLineNumbers title="Docker Model Runner via Proxy - cURL Streaming" +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "llama-3.1", + "messages": [{"role": "user", "content": "hello from litellm"}], + "stream": true + }' +``` + + + + +For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy). + +## API Reference + +For detailed API information, see the [Docker Model Runner API Reference](https://docs.docker.com/ai/model-runner/api-reference/). + diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md index c5014fc2ff3..e04225e1f85 100644 --- a/docs/my-website/docs/providers/gemini.md +++ b/docs/my-website/docs/providers/gemini.md @@ -70,7 +70,11 @@ LiteLLM translates OpenAI's `reasoning_effort` to Gemini's `thinking` parameter. Note: Reasoning cannot be turned off on Gemini 2.5 Pro models. ::: -**Mapping** +:::tip Gemini 3 Models +For **Gemini 3+ models** (e.g., `gemini-3-pro-preview`), LiteLLM automatically maps `reasoning_effort` to the new `thinking_level` parameter instead of `thinking_budget`. The `thinking_level` parameter uses `"low"` or `"high"` values for better control over reasoning depth. +::: + +**Mapping for Gemini 2.5 and earlier models** | reasoning_effort | thinking | Notes | | ---------------- | -------- | ----- | @@ -80,6 +84,17 @@ Note: Reasoning cannot be turned off on Gemini 2.5 Pro models. | "medium" | "budget_tokens": 2048 | | | "high" | "budget_tokens": 4096 | | +**Mapping for Gemini 3+ models** + +| reasoning_effort | thinking_level | Notes | +| ---------------- | -------------- | ----- | +| "minimal" | "low" | Minimizes latency and cost | +| "low" | "low" | Best for simple instruction following or chat | +| "medium" | "high" | Maps to high (medium not yet available) | +| "high" | "high" | Maximizes reasoning depth | +| "disable" | "low" | Cannot fully disable thinking in Gemini 3 | +| "none" | "low" | Cannot fully disable thinking in Gemini 3 | + @@ -137,6 +152,59 @@ curl http://0.0.0.0:4000/v1/chat/completions \ +### Gemini 3+ Models - `thinking_level` Parameter + +For Gemini 3+ models (e.g., `gemini-3-pro-preview`), you can use the new `thinking_level` parameter directly: + + + + +```python +from litellm import completion + +# Use thinking_level for Gemini 3 models +resp = completion( + model="gemini/gemini-3-pro-preview", + messages=[{"role": "user", "content": "Solve this complex math problem step by step."}], + reasoning_effort="high", # Options: "low" or "high" +) + +# Low thinking level for faster, simpler tasks +resp = completion( + model="gemini/gemini-3-pro-preview", + messages=[{"role": "user", "content": "What is the weather today?"}], + reasoning_effort="low", # Minimizes latency and cost +) +``` + + + + + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer " \ + -d '{ + "model": "gemini-3-pro-preview", + "messages": [{"role": "user", "content": "Solve this complex problem."}], + "reasoning_effort": "high" + }' +``` + + + + +:::warning +**Temperature Recommendation for Gemini 3 Models** + +For Gemini 3 models, LiteLLM defaults `temperature` to `1.0` and strongly recommends keeping it at this default. Setting `temperature < 1.0` can cause: +- Infinite loops +- Degraded reasoning performance +- Failure on complex tasks + +LiteLLM will automatically set `temperature=1.0` if not specified for Gemini 3+ models. +::: **Expected Response** @@ -951,6 +1019,297 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +## Thought Signatures + +Thought signatures are encrypted representations of the model's internal reasoning process for a given turn in a conversation. By passing thought signatures back to the model in subsequent requests, you provide it with the context of its previous thoughts, allowing it to build upon its reasoning and maintain a coherent line of inquiry. + +Thought signatures are particularly important for multi-turn function calling scenarios where the model needs to maintain context across multiple tool invocations. + +### How Thought Signatures Work + +- **Function calls with signatures**: When Gemini returns a function call, it includes a `thought_signature` in the response +- **Preservation**: LiteLLM automatically extracts and stores thought signatures in `provider_specific_fields` of tool calls +- **Return in conversation history**: When you include the assistant's message with tool calls in subsequent requests, LiteLLM automatically preserves and returns the thought signatures to Gemini +- **Parallel function calls**: Only the first function call in a parallel set has a thought signature +- **Sequential function calls**: Each function call in a multi-step sequence has its own signature + +### Enabling Thought Signatures + +To enable thought signatures, you need to enable thinking/reasoning: + + + + +```python +from litellm import completion + +response = completion( + model="gemini/gemini-2.5-flash", + messages=[{"role": "user", "content": "What's the weather in Tokyo?"}], + tools=[...], + reasoning_effort="low", # Enable thinking to get thought signatures +) +``` + + + + +```bash +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gemini-2.5-flash", + "messages": [{"role": "user", "content": "What'\''s the weather in Tokyo?"}], + "tools": [...], + "reasoning_effort": "low" + }' +``` + + + + +### Multi-Turn Function Calling with Thought Signatures + +When building conversation history for multi-turn function calling, you must include the thought signatures from previous responses. LiteLLM handles this automatically when you append the full assistant message to your conversation history. + + + + +```python +from openai import OpenAI +import json + +client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000") + +def get_current_temperature(location: str) -> dict: + """Gets the current weather temperature for a given location.""" + return {"temperature": 30, "unit": "celsius"} + +def set_thermostat_temperature(temperature: int) -> dict: + """Sets the thermostat to a desired temperature.""" + return {"status": "success"} + +get_weather_declaration = { + "name": "get_current_temperature", + "description": "Gets the current weather temperature for a given location.", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, +} + +set_thermostat_declaration = { + "name": "set_thermostat_temperature", + "description": "Sets the thermostat to a desired temperature.", + "parameters": { + "type": "object", + "properties": {"temperature": {"type": "integer"}}, + "required": ["temperature"], + }, +} + +# Initial request +messages = [ + {"role": "user", "content": "If it's too hot or too cold in London, set the thermostat to a comfortable level."} +] + +response = client.chat.completions.create( + model="gemini-2.5-flash", + messages=messages, + tools=[get_weather_declaration, set_thermostat_declaration], + reasoning_effort="low" +) + +# Append the assistant's message (includes thought signatures automatically) +messages.append(response.choices[0].message) + +# Execute tool calls and append results +for tool_call in response.choices[0].message.tool_calls: + if tool_call.function.name == "get_current_temperature": + result = get_current_temperature(**json.loads(tool_call.function.arguments)) + messages.append({ + "role": "tool", + "content": json.dumps(result), + "tool_call_id": tool_call.id + }) + +# Second request - thought signatures are automatically preserved +response2 = client.chat.completions.create( + model="gemini-2.5-flash", + messages=messages, + tools=[get_weather_declaration, set_thermostat_declaration], + reasoning_effort="low" +) + +print(response2.choices[0].message.content) +``` + + + + +```bash +# Step 1: Initial request +curl --location 'http://localhost:4000/v1/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer sk-1234' \ + --data '{ + "model": "gemini-2.5-flash", + "messages": [ + { + "role": "user", + "content": "If it'\''s too hot or too cold in London, set the thermostat to a comfortable level." + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_temperature", + "description": "Gets the current weather temperature for a given location.", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + } + }, + { + "type": "function", + "function": { + "name": "set_thermostat_temperature", + "description": "Sets the thermostat to a desired temperature.", + "parameters": { + "type": "object", + "properties": { + "temperature": {"type": "integer"} + }, + "required": ["temperature"] + } + } + } + ], + "tool_choice": "auto", + "reasoning_effort": "low" + }' +``` + +The response will include tool calls with thought signatures in `provider_specific_fields`: + +```json +{ + "choices": [{ + "message": { + "role": "assistant", + "tool_calls": [{ + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_current_temperature", + "arguments": "{\"location\": \"London\"}" + }, + "index": 0, + "provider_specific_fields": { + "thought_signature": "CpcHAdHtim9+q4rstcbvQC0ic4x1/vqQlCJWgE+UZ6dTLYGHMMBkF/AxqL5UmP6SY46uYC8t4BTFiXG5zkw6EMJ...==" + } + }] + } + }] +} +``` + +```bash +# Step 2: Follow-up request with tool response +# Include the assistant message from Step 1 (with thought signatures in provider_specific_fields) +curl --location 'http://localhost:4000/v1/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer sk-1234' \ + --data '{ + "model": "gemini-2.5-flash", + "messages": [ + { + "role": "user", + "content": "If it'\''s too hot or too cold in London, set the thermostat to a comfortable level." + }, + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_c130b9f8c2c042e9b65e39a88245", + "type": "function", + "function": { + "name": "get_current_temperature", + "arguments": "{\"location\": \"London\"}" + }, + "index": 0, + "provider_specific_fields": { + "thought_signature": "CpcHAdHtim9+q4rstcbvQC0ic4x1/vqQlCJWgE+UZ6dTLYGHMMBkF/AxqL5UmP6SY46uYC8t4BTFiXG5zkw6EMJ...==" + } + } + ] + }, + { + "role": "tool", + "content": "{\"temperature\": 30, \"unit\": \"celsius\"}", + "tool_call_id": "call_c130b9f8c2c042e9b65e39a88245" + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_temperature", + "description": "Gets the current weather temperature for a given location.", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + } + }, + { + "type": "function", + "function": { + "name": "set_thermostat_temperature", + "description": "Sets the thermostat to a desired temperature.", + "parameters": { + "type": "object", + "properties": { + "temperature": {"type": "integer"} + }, + "required": ["temperature"] + } + } + } + ], + "tool_choice": "auto", + "reasoning_effort": "low" + }' +``` + + + + +### Important Notes + +1. **Automatic Handling**: LiteLLM automatically extracts thought signatures from Gemini responses and preserves them when you include assistant messages in conversation history. You don't need to manually extract or manage them. + +2. **Parallel Function Calls**: When the model makes parallel function calls, only the first function call will have a thought signature. Subsequent parallel calls won't have signatures. + +3. **Sequential Function Calls**: In multi-step function calling scenarios, each step's first function call will have its own thought signature that must be preserved. + +4. **Required for Context**: Thought signatures are essential for maintaining reasoning context across multi-turn conversations with function calling. Without them, the model may lose context of its previous reasoning. + +5. **Format**: Thought signatures are stored in `provider_specific_fields.thought_signature` of tool calls in the response, and are automatically included when you append the assistant message to your conversation history. + +6. **Chat Completions Clients**: With chat completions clients where you cannot control whether or not the previous assistant message is included as-is (ex langchain's ChatOpenAI), LiteLLM also preserves the thought signature by appending it to the tool call id (`call_123__thought__`) and extracting it back out before sending the outbound request to Gemini. + ## JSON Mode @@ -1022,6 +1381,56 @@ LiteLLM Supports the following image types passed in `url` - Images with direct links - https://storage.googleapis.com/github-repo/img/gemini/intro/landmark3.jpg - Image in local storage - ./localimage.jpeg +## Image Resolution Control (Gemini 3+) + +For Gemini 3+ models, LiteLLM supports per-part media resolution control using OpenAI's `detail` parameter. This allows you to specify different resolution levels for individual images in your request. + +**Supported `detail` values:** +- `"low"` - Maps to `media_resolution: "low"` (280 tokens for images, 70 tokens per frame for videos) +- `"high"` - Maps to `media_resolution: "high"` (1120 tokens for images) +- `"auto"` or `None` - Model decides optimal resolution (no `media_resolution` set) + +**Usage Example:** + +```python +from litellm import completion + +messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://example.com/chart.png", + "detail": "high" # High resolution for detailed chart analysis + } + }, + { + "type": "text", + "text": "Analyze this chart" + }, + { + "type": "image_url", + "image_url": { + "url": "https://example.com/icon.png", + "detail": "low" # Low resolution for simple icon + } + } + ] + } +] + +response = completion( + model="gemini/gemini-3-pro-preview", + messages=messages, +) +``` + +:::info +**Per-Part Resolution:** Each image in your request can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This feature is only available for Gemini 3+ models. +::: + ## Sample Usage ```python import os diff --git a/docs/my-website/docs/providers/groq.md b/docs/my-website/docs/providers/groq.md index 59668b5eb5f..ebed31f720f 100644 --- a/docs/my-website/docs/providers/groq.md +++ b/docs/my-website/docs/providers/groq.md @@ -290,7 +290,7 @@ response = completion( { "type": "image_url", "image_url": { - "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + "url": "https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png" } } ] @@ -342,7 +342,7 @@ response = client.chat.completions.create( { "type": "image_url", "image_url": { - "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + "url": "https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png" } } ] diff --git a/docs/my-website/docs/providers/huggingface.md b/docs/my-website/docs/providers/huggingface.md index 399d49b5f46..985351e9f69 100644 --- a/docs/my-website/docs/providers/huggingface.md +++ b/docs/my-website/docs/providers/huggingface.md @@ -130,7 +130,7 @@ messages=[ { "type": "image_url", "image_url": { - "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", + "url": "https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png", } }, ], @@ -250,7 +250,7 @@ messages=[ { "type": "image_url", "image_url": { - "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", + "url": "https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png", } }, ], diff --git a/docs/my-website/docs/providers/oci.md b/docs/my-website/docs/providers/oci.md index cea5d6824a0..ce6fe18dd6f 100644 --- a/docs/my-website/docs/providers/oci.md +++ b/docs/my-website/docs/providers/oci.md @@ -58,12 +58,11 @@ This method is an alternative when using the LiteLLM SDK on Oracle Cloud Infrast ## Usage - + Input the parameters obtained from the OCI signing key creation process into the `completion` function: ```python -import os from litellm import completion messages = [{"role": "user", "content": "Hey! how's it going?"}] @@ -86,7 +85,7 @@ print(response) ``` - + Use the OCI SDK `Signer` for authentication: @@ -153,7 +152,6 @@ For applications running on OCI compute instances: from litellm import completion from oci.auth.signers import InstancePrincipalsSecurityTokenSigner -oci.auth.signers.get_oke_workload_identity_resource_principal_signer() # Use instance principal authentication signer = InstancePrincipalsSecurityTokenSigner() @@ -168,7 +166,7 @@ response = completion( print(response) ``` -**Use workload identity authentication** +**Workload Identity Authentication** For applications running in Oracle Kubernetes Engine (OKE): @@ -176,7 +174,7 @@ For applications running in Oracle Kubernetes Engine (OKE): from litellm import completion from oci.auth.signers import get_oke_workload_identity_resource_principal_signer -# Use instance principal authentication +# Use workload identity authentication signer = get_oke_workload_identity_resource_principal_signer() messages = [{"role": "user", "content": "Hey! how's it going?"}] @@ -196,10 +194,9 @@ print(response) Just set `stream=True` when calling completion. - + ```python -import os from litellm import completion messages = [{"role": "user", "content": "Hey! how's it going?"}] @@ -224,7 +221,7 @@ for chunk in response: ``` - + ```python from litellm import completion @@ -258,7 +255,27 @@ for chunk in response: ### Using Cohere Models - + + +```python +from litellm import completion + +messages = [{"role": "user", "content": "Explain quantum computing"}] +response = completion( + model="oci/cohere.command-latest", + messages=messages, + oci_region="us-chicago-1", + oci_user=, + oci_fingerprint=, + oci_tenancy=, + oci_key=, + oci_compartment_id=, +) +print(response) +``` + + + ```python from litellm import completion @@ -283,19 +300,28 @@ print(response) ``` - + + +## Using Dedicated Endpoints + +OCI supports dedicated endpoints for hosting models. Use the `oci_serving_mode="DEDICATED"` parameter along with `oci_endpoint_id` to specify the endpoint ID. + + + ```python from litellm import completion -messages = [{"role": "user", "content": "Explain quantum computing"}] +messages = [{"role": "user", "content": "Hey! how's it going?"}] response = completion( - model="oci/cohere.command-latest", + model="oci/xai.grok-4", # Must match the model type hosted on the endpoint messages=messages, - oci_region="us-chicago-1", + oci_region=, oci_user=, oci_fingerprint=, oci_tenancy=, + oci_serving_mode="DEDICATED", + oci_endpoint_id="ocid1.generativeaiendpoint.oc1...", # Your dedicated endpoint OCID oci_key=, oci_compartment_id=, ) @@ -303,4 +329,69 @@ print(response) ``` - \ No newline at end of file + + +```python +from litellm import completion +from oci.signer import Signer + +signer = Signer( + tenancy="ocid1.tenancy.oc1..", + user="ocid1.user.oc1..", + fingerprint="xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx", + private_key_file_location="~/.oci/key.pem", +) + +messages = [{"role": "user", "content": "Hey! how's it going?"}] +response = completion( + model="oci/xai.grok-4", # Must match the model type hosted on the endpoint + messages=messages, + oci_signer=signer, + oci_region="us-chicago-1", + oci_serving_mode="DEDICATED", + oci_endpoint_id="ocid1.generativeaiendpoint.oc1...", # Your dedicated endpoint OCID + oci_compartment_id="", +) +print(response) +``` + + + + +**Important:** When using `oci_serving_mode="DEDICATED"`: +- The `model` parameter **must match the type of model hosted on your dedicated endpoint** (e.g., use `"oci/cohere.command-latest"` for Cohere models, `"oci/xai.grok-4"` for Grok models) +- The model name determines the API format and vendor-specific handling (Cohere vs Generic) +- The `oci_endpoint_id` parameter specifies your dedicated endpoint's OCID +- If `oci_endpoint_id` is not provided, the `model` parameter will be used as the endpoint ID (for backward compatibility) + +**Example with Cohere Dedicated Endpoint:** +```python +# For a dedicated endpoint hosting a Cohere model +response = completion( + model="oci/cohere.command-latest", # Use Cohere model name to get Cohere API format + messages=messages, + oci_region="us-chicago-1", + oci_user=, + oci_fingerprint=, + oci_tenancy=, + oci_serving_mode="DEDICATED", + oci_endpoint_id="ocid1.generativeaiendpoint.oc1...", # Your Cohere endpoint OCID + oci_key=, + oci_compartment_id=, +) +``` + +## Optional Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `oci_region` | string | `us-ashburn-1` | OCI region where the GenAI service is deployed | +| `oci_serving_mode` | string | `ON_DEMAND` | Service mode: `ON_DEMAND` for managed models or `DEDICATED` for dedicated endpoints | +| `oci_endpoint_id` | string | Same as `model` | (For DEDICATED mode) The OCID of your dedicated endpoint | +| `oci_compartment_id` | string | **Required** | The OCID of the OCI compartment containing your resources | +| `oci_user` | string | - | (Manual auth) The OCID of the OCI user | +| `oci_fingerprint` | string | - | (Manual auth) The fingerprint of the API signing key | +| `oci_tenancy` | string | - | (Manual auth) The OCID of your OCI tenancy | +| `oci_key` | string | - | (Manual auth) The private key content as a string | +| `oci_key_file` | string | - | (Manual auth) Path to the private key file | +| `oci_signer` | object | - | (SDK auth) OCI SDK Signer object for authentication | \ No newline at end of file diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md index 99d17d8b21e..6f46807c89a 100644 --- a/docs/my-website/docs/providers/openai.md +++ b/docs/my-website/docs/providers/openai.md @@ -29,6 +29,18 @@ response = completion( ) ``` +:::info Metadata passthrough (preview) +When `litellm.enable_preview_features = True`, LiteLLM forwards only the values inside `metadata` to OpenAI. + +```python +completion( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + metadata= {"custom_meta_key": "value"}, +) +``` +::: + ### Usage - LiteLLM Proxy Server Here's how to call OpenAI models with the LiteLLM Proxy Server @@ -176,6 +188,9 @@ os.environ["OPENAI_BASE_URL"] = "https://your_host/v1" # OPTIONAL | gpt-5-mini-2025-08-07 | `response = completion(model="gpt-5-mini-2025-08-07", messages=messages)` | | gpt-5-nano-2025-08-07 | `response = completion(model="gpt-5-nano-2025-08-07", messages=messages)` | | gpt-5-pro | `response = completion(model="gpt-5-pro", messages=messages)` | +| gpt-5.1 | `response = completion(model="gpt-5.1", messages=messages)` | +| gpt-5.1-codex | `response = completion(model="gpt-5.1-codex", messages=messages)` | +| gpt-5.1-codex-mini | `response = completion(model="gpt-5.1-codex-mini", messages=messages)` | | gpt-4.1 | `response = completion(model="gpt-4.1", messages=messages)` | | gpt-4.1-mini | `response = completion(model="gpt-4.1-mini", messages=messages)` | | gpt-4.1-nano | `response = completion(model="gpt-4.1-nano", messages=messages)` | @@ -237,7 +252,7 @@ response = completion( { "type": "image_url", "image_url": { - "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + "url": "https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png" } } ] @@ -477,6 +492,8 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ | `gpt-5-mini` | `medium` | `none`, `minimal`, `low`, `medium`, `high` | | `gpt-5-nano` | `none` | `none`, `low`, `medium`, `high` | | `gpt-5-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) | +| `gpt-5.1-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) | +| `gpt-5.1-codex-mini` | `adaptive` | `low`, `medium`, `high` (no `minimal`) | | `gpt-5-pro` | `high` | `high` only | **Note:** @@ -490,7 +507,9 @@ See [OpenAI Reasoning documentation](https://platform.openai.com/docs/guides/rea The `verbosity` parameter controls the length and detail of responses from GPT-5 family models. It accepts three values: `"low"`, `"medium"`, or `"high"`. -**Supported models:** All GPT-5 family models (`gpt-5`, `gpt-5.1`, `gpt-5-mini`, `gpt-5-nano`, `gpt-5-codex`, `gpt-5-pro`) +**Supported models:** `gpt-5`, `gpt-5.1`, `gpt-5-mini`, `gpt-5-nano`, `gpt-5-pro` + +**Note:** GPT-5-Codex models (`gpt-5-codex`, `gpt-5.1-codex`, `gpt-5.1-codex-mini`) do **not** support the `verbosity` parameter. **Use cases:** - **`"low"`**: Best for concise answers or simple code generation (e.g., SQL queries) diff --git a/docs/my-website/docs/providers/snowflake.md b/docs/my-website/docs/providers/snowflake.md index 40deef87805..483bf939fe6 100644 --- a/docs/my-website/docs/providers/snowflake.md +++ b/docs/my-website/docs/providers/snowflake.md @@ -3,20 +3,15 @@ import TabItem from '@theme/TabItem'; # Snowflake -| Property | Details | -|-------|-------| -| Description | The Snowflake Cortex LLM REST API lets you access the COMPLETE function via HTTP POST requests| -| Provider Route on LiteLLM | `snowflake/` | -| Link to Provider Doc | [Snowflake ↗](https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-llm-rest-api) | -| Base URL | `https://{account-id}.snowflakecomputing.com/api/v2/cortex/inference:complete` | -| Supported OpenAI Endpoints | `/chat/completions`, `/completions` | +| Property | Details | +|----------------------------|-----------------------------------------------------------------------------------------------------------| +| Description | The Snowflake Cortex LLM REST API lets you access the COMPLETE and EMBED functions via HTTP POST requests | +| Provider Route on LiteLLM | `snowflake/` | +| Link to Provider Doc | [Snowflake ↗](https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-llm-rest-api) | +| Base URLs | `https://{account-id}.snowflakecomputing.com/api/v2/cortex/inference:complete`,`https://{account-id}.snowflakecomputing.com/api/v2/cortex/inference:embed`| +| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings` | - -Currently, Snowflake's REST API does not have an endpoint for `snowflake-arctic-embed` embedding models. If you want to use these embedding models with Litellm, you can call them through our Hugging Face provider. - -Find the Arctic Embed models [here](https://huggingface.co/collections/Snowflake/arctic-embed-661fd57d50fab5fc314e4c18) on Hugging Face. - ## Supported OpenAI Parameters ``` "temperature", @@ -29,6 +24,9 @@ Find the Arctic Embed models [here](https://huggingface.co/collections/Snowflake Snowflake does have API keys. Instead, you access the Snowflake API with your JWT token and account identifier. +It is also possible to use [programmatic access tokens](https://docs.snowflake.com/en/user-guide/programmatic-access-tokens) (PAT). It can be defined by using 'pat/' prefix + + ```python import os os.environ["SNOWFLAKE_JWT"] = "YOUR JWT" @@ -37,17 +35,38 @@ os.environ["SNOWFLAKE_ACCOUNT_ID"] = "YOUR ACCOUNT IDENTIFIER" ## Usage ```python -from litellm import completion +from litellm import completion, embedding ## set ENV variables -os.environ["SNOWFLAKE_JWT"] = "YOUR JWT" +os.environ["SNOWFLAKE_JWT"] = "JWT_TOKEN" os.environ["SNOWFLAKE_ACCOUNT_ID"] = "YOUR ACCOUNT IDENTIFIER" -# Snowflake call +# Snowflake completion call response = completion( model="snowflake/mistral-7b", messages = [{ "content": "Hello, how are you?","role": "user"}] ) + +# Snowflake embedding call +response = embedding( + model="snowflake/mistral-7b", + input = ["My text"] +) + +# Pass`api_key` and `account_id` as parameters +response = completion( + model="snowflake/mistral-7b", + messages = [{ "content": "Hello, how are you?","role": "user"}], + account_id="AAAA-BBBB", + api_key="JWT_TOKEN" +) + +# using PAT +response = completion( + model="snowflake/mistral-7b", + messages = [{ "content": "Hello, how are you?","role": "user"}], + api_key="pat/PAT_TOKEN" +) ``` ## Usage with LiteLLM Proxy diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 874b637e4db..70babea3814 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -1741,7 +1741,7 @@ response = litellm.completion( { "type": "image_url", "image_url": { - "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + "url": "https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png" } } ] diff --git a/docs/my-website/docs/providers/xai.md b/docs/my-website/docs/providers/xai.md index 49a3640991d..afeecc21528 100644 --- a/docs/my-website/docs/providers/xai.md +++ b/docs/my-website/docs/providers/xai.md @@ -11,6 +11,68 @@ https://docs.x.ai/docs ::: +## Supported Models + + + +**Latest Release** - Grok 4.1 Fast: Optimized for high-performance agentic tool calling with 2M context and prompt caching. + +| Model | Context | Features | +|-------|---------|----------| +| `xai/grok-4-1-fast-reasoning` | 2M tokens | **Reasoning**, Function calling, Vision, Audio, Web search, Caching | +| `xai/grok-4-1-fast-non-reasoning` | 2M tokens | Function calling, Vision, Audio, Web search, Caching | + +**When to use:** +- ✅ **Reasoning model**: Complex analysis, planning, multi-step reasoning problems +- ✅ **Non-reasoning model**: Simple queries, faster responses, lower token usage + +**Example:** +```python +from litellm import completion + +# With reasoning +response = completion( + model="xai/grok-4-1-fast-reasoning", + messages=[{"role": "user", "content": "Analyze this problem step by step..."}] +) + +# Without reasoning +response = completion( + model="xai/grok-4-1-fast-non-reasoning", + messages=[{"role": "user", "content": "What's 2+2?"}] +) +``` + +--- + +### All Available Models + +| Model Family | Model | Context | Features | +|--------------|-------|---------|----------| +| **Grok 4.1** | `xai/grok-4-1-fast-reasoning` | 2M | **Reasoning**, Tools, Vision, Audio, Web search, Caching | +| | `xai/grok-4-1-fast-non-reasoning` | 2M | Tools, Vision, Audio, Web search, Caching | +| **Grok 4** | `xai/grok-4` | 256K | Tools, Web search | +| | `xai/grok-4-0709` | 256K | Tools, Web search | +| | `xai/grok-4-fast-reasoning` | 2M | **Reasoning**, Tools, Web search | +| | `xai/grok-4-fast-non-reasoning` | 2M | Tools, Web search | +| **Grok 3** | `xai/grok-3` | 131K | Tools, Web search | +| | `xai/grok-3-mini` | 131K | Tools, Web search | +| | `xai/grok-3-fast-beta` | 131K | Tools, Web search | +| **Grok Code** | `xai/grok-code-fast` | 256K | **Reasoning**, Tools, Code generation, Caching | +| **Grok 2** | `xai/grok-2` | 131K | Tools, **Vision** | +| | `xai/grok-2-vision-latest` | 32K | Tools, **Vision** | + +**Features:** +- **Reasoning** = Chain-of-thought reasoning with reasoning tokens +- **Tools** = Function calling / Tool use +- **Web search** = Live internet search +- **Vision** = Image understanding +- **Audio** = Audio input support +- **Caching** = Prompt caching for cost savings +- **Code generation** = Optimized for code tasks + +**Pricing:** See [xAI's pricing page](https://docs.x.ai/docs/models) for current rates. + ## API Key ```python # env variable diff --git a/docs/my-website/docs/proxy/admin_ui_sso.md b/docs/my-website/docs/proxy/admin_ui_sso.md index ae082848b6b..0438c264685 100644 --- a/docs/my-website/docs/proxy/admin_ui_sso.md +++ b/docs/my-website/docs/proxy/admin_ui_sso.md @@ -380,3 +380,54 @@ If you need to inspect the JWT fields received from your SSO provider by LiteLLM Once redirected, you should see a page called "SSO Debug Information". This page displays the JWT fields received from your SSO provider (as shown in the image above) + +## Advanced + +### Manage User Roles via Azure App Roles + +Centralize role management by defining user permissions in Azure Entra ID. LiteLLM will automatically assign roles based on your Azure configuration when users sign in—no need to manually manage roles in LiteLLM. + +#### Step 1: Create App Roles on Azure App Registration + +1. Navigate to your App Registration on https://portal.azure.com/ +2. Go to **App roles** > **Create app role** +3. Configure the app role using one of the [supported LiteLLM roles](./access_control.md#global-proxy-roles): + - **Display name**: Admin Viewer (or your preferred display name) + - **Value**: `proxy_admin_viewer` (must match one of the LiteLLM role values exactly) +4. Click **Apply** to save the role +5. Repeat for each LiteLLM role you want to use + + +**Supported LiteLLM role values** (see [full role documentation](./access_control.md#global-proxy-roles)): +- `proxy_admin` - Full admin access +- `proxy_admin_viewer` - Read-only admin access +- `internal_user` - Can create/view/delete own keys +- `internal_user_viewer` - Can view own keys (read-only) + + + +--- + +#### Step 2: Assign Users to App Roles + +1. Navigate to **Enterprise Applications** on https://portal.azure.com/ +2. Select your LiteLLM application +3. Go to **Users and groups** > **Add user/group** +4. Select the user +5. Under **Select a role**, choose the app role you created (e.g., `proxy_admin_viewer`) +6. Click **Assign** to save + + + +--- + +#### Step 3: Sign in and verify + +1. Sign in to the LiteLLM UI via SSO +2. LiteLLM will automatically extract the app role from the JWT token +3. The user will be assigned the corresponding role (you can verify this in the UI by checking the user profile dropdown) + + + +**Note:** The role from Entra ID will take precedence over any existing role in the LiteLLM database. This ensures your SSO provider is the authoritative source for user roles. + diff --git a/docs/my-website/docs/proxy/ai_hub.md b/docs/my-website/docs/proxy/ai_hub.md new file mode 100644 index 00000000000..a7865db6cdb --- /dev/null +++ b/docs/my-website/docs/proxy/ai_hub.md @@ -0,0 +1,240 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# AI Hub + +Share models and agents with your organization. Show developers what's available without needing to rebuild them. + +This feature is **available in v1.74.3-stable and above**. + +## Overview + +Admin can select models/agents to expose on public AI hub → Users go to the public url and see what's available. + + + +## Models + +### How to use + +#### 1. Go to the Admin UI + +Navigate to the Model Hub page in the Admin UI (`PROXY_BASE_URL/ui/?login=success&page=model-hub-table`) + + + +#### 2. Select the models you want to expose + +Click on `Select Models to Make Public` and select the models you want to expose. + + + +#### 3. Confirm the changes + + + +#### 4. Success! + +Go to the public url (`PROXY_BASE_URL/ui/model_hub_table`) and see available models. + + + +### API Endpoints + +- `GET /public/model_hub` – returns the list of public model groups. Requires a valid user API key. +- `GET /public/model_hub/info` – returns metadata (docs title, version, useful links) for the public model hub. + +## Agents + +:::info +Agents are only available in v1.79.4-stable and above. +::: + +Share pre-built agents (A2A spec) across your organization. Users can discover and use agents without rebuilding them. + +[**Demo Video**](https://drive.google.com/file/d/1r-_Rtiu04RW5Fwwu3_eshtA1oZtC3_DH/view?usp=sharing) + +### 1. Create an agent + +Create an agent that follows the [A2A spec](https://a2a.dev/). + + + + + + + + +```bash +curl -X POST 'http://0.0.0.0:4000/v1/agents' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data '{ + "agent_name": "hello-world-agent", + "agent_card_params": { + "protocolVersion": "1.0", + "name": "Hello World Agent", + "description": "Just a hello world agent", + "url": "http://localhost:9999/", + "version": "1.0.0", + "defaultInputModes": ["text"], + "defaultOutputModes": ["text"], + "capabilities": { + "streaming": true + }, + "skills": [ + { + "id": "hello_world", + "name": "Returns hello world", + "description": "just returns hello world", + "tags": ["hello world"], + "examples": ["hi", "hello world"] + } + ] + } +}' +``` + +**Expected Response** + +```json +{ + "agent_id": "123e4567-e89b-12d3-a456-426614174000", + "agent_name": "hello-world-agent", + "agent_card_params": { + "protocolVersion": "1.0", + "name": "Hello World Agent", + "description": "Just a hello world agent", + "url": "http://localhost:9999/", + "version": "1.0.0", + "defaultInputModes": ["text"], + "defaultOutputModes": ["text"], + "capabilities": { + "streaming": true + }, + "skills": [ + { + "id": "hello_world", + "name": "Returns hello world", + "description": "just returns hello world", + "tags": ["hello world"], + "examples": ["hi", "hello world"] + } + ] + }, + "created_at": "2025-11-15T10:30:00Z", + "created_by": "user123" +} +``` + + + + +### 2. Make agent public + +Make the agent discoverable on the AI Hub. + + + + +Navigate to the Agents Tab on the AI Hub page + + + +Select the agents you want to make public and click on `Make Public` button. + + + + + + +**Option 1: Make single agent public** + +```bash +curl -X POST 'http://0.0.0.0:4000/v1/agents/123e4567-e89b-12d3-a456-426614174000/make_public' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' +``` + +**Option 2: Make multiple agents public** + + +```bash +curl -X POST 'http://0.0.0.0:4000/v1/agents/make_public' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data '{ + "agent_ids": [ + "123e4567-e89b-12d3-a456-426614174000", + "123e4567-e89b-12d3-a456-426614174001" + ] +}' +``` + +**Expected Response** + +```json +{ + "message": "Successfully updated public agent groups", + "public_agent_groups": [ + "123e4567-e89b-12d3-a456-426614174000" + ], + "updated_by": "user123" +} +``` + + + + + + + +### 3. View public agents + +Users can now discover the agent via the public endpoint. + + + + + + + + + +```bash +curl -X GET 'http://0.0.0.0:4000/public/agent_hub' \ +--header 'Authorization: Bearer ' +``` + +**Expected Response** + +```json +[ + { + "protocolVersion": "1.0", + "name": "Hello World Agent", + "description": "Just a hello world agent", + "url": "http://localhost:9999/", + "version": "1.0.0", + "defaultInputModes": ["text"], + "defaultOutputModes": ["text"], + "capabilities": { + "streaming": true + }, + "skills": [ + { + "id": "hello_world", + "name": "Returns hello world", + "description": "just returns hello world", + "tags": ["hello world"], + "examples": ["hi", "hello world"] + } + ] + } +] +``` + + + + diff --git a/docs/my-website/docs/proxy/cli_sso.md b/docs/my-website/docs/proxy/cli_sso.md index f7669d6a25c..cde6bf266d4 100644 --- a/docs/my-website/docs/proxy/cli_sso.md +++ b/docs/my-website/docs/proxy/cli_sso.md @@ -9,6 +9,26 @@ Use the litellm cli to authenticate to the LiteLLM Gateway. This is great if you ## Usage +### Prerequisites - Start LiteLLM Proxy with Beta Flag + +:::warning[Beta Feature - Required] + +CLI SSO Authentication is currently in beta. You must set this environment variable **when starting up your LiteLLM Proxy**: + +```bash +export EXPERIMENTAL_UI_LOGIN="True" +litellm --config config.yaml +``` + +Or add it to your proxy startup command: + +```bash +EXPERIMENTAL_UI_LOGIN="True" litellm --config config.yaml +``` + +::: + +### Steps 1. **Install the CLI** @@ -33,6 +53,8 @@ Use the litellm cli to authenticate to the LiteLLM Gateway. This is great if you 2. **Set up environment variables** + On your local machine, set the proxy URL: + ```bash export LITELLM_PROXY_URL=http://localhost:4000 ``` diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 4d02d5729bf..67b5ad26fb9 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -655,6 +655,7 @@ router_settings: | LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRY_TIMEOUT | Timeout for retries of parallel requests in LiteLLM | LITELLM_MIGRATION_DIR | Custom migrations directory for prisma migrations, used for baselining db in read-only file systems. | LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM +| LITELLM_UI_API_DOC_BASE_URL | Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. Defaults to `PROXY_BASE_URL` when unset. | LITELM_ENVIRONMENT | Environment of LiteLLM Instance, used by logging services. Currently only used by DeepEval. | LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false. | LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours). diff --git a/docs/my-website/docs/proxy/control_plane_and_data_plane.md b/docs/my-website/docs/proxy/control_plane_and_data_plane.md index db0b7884c92..b0fe2b71ee2 100644 --- a/docs/my-website/docs/proxy/control_plane_and_data_plane.md +++ b/docs/my-website/docs/proxy/control_plane_and_data_plane.md @@ -163,6 +163,10 @@ DISABLE_LLM_API_ENDPOINTS=true - `/config/*` - Configuration updates - All other administrative endpoints +### `LITELLM_UI_API_DOC_BASE_URL` + +Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. + ## Usage Patterns diff --git a/docs/my-website/docs/proxy/db_info.md b/docs/my-website/docs/proxy/db_info.md index 946089bf147..5ef9fa55043 100644 --- a/docs/my-website/docs/proxy/db_info.md +++ b/docs/my-website/docs/proxy/db_info.md @@ -46,8 +46,8 @@ You can see the full DB Schema [here](https://github.com/BerriAI/litellm/blob/ma | Table Name | Description | Row Insert Frequency | |------------|-------------|---------------------| -| LiteLLM_SpendLogs | Detailed logs of all API requests. Records token usage, spend, and timing information. Tracks which models and keys were used. | **High - every LLM API request - Success or Failure** | -| LiteLLM_AuditLog | Tracks changes to system configuration. Records who made changes and what was modified. Maintains history of updates to teams, users, and models. | **Off by default**, **High - when enabled** | +| LiteLLM_SpendLogs | Detailed logs of all API requests. Records token usage, spend, and timing information. Tracks which models and keys were used. | **Medium - this is a batch process that runs on an interval.** | +| LiteLLM_AuditLog | Tracks changes to system configuration. Records who made changes and what was modified. Maintains history of updates to teams, users, and models. | **Off by default**, **High - Runs on every change to an entity** | ## Disable `LiteLLM_SpendLogs` diff --git a/docs/my-website/docs/proxy/dynamic_logging.md b/docs/my-website/docs/proxy/dynamic_logging.md index 3bc9f72b033..42df221bb84 100644 --- a/docs/my-website/docs/proxy/dynamic_logging.md +++ b/docs/my-website/docs/proxy/dynamic_logging.md @@ -211,4 +211,64 @@ x-litellm-disable-callbacks: LANGFUSE,datadog,PROMETHEUS x-litellm-disable-callbacks: langfuse,DATADOG,prometheus ``` +--- + +## Disabling Dynamic Callback Management (Enterprise) + +Some organizations have compliance requirements where **all requests must be logged under all circumstances**. For these cases, you can disable dynamic callback management entirely to ensure users cannot disable any logging callbacks. + +### Use Case + +This is designed for enterprise scenarios where: +- **Compliance requirements** mandate that all API requests must be logged +- **Audit trails** must be complete with no gaps +- **Security policies** require all traffic to be monitored +- **No exceptions** can be made for callback disabling + +### How to Disable + +Set `allow_dynamic_callback_disabling` to `false` in your config.yaml: + +```yaml showLineNumbers title="config.yaml" +litellm_settings: + allow_dynamic_callback_disabling: false +``` + +### Effect + +When disabled: +- The `x-litellm-disable-callbacks` header will be **ignored** +- All configured callbacks will **always execute** for every request +- Users cannot bypass logging through headers or request metadata +- All requests are guaranteed to be logged per your proxy configuration + +### Example: Compliance Logging Setup + +Here's a complete example for an organization requiring guaranteed logging: + +```yaml showLineNumbers title="config.yaml" +# config.yaml +model_list: + - model_name: gpt-4 + litellm_params: + model: openai/gpt-4 + api_key: os.environ/OPENAI_API_KEY + +litellm_settings: + callbacks: ["langfuse", "datadog", "s3"] + # Disable dynamic callback disabling for compliance + allow_dynamic_callback_disabling: false +``` + +With this configuration: +- All requests will be logged to Langfuse, Datadog, and S3 +- Users cannot disable any of these callbacks via headers +- Complete audit trail is guaranteed for compliance requirements + +:::info + +**Default Behavior**: Dynamic callback disabling is **enabled by default** (`allow_dynamic_callback_disabling: true`). You must explicitly set it to `false` to enforce guaranteed logging. + +::: + diff --git a/docs/my-website/docs/proxy/enterprise.md b/docs/my-website/docs/proxy/enterprise.md index b5d8ab59077..cfd6ab31015 100644 --- a/docs/my-website/docs/proxy/enterprise.md +++ b/docs/my-website/docs/proxy/enterprise.md @@ -901,9 +901,11 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ' ``` -## Public Model Hub +## Public AI Hub -Share a public page of available models for users +Share a public page of available models and agents for users + +[Learn more](./ai_hub.md) diff --git a/docs/my-website/docs/proxy/guardrails/custom_guardrail.md b/docs/my-website/docs/proxy/guardrails/custom_guardrail.md index b8ba64d333a..365fdf81aa5 100644 --- a/docs/my-website/docs/proxy/guardrails/custom_guardrail.md +++ b/docs/my-website/docs/proxy/guardrails/custom_guardrail.md @@ -4,151 +4,86 @@ import TabItem from '@theme/TabItem'; # Custom Guardrail -Use this is you want to write code to run a custom guardrail +Use this if you want to write code to run a custom guardrail ## Quick Start ### 1. Write a `CustomGuardrail` Class -A CustomGuardrail has 4 methods to enforce guardrails -- `async_pre_call_hook` - (Optional) modify input or reject request before making LLM API call -- `async_moderation_hook` - (Optional) reject request, runs while making LLM API call (help to lower latency) -- `async_post_call_success_hook`- (Optional) apply guardrail on input/output, runs after making LLM API call -- `async_post_call_streaming_iterator_hook` - (Optional) pass the entire stream to the guardrail - - -**[See detailed spec of methods here](#customguardrail-methods)** +The simplest way to create a custom guardrail is by implementing the `apply_guardrail` method. This method is called to check text content and can block requests by raising an exception. **Example `CustomGuardrail` Class** -Create a new file called `custom_guardrail.py` and add this code to it +Create a new file called `custom_guardrail.py` and add this code to it: + ```python -from typing import Any, AsyncGenerator, Literal, Optional, Union - -import litellm -from litellm._logging import verbose_proxy_logger -from litellm.caching.caching import DualCache +import os +from typing import Optional, List from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.proxy._types import UserAPIKeyAuth -from litellm.types.utils import ModelResponseStream - +from litellm.types.guardrails import PiiEntityType +from litellm._logging import verbose_proxy_logger +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, +) class myCustomGuardrail(CustomGuardrail): - def __init__( - self, - **kwargs, - ): - # store kwargs as optional_params - self.optional_params = kwargs - + def __init__(self, api_key: Optional[str] = None, api_base: Optional[str] = None, **kwargs): + self.api_key = api_key or os.getenv("MY_GUARDRAIL_API_KEY") + self.api_base = api_base or os.getenv("MY_GUARDRAIL_API_BASE", "https://api.myguardrail.com") super().__init__(**kwargs) - async def async_pre_call_hook( + async def apply_guardrail( self, - user_api_key_dict: UserAPIKeyAuth, - cache: DualCache, - data: dict, - call_type: Literal[ - "completion", - "text_completion", - "embeddings", - "image_generation", - "moderation", - "audio_transcription", - "pass_through_endpoint", - "rerank" - ], - ) -> Optional[Union[Exception, str, dict]]: + text: str, # IMPORTANT: This is the text to check against your guardrail rules. It's extracted from the request or response across all LLM call types. + language: Optional[str] = None, # ignore + entities: Optional[List[PiiEntityType]] = None, # ignore + request_data: Optional[dict] = None, # ignore + ) -> str: """ - Runs before the LLM API call - Runs on only Input - Use this if you want to MODIFY the input + Check text content against your guardrail rules. + Raise an exception to block the request. + Return the text (optionally modified) to allow it through. """ + result = await self._check_with_api(text, request_data) + + if result.get("action") == "BLOCK": + raise Exception(f"Content blocked: {result.get('reason', 'Policy violation')}") + + return text - # In this guardrail, if a user inputs `litellm` we will mask it and then send it to the LLM - _messages = data.get("messages") - if _messages: - for message in _messages: - _content = message.get("content") - if isinstance(_content, str): - if "litellm" in _content.lower(): - _content = _content.replace("litellm", "********") - message["content"] = _content - - verbose_proxy_logger.debug( - "async_pre_call_hook: Message after masking %s", _messages + async def _check_with_api(self, text: str, request_data: Optional[dict]) -> dict: + async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback) + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.api_key}", + } + + response = await async_client.post( + f"{self.api_base}/check", + headers=headers, + json={"text": text}, + timeout=5, ) - - return data - - async def async_moderation_hook( - self, - data: dict, - user_api_key_dict: UserAPIKeyAuth, - call_type: Literal["completion", "embeddings", "image_generation", "moderation", "audio_transcription"], - ): - """ - Runs in parallel to LLM API call - Runs on only Input - - This can NOT modify the input, only used to reject or accept a call before going to LLM API - """ - - # this works the same as async_pre_call_hook, but just runs in parallel as the LLM API Call - # In this guardrail, if a user inputs `litellm` we will mask it. - _messages = data.get("messages") - if _messages: - for message in _messages: - _content = message.get("content") - if isinstance(_content, str): - if "litellm" in _content.lower(): - raise ValueError("Guardrail failed words - `litellm` detected") - - async def async_post_call_success_hook( - self, - data: dict, - user_api_key_dict: UserAPIKeyAuth, - response, - ): - """ - Runs on response from LLM API call - - It can be used to reject a response - - If a response contains the word "coffee" -> we will raise an exception - """ - verbose_proxy_logger.debug("async_pre_call_hook response: %s", response) - if isinstance(response, litellm.ModelResponse): - for choice in response.choices: - if isinstance(choice, litellm.Choices): - verbose_proxy_logger.debug("async_pre_call_hook choice: %s", choice) - if ( - choice.message.content - and isinstance(choice.message.content, str) - and "coffee" in choice.message.content - ): - raise ValueError("Guardrail failed Coffee Detected") - - async def async_post_call_streaming_iterator_hook( - self, - user_api_key_dict: UserAPIKeyAuth, - response: Any, - request_data: dict, - ) -> AsyncGenerator[ModelResponseStream, None]: - """ - Passes the entire stream to the guardrail - - This is useful for guardrails that need to see the entire response, such as PII masking. - - See Aim guardrail implementation for an example - https://github.com/BerriAI/litellm/blob/d0e022cfacb8e9ebc5409bb652059b6fd97b45c0/litellm/proxy/guardrails/guardrail_hooks/aim.py#L168 - - Triggered by mode: 'post_call' - """ - async for item in response: - yield item - + + response.raise_for_status() + return response.json() ``` +:::tip Advanced: Using Individual Event Hooks + +If you need more fine-grained control, you can implement individual event hooks instead of (or in addition to) `apply_guardrail`: + +- `async_pre_call_hook` - Modify input or reject request before making LLM API call +- `async_moderation_hook` - Reject request, runs in parallel with LLM API call (helps lower latency) +- `async_post_call_success_hook` - Apply guardrail on input/output, runs after making LLM API call +- `async_post_call_streaming_iterator_hook` - Pass the entire stream to the guardrail + +**[See examples of individual event hooks here](#advanced-individual-event-hooks)** | **[See detailed spec of methods here](#customguardrail-methods)** + +::: + ### 2. Pass your custom guardrail class in LiteLLM `config.yaml` In the config below, we point the guardrail to our custom guardrail by setting `guardrail: custom_guardrail.myCustomGuardrail` @@ -166,9 +101,32 @@ model_list: api_key: os.environ/OPENAI_API_KEY guardrails: - - guardrail_name: "custom-pre-guard" + - guardrail_name: "my-custom-guardrail" litellm_params: guardrail: custom_guardrail.myCustomGuardrail # 👈 Key change + mode: "during_call" # runs apply_guardrail method + api_key: os.environ/MY_GUARDRAIL_API_KEY + api_base: https://api.myguardrail.com +``` + +:::info Mode Options + +- `during_call` - Default mode, runs `apply_guardrail` method (or `async_moderation_hook` if using individual hooks) +- `pre_call` - Runs `async_pre_call_hook` for input modification +- `post_call` - Runs `async_post_call_success_hook` for output validation + +::: + +
+Advanced: Multiple modes with individual event hooks + +If you're using individual event hooks, you can configure multiple guardrails with different modes: + +```yaml +guardrails: + - guardrail_name: "custom-pre-guard" + litellm_params: + guardrail: custom_guardrail.myCustomGuardrail mode: "pre_call" # runs async_pre_call_hook - guardrail_name: "custom-during-guard" litellm_params: @@ -180,6 +138,8 @@ guardrails: mode: "post_call" # runs async_post_call_success_hook ``` +
+ ### 3. Start LiteLLM Gateway @@ -218,15 +178,76 @@ litellm --config config.yaml --detailed_debug ### 4. Test it -#### Test `"custom-pre-guard"` - - **[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** + + + +This request will be blocked if it violates your guardrail policy: + +```shell +curl -i -X POST http://localhost:4000/v1/chat/completions \ +-H "Content-Type: application/json" \ +-H "Authorization: Bearer sk-1234" \ +-d '{ + "model": "gpt-4", + "messages": [ + { + "role": "user", + "content": "Content that violates policy" + } + ], + "guardrails": ["my-custom-guardrail"] +}' +``` + +Expected response when blocked: + +```json +{ + "error": { + "message": "Content blocked: Policy violation", + "type": "None", + "param": "None", + "code": "500" + } +} +``` + + + + + +This request passes the guardrail: + +```shell +curl -i http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-4", + "messages": [ + {"role": "user", "content": "What is the weather like today?"} + ], + "guardrails": ["my-custom-guardrail"] + }' +``` + + + + + +
+Advanced: Testing individual event hooks + +If you're using individual event hooks, you can test each mode separately: + +#### Test `"custom-pre-guard"` + -Expect this to mask the word `litellm` before sending the request to the LLM API. [This runs the `async_pre_call_hook`](#1-write-a-customguardrail-class) +Expect this to mask the word `litellm` before sending the request to the LLM API. [This runs the `async_pre_call_hook`](#advanced-individual-event-hooks) ```shell curl -i -X POST http://localhost:4000/v1/chat/completions \ @@ -244,37 +265,6 @@ curl -i -X POST http://localhost:4000/v1/chat/completions \ }' ``` -Expected response after pre-guard - -```json -{ - "id": "chatcmpl-9zREDkBIG20RJB4pMlyutmi1hXQWc", - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "It looks like you've chosen a string of asterisks. This could be a way to censor or hide certain text. However, without more context, I can't provide a specific word or phrase. If there's something specific you'd like me to say or if you need help with a topic, feel free to let me know!", - "role": "assistant", - "tool_calls": null, - "function_call": null - } - } - ], - "created": 1724429701, - "model": "gpt-4o-2024-05-13", - "object": "chat.completion", - "system_fingerprint": "fp_3aa7262c27", - "usage": { - "completion_tokens": 65, - "prompt_tokens": 14, - "total_tokens": 79 - }, - "service_tier": null -} - -``` - @@ -282,7 +272,7 @@ Expected response after pre-guard ```shell curl -i http://localhost:4000/v1/chat/completions \ -H "Content-Type: application/json" \ - -H "Authorization: Bearer sk-npnwjPQciVRok5yNZgKmFQ" \ + -H "Authorization: Bearer sk-1234" \ -d '{ "model": "gpt-3.5-turbo", "messages": [ @@ -294,20 +284,14 @@ curl -i http://localhost:4000/v1/chat/completions \ - - #### Test `"custom-during-guard"` - -**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** - -Expect this to fail since since `litellm` is in the message content. [This runs the `async_moderation_hook`](#1-write-a-customguardrail-class) - +Expect this to fail since `litellm` is in the message content. [This runs the `async_moderation_hook`](#advanced-individual-event-hooks) ```shell curl -i -X POST http://localhost:4000/v1/chat/completions \ @@ -325,7 +309,7 @@ curl -i -X POST http://localhost:4000/v1/chat/completions \ }' ``` -Expected response after running during-guard +Expected response: ```json { @@ -345,7 +329,7 @@ Expected response after running during-guard ```shell curl -i http://localhost:4000/v1/chat/completions \ -H "Content-Type: application/json" \ - -H "Authorization: Bearer sk-npnwjPQciVRok5yNZgKmFQ" \ + -H "Authorization: Bearer sk-1234" \ -d '{ "model": "gpt-3.5-turbo", "messages": [ @@ -357,21 +341,14 @@ curl -i http://localhost:4000/v1/chat/completions \ - - #### Test `"custom-post-guard"` - - -**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** - -Expect this to fail since since `coffee` will be in the response content. [This runs the `async_post_call_success_hook`](#1-write-a-customguardrail-class) - +Expect this to fail since `coffee` will be in the response content. [This runs the `async_post_call_success_hook`](#advanced-individual-event-hooks) ```shell curl -i -X POST http://localhost:4000/v1/chat/completions \ @@ -389,7 +366,7 @@ curl -i -X POST http://localhost:4000/v1/chat/completions \ }' ``` -Expected response after running during-guard +Expected response: ```json { @@ -407,7 +384,7 @@ Expected response after running during-guard ```shell - curl -i -X POST http://localhost:4000/v1/chat/completions \ +curl -i -X POST http://localhost:4000/v1/chat/completions \ -H "Content-Type: application/json" \ -H "Authorization: Bearer sk-1234" \ -d '{ @@ -424,9 +401,10 @@ Expected response after running during-guard - +
+ ## ✨ Pass additional parameters to guardrail :::info @@ -539,10 +517,162 @@ The `get_guardrail_dynamic_request_body_params` method will return: } ``` +## Advanced: Individual Event Hooks + +Pro: More flexibility +Con: You need to implement this for each LLM call type (chat completions, text completions, embeddings, image generation, moderation, audio transcription, pass through endpoint, rerank, etc. ) + +For more fine-grained control over when and how your guardrail runs, you can implement individual event hooks. This gives you flexibility to: +- Modify inputs before the LLM call +- Run checks in parallel with the LLM call (lower latency) +- Validate or modify outputs after the LLM call +- Process streaming responses + +### Example with Individual Event Hooks + +```python +from typing import Any, AsyncGenerator, Literal, Optional, Union + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.caching.caching import DualCache +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.utils import ModelResponseStream, CallTypes + + +class myCustomGuardrail(CustomGuardrail): + def __init__( + self, + **kwargs, + ): + # store kwargs as optional_params + self.optional_params = kwargs + + super().__init__(**kwargs) + + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: dict, + call_type: Optional[CallTypes], + ) -> Optional[Union[Exception, str, dict]]: + """ + Runs before the LLM API call + Runs on only Input + Use this if you want to MODIFY the input + """ + + # In this guardrail, if a user inputs `litellm` we will mask it and then send it to the LLM + _messages = data.get("messages") + if _messages: + for message in _messages: + _content = message.get("content") + if isinstance(_content, str): + if "litellm" in _content.lower(): + _content = _content.replace("litellm", "********") + message["content"] = _content + + verbose_proxy_logger.debug( + "async_pre_call_hook: Message after masking %s", _messages + ) + + return data + + async def async_moderation_hook( + self, + data: dict, + user_api_key_dict: UserAPIKeyAuth, + call_type: Literal["completion", "embeddings", "image_generation", "moderation", "audio_transcription"], + ): + """ + Runs in parallel to LLM API call + Runs on only Input + + This can NOT modify the input, only used to reject or accept a call before going to LLM API + """ + + # this works the same as async_pre_call_hook, but just runs in parallel as the LLM API Call + # In this guardrail, if a user inputs `litellm` we will mask it. + _messages = data.get("messages") + if _messages: + for message in _messages: + _content = message.get("content") + if isinstance(_content, str): + if "litellm" in _content.lower(): + raise ValueError("Guardrail failed words - `litellm` detected") + + async def async_post_call_success_hook( + self, + data: dict, + user_api_key_dict: UserAPIKeyAuth, + response, + ): + """ + Runs on response from LLM API call + + It can be used to reject a response + + If a response contains the word "coffee" -> we will raise an exception + """ + verbose_proxy_logger.debug("async_pre_call_hook response: %s", response) + if isinstance(response, litellm.ModelResponse): + for choice in response.choices: + if isinstance(choice, litellm.Choices): + verbose_proxy_logger.debug("async_pre_call_hook choice: %s", choice) + if ( + choice.message.content + and isinstance(choice.message.content, str) + and "coffee" in choice.message.content + ): + raise ValueError("Guardrail failed Coffee Detected") + + async def async_post_call_streaming_iterator_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + response: Any, + request_data: dict, + ) -> AsyncGenerator[ModelResponseStream, None]: + """ + Passes the entire stream to the guardrail + + This is useful for guardrails that need to see the entire response, such as PII masking. + + See Aim guardrail implementation for an example - https://github.com/BerriAI/litellm/blob/d0e022cfacb8e9ebc5409bb652059b6fd97b45c0/litellm/proxy/guardrails/guardrail_hooks/aim.py#L168 + + Triggered by mode: 'post_call' + """ + async for item in response: + yield item + +``` + ## **CustomGuardrail methods** | Component | Description | Optional | Checked Data | Can Modify Input | Can Modify Output | Can Fail Call | |-----------|-------------|----------|--------------|------------------|-------------------|----------------| +| `apply_guardrail` | Simple method to check and optionally modify text | ✅ | INPUT or OUTPUT | ✅ | ✅ | ✅ | | `async_pre_call_hook` | A hook that runs before the LLM API call | ✅ | INPUT | ✅ | ❌ | ✅ | | `async_moderation_hook` | A hook that runs during the LLM API call| ✅ | INPUT | ❌ | ❌ | ✅ | | `async_post_call_success_hook` | A hook that runs after a successful LLM API call| ✅ | INPUT, OUTPUT | ❌ | ✅ | ✅ | +| `async_post_call_streaming_iterator_hook` | A hook that processes streaming responses | ✅ | OUTPUT | ❌ | ✅ | ✅ | + + +## Frequently Asked Questions + +**Q. Is `apply_guardrail` relevant both in the request and in the response (pre_call, during_call and post_call hooks)?** + +**A.** Yes, one function works in both - See implementation [here](https://github.com/BerriAI/litellm/blob/0292b84dc47473ddeff29bd5a86f529bc523034b/litellm/proxy/utils.py#L825) + +**Q. What do I get in the inputs of `apply_guardrail`? What does each field represent (what is text, language, entities, request_data)?** + +**A.** The main one you should care about is 'text' - this is what you'll want to send to your api for verification - See implementation [here](https://github.com/BerriAI/litellm/blob/0292b84dc47473ddeff29bd5a86f529bc523034b/litellm/llms/anthropic/chat/guardrail_translation/handler.py#L102) + +**Q. Is this function agnostic to the LLM provider? Meaning does it pass the same values for OpenAI and Anthropic for example? + +**A.** Yes + +**Q. How do I know if my guardrail is running?** + +**A.** If you implement `apply_guardrail`, you can query the guardrail directly via [the `/apply_guardrail` API](../../apply_guardrail). \ No newline at end of file diff --git a/docs/my-website/docs/proxy/guardrails/grayswan.md b/docs/my-website/docs/proxy/guardrails/grayswan.md index b510c870a1e..7cc75b9f3b6 100644 --- a/docs/my-website/docs/proxy/guardrails/grayswan.md +++ b/docs/my-website/docs/proxy/guardrails/grayswan.md @@ -142,8 +142,8 @@ Provides the strongest enforcement by inspecting both prompts and responses. |---------------------------------------|-----------------|-------------| | `api_key` | string | Gray Swan Cygnal API key. Reads from `GRAYSWAN_API_KEY` if omitted. | | `mode` | string or list | Guardrail stages (`pre_call`, `during_call`, `post_call`). | -| `optional_params.on_flagged_action` | string | `monitor` (log only) or `block` (raise `HTTPException`). | +| `optional_params.on_flagged_action` | string | `monitor` (log only), `block` (raise `HTTPException`), or `passthrough` (include detection info in response without blocking). | | `.optional_params.violation_threshold`| number (0-1) | Scores at or above this value are considered violations. | -| `optional_params.reasoning_mode` | string | `off`, `hybrid`, or `thinking`. Enables Cygnal’s reasoning capabilities. | +| `optional_params.reasoning_mode` | string | `off`, `hybrid`, or `thinking`. Enables Cygnal's reasoning capabilities. | | `optional_params.categories` | object | Map of custom category names to descriptions. | | `optional_params.policy_id` | string | Gray Swan policy identifier. | diff --git a/docs/my-website/docs/proxy/guardrails/ibm_guardrails.md b/docs/my-website/docs/proxy/guardrails/ibm_guardrails.md index 0c13d2dcea9..43ba6622078 100644 --- a/docs/my-website/docs/proxy/guardrails/ibm_guardrails.md +++ b/docs/my-website/docs/proxy/guardrails/ibm_guardrails.md @@ -95,6 +95,7 @@ curl -i http://localhost:4000/v1/chat/completions \ These go under `optional_params`: - `detector_params` - dict - Parameters to pass to your detector +- `extra_headers` - dict - Additional headers to inject into requests to IBM Guardrails, as a key-value dict. - `score_threshold` - float - Only count detections above this score (0.0 to 1.0) - `block_on_detection` - bool - Block the request when violations found. Default: `true` diff --git a/docs/my-website/docs/proxy/guardrails/tool_permission.md b/docs/my-website/docs/proxy/guardrails/tool_permission.md index 9ed05ed46a8..22ecdd2251e 100644 --- a/docs/my-website/docs/proxy/guardrails/tool_permission.md +++ b/docs/my-website/docs/proxy/guardrails/tool_permission.md @@ -46,6 +46,43 @@ guardrails: - `pre_call` Run **before** LLM call, on **input** - `post_call` Run **after** LLM call, on **input & output** +### `on_disallowed_action` behavior + +| Value | What happens | +| --- | --- | +| `block` | The request is immediately rejected. Pre-call checks raise a `400` HTTP error. Post-call checks raise `GuardrailRaisedException`, so the proxy responds with an error instead of the model output. Use when invoking the forbidden tool must halt the workflow. | +| `rewrite` | LiteLLM silently strips disallowed tools from the payload before it reaches the model (pre-call) or rewrites the model response/tool calls after the fact. The guardrail inserts error text into `message.content`/`tool_result` entries so the client learns the tool was blocked while the rest of the completion continues. Use when you want graceful degradation instead of hard failures. | + +### Custom denial message + +Set `violation_message_template` when you want the guardrail to return a branded error (e.g., “this violates our org policy…”). LiteLLM replaces placeholders from the denied tool: + +- `{tool_name}` – the tool/function name (e.g., `Read`) +- `{rule_id}` – the matching rule ID (or `None` when the default action kicks in) +- `{default_message}` – the original LiteLLM message if you need to append it + +Example: + +```yaml +guardrails: + - guardrail_name: "tool-permission-guardrail" + litellm_params: + guardrail: tool_permission + mode: "post_call" + violation_message_template: "this violates our org policy, we don't support executing {tool_name} commands" + rules: + - id: "allow_bash" + tool_name: "Bash" + decision: "allow" + - id: "deny_read" + tool_name: "Read" + decision: "deny" + default_action: "deny" + on_disallowed_action: "block" +``` + +If a request tries to invoke `Read`, the proxy now returns “this violates our org policy, we don't support executing Read commands” instead of the stock error text. Omit the field to keep the default messaging. + ### 2. Start the Proxy ```shell @@ -57,7 +94,7 @@ litellm --config config.yaml --port 4000 -**Block requset** +**Block request (`on_disallowed_action: block`)** ```bash # Test @@ -96,7 +133,7 @@ curl -X POST "http://localhost:4000/v1/chat/completions" \ -**Rewrite requset** +**Rewrite request (`on_disallowed_action: rewrite`)** ```bash # Test @@ -118,7 +155,7 @@ curl -X POST "http://localhost:4000/v1/chat/completions" \ }' ``` -**Expected response:** +**Expected response (tool removed, completion continues):** ```json { diff --git a/docs/my-website/docs/proxy/litellm_managed_files.md b/docs/my-website/docs/proxy/litellm_managed_files.md index ab0e4b3a751..7aba173f35b 100644 --- a/docs/my-website/docs/proxy/litellm_managed_files.md +++ b/docs/my-website/docs/proxy/litellm_managed_files.md @@ -21,7 +21,7 @@ Available via the `litellm[proxy]` package or any `litellm` docker image. | Proxy | ✅ | | | SDK | ❌ | Requires postgres DB for storing file ids. | | Available across all providers | ✅ | | -| Supported endpoints | `/chat/completions`, `/batch`, `/fine_tuning` | | +| Supported endpoints | `/chat/completions`, `/batch`, `/fine_tuning`, `/responses` | | ## Usage @@ -424,4 +424,4 @@ No, as of `v1.71.2` users can only view/edit/delete files they have created. ## See Also - [Managed Files w/ Finetuning APIs](../../docs/proxy/managed_finetuning) -- [Managed Files w/ Batch APIs](../../docs/proxy/managed_batch) \ No newline at end of file +- [Managed Files w/ Batch APIs](../../docs/proxy/managed_batches) \ No newline at end of file diff --git a/docs/my-website/docs/proxy/managed_batches.md b/docs/my-website/docs/proxy/managed_batches.md index 431d313fc18..4bd3b12d3af 100644 --- a/docs/my-website/docs/proxy/managed_batches.md +++ b/docs/my-website/docs/proxy/managed_batches.md @@ -260,4 +260,15 @@ print(f"status: {status}") When a `target_model_names` is specified, the file is written to all deployments that match the `target_model_names`. -No additional infrastructure is required. \ No newline at end of file +No additional infrastructure is required. + +## Could the batch be created at the eastus-01 deployment but a subsequent get of the batch could be routed to (a different) eastus2-01 deployment ? + +**A.** You can loadbalance b/w multiple models for the initial create batch. Once that's created - we return a file id, which encodes the model deployment used, so it's sticky and only sends any get/delete to that deployment. + + + + + + + diff --git a/docs/my-website/docs/proxy/management_cli.md b/docs/my-website/docs/proxy/management_cli.md index 9ecc2ae8a34..23a56842105 100644 --- a/docs/my-website/docs/proxy/management_cli.md +++ b/docs/my-website/docs/proxy/management_cli.md @@ -67,7 +67,26 @@ For an indepth guide, see [CLI Authentication](./cli_sso). ::: +### Prerequisites +:::warning[Beta Feature - Required Environment Variable] + +CLI SSO Authentication is currently in beta. You must set this environment variable **when starting up your LiteLLM Proxy**: + +```bash +export EXPERIMENTAL_UI_LOGIN="True" +litellm --config config.yaml +``` + +Or add it to your proxy startup command: + +```bash +EXPERIMENTAL_UI_LOGIN="True" litellm --config config.yaml +``` + +::: + +### Steps 1. **Set up the proxy URL** diff --git a/docs/my-website/docs/proxy/model_compare_ui.md b/docs/my-website/docs/proxy/model_compare_ui.md new file mode 100644 index 00000000000..a3fb236393f --- /dev/null +++ b/docs/my-website/docs/proxy/model_compare_ui.md @@ -0,0 +1,193 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Model Compare Playground UI + +Compare multiple LLM models side-by-side in an interactive playground interface. Evaluate model responses, performance metrics, and costs to make informed decisions about which models work best for your use case. + +This feature is **available in v1.80.0-stable and above**. + +## Overview + +The Model Compare Playground UI enables side-by-side comparison of up to 3 different LLM models simultaneously. Configure models, parameters, and test prompts to evaluate and compare model responses with detailed metrics including latency, token usage, and cost. + + + +## Getting Started + +### Accessing the Model Compare UI + +#### 1. Navigate to the Playground + +Go to the Playground page in the Admin UI (`PROXY_BASE_URL/ui/?login=success&page=llm-playground`) + + + +#### 2. Switch to Compare Tab + +Click on the **Compare** tab in the Playground interface. + +## Configuration + +### Setting Up Models + +#### 1. Select Models to Compare + +You can compare up to 3 models simultaneously. For each comparison panel: + +- Click on the model dropdown to see available models +- Select a model from your configured endpoints +- Models are loaded from your LiteLLM proxy configuration + + + +#### 2. Configure Model Parameters + +Each model panel supports individual parameter configuration: + +**Basic Parameters:** + +- **Temperature**: Controls randomness (0.0 to 2.0) +- **Max Tokens**: Maximum tokens in the response + +**Advanced Parameters:** + +- Enable "Use Advanced Params" to configure additional model-specific parameters +- Supports all parameters available for the selected model/provider + + + +#### 3. Apply Parameters Across Models + +Use the "Sync Settings Across Models" toggle to synchronize parameters (tags, guardrails, temperature, max tokens, etc.) across all comparison panels for consistent testing. + + + +### Guardrails + +Configure and test guardrails directly in the playground: + +1. Click on the guardrails selector in a model panel +2. Select one or more guardrails from your configured list +3. Test how different models respond to guardrail filtering +4. Compare guardrail behavior across models + + + +### Tags + +Apply tags to organize and filter your comparisons: + +1. Select tags from the tag dropdown +2. Tags help categorize and track different test scenarios + + + +### Vector Stores + +Configure vector store retrieval for RAG (Retrieval Augmented Generation) comparisons: + +1. Select vector stores from the dropdown +2. Compare how different models utilize retrieved context +3. Evaluate RAG performance across models + + + +## Running Comparisons + +### 1. Enter Your Prompt + +Type your test prompt in the message input area. You can: + +- Enter a single message for all models +- Use suggested prompts for quick testing +- Build multi-turn conversations + + + +### 2. Send Request + +Click the send button (or press Enter) to start the comparison. All selected models will process the request simultaneously. + +### 3. View Responses + +Responses appear side-by-side in each model panel, making it easy to compare: + +- Response quality and content +- Response length and structure +- Model-specific formatting + + + +## Comparison Metrics + +Each comparison panel displays detailed metrics to help you evaluate model performance: + +### Time To First Token (TTFT) + +Measures the latency from request submission to the first token received. Lower values indicate faster initial response times. + +### Token Usage + +- **Input Tokens**: Number of tokens in the prompt/request +- **Output Tokens**: Number of tokens in the model's response +- **Reasoning Tokens**: Tokens used for reasoning (if applicable, e.g., o1 models) + +### Total Latency + +Complete time from request to final response, including streaming time. + +### Cost + +If cost tracking is enabled in your LiteLLM configuration, you'll see: + +- Cost per request +- Cost breakdown by input/output tokens +- Comparison of costs across models + + + +## Use Cases + +### Model Selection + +Compare multiple models on the same prompt to determine which performs best for your specific use case: + +- Response quality +- Response time +- Cost efficiency +- Token usage + +### Parameter Tuning + +Test different parameter configurations across models to find optimal settings: + +- Temperature variations +- Max token limits +- Advanced parameter combinations + +### Guardrail Testing + +Evaluate how different models respond to safety filters and guardrails: + +- Filter effectiveness +- False positive rates +- Model-specific guardrail behavior + +### A/B Testing + +Use tags and multiple comparisons to run structured A/B tests: + +- Compare model versions +- Test prompt variations +- Evaluate feature rollouts + +--- + +## Related Features + +- [Playground Chat UI](./playground.md) - Single model testing interface +- [Model Management](./model_management.md) - Configure and manage models +- [Guardrails](./guardrails.md) - Set up safety filters +- [AI Hub](./ai_hub.md) - Share models and agents with your organization diff --git a/docs/my-website/docs/proxy/model_hub.md b/docs/my-website/docs/proxy/model_hub.md deleted file mode 100644 index 6c12194d751..00000000000 --- a/docs/my-website/docs/proxy/model_hub.md +++ /dev/null @@ -1,53 +0,0 @@ -import Image from '@theme/IdealImage'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - -# Model Hub - -Tell developers what models are available on the proxy. - -This feature is **available in v1.74.3-stable and above**. - -## Overview - -Admin can select models to expose on public model hub -> Users can go to the public url (`/ui/model_hub_table`) and see available models. - - - -## How to use - -### 1. Go to the Admin UI - -Navigate to the Model Hub page in the Admin UI (`PROXY_BASE_URL/ui/?login=success&page=model-hub-table`) - - - -### 2. Select the models you want to expose - -Click on `Make Public` and select the models you want to expose. - - - -### 3. Confirm the changes - - - -### 4. Success! - -Go to the public url (`PROXY_BASE_URL/ui/model_hub_table`) and see available models. - - - -## API Endpoints - -LiteLLM also exposes REST endpoints: - -- `GET /public/model_hub` – returns the list of public model groups. Requires a valid user API key. -- `GET /public/model_hub/info` – returns metadata (docs title, version, useful links) for the public model hub. -- `GET /public/providers` – returns a sorted list of all providers supported by LiteLLM. No authentication required. - -Example: - -```bash -curl -s PROXY_BASE_URL/public/providers | jq -``` diff --git a/docs/my-website/docs/proxy/prometheus.md b/docs/my-website/docs/proxy/prometheus.md index 370a6540f9c..2dae463514a 100644 --- a/docs/my-website/docs/proxy/prometheus.md +++ b/docs/my-website/docs/proxy/prometheus.md @@ -14,11 +14,12 @@ If you're using the LiteLLM CLI with `litellm --config proxy_config.yaml` then y Add this to your proxy config.yaml ```yaml model_list: - - model_name: gpt-4o + - model_name: gpt-4o litellm_params: model: gpt-4o litellm_settings: - callbacks: ["prometheus"] + callbacks: + - prometheus ``` Start the proxy diff --git a/docs/my-website/docs/proxy/ui.md b/docs/my-website/docs/proxy/ui.md index f7419d20740..f6fa02fb69b 100644 --- a/docs/my-website/docs/proxy/ui.md +++ b/docs/my-website/docs/proxy/ui.md @@ -59,11 +59,13 @@ Allow others to create/delete their own keys. The Admin UI provides comprehensive model management capabilities: - **Add Models**: Add new models through the UI without restarting the proxy -- **Model Hub**: Make models public for developers to discover available models +- **AI Hub**: Make models and agents public for developers to discover what's available - **Price Data Sync**: Keep model pricing data up to date by syncing from GitHub For detailed information on model management, see [Model Management](./model_management.md). +For information on sharing models and agents, see [AI Hub](./ai_hub.md). + :::tip Sync Model Pricing Data [Sync model pricing data from GitHub](./sync_models_github.md) to keep your model cost information current. ::: diff --git a/docs/my-website/docs/proxy/ui_logs.md b/docs/my-website/docs/proxy/ui_logs.md index cd2ee982232..61f328011c3 100644 --- a/docs/my-website/docs/proxy/ui_logs.md +++ b/docs/my-website/docs/proxy/ui_logs.md @@ -76,8 +76,6 @@ Set `SPEND_LOG_CLEANUP_BATCH_SIZE` to control how many logs are deleted per batc For detailed architecture and how it works, see [Spend Logs Deletion](../proxy/spend_logs_deletion). +## What gets logged? - - - - +[Here's a schema](https://github.com/BerriAI/litellm/blob/1cdd4065a645021aea931afb9494e7694b4ec64b/schema.prisma#L285) breakdown of what gets logged. diff --git a/docs/my-website/docs/secret_managers/aws_secret_manager.md b/docs/my-website/docs/secret_managers/aws_secret_manager.md index 44fa23a4ae5..5b7ab1e3e7b 100644 --- a/docs/my-website/docs/secret_managers/aws_secret_manager.md +++ b/docs/my-website/docs/secret_managers/aws_secret_manager.md @@ -110,3 +110,57 @@ The `primary_secret_name` allows you to read multiple keys from a single AWS Sec This reduces the number of AWS Secrets you need to manage. +## IAM Role Assumption + +Use IAM roles instead of static AWS credentials for better security. + +### Basic IAM Role + +```yaml +general_settings: + key_management_system: "aws_secret_manager" + key_management_settings: + store_virtual_keys: true + aws_region_name: "us-east-1" + aws_role_name: "arn:aws:iam::123456789012:role/LiteLLMSecretManagerRole" + aws_session_name: "litellm-session" +``` + +### Cross-Account Access + +```yaml +general_settings: + key_management_system: "aws_secret_manager" + key_management_settings: + store_virtual_keys: true + aws_region_name: "us-east-1" + aws_role_name: "arn:aws:iam::999999999999:role/CrossAccountRole" + aws_external_id: "unique-external-id" +``` + +### EKS with IRSA + +```yaml +general_settings: + key_management_system: "aws_secret_manager" + key_management_settings: + store_virtual_keys: true + aws_region_name: "us-east-1" + aws_role_name: "arn:aws:iam::123456789012:role/LiteLLMServiceAccountRole" + aws_web_identity_token: "os.environ/AWS_WEB_IDENTITY_TOKEN_FILE" +``` + +### Configuration Parameters + +| Parameter | Description | +|-----------|-------------| +| `aws_region_name` | AWS region | +| `aws_role_name` | IAM role ARN to assume | +| `aws_session_name` | Session name (optional) | +| `aws_external_id` | External ID for cross-account | +| `aws_profile_name` | AWS profile from `~/.aws/credentials` | +| `aws_web_identity_token` | OIDC token path for IRSA | +| `aws_sts_endpoint` | Custom STS endpoint for VPC | + + + diff --git a/docs/my-website/docs/tutorials/claude_responses_api.md b/docs/my-website/docs/tutorials/claude_responses_api.md index 0dbb4a2f1e7..aafeccceaf5 100644 --- a/docs/my-website/docs/tutorials/claude_responses_api.md +++ b/docs/my-website/docs/tutorials/claude_responses_api.md @@ -105,7 +105,7 @@ LITELLM_MASTER_KEY gives claude access to all proxy models, whereas a virtual ke Alternatively, use the Anthropic pass-through endpoint: ```bash -export ANTHROPIC_BASE_URL="http://0.0.0.0:4000" +export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic" export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY" ``` @@ -221,7 +221,6 @@ You can also connect MCP servers to Claude Code via LiteLLM Proxy. Limitations: - Currently, only HTTP MCP servers are supported -- Does not work in Cursor IDE yet. ::: diff --git a/docs/my-website/docs/vector_store_files.md b/docs/my-website/docs/vector_store_files.md index d5544f34994..1a972ebc43f 100644 --- a/docs/my-website/docs/vector_store_files.md +++ b/docs/my-website/docs/vector_store_files.md @@ -1,4 +1,4 @@ -# /vector_stores/{vector_store_id}/files +# /vector_stores/\{vector_store_id\}/files Vector store files represent the individual files that live inside a vector store. @@ -26,7 +26,7 @@ Vector store support currently works **only with OpenAI vector stores and OpenAI ## Create vector store file -`POST http://localhost:4000/v1/vector_stores/{vector_store_id}/files` +POST http://localhost:4000/v1/vector_stores/{vector_store_id}/files ```python from openai import OpenAI @@ -53,7 +53,7 @@ print(vector_store_file) ## List vector store files -`GET http://localhost:4000/v1/vector_stores/{vector_store_id}/files` +GET http://localhost:4000/v1/vector_stores/{vector_store_id}/files Parameters: @@ -72,7 +72,7 @@ print(vector_store_files) ## Retrieve vector store file -`GET http://localhost:4000/v1/vector_stores/{vector_store_id}/files/{file_id}` +GET http://localhost:4000/v1/vector_stores/{vector_store_id}/files/{file_id} ```python vector_store_file = client.vector_stores.files.retrieve( @@ -84,7 +84,7 @@ print(vector_store_file) ## Delete vector store file -`DELETE http://localhost:4000/v1/vector_stores/{vector_store_id}/files/{file_id}` +DELETE http://localhost:4000/v1/vector_stores/{vector_store_id}/files/{file_id} ```python deleted_vector_store_file = client.vector_stores.files.delete( @@ -101,14 +101,14 @@ When you need raw content chunks or attribute updates, call the LiteLLM Proxy di ### Retrieve file content ```bash -curl -X GET "http://localhost:4000/v1/vector_stores/{vector_store_id}/files/{file_id}/content" \ +curl -X GET "http://localhost:4000/v1/vector_stores/\{vector_store_id\}/files/\{file_id\}/content" \ -H "Authorization: Bearer sk-1234" ``` ### Update file attributes ```bash -curl -X POST "http://localhost:4000/v1/vector_stores/{vector_store_id}/files/{file_id}" \ +curl -X POST "http://localhost:4000/v1/vector_stores/\{vector_store_id\}/files/\{file_id\}" \ -H "Authorization: Bearer sk-1234" \ -H "Content-Type: application/json" \ -d '{ diff --git a/docs/my-website/docusaurus.config.js b/docs/my-website/docusaurus.config.js index cec0479f673..32d5d800b71 100644 --- a/docs/my-website/docusaurus.config.js +++ b/docs/my-website/docusaurus.config.js @@ -101,6 +101,21 @@ const config = { include: ['**/*.{md,mdx}'], }, ], + [ + '@docusaurus/plugin-content-blog', + { + id: 'blog', + path: './blog', + routeBasePath: 'blog', + blogTitle: 'Blog', + blogSidebarTitle: 'All Posts', + blogSidebarCount: 'ALL', + postsPerPage: 10, + showReadingTime: false, + sortPosts: 'descending', + include: ['**/index.{md,mdx}'], + }, + ], () => ({ name: 'cripchat', @@ -129,6 +144,7 @@ const config = { docs: { sidebarPath: require.resolve('./sidebars.js'), }, + blog: false, // Disable the default blog plugin from preset-classic theme: { customCss: require.resolve('./src/css/custom.css'), }, @@ -177,6 +193,7 @@ const config = { to: "docs/enterprise" }, { to: '/release_notes', label: 'Release Notes', position: 'left' }, + { to: '/blog', label: 'Blog', position: 'left' }, { href: 'https://models.litellm.ai/', label: '💸 LLM Model Cost Map', @@ -231,6 +248,11 @@ const config = { ], copyright: `Copyright © ${new Date().getFullYear()} liteLLM`, }, + colorMode: { + defaultMode: 'light', + disableSwitch: false, + respectPrefersColorScheme: true, + }, prism: { theme: lightCodeTheme, darkTheme: darkCodeTheme, diff --git a/docs/my-website/img/add_agent.png b/docs/my-website/img/add_agent.png new file mode 100644 index 00000000000..f9a96b95e30 Binary files /dev/null and b/docs/my-website/img/add_agent.png differ diff --git a/docs/my-website/img/agent_hub_clean.png b/docs/my-website/img/agent_hub_clean.png new file mode 100644 index 00000000000..89537566f08 Binary files /dev/null and b/docs/my-website/img/agent_hub_clean.png differ diff --git a/docs/my-website/img/ai_hub_with_agents.png b/docs/my-website/img/ai_hub_with_agents.png new file mode 100644 index 00000000000..f61214636c1 Binary files /dev/null and b/docs/my-website/img/ai_hub_with_agents.png differ diff --git a/docs/my-website/img/app_role2.png b/docs/my-website/img/app_role2.png new file mode 100644 index 00000000000..81eaf8f96ae Binary files /dev/null and b/docs/my-website/img/app_role2.png differ diff --git a/docs/my-website/img/app_role3.png b/docs/my-website/img/app_role3.png new file mode 100644 index 00000000000..e11d73ccc21 Binary files /dev/null and b/docs/my-website/img/app_role3.png differ diff --git a/docs/my-website/img/app_roles.png b/docs/my-website/img/app_roles.png new file mode 100644 index 00000000000..4587ab3a058 Binary files /dev/null and b/docs/my-website/img/app_roles.png differ diff --git a/docs/my-website/img/favicon_converted.ico b/docs/my-website/img/favicon_converted.ico new file mode 100644 index 00000000000..7c45601d5c3 Binary files /dev/null and b/docs/my-website/img/favicon_converted.ico differ diff --git a/docs/my-website/img/make_agents_public.png b/docs/my-website/img/make_agents_public.png new file mode 100644 index 00000000000..25cf57ae751 Binary files /dev/null and b/docs/my-website/img/make_agents_public.png differ diff --git a/docs/my-website/img/model_compare_overview.png b/docs/my-website/img/model_compare_overview.png new file mode 100644 index 00000000000..f4af0eaee3c Binary files /dev/null and b/docs/my-website/img/model_compare_overview.png differ diff --git a/docs/my-website/img/public_agent_hub.png b/docs/my-website/img/public_agent_hub.png new file mode 100644 index 00000000000..24f47da12b0 Binary files /dev/null and b/docs/my-website/img/public_agent_hub.png differ diff --git a/docs/my-website/img/ui_model_compare_cost_metrics.png b/docs/my-website/img/ui_model_compare_cost_metrics.png new file mode 100644 index 00000000000..b4639348c88 Binary files /dev/null and b/docs/my-website/img/ui_model_compare_cost_metrics.png differ diff --git a/docs/my-website/img/ui_model_compare_enter_prompt.png b/docs/my-website/img/ui_model_compare_enter_prompt.png new file mode 100644 index 00000000000..af643abf6b8 Binary files /dev/null and b/docs/my-website/img/ui_model_compare_enter_prompt.png differ diff --git a/docs/my-website/img/ui_model_compare_guardrails_config.png b/docs/my-website/img/ui_model_compare_guardrails_config.png new file mode 100644 index 00000000000..a85f9901299 Binary files /dev/null and b/docs/my-website/img/ui_model_compare_guardrails_config.png differ diff --git a/docs/my-website/img/ui_model_compare_model_parameters.png b/docs/my-website/img/ui_model_compare_model_parameters.png new file mode 100644 index 00000000000..1ad0dfc4095 Binary files /dev/null and b/docs/my-website/img/ui_model_compare_model_parameters.png differ diff --git a/docs/my-website/img/ui_model_compare_overview.png b/docs/my-website/img/ui_model_compare_overview.png new file mode 100644 index 00000000000..f4af0eaee3c Binary files /dev/null and b/docs/my-website/img/ui_model_compare_overview.png differ diff --git a/docs/my-website/img/ui_model_compare_responses.png b/docs/my-website/img/ui_model_compare_responses.png new file mode 100644 index 00000000000..5d207cd0155 Binary files /dev/null and b/docs/my-website/img/ui_model_compare_responses.png differ diff --git a/docs/my-website/img/ui_model_compare_select_model.png b/docs/my-website/img/ui_model_compare_select_model.png new file mode 100644 index 00000000000..ba7bf948fcc Binary files /dev/null and b/docs/my-website/img/ui_model_compare_select_model.png differ diff --git a/docs/my-website/img/ui_model_compare_sync_across_models.png b/docs/my-website/img/ui_model_compare_sync_across_models.png new file mode 100644 index 00000000000..d59696a4bd2 Binary files /dev/null and b/docs/my-website/img/ui_model_compare_sync_across_models.png differ diff --git a/docs/my-website/img/ui_model_compare_tags_config.png b/docs/my-website/img/ui_model_compare_tags_config.png new file mode 100644 index 00000000000..bf36d9a987e Binary files /dev/null and b/docs/my-website/img/ui_model_compare_tags_config.png differ diff --git a/docs/my-website/img/ui_model_compare_vector_stores_config.png b/docs/my-website/img/ui_model_compare_vector_stores_config.png new file mode 100644 index 00000000000..b3bae046abf Binary files /dev/null and b/docs/my-website/img/ui_model_compare_vector_stores_config.png differ diff --git a/docs/my-website/img/ui_playground_navigation.png b/docs/my-website/img/ui_playground_navigation.png new file mode 100644 index 00000000000..202224b4069 Binary files /dev/null and b/docs/my-website/img/ui_playground_navigation.png differ diff --git a/docs/my-website/package-lock.json b/docs/my-website/package-lock.json index 3fce178f9aa..67f0ea79e68 100644 --- a/docs/my-website/package-lock.json +++ b/docs/my-website/package-lock.json @@ -11378,9 +11378,9 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.5.253", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.253.tgz", - "integrity": "sha512-O0tpQ/35rrgdiGQ0/OFWhy1itmd9A6TY9uQzlqj3hKSu/aYpe7UIn5d7CU2N9myH6biZiWF3VMZVuup8pw5U9w==", + "version": "1.5.254", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.254.tgz", + "integrity": "sha512-DcUsWpVhv9svsKRxnSCZ86SjD+sp32SGidNB37KpqXJncp1mfUgKbHvBomE89WJDbfVKw1mdv5+ikrvd43r+Bg==", "license": "ISC" }, "node_modules/emoji-regex": { @@ -11601,6 +11601,19 @@ "node": ">=8.0.0" } }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/esrecurse": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", @@ -12619,6 +12632,28 @@ "node": ">=6.0" } }, + "node_modules/gray-matter/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/gray-matter/node_modules/js-yaml": { + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/gzip-size": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/gzip-size/-/gzip-size-6.0.0.tgz", @@ -21028,6 +21063,12 @@ "wbuf": "^1.7.3" } }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "license": "BSD-3-Clause" + }, "node_modules/srcset": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/srcset/-/srcset-4.0.0.tgz", diff --git a/docs/my-website/package.json b/docs/my-website/package.json index a457355486b..784a5e4b578 100644 --- a/docs/my-website/package.json +++ b/docs/my-website/package.json @@ -48,10 +48,16 @@ "node": ">=16.14", "npm": ">=8.3.0" }, + "resolutions": { + "webpack-dev-server": ">=5.2.1", + "form-data": ">=4.0.4", + "mermaid": ">=11.10.0", + "gray-matter": "4.0.3" + }, "overrides": { "webpack-dev-server": ">=5.2.1", "form-data": ">=4.0.4", "mermaid": ">=11.10.0", - "js-yaml": ">=4.1.1" + "gray-matter": "4.0.3" } } diff --git a/docs/my-website/release_notes/authors.yml b/docs/my-website/release_notes/authors.yml new file mode 100644 index 00000000000..aaa3d51ec97 --- /dev/null +++ b/docs/my-website/release_notes/authors.yml @@ -0,0 +1,18 @@ +krrish: + name: Krrish Dholakia + title: CEO, LiteLLM + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + +ishaan: + name: Ishaan Jaffer + title: CTO, LiteLLM + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + +# Alias for typo in name +ishaan-alt: + name: Ishaan Jaff + title: CTO, LiteLLM + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg diff --git a/docs/my-website/release_notes/v1.80.0-stable/index.md b/docs/my-website/release_notes/v1.80.0-stable/index.md index de07408db57..9c643a48adb 100644 --- a/docs/my-website/release_notes/v1.80.0-stable/index.md +++ b/docs/my-website/release_notes/v1.80.0-stable/index.md @@ -1,5 +1,5 @@ --- -title: "v1.80.0-stable - RunwayML Provider Support" +title: "[Preview] v1.80.0-stable - Agent Hub Support" slug: "v1-80-0" date: 2025-11-15T10:00:00 authors: @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.80.0.rc.1 +ghcr.io/berriai/litellm:v1.80.0.rc.2 ``` @@ -45,7 +45,8 @@ pip install litellm==1.80.0 ## Key Highlights -- **🆕 RunwayML Provider** - Complete video generation, image generation, and text-to-speech support +- **🆕 Agent Hub Support** - Register and make agents public for your organization +- **RunwayML Provider** - Complete video generation, image generation, and text-to-speech support - **GPT-5.1 Family Support** - Day-0 support for OpenAI's latest GPT-5.1 and GPT-5.1-Codex models - **Prometheus OSS** - Prometheus metrics now available in open-source version - **Vector Store Files API** - Complete OpenAI-compatible Vector Store Files API with full CRUD operations @@ -53,6 +54,46 @@ pip install litellm==1.80.0 --- +### Agent Hub + + + +This release adds support for registering and making agents public for your organization. This is great for **Proxy Admins** who want a central place to make agents built in their organization, discoverable to their users. + +Here's the flow: +1. Add agent to litellm. +2. Make it public. +3. Allow anyone to discover it on the public AI Hub page. + +[**Get Started with Agent Hub**](../../docs/proxy/ai_hub) + + +### Performance – `/embeddings` 13× Lower p95 Latency + +This update significantly improves `/embeddings` latency by routing it through the same optimized pipeline as `/chat/completions`, benefiting from all previously applied networking optimizations. + +### Results + +| Metric | Before | After | Improvement | +| --- | --- | --- | --- | +| p95 latency | 5,700 ms | **430 ms** | −92% (~13× faster)** | +| p99 latency | 7,200 ms | **780 ms** | −89% | +| Average latency | 844 ms | **262 ms** | −69% | +| Median latency | 290 ms | **230 ms** | −21% | +| RPS | 1,216.7 | **1,219.7** | **+0.25%** | + +### Test Setup + +| Category | Specification | +| --- | --- | +| **Load Testing** | Locust: 1,000 concurrent users, 500 ramp-up | +| **System** | 4 vCPUs, 8 GB RAM, 4 workers, 4 instances | +| **Database** | PostgreSQL (Redis unused) | +| **Configuration** | [config.yaml](https://gist.github.com/AlexsanderHamir/550791675fd752befcac6a9e44024652) | +| **Load Script** | [no_cache_hits.py](https://gist.github.com/AlexsanderHamir/99d673bf74cdd81fd39f59fa9048f2e8) | + +--- + ### 🆕 RunwayML Complete integration for RunwayML's Gen-4 family of models, supporting video generation, image generation, and text-to-speech. @@ -97,7 +138,7 @@ litellm_settings: --- -### Vector Store Files API - Stable Release +### Vector Store Files API Complete OpenAI-compatible Vector Store Files API now stable, enabling full file lifecycle management within vector stores. @@ -120,7 +161,28 @@ curl --location 'http://localhost:4000/v1/vector_stores/vs_123/files' \ }' ``` -[Get Started with Vector Stores](../../docs/vector_stores) +[Get Started with Vector Stores](../../docs/vector_store_files) + +--- + +## New Providers and Endpoints + +### New Providers + +| Provider | Supported Endpoints | Description | +| -------- | ------------------- | ----------- | +| **[RunwayML](../../docs/providers/runwayml/videos)** | `/v1/videos`, `/v1/images/generations`, `/v1/audio/speech` | Gen-4 video generation, image generation, and text-to-speech | + +### New LLM API Endpoints + +| Endpoint | Method | Description | Documentation | +| -------- | ------ | ----------- | ------------- | +| `/v1/vector_stores/{vector_store_id}/files` | POST | Create vector store file | [Docs](../../docs/vector_store_files) | +| `/v1/vector_stores/{vector_store_id}/files` | GET | List vector store files | [Docs](../../docs/vector_store_files) | +| `/v1/vector_stores/{vector_store_id}/files/{file_id}` | GET | Retrieve vector store file | [Docs](../../docs/vector_store_files) | +| `/v1/vector_stores/{vector_store_id}/files/{file_id}/content` | GET | Retrieve file content | [Docs](../../docs/vector_store_files) | +| `/v1/vector_stores/{vector_store_id}/files/{file_id}` | DELETE | Delete vector store file | [Docs](../../docs/vector_store_files) | +| `/v1/vector_stores/{vector_store_id}` | DELETE | Delete vector store | [Docs](../../docs/vector_store_files) | --- diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 92592d3a473..08857ce07b4 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -146,13 +146,14 @@ const sidebars = { type: "category", label: "Admin UI", items: [ + "proxy/ui", "proxy/admin_ui_sso", "proxy/custom_root_ui", "proxy/custom_sso", - "proxy/model_hub", + "proxy/ai_hub", + "proxy/model_compare_ui", "proxy/public_teams", "proxy/self_serve", - "proxy/ui", "proxy/ui/bulk_edit_users", "proxy/ui_credentials", "tutorials/scim_litellm", @@ -530,13 +531,39 @@ const sidebars = { "providers/bedrock_vector_store", ] }, - "providers/milvus_vector_stores", "providers/litellm_proxy", - "providers/meta_llama", - "providers/mistral", + "providers/ai21", + "providers/aiml", + "providers/aleph_alpha", + "providers/anyscale", + "providers/baseten", + "providers/bytez", + "providers/cerebras", + "providers/clarifai", + "providers/cloudflare_workers", "providers/codestral", "providers/cohere", - "providers/anyscale", + "providers/cometapi", + "providers/compactifai", + "providers/custom_llm_server", + "providers/dashscope", + "providers/databricks", + "providers/datarobot", + "providers/deepgram", + "providers/deepinfra", + "providers/deepseek", + "providers/docker_model_runner", + "providers/elevenlabs", + "providers/fal_ai", + "providers/featherless_ai", + "providers/fireworks_ai", + "providers/friendliai", + "providers/galadriel", + "providers/github", + "providers/github_copilot", + "providers/gradient_ai", + "providers/groq", + "providers/heroku", { type: "category", label: "HuggingFace", @@ -546,10 +573,21 @@ const sidebars = { ] }, "providers/hyperbolic", - "providers/databricks", - "providers/deepgram", - "providers/watsonx", - "providers/predibase", + "providers/infinity", + "providers/jina_ai", + "providers/lambda_ai", + "providers/lemonade", + "providers/llamafile", + "providers/lm_studio", + "providers/meta_llama", + "providers/milvus_vector_stores", + "providers/mistral", + "providers/moonshot", + "providers/morph", + "providers/nebius", + "providers/nlp_cloud", + "providers/novita", + { type: "doc", id: "providers/nscale", label: "Nscale (EU Sovereign)" }, { type: "category", label: "Nvidia NIM", @@ -558,37 +596,13 @@ const sidebars = { "providers/nvidia_nim_rerank", ] }, - { type: "doc", id: "providers/nscale", label: "Nscale (EU Sovereign)" }, - "providers/xai", - "providers/moonshot", - "providers/lm_studio", - "providers/cerebras", - "providers/volcano", - "providers/triton-inference-server", + "providers/oci", "providers/ollama", + "providers/openrouter", + "providers/ovhcloud", "providers/perplexity", - "providers/friendliai", - "providers/galadriel", - "providers/topaz", - "providers/groq", - "providers/deepseek", - "providers/elevenlabs", - "providers/fal_ai", - "providers/fireworks_ai", - "providers/clarifai", - "providers/compactifai", - "providers/lemonade", - "providers/vllm", - "providers/llamafile", - "providers/infinity", - "providers/xinference", - "providers/aiml", - "providers/cloudflare_workers", - "providers/deepinfra", - "providers/github", - "providers/github_copilot", - "providers/ai21", - "providers/nlp_cloud", + "providers/petals", + "providers/predibase", "providers/recraft", "providers/replicate", { @@ -599,32 +613,20 @@ const sidebars = { "providers/runwayml/videos", ] }, + "providers/sambanova", + "providers/snowflake", "providers/togetherai", + "providers/topaz", + "providers/triton-inference-server", "providers/v0", "providers/vercel_ai_gateway", - "providers/morph", - "providers/lambda_ai", - "providers/novita", + "providers/vllm", + "providers/volcano", "providers/voyage", - "providers/jina_ai", - "providers/aleph_alpha", - "providers/baseten", - "providers/openrouter", - "providers/sambanova", - "providers/custom_llm_server", - "providers/petals", - "providers/snowflake", - "providers/gradient_ai", - "providers/featherless_ai", - "providers/nebius", - "providers/dashscope", - "providers/bytez", - "providers/heroku", - "providers/oci", - "providers/datarobot", - "providers/ovhcloud", "providers/wandb_inference", - "providers/cometapi", + "providers/watsonx", + "providers/xai", + "providers/xinference", ], }, { diff --git a/docs/my-website/static/img/favicon.ico b/docs/my-website/static/img/favicon.ico index 88caa2b8315..7c45601d5c3 100644 Binary files a/docs/my-website/static/img/favicon.ico and b/docs/my-website/static/img/favicon.ico differ diff --git a/enterprise/dist/litellm_enterprise-0.1.22-py3-none-any.whl b/enterprise/dist/litellm_enterprise-0.1.22-py3-none-any.whl new file mode 100644 index 00000000000..6ad5b7041c5 Binary files /dev/null and b/enterprise/dist/litellm_enterprise-0.1.22-py3-none-any.whl differ diff --git a/enterprise/dist/litellm_enterprise-0.1.22.tar.gz b/enterprise/dist/litellm_enterprise-0.1.22.tar.gz new file mode 100644 index 00000000000..9db2c14b12f Binary files /dev/null and b/enterprise/dist/litellm_enterprise-0.1.22.tar.gz differ diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py b/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py index ff3e9a744c1..8824f4c02de 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py @@ -40,7 +40,7 @@ class EnterpriseCallbackControls: ######################################################### # premium user check ######################################################### - if not EnterpriseCallbackControls._premium_user_check(): + if not EnterpriseCallbackControls._should_allow_dynamic_callback_disabling(): return False ######################################################### if isinstance(callback, str): @@ -84,8 +84,15 @@ class EnterpriseCallbackControls: return None @staticmethod - def _premium_user_check(): + def _should_allow_dynamic_callback_disabling(): + import litellm from litellm.proxy.proxy_server import premium_user + + # Check if admin has disabled this feature + if litellm.allow_dynamic_callback_disabling is not True: + verbose_logger.debug("Dynamic callback disabling is disabled by admin via litellm.allow_dynamic_callback_disabling") + return False + if premium_user: return True verbose_logger.warning(f"Disabling callbacks using request headers is an enterprise feature. {CommonProxyErrors.not_premium_user.value}") diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 80cc77883fe..608bb495885 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -296,6 +296,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): file_ids, user_api_key_dict.parent_otel_span ) + data["model_file_id_mapping"] = model_file_id_mapping + elif call_type == CallTypes.aresponses.value or call_type == CallTypes.responses.value: + # Handle managed files in responses API input + input_data = data.get("input") + if input_data: + file_ids = self.get_file_ids_from_responses_input(input_data) + if file_ids: + model_file_id_mapping = await self.get_model_file_id_mapping( + file_ids, user_api_key_dict.parent_otel_span + ) data["model_file_id_mapping"] = model_file_id_mapping elif call_type == CallTypes.afile_content.value: retrieve_file_id = cast(Optional[str], data.get("file_id")) @@ -453,6 +463,47 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): file_ids.append(file_id) return file_ids + def get_file_ids_from_responses_input( + self, input: Union[str, List[Dict[str, Any]]] + ) -> List[str]: + """ + Gets file ids from responses API input. + + The input can be: + - A string (no files) + - A list of input items, where each item can have: + - type: "input_file" with file_id + - content: a list that can contain items with type: "input_file" and file_id + """ + file_ids: List[str] = [] + + if isinstance(input, str): + return file_ids + + if not isinstance(input, list): + return file_ids + + for item in input: + if not isinstance(item, dict): + continue + + # Check for direct input_file type + if item.get("type") == "input_file": + file_id = item.get("file_id") + if file_id: + file_ids.append(file_id) + + # Check for input_file in content array + content = item.get("content") + if isinstance(content, list): + for content_item in content: + if isinstance(content_item, dict) and content_item.get("type") == "input_file": + file_id = content_item.get("file_id") + if file_id: + file_ids.append(file_id) + + return file_ids + async def get_model_file_id_mapping( self, file_ids: List[str], litellm_parent_otel_span: Span ) -> dict: @@ -478,7 +529,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): for file_id in file_ids: ## CHECK IF FILE ID IS MANAGED BY LITELM is_base64_unified_file_id = _is_base64_encoded_unified_file_id(file_id) - if is_base64_unified_file_id: litellm_managed_file_ids.append(file_id) @@ -489,6 +539,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): unified_file_object = await self.get_unified_file_id( file_id, litellm_parent_otel_span ) + if unified_file_object: file_id_mapping[file_id] = unified_file_object.model_mappings @@ -764,18 +815,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): llm_router: Router, **data: Dict, ) -> OpenAIFileObject: - file_id = convert_b64_uid_to_unified_uid(file_id) + + # file_id = convert_b64_uid_to_unified_uid(file_id) model_file_id_mapping = await self.get_model_file_id_mapping( [file_id], litellm_parent_otel_span ) + specific_model_file_id_mapping = model_file_id_mapping.get(file_id) if specific_model_file_id_mapping: - for model_id, file_id in specific_model_file_id_mapping.items(): - await llm_router.afile_delete(model=model_id, file_id=file_id, **data) # type: ignore + for model_id, model_file_id in specific_model_file_id_mapping.items(): + await llm_router.afile_delete(model=model_id, file_id=model_file_id, **data) # type: ignore stored_file_object = await self.delete_unified_file_id( file_id, litellm_parent_otel_span ) + if stored_file_object: return stored_file_object else: @@ -796,6 +850,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_file_id_mapping or await self.get_model_file_id_mapping([file_id], litellm_parent_otel_span) ) + specific_model_file_id_mapping = model_file_id_mapping.get(file_id) if specific_model_file_id_mapping: diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index aec888ddc94..2c1fa9945bb 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-enterprise" -version = "0.1.21" +version = "0.1.22" description = "Package for LiteLLM Enterprise features" authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.1.21" +version = "0.1.22" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-enterprise==", diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.6-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.6-py3-none-any.whl new file mode 100644 index 00000000000..346c07b06ea Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.6-py3-none-any.whl differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.6.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.6.tar.gz new file mode 100644 index 00000000000..3a25d44425d Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.6.tar.gz differ diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251115120021_baseline_diff/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251115120021_baseline_diff/migration.sql new file mode 100644 index 00000000000..2f725d83806 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251115120021_baseline_diff/migration.sql @@ -0,0 +1,2 @@ +-- This is an empty migration. + diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251115120539_baseline_diff/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251115120539_baseline_diff/migration.sql new file mode 100644 index 00000000000..2f725d83806 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251115120539_baseline_diff/migration.sql @@ -0,0 +1,2 @@ +-- This is an empty migration. + diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251119131227_add_prompt_versioning/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251119131227_add_prompt_versioning/migration.sql new file mode 100644 index 00000000000..a9d9528bd24 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251119131227_add_prompt_versioning/migration.sql @@ -0,0 +1,12 @@ +-- DropIndex +DROP INDEX "LiteLLM_PromptTable_prompt_id_key"; + +-- AlterTable +ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "version" INTEGER NOT NULL DEFAULT 1; + +-- CreateIndex +CREATE INDEX "LiteLLM_PromptTable_prompt_id_idx" ON "LiteLLM_PromptTable"("prompt_id"); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_PromptTable_prompt_id_version_key" ON "LiteLLM_PromptTable"("prompt_id", "version"); + diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index d6b7cebbd14..6cfbb90c362 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -561,11 +561,15 @@ model LiteLLM_GuardrailsTable { // Prompt table for storing prompt configurations model LiteLLM_PromptTable { id String @id @default(uuid()) - prompt_id String @unique + prompt_id String + version Int @default(1) litellm_params Json prompt_info Json? created_at DateTime @default(now()) updated_at DateTime @updatedAt + + @@unique([prompt_id, version]) + @@index([prompt_id]) } model LiteLLM_HealthCheckTable { diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index c29492558a3..78e34ccd01a 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.4.5" +version = "0.4.6" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.4.5" +version = "0.4.6" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/litellm/__init__.py b/litellm/__init__.py index 170566a0164..51be5ee2e29 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -181,22 +181,22 @@ prometheus_initialize_budget_metrics: Optional[bool] = False require_auth_for_metrics_endpoint: Optional[bool] = False argilla_batch_size: Optional[int] = None datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload. -gcs_pub_sub_use_v1: Optional[ - bool -] = False # if you want to use v1 gcs pubsub logged payload -generic_api_use_v1: Optional[ - bool -] = False # if you want to use v1 generic api logged payload +gcs_pub_sub_use_v1: Optional[bool] = ( + False # if you want to use v1 gcs pubsub logged payload +) +generic_api_use_v1: Optional[bool] = ( + False # if you want to use v1 generic api logged payload +) argilla_transformation_object: Optional[Dict[str, Any]] = None -_async_input_callback: List[ - Union[str, Callable, CustomLogger] -] = [] # internal variable - async custom callbacks are routed here. -_async_success_callback: List[ - Union[str, Callable, CustomLogger] -] = [] # internal variable - async custom callbacks are routed here. -_async_failure_callback: List[ - Union[str, Callable, CustomLogger] -] = [] # internal variable - async custom callbacks are routed here. +_async_input_callback: List[Union[str, Callable, CustomLogger]] = ( + [] +) # internal variable - async custom callbacks are routed here. +_async_success_callback: List[Union[str, Callable, CustomLogger]] = ( + [] +) # internal variable - async custom callbacks are routed here. +_async_failure_callback: List[Union[str, Callable, CustomLogger]] = ( + [] +) # internal variable - async custom callbacks are routed here. pre_call_rules: List[Callable] = [] post_call_rules: List[Callable] = [] turn_off_message_logging: Optional[bool] = False @@ -204,18 +204,18 @@ log_raw_request_response: bool = False redact_messages_in_exceptions: Optional[bool] = False redact_user_api_key_info: Optional[bool] = False filter_invalid_headers: Optional[bool] = False -add_user_information_to_llm_headers: Optional[ - bool -] = None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers +add_user_information_to_llm_headers: Optional[bool] = ( + None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers +) store_audit_logs = False # Enterprise feature, allow users to see audit logs ### end of callbacks ############# -email: Optional[ - str -] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -token: Optional[ - str -] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +email: Optional[str] = ( + None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +) +token: Optional[str] = ( + None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +) telemetry = True max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False)) @@ -271,9 +271,9 @@ use_client: bool = False ssl_verify: Union[str, bool] = True ssl_security_level: Optional[str] = None ssl_certificate: Optional[str] = None -ssl_ecdh_curve: Optional[ - str -] = None # Set to 'X25519' to disable PQC and improve performance +ssl_ecdh_curve: Optional[str] = ( + None # Set to 'X25519' to disable PQC and improve performance +) disable_streaming_logging: bool = False disable_token_counter: bool = False disable_add_transform_inline_image_block: bool = False @@ -319,20 +319,24 @@ enable_loadbalancing_on_batch_endpoints: Optional[bool] = None enable_caching_on_provider_specific_optional_params: bool = ( False # feature-flag for caching on optional params - e.g. 'top_k' ) -caching: bool = False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -caching_with_models: bool = False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -cache: Optional[ - Cache -] = None # cache object <- use this - https://docs.litellm.ai/docs/caching +caching: bool = ( + False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +) +caching_with_models: bool = ( + False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +) +cache: Optional[Cache] = ( + None # cache object <- use this - https://docs.litellm.ai/docs/caching +) default_in_memory_ttl: Optional[float] = None default_redis_ttl: Optional[float] = None default_redis_batch_cache_expiry: Optional[float] = None model_alias_map: Dict[str, str] = {} model_group_settings: Optional["ModelGroupSettings"] = None max_budget: float = 0.0 # set the max budget across all providers -budget_duration: Optional[ - str -] = None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). +budget_duration: Optional[str] = ( + None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). +) default_soft_budget: float = ( DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0 ) @@ -341,7 +345,9 @@ forward_traceparent_to_llm_provider: bool = False _current_cost = 0.0 # private variable, used if max budget is set error_logs: Dict = {} -add_function_to_prompt: bool = False # if function calling not supported by api, append function call details to system prompt +add_function_to_prompt: bool = ( + False # if function calling not supported by api, append function call details to system prompt +) client_session: Optional[httpx.Client] = None aclient_session: Optional[httpx.AsyncClient] = None model_fallbacks: Optional[List] = None # Deprecated for 'litellm.fallbacks' @@ -379,8 +385,12 @@ prometheus_metrics_config: Optional[List] = None disable_add_prefix_to_prompt: bool = ( False # used by anthropic, to disable adding prefix to prompt ) -disable_copilot_system_to_assistant: bool = False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. +disable_copilot_system_to_assistant: bool = ( + False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. +) +public_mcp_servers: Optional[List[str]] = None public_model_groups: Optional[List[str]] = None +public_agent_groups: Optional[List[str]] = None public_model_groups_links: Dict[str, str] = {} #### REQUEST PRIORITIZATION ####### priority_reservation: Optional[Dict[str, Union[float, PriorityReservationDict]]] = None @@ -390,13 +400,17 @@ priority_reservation_settings: "PriorityReservationSettings" = ( ######## Networking Settings ######## -use_aiohttp_transport: bool = True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead. +use_aiohttp_transport: bool = ( + True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead. +) aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings disable_aiohttp_transport: bool = False # Set this to true to use httpx instead disable_aiohttp_trust_env: bool = ( False # When False, aiohttp will respect HTTP(S)_PROXY env vars ) -force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. +force_ipv4: bool = ( + False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. +) module_level_aclient = AsyncHTTPHandler( timeout=request_timeout, client_alias="module level aclient" ) @@ -410,13 +424,14 @@ fallbacks: Optional[List] = None context_window_fallbacks: Optional[List] = None content_policy_fallbacks: Optional[List] = None allowed_fails: int = 3 -num_retries_per_request: Optional[ - int -] = None # for the request overall (incl. fallbacks + model retries) +allow_dynamic_callback_disabling: bool = True +num_retries_per_request: Optional[int] = ( + None # for the request overall (incl. fallbacks + model retries) +) ####### SECRET MANAGERS ##################### -secret_manager_client: Optional[ - Any -] = None # list of instantiated key management clients - e.g. azure kv, infisical, etc. +secret_manager_client: Optional[Any] = ( + None # list of instantiated key management clients - e.g. azure kv, infisical, etc. +) _google_kms_resource_name: Optional[str] = None _key_management_system: Optional[KeyManagementSystem] = None _key_management_settings: KeyManagementSettings = KeyManagementSettings() @@ -426,9 +441,9 @@ output_parse_pii: bool = False from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map model_cost = get_model_cost_map(url=model_cost_map_url) -cost_discount_config: Dict[ - str, float -] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount +cost_discount_config: Dict[str, float] = ( + {} +) # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount custom_prompt_dict: Dict[str, dict] = {} check_provider_endpoint = False @@ -548,6 +563,7 @@ wandb_models: Set = set(WANDB_MODELS) ovhcloud_models: Set = set() ovhcloud_embedding_models: Set = set() lemonade_models: Set = set() +docker_model_runner_models: Set = set() def is_bedrock_pricing_only_model(key: str) -> bool: @@ -782,6 +798,8 @@ def add_known_models(): ovhcloud_embedding_models.add(key) elif value.get("litellm_provider") == "lemonade": lemonade_models.add(key) + elif value.get("litellm_provider") == "docker_model_runner": + docker_model_runner_models.add(key) add_known_models() @@ -885,6 +903,7 @@ model_list = list( | wandb_models | ovhcloud_models | lemonade_models + | docker_model_runner_models | set(clarifai_models) ) @@ -1328,10 +1347,14 @@ from .llms.watsonx.completion.transformation import IBMWatsonXAIConfig from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig from .llms.github_copilot.chat.transformation import GithubCopilotConfig +from .llms.github_copilot.responses.transformation import ( + GithubCopilotResponsesAPIConfig, +) from .llms.nebius.chat.transformation import NebiusConfig from .llms.wandb.chat.transformation import WandbConfig from .llms.dashscope.chat.transformation import DashScopeChatConfig from .llms.moonshot.chat.transformation import MoonshotChatConfig +from .llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig from .llms.v0.chat.transformation import V0ChatConfig from .llms.oci.chat.transformation import OCIChatConfig from .llms.morph.chat.transformation import MorphChatConfig @@ -1342,6 +1365,7 @@ from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig from .llms.ovhcloud.embedding.transformation import OVHCloudEmbeddingConfig from .llms.cometapi.embed.transformation import CometAPIEmbeddingConfig from .llms.lemonade.chat.transformation import LemonadeChatConfig +from .llms.snowflake.embedding.transformation import SnowflakeEmbeddingConfig from .main import * # type: ignore from .integrations import * from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients @@ -1423,12 +1447,12 @@ from .types.llms.custom_llm import CustomLLMItem from .types.utils import GenericStreamingChunk custom_provider_map: List[CustomLLMItem] = [] -_custom_providers: List[ - str -] = [] # internal helper util, used to track names of custom providers -disable_hf_tokenizer_download: Optional[ - bool -] = None # disable huggingface tokenizer download. Defaults to openai clk100 +_custom_providers: List[str] = ( + [] +) # internal helper util, used to track names of custom providers +disable_hf_tokenizer_download: Optional[bool] = ( + None # disable huggingface tokenizer download. Defaults to openai clk100 +) global_disable_no_log_param: bool = False ### CLI UTILITIES ### diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 48521e5fba0..838ee95b2b5 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -17,6 +17,7 @@ from functools import partial from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast import httpx +from openai.types.batch import BatchRequestCounts import litellm from litellm._logging import verbose_logger @@ -223,10 +224,12 @@ def create_batch( api_key=optional_params.api_key, logging_obj=litellm_logging_obj, _is_async=_is_async, - client=client - if client is not None - and isinstance(client, (HTTPHandler, AsyncHTTPHandler)) - else None, + client=( + client + if client is not None + and isinstance(client, (HTTPHandler, AsyncHTTPHandler)) + else None + ), timeout=timeout, model=model, ) @@ -609,10 +612,12 @@ def retrieve_batch( function_id="batch_retrieve", ), _is_async=_is_async, - client=client - if client is not None - and isinstance(client, (HTTPHandler, AsyncHTTPHandler)) - else None, + client=( + client + if client is not None + and isinstance(client, (HTTPHandler, AsyncHTTPHandler)) + else None + ), timeout=timeout, model=model, ) @@ -799,6 +804,7 @@ def list_batches( async def acancel_batch( batch_id: str, + model: Optional[str] = None, custom_llm_provider: Literal["openai", "azure"] = "openai", metadata: Optional[Dict[str, str]] = None, extra_headers: Optional[Dict[str, str]] = None, @@ -813,11 +819,13 @@ async def acancel_batch( try: loop = asyncio.get_event_loop() kwargs["acancel_batch"] = True + model = kwargs.pop("model", None) # Use a partial function to pass your keyword arguments func = partial( cancel_batch, batch_id, + model, custom_llm_provider, metadata, extra_headers, @@ -840,7 +848,8 @@ async def acancel_batch( def cancel_batch( batch_id: str, - custom_llm_provider: Literal["openai", "azure"] = "openai", + model: Optional[str] = None, + custom_llm_provider: Union[Literal["openai", "azure"], str] = "openai", metadata: Optional[Dict[str, str]] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, @@ -852,6 +861,17 @@ def cancel_batch( LiteLLM Equivalent of POST https://api.openai.com/v1/batches/{batch_id}/cancel """ try: + + try: + if model is not None: + _, custom_llm_provider, _, _ = get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + ) + except Exception as e: + verbose_logger.exception( + f"litellm.batches.main.py::cancel_batch() - Error inferring custom_llm_provider - {str(e)}" + ) optional_params = GenericLiteLLMParams(**kwargs) litellm_params = get_litellm_params( custom_llm_provider=custom_llm_provider, @@ -1005,21 +1025,28 @@ def _handle_async_invoke_status( created_at=status_response["submitTime"], in_progress_at=status_response["lastModifiedTime"], completed_at=status_response.get("endTime"), - failed_at=status_response.get("endTime") - if status_response["status"] == "failed" - else None, - request_counts={ - "total": 1, - "completed": 1 if status_response["status"] == "completed" else 0, - "failed": 1 if status_response["status"] == "failed" else 0, - }, - metadata={ - "output_file_id": status_response["outputDataConfig"][ - "s3OutputDataConfig" - ]["s3Uri"], - "failure_message": status_response.get("failureMessage"), - "model_arn": status_response["modelArn"], - }, + failed_at=( + status_response.get("endTime") + if status_response["status"] == "failed" + else None + ), + request_counts=BatchRequestCounts( + total=1, + completed=1 if status_response["status"] == "completed" else 0, + failed=1 if status_response["status"] == "failed" else 0, + ), + metadata=dict( + **{ + "output_file_id": status_response["outputDataConfig"][ + "s3OutputDataConfig" + ]["s3Uri"], + "failure_message": status_response.get("failureMessage") or "", + "model_arn": status_response["modelArn"], + } + ), + completion_window="24h", + endpoint="/v1/embeddings", + input_file_id="", ) return result diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 55ae47fe461..8d6a7296385 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -193,7 +193,7 @@ class RedisCache(BaseCache): connection_pool=self.async_redis_conn_pool, **self.redis_kwargs ) in_memory_llm_clients_cache.set_cache( - key="async-redis-client", value=self.redis_async_client + key="async-redis-client", value=redis_async_client ) self.redis_async_client = redis_async_client # type: ignore diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 8c3ebd51036..f39d1bfb5fe 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -26,7 +26,12 @@ from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.bridges.completion_transformation import ( CompletionTransformationBridge, ) -from litellm.types.llms.openai import ChatCompletionToolParamFunctionChunk, Reasoning +from litellm.types.llms.openai import ( + ChatCompletionToolParamFunctionChunk, + Reasoning, + ResponsesAPIOptionalRequestParams, + ResponsesAPIStreamEvents, +) if TYPE_CHECKING: from openai.types.responses import ResponseInputImageParam @@ -88,6 +93,35 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): choice = Choices(message=msg, finish_reason="stop", index=index) return choice, index + 1 + # Handle function_call items (e.g., from GPT-5 Codex format) + if item_type == "function_call": + # Extract provider_specific_fields if present and pass through as-is + provider_specific_fields = item.get("provider_specific_fields") + if provider_specific_fields and not isinstance(provider_specific_fields, dict): + provider_specific_fields = dict(provider_specific_fields) if hasattr(provider_specific_fields, "__dict__") else {} + + tool_call_dict = { + "id": item.get("call_id") or item.get("id", ""), + "function": { + "name": item.get("name", ""), + "arguments": item.get("arguments", ""), + }, + "type": "function", + } + + # Pass through provider_specific_fields as-is if present + if provider_specific_fields: + tool_call_dict["provider_specific_fields"] = provider_specific_fields + # Also add to function's provider_specific_fields for consistency + tool_call_dict["function"]["provider_specific_fields"] = provider_specific_fields + + msg = Message( + content=None, + tool_calls=[tool_call_dict], + ) + choice = Choices(message=msg, finish_reason="tool_calls", index=index) + return choice, index + 1 + # Unknown or unsupported type return None, index @@ -165,13 +199,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): litellm_logging_obj: "LiteLLMLoggingObj", client: Optional[Any] = None, ) -> dict: - from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams - ( input_items, instructions, ) = self.convert_chat_completion_messages_to_responses_api(messages) + optional_params = self._extract_extra_body_params(optional_params) + # Build responses API request using the reverse transformation logic responses_api_request = ResponsesAPIOptionalRequestParams() @@ -194,9 +228,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys(): responses_api_request[key] = value # type: ignore - elif key in ("metadata"): + elif key == "metadata": responses_api_request["metadata"] = value - elif key in ("previous_response_id"): + elif key == "previous_response_id": responses_api_request["previous_response_id"] = value elif key == "reasoning_effort": responses_api_request["reasoning"] = self._map_reasoning_effort(value) @@ -252,7 +286,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return request_data - def transform_response( + def transform_response( # noqa: PLR0915 self, model: str, raw_response: "BaseModel", @@ -316,18 +350,37 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): reasoning_content = None # flush reasoning content index += 1 elif isinstance(item, ResponseFunctionToolCall): + + provider_specific_fields = None + if hasattr(item, "provider_specific_fields") and item.provider_specific_fields: + provider_specific_fields = item.provider_specific_fields + if not isinstance(provider_specific_fields, dict): + provider_specific_fields = dict(provider_specific_fields) if hasattr(provider_specific_fields, "__dict__") else {} + elif hasattr(item, "get") and callable(item.get): + provider_fields = item.get("provider_specific_fields") + if provider_fields: + provider_specific_fields = provider_fields if isinstance(provider_fields, dict) else (dict(provider_fields) if hasattr(provider_fields, "__dict__") else {}) + + function_dict: Dict[str, Any] = { + "name": item.name, + "arguments": item.arguments, + } + + if provider_specific_fields: + function_dict["provider_specific_fields"] = provider_specific_fields + + tool_call_dict: Dict[str, Any] = { + "id": item.call_id, + "function": function_dict, + "type": "function", + } + + if provider_specific_fields: + tool_call_dict["provider_specific_fields"] = provider_specific_fields + msg = Message( content=None, - tool_calls=[ - { - "id": item.call_id, - "function": { - "name": item.name, - "arguments": item.arguments, - }, - "type": "function", - } - ], + tool_calls=[tool_call_dict], reasoning_content=reasoning_content, ) @@ -538,6 +591,35 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return cast(List["ALL_RESPONSES_API_TOOL_PARAMS"], responses_tools) + def _extract_extra_body_params(self, optional_params: dict): + """ + Extract extra_body from optional_params and separate supported Responses API params + from unsupported ones. Supported params are moved to top-level optional_params, + unsupported params remain in extra_body. + """ + # Extract extra_body and separate supported params from unsupported ones + extra_body = optional_params.pop("extra_body", None) or {} + if not extra_body: + return optional_params + + supported_responses_api_params = set( + ResponsesAPIOptionalRequestParams.__annotations__.keys() + ) + # Also include params we handle specially + supported_responses_api_params.update({ + "previous_response_id", + "reasoning_effort", # We map this to "reasoning" + }) + + # Extract supported params from extra_body and merge into optional_params + extra_body_copy = extra_body.copy() + for key, value in extra_body_copy.items(): + if key in supported_responses_api_params: + # Prefer extra_body value if it exists (may have more complete info like summary in reasoning_effort) + optional_params[key] = extra_body.pop(key) + + return optional_params + def _map_reasoning_effort(self, reasoning_effort: Union[str, Dict[str, Any]]) -> Optional[Reasoning]: # If dict is passed, convert it directly to Reasoning object if isinstance(reasoning_effort, dict): @@ -596,7 +678,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): return self.chunk_parser(json.loads(str_line)) - def chunk_parser( + def chunk_parser( # noqa: PLR0915 self, chunk: dict ) -> Union["GenericStreamingChunk", "ModelResponseStream"]: # Transform responses API streaming chunk to chat completion format @@ -619,6 +701,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): # Handle different event types from responses API event_type = parsed_chunk.get("type") + if isinstance(event_type, ResponsesAPIStreamEvents): + event_type = event_type.value verbose_logger.debug(f"Chat provider: Processing event type: {event_type}") if event_type == "response.created": @@ -631,17 +715,33 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): # New output item added output_item = parsed_chunk.get("item", {}) if output_item.get("type") == "function_call": + # Extract provider_specific_fields if present + provider_specific_fields = output_item.get("provider_specific_fields") + if provider_specific_fields and not isinstance(provider_specific_fields, dict): + provider_specific_fields = dict(provider_specific_fields) if hasattr(provider_specific_fields, "__dict__") else {} + + function_chunk = ChatCompletionToolCallFunctionChunk( + name=output_item.get("name", None), + arguments=parsed_chunk.get("arguments", ""), + ) + + if provider_specific_fields: + function_chunk["provider_specific_fields"] = provider_specific_fields + + tool_call_chunk = ChatCompletionToolCallChunk( + id=output_item.get("call_id"), + index=0, + type="function", + function=function_chunk, + ) + + # Add provider_specific_fields if present + if provider_specific_fields: + tool_call_chunk.provider_specific_fields = provider_specific_fields # type: ignore + return GenericStreamingChunk( text="", - tool_use=ChatCompletionToolCallChunk( - id=output_item.get("call_id"), - index=0, - type="function", - function=ChatCompletionToolCallFunctionChunk( - name=parsed_chunk.get("name", None), - arguments=parsed_chunk.get("arguments", ""), - ), - ), + tool_use=tool_call_chunk, is_finished=False, finish_reason="", usage=None, @@ -677,17 +777,34 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): # New output item added output_item = parsed_chunk.get("item", {}) if output_item.get("type") == "function_call": + # Extract provider_specific_fields if present + provider_specific_fields = output_item.get("provider_specific_fields") + if provider_specific_fields and not isinstance(provider_specific_fields, dict): + provider_specific_fields = dict(provider_specific_fields) if hasattr(provider_specific_fields, "__dict__") else {} + + function_chunk = ChatCompletionToolCallFunctionChunk( + name=output_item.get("name", None), + arguments="", # responses API sends everything again, we don't + ) + + # Add provider_specific_fields to function if present + if provider_specific_fields: + function_chunk["provider_specific_fields"] = provider_specific_fields + + tool_call_chunk = ChatCompletionToolCallChunk( + id=output_item.get("call_id"), + index=0, + type="function", + function=function_chunk, + ) + + # Add provider_specific_fields if present + if provider_specific_fields: + tool_call_chunk.provider_specific_fields = provider_specific_fields # type: ignore + return GenericStreamingChunk( text="", - tool_use=ChatCompletionToolCallChunk( - id=output_item.get("call_id"), - index=0, - type="function", - function=ChatCompletionToolCallFunctionChunk( - name=parsed_chunk.get("name", None), - arguments="", # responses API sends everything again, we don't - ), - ), + tool_use=tool_call_chunk, is_finished=True, finish_reason="tool_calls", usage=None, diff --git a/litellm/constants.py b/litellm/constants.py index fc26e1cf817..2771b9b9c8f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1,7 +1,9 @@ import os from typing import List, Literal -DEFAULT_HEALTH_CHECK_PROMPT = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm")) +DEFAULT_HEALTH_CHECK_PROMPT = str( + os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm") +) AZURE_DEFAULT_RESPONSES_API_VERSION = str( os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview") ) @@ -18,7 +20,9 @@ DEFAULT_SQS_FLUSH_INTERVAL_SECONDS = int( DEFAULT_NUM_WORKERS_LITELLM_PROXY = int( os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1) ) -DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = int(os.getenv("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1)) +DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = int( + os.getenv("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1) +) DEFAULT_SQS_BATCH_SIZE = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512)) SQS_SEND_MESSAGE_ACTION = "SendMessage" SQS_API_VERSION = "2012-11-05" @@ -85,8 +89,12 @@ MAX_TOKEN_TRIMMING_ATTEMPTS = int( os.getenv("MAX_TOKEN_TRIMMING_ATTEMPTS", 10) ) # Maximum number of attempts to trim the message -RUNWAYML_DEFAULT_API_VERSION = str(os.getenv("RUNWAYML_DEFAULT_API_VERSION", "2024-11-06")) -RUNWAYML_POLLING_TIMEOUT = int(os.getenv("RUNWAYML_POLLING_TIMEOUT", 600)) # 10 minutes default for image generation +RUNWAYML_DEFAULT_API_VERSION = str( + os.getenv("RUNWAYML_DEFAULT_API_VERSION", "2024-11-06") +) +RUNWAYML_POLLING_TIMEOUT = int( + os.getenv("RUNWAYML_POLLING_TIMEOUT", 600) +) # 10 minutes default for image generation ########## Networking constants ############################################################## _DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600 # 1 hour, re-use the same httpx client for 1 hour @@ -110,22 +118,21 @@ REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES = ( DEFAULT_SSL_CIPHERS = os.getenv( "LITELLM_SSL_CIPHERS", # Priority 1: TLS 1.3 ciphers (fastest, ~50ms handshake) - "TLS_AES_256_GCM_SHA384:" # Fastest observed in testing - "TLS_AES_128_GCM_SHA256:" # Slightly faster than 256-bit - "TLS_CHACHA20_POLY1305_SHA256:" # Fast on ARM/mobile + "TLS_AES_256_GCM_SHA384:" # Fastest observed in testing + "TLS_AES_128_GCM_SHA256:" # Slightly faster than 256-bit + "TLS_CHACHA20_POLY1305_SHA256:" # Fast on ARM/mobile # Priority 2: TLS 1.2 ECDHE+GCM (fast, ~100ms handshake, widely supported) "ECDHE-RSA-AES256-GCM-SHA384:" "ECDHE-RSA-AES128-GCM-SHA256:" "ECDHE-ECDSA-AES256-GCM-SHA384:" "ECDHE-ECDSA-AES128-GCM-SHA256:" # Priority 3: Additional modern ciphers (good balance) - "ECDHE-RSA-CHACHA20-POLY1305:" - "ECDHE-ECDSA-CHACHA20-POLY1305:" + "ECDHE-RSA-CHACHA20-POLY1305:" "ECDHE-ECDSA-CHACHA20-POLY1305:" # Priority 4: Widely compatible fallbacks (slower but universally supported) - "ECDHE-RSA-AES256-SHA384:" # Common fallback - "ECDHE-RSA-AES128-SHA256:" # Very widely supported - "AES256-GCM-SHA384:" # Non-PFS fallback (compatibility) - "AES128-GCM-SHA256", # Last resort (maximum compatibility) + "ECDHE-RSA-AES256-SHA384:" # Common fallback + "ECDHE-RSA-AES128-SHA256:" # Very widely supported + "AES256-GCM-SHA384:" # Non-PFS fallback (compatibility) + "AES128-GCM-SHA256", # Last resort (maximum compatibility) ) ########### v2 Architecture constants for managing writing updates to the database ########### @@ -199,6 +206,7 @@ REPEATED_STREAMING_CHUNK_LIMIT = int( os.getenv("REPEATED_STREAMING_CHUNK_LIMIT", 100) ) # catch if model starts looping the same chunk while streaming. Uses high default to prevent false positives. DEFAULT_MAX_LRU_CACHE_SIZE = int(os.getenv("DEFAULT_MAX_LRU_CACHE_SIZE", 16)) +_REALTIME_BODY_CACHE_SIZE = 1000 # Keep realtime helper caches bounded; workloads rarely exceed 1k models/intents INITIAL_RETRY_DELAY = float(os.getenv("INITIAL_RETRY_DELAY", 0.5)) MAX_RETRY_DELAY = float(os.getenv("MAX_RETRY_DELAY", 8.0)) JITTER = float(os.getenv("JITTER", 0.75)) @@ -283,7 +291,9 @@ ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES = { DEFAULT_IMAGE_ENDPOINT_MODEL = "dall-e-2" DEFAULT_VIDEO_ENDPOINT_MODEL = "sora-2" -DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS = int(os.getenv("DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS", 8)) +DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS = int( + os.getenv("DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS", 8) +) ### DATAFORSEO CONSTANTS ### DEFAULT_DATAFORSEO_LOCATION_CODE = int( @@ -372,7 +382,8 @@ LITELLM_CHAT_PROVIDERS = [ "vercel_ai_gateway", "wandb", "ovhcloud", - "lemonade" + "lemonade", + "docker_model_runner", ] LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [ @@ -559,6 +570,7 @@ openai_compatible_providers: List = [ "wandb", "cometapi", "clarifai", + "docker_model_runner", ] openai_text_completion_compatible_providers: List = ( [ # providers that support `/v1/completions` @@ -632,7 +644,7 @@ clarifai_models: set = set( "clarifai/qwen.qwenLM.Qwen3-14B", "clarifai/qwen.qwenLM.QwQ-32B-AWQ", "clarifai/anthropic.completion.claude-3_5-haiku", - "clarifai/anthropic.completion.claude-3_7-sonnet", + "clarifai/anthropic.completion.claude-3_7-sonnet", ] ) @@ -798,28 +810,22 @@ WANDB_MODELS: set = set( # openai models "openai/gpt-oss-120b", "openai/gpt-oss-20b", - # zai-org models "zai-org/GLM-4.5", - # Qwen models "Qwen/Qwen3-235B-A22B-Instruct-2507", "Qwen/Qwen3-Coder-480B-A35B-Instruct", "Qwen/Qwen3-235B-A22B-Thinking-2507", - # moonshotai "moonshotai/Kimi-K2-Instruct", - # meta models "meta-llama/Llama-3.1-8B-Instruct", "meta-llama/Llama-3.3-70B-Instruct", "meta-llama/Llama-4-Scout-17B-16E-Instruct", - # deepseek-ai "deepseek-ai/DeepSeek-V3.1", "deepseek-ai/DeepSeek-R1-0528", "deepseek-ai/DeepSeek-V3-0324", - # microsoft "microsoft/Phi-4-mini-instruct", ] @@ -1033,13 +1039,17 @@ LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME = "litellm_internal_jobs" # Key Rotation Constants LITELLM_KEY_ROTATION_ENABLED = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false") -LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS = int(os.getenv("LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS", 86400)) # 24 hours default +LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS = int( + os.getenv("LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS", 86400) +) # 24 hours default UI_SESSION_TOKEN_TEAM_ID = "litellm-dashboard" LITELLM_PROXY_ADMIN_NAME = "default_user_id" ########################### CLI SSO AUTHENTICATION CONSTANTS ########################### LITELLM_CLI_SOURCE_IDENTIFIER = "litellm-cli" LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token" +CLI_SSO_SESSION_CACHE_KEY_PREFIX = "cli_sso_session" +CLI_JWT_TOKEN_NAME = "cli-jwt-token" ########################### DB CRON JOB NAMES ########################### DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job" @@ -1061,14 +1071,28 @@ PROXY_BATCH_POLLING_INTERVAL = int(os.getenv("PROXY_BATCH_POLLING_INTERVAL", 360 PROXY_BUDGET_RESCHEDULER_MAX_TIME = int( os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605) ) -PROXY_BATCH_WRITE_AT = int(os.getenv("PROXY_BATCH_WRITE_AT", 10)) # in seconds, increased from 10 +PROXY_BATCH_WRITE_AT = int( + os.getenv("PROXY_BATCH_WRITE_AT", 10) +) # in seconds, increased from 10 # APScheduler Configuration - MEMORY LEAK FIX # These settings prevent memory leaks in APScheduler's normalize() and _apply_jitter() functions -APSCHEDULER_COALESCE = os.getenv("APSCHEDULER_COALESCE", "True").lower() in ["true", "1"] # collapse many missed runs into one -APSCHEDULER_MISFIRE_GRACE_TIME = int(os.getenv("APSCHEDULER_MISFIRE_GRACE_TIME", 3600)) # ignore runs older than 1 hour (was 120) -APSCHEDULER_MAX_INSTANCES = int(os.getenv("APSCHEDULER_MAX_INSTANCES", 1)) # prevent concurrent job instances -APSCHEDULER_REPLACE_EXISTING = os.getenv("APSCHEDULER_REPLACE_EXISTING", "True").lower() in ["true", "1"] # always replace existing jobs +APSCHEDULER_COALESCE = os.getenv("APSCHEDULER_COALESCE", "True").lower() in [ + "true", + "1", +] # collapse many missed runs into one +APSCHEDULER_MISFIRE_GRACE_TIME = int( + os.getenv("APSCHEDULER_MISFIRE_GRACE_TIME", 3600) +) # ignore runs older than 1 hour (was 120) +APSCHEDULER_MAX_INSTANCES = int( + os.getenv("APSCHEDULER_MAX_INSTANCES", 1) +) # prevent concurrent job instances +APSCHEDULER_REPLACE_EXISTING = os.getenv( + "APSCHEDULER_REPLACE_EXISTING", "True" +).lower() in [ + "true", + "1", +] # always replace existing jobs DEFAULT_HEALTH_CHECK_INTERVAL = int( os.getenv("DEFAULT_HEALTH_CHECK_INTERVAL", 300) @@ -1098,6 +1122,8 @@ SECRET_MANAGER_REFRESH_INTERVAL = int( ) LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [ "default_internal_user_params", + "public_mcp_servers", + "public_agent_groups", "public_model_groups", "public_model_groups_links", ] diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index d1c7ede6552..0f5195e31af 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -133,6 +133,25 @@ def _cost_per_token_custom_pricing_helper( return None +def _transcription_usage_has_token_details( + usage_block: Optional[Usage], +) -> bool: + if usage_block is None: + return False + + prompt_tokens_val = getattr(usage_block, "prompt_tokens", 0) or 0 + completion_tokens_val = getattr(usage_block, "completion_tokens", 0) or 0 + prompt_details = getattr(usage_block, "prompt_tokens_details", None) + + if prompt_details is not None: + audio_token_count = getattr(prompt_details, "audio_tokens", 0) or 0 + text_token_count = getattr(prompt_details, "text_tokens", 0) or 0 + if audio_token_count > 0 or text_token_count > 0: + return True + + return (prompt_tokens_val > 0) or (completion_tokens_val > 0) + + def cost_per_token( # noqa: PLR0915 model: str = "", prompt_tokens: int = 0, @@ -324,19 +343,18 @@ def cost_per_token( # noqa: PLR0915 usage=usage_block, model=model, custom_llm_provider=custom_llm_provider ) elif call_type == "atranscription" or call_type == "transcription": - - if model == "gpt-4o-mini-transcribe": + if _transcription_usage_has_token_details(usage_block): return openai_cost_per_token( - model=model, + model=model_without_prefix, usage=usage_block, service_tier=service_tier, ) - else: - return openai_cost_per_second( - model=model, - custom_llm_provider=custom_llm_provider, - duration=audio_transcription_file_duration, - ) + + return openai_cost_per_second( + model=model_without_prefix, + custom_llm_provider=custom_llm_provider, + duration=audio_transcription_file_duration, + ) elif call_type == "search" or call_type == "asearch": # Search providers use per-query pricing from litellm.search import search_provider_cost_per_query diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 6aa671a5011..943cc6b2d53 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -5,17 +5,24 @@ LiteLLM Proxy uses this MCP Client to connnect to other MCP servers. import asyncio import base64 from datetime import timedelta -from typing import Callable, Dict, List, Optional, Union +from typing import Awaitable, Callable, Dict, List, Optional, TypeVar, Union import httpx -from mcp import ClientSession, StdioServerParameters +from mcp import ClientSession, ReadResourceResult, Resource, StdioServerParameters from mcp.client.sse import sse_client from mcp.client.stdio import stdio_client from mcp.client.streamable_http import streamablehttp_client -from mcp.types import CallToolRequestParams as MCPCallToolRequestParams +from mcp.types import ( + CallToolRequestParams as MCPCallToolRequestParams, + GetPromptRequestParams, + GetPromptResult, + Prompt, + ResourceTemplate, +) from mcp.types import CallToolResult as MCPCallToolResult from mcp.types import TextContent from mcp.types import Tool as MCPTool +from pydantic import AnyUrl from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import get_ssl_configuration @@ -34,6 +41,9 @@ def to_basic_auth(auth_value: str) -> str: return base64.b64encode(auth_value.encode("utf-8")).decode() +TSessionResult = TypeVar("TSessionResult") + + class MCPClient: """ MCP Client supporting: @@ -58,12 +68,6 @@ class MCPClient: self.auth_type: MCPAuthType = auth_type self.timeout: float = timeout self._mcp_auth_value: Optional[Union[str, Dict[str, str]]] = None - self._session: Optional[ClientSession] = None - self._context = None - self._transport_ctx = None - self._transport = None - self._session_ctx = None - self._task: Optional[asyncio.Task] = None self.stdio_config: Optional[MCPStdioConfig] = stdio_config self.extra_headers: Optional[Dict[str, str]] = extra_headers self.ssl_verify: Optional[VerifyTypes] = ssl_verify @@ -71,33 +75,14 @@ class MCPClient: if auth_value: self.update_auth_value(auth_value) - async def __aenter__(self): - """ - Enable async context manager support. - Initializes the transport and session. - """ - try: - await self.connect() - return self - except Exception: - await self.disconnect() - raise - - async def connect(self): - """Initialize the transport and session.""" - if self._session: - verbose_logger.debug( - f"MCP client already connected to {self.server_url or 'stdio'}" - ) - return # Already connected - - verbose_logger.info( - f"MCP client connecting to {self.server_url or 'stdio'} via {self.transport_type}" - ) + async def run_with_session( + self, operation: Callable[[ClientSession], Awaitable[TSessionResult]] + ) -> TSessionResult: + """Open a session, run the provided coroutine, and clean up.""" + transport_ctx = None try: if self.transport_type == MCPTransport.stdio: - # For stdio transport, use stdio_client with command-line parameters if not self.stdio_config: raise ValueError("stdio_config is required for stdio transport") @@ -106,117 +91,43 @@ class MCPClient: args=self.stdio_config.get("args", []), env=self.stdio_config.get("env", {}), ) - - self._transport_ctx = stdio_client(server_params) - self._transport = await self._transport_ctx.__aenter__() - self._session_ctx = ClientSession( - self._transport[0], self._transport[1] - ) - self._session = await self._session_ctx.__aenter__() - await self._session.initialize() - verbose_logger.info( - f"MCP client successfully connected via stdio: {self.stdio_config.get('command', '')}" - ) + transport_ctx = stdio_client(server_params) elif self.transport_type == MCPTransport.sse: headers = self._get_auth_headers() httpx_client_factory = self._create_httpx_client_factory() - self._transport_ctx = sse_client( + transport_ctx = sse_client( url=self.server_url, timeout=self.timeout, headers=headers, httpx_client_factory=httpx_client_factory, ) - self._transport = await self._transport_ctx.__aenter__() - self._session_ctx = ClientSession( - self._transport[0], self._transport[1] - ) - self._session = await self._session_ctx.__aenter__() - await self._session.initialize() - verbose_logger.info( - f"MCP client successfully connected via SSE to {self.server_url}" - ) - else: # http + else: headers = self._get_auth_headers() httpx_client_factory = self._create_httpx_client_factory() verbose_logger.debug( "litellm headers for streamablehttp_client: %s", headers ) - self._transport_ctx = streamablehttp_client( + transport_ctx = streamablehttp_client( url=self.server_url, timeout=timedelta(seconds=self.timeout), headers=headers, httpx_client_factory=httpx_client_factory, ) - self._transport = await self._transport_ctx.__aenter__() - self._session_ctx = ClientSession( - self._transport[0], self._transport[1] - ) - self._session = await self._session_ctx.__aenter__() - await self._session.initialize() - verbose_logger.info( - f"MCP client successfully connected via HTTP to {self.server_url}" - ) - except ValueError as e: - # Re-raise ValueError exceptions (like missing stdio_config) - verbose_logger.warning(f"MCP client connection failed: {str(e)}") - await self.disconnect() + + if transport_ctx is None: + raise RuntimeError("Failed to create transport context") + + async with transport_ctx as transport: + read_stream, write_stream = transport[0], transport[1] + session_ctx = ClientSession(read_stream, write_stream) + async with session_ctx as session: + await session.initialize() + return await operation(session) + except Exception: + verbose_logger.warning( + "MCP client run_with_session failed for %s", self.server_url or "stdio" + ) raise - except Exception as e: - verbose_logger.warning(f"MCP client connection failed: {str(e)}") - await self.disconnect() - # Don't raise other exceptions, let the calling code handle it gracefully - # This allows the server manager to continue with other servers - # Instead of raising, we'll let the calling code handle the failure - pass - - async def __aexit__(self, exc_type, exc_val, exc_tb): - """Cleanup when exiting context manager.""" - await self.disconnect() - - async def disconnect(self): - """Clean up session and connections.""" - verbose_logger.info( - f"MCP client disconnecting from {self.server_url or 'stdio'}" - ) - - if self._task and not self._task.done(): - verbose_logger.debug("MCP client cancelling background task") - self._task.cancel() - try: - await self._task - except asyncio.CancelledError: - pass - - if self._session: - try: - verbose_logger.debug("MCP client closing session") - await self._session_ctx.__aexit__(None, None, None) # type: ignore - except Exception as e: - verbose_logger.debug( - f"Error closing MCP session: {type(e).__name__}: {str(e)}" - ) - pass - self._session = None - self._session_ctx = None - - if self._transport_ctx: - try: - verbose_logger.debug("MCP client closing transport") - await self._transport_ctx.__aexit__(None, None, None) - except Exception as e: - verbose_logger.debug( - f"Error closing MCP transport: {type(e).__name__}: {str(e)}" - ) - pass - self._transport_ctx = None - self._transport = None - - if self._context: - try: - await self._context.__aexit__(None, None, None) # type: ignore - except Exception: - pass - self._context = None def update_auth_value(self, mcp_auth_value: Union[str, Dict[str, str]]): """ @@ -294,24 +205,11 @@ class MCPClient: f"MCP client listing tools from {self.server_url or 'stdio'}" ) - if not self._session: - verbose_logger.debug("MCP client session not found, attempting to connect") - try: - await self.connect() - except Exception as e: - verbose_logger.error( - f"MCP client connection failed during list_tools: {type(e).__name__}: {str(e)}" - ) - return [] - - if self._session is None: - verbose_logger.error( - "MCP client session is not initialized after connection attempt" - ) - return [] + async def _list_tools_operation(session: ClientSession): + return await session.list_tools() try: - result = await self._session.list_tools() + result = await self.run_with_session(_list_tools_operation) tool_count = len(result.tools) tool_names = [tool.name for tool in result.tools] verbose_logger.info( @@ -320,11 +218,10 @@ class MCPClient: return result.tools except asyncio.CancelledError: verbose_logger.warning("MCP client list_tools was cancelled") - await self.disconnect() raise except Exception as e: error_type = type(e).__name__ - verbose_logger.error( + verbose_logger.exception( f"MCP client list_tools failed - " f"Error Type: {error_type}, " f"Error: {str(e)}, " @@ -339,7 +236,6 @@ class MCPClient: "the MCP server may have crashed, disconnected, or timed out" ) - await self.disconnect() # Return empty list instead of raising to allow graceful degradation return [] @@ -353,55 +249,21 @@ class MCPClient: f"MCP client calling tool '{call_tool_request_params.name}' with arguments: {call_tool_request_params.arguments}" ) - if not self._session: - verbose_logger.warning( - "MCP client session not found, attempting to connect" - ) - try: - await self.connect() - except Exception as e: - verbose_logger.error( - f"MCP client connection failed before tool call: {type(e).__name__}: {str(e)}" - ) - return MCPCallToolResult( - content=[TextContent(type="text", text=f"{str(e)}")], isError=True - ) - - if self._session is None: - verbose_logger.error( - "MCP client session is not initialized after connection attempt" - ) - return MCPCallToolResult( - content=[ - TextContent( - type="text", text="MCP client session is not initialized" - ) - ], - isError=True, - ) - - # Check session and transport state before calling tool - verbose_logger.debug( - f"MCP client state before tool call - " - f"session: {'active' if self._session else 'none'}, " - f"transport: {'active' if self._transport else 'none'}, " - f"session_ctx: {'active' if self._session_ctx else 'none'}, " - f"transport_ctx: {'active' if self._transport_ctx else 'none'}" - ) - - try: + async def _call_tool_operation(session: ClientSession): verbose_logger.debug("MCP client sending tool call to session") - tool_result = await self._session.call_tool( + return await session.call_tool( name=call_tool_request_params.name, arguments=call_tool_request_params.arguments, ) + + try: + tool_result = await self.run_with_session(_call_tool_operation) verbose_logger.info( f"MCP client tool call '{call_tool_request_params.name}' completed successfully" ) return tool_result except asyncio.CancelledError: verbose_logger.warning("MCP client tool call was cancelled") - await self.disconnect() raise except Exception as e: import traceback @@ -424,11 +286,9 @@ class MCPClient: if "BrokenResourceError" in error_type or "Broken" in error_type: verbose_logger.error( "MCP client detected broken connection/stream - " - "the MCP server may have crashed, disconnected, or timed out. " - "Session and transport will be disconnected." + "the MCP server may have crashed, disconnected, or timed out." ) - await self.disconnect() # Return a default error result instead of raising return MCPCallToolResult( content=[ @@ -436,3 +296,218 @@ class MCPClient: ], # Empty content for error case isError=True, ) + + async def list_prompts(self) -> List[Prompt]: + """List available prompts from the server.""" + verbose_logger.debug( + f"MCP client listing tools from {self.server_url or 'stdio'}" + ) + + async def _list_prompts_operation(session: ClientSession): + return await session.list_prompts() + + try: + result = await self.run_with_session(_list_prompts_operation) + prompt_count = len(result.prompts) + prompt_names = [prompt.name for prompt in result.prompts] + verbose_logger.info( + f"MCP client listed {prompt_count} tools from {self.server_url or 'stdio'}: {prompt_names}" + ) + return result.prompts + except asyncio.CancelledError: + verbose_logger.warning("MCP client list_prompts was cancelled") + raise + except Exception as e: + error_type = type(e).__name__ + verbose_logger.error( + f"MCP client list_prompts failed - " + f"Error Type: {error_type}, " + f"Error: {str(e)}, " + f"Server: {self.server_url or 'stdio'}, " + f"Transport: {self.transport_type}" + ) + + # Check if it's a stream/connection error + if "BrokenResourceError" in error_type or "Broken" in error_type: + verbose_logger.error( + "MCP client detected broken connection/stream during list_tools - " + "the MCP server may have crashed, disconnected, or timed out" + ) + + # Return empty list instead of raising to allow graceful degradation + return [] + + async def get_prompt( + self, get_prompt_request_params: GetPromptRequestParams + ) -> GetPromptResult: + """Fetch a prompt definition from the MCP server.""" + verbose_logger.info( + f"MCP client fetching prompt '{get_prompt_request_params.name}' with arguments: {get_prompt_request_params.arguments}" + ) + + async def _get_prompt_operation(session: ClientSession): + verbose_logger.debug("MCP client sending get_prompt request to session") + return await session.get_prompt( + name=get_prompt_request_params.name, + arguments=get_prompt_request_params.arguments, + ) + + try: + get_prompt_result = await self.run_with_session(_get_prompt_operation) + verbose_logger.info( + f"MCP client get_prompt '{get_prompt_request_params.name}' completed successfully" + ) + return get_prompt_result + except asyncio.CancelledError: + verbose_logger.warning("MCP client get_prompt was cancelled") + raise + except Exception as e: + import traceback + + error_trace = traceback.format_exc() + verbose_logger.debug(f"MCP client get_prompt traceback:\n{error_trace}") + + # Log detailed error information + error_type = type(e).__name__ + verbose_logger.error( + f"MCP client get_prompt failed - " + f"Error Type: {error_type}, " + f"Error: {str(e)}, " + f"Prompt: {get_prompt_request_params.name}, " + f"Server: {self.server_url or 'stdio'}, " + f"Transport: {self.transport_type}" + ) + + # Check if it's a stream/connection error + if "BrokenResourceError" in error_type or "Broken" in error_type: + verbose_logger.error( + "MCP client detected broken connection/stream during get_prompt - " + "the MCP server may have crashed, disconnected, or timed out." + ) + + raise + + async def list_resources(self) -> list[Resource]: + """List available resources from the server.""" + verbose_logger.debug( + f"MCP client listing resources from {self.server_url or 'stdio'}" + ) + + async def _list_resources_operation(session: ClientSession): + return await session.list_resources() + + try: + result = await self.run_with_session(_list_resources_operation) + resource_count = len(result.resources) + resource_names = [resource.name for resource in result.resources] + verbose_logger.info( + f"MCP client listed {resource_count} resources from {self.server_url or 'stdio'}: {resource_names}" + ) + return result.resources + except asyncio.CancelledError: + verbose_logger.warning("MCP client list_resources was cancelled") + raise + except Exception as e: + error_type = type(e).__name__ + verbose_logger.error( + f"MCP client list_resources failed - " + f"Error Type: {error_type}, " + f"Error: {str(e)}, " + f"Server: {self.server_url or 'stdio'}, " + f"Transport: {self.transport_type}" + ) + + # Check if it's a stream/connection error + if "BrokenResourceError" in error_type or "Broken" in error_type: + verbose_logger.error( + "MCP client detected broken connection/stream during list_resources - " + "the MCP server may have crashed, disconnected, or timed out" + ) + + # Return empty list instead of raising to allow graceful degradation + return [] + + async def list_resource_templates(self) -> list[ResourceTemplate]: + """List available resource templates from the server.""" + verbose_logger.debug( + f"MCP client listing resource templates from {self.server_url or 'stdio'}" + ) + + async def _list_resource_templates_operation(session: ClientSession): + return await session.list_resource_templates() + + try: + result = await self.run_with_session(_list_resource_templates_operation) + resource_template_count = len(result.resourceTemplates) + resource_template_names = [ + resourceTemplate.name for resourceTemplate in result.resourceTemplates + ] + verbose_logger.info( + f"MCP client listed {resource_template_count} resource templates from {self.server_url or 'stdio'}: {resource_template_names}" + ) + return result.resourceTemplates + except asyncio.CancelledError: + verbose_logger.warning("MCP client list_resource_templates was cancelled") + raise + except Exception as e: + error_type = type(e).__name__ + verbose_logger.error( + f"MCP client list_resource_templates failed - " + f"Error Type: {error_type}, " + f"Error: {str(e)}, " + f"Server: {self.server_url or 'stdio'}, " + f"Transport: {self.transport_type}" + ) + + # Check if it's a stream/connection error + if "BrokenResourceError" in error_type or "Broken" in error_type: + verbose_logger.error( + "MCP client detected broken connection/stream during list_resource_templates - " + "the MCP server may have crashed, disconnected, or timed out" + ) + + # Return empty list instead of raising to allow graceful degradation + return [] + + async def read_resource(self, url: AnyUrl) -> ReadResourceResult: + """Fetch resource contents from the MCP server.""" + verbose_logger.info(f"MCP client fetching resource '{url}'") + + async def _read_resource_operation(session: ClientSession): + verbose_logger.debug("MCP client sending read_resource request to session") + return await session.read_resource(url) + + try: + read_resource_result = await self.run_with_session(_read_resource_operation) + verbose_logger.info( + f"MCP client read_resource '{url}' completed successfully" + ) + return read_resource_result + except asyncio.CancelledError: + verbose_logger.warning("MCP client read_resource was cancelled") + raise + except Exception as e: + import traceback + + error_trace = traceback.format_exc() + verbose_logger.debug(f"MCP client read_resource traceback:\n{error_trace}") + + # Log detailed error information + error_type = type(e).__name__ + verbose_logger.error( + f"MCP client read_resource failed - " + f"Error Type: {error_type}, " + f"Error: {str(e)}, " + f"Url: {url}, " + f"Server: {self.server_url or 'stdio'}, " + f"Transport: {self.transport_type}" + ) + + # Check if it's a stream/connection error + if "BrokenResourceError" in error_type or "Broken" in error_type: + verbose_logger.error( + "MCP client detected broken connection/stream during read_resource - " + "the MCP server may have crashed, disconnected, or timed out." + ) + + raise diff --git a/litellm/files/main.py b/litellm/files/main.py index 9c85fa10565..535772fa42c 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -95,7 +95,9 @@ async def acreate_file( def create_file( file: FileTypes, purpose: Literal["assistants", "batch", "fine-tune"], - custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai", "bedrock"]] = None, + custom_llm_provider: Optional[ + Literal["openai", "azure", "vertex_ai", "bedrock"] + ] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -155,10 +157,12 @@ def create_file( api_key=optional_params.api_key, logging_obj=logging_obj, _is_async=_is_async, - client=client - if client is not None - and isinstance(client, (HTTPHandler, AsyncHTTPHandler)) - else None, + client=( + client + if client is not None + and isinstance(client, (HTTPHandler, AsyncHTTPHandler)) + else None + ), timeout=timeout, ) elif custom_llm_provider == "openai": @@ -441,12 +445,14 @@ async def afile_delete( """ try: loop = asyncio.get_event_loop() + model = kwargs.pop("model", None) kwargs["is_async"] = True # Use a partial function to pass your keyword arguments func = partial( file_delete, file_id, + model, custom_llm_provider, extra_headers, extra_body, @@ -470,7 +476,8 @@ async def afile_delete( @client def file_delete( file_id: str, - custom_llm_provider: Literal["openai", "azure"] = "openai", + model: Optional[str] = None, + custom_llm_provider: Union[Literal["openai", "azure"], str] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -481,6 +488,13 @@ def file_delete( LiteLLM Equivalent of DELETE https://api.openai.com/v1/files """ try: + try: + if model is not None: + _, custom_llm_provider, _, _ = get_llm_provider( + model, custom_llm_provider + ) + except Exception: + pass optional_params = GenericLiteLLMParams(**kwargs) litellm_params_dict = get_litellm_params(**kwargs) ### TIMEOUT LOGIC ### @@ -566,7 +580,7 @@ def file_delete( ) else: raise litellm.exceptions.BadRequestError( - message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format( + message="LiteLLM doesn't support {} for 'delete_batch'. Only 'openai' is supported.".format( custom_llm_provider ), model="n/a", diff --git a/litellm/integrations/arize/_utils.py b/litellm/integrations/arize/_utils.py index 10597d6e713..c9a1531b5d4 100644 --- a/litellm/integrations/arize/_utils.py +++ b/litellm/integrations/arize/_utils.py @@ -208,7 +208,10 @@ def set_attributes( ) try: + # Remove secret_fields to prevent leaking sensitive data (e.g., authorization headers) optional_params = kwargs.get("optional_params", {}) + if isinstance(optional_params, dict): + optional_params.pop("secret_fields", None) litellm_params = kwargs.get("litellm_params", {}) standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get( "standard_logging_object" diff --git a/litellm/integrations/arize/arize_phoenix.py b/litellm/integrations/arize/arize_phoenix.py index 60566ee55c0..666d322cb2e 100644 --- a/litellm/integrations/arize/arize_phoenix.py +++ b/litellm/integrations/arize/arize_phoenix.py @@ -1,5 +1,4 @@ import os -import urllib.parse from typing import TYPE_CHECKING, Any, Union from litellm._logging import verbose_logger @@ -23,7 +22,7 @@ else: Span = Any -ARIZE_HOSTED_PHOENIX_ENDPOINT = "https://app.phoenix.arize.com/v1/traces" +ARIZE_HOSTED_PHOENIX_ENDPOINT = "https://otlp.arize.com/v1/traces" class ArizePhoenixLogger: @@ -41,38 +40,53 @@ class ArizePhoenixLogger: ArizePhoenixConfig: A Pydantic model containing Arize Phoenix configuration. """ api_key = os.environ.get("PHOENIX_API_KEY", None) - grpc_endpoint = os.environ.get("PHOENIX_COLLECTOR_ENDPOINT", None) - http_endpoint = os.environ.get("PHOENIX_COLLECTOR_HTTP_ENDPOINT", None) + + collector_endpoint = os.environ.get("PHOENIX_COLLECTOR_HTTP_ENDPOINT", None) + + if not collector_endpoint: + grpc_endpoint = os.environ.get("PHOENIX_COLLECTOR_ENDPOINT", None) + http_endpoint = os.environ.get("PHOENIX_COLLECTOR_HTTP_ENDPOINT", None) + collector_endpoint = http_endpoint or grpc_endpoint endpoint = None protocol: Protocol = "otlp_http" - if http_endpoint: - endpoint = http_endpoint - protocol = "otlp_http" - elif grpc_endpoint: - endpoint = grpc_endpoint - protocol = "otlp_grpc" + if collector_endpoint: + # Parse the endpoint to determine protocol + if collector_endpoint.startswith("grpc://") or (":4317" in collector_endpoint and "/v1/traces" not in collector_endpoint): + endpoint = collector_endpoint + protocol = "otlp_grpc" + else: + # Phoenix Cloud endpoints (app.phoenix.arize.com) include the space in the URL + if "app.phoenix.arize.com" in collector_endpoint: + endpoint = collector_endpoint + protocol = "otlp_http" + # For other HTTP endpoints, ensure they have the correct path + elif "/v1/traces" not in collector_endpoint: + if collector_endpoint.endswith("/v1"): + endpoint = collector_endpoint + "/traces" + elif collector_endpoint.endswith("/"): + endpoint = f"{collector_endpoint}v1/traces" + else: + endpoint = f"{collector_endpoint}/v1/traces" + else: + endpoint = collector_endpoint + protocol = "otlp_http" else: - endpoint = ARIZE_HOSTED_PHOENIX_ENDPOINT + # If no endpoint specified, self hosted phoenix + endpoint = "http://localhost:6006/v1/traces" protocol = "otlp_http" verbose_logger.debug( - f"No PHOENIX_COLLECTOR_ENDPOINT or PHOENIX_COLLECTOR_HTTP_ENDPOINT found, using default endpoint with http: {ARIZE_HOSTED_PHOENIX_ENDPOINT}" + f"No PHOENIX_COLLECTOR_ENDPOINT found, using default local Phoenix endpoint: {endpoint}" ) otlp_auth_headers = None - # If the endpoint is the Arize hosted Phoenix endpoint, use the api_key as the auth header as currently it is uses - # a slightly different auth header format than self hosted phoenix - if endpoint == ARIZE_HOSTED_PHOENIX_ENDPOINT: - if api_key is None: - raise ValueError( - "PHOENIX_API_KEY must be set when the Arize hosted Phoenix endpoint is used." - ) - otlp_auth_headers = f"api_key={api_key}" - elif api_key is not None: - # api_key/auth is optional for self hosted phoenix - otlp_auth_headers = ( - f"Authorization={urllib.parse.quote(f'Bearer {api_key}')}" + if api_key is not None: + otlp_auth_headers = f"Authorization=Bearer {api_key}" + elif "app.phoenix.arize.com" in endpoint: + # Phoenix Cloud requires an API key + raise ValueError( + "PHOENIX_API_KEY must be set when using Phoenix Cloud (app.phoenix.arize.com)." ) return ArizePhoenixConfig( diff --git a/litellm/integrations/callback_configs.json b/litellm/integrations/callback_configs.json new file mode 100644 index 00000000000..d8a96e71769 --- /dev/null +++ b/litellm/integrations/callback_configs.json @@ -0,0 +1,404 @@ +[ + { + "id": "arize", + "displayName": "Arize", + "logo": "arize.png", + "supports_key_team_logging": true, + "dynamic_params": { + "arize_api_key": { + "type": "password", + "ui_name": "API Key", + "description": "Arize API key for authentication", + "required": true + }, + "arize_space_key": { + "type": "password", + "ui_name": "Space Key", + "description": "Arize Space key to identify your workspace", + "required": true + } + }, + "description": "Arize Logging Integration" + }, + { + "id": "braintrust", + "displayName": "Braintrust", + "logo": "braintrust.png", + "supports_key_team_logging": false, + "dynamic_params": { + "braintrust_api_key": { + "type": "password", + "ui_name": "API Key", + "description": "Braintrust API key for authentication", + "required": true + }, + "braintrust_project_name": { + "type": "text", + "ui_name": "Project Name", + "description": "Name of the Braintrust project to log to", + "required": true + } + }, + "description": "Braintrust Logging Integration" + }, + { + "id": "custom_callback_api", + "displayName": "Custom Callback API", + "logo": "custom.svg", + "supports_key_team_logging": true, + "dynamic_params": { + "custom_callback_api_url": { + "type": "text", + "ui_name": "Callback URL", + "description": "Your custom webhook/API endpoint URL to receive logs", + "required": true + }, + "custom_callback_api_headers": { + "type": "text", + "ui_name": "Headers (JSON)", + "description": "Custom HTTP headers as JSON string (e.g., {\"Authorization\": \"Bearer token\"})", + "required": false + } + }, + "description": "Custom Callback API Logging Integration" + }, + { + "id": "datadog", + "displayName": "Datadog", + "logo": "datadog.png", + "supports_key_team_logging": false, + "dynamic_params": { + "dd_api_key": { + "type": "password", + "ui_name": "API Key", + "description": "Datadog API key for authentication", + "required": true + }, + "dd_site": { + "type": "text", + "ui_name": "Site", + "description": "Datadog site URL (e.g., us5.datadoghq.com)", + "required": true + } + }, + "description": "Datadog Logging Integration" + }, + { + "id": "lago", + "displayName": "Lago", + "logo": "lago.svg", + "supports_key_team_logging": false, + "dynamic_params": { + "lago_api_url": { + "type": "text", + "ui_name": "API URL", + "description": "Lago API base URL", + "required": true + }, + "lago_api_key": { + "type": "password", + "ui_name": "API Key", + "description": "Lago API key for authentication", + "required": true + } + }, + "description": "Lago Billing Logging Integration" + }, + { + "id": "langfuse", + "displayName": "Langfuse", + "logo": "langfuse.png", + "supports_key_team_logging": true, + "dynamic_params": { + "langfuse_public_key": { + "type": "text", + "ui_name": "Public Key", + "description": "Langfuse public key", + "required": true + }, + "langfuse_secret_key": { + "type": "password", + "ui_name": "Secret Key", + "description": "Langfuse secret key for authentication", + "required": true + }, + "langfuse_host": { + "type": "text", + "ui_name": "Host URL", + "description": "Langfuse host URL (default: https://cloud.langfuse.com)", + "required": false + } + }, + "description": "Langfuse v2 Logging Integration" + }, + { + "id": "langfuse_otel", + "displayName": "Langfuse OTEL", + "logo": "langfuse.png", + "supports_key_team_logging": true, + "dynamic_params": { + "langfuse_public_key": { + "type": "text", + "ui_name": "Public Key", + "description": "Langfuse public key", + "required": true + }, + "langfuse_secret_key": { + "type": "password", + "ui_name": "Secret Key", + "description": "Langfuse secret key for authentication", + "required": true + }, + "langfuse_host": { + "type": "text", + "ui_name": "Host URL", + "description": "Langfuse host URL (default: https://cloud.langfuse.com)", + "required": false + } + }, + "description": "Langfuse v3 OTEL Logging Integration" + }, + { + "id": "langsmith", + "displayName": "LangSmith", + "logo": "langsmith.png", + "supports_key_team_logging": true, + "dynamic_params": { + "langsmith_api_key": { + "type": "password", + "ui_name": "API Key", + "description": "LangSmith API key for authentication", + "required": true + }, + "langsmith_project": { + "type": "text", + "ui_name": "Project Name", + "description": "LangSmith project name (default: litellm-completion)", + "required": false + }, + "langsmith_base_url": { + "type": "text", + "ui_name": "Base URL", + "description": "LangSmith base URL (default: https://api.smith.langchain.com)", + "required": false + }, + "langsmith_sampling_rate": { + "type": "number", + "ui_name": "Sampling Rate", + "description": "Sampling rate for logging (0.0 to 1.0, default: 1.0)", + "required": false + } + }, + "description": "Langsmith Logging Integration" + }, + { + "id": "openmeter", + "displayName": "OpenMeter", + "logo": "openmeter.png", + "supports_key_team_logging": false, + "dynamic_params": { + "openmeter_api_key": { + "type": "password", + "ui_name": "API Key", + "description": "OpenMeter API key for authentication", + "required": true + }, + "openmeter_base_url": { + "type": "text", + "ui_name": "Base URL", + "description": "OpenMeter base URL (default: https://openmeter.cloud)", + "required": false + } + }, + "description": "OpenMeter Logging Integration" + }, + { + "id": "otel", + "displayName": "Open Telemetry", + "logo": "otel.png", + "supports_key_team_logging": false, + "dynamic_params": { + "otel_endpoint": { + "type": "text", + "ui_name": "Endpoint URL", + "description": "OpenTelemetry collector endpoint URL", + "required": true + }, + "otel_headers": { + "type": "text", + "ui_name": "Headers", + "description": "Headers for OTEL exporter (e.g., x-honeycomb-team=YOUR_API_KEY)", + "required": false + } + }, + "description": "OpenTelemetry Logging Integration" + }, + { + "id": "s3", + "displayName": "S3", + "logo": "aws.svg", + "supports_key_team_logging": false, + "dynamic_params": { + "s3_bucket_name": { + "type": "text", + "ui_name": "Bucket Name", + "description": "AWS S3 bucket name to store logs", + "required": true + }, + "s3_region_name": { + "type": "text", + "ui_name": "AWS Region", + "description": "AWS region name (e.g., us-east-1)", + "required": false + }, + "s3_aws_access_key_id": { + "type": "password", + "ui_name": "AWS Access Key ID", + "description": "AWS access key ID for authentication", + "required": false + }, + "s3_aws_secret_access_key": { + "type": "password", + "ui_name": "AWS Secret Access Key", + "description": "AWS secret access key for authentication", + "required": false + }, + "s3_aws_session_token": { + "type": "password", + "ui_name": "AWS Session Token", + "description": "AWS session token for temporary credentials", + "required": false + }, + "s3_endpoint_url": { + "type": "text", + "ui_name": "S3 Endpoint URL", + "description": "Custom S3 endpoint URL (for MinIO or custom S3-compatible services)", + "required": false + }, + "s3_path": { + "type": "text", + "ui_name": "S3 Path Prefix", + "description": "Path prefix within the bucket for organizing logs", + "required": false + } + }, + "description": "S3 Bucket (AWS) Logging Integration" + }, + { + "id": "sqs", + "displayName": "SQS", + "logo": "aws.svg", + "supports_key_team_logging": false, + "dynamic_params": { + "sqs_queue_url": { + "type": "text", + "ui_name": "Queue URL", + "description": "AWS SQS Queue URL", + "required": true + }, + "sqs_region_name": { + "type": "text", + "ui_name": "AWS Region", + "description": "AWS region name (e.g., us-east-1)", + "required": false + }, + "sqs_aws_access_key_id": { + "type": "password", + "ui_name": "AWS Access Key ID", + "description": "AWS access key ID for authentication", + "required": false + }, + "sqs_aws_secret_access_key": { + "type": "password", + "ui_name": "AWS Secret Access Key", + "description": "AWS secret access key for authentication", + "required": false + }, + "sqs_aws_session_token": { + "type": "password", + "ui_name": "AWS Session Token", + "description": "AWS session token for temporary credentials", + "required": false + }, + "sqs_aws_session_name": { + "type": "text", + "ui_name": "AWS Session Name", + "description": "Name for AWS session", + "required": false + }, + "sqs_aws_profile_name": { + "type": "text", + "ui_name": "AWS Profile Name", + "description": "AWS profile name from credentials file", + "required": false + }, + "sqs_aws_role_name": { + "type": "text", + "ui_name": "AWS Role Name", + "description": "AWS IAM role name to assume", + "required": false + }, + "sqs_aws_web_identity_token": { + "type": "password", + "ui_name": "AWS Web Identity Token", + "description": "AWS web identity token for authentication", + "required": false + }, + "sqs_aws_sts_endpoint": { + "type": "text", + "ui_name": "AWS STS Endpoint", + "description": "AWS STS endpoint URL", + "required": false + }, + "sqs_endpoint_url": { + "type": "text", + "ui_name": "SQS Endpoint URL", + "description": "Custom SQS endpoint URL (for LocalStack or custom endpoints)", + "required": false + }, + "sqs_api_version": { + "type": "text", + "ui_name": "API Version", + "description": "SQS API version", + "required": false + }, + "sqs_use_ssl": { + "type": "boolean", + "ui_name": "Use SSL", + "description": "Whether to use SSL for SQS connections", + "required": false + }, + "sqs_verify": { + "type": "boolean", + "ui_name": "Verify SSL", + "description": "Whether to verify SSL certificates", + "required": false + }, + "sqs_strip_base64_files": { + "type": "boolean", + "ui_name": "Strip Base64 Files", + "description": "Remove base64-encoded files from logs to reduce payload size", + "required": false + }, + "sqs_aws_use_application_level_encryption": { + "type": "boolean", + "ui_name": "Use Application-Level Encryption", + "description": "Enable application-level encryption for SQS messages", + "required": false + }, + "sqs_app_encryption_key_b64": { + "type": "password", + "ui_name": "Encryption Key (Base64)", + "description": "Base64-encoded encryption key for application-level encryption", + "required": false + }, + "sqs_app_encryption_aad": { + "type": "text", + "ui_name": "Encryption AAD", + "description": "Additional authenticated data for encryption", + "required": false + } + }, + "description": "SQS Queue (AWS) Logging Integration" + } +] diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index b50d05ed2ec..b52f1b3095e 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -36,6 +36,7 @@ class CustomGuardrail(CustomLogger): default_on: bool = False, mask_request_content: bool = False, mask_response_content: bool = False, + violation_message_template: Optional[str] = None, **kwargs, ): """ @@ -57,12 +58,34 @@ class CustomGuardrail(CustomLogger): self.default_on: bool = default_on self.mask_request_content: bool = mask_request_content self.mask_response_content: bool = mask_response_content + self.violation_message_template: Optional[str] = violation_message_template if supported_event_hooks: ## validate event_hook is in supported_event_hooks self._validate_event_hook(event_hook, supported_event_hooks) super().__init__(**kwargs) + def render_violation_message( + self, default: str, context: Optional[Dict[str, Any]] = None + ) -> str: + """Return a custom violation message if template is configured.""" + + if not self.violation_message_template: + return default + + format_context: Dict[str, Any] = {"default_message": default} + if context: + format_context.update(context) + try: + return self.violation_message_template.format(**format_context) + except Exception as e: + verbose_logger.warning( + "Failed to format violation message template for guardrail %s: %s", + self.guardrail_name, + e, + ) + return default + @staticmethod def get_config_model() -> Optional[Type["GuardrailConfigModel"]]: """ @@ -279,7 +302,7 @@ class CustomGuardrail(CustomLogger): data, self.event_hook ) if result is not None: - return result + return result return True def _event_hook_is_event_type(self, event_type: GuardrailEventHooks) -> bool: diff --git a/litellm/integrations/dotprompt/__init__.py b/litellm/integrations/dotprompt/__init__.py index 3af7fbf6dd3..3847c8fa192 100644 --- a/litellm/integrations/dotprompt/__init__.py +++ b/litellm/integrations/dotprompt/__init__.py @@ -25,6 +25,23 @@ def set_global_prompt_directory(directory: str) -> None: litellm.global_prompt_directory = directory # type: ignore +def _get_prompt_data_from_dotprompt_content(dotprompt_content: str) -> dict: + """ + Get the prompt data from the dotprompt content. + + The UI stores prompts under `dotprompt_content` in the database. This function parses the content and returns the prompt data in the format expected by the prompt manager. + """ + from .prompt_manager import PromptManager + + # Parse the dotprompt content to extract frontmatter and content + temp_manager = PromptManager() + metadata, content = temp_manager._parse_frontmatter(dotprompt_content) + + # Convert to prompt_data format + return { + "content": content.strip(), + "metadata": metadata + } def prompt_initializer( litellm_params: "PromptLiteLLMParams", prompt_spec: "PromptSpec" @@ -41,6 +58,11 @@ def prompt_initializer( ) prompt_file = getattr(litellm_params, "prompt_file", None) + + # Handle dotprompt_content from database + dotprompt_content = getattr(litellm_params, "dotprompt_content", None) + if dotprompt_content and not prompt_data and not prompt_file: + prompt_data = _get_prompt_data_from_dotprompt_content(dotprompt_content) try: dot_prompt_manager = DotpromptManager( diff --git a/litellm/integrations/dotprompt/dotprompt_manager.py b/litellm/integrations/dotprompt/dotprompt_manager.py index 0f0d7b938f3..7aaa6cc9628 100644 --- a/litellm/integrations/dotprompt/dotprompt_manager.py +++ b/litellm/integrations/dotprompt/dotprompt_manager.py @@ -108,7 +108,7 @@ class DotpromptManager(CustomPromptManagement): Compile a .prompt file into a PromptManagementClient structure. This method: - 1. Loads the prompt template from the .prompt file + 1. Loads the prompt template from the .prompt file (with optional version) 2. Renders it with the provided variables 3. Converts the rendered text into chat messages 4. Extracts model and optional parameters from metadata @@ -116,13 +116,22 @@ class DotpromptManager(CustomPromptManagement): try: - # Get the prompt template - template = self.prompt_manager.get_prompt(prompt_id) + # Get the prompt template (versioned or base) + template = self.prompt_manager.get_prompt( + prompt_id=prompt_id, version=prompt_version + ) if template is None: - raise ValueError(f"Prompt '{prompt_id}' not found in prompt directory") + version_str = f" (version {prompt_version})" if prompt_version else "" + raise ValueError( + f"Prompt '{prompt_id}'{version_str} not found in prompt directory" + ) - # Render the template with variables - rendered_content = self.prompt_manager.render(prompt_id, prompt_variables) + # Render the template with variables (pass version for proper lookup) + rendered_content = self.prompt_manager.render( + prompt_id=prompt_id, + prompt_variables=prompt_variables, + version=prompt_version, + ) # Convert rendered content to chat messages messages = self._convert_to_messages(rendered_content) diff --git a/litellm/integrations/dotprompt/prompt_manager.py b/litellm/integrations/dotprompt/prompt_manager.py index 9623ddab5fb..fc5a325ffe1 100644 --- a/litellm/integrations/dotprompt/prompt_manager.py +++ b/litellm/integrations/dotprompt/prompt_manager.py @@ -183,7 +183,10 @@ class PromptManager: return frontmatter, template_content def render( - self, prompt_id: str, prompt_variables: Optional[Dict[str, Any]] = None + self, + prompt_id: str, + prompt_variables: Optional[Dict[str, Any]] = None, + version: Optional[int] = None, ) -> str: """ Render a prompt template with the given variables. @@ -191,6 +194,7 @@ class PromptManager: Args: prompt_id: The ID of the prompt template to render prompt_variables: Variables to substitute in the template + version: Optional version number. If provided, looks for {prompt_id}.v{version} Returns: The rendered prompt string @@ -199,13 +203,16 @@ class PromptManager: KeyError: If prompt_id is not found ValueError: If template rendering fails """ - if prompt_id not in self.prompts: + # Get the template (versioned or base) + template = self.get_prompt(prompt_id=prompt_id, version=version) + + if template is None: available_prompts = list(self.prompts.keys()) + version_str = f" (version {version})" if version else "" raise KeyError( - f"Prompt '{prompt_id}' not found. Available prompts: {available_prompts}" + f"Prompt '{prompt_id}'{version_str} not found. Available prompts: {available_prompts}" ) - template = self.prompts[prompt_id] variables = prompt_variables or {} # Validate input variables against schema if defined @@ -254,8 +261,26 @@ class PromptManager: return type_mapping.get(schema_type.lower(), str) # type: ignore - def get_prompt(self, prompt_id: str) -> Optional[PromptTemplate]: - """Get a prompt template by ID.""" + def get_prompt( + self, prompt_id: str, version: Optional[int] = None + ) -> Optional[PromptTemplate]: + """ + Get a prompt template by ID and optional version. + + Args: + prompt_id: The base prompt ID + version: Optional version number. If provided, looks for {prompt_id}.v{version} + + Returns: + The prompt template if found, None otherwise + """ + if version is not None: + # Try versioned prompt first: prompt_id.v{version} + versioned_id = f"{prompt_id}.v{version}" + if versioned_id in self.prompts: + return self.prompts[versioned_id] + + # Fall back to base prompt_id return self.prompts.get(prompt_id) def list_prompts(self) -> List[str]: diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index c2a2cc77950..12eb00efa9e 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -228,6 +228,8 @@ class LangFuseLogger: functions = optional_params.pop("functions", None) tools = optional_params.pop("tools", None) + # Remove secret_fields to prevent leaking sensitive data (e.g., authorization headers) + optional_params.pop("secret_fields", None) if functions is not None: prompt["functions"] = functions if tools is not None: diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 53b7825b3d3..468fbc91408 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -1773,6 +1773,10 @@ class OpenTelemetry(CustomLogger): """ Create a span for the received proxy server request. """ + # don't create proxy parent spans for arize phoenix + if self.callback_name == "arize_phoenix": + return None + return self.tracer.start_span( name="Received Proxy Server Request", start_time=self._to_ns(start_time), diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index fb25c5ed840..eefe680217d 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -693,12 +693,12 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) # type: ignore dynamic_api_key = api_key or get_secret_str("NOVITA_API_KEY") elif custom_llm_provider == "snowflake": - api_base = ( - api_base - or get_secret_str("SNOWFLAKE_API_BASE") - or f"https://{get_secret('SNOWFLAKE_ACCOUNT_ID')}.snowflakecomputing.com/api/v2/cortex/inference:complete" - ) # type: ignore - dynamic_api_key = api_key or get_secret_str("SNOWFLAKE_JWT") + ( + api_base, + dynamic_api_key, + ) = litellm.SnowflakeConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) elif custom_llm_provider == "gradient_ai": ( api_base, @@ -741,6 +741,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.MoonshotChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "docker_model_runner": + ( + api_base, + dynamic_api_key, + ) = litellm.DockerModelRunnerChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) elif custom_llm_provider == "v0": ( api_base, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 6bad7ee29e2..6a36eb6dc3b 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -585,7 +585,10 @@ class Logging(LiteLLMLoggingBaseClass): custom_logger = ( prompt_management_logger or self.get_custom_logger_for_prompt_management( - model=model, non_default_params=non_default_params + model=model, + non_default_params=non_default_params, + prompt_id=prompt_id, + dynamic_callback_params=self.standard_callback_dynamic_params, ) ) @@ -622,7 +625,11 @@ class Logging(LiteLLMLoggingBaseClass): custom_logger = ( prompt_management_logger or self.get_custom_logger_for_prompt_management( - model=model, tools=tools, non_default_params=non_default_params + model=model, + tools=tools, + non_default_params=non_default_params, + prompt_id=prompt_id, + dynamic_callback_params=self.standard_callback_dynamic_params, ) ) @@ -646,19 +653,69 @@ class Logging(LiteLLMLoggingBaseClass): self.messages = messages return model, messages, non_default_params + def _auto_detect_prompt_management_logger( + self, + prompt_id: str, + dynamic_callback_params: StandardCallbackDynamicParams, + ) -> Optional[CustomLogger]: + """ + Auto-detect which prompt management system owns the given prompt_id. + + This allows a user to just pass prompt_id in the completion call and it will be auto-detected which system owns this prompt. + + Args: + prompt_id: The prompt ID to check + dynamic_callback_params: Dynamic callback parameters for should_run_prompt_management checks + + Returns: + A CustomLogger instance if a matching prompt management system is found, None otherwise + """ + prompt_management_loggers = ( + litellm.logging_callback_manager.get_custom_loggers_for_type( + callback_type=CustomPromptManagement + ) + ) + + for logger in prompt_management_loggers: + if isinstance(logger, CustomPromptManagement): + try: + if logger.should_run_prompt_management( + prompt_id=prompt_id, + dynamic_callback_params=dynamic_callback_params, + ): + self.model_call_details["prompt_integration"] = ( + logger.__class__.__name__ + ) + return logger + except Exception: + # If check fails, continue to next logger + continue + + return None + def get_custom_logger_for_prompt_management( - self, model: str, non_default_params: Dict, tools: Optional[List[Dict]] = None + self, + model: str, + non_default_params: Dict, + tools: Optional[List[Dict]] = None, + prompt_id: Optional[str] = None, + dynamic_callback_params: Optional[StandardCallbackDynamicParams] = None, ) -> Optional[CustomLogger]: """ Get a custom logger for prompt management based on model name or available callbacks. Args: model: The model name to check for prompt management integration + non_default_params: Non-default parameters passed to the completion call + tools: Optional tools passed to the completion call + prompt_id: Optional prompt ID to auto-detect which system owns this prompt + dynamic_callback_params: Dynamic callback parameters for should_run_prompt_management checks Returns: A CustomLogger instance if one is found, None otherwise """ # First check if model starts with a known custom logger compatible callback + # This takes precedence for backward compatibility for callback_name in litellm._known_custom_logger_compatible_callbacks: if model.startswith(callback_name): custom_logger = _init_custom_logger_compatible_class( @@ -670,7 +727,16 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["prompt_integration"] = model.split("/")[0] return custom_logger - # Then check for any registered CustomPromptManagement loggers + # If prompt_id is provided, try to auto-detect which system has this prompt + if prompt_id and dynamic_callback_params is not None: + auto_detected_logger = self._auto_detect_prompt_management_logger( + prompt_id=prompt_id, + dynamic_callback_params=dynamic_callback_params, + ) + if auto_detected_logger is not None: + return auto_detected_logger + + # Then check for any registered CustomPromptManagement loggers (fallback) prompt_management_loggers = ( litellm.logging_callback_manager.get_custom_loggers_for_type( callback_type=CustomPromptManagement @@ -3488,8 +3554,19 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 otel_config = OpenTelemetryConfig( exporter=arize_phoenix_config.protocol, endpoint=arize_phoenix_config.endpoint, + headers=arize_phoenix_config.otlp_auth_headers, ) + # Set Phoenix project name from environment variable + phoenix_project_name = os.environ.get("PHOENIX_PROJECT_NAME", None) + if phoenix_project_name: + existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "") + # Add openinference.project.name attribute + if existing_attrs: + os.environ["OTEL_RESOURCE_ATTRIBUTES"] = f"{existing_attrs},openinference.project.name={phoenix_project_name}" + else: + os.environ["OTEL_RESOURCE_ATTRIBUTES"] = f"openinference.project.name={phoenix_project_name}" + # auth can be disabled on local deployments of arize phoenix if arize_phoenix_config.otlp_auth_headers is not None: os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = ( diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index eff5376e49e..9717f442b82 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -408,6 +408,7 @@ class CompletionTokensDetailsResult(TypedDict): audio_tokens: int text_tokens: int reasoning_tokens: int + image_tokens: int def _parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResult: @@ -432,11 +433,19 @@ def _parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsRes ) or 0 ) + image_tokens = ( + cast( + Optional[int], + getattr(usage.completion_tokens_details, "image_tokens", 0), + ) + or 0 + ) return CompletionTokensDetailsResult( audio_tokens=audio_tokens, text_tokens=text_tokens, reasoning_tokens=reasoning_tokens, + image_tokens=image_tokens, ) @@ -565,12 +574,14 @@ def generic_cost_per_token( text_tokens = 0 audio_tokens = 0 reasoning_tokens = 0 + image_tokens = 0 is_text_tokens_total = False if usage.completion_tokens_details is not None: completion_tokens_details = _parse_completion_tokens_details(usage) audio_tokens = completion_tokens_details["audio_tokens"] text_tokens = completion_tokens_details["text_tokens"] reasoning_tokens = completion_tokens_details["reasoning_tokens"] + image_tokens = completion_tokens_details["image_tokens"] if text_tokens == 0: text_tokens = usage.completion_tokens @@ -585,6 +596,9 @@ def generic_cost_per_token( _output_cost_per_reasoning_token = _get_cost_per_unit( model_info, "output_cost_per_reasoning_token", None ) + _output_cost_per_image_token = _get_cost_per_unit( + model_info, "output_cost_per_image_token", None + ) ## AUDIO COST if not is_text_tokens_total and audio_tokens is not None and audio_tokens > 0: @@ -604,6 +618,15 @@ def generic_cost_per_token( ) completion_cost += float(reasoning_tokens) * _output_cost_per_reasoning_token + ## IMAGE COST + if not is_text_tokens_total and image_tokens and image_tokens > 0: + _output_cost_per_image_token = ( + _output_cost_per_image_token + if _output_cost_per_image_token is not None + else completion_base_cost + ) + completion_cost += float(image_tokens) * _output_cost_per_image_token + return prompt_cost, completion_cost diff --git a/litellm/litellm_core_utils/llm_response_utils/get_formatted_prompt.py b/litellm/litellm_core_utils/llm_response_utils/get_formatted_prompt.py index fffaad79b9e..f7406398a46 100644 --- a/litellm/litellm_core_utils/llm_response_utils/get_formatted_prompt.py +++ b/litellm/litellm_core_utils/llm_response_utils/get_formatted_prompt.py @@ -4,6 +4,7 @@ from typing import List, Literal def get_formatted_prompt( data: dict, call_type: Literal[ + "acompletion", "completion", "embedding", "image_generation", @@ -18,7 +19,7 @@ def get_formatted_prompt( Returns a string. """ prompt = "" - if call_type == "completion": + if call_type == "acompletion" or call_type == "completion": for message in data["messages"]: if message.get("content", None) is not None: content = message.get("content") diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 69e3cc43322..c50ceeabdb2 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -437,6 +437,66 @@ def update_messages_with_model_file_ids( return messages +def update_responses_input_with_model_file_ids( + input: Any, +) -> Union[str, List[Dict[str, Any]]]: + """ + Updates responses API input with provider-specific file IDs. + File IDs are always inside the content array, not as direct input_file items. + + For managed files (unified file IDs), decodes the base64-encoded unified file ID + and extracts the llm_output_file_id directly. + """ + from litellm.proxy.openai_files_endpoints.common_utils import ( + _is_base64_encoded_unified_file_id, + convert_b64_uid_to_unified_uid, + ) + + if isinstance(input, str): + return input + + if not isinstance(input, list): + return input + + updated_input = [] + for item in input: + if not isinstance(item, dict): + updated_input.append(item) + continue + + updated_item = item.copy() + content = item.get("content") + if isinstance(content, list): + updated_content = [] + for content_item in content: + if isinstance(content_item, dict) and content_item.get("type") == "input_file": + file_id = content_item.get("file_id") + if file_id: + # Check if this is a managed file ID (base64-encoded unified file ID) + is_unified_file_id = _is_base64_encoded_unified_file_id(file_id) + if is_unified_file_id: + unified_file_id = convert_b64_uid_to_unified_uid(file_id) + if "llm_output_file_id," in unified_file_id: + provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0] + else: + # Fallback: keep original if we can't extract + provider_file_id = file_id + updated_content_item = content_item.copy() + updated_content_item["file_id"] = provider_file_id + updated_content.append(updated_content_item) + else: + updated_content.append(content_item) + else: + updated_content.append(content_item) + else: + updated_content.append(content_item) + updated_item["content"] = updated_content + + updated_input.append(updated_item) + + return updated_input + + def extract_file_data(file_data: FileTypes) -> ExtractedFileData: """ Extracts and processes file data from various input formats. diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 717c2607657..262692d6d1a 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1,3 +1,4 @@ +import base64 import copy import hashlib import json @@ -5,7 +6,7 @@ import mimetypes import re import xml.etree.ElementTree as ET from enum import Enum -from typing import Any, List, Optional, Tuple, cast, overload +from typing import Any, List, Optional, Tuple, Union, cast, overload from jinja2.sandbox import ImmutableSandboxedEnvironment @@ -57,6 +58,10 @@ def prompt_injection_detection_default_pt(): BAD_MESSAGE_ERROR_STR = "Invalid Message " +# Separator used to embed Gemini thought signatures in tool call IDs +# See: https://ai.google.dev/gemini-api/docs/thought-signatures +THOUGHT_SIGNATURE_SEPARATOR = "__thought__" + # used to interweave user messages, to ensure user/assistant alternating DEFAULT_USER_CONTINUE_MESSAGE = { "role": "user", @@ -905,6 +910,64 @@ def convert_to_anthropic_image_obj( ) +def create_anthropic_image_param( + image_url_input: Union[str, dict], + format: Optional[str] = None, + is_bedrock_invoke: bool = False +) -> AnthropicMessagesImageParam: + """ + Create an AnthropicMessagesImageParam from an image URL input. + + Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding. + """ + # Extract URL and format from input + if isinstance(image_url_input, str): + image_url = image_url_input + else: + image_url = image_url_input.get("url", "") + if format is None: + format = image_url_input.get("format") + + # Check if the image URL is an HTTP/HTTPS URL + if image_url.startswith("http://") or image_url.startswith("https://"): + # For Bedrock invoke, always convert URLs to base64 (Bedrock invoke doesn't support URLs) + if is_bedrock_invoke or image_url.startswith("http://"): + base64_url = convert_url_to_base64(url=image_url) + image_chunk = convert_to_anthropic_image_obj( + openai_image_url=base64_url, format=format + ) + return AnthropicMessagesImageParam( + type="image", + source=AnthropicContentParamSource( + type="base64", + media_type=image_chunk["media_type"], + data=image_chunk["data"], + ), + ) + else: + # HTTPS URL - pass directly for regular Anthropic + return AnthropicMessagesImageParam( + type="image", + source=AnthropicContentParamSourceUrl( + type="url", + url=image_url, + ), + ) + else: + # Convert to base64 for data URIs or other formats + image_chunk = convert_to_anthropic_image_obj( + openai_image_url=image_url, format=format + ) + return AnthropicMessagesImageParam( + type="image", + source=AnthropicContentParamSource( + type="base64", + media_type=image_chunk["media_type"], + data=image_chunk["data"], + ), + ) + + # The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex # ------------------------------------------------------------------------------ def convert_to_anthropic_tool_result_xml(message: dict) -> str: @@ -1007,15 +1070,35 @@ def anthropic_messages_pt_xml(messages: list): if isinstance(messages[msg_i]["content"], list): for m in messages[msg_i]["content"]: if m.get("type", "") == "image_url": - format = m["image_url"].get("format") - user_content.append( - { - "type": "image", - "source": convert_to_anthropic_image_obj( - m["image_url"]["url"], format=format - ), - } - ) + format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None + image_param = create_anthropic_image_param(m["image_url"], format=format) + # Convert to dict format for XML version + source = image_param["source"] + if isinstance(source, dict) and source.get("type") == "url": + # Type narrowing for URL source + url_source = cast(AnthropicContentParamSourceUrl, source) + user_content.append( + { + "type": "image", + "source": { + "type": "url", + "url": url_source["url"], + }, + } + ) + else: + # Type narrowing for base64 source + base64_source = cast(AnthropicContentParamSource, source) + user_content.append( + { + "type": "image", + "source": { + "type": "base64", + "media_type": base64_source["media_type"], + "data": base64_source["data"], + }, + } + ) elif m.get("type", "") == "text": user_content.append({"type": "text", "text": m["text"]}) else: @@ -1161,8 +1244,94 @@ def _gemini_tool_call_invoke_helper( return function_call +def _encode_tool_call_id_with_signature( + tool_call_id: str, thought_signature: Optional[str] +) -> str: + """ + Embed thought signature into tool call ID for OpenAI client compatibility. + + Args: + tool_call_id: The tool call ID (e.g., "call_abc123...") + thought_signature: Base64-encoded signature from Gemini response + + Returns: + Tool call ID with embedded signature if present, otherwise original ID + Format: call___thought__ + + See: https://ai.google.dev/gemini-api/docs/thought-signatures + """ + if thought_signature: + return f"{tool_call_id}{THOUGHT_SIGNATURE_SEPARATOR}{thought_signature}" + return tool_call_id + + +def _get_thought_signature_from_tool( + tool: dict, model: Optional[str] = None +) -> Optional[str]: + """Extract thought signature from tool call's provider_specific_fields. + + If not provided try to extract thought signature from tool call id + + Checks both tool.provider_specific_fields and tool.function.provider_specific_fields. + If no signature is found and model is gemini-3, returns a dummy signature. + """ + # First check tool's provider_specific_fields + provider_fields = tool.get("provider_specific_fields") or {} + if isinstance(provider_fields, dict): + signature = provider_fields.get("thought_signature") + if signature: + return signature + + # Then check function's provider_specific_fields + function = tool.get("function") + if function: + if isinstance(function, dict): + func_provider_fields = function.get("provider_specific_fields") or {} + if isinstance(func_provider_fields, dict): + signature = func_provider_fields.get("thought_signature") + if signature: + return signature + elif ( + hasattr(function, "provider_specific_fields") + and function.provider_specific_fields + ): + if isinstance(function.provider_specific_fields, dict): + signature = function.provider_specific_fields.get("thought_signature") + if signature: + return signature + # Check if thought signature is embedded in tool call ID + tool_call_id = tool.get("id") + if tool_call_id and THOUGHT_SIGNATURE_SEPARATOR in tool_call_id: + parts = tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1) + if len(parts) == 2: + _, signature = parts + return signature + # If no signature found and model is gemini-3, return dummy signature + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + if model and VertexGeminiConfig._is_gemini_3_or_newer(model): + return _get_dummy_thought_signature() + return None + + +def _get_dummy_thought_signature() -> str: + """Generate a dummy thought signature for models that require it. + + This is used when transferring conversation history from older models + (like gemini-2.5-flash) to gemini-3, which requires thought_signature + for strict validation. + """ + # Return a base64-encoded dummy signature string + # Below dummy signature is recommended by google - https://ai.google.dev/gemini-api/docs/thought-signatures#faqs + dummy_data = b"skip_thought_signature_validator" + return base64.b64encode(dummy_data).decode("utf-8") + + def convert_to_gemini_tool_call_invoke( message: ChatCompletionAssistantMessage, + model: Optional[str] = None, ) -> List[VertexPartType]: """ OpenAI tool invokes: @@ -1207,18 +1376,26 @@ def convert_to_gemini_tool_call_invoke( _parts_list: List[VertexPartType] = [] tool_calls = message.get("tool_calls", None) function_call = message.get("function_call", None) + if tool_calls is not None: - for tool in tool_calls: + for idx, tool in enumerate(tool_calls): if "function" in tool: - gemini_function_call: Optional[VertexFunctionCall] = ( - _gemini_tool_call_invoke_helper( - function_call_params=tool["function"] - ) + gemini_function_call: Optional[ + VertexFunctionCall + ] = _gemini_tool_call_invoke_helper( + function_call_params=tool["function"] ) if gemini_function_call is not None: - _parts_list.append( - VertexPartType(function_call=gemini_function_call) + part_dict: VertexPartType = { + "function_call": gemini_function_call + } + thought_signature = _get_thought_signature_from_tool( + dict(tool), model=model ) + if thought_signature: + part_dict["thoughtSignature"] = thought_signature + + _parts_list.append(part_dict) else: # don't silently drop params. Make it clear to user what's happening. raise Exception( "function_call missing. Received tool call with 'type': 'function'. No function call in argument - {}".format( @@ -1230,7 +1407,36 @@ def convert_to_gemini_tool_call_invoke( function_call_params=function_call ) if gemini_function_call is not None: - _parts_list.append(VertexPartType(function_call=gemini_function_call)) + part_dict_function: VertexPartType = { + "function_call": gemini_function_call + } + + # Extract thought signature from function_call's provider_specific_fields + thought_signature = None + provider_fields = ( + function_call.get("provider_specific_fields") + if isinstance(function_call, dict) + else {} + ) + if isinstance(provider_fields, dict): + thought_signature = provider_fields.get("thought_signature") + + # If no signature found and model is gemini-3, use dummy signature + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + if ( + not thought_signature + and model + and VertexGeminiConfig._is_gemini_3_or_newer(model) + ): + thought_signature = _get_dummy_thought_signature() + + if thought_signature: + part_dict_function["thoughtSignature"] = thought_signature + + _parts_list.append(part_dict_function) else: # don't silently drop params. Make it clear to user what's happening. raise Exception( "function_call missing. Received tool call with 'type': 'function'. No function call in argument - {}".format( @@ -1363,24 +1569,9 @@ def convert_to_anthropic_tool_result( ) ) elif content["type"] == "image_url": - if isinstance(content["image_url"], str): - image_chunk = convert_to_anthropic_image_obj( - content["image_url"], format=None - ) - else: - format = content["image_url"].get("format") - image_chunk = convert_to_anthropic_image_obj( - content["image_url"]["url"], format=format - ) + format = content["image_url"].get("format") if isinstance(content["image_url"], dict) else None anthropic_content_list.append( - AnthropicMessagesImageParam( - type="image", - source=AnthropicContentParamSource( - type="base64", - media_type=image_chunk["media_type"], - data=image_chunk["data"], - ), - ) + create_anthropic_image_param(content["image_url"], format=format) ) anthropic_content = anthropic_content_list @@ -1711,30 +1902,31 @@ def anthropic_messages_pt( # noqa: PLR0915 for m in user_message_types_block["content"]: if m.get("type", "") == "image_url": m = cast(ChatCompletionImageObject, m) - format: Optional[str] = None - if isinstance(m["image_url"], str): - image_chunk = convert_to_anthropic_image_obj( - openai_image_url=m["image_url"], format=None - ) + format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None + # Convert ChatCompletionImageUrlObject to dict if needed + image_url_value = m["image_url"] + if isinstance(image_url_value, str): + image_url_input: Union[str, dict[str, Any]] = image_url_value else: - format = m["image_url"].get("format") - image_chunk = convert_to_anthropic_image_obj( - openai_image_url=m["image_url"]["url"], - format=format, - ) - - _anthropic_content_element = ( - _anthropic_content_element_factory(image_chunk) - ) + # ChatCompletionImageUrlObject or dict case - convert to dict + image_url_input = { + "url": image_url_value["url"], + "format": image_url_value.get("format"), + } + # Bedrock invoke models have format: invoke/... + is_bedrock_invoke = model.lower().startswith("invoke/") + _anthropic_content_element = create_anthropic_image_param( + image_url_input, format=format, is_bedrock_invoke=is_bedrock_invoke + ) _content_element = add_cache_control_to_content( anthropic_content_element=_anthropic_content_element, original_content_element=dict(m), ) if "cache_control" in _content_element: - _anthropic_content_element["cache_control"] = ( - _content_element["cache_control"] - ) + _anthropic_content_element[ + "cache_control" + ] = _content_element["cache_control"] user_content.append(_anthropic_content_element) elif m.get("type", "") == "text": m = cast(ChatCompletionTextObject, m) @@ -1772,9 +1964,9 @@ def anthropic_messages_pt( # noqa: PLR0915 ) if "cache_control" in _content_element: - _anthropic_content_text_element["cache_control"] = ( - _content_element["cache_control"] - ) + _anthropic_content_text_element[ + "cache_control" + ] = _content_element["cache_control"] user_content.append(_anthropic_content_text_element) @@ -2496,7 +2688,6 @@ def stringify_json_tool_call_content(messages: List) -> List: ###### AMAZON BEDROCK ####### -import base64 from email.message import Message import httpx @@ -2541,17 +2732,19 @@ class BedrockImageProcessor: """Handles both sync and async image processing for Bedrock conversations.""" @staticmethod - def _post_call_image_processing(response: httpx.Response, image_url: str = "") -> Tuple[str, str]: + def _post_call_image_processing( + response: httpx.Response, image_url: str = "" + ) -> Tuple[str, str]: # Check the response's content type to ensure it is an image content_type = response.headers.get("content-type") - + # Use helper function to infer content type with fallback logic content_type = infer_content_type_from_url_and_content( url=image_url, content=response.content, current_content_type=content_type, ) - + content_type = _parse_content_type(content_type) # Convert the image content to base64 bytes @@ -2570,7 +2763,9 @@ class BedrockImageProcessor: response = await client.get(image_url, follow_redirects=True) response.raise_for_status() # Raise an exception for HTTP errors - return BedrockImageProcessor._post_call_image_processing(response, image_url) + return BedrockImageProcessor._post_call_image_processing( + response, image_url + ) except Exception as e: raise e @@ -2583,7 +2778,9 @@ class BedrockImageProcessor: response = client.get(image_url, follow_redirects=True) response.raise_for_status() # Raise an exception for HTTP errors - return BedrockImageProcessor._post_call_image_processing(response, image_url) + return BedrockImageProcessor._post_call_image_processing( + response, image_url + ) except Exception as e: raise e @@ -2914,21 +3111,33 @@ def _convert_to_bedrock_tool_call_result( """ - """ - content_str: str = "" + tool_result_content_blocks:List[BedrockToolResultContentBlock] = [] if isinstance(message["content"], str): - content_str = message["content"] + tool_result_content_blocks.append(BedrockToolResultContentBlock(text=message["content"])) elif isinstance(message["content"], List): content_list = message["content"] for content in content_list: if content["type"] == "text": - content_str += content["text"] + tool_result_content_blocks.append(BedrockToolResultContentBlock(text=content["text"])) + elif content["type"] == "image_url": + format: Optional[str] = None + if isinstance(content["image_url"], dict): + image_url = content["image_url"]["url"] + format = content["image_url"].get("format") + else: + image_url = content["image_url"] + _block:BedrockContentBlock = BedrockImageProcessor.process_image_sync( + image_url=image_url, + format=format, + ) + if "image" in _block: + tool_result_content_blocks.append(BedrockToolResultContentBlock(image=_block["image"])) message.get("name", "") id = str(message.get("tool_call_id", str(uuid.uuid4()))) - tool_result_content_block = BedrockToolResultContentBlock(text=content_str) tool_result = BedrockToolResultBlock( - content=[tool_result_content_block], + content=tool_result_content_blocks, toolUseId=id, ) @@ -3840,7 +4049,9 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 ) elif element["type"] == "text": # AWS Bedrock doesn't allow empty or whitespace-only text content, so use placeholder for empty strings - text_content = element["text"] if element["text"].strip() else "." + text_content = ( + element["text"] if element["text"].strip() else "." + ) assistants_part = BedrockContentBlock(text=text_content) assistants_parts.append(assistants_part) elif element["type"] == "image_url": diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index ea0bed30416..206810943ca 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -42,7 +42,11 @@ class SensitiveDataMasker: else: return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}{value_str[-self.visible_suffix:]}" - def is_sensitive_key(self, key: str) -> bool: + def is_sensitive_key(self, key: str, excluded_keys: Optional[Set[str]] = None) -> bool: + # Check if key is in excluded_keys first (exact match) + if excluded_keys and key in excluded_keys: + return False + key_lower = str(key).lower() # Split on underscores and check if any segment matches the pattern # This avoids false positives like "max_tokens" matching "token" @@ -59,6 +63,7 @@ class SensitiveDataMasker: data: Dict[str, Any], depth: int = 0, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER, + excluded_keys: Optional[Set[str]] = None, ) -> Dict[str, Any]: if depth >= max_depth: return data @@ -67,10 +72,10 @@ class SensitiveDataMasker: for k, v in data.items(): try: if isinstance(v, dict): - masked_data[k] = self.mask_dict(v, depth + 1) + masked_data[k] = self.mask_dict(v, depth + 1, max_depth, excluded_keys) elif hasattr(v, "__dict__") and not isinstance(v, type): - masked_data[k] = self.mask_dict(vars(v), depth + 1) - elif self.is_sensitive_key(k): + masked_data[k] = self.mask_dict(vars(v), depth + 1, max_depth, excluded_keys) + elif self.is_sensitive_key(k, excluded_keys): str_value = str(v) if v is not None else "" masked_data[k] = self._mask_value(str_value) else: diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 2f85c7aef60..ddcf81b5ba5 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -137,6 +137,7 @@ class ChunkProcessor: "name": None, "type": None, "arguments": [], + "provider_specific_fields": None, } if hasattr(tool_call, "id") and tool_call.id: @@ -156,22 +157,48 @@ class ChunkProcessor: tool_call_map[index]["arguments"].append( tool_call.function.arguments ) + + # Preserve provider_specific_fields from streaming chunks + provider_fields = None + if hasattr(tool_call, "provider_specific_fields") and tool_call.provider_specific_fields: + provider_fields = tool_call.provider_specific_fields + elif hasattr(tool_call, "function") and hasattr(tool_call.function, "provider_specific_fields") and tool_call.function.provider_specific_fields: + provider_fields = tool_call.function.provider_specific_fields + + if provider_fields: + # Merge provider_specific_fields if multiple chunks have them + if tool_call_map[index]["provider_specific_fields"] is None: + tool_call_map[index]["provider_specific_fields"] = {} + if isinstance(provider_fields, dict): + tool_call_map[index]["provider_specific_fields"].update( + provider_fields + ) # Convert the map to a list of tool calls for index in sorted(tool_call_map.keys()): tool_call_data = tool_call_map[index] if tool_call_data["id"] and tool_call_data["name"]: combined_arguments = "".join(tool_call_data["arguments"]) or "{}" - tool_calls_list.append( - ChatCompletionMessageToolCall( - id=tool_call_data["id"], - function=Function( - arguments=combined_arguments, - name=tool_call_data["name"], - ), - type=tool_call_data["type"] or "function", - ) + + # Build function - provider_specific_fields should be on tool_call level, not function level + function = Function( + arguments=combined_arguments, + name=tool_call_data["name"], ) + + # Prepare params for ChatCompletionMessageToolCall + tool_call_params = { + "id": tool_call_data["id"], + "function": function, + "type": tool_call_data["type"] or "function", + } + + # Add provider_specific_fields if present (for thought signatures in Gemini 3) + if tool_call_data.get("provider_specific_fields"): + tool_call_params["provider_specific_fields"] = tool_call_data["provider_specific_fields"] + + tool_call = ChatCompletionMessageToolCall(**tool_call_params) + tool_calls_list.append(tool_call) return tool_calls_list diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index a786f06921f..0e905014fe2 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -3,6 +3,7 @@ from typing import ( TYPE_CHECKING, Any, AsyncIterator, + Dict, List, Literal, Optional, @@ -129,6 +130,39 @@ class LiteLLMAnthropicMessagesAdapter: ### FOR [BETA] `/v1/messages` endpoint support + def _extract_signature_from_tool_call( + self, tool_call: Any + ) -> Optional[str]: + """ + Extract signature from a tool call's provider_specific_fields. + Only checks provider_specific_fields, not thinking blocks. + """ + signature = None + + if hasattr(tool_call, "provider_specific_fields") and tool_call.provider_specific_fields: + if "thought_signature" in tool_call.provider_specific_fields: + signature = tool_call.provider_specific_fields["thought_signature"] + elif ( + hasattr(tool_call.function, "provider_specific_fields") + and tool_call.function.provider_specific_fields + ): + if "thought_signature" in tool_call.function.provider_specific_fields: + signature = tool_call.function.provider_specific_fields["thought_signature"] + + return signature + + def _extract_signature_from_tool_use_content( + self, content: Dict[str, Any] + ) -> Optional[str]: + """ + Extract signature from a tool_use content block's provider_specific_fields. + """ + provider_specific_fields = content.get("provider_specific_fields", {}) + if provider_specific_fields: + return provider_specific_fields.get("signature") + return None + + def translatable_anthropic_params(self) -> List: """ Which anthropic params, we need to translate to the openai format. @@ -263,10 +297,18 @@ class LiteLLMAnthropicMessagesAdapter: else: assistant_message_str += content.get("text", "") elif content.get("type") == "tool_use": - function_chunk = ChatCompletionToolCallFunctionChunk( - name=content.get("name", ""), - arguments=json.dumps(content.get("input", {})), - ) + function_chunk: ChatCompletionToolCallFunctionChunk = { + "name": content.get("name", ""), + "arguments": json.dumps(content.get("input", {})), + } + signature = self._extract_signature_from_tool_use_content(content) + + if signature: + provider_specific_fields: Dict[str, Any] = ( + function_chunk.get("provider_specific_fields") or {} + ) + provider_specific_fields["thought_signature"] = signature + function_chunk["provider_specific_fields"] = provider_specific_fields tool_calls.append( ChatCompletionAssistantToolCall( @@ -512,18 +554,27 @@ class LiteLLMAnthropicMessagesAdapter: and len(choice.message.tool_calls) > 0 ): for tool_call in choice.message.tool_calls: - new_content.append( - AnthropicResponseContentBlockToolUse( - type="tool_use", - id=tool_call.id, - name=tool_call.function.name or "", - input=( - json.loads(tool_call.function.arguments) - if tool_call.function.arguments - else {} - ), - ) + # Extract signature from provider_specific_fields only + signature = self._extract_signature_from_tool_call(tool_call) + + provider_specific_fields = {} + if signature: + provider_specific_fields["signature"] = signature + + tool_use_block = AnthropicResponseContentBlockToolUse( + type="tool_use", + id=tool_call.id, + name=tool_call.function.name or "", + input=( + json.loads(tool_call.function.arguments) + if tool_call.function.arguments + else {} + ), ) + # Add provider_specific_fields if signature is present + if provider_specific_fields: + tool_use_block.provider_specific_fields = provider_specific_fields + new_content.append(tool_use_block) # Handle text content elif choice.message.content is not None: new_content.append( diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index 23c04e640c4..8e5581206de 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -10,6 +10,7 @@ from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from ....litellm_core_utils.realtime_streaming import RealTimeStreaming +from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from ..azure import AzureChatCompletion # BACKEND_WS_URL = "ws://localhost:8080/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01" @@ -61,12 +62,14 @@ class AzureOpenAIRealtime(AzureChatCompletion): url = self._construct_url(api_base, model, api_version) try: + ssl_context = get_shared_realtime_ssl_context() async with websockets.connect( # type: ignore url, extra_headers={ "api-key": api_key, # type: ignore }, max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, + ssl=ssl_context, ) as backend_ws: realtime_streaming = RealTimeStreaming( websocket, cast(ClientConnection, backend_ws), logging_obj diff --git a/litellm/llms/azure/text_to_speech/transformation.py b/litellm/llms/azure/text_to_speech/transformation.py index 0f8911ac2b8..df582c3c09b 100644 --- a/litellm/llms/azure/text_to_speech/transformation.py +++ b/litellm/llms/azure/text_to_speech/transformation.py @@ -382,6 +382,15 @@ class AzureAVATextToSpeechConfig(BaseTextToSpeechConfig): return f"https://{region}.{self.TTS_SPEECH_DOMAIN}{self.TTS_ENDPOINT_PATH}" return f"https://{self.TTS_SPEECH_DOMAIN}{self.TTS_ENDPOINT_PATH}" + + def is_ssml_input(self, input: str) -> bool: + """ + Returns True if input is SSML, False otherwise + + Based on https://www.w3.org/TR/speech-synthesis/ all SSML must start with + """ + return "" in input or ", it's passed through as-is without transformation + Returns: TextToSpeechRequestData: Contains SSML body and Azure-specific headers """ @@ -414,7 +426,15 @@ class AzureAVATextToSpeechConfig(BaseTextToSpeechConfig): ) headers["X-Microsoft-OutputFormat"] = output_format - # Build SSML + # Auto-detect SSML: if input contains , pass it through as-is + # Similar to Vertex AI behavior - check if input looks like SSML + if self.is_ssml_input(input=input): + return TextToSpeechRequestData( + ssml_body=input, + headers=headers, + ) + + # Build SSML from plain text rate = optional_params.get("rate", "0%") style = optional_params.get("style") styledegree = optional_params.get("styledegree") diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 53cbafcbe6a..b35e86cabd2 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -51,7 +51,11 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallFunctionChunk, ChatCompletionUsageBlock, ) -from litellm.types.utils import ChatCompletionMessageToolCall, Choices, Delta +from litellm.types.utils import ( + ChatCompletionMessageToolCall, + Choices, + Delta, +) from litellm.types.utils import GenericStreamingChunk as GChunk from litellm.types.utils import ( ModelResponse, @@ -493,9 +497,9 @@ class BedrockLLM(BaseAWSLLM): content=None, ) model_response.choices[0].message = _message # type: ignore - model_response._hidden_params["original_response"] = ( - outputText # allow user to access raw anthropic tool calling response - ) + model_response._hidden_params[ + "original_response" + ] = outputText # allow user to access raw anthropic tool calling response if ( _is_function_call is True and stream is not None @@ -793,9 +797,9 @@ class BedrockLLM(BaseAWSLLM): ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in inference_params[k] = v if stream is True: - inference_params["stream"] = ( - True # cohere requires stream = True in inference params - ) + inference_params[ + "stream" + ] = True # cohere requires stream = True in inference params data = json.dumps({"prompt": prompt, **inference_params}) elif provider == "anthropic": if model.startswith("anthropic.claude-3"): @@ -1184,6 +1188,7 @@ class AWSEventStreamDecoder: self.parser = EventStreamJSONParser() self.content_blocks: List[ContentBlockDeltaEvent] = [] self.tool_calls_index: Optional[int] = None + self.response_id: Optional[str] = None def check_empty_tool_call_args(self) -> bool: """ @@ -1245,8 +1250,169 @@ class AWSEventStreamDecoder: thinking_blocks_list.append(_thinking_block) return thinking_blocks_list + def _initialize_converse_response_id(self, chunk_data: dict): + """Initialize response_id from chunk data if not already set.""" + if self.response_id is None: + if "messageStart" in chunk_data: + conversation_id = chunk_data["messageStart"].get("conversationId") + if conversation_id: + self.response_id = f"chatcmpl-{conversation_id}" + else: + # Fallback to generating a UUID if the first chunk is not messageStart + self.response_id = f"chatcmpl-{uuid.uuid4()}" + + def _handle_converse_start_event( + self, + start_obj: ContentBlockStartEvent, + ) -> Tuple[ + Optional[ChatCompletionToolCallChunk], + dict, + Optional[ + List[ + Union[ + ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock + ] + ] + ], + ]: + """Handle 'start' event in converse chunk parsing.""" + tool_use: Optional[ChatCompletionToolCallChunk] = None + provider_specific_fields: dict = {} + thinking_blocks: Optional[ + List[ + Union[ + ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock + ] + ] + ] = None + + self.content_blocks = [] # reset + if start_obj is not None: + if "toolUse" in start_obj and start_obj["toolUse"] is not None: + ## check tool name was formatted by litellm + _response_tool_name = start_obj["toolUse"]["name"] + response_tool_name = get_bedrock_tool_name( + response_tool_name=_response_tool_name + ) + self.tool_calls_index = ( + 0 + if self.tool_calls_index is None + else self.tool_calls_index + 1 + ) + tool_use = { + "id": start_obj["toolUse"]["toolUseId"], + "type": "function", + "function": { + "name": response_tool_name, + "arguments": "", + }, + "index": self.tool_calls_index, + } + elif ( + "reasoningContent" in start_obj + and start_obj["reasoningContent"] is not None + ): # redacted thinking can be in start object + thinking_blocks = self.translate_thinking_blocks( + start_obj["reasoningContent"] + ) + provider_specific_fields = { + "reasoningContent": start_obj["reasoningContent"], + } + return tool_use, provider_specific_fields, thinking_blocks + + def _handle_converse_delta_event( + self, + delta_obj: ContentBlockDeltaEvent, + index: int, + ) -> Tuple[ + str, + Optional[ChatCompletionToolCallChunk], + dict, + Optional[str], + Optional[ + List[ + Union[ + ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock + ] + ] + ], + ]: + """Handle 'delta' event in converse chunk parsing.""" + text = "" + tool_use: Optional[ChatCompletionToolCallChunk] = None + provider_specific_fields: dict = {} + reasoning_content: Optional[str] = None + thinking_blocks: Optional[ + List[ + Union[ + ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock + ] + ] + ] = None + + self.content_blocks.append(delta_obj) + if "text" in delta_obj: + text = delta_obj["text"] + elif "toolUse" in delta_obj: + tool_use = { + "id": None, + "type": "function", + "function": { + "name": None, + "arguments": delta_obj["toolUse"]["input"], + }, + "index": ( + self.tool_calls_index + if self.tool_calls_index is not None + else index + ), + } + elif "reasoningContent" in delta_obj: + provider_specific_fields = { + "reasoningContent": delta_obj["reasoningContent"], + } + reasoning_content = self.extract_reasoning_content_str( + delta_obj["reasoningContent"] + ) + thinking_blocks = self.translate_thinking_blocks( + delta_obj["reasoningContent"] + ) + if ( + thinking_blocks + and len(thinking_blocks) > 0 + and reasoning_content is None + ): + reasoning_content = "" # set to non-empty string to ensure consistency with Anthropic + return text, tool_use, provider_specific_fields, reasoning_content, thinking_blocks + + def _handle_converse_stop_event( + self, index: int + ) -> Optional[ChatCompletionToolCallChunk]: + """Handle stop/contentBlockIndex event in converse chunk parsing.""" + tool_use: Optional[ChatCompletionToolCallChunk] = None + is_empty = self.check_empty_tool_call_args() + if is_empty: + tool_use = { + "id": None, + "type": "function", + "function": { + "name": None, + "arguments": "{}", + }, + "index": ( + self.tool_calls_index + if self.tool_calls_index is not None + else index + ), + } + return tool_use + def converse_chunk_parser(self, chunk_data: dict) -> ModelResponseStream: try: + # Capture the conversationId from the first messageStart event + # and use it as the consistent ID for all subsequent chunks. + self._initialize_converse_response_id(chunk_data) + verbose_logger.debug("\n\nRaw Chunk: {}\n\n".format(chunk_data)) text = "" tool_use: Optional[ChatCompletionToolCallChunk] = None @@ -1265,91 +1431,22 @@ class AWSEventStreamDecoder: index = int(chunk_data.get("contentBlockIndex", 0)) if "start" in chunk_data: start_obj = ContentBlockStartEvent(**chunk_data["start"]) - self.content_blocks = [] # reset - if start_obj is not None: - if "toolUse" in start_obj and start_obj["toolUse"] is not None: - ## check tool name was formatted by litellm - _response_tool_name = start_obj["toolUse"]["name"] - response_tool_name = get_bedrock_tool_name( - response_tool_name=_response_tool_name - ) - self.tool_calls_index = ( - 0 - if self.tool_calls_index is None - else self.tool_calls_index + 1 - ) - tool_use = { - "id": start_obj["toolUse"]["toolUseId"], - "type": "function", - "function": { - "name": response_tool_name, - "arguments": "", - }, - "index": self.tool_calls_index, - } - elif ( - "reasoningContent" in start_obj - and start_obj["reasoningContent"] is not None - ): # redacted thinking can be in start object - thinking_blocks = self.translate_thinking_blocks( - start_obj["reasoningContent"] - ) - provider_specific_fields = { - "reasoningContent": start_obj["reasoningContent"], - } + tool_use, provider_specific_fields, thinking_blocks = ( + self._handle_converse_start_event(start_obj) + ) elif "delta" in chunk_data: delta_obj = ContentBlockDeltaEvent(**chunk_data["delta"]) - self.content_blocks.append(delta_obj) - if "text" in delta_obj: - text = delta_obj["text"] - elif "toolUse" in delta_obj: - tool_use = { - "id": None, - "type": "function", - "function": { - "name": None, - "arguments": delta_obj["toolUse"]["input"], - }, - "index": ( - self.tool_calls_index - if self.tool_calls_index is not None - else index - ), - } - elif "reasoningContent" in delta_obj: - provider_specific_fields = { - "reasoningContent": delta_obj["reasoningContent"], - } - reasoning_content = self.extract_reasoning_content_str( - delta_obj["reasoningContent"] - ) - thinking_blocks = self.translate_thinking_blocks( - delta_obj["reasoningContent"] - ) - if ( - thinking_blocks - and len(thinking_blocks) > 0 - and reasoning_content is None - ): - reasoning_content = "" # set to non-empty string to ensure consistency with Anthropic + ( + text, + tool_use, + provider_specific_fields, + reasoning_content, + thinking_blocks, + ) = self._handle_converse_delta_event(delta_obj, index) elif ( "contentBlockIndex" in chunk_data ): # stop block, no 'start' or 'delta' object - is_empty = self.check_empty_tool_call_args() - if is_empty: - tool_use = { - "id": None, - "type": "function", - "function": { - "name": None, - "arguments": "{}", - }, - "index": ( - self.tool_calls_index - if self.tool_calls_index is not None - else index - ), - } + tool_use = self._handle_converse_stop_event(index) elif "stopReason" in chunk_data: finish_reason = map_finish_reason(chunk_data.get("stopReason", "stop")) elif "usage" in chunk_data: @@ -1378,6 +1475,7 @@ class AWSEventStreamDecoder: ), ) ], + id=self.response_id, usage=usage, provider_specific_fields=model_response_provider_specific_fields, ) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 37b4af306a1..c35e910ab08 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -87,6 +87,60 @@ def _prepare_request_data_and_content( return request_data, request_content +# Cache for SSL contexts to avoid creating duplicate contexts with the same configuration +# Key: tuple of (cafile, ssl_security_level, ssl_ecdh_curve) +# Value: ssl.SSLContext +_ssl_context_cache: Dict[Tuple[Optional[str], Optional[str], Optional[str]], ssl.SSLContext] = {} + + +def _create_ssl_context( + cafile: Optional[str], + ssl_security_level: Optional[str], + ssl_ecdh_curve: Optional[str], +) -> ssl.SSLContext: + """ + Create an SSL context with the given configuration. + This is separated from get_ssl_configuration to enable caching. + """ + custom_ssl_context = ssl.create_default_context(cafile=cafile) + + # Optimize SSL handshake performance + # Set minimum TLS version to 1.2 for better performance + custom_ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2 + + # Configure cipher suites for optimal performance + if ssl_security_level and isinstance(ssl_security_level, str): + # User provided custom cipher configuration (e.g., via SSL_SECURITY_LEVEL env var) + custom_ssl_context.set_ciphers(ssl_security_level) + else: + # Use optimized cipher list that strongly prefers fast ciphers + # but falls back to widely compatible ones + custom_ssl_context.set_ciphers(DEFAULT_SSL_CIPHERS) + + # Configure ECDH curve for key exchange (e.g., to disable PQC and improve performance) + # Set SSL_ECDH_CURVE env var or litellm.ssl_ecdh_curve to 'X25519' to disable PQC + # Common valid curves: X25519, prime256v1, secp384r1, secp521r1 + if ssl_ecdh_curve and isinstance(ssl_ecdh_curve, str): + try: + custom_ssl_context.set_ecdh_curve(ssl_ecdh_curve) + verbose_logger.debug(f"SSL ECDH curve set to: {ssl_ecdh_curve}") + except AttributeError: + verbose_logger.warning( + f"SSL ECDH curve configuration not supported. " + f"Python version: {sys.version.split()[0]}, OpenSSL version: {ssl.OPENSSL_VERSION}. " + f"Requested curve: {ssl_ecdh_curve}. Continuing with default curves." + ) + except ValueError as e: + # Invalid curve name + verbose_logger.warning( + f"Invalid SSL ECDH curve name: '{ssl_ecdh_curve}'. {e}. " + f"Common valid curves: X25519, prime256v1, secp384r1, secp521r1. " + f"Continuing with default curves (including PQC)." + ) + + return custom_ssl_context + + def get_ssl_configuration( ssl_verify: Optional[VerifyTypes] = None, ) -> Union[bool, str, ssl.SSLContext]: @@ -102,6 +156,9 @@ def get_ssl_configuration( If ssl_security_level is set, it will apply the security level to the SSL context. + SSL contexts are cached to avoid creating duplicate contexts with the same configuration, + which reduces memory allocation and improves performance. + Args: ssl_verify: SSL verification setting. Can be: - None: Use default from environment/litellm settings @@ -128,6 +185,7 @@ def get_ssl_configuration( ssl_verify = ssl_verify_bool ssl_security_level = os.getenv("SSL_SECURITY_LEVEL", litellm.ssl_security_level) + ssl_ecdh_curve = os.getenv("SSL_ECDH_CURVE", litellm.ssl_ecdh_curve) cafile = None if isinstance(ssl_verify, str) and os.path.exists(ssl_verify): @@ -140,49 +198,37 @@ def get_ssl_configuration( cafile = certifi.where() if ssl_verify is not False: - custom_ssl_context = ssl.create_default_context(cafile=cafile) - - # Optimize SSL handshake performance - # Set minimum TLS version to 1.2 for better performance - custom_ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2 - - # Configure cipher suites for optimal performance - if ssl_security_level and isinstance(ssl_security_level, str): - # User provided custom cipher configuration (e.g., via SSL_SECURITY_LEVEL env var) - custom_ssl_context.set_ciphers(ssl_security_level) - else: - # Use optimized cipher list that strongly prefers fast ciphers - # but falls back to widely compatible ones - custom_ssl_context.set_ciphers(DEFAULT_SSL_CIPHERS) - - # Configure ECDH curve for key exchange (e.g., to disable PQC and improve performance) - # Set SSL_ECDH_CURVE env var or litellm.ssl_ecdh_curve to 'X25519' to disable PQC - # Common valid curves: X25519, prime256v1, secp384r1, secp521r1 - ssl_ecdh_curve = os.getenv("SSL_ECDH_CURVE", litellm.ssl_ecdh_curve) - if ssl_ecdh_curve and isinstance(ssl_ecdh_curve, str): - try: - custom_ssl_context.set_ecdh_curve(ssl_ecdh_curve) - verbose_logger.debug(f"SSL ECDH curve set to: {ssl_ecdh_curve}") - except AttributeError: - verbose_logger.warning( - f"SSL ECDH curve configuration not supported. " - f"Python version: {sys.version.split()[0]}, OpenSSL version: {ssl.OPENSSL_VERSION}. " - f"Requested curve: {ssl_ecdh_curve}. Continuing with default curves." - ) - except ValueError as e: - # Invalid curve name - verbose_logger.warning( - f"Invalid SSL ECDH curve name: '{ssl_ecdh_curve}'. {e}. " - f"Common valid curves: X25519, prime256v1, secp384r1, secp521r1. " - f"Continuing with default curves (including PQC)." - ) - - # Use our custom SSL context instead of the original ssl_verify value - return custom_ssl_context + # Create cache key from configuration parameters + cache_key = (cafile, ssl_security_level, ssl_ecdh_curve) + + # Check if we have a cached SSL context for this configuration + if cache_key not in _ssl_context_cache: + _ssl_context_cache[cache_key] = _create_ssl_context( + cafile=cafile, + ssl_security_level=ssl_security_level, + ssl_ecdh_curve=ssl_ecdh_curve, + ) + + # Return the cached SSL context + return _ssl_context_cache[cache_key] return ssl_verify +_shared_realtime_ssl_context: Optional[Union[bool, str, ssl.SSLContext]] = None + + +def get_shared_realtime_ssl_context() -> Union[bool, str, ssl.SSLContext]: + """ + Lazily create the SSL context reused by realtime websocket clients so we avoid + import-order cycles during startup while keeping a single shared configuration. + """ + global _shared_realtime_ssl_context + if _shared_realtime_ssl_context is None: + _shared_realtime_ssl_context = get_ssl_configuration() + return _shared_realtime_ssl_context + + def mask_sensitive_info(error_message): # Find the start of the key parameter if isinstance(error_message, str): diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 861da9d9a27..a0e8190cc65 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -38,6 +38,7 @@ from litellm.llms.base_llm.google_genai.transformation import ( BaseGoogleGenAIGenerateContentConfig, ) from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from .http_handler import get_shared_realtime_ssl_context from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) @@ -1140,6 +1141,7 @@ class BaseLLMHTTPHandler: atranscription: bool = False, headers: Optional[Dict[str, Any]] = None, provider_config: Optional[BaseAudioTranscriptionConfig] = None, + shared_session: Optional["ClientSession"] = None, ) -> Union[TranscriptionResponse, Coroutine[Any, Any, TranscriptionResponse]]: if provider_config is None: raise ValueError( @@ -1162,6 +1164,7 @@ class BaseLLMHTTPHandler: client=client, headers=headers, provider_config=provider_config, + shared_session=shared_session, ) # Prepare the request @@ -1226,6 +1229,7 @@ class BaseLLMHTTPHandler: client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, headers: Optional[Dict[str, Any]] = None, provider_config: Optional[BaseAudioTranscriptionConfig] = None, + shared_session: Optional["ClientSession"] = None, ) -> TranscriptionResponse: if provider_config is None: raise ValueError( @@ -1254,6 +1258,7 @@ class BaseLLMHTTPHandler: async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + shared_session=shared_session, ) else: async_httpx_client = client @@ -3608,10 +3613,12 @@ class BaseLLMHTTPHandler: ) try: + ssl_context = get_shared_realtime_ssl_context() async with websockets.connect( # type: ignore url, extra_headers=headers, max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, + ssl=ssl_context, ) as backend_ws: realtime_streaming = RealTimeStreaming( websocket, @@ -4107,7 +4114,7 @@ class BaseLLMHTTPHandler: sync_httpx_client = client headers = video_generation_provider_config.validate_environment( - api_key=api_key, + api_key=api_key or litellm_params.get("api_key", None), headers=video_generation_optional_request_params.get("extra_headers", {}) or {}, model=model, @@ -4207,7 +4214,7 @@ class BaseLLMHTTPHandler: async_httpx_client = client headers = video_generation_provider_config.validate_environment( - api_key=api_key, + api_key=api_key or litellm_params.get("api_key", None), headers=video_generation_optional_request_params.get("extra_headers", {}) or {}, model=model, diff --git a/litellm/llms/docker_model_runner/chat/transformation.py b/litellm/llms/docker_model_runner/chat/transformation.py new file mode 100644 index 00000000000..3d84b24a01c --- /dev/null +++ b/litellm/llms/docker_model_runner/chat/transformation.py @@ -0,0 +1,144 @@ +""" +Translates from OpenAI's `/v1/chat/completions` to Docker Model Runner's `/engines/{engine}/v1/chat/completions` + +Docker Model Runner API Reference: https://docs.docker.com/ai/model-runner/api-reference/ +""" + +from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload + +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + handle_messages_with_content_list_to_str_conversion, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues + +from ...openai.chat.gpt_transformation import OpenAIGPTConfig + + +class DockerModelRunnerChatConfig(OpenAIGPTConfig): + """ + Configuration for Docker Model Runner API. + + Docker Model Runner uses URLs in the format: /engines/{engine}/v1/chat/completions + The engine name (e.g., "llama.cpp") is part of the API endpoint path. + """ + + @overload + def _transform_messages( + self, messages: List[AllMessageValues], model: str, is_async: Literal[True] + ) -> Coroutine[Any, Any, List[AllMessageValues]]: + ... + + @overload + def _transform_messages( + self, + messages: List[AllMessageValues], + model: str, + is_async: Literal[False] = False, + ) -> List[AllMessageValues]: + ... + + def _transform_messages( + self, messages: List[AllMessageValues], model: str, is_async: bool = False + ) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]: + """ + Docker Model Runner is OpenAI-compatible, so we use standard message transformation. + """ + messages = handle_messages_with_content_list_to_str_conversion(messages) + if is_async: + return super()._transform_messages( + messages=messages, model=model, is_async=True + ) + else: + return super()._transform_messages( + messages=messages, model=model, is_async=False + ) + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + """ + Get API base and key for Docker Model Runner. + + Default API base: http://localhost:22088/engines/llama.cpp + The engine path should be included in the api_base. + """ + api_base = ( + api_base + or get_secret_str("DOCKER_MODEL_RUNNER_API_BASE") + or "http://localhost:22088/engines/llama.cpp" + ) # type: ignore + # Docker Model Runner may not require authentication for local instances + dynamic_api_key = api_key or get_secret_str("DOCKER_MODEL_RUNNER_API_KEY") or "dummy-key" + return api_base, dynamic_api_key + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Build the complete URL for Docker Model Runner API. + + Docker Model Runner uses URLs in the format: /engines/{engine}/v1/chat/completions + + The engine name should be specified in the api_base: + - api_base="http://model-runner.docker.internal/engines/llama.cpp" + - Default: "http://localhost:22088/engines/llama.cpp" + + Args: + api_base: Base URL for the Docker Model Runner instance including engine path + api_key: API key (may not be required for local instances) + model: Model name (e.g., "llama-3.1") + optional_params: Optional parameters + litellm_params: LiteLLM parameters + stream: Whether streaming is enabled + + Returns: + Complete URL for the API call + """ + if not api_base: + api_base = "http://localhost:22088/engines/llama.cpp" + + # Remove trailing slashes from api_base + api_base = api_base.rstrip("/") + + # Build the URL: {api_base}/v1/chat/completions + # api_base is expected to already contain the engine path + complete_url = f"{api_base}/v1/chat/completions" + + return complete_url + + def get_supported_openai_params(self, model: str) -> list: + """ + Get the supported OpenAI params for Docker Model Runner. + + Docker Model Runner is OpenAI-compatible and supports standard parameters. + """ + return super().get_supported_openai_params(model=model) + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Map OpenAI parameters to Docker Model Runner parameters. + + Docker Model Runner is OpenAI-compatible, so most parameters map directly. + """ + supported_openai_params = self.get_supported_openai_params(model) + for param, value in non_default_params.items(): + if param == "max_completion_tokens": + optional_params["max_tokens"] = value + elif param in supported_openai_params: + optional_params[param] = value + + return optional_params + diff --git a/litellm/llms/gemini/chat/transformation.py b/litellm/llms/gemini/chat/transformation.py index e889126883c..c5e2d8b3dac 100644 --- a/litellm/llms/gemini/chat/transformation.py +++ b/litellm/llms/gemini/chat/transformation.py @@ -99,7 +99,7 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig): return supported_params def _transform_messages( - self, messages: List[AllMessageValues] + self, messages: List[AllMessageValues], model: Optional[str] = None ) -> List[ContentType]: """ Google AI Studio Gemini does not support HTTP/HTTPS URLs for files. @@ -140,4 +140,4 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig): except Exception: # If conversion fails, leave as is and let the API handle it pass - return _gemini_convert_messages_with_history(messages=messages) + return _gemini_convert_messages_with_history(messages=messages, model=model) diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py index d1ae47af269..ce2519e9177 100644 --- a/litellm/llms/gemini/videos/transformation.py +++ b/litellm/llms/gemini/videos/transformation.py @@ -15,17 +15,16 @@ from litellm.images.utils import ImageEditRequestUtils import litellm from litellm.types.llms.gemini import GeminiLongRunningOperationResponse, GeminiVideoGenerationInstance, GeminiVideoGenerationParameters, GeminiVideoGenerationRequest from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS +from litellm.llms.base_llm.videos.transformation import BaseVideoConfig + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from ...base_llm.videos.transformation import BaseVideoConfig as _BaseVideoConfig from ...base_llm.chat.transformation import BaseLLMException as _BaseLLMException LiteLLMLoggingObj = _LiteLLMLoggingObj - BaseVideoConfig = _BaseVideoConfig BaseLLMException = _BaseLLMException else: LiteLLMLoggingObj = Any - BaseVideoConfig = Any BaseLLMException = Any diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py new file mode 100644 index 00000000000..cc96e3415f3 --- /dev/null +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -0,0 +1,317 @@ +""" +GitHub Copilot Responses API Configuration. + +This module provides the configuration for GitHub Copilot's Responses API, +which is required for models like gpt-5.1-codex that only support the /responses endpoint. + +Implementation based on analysis of the copilot-api project by caozhiyuan: +https://github.com/caozhiyuan/copilot-api +""" +from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from uuid import uuid4 + +from litellm._logging import verbose_logger +from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.exceptions import AuthenticationError +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.types.llms.openai import ( + ResponseInputParam, + ResponsesAPIOptionalRequestParams, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + +from ..authenticator import Authenticator +from ..common_utils import GetAPIKeyError + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + +# GitHub Copilot API Constants (from copilot-api) +COPILOT_VERSION = "0.26.7" +EDITOR_PLUGIN_VERSION = f"copilot-chat/{COPILOT_VERSION}" +USER_AGENT = f"GitHubCopilotChat/{COPILOT_VERSION}" +API_VERSION = "2025-04-01" + + +class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): + """ + Configuration for GitHub Copilot's Responses API. + + Inherits from OpenAIResponsesAPIConfig since GitHub Copilot's Responses API + is compatible with OpenAI's Responses API specification. + + Key differences from OpenAI: + - Uses OAuth Device Flow authentication (handled by Authenticator) + - Uses api.githubcopilot.com as the API base + - Requires specific headers for VSCode/Copilot integration + - Supports vision requests with special header + - Requires X-Initiator header based on input analysis + + Reference: https://api.githubcopilot.com/ + """ + + GITHUB_COPILOT_API_BASE = "https://api.githubcopilot.com" + + def __init__(self) -> None: + super().__init__() + self.authenticator = Authenticator() + + @property + def custom_llm_provider(self) -> LlmProviders: + """Return the GitHub Copilot provider identifier.""" + return LlmProviders.GITHUB_COPILOT + + def get_supported_openai_params(self, model: str) -> list: + """ + Get supported parameters for GitHub Copilot Responses API. + + GitHub Copilot supports all standard OpenAI Responses API parameters. + """ + return super().get_supported_openai_params(model) + + def map_openai_params( + self, + response_api_optional_params: ResponsesAPIOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict: + """ + Map parameters for GitHub Copilot Responses API. + + GitHub Copilot uses the same parameter format as OpenAI, + so no transformation is needed. + """ + return dict(response_api_optional_params) + + def validate_environment( + self, + headers: dict, + model: str, + litellm_params: Optional[GenericLiteLLMParams], + ) -> dict: + """ + Validate environment and set up headers for GitHub Copilot API. + + Uses the Authenticator to obtain GitHub Copilot API key via OAuth Device Flow, + then configures all required headers for the Responses API. + + Headers include: + - Authorization with API key + - Standard GitHub Copilot headers (editor-version, user-agent, etc.) + - X-Initiator based on input analysis + - copilot-vision-request if vision content detected + - User-provided extra_headers (merged with priority) + """ + try: + # Get GitHub Copilot API key via OAuth + api_key = self.authenticator.get_api_key() + + if not api_key: + raise AuthenticationError( + model=model, + llm_provider="github_copilot", + message="GitHub Copilot API key is required. Please authenticate via OAuth Device Flow.", + ) + + # Get default headers (from copilot-api configuration) + default_headers = self._get_default_headers(api_key) + + # Merge with existing headers (user's extra_headers take priority) + merged_headers = {**default_headers, **headers} + + # Analyze input to determine additional headers + input_param = self._get_input_from_params(litellm_params) + + # Add X-Initiator header based on input analysis + if input_param is not None: + initiator = self._get_initiator(input_param) + merged_headers["X-Initiator"] = initiator + verbose_logger.debug( + f"GitHub Copilot Responses API: Set X-Initiator={initiator}" + ) + + # Add vision header if input contains images + if self._has_vision_input(input_param): + merged_headers["copilot-vision-request"] = "true" + verbose_logger.debug( + "GitHub Copilot Responses API: Enabled vision request" + ) + + verbose_logger.debug( + f"GitHub Copilot Responses API: Successfully configured headers for model {model}" + ) + + return merged_headers + + except GetAPIKeyError as e: + raise AuthenticationError( + model=model, + llm_provider="github_copilot", + message=str(e), + ) + + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Get the complete URL for GitHub Copilot Responses API endpoint. + + Returns: https://api.githubcopilot.com/responses + + Note: Currently only supports individual accounts. + Business/enterprise accounts (api.business.githubcopilot.com) can be + added in the future by detecting account type. + """ + # Use provided api_base or fall back to authenticator's base or default + api_base = ( + api_base + or self.authenticator.get_api_base() + or self.GITHUB_COPILOT_API_BASE + ) + + # Remove trailing slashes + api_base = api_base.rstrip("/") + + # Return the responses endpoint + return f"{api_base}/responses" + + # ==================== Helper Methods ==================== + + def _get_default_headers(self, api_key: str) -> Dict[str, str]: + """ + Get default headers for GitHub Copilot Responses API. + + Based on copilot-api's header configuration. + """ + return { + "Authorization": f"Bearer {api_key}", + "content-type": "application/json", + "copilot-integration-id": "vscode-chat", + "editor-version": "vscode/1.95.0", # Fixed version for stability + "editor-plugin-version": EDITOR_PLUGIN_VERSION, + "user-agent": USER_AGENT, + "openai-intent": "conversation-panel", + "x-github-api-version": API_VERSION, + "x-request-id": str(uuid4()), + "x-vscode-user-agent-library-version": "electron-fetch", + } + + def _get_input_from_params( + self, litellm_params: Optional[GenericLiteLLMParams] + ) -> Optional[Union[str, ResponseInputParam]]: + """ + Extract input parameter from litellm_params. + + The input parameter contains the conversation history and is needed + for vision detection and initiator determination. + """ + if litellm_params is None: + return None + + # Try to get input from litellm_params + # This might be in different locations depending on how LiteLLM structures it + if hasattr(litellm_params, "input"): + return litellm_params.input + + # If not found, return None and let the API handle it + return None + + def _get_initiator(self, input_param: Union[str, ResponseInputParam]) -> str: + """ + Determine X-Initiator header value based on input analysis. + + Based on copilot-api's hasAgentInitiator logic: + - Returns "agent" if input contains assistant role or items without role + - Returns "user" otherwise + + Args: + input_param: The input parameter (string or list of input items) + + Returns: + "agent" or "user" + """ + # If input is a string, it's user-initiated + if isinstance(input_param, str): + return "user" + + # If input is a list, analyze items + if isinstance(input_param, list): + for item in input_param: + if not isinstance(item, dict): + continue + + # Check if item has no role (agent-initiated) + if "role" not in item or not item.get("role"): + return "agent" + + # Check if role is assistant (agent-initiated) + role = item.get("role") + if isinstance(role, str) and role.lower() == "assistant": + return "agent" + + # Default to user-initiated + return "user" + + def _has_vision_input(self, input_param: Union[str, ResponseInputParam]) -> bool: + """ + Check if input contains vision content (images). + + Based on copilot-api's hasVisionInput and containsVisionContent logic. + Recursively searches for input_image type in the input structure. + + Args: + input_param: The input parameter to analyze + + Returns: + True if input contains image content, False otherwise + """ + return self._contains_vision_content(input_param) + + def _contains_vision_content( + self, value: Any, depth: int = 0, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH + ) -> bool: + """ + Recursively check if a value contains vision content. + + Looks for items with type="input_image" in the structure. + """ + if depth > max_depth: + verbose_logger.warning( + f"[GitHub Copilot] Max recursion depth {max_depth} reached while checking for vision content" + ) + return False + + if value is None: + return False + + # Check arrays + if isinstance(value, list): + return any( + self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) + for item in value + ) + + # Only check dict/object types + if not isinstance(value, dict): + return False + + # Check if this item is an input_image + item_type = value.get("type") + if isinstance(item_type, str) and item_type.lower() == "input_image": + return True + + # Check content field recursively + if "content" in value and isinstance(value["content"], list): + return any( + self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) + for item in value["content"] + ) + + return False diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 167ba26bacb..f0e2db9a08b 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -765,9 +765,10 @@ class OCIChatConfig(BaseConfig): ) if oci_serving_mode == "DEDICATED": + oci_endpoint_id = optional_params.get("oci_endpoint_id", model) servingMode = OCIServingMode( servingType="DEDICATED", - endpointId=model, + endpointId=oci_endpoint_id, ) else: servingMode = OCIServingMode( diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index e1fb3f12602..882309bb2fa 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -11,6 +11,7 @@ from litellm.types.realtime import RealtimeQueryParams from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from ....litellm_core_utils.realtime_streaming import RealTimeStreaming +from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from ..openai import OpenAIChatCompletion @@ -55,6 +56,7 @@ class OpenAIRealtime(OpenAIChatCompletion): url = self._construct_url(api_base, query_params) try: + ssl_context = get_shared_realtime_ssl_context() async with websockets.connect( # type: ignore url, extra_headers={ @@ -62,6 +64,7 @@ class OpenAIRealtime(OpenAIChatCompletion): "OpenAI-Beta": "realtime=v1", }, max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, + ssl=ssl_context, ) as backend_ws: realtime_streaming = RealTimeStreaming( websocket, cast(ClientConnection, backend_ws), logging_obj diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index 4d60b8a8310..e241d2c1c7d 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -1,10 +1,13 @@ -from typing import Optional, Union, cast +from typing import TYPE_CHECKING, Optional, Union, cast import httpx from openai import AsyncOpenAI, OpenAI from pydantic import BaseModel import litellm + +if TYPE_CHECKING: + from aiohttp import ClientSession from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.audio_transcription.transformation import ( @@ -89,6 +92,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): client=None, atranscription: bool = False, provider_config: Optional[BaseAudioTranscriptionConfig] = None, + shared_session: Optional["ClientSession"] = None, ) -> TranscriptionResponse: """ Handle audio transcription request @@ -116,6 +120,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): client=client, max_retries=max_retries, logging_obj=logging_obj, + shared_session=shared_session, ) openai_client: OpenAI = self._get_openai_client( # type: ignore @@ -170,6 +175,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): api_base: Optional[str] = None, client=None, max_retries=None, + shared_session: Optional["ClientSession"] = None, ): try: openai_aclient: AsyncOpenAI = self._get_openai_client( # type: ignore @@ -179,6 +185,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): timeout=timeout, max_retries=max_retries, client=client, + shared_session=shared_session, ) ## LOGGING diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index 8f5d41fe467..d1d3fc2919e 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -4,6 +4,7 @@ from typing import cast import httpx from httpx._types import RequestFiles +from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.types.videos.main import VideoCreateOptionalRequestParams from litellm.types.llms.openai import CreateVideoRequest from litellm.types.router import GenericLiteLLMParams @@ -15,15 +16,12 @@ from litellm.llms.openai.image_edit.transformation import ImageEditRequestUtils if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from ...base_llm.videos.transformation import BaseVideoConfig as _BaseVideoConfig from ...base_llm.chat.transformation import BaseLLMException as _BaseLLMException LiteLLMLoggingObj = _LiteLLMLoggingObj - BaseVideoConfig = _BaseVideoConfig BaseLLMException = _BaseLLMException else: LiteLLMLoggingObj = Any - BaseVideoConfig = Any BaseLLMException = Any diff --git a/litellm/llms/snowflake/chat/transformation.py b/litellm/llms/snowflake/chat/transformation.py index 4c0258d9f4b..62ede0aeaf8 100644 --- a/litellm/llms/snowflake/chat/transformation.py +++ b/litellm/llms/snowflake/chat/transformation.py @@ -7,12 +7,14 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import httpx -from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ChatCompletionMessageToolCall, Function, ModelResponse from ...openai_like.chat.transformation import OpenAIGPTConfig +from ..utils import SnowflakeBaseConfig + + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -21,7 +23,7 @@ else: LiteLLMLoggingObj = Any -class SnowflakeConfig(OpenAIGPTConfig): +class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): """ Reference: https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-llm-rest-api @@ -33,40 +35,6 @@ class SnowflakeConfig(OpenAIGPTConfig): def get_config(cls): return super().get_config() - def get_supported_openai_params(self, model: str) -> List[str]: - return [ - "temperature", - "max_tokens", - "top_p", - "response_format", - "tools", - "tool_choice", - ] - - def map_openai_params( - self, - non_default_params: dict, - optional_params: dict, - model: str, - drop_params: bool, - ) -> dict: - """ - If any supported_openai_params are in non_default_params, add them to optional_params, so they are used in API call - - Args: - non_default_params (dict): Non-default parameters to filter. - optional_params (dict): Optional parameters to update. - model (str): Model name for parameter support check. - - Returns: - dict: Updated optional_params with supported non-default parameters. - """ - supported_openai_params = self.get_supported_openai_params(model) - for param, value in non_default_params.items(): - if param in supported_openai_params: - optional_params[param] = value - return optional_params - def _transform_tool_calls_from_snowflake_to_openai( self, content_list: List[Dict[str, Any]] ) -> Tuple[str, Optional[List[ChatCompletionMessageToolCall]]]: @@ -169,53 +137,6 @@ class SnowflakeConfig(OpenAIGPTConfig): returned_response._hidden_params["model"] = model return returned_response - def validate_environment( - self, - headers: dict, - model: str, - messages: List[AllMessageValues], - optional_params: dict, - litellm_params: dict, - api_key: Optional[str] = None, - api_base: Optional[str] = None, - ) -> dict: - """ - Return headers to use for Snowflake completion request - - Snowflake REST API Ref: https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-llm-rest-api#api-reference - Expected headers: - { - "Content-Type": "application/json", - "Accept": "application/json", - "Authorization": "Bearer " + , - "X-Snowflake-Authorization-Token-Type": "KEYPAIR_JWT" - } - """ - - if api_key is None: - raise ValueError("Missing Snowflake JWT key") - - headers.update( - { - "Content-Type": "application/json", - "Accept": "application/json", - "Authorization": "Bearer " + api_key, - "X-Snowflake-Authorization-Token-Type": "KEYPAIR_JWT", - } - ) - return headers - - def _get_openai_compatible_provider_info( - self, api_base: Optional[str], api_key: Optional[str] - ) -> Tuple[Optional[str], Optional[str]]: - api_base = ( - api_base - or f"""https://{get_secret_str("SNOWFLAKE_ACCOUNT_ID")}.snowflakecomputing.com/api/v2/cortex/inference:complete""" - or get_secret_str("SNOWFLAKE_API_BASE") - ) - dynamic_api_key = api_key or get_secret_str("SNOWFLAKE_JWT") - return api_base, dynamic_api_key - def get_complete_url( self, api_base: Optional[str], @@ -228,10 +149,10 @@ class SnowflakeConfig(OpenAIGPTConfig): """ If api_base is not provided, use the default DeepSeek /chat/completions endpoint. """ - if not api_base: - api_base = f"""https://{get_secret_str("SNOWFLAKE_ACCOUNT_ID")}.snowflakecomputing.com/api/v2/cortex/inference:complete""" - return api_base + api_base = self._get_api_base(api_base, optional_params) + + return f"{api_base}/cortex/inference:complete" def _transform_tools(self, tools: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """ @@ -279,9 +200,7 @@ class SnowflakeConfig(OpenAIGPTConfig): } # Add description if present if "description" in function: - snowflake_tool["tool_spec"]["description"] = function[ - "description" - ] + snowflake_tool["tool_spec"]["description"] = function["description"] snowflake_tools.append(snowflake_tool) diff --git a/litellm/llms/snowflake/embedding/transformation.py b/litellm/llms/snowflake/embedding/transformation.py new file mode 100644 index 00000000000..83716f3ef26 --- /dev/null +++ b/litellm/llms/snowflake/embedding/transformation.py @@ -0,0 +1,69 @@ +from typing import Optional, Union + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig +from litellm.types.llms.openai import AllEmbeddingInputValues +from litellm.types.utils import EmbeddingResponse + +from ..utils import SnowflakeException, SnowflakeBaseConfig + + +class SnowflakeEmbeddingConfig(SnowflakeBaseConfig, BaseEmbeddingConfig): + """ + source: https://docs.snowflake.com/developer-guide/snowflake-rest-api/reference/cortex-embed + """ + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + api_base = self._get_api_base(api_base, optional_params) + + return f"{api_base}/cortex/inference:embed" + + def transform_embedding_request( + self, + model: str, + input: AllEmbeddingInputValues, + optional_params: dict, + headers: dict, + ) -> dict: + return {"text": input, "model": model, **optional_params} + + def transform_embedding_response( + self, + model: str, + raw_response: httpx.Response, + model_response: EmbeddingResponse, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + request_data: dict, + optional_params: dict, + litellm_params: dict, + ) -> EmbeddingResponse: + response_json = raw_response.json() + # convert embeddings to 1d array + for item in response_json["data"]: + item["embedding"] = item["embedding"][0] + returned_response = EmbeddingResponse(**response_json) + + returned_response.model = "snowflake/" + (returned_response.model or "") + + if model is not None: + returned_response._hidden_params["model"] = model + return returned_response + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + return SnowflakeException( + message=error_message, status_code=status_code, headers=headers + ) diff --git a/litellm/llms/snowflake/utils.py b/litellm/llms/snowflake/utils.py new file mode 100644 index 00000000000..9d458f6ece3 --- /dev/null +++ b/litellm/llms/snowflake/utils.py @@ -0,0 +1,118 @@ +from typing import TYPE_CHECKING, Any, List, Optional, Tuple + +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues +from litellm.llms.base_llm.chat.transformation import BaseLLMException + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class SnowflakeException(BaseLLMException): + """Snowflake AI Endpoints exception handling class""" + + pass + + +class SnowflakeBaseConfig: + def get_supported_openai_params(self, model: str) -> List[str]: + return [ + "temperature", + "max_tokens", + "top_p", + "response_format", + "tools", + "tool_choice", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + If any supported_openai_params are in non_default_params, add them to optional_params, so they are used in API call + + Args: + non_default_params (dict): Non-default parameters to filter. + optional_params (dict): Optional parameters to update. + model (str): Model name for parameter support check. + + Returns: + dict: Updated optional_params with supported non-default parameters. + """ + supported_openai_params = self.get_supported_openai_params(model) + for param, value in non_default_params.items(): + if param in supported_openai_params: + optional_params[param] = value + return optional_params + + def _get_api_base(self, api_base, optional_params): + if not api_base: + if "account_id" in optional_params: + account_id = optional_params.pop("account_id") + else: + account_id = get_secret_str("SNOWFLAKE_ACCOUNT_ID") + if account_id is None: + raise ValueError("Missing snowflake account_id") + api_base = f"https://{account_id}.snowflakecomputing.com/api/v2" + + api_base = api_base.rstrip("/") + if not api_base.endswith("/api/v2"): + api_base += "/api/v2" + return api_base + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Return headers to use for Snowflake completion request + + Snowflake REST API Ref: https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-llm-rest-api#api-reference + Expected headers: + { + "Content-Type": "application/json", + "Accept": "application/json", + "Authorization": "Bearer " + , + "X-Snowflake-Authorization-Token-Type": "KEYPAIR_JWT" + } + """ + + auth_type = "KEYPAIR_JWT" + + if api_key is None: + raise ValueError("Missing Snowflake JWT key") + else: + pat_key_prefix = "pat/" + if api_key.startswith(pat_key_prefix): + api_key = api_key[len(pat_key_prefix) :] + auth_type = "PROGRAMMATIC_ACCESS_TOKEN" + + headers.update( + { + "Content-Type": "application/json", + "Accept": "application/json", + "Authorization": "Bearer " + api_key, + "X-Snowflake-Authorization-Token-Type": auth_type, + } + ) + return headers + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + dynamic_api_key = api_key or get_secret_str("SNOWFLAKE_JWT") + return api_base, dynamic_api_key diff --git a/litellm/llms/vertex_ai/context_caching/transformation.py b/litellm/llms/vertex_ai/context_caching/transformation.py index bb40b7665c1..bc5c1b451f1 100644 --- a/litellm/llms/vertex_ai/context_caching/transformation.py +++ b/litellm/llms/vertex_ai/context_caching/transformation.py @@ -173,7 +173,7 @@ def transform_openai_messages_to_gemini_context_caching( supports_system_message=supports_system_message, messages=messages ) - transformed_messages = _gemini_convert_messages_with_history(messages=new_messages) + transformed_messages = _gemini_convert_messages_with_history(messages=new_messages, model=model) model_name = "models/{}".format(model) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 08c91a6fad1..e4fcd35b954 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -64,7 +64,25 @@ else: LiteLLMLoggingObj = Any -def _process_gemini_image(image_url: str, format: Optional[str] = None) -> PartType: +def _map_openai_detail_to_media_resolution( + detail: Optional[str], +) -> Optional[Literal["low", "medium", "high"]]: + """ + Map OpenAI's "detail" parameter to Gemini's "media_resolution" parameter. + """ + if detail == "low": + return "low" + elif detail == "high": + return "high" + # "auto" or None means let the model decide, so we don't set media_resolution + return None + + +def _process_gemini_image( + image_url: str, + format: Optional[str] = None, + media_resolution: Optional[Literal["low", "medium", "high"]] = None, +) -> PartType: """ Given an image URL, return the appropriate PartType for Gemini """ @@ -99,8 +117,19 @@ def _process_gemini_image(image_url: str, format: Optional[str] = None) -> PartT elif "http://" in image_url or "https://" in image_url or "base64" in image_url: # https links for unsupported mime types and base64 images image = convert_to_anthropic_image_obj(image_url, format=format) - _blob = BlobType(data=image["data"], mime_type=image["media_type"]) - return PartType(inline_data=_blob) + _blob: BlobType = {"data": image["data"], "mime_type": image["media_type"]} + if media_resolution is not None: + _blob["media_resolution"] = media_resolution + + # Convert snake_case keys to camelCase for JSON serialization + # The TypedDict uses snake_case, but the API expects camelCase + _blob_dict = dict(_blob) + if "media_resolution" in _blob_dict: + _blob_dict["mediaResolution"] = _blob_dict.pop("media_resolution") + if "mime_type" in _blob_dict: + _blob_dict["mimeType"] = _blob_dict.pop("mime_type") + + return PartType(inline_data=cast(BlobType, _blob_dict)) raise Exception("Invalid image received - {}".format(image_url)) except Exception as e: raise e @@ -166,6 +195,7 @@ def check_if_part_exists_in_parts( def _gemini_convert_messages_with_history( # noqa: PLR0915 messages: List[AllMessageValues], + model: Optional[str] = None, ) -> List[ContentType]: """ Converts given messages from OpenAI format to Gemini format @@ -205,13 +235,18 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 element = cast(ChatCompletionImageObject, element) img_element = element format: Optional[str] = None + media_resolution: Optional[Literal["low", "medium", "high"]] = None if isinstance(img_element["image_url"], dict): image_url = img_element["image_url"]["url"] format = img_element["image_url"].get("format") + detail = img_element["image_url"].get("detail") + media_resolution = _map_openai_detail_to_media_resolution(detail) else: image_url = img_element["image_url"] _part = _process_gemini_image( - image_url=image_url, format=format + image_url=image_url, + format=format, + media_resolution=media_resolution, ) _parts.append(_part) elif element["type"] == "input_audio": @@ -250,7 +285,8 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 ) try: _part = _process_gemini_image( - image_url=passed_file, format=format + image_url=passed_file, + format=format, ) _parts.append(_part) except Exception: @@ -344,7 +380,7 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 or assistant_msg.get("function_call") is not None ): # support assistant tool invoke conversion gemini_tool_call_parts = convert_to_gemini_tool_call_invoke( - assistant_msg + assistant_msg, model=model ) ## check if gemini_tool_call already exists in assistant_content for gemini_tool_call_part in gemini_tool_call_parts: @@ -448,11 +484,11 @@ def _transform_request_body( try: if custom_llm_provider == "gemini": content = litellm.GoogleAIStudioGeminiConfig()._transform_messages( - messages=messages + messages=messages, model=model ) else: content = litellm.VertexGeminiConfig()._transform_messages( - messages=messages + messages=messages, model=model ) tools: Optional[Tools] = optional_params.pop("tools", None) tool_choice: Optional[ToolConfig] = optional_params.pop("tool_choice", None) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index cbd8cf320c7..ab594c79ef4 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -35,6 +35,9 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE, DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO, ) +from litellm.litellm_core_utils.prompt_templates.factory import ( + _encode_tool_call_id_with_signature, +) from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -217,8 +220,26 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): @classmethod def get_config(cls): return super().get_config() - + + @staticmethod + def _is_gemini_3_or_newer(model: str) -> bool: + """ + Check if the model is Gemini 3 Pro or newer. + + Gemini 3 models include: + - gemini-3-pro-preview + - Any future Gemini 3.x models + """ + # Check for Gemini 3 models + if "gemini-3" in model: + return True + + return False + def _supports_penalty_parameters(self, model: str) -> bool: + # Gemini 3 models do not support penalty parameters + if VertexGeminiConfig._is_gemini_3_or_newer(model): + return False unsupported_models = ["gemini-2.5-pro-preview-06-05"] if model in unsupported_models: return False @@ -245,11 +266,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "parallel_tool_calls", "web_search_options", ] - + # Add penalty parameters only for non-preview models if self._supports_penalty_parameters(model): supported_params.extend(["frequency_penalty", "presence_penalty"]) - + if supports_reasoning(model): supported_params.append("reasoning_effort") supported_params.append("thinking") @@ -293,14 +314,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) -> Tuple[dict, Optional[dict]]: """ Extract location configuration from googleMaps tool for Vertex AI toolConfig. - + Supports two interface styles: 1. Nested (recommended): {"enableWidget": "...", "retrievalConfig": {"latitude": ..., "longitude": ...}} 2. Flat (backward compat): {"enableWidget": "...", "latitude": ..., "longitude": ...} - + Args: google_maps_config: The googleMaps tool configuration from LiteLLM - + Returns: Tuple of (cleaned_google_maps_config, retrieval_config): - cleaned_google_maps_config: googleMaps config without location fields @@ -310,7 +331,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): latitude = google_maps_config.get("latitude") longitude = google_maps_config.get("longitude") language_code = google_maps_config.get("languageCode") - + if latitude is not None and longitude is not None: retrieval_config = { "latLng": { @@ -320,21 +341,17 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): } if language_code is not None: retrieval_config["languageCode"] = language_code - + # Remove location fields from tool definition cleaned_config = { k: v for k, v in google_maps_config.items() if k not in ["latitude", "longitude", "languageCode"] } - + return cleaned_config, retrieval_config - - def get_tool_value( - self, - tool: dict, - tool_name: str - ) -> Optional[dict]: + + def get_tool_value(self, tool: dict, tool_name: str) -> Optional[dict]: """ Helper function to get tool value handling both camelCase and underscore_case variants @@ -358,19 +375,19 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): else: return None - def _map_function( # noqa: PLR0915 + def _map_function( # noqa: PLR0915 self, value: List[dict], optional_params: dict ) -> List[Tools]: """ Map OpenAI-style tools/functions to Vertex AI format. - + Args: value: List of tool definitions optional_params: Request-scoped parameters to store retrieval config - + Returns: List of mapped tools in Vertex AI format - + Side effects: May add 'toolConfig' with 'retrievalConfig' to optional_params if googleMaps tools contain location data @@ -417,25 +434,43 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): tool_name = list(tool.keys())[0] if len(tool.keys()) == 1 else None if tool_name and ( - tool_name == "codeExecution" or tool_name == VertexToolName.CODE_EXECUTION.value + tool_name == "codeExecution" + or tool_name == VertexToolName.CODE_EXECUTION.value ): # code_execution maintained for backwards compatibility code_execution = self.get_tool_value(tool, "codeExecution") elif tool_name and tool_name == VertexToolName.GOOGLE_SEARCH.value: - googleSearch = self.get_tool_value(tool, VertexToolName.GOOGLE_SEARCH.value) - elif tool_name and tool_name == VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value: - googleSearchRetrieval = self.get_tool_value(tool, VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value) + googleSearch = self.get_tool_value( + tool, VertexToolName.GOOGLE_SEARCH.value + ) + elif ( + tool_name and tool_name == VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value + ): + googleSearchRetrieval = self.get_tool_value( + tool, VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value + ) elif tool_name and tool_name == VertexToolName.ENTERPRISE_WEB_SEARCH.value: - enterpriseWebSearch = self.get_tool_value(tool, VertexToolName.ENTERPRISE_WEB_SEARCH.value) - elif tool_name and (tool_name == VertexToolName.URL_CONTEXT.value or tool_name == "urlContext"): + enterpriseWebSearch = self.get_tool_value( + tool, VertexToolName.ENTERPRISE_WEB_SEARCH.value + ) + elif tool_name and ( + tool_name == VertexToolName.URL_CONTEXT.value + or tool_name == "urlContext" + ): urlContext = self.get_tool_value(tool, tool_name) elif tool_name and ( - tool_name == VertexToolName.GOOGLE_MAPS.value or tool_name == "google_maps" + tool_name == VertexToolName.GOOGLE_MAPS.value + or tool_name == "google_maps" ): - google_maps_value = self.get_tool_value(tool, VertexToolName.GOOGLE_MAPS.value) - + google_maps_value = self.get_tool_value( + tool, VertexToolName.GOOGLE_MAPS.value + ) + # Extract and transform location configuration for toolConfig if google_maps_value is not None: - googleMaps, google_maps_retrieval_config = self._extract_google_maps_retrieval_config( + ( + googleMaps, + google_maps_retrieval_config, + ) = self._extract_google_maps_retrieval_config( google_maps_config=google_maps_value ) elif openai_function_object is not None: @@ -475,13 +510,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): _tools[VertexToolName.URL_CONTEXT.value] = urlContext if googleMaps is not None: _tools[VertexToolName.GOOGLE_MAPS.value] = googleMaps - + # Add retrieval config to toolConfig if googleMaps has location data if google_maps_retrieval_config is not None: if "toolConfig" not in optional_params: optional_params["toolConfig"] = {} - optional_params["toolConfig"]["retrievalConfig"] = google_maps_retrieval_config - + optional_params["toolConfig"][ + "retrievalConfig" + ] = google_maps_retrieval_config + return [_tools] def _map_response_schema(self, value: dict) -> dict: @@ -575,10 +612,83 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): else: raise ValueError(f"Invalid reasoning effort: {reasoning_effort}") + @staticmethod + def _map_reasoning_effort_to_thinking_level( + reasoning_effort: str, + model: Optional[str] = None, + ) -> GeminiThinkingConfig: + """ + Map reasoning_effort to thinking_level for Gemini 3+ models. + Args: + reasoning_effort: The reasoning effort value + model: The model name + + Returns: + GeminiThinkingConfig with thinkingLevel and includeThoughts + """ + if reasoning_effort == "minimal": + return {"thinkingLevel": "low", "includeThoughts": True} + elif reasoning_effort == "low": + return {"thinkingLevel": "low", "includeThoughts": True} + elif reasoning_effort == "medium": + return { + "thinkingLevel": "high", + "includeThoughts": True, + } # medium is not out yet + elif reasoning_effort == "high": + return {"thinkingLevel": "high", "includeThoughts": True} + elif reasoning_effort == "disable": + # Gemini 3 cannot fully disable thinking, so we use "low" but hide thoughts + return {"thinkingLevel": "low", "includeThoughts": False} + elif reasoning_effort == "none": + return {"thinkingLevel": "low", "includeThoughts": False} + else: + raise ValueError(f"Invalid reasoning effort: {reasoning_effort}") + @staticmethod def _is_thinking_budget_zero(thinking_budget: Optional[int]) -> bool: return thinking_budget is not None and thinking_budget == 0 + @staticmethod + def _validate_thinking_config_conflicts( + optional_params: Dict, + param_name: str, + param_description: str = "thinking_budget", + ) -> None: + """ + Validate that thinking_level and thinking_budget are not both specified. + """ + if "thinkingConfig" in optional_params: + existing_config = optional_params["thinkingConfig"] + if "thinkingLevel" in existing_config: + raise litellm.utils.UnsupportedParamsError( + message=( + f"Cannot specify both `{param_name}` (which maps to `{param_description}`) " + "and `thinking_level` in the same request. " + "For Gemini 3 models, use `thinking_level` instead." + ), + status_code=400, + ) + + @staticmethod + def _validate_thinking_level_conflicts( + optional_params: Dict, + ) -> None: + """ + Validate that thinking_level and thinking_budget are not both specified. + Called when setting thinking_level. + """ + if "thinkingConfig" in optional_params: + existing_config = optional_params["thinkingConfig"] + if "thinkingBudget" in existing_config: + raise litellm.utils.UnsupportedParamsError( + message=( + "Cannot specify both `thinking_level` and `thinking_budget` in the same request. " + "For Gemini 3 models, use `thinking_level` instead of `thinking_budget`." + ), + status_code=400, + ) + @staticmethod def _map_thinking_param( thinking_param: AnthropicThinkingParam, @@ -672,6 +782,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) -> Dict: for param, value in non_default_params.items(): if param == "temperature": + if VertexGeminiConfig._is_gemini_3_or_newer(model): + if value is not None and value < 1.0: + verbose_logger.info( + f"Warning: Setting temperature < 1.0 for Gemini 3 models ({model}) " + "can cause infinite loops, degraded reasoning performance, and failure on complex tasks. " + "Strongly recommended to use temperature = 1.0 (default)." + ) optional_params["temperature"] = value elif param == "top_p": optional_params["top_p"] = value @@ -734,12 +851,31 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): elif param == "seed": optional_params["seed"] = value elif param == "reasoning_effort" and isinstance(value, str): - optional_params[ - "thinkingConfig" - ] = VertexGeminiConfig._map_reasoning_effort_to_thinking_budget( - value, model + # Validate no conflict with thinking_level + VertexGeminiConfig._validate_thinking_config_conflicts( + optional_params=optional_params, + param_name="reasoning_effort", + param_description="thinking_budget", ) + if VertexGeminiConfig._is_gemini_3_or_newer(model): + optional_params[ + "thinkingConfig" + ] = VertexGeminiConfig._map_reasoning_effort_to_thinking_level( + value, model + ) + else: + optional_params[ + "thinkingConfig" + ] = VertexGeminiConfig._map_reasoning_effort_to_thinking_budget( + value, model + ) elif param == "thinking": + # Validate no conflict with thinking_level + VertexGeminiConfig._validate_thinking_config_conflicts( + optional_params=optional_params, + param_name="thinking", + param_description="thinking_budget", + ) optional_params[ "thinkingConfig" ] = VertexGeminiConfig._map_thinking_param( @@ -764,6 +900,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): elif "AUDIO" not in optional_params["responseModalities"]: optional_params["responseModalities"].append("AUDIO") + # Set default temperature to 1.0 for Gemini 3 models if not specified + if VertexGeminiConfig._is_gemini_3_or_newer(model): + if "temperature" not in optional_params: + optional_params["temperature"] = 1.0 + thinking_config = optional_params.get("thinkingConfig", {}) + if ( + "thinkingLevel" not in thinking_config + and "thinkingBudget" not in thinking_config + ): + thinking_config["thinkingLevel"] = "low" + optional_params["thinkingConfig"] = thinking_config + return optional_params def get_mapped_special_auth_params(self) -> dict: @@ -911,8 +1059,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): pass _content_str += text_content elif "inlineData" in part: - mime_type = part["inlineData"]["mimeType"] - data = part["inlineData"]["data"] + inline_data = part.get("inlineData", {}) + mime_type = inline_data.get("mimeType", "") + data = inline_data.get("data", "") # Check if inline data is audio or image - if so, exclude from text content # Images and audio are now handled separately in their respective response fields if mime_type.startswith("audio/") or mime_type.startswith("image/"): @@ -956,8 +1105,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): images: List[ImageURLListItem] = [] for part in parts: if "inlineData" in part: - mime_type = part["inlineData"]["mimeType"] - data = part["inlineData"]["data"] + inline_data = part.get("inlineData", {}) + mime_type = inline_data.get("mimeType", "") + data = inline_data.get("data", "") if mime_type.startswith("image/"): # Convert base64 data to data URI format data_uri = f"data:{mime_type};base64,{data}" @@ -998,8 +1148,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): pass elif "inlineData" in part: - mime_type = part["inlineData"]["mimeType"] - data = part["inlineData"]["data"] + inline_data = part.get("inlineData", {}) + mime_type = inline_data.get("mimeType", "") + data = inline_data.get("data", "") if mime_type.startswith("audio/"): expires_at = int(time.time()) + (24 * 60 * 60) @@ -1025,19 +1176,41 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): _tools: List[ChatCompletionToolCallChunk] = [] for part in parts: if "functionCall" in part: - _function_chunk = ChatCompletionToolCallFunctionChunk( - name=part["functionCall"]["name"], - arguments=json.dumps(part["functionCall"]["args"], ensure_ascii=False), - ) + _function_chunk: ChatCompletionToolCallFunctionChunk = { + "name": part["functionCall"]["name"], + "arguments": json.dumps( + part["functionCall"]["args"], ensure_ascii=False + ), + } + # Extract thought signature if present + thought_signature = part.get("thoughtSignature") + if is_function_call is True: - function = _function_chunk + function_dict: Dict[str, Any] = dict(_function_chunk) + if thought_signature: + if "provider_specific_fields" not in function_dict: + function_dict["provider_specific_fields"] = {} + function_dict["provider_specific_fields"][ + "thought_signature" + ] = thought_signature + function = cast(ChatCompletionToolCallFunctionChunk, function_dict) else: - _tool_response_chunk = ChatCompletionToolCallChunk( - id=f"call_{uuid.uuid4().hex[:28]}", - type="function", - function=_function_chunk, - index=cumulative_tool_call_idx, - ) + _tool_response_chunk: ChatCompletionToolCallChunk = { + "id": f"call_{uuid.uuid4().hex[:28]}", + "type": "function", + "function": _function_chunk, + "index": cumulative_tool_call_idx, + } + # Embed thought signature in ID for OpenAI client compatibility + if thought_signature: + _tool_response_chunk[ + "id" + ] = _encode_tool_call_id_with_signature( + _tool_response_chunk["id"] or "", thought_signature + ) + _tool_response_chunk["provider_specific_fields"] = { # type: ignore + "thought_signature": thought_signature + } _tools.append(_tool_response_chunk) cumulative_tool_call_idx += 1 if len(_tools) == 0: @@ -1174,7 +1347,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return False @staticmethod - def _calculate_usage( + def _calculate_usage( # noqa: PLR0915 completion_response: Union[ GenerateContentResponseBody, BidiGenerateContentServerMessage ], @@ -1209,6 +1382,30 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): response_tokens_details.audio_tokens = detail.get("tokenCount", 0) ######################################################### + ## CANDIDATES TOKEN DETAILS (e.g., for image generation models) ## + if "candidatesTokensDetails" in usage_metadata: + if response_tokens_details is None: + response_tokens_details = CompletionTokensDetailsWrapper() + for detail in usage_metadata["candidatesTokensDetails"]: + modality = detail.get("modality") + token_count = detail.get("tokenCount", 0) + if modality == "TEXT": + response_tokens_details.text_tokens = token_count + elif modality == "AUDIO": + response_tokens_details.audio_tokens = token_count + elif modality == "IMAGE": + response_tokens_details.image_tokens = token_count + + # Calculate text_tokens if not explicitly provided in candidatesTokensDetails + # candidatesTokenCount includes all modalities, so: text = total - (image + audio) + if response_tokens_details.text_tokens is None: + candidates_token_count = usage_metadata.get("candidatesTokenCount", 0) + image_tokens = response_tokens_details.image_tokens or 0 + audio_tokens_candidate = response_tokens_details.audio_tokens or 0 + calculated_text_tokens = candidates_token_count - image_tokens - audio_tokens_candidate + response_tokens_details.text_tokens = calculated_text_tokens + ######################################################### + if "promptTokensDetails" in usage_metadata: for detail in usage_metadata["promptTokensDetails"]: if detail["modality"] == "AUDIO": @@ -1217,6 +1414,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): text_tokens = detail.get("tokenCount", 0) if "thoughtsTokenCount" in usage_metadata: reasoning_tokens = usage_metadata["thoughtsTokenCount"] + # Also add reasoning tokens to response_tokens_details + if response_tokens_details is None: + response_tokens_details = CompletionTokensDetailsWrapper() + response_tokens_details.reasoning_tokens = reasoning_tokens ## adjust 'text_tokens' to subtract cached tokens if ( @@ -1374,7 +1575,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): annotations: List[ChatCompletionAnnotation] = [] - for metadata in grounding_metadata: # Extract groundingSupports - these map text segments to sources grounding_supports = metadata.get("groundingSupports", []) @@ -1395,23 +1595,23 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): segment = support.get("segment", {}) start_index = segment.get("startIndex") end_index = segment.get("endIndex") - + # Get the chunk indices for this support chunk_indices = support.get("groundingChunkIndices", []) - + if start_index is not None and end_index is not None and chunk_indices: # Use the first chunk's URL for the annotation first_chunk_idx = chunk_indices[0] if first_chunk_idx in chunk_to_uri_map: uri_info = chunk_to_uri_map[first_chunk_idx] - + url_citation: ChatCompletionAnnotationURLCitation = { "start_index": start_index, "end_index": end_index, "url": uri_info["url"], "title": uri_info["title"], } - + annotation: ChatCompletionAnnotation = { "type": "url_citation", "url_citation": url_citation, @@ -1451,6 +1651,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): tools: Optional[List[ChatCompletionToolCallChunk]] = [] functions: Optional[ChatCompletionToolCallFunctionChunk] = None thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None + reasoning_content: Optional[str] = None for idx, candidate in enumerate(_candidates): if "content" not in candidate: @@ -1511,9 +1712,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): chat_completion_message["reasoning_content"] = reasoning_content if candidate_grounding_metadata: - annotations = VertexGeminiConfig._convert_grounding_metadata_to_annotations( - grounding_metadata=candidate_grounding_metadata, - content_text=content, + annotations = ( + VertexGeminiConfig._convert_grounding_metadata_to_annotations( + grounding_metadata=candidate_grounding_metadata, + content_text=content, + ) ) if annotations: chat_completion_message["annotations"] = annotations # type: ignore @@ -1541,6 +1744,22 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if thinking_blocks is not None: chat_completion_message["thinking_blocks"] = thinking_blocks # type: ignore + # Convert thinking_blocks to reasoning_content for streaming + # This ensures reasoning_content is available in streaming responses + if ( + isinstance(model_response, ModelResponseStream) + and reasoning_content is None + ): + reasoning_content_parts = [] + for block in thinking_blocks: + thinking_text = block.get("thinking") + if thinking_text: + reasoning_content_parts.append(thinking_text) + + if reasoning_content_parts: + reasoning_content = "\n".join(reasoning_content_parts) + chat_completion_message["reasoning_content"] = reasoning_content + if isinstance(model_response, ModelResponseStream): choice = VertexGeminiConfig._create_streaming_choice( chat_completion_message=chat_completion_message, @@ -1718,9 +1937,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return model_response def _transform_messages( - self, messages: List[AllMessageValues] + self, messages: List[AllMessageValues], model: Optional[str] = None ) -> List[ContentType]: - return _gemini_convert_messages_with_history(messages=messages) + return _gemini_convert_messages_with_history(messages=messages, model=model) def get_error_class( self, error_message: str, status_code: int, headers: Union[Dict, httpx.Headers] @@ -1779,7 +1998,9 @@ async def make_call( ) try: - response = await client.post(api_base, headers=headers, data=data, stream=True, logging_obj=logging_obj) + response = await client.post( + api_base, headers=headers, data=data, stream=True, logging_obj=logging_obj + ) response.raise_for_status() except httpx.HTTPStatusError as e: exception_string = str(await e.response.aread()) @@ -1826,7 +2047,9 @@ def make_sync_call( if client is None: client = HTTPHandler() # Create a new client if none provided - response = client.post(api_base, headers=headers, data=data, stream=True, logging_obj=logging_obj) + response = client.post( + api_base, headers=headers, data=data, stream=True, logging_obj=logging_obj + ) if response.status_code != 200 and response.status_code != 201: raise VertexAIError( @@ -1881,7 +2104,6 @@ class VertexLLM(VertexBase): gemini_api_key: Optional[str] = None, extra_headers: Optional[dict] = None, ) -> CustomStreamWrapper: - should_use_v1beta1_features = self.is_using_v1beta1_features( optional_params=optional_params ) @@ -1918,8 +2140,8 @@ class VertexLLM(VertexBase): **data, vertex_project=vertex_project, vertex_location=vertex_location, - vertex_auth_header=auth_header) # type: ignore - + vertex_auth_header=auth_header, + ) # type: ignore ## LOGGING logging_obj.pre_call( @@ -2012,7 +2234,8 @@ class VertexLLM(VertexBase): **data, vertex_project=vertex_project, vertex_location=vertex_location, - vertex_auth_header=auth_header) # type: ignore + vertex_auth_header=auth_header, + ) # type: ignore _async_client_params = {} if timeout: @@ -2036,7 +2259,10 @@ class VertexLLM(VertexBase): try: response = await client.post( - api_base, headers=headers, json=cast(dict, request_body), logging_obj=logging_obj + api_base, + headers=headers, + json=cast(dict, request_body), + logging_obj=logging_obj, ) # type: ignore response.raise_for_status() except httpx.HTTPStatusError as err: @@ -2190,9 +2416,10 @@ class VertexLLM(VertexBase): ## TRANSFORMATION ## data = sync_transform_request_body( **transform_request_params, - vertex_project=vertex_project, + vertex_project=vertex_project, vertex_location=vertex_location, - vertex_auth_header=auth_header) + vertex_auth_header=auth_header, + ) ## LOGGING logging_obj.pre_call( diff --git a/litellm/llms/vertex_ai/image_edit/__init__.py b/litellm/llms/vertex_ai/image_edit/__init__.py new file mode 100644 index 00000000000..44914e861a7 --- /dev/null +++ b/litellm/llms/vertex_ai/image_edit/__init__.py @@ -0,0 +1,39 @@ +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.vertex_ai.common_utils import VertexAIModelRoute, get_vertex_ai_model_route + +from .cost_calculator import cost_calculator +from .vertex_gemini_transformation import VertexAIGeminiImageEditConfig +from .vertex_imagen_transformation import VertexAIImagenImageEditConfig + +__all__ = [ + "VertexAIGeminiImageEditConfig", + "VertexAIImagenImageEditConfig", + "get_vertex_ai_image_edit_config", + "cost_calculator" +] + + +def get_vertex_ai_image_edit_config(model: str) -> BaseImageEditConfig: + """ + Get the appropriate image edit config for a Vertex AI model. + + Routes to the correct transformation class based on the model type: + - Gemini models use generateContent API (VertexAIGeminiImageEditConfig) + - Imagen models use predict API (VertexAIImagenImageEditConfig) + + Args: + model: The model name (e.g., "gemini-2.5-flash", "imagegeneration@006") + + Returns: + BaseImageEditConfig: The appropriate configuration class + """ + # Determine the model route + model_route = get_vertex_ai_model_route(model) + + if model_route == VertexAIModelRoute.GEMINI: + # Gemini models use generateContent API + return VertexAIGeminiImageEditConfig() + else: + # Default to Imagen for other models (imagegeneration, etc.) + # This includes NON_GEMINI models like imagegeneration@006 + return VertexAIImagenImageEditConfig() diff --git a/litellm/llms/vertex_ai/image_edit/cost_calculator.py b/litellm/llms/vertex_ai/image_edit/cost_calculator.py new file mode 100644 index 00000000000..b346622a336 --- /dev/null +++ b/litellm/llms/vertex_ai/image_edit/cost_calculator.py @@ -0,0 +1,34 @@ +""" +Vertex AI Image Edit Cost Calculator +""" + +from typing import Any + +import litellm +from litellm.types.utils import ImageResponse + + +def cost_calculator( + model: str, + image_response: Any, +) -> float: + """ + Vertex AI image edit cost calculator. + + Mirrors image generation pricing: charge per returned image based on + model metadata (`output_cost_per_image`). + """ + model_info = litellm.get_model_info( + model=model, + custom_llm_provider="vertex_ai", + ) + + output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0 + + if not isinstance(image_response, ImageResponse): + raise ValueError( + f"image_response must be of type ImageResponse got type={type(image_response)}" + ) + + num_images = len(image_response.data or []) + return output_cost_per_image * num_images diff --git a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py new file mode 100644 index 00000000000..469340f6bba --- /dev/null +++ b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py @@ -0,0 +1,263 @@ +import base64 +import json +import os +from io import BufferedReader, BytesIO +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast + +import httpx +from httpx._types import RequestFiles + +import litellm + +from litellm.images.utils import ImageEditRequestUtils +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM +from litellm.secret_managers.main import get_secret_str +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import FileTypes, ImageObject, ImageResponse, OpenAIImage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM): + """ + Vertex AI Gemini Image Edit Configuration + + Uses generateContent API for Gemini models on Vertex AI + """ + SUPPORTED_PARAMS: List[str] = ["size"] + + def __init__(self) -> None: + BaseImageEditConfig.__init__(self) + VertexLLM.__init__(self) + + def get_supported_openai_params(self, model: str) -> List[str]: + return list(self.SUPPORTED_PARAMS) + + def map_openai_params( + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict[str, Any]: + supported_params = self.get_supported_openai_params(model) + filtered_params = { + key: value + for key, value in image_edit_optional_params.items() + if key in supported_params + } + + mapped_params: Dict[str, Any] = {} + + if "size" in filtered_params: + mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio( + filtered_params["size"] # type: ignore[arg-type] + ) + + return mapped_params + + def _resolve_vertex_project(self) -> Optional[str]: + return ( + getattr(self, "_vertex_project", None) + or os.environ.get("VERTEXAI_PROJECT") + or getattr(litellm, "vertex_project", None) + or get_secret_str("VERTEXAI_PROJECT") + ) + + def _resolve_vertex_location(self) -> Optional[str]: + return ( + getattr(self, "_vertex_location", None) + or os.environ.get("VERTEXAI_LOCATION") + or os.environ.get("VERTEX_LOCATION") + or getattr(litellm, "vertex_location", None) + or get_secret_str("VERTEXAI_LOCATION") + or get_secret_str("VERTEX_LOCATION") + ) + + def _resolve_vertex_credentials(self) -> Optional[str]: + return ( + getattr(self, "_vertex_credentials", None) + or os.environ.get("VERTEXAI_CREDENTIALS") + or getattr(litellm, "vertex_credentials", None) + or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + or get_secret_str("VERTEXAI_CREDENTIALS") + ) + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + headers = headers or {} + vertex_project = self._resolve_vertex_project() + vertex_credentials = self._resolve_vertex_credentials() + access_token, _ = self._ensure_access_token( + credentials=vertex_credentials, + project_id=vertex_project, + custom_llm_provider="vertex_ai", + ) + return self.set_headers(access_token, headers) + + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Get the complete URL for Vertex AI Gemini generateContent API + """ + vertex_project = self._resolve_vertex_project() + vertex_location = self._resolve_vertex_location() + + if not vertex_project or not vertex_location: + raise ValueError("vertex_project and vertex_location are required for Vertex AI") + + # Use the model name as provided, handling vertex_ai prefix + model_name = model + if model.startswith("vertex_ai/"): + model_name = model.replace("vertex_ai/", "") + + if api_base: + base_url = api_base.rstrip("/") + else: + base_url = f"https://{vertex_location}-aiplatform.googleapis.com" + + return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:generateContent" + + def transform_image_edit_request( # type: ignore[override] + self, + model: str, + prompt: str, + image: FileTypes, + image_edit_optional_request_params: Dict[str, Any], + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[Dict[str, Any], Optional[RequestFiles]]: + inline_parts = self._prepare_inline_image_parts(image) + if not inline_parts: + raise ValueError("Vertex AI Gemini image edit requires at least one image.") + + # Correct format for Vertex AI Gemini image editing + contents = { + "role": "USER", + "parts": inline_parts + [{"text": prompt}] + } + + request_body: Dict[str, Any] = {"contents": contents} + + # Generation config with proper structure for image editing + generation_config: Dict[str, Any] = { + "response_modalities": ["IMAGE"] + } + + # Add image-specific configuration + image_config: Dict[str, Any] = {} + if "aspectRatio" in image_edit_optional_request_params: + image_config["aspect_ratio"] = image_edit_optional_request_params["aspectRatio"] + + if image_config: + generation_config["image_config"] = image_config + + request_body["generationConfig"] = generation_config + + payload: Any = json.dumps(request_body) + empty_files = cast(RequestFiles, []) + return cast(Tuple[Dict[str, Any], Optional[RequestFiles]], (payload, empty_files)) + + def transform_image_edit_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: Any, + ) -> ImageResponse: + model_response = ImageResponse() + try: + response_json = raw_response.json() + except Exception as exc: + raise self.get_error_class( + error_message=f"Error transforming image edit response: {exc}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + candidates = response_json.get("candidates", []) + data_list: List[ImageObject] = [] + + for candidate in candidates: + content = candidate.get("content", {}) + parts = content.get("parts", []) + for part in parts: + inline_data = part.get("inlineData") + if inline_data and inline_data.get("data"): + data_list.append( + ImageObject( + b64_json=inline_data["data"], + url=None, + ) + ) + + model_response.data = cast(List[OpenAIImage], data_list) + return model_response + + def _map_size_to_aspect_ratio(self, size: str) -> str: + """Map OpenAI size format to Gemini aspect ratio format""" + aspect_ratio_map = { + "1024x1024": "1:1", + "1792x1024": "16:9", + "1024x1792": "9:16", + "1280x896": "4:3", + "896x1280": "3:4", + } + return aspect_ratio_map.get(size, "1:1") + + def _prepare_inline_image_parts( + self, image: Union[FileTypes, List[FileTypes]] + ) -> List[Dict[str, Any]]: + images: List[FileTypes] + if isinstance(image, list): + images = image + else: + images = [image] + + inline_parts: List[Dict[str, Any]] = [] + for img in images: + if img is None: + continue + + mime_type = ImageEditRequestUtils.get_image_content_type(img) + image_bytes = self._read_all_bytes(img) + inline_parts.append( + { + "inlineData": { + "mimeType": mime_type, + "data": base64.b64encode(image_bytes).decode("utf-8"), + } + } + ) + + return inline_parts + + def _read_all_bytes(self, image: FileTypes) -> bytes: + if isinstance(image, bytes): + return image + if isinstance(image, BytesIO): + current_pos = image.tell() + image.seek(0) + data = image.read() + image.seek(current_pos) + return data + if isinstance(image, BufferedReader): + current_pos = image.tell() + image.seek(0) + data = image.read() + image.seek(current_pos) + return data + raise ValueError("Unsupported image type for Vertex AI Gemini image edit.") diff --git a/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py new file mode 100644 index 00000000000..ad650e38499 --- /dev/null +++ b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py @@ -0,0 +1,353 @@ +import base64 +import json +import os +from io import BufferedRandom, BufferedReader, BytesIO +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast + +import httpx +from httpx._types import RequestFiles + +import litellm + +from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM +from litellm.secret_managers.main import get_secret_str +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import FileTypes, ImageObject, ImageResponse, OpenAIImage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): + """ + Vertex AI Imagen Image Edit Configuration + + Uses predict API for Imagen models on Vertex AI + """ + SUPPORTED_PARAMS: List[str] = ["n", "size", "mask"] + + def __init__(self) -> None: + BaseImageEditConfig.__init__(self) + VertexLLM.__init__(self) + + def get_supported_openai_params(self, model: str) -> List[str]: + return list(self.SUPPORTED_PARAMS) + + def map_openai_params( + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict[str, Any]: + supported_params = self.get_supported_openai_params(model) + filtered_params = { + key: value + for key, value in image_edit_optional_params.items() + if key in supported_params + } + + mapped_params: Dict[str, Any] = {} + + # Map OpenAI parameters to Imagen format + if "n" in filtered_params: + mapped_params["sampleCount"] = filtered_params["n"] + + if "size" in filtered_params: + mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio( + filtered_params["size"] # type: ignore[arg-type] + ) + + if "mask" in filtered_params: + mapped_params["mask"] = filtered_params["mask"] + + return mapped_params + + def _resolve_vertex_project(self) -> Optional[str]: + return ( + getattr(self, "_vertex_project", None) + or os.environ.get("VERTEXAI_PROJECT") + or getattr(litellm, "vertex_project", None) + or get_secret_str("VERTEXAI_PROJECT") + ) + + def _resolve_vertex_location(self) -> Optional[str]: + return ( + getattr(self, "_vertex_location", None) + or os.environ.get("VERTEXAI_LOCATION") + or os.environ.get("VERTEX_LOCATION") + or getattr(litellm, "vertex_location", None) + or get_secret_str("VERTEXAI_LOCATION") + or get_secret_str("VERTEX_LOCATION") + ) + + def _resolve_vertex_credentials(self) -> Optional[str]: + return ( + getattr(self, "_vertex_credentials", None) + or os.environ.get("VERTEXAI_CREDENTIALS") + or getattr(litellm, "vertex_credentials", None) + or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + or get_secret_str("VERTEXAI_CREDENTIALS") + ) + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + headers = headers or {} + vertex_project = self._resolve_vertex_project() + vertex_credentials = self._resolve_vertex_credentials() + access_token, _ = self._ensure_access_token( + credentials=vertex_credentials, + project_id=vertex_project, + custom_llm_provider="vertex_ai", + ) + return self.set_headers(access_token, headers) + + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Get the complete URL for Vertex AI Imagen predict API + """ + vertex_project = self._resolve_vertex_project() + vertex_location = self._resolve_vertex_location() + + if not vertex_project or not vertex_location: + raise ValueError("vertex_project and vertex_location are required for Vertex AI") + + # Use the model name as provided, handling vertex_ai prefix + model_name = model + if model.startswith("vertex_ai/"): + model_name = model.replace("vertex_ai/", "") + + if api_base: + base_url = api_base.rstrip("/") + else: + base_url = f"https://{vertex_location}-aiplatform.googleapis.com" + + return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:predict" + + def transform_image_edit_request( # type: ignore[override] + self, + model: str, + prompt: str, + image: FileTypes, + image_edit_optional_request_params: Dict[str, Any], + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[Dict[str, Any], Optional[RequestFiles]]: + # Prepare reference images in the correct Imagen format + reference_images = self._prepare_reference_images(image, image_edit_optional_request_params) + if not reference_images: + raise ValueError("Vertex AI Imagen image edit requires at least one reference image.") + + # Correct Imagen instances format + instances = [ + { + "prompt": prompt, + "referenceImages": reference_images + } + ] + + # Extract OpenAI parameters and set sensible defaults for Vertex AI-specific parameters + sample_count = image_edit_optional_request_params.get("sampleCount", 1) + # Use sensible defaults for Vertex AI-specific parameters (not exposed to users) + edit_mode = "EDIT_MODE_INPAINT_INSERTION" # Default edit mode + base_steps = 50 # Default number of steps + + # Imagen parameters with correct structure + parameters = { + "sampleCount": sample_count, + "editMode": edit_mode, + "editConfig": { + "baseSteps": base_steps + } + } + + # Set default values for Vertex AI-specific parameters (not configurable by users via OpenAI API) + parameters["guidanceScale"] = 7.5 # Default guidance scale + parameters["seed"] = None # Let Vertex AI choose random seed + + request_body: Dict[str, Any] = { + "instances": instances, + "parameters": parameters + } + + payload: Any = json.dumps(request_body) + empty_files = cast(RequestFiles, []) + return cast(Tuple[Dict[str, Any], Optional[RequestFiles]], (payload, empty_files)) + + def transform_image_edit_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: Any, + ) -> ImageResponse: + model_response = ImageResponse() + try: + response_json = raw_response.json() + except Exception as exc: + raise self.get_error_class( + error_message=f"Error transforming image edit response: {exc}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + predictions = response_json.get("predictions", []) + data_list: List[ImageObject] = [] + + for prediction in predictions: + # Imagen returns images as bytesBase64Encoded + if "bytesBase64Encoded" in prediction: + data_list.append( + ImageObject( + b64_json=prediction["bytesBase64Encoded"], + url=None, + ) + ) + + model_response.data = cast(List[OpenAIImage], data_list) + return model_response + + def _map_size_to_aspect_ratio(self, size: str) -> str: + """Map OpenAI size format to Imagen aspect ratio format""" + aspect_ratio_map = { + "1024x1024": "1:1", + "1792x1024": "16:9", + "1024x1792": "9:16", + "1280x896": "4:3", + "896x1280": "3:4", + } + return aspect_ratio_map.get(size, "1:1") + + def _prepare_reference_images( + self, image: Union[FileTypes, List[FileTypes]], + image_edit_optional_request_params: Dict[str, Any] + ) -> List[Dict[str, Any]]: + """ + Prepare reference images in the correct Imagen API format + """ + images: List[FileTypes] + if isinstance(image, list): + images = image + else: + images = [image] + + reference_images: List[Dict[str, Any]] = [] + + for idx, img in enumerate(images): + if img is None: + continue + + image_bytes = self._read_all_bytes(img) + base64_data = base64.b64encode(image_bytes).decode("utf-8") + + # Create reference image structure + reference_image = { + "referenceType": "REFERENCE_TYPE_RAW", + "referenceId": idx + 1, + "referenceImage": { + "bytesBase64Encoded": base64_data + } + } + + reference_images.append(reference_image) + + # Handle mask image if provided (for inpainting) + mask_image = image_edit_optional_request_params.get("mask") + if mask_image is not None: + mask_bytes = self._read_all_bytes(mask_image) + mask_base64 = base64.b64encode(mask_bytes).decode("utf-8") + + mask_reference = { + "referenceType": "REFERENCE_TYPE_MASK", + "referenceId": len(reference_images) + 1, + "referenceImage": { + "bytesBase64Encoded": mask_base64 + }, + "maskImageConfig": { + "maskMode": "MASK_MODE_USER_PROVIDED", + "dilation": 0.03 # Default dilation value (not configurable via OpenAI API) + } + } + reference_images.append(mask_reference) + + return reference_images + + def _read_all_bytes( + self, image: Any, depth: int = 0, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH + ) -> bytes: + if depth > max_depth: + raise ValueError( + f"Max recursion depth {max_depth} reached while reading image bytes for Vertex AI Imagen image edit." + ) + + if isinstance(image, (list, tuple)): + for item in image: + if item is not None: + return self._read_all_bytes(item, depth=depth + 1, max_depth=max_depth) + raise ValueError("Unsupported image type for Vertex AI Imagen image edit.") + + if isinstance(image, dict): + for key in ("data", "bytes", "content"): + if key in image and image[key] is not None: + value = image[key] + if isinstance(value, str): + try: + return base64.b64decode(value) + except Exception: + continue + return self._read_all_bytes(value, depth=depth + 1, max_depth=max_depth) + if "path" in image: + return self._read_all_bytes(image["path"], depth=depth + 1, max_depth=max_depth) + + if isinstance(image, bytes): + return image + if isinstance(image, bytearray): + return bytes(image) + if isinstance(image, BytesIO): + current_pos = image.tell() + image.seek(0) + data = image.read() + image.seek(current_pos) + return data + if isinstance(image, (BufferedReader, BufferedRandom)): + stream_pos: Optional[int] = None + try: + stream_pos = image.tell() + except Exception: + stream_pos = None + if stream_pos is not None: + image.seek(0) + data = image.read() + if stream_pos is not None: + image.seek(stream_pos) + return data + if isinstance(image, (str, Path)): + path_obj = Path(image) + if not path_obj.exists(): + raise ValueError( + f"Mask/image path does not exist for Vertex AI Imagen image edit: {path_obj}" + ) + return path_obj.read_bytes() + if hasattr(image, "read"): + data = image.read() + if isinstance(data, str): + data = data.encode("utf-8") + return data + raise ValueError( + f"Unsupported image type for Vertex AI Imagen image edit. Got type={type(image)}" + ) diff --git a/litellm/main.py b/litellm/main.py index 412d7f1c38e..8cfc1ab1ba3 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -105,7 +105,7 @@ from litellm.utils import ( ProviderConfigManager, Usage, _get_model_info_helper, - add_openai_metadata, + get_requester_metadata, add_provider_specific_params_to_optional_params, async_mock_completion_streaming_obj, convert_to_model_response_object, @@ -2086,10 +2086,12 @@ def completion( # type: ignore # noqa: PLR0915 if extra_headers is not None: optional_params["extra_headers"] = extra_headers - if litellm.enable_preview_features: - metadata_payload = add_openai_metadata(metadata) - if metadata_payload is not None: - optional_params["metadata"] = metadata_payload + if ( + litellm.enable_preview_features and metadata is not None + ): # [PREVIEW] allow metadata to be passed to OPENAI + openai_metadata = get_requester_metadata(metadata) + if openai_metadata is not None: + optional_params["metadata"] = openai_metadata ## LOAD CONFIG - if set config = litellm.OpenAIConfig.get_config() @@ -4822,6 +4824,22 @@ def embedding( # noqa: PLR0915 print_verbose=print_verbose, litellm_params=litellm_params_dict, ) + elif custom_llm_provider == "snowflake": + api_key = api_key or get_secret_str("SNOWFLAKE_JWT") + response = base_llm_http_handler.embedding( + model=model, + input=input, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + logging_obj=logging, + timeout=timeout, + model_response=EmbeddingResponse(), + optional_params=optional_params, + client=client, + aembedding=aembedding, + litellm_params={}, + ) else: raise LiteLLMUnknownProvider( model=model, custom_llm_provider=custom_llm_provider @@ -5500,6 +5518,7 @@ def transcription( atranscription = kwargs.pop("atranscription", False) litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore extra_headers = kwargs.get("extra_headers", None) + shared_session = kwargs.get("shared_session", None) kwargs.pop("tags", []) non_default_params = get_non_default_transcription_params(kwargs) @@ -5637,6 +5656,7 @@ def transcription( api_key=api_key, provider_config=provider_config, litellm_params=litellm_params_dict, + shared_session=shared_session, ) elif provider_config is not None: response = base_llm_http_handler.audio_transcriptions( @@ -5663,6 +5683,7 @@ def transcription( custom_llm_provider=custom_llm_provider, headers={}, provider_config=provider_config, + shared_session=shared_session, ) # Calculate and add duration if response is missing it diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 367f788dd34..8f71c2985be 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -462,39 +462,45 @@ "cache_creation_input_token_cost": 1.25e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, - "litellm_provider": "bedrock", + "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", "supports_assistant_prefill": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 }, "anthropic.claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, - "litellm_provider": "bedrock", + "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", "supports_assistant_prefill": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 }, "anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -702,6 +708,36 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, + "anthropic.claude-sonnet-4-5-20250929-v1:0": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "input_cost_per_token_above_200k_tokens": 6e-06, + "output_cost_per_token_above_200k_tokens": 2.25e-05, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, "anthropic.claude-v1": { "input_cost_per_token": 8e-06, "litellm_provider": "bedrock", @@ -930,20 +966,23 @@ "cache_creation_input_token_cost": 1.375e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, - "litellm_provider": "bedrock", + "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", "supports_assistant_prefill": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 }, "apac.anthropic.claude-3-sonnet-20240229-v1:0": { "input_cost_per_token": 3e-06, @@ -1224,6 +1263,228 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "azure/eu/gpt-5-2025-08-07": { + "cache_read_input_token_cost": 1.375e-07, + "input_cost_per_token": 1.375e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/eu/gpt-5-mini-2025-08-07": { + "cache_read_input_token_cost": 2.75e-08, + "input_cost_per_token": 2.75e-07, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/eu/gpt-5.1": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.38e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/eu/gpt-5.1-chat": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.38e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/eu/gpt-5.1-codex": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.38e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 1.1e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/eu/gpt-5.1-codex-mini": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 2.75e-07, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 2.2e-06, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/eu/gpt-5-nano-2025-08-07": { + "cache_read_input_token_cost": 5.5e-09, + "input_cost_per_token": 5.5e-08, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-07, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/eu/o1-2024-12-17": { "cache_read_input_token_cost": 8.25e-06, "input_cost_per_token": 1.65e-05, @@ -1366,6 +1627,132 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/global/gpt-5.1": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/global/gpt-5.1-chat": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/global/gpt-5.1-codex": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/global/gpt-5.1-codex-mini": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_token": 2.5e-07, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 2e-06, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/gpt-3.5-turbo": { "input_cost_per_token": 5e-07, "litellm_provider": "azure", @@ -1882,6 +2269,68 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/gpt-audio-2025-08-28": { + "input_cost_per_audio_token": 4e-05, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_audio_token": 8e-05, + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "azure/gpt-audio-mini-2025-10-06": { + "input_cost_per_audio_token": 1e-05, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, "azure/gpt-4o-audio-preview-2024-12-17": { "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 2.5e-06, @@ -1995,6 +2444,70 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "azure/gpt-realtime-2025-08-28": { + "cache_creation_input_audio_token_cost": 4e-06, + "cache_read_input_token_cost": 4e-06, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1.6e-05, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/gpt-realtime-mini-2025-10-06": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "azure/gpt-4o-mini-transcribe": { "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 1.25e-06, @@ -2082,6 +2595,155 @@ "/v1/audio/transcriptions" ] }, + "azure/gpt-4o-transcribe-diarize": { + "input_cost_per_audio_token": 6e-06, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "azure", + "max_input_tokens": 16000, + "max_output_tokens": 2000, + "mode": "audio_transcription", + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, + "azure/gpt-5.1-2025-11-13": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true + }, + "azure/gpt-5.1-chat-2025-11-13": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_native_streaming": true, + "supports_parallel_function_calling": false, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": false, + "supports_vision": true + }, + "azure/gpt-5.1-codex-2025-11-13": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/gpt-5.1-codex-mini-2025-11-13": { + "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_priority": 4.5e-08, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_priority": 4.5e-07, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 2e-06, + "output_cost_per_token_priority": 3.6e-06, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/gpt-5": { "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, @@ -2398,6 +3060,132 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/gpt-5.1": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/gpt-5.1-chat": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/gpt-5.1-codex": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/gpt-5.1-codex-mini": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_token": 2.5e-07, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 2e-06, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/gpt-image-1": { "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", @@ -2738,14 +3526,14 @@ }, "azure/o3-2025-04-16": { "deprecation_date": "2026-04-16", - "cache_read_input_token_cost": 2.5e-06, - "input_cost_per_token": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", - "output_cost_per_token": 4e-05, + "output_cost_per_token": 8e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -3004,6 +3792,107 @@ "litellm_provider": "azure", "mode": "audio_speech" }, + "azure/us/gpt-4.1-2025-04-14": { + "deprecation_date": "2026-11-04", + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "litellm_provider": "azure", + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "output_cost_per_token_batches": 4.4e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": false + }, + "azure/us/gpt-4.1-mini-2025-04-14": { + "deprecation_date": "2026-11-04", + "cache_read_input_token_cost": 1.1e-07, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_batches": 2.2e-07, + "litellm_provider": "azure", + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.76e-06, + "output_cost_per_token_batches": 8.8e-07, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": false + }, + "azure/us/gpt-4.1-nano-2025-04-14": { + "deprecation_date": "2026-11-04", + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_token": 1.1e-07, + "input_cost_per_token_batches": 6e-08, + "litellm_provider": "azure", + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4.4e-07, + "output_cost_per_token_batches": 2.2e-07, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/us/gpt-4o-2024-08-06": { "deprecation_date": "2026-02-27", "cache_read_input_token_cost": 1.375e-06, @@ -3118,6 +4007,228 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "azure/us/gpt-5-2025-08-07": { + "cache_read_input_token_cost": 1.375e-07, + "input_cost_per_token": 1.375e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/us/gpt-5-mini-2025-08-07": { + "cache_read_input_token_cost": 2.75e-08, + "input_cost_per_token": 2.75e-07, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/us/gpt-5-nano-2025-08-07": { + "cache_read_input_token_cost": 5.5e-09, + "input_cost_per_token": 5.5e-08, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-07, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/us/gpt-5.1": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.38e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/us/gpt-5.1-chat": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.38e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/us/gpt-5.1-codex": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.38e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 1.1e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/us/gpt-5.1-codex-mini": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 2.75e-07, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 2.2e-06, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/us/o1-2024-12-17": { "cache_read_input_token_cost": 8.25e-06, "input_cost_per_token": 1.65e-05, @@ -3163,6 +4274,36 @@ "supports_prompt_caching": true, "supports_vision": false }, + "azure/us/o3-2025-04-16": { + "deprecation_date": "2026-04-16", + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/us/o3-mini-2025-01-31": { "cache_read_input_token_cost": 6.05e-07, "input_cost_per_token": 1.21e-06, @@ -3179,6 +4320,23 @@ "supports_tool_choice": true, "supports_vision": false }, + "azure/us/o4-mini-2025-04-16": { + "cache_read_input_token_cost": 3.1e-07, + "input_cost_per_token": 1.21e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 4.84e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/whisper-1": { "input_cost_per_second": 0.0001, "litellm_provider": "azure", @@ -4445,6 +5603,20 @@ "supports_tool_choice": true, "supports_vision": true }, + "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": { + "input_cost_per_token": 3.3e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.65e-05, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", @@ -4572,6 +5744,20 @@ "supports_tool_choice": true, "supports_vision": true }, + "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": { + "input_cost_per_token": 3.3e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.65e-05, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", @@ -4778,7 +5964,7 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "cerebras/openai/gpt-oss-120b": { + "cerebras/gpt-oss-120b": { "input_cost_per_token": 2.5e-07, "litellm_provider": "cerebras", "max_input_tokens": 131072, @@ -5302,6 +6488,31 @@ "supports_web_search": true, "tool_use_system_prompt_tokens": 346 }, + "claude-sonnet-4-5-20250929-v1:0": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "input_cost_per_token_above_200k_tokens": 6e-06, + "output_cost_per_token_above_200k_tokens": 2.25e-05, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, "claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, @@ -8257,20 +9468,23 @@ "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "deprecation_date": "2026-10-15", - "litellm_provider": "bedrock", + "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", "supports_assistant_prefill": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 }, "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -8751,6 +9965,18 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "fireworks_ai/accounts/fireworks/models/deepseek-v3p1-terminus": { + "input_cost_per_token": 5.6e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.68e-06, + "source": "https://fireworks.ai/pricing", + "supports_response_schema": true, + "supports_tool_choice": true + }, "fireworks_ai/accounts/fireworks/models/firefunction-v2": { "input_cost_per_token": 9e-07, "litellm_provider": "fireworks_ai", @@ -8829,6 +10055,20 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "fireworks_ai/accounts/fireworks/models/kimi-k2-thinking": { + "input_cost_per_token": 6e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://fireworks.ai/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_web_search": true + }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct": { "input_cost_per_token": 3e-06, "litellm_provider": "fireworks_ai", @@ -10210,6 +11450,39 @@ "supports_web_search": true, "tpm": 8000000 }, + "gemini-3-pro-image-preview": { + "input_cost_per_image": 0.0011, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 65536, + "mode": "image_generation", + "output_cost_per_image": 0.134, + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_batches": 6e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true + }, "gemini-2.5-flash-lite": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 5e-07, @@ -10615,6 +11888,102 @@ "supports_vision": true, "supports_web_search": true }, + "gemini-3-pro-preview": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "vertex_ai-language-models", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_above_200k_tokens": 1.8e-05, + "output_cost_per_token_batches": 6e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true + }, + "vertex_ai/gemini-3-pro-preview": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "vertex_ai", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_above_200k_tokens": 1.8e-05, + "output_cost_per_token_batches": 6e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true + }, "gemini-2.5-pro-exp-03-25": { "cache_read_input_token_cost": 3.125e-07, "input_cost_per_token": 1.25e-06, @@ -11818,6 +13187,41 @@ "supports_web_search": true, "tpm": 8000000 }, + "gemini/gemini-3-pro-image-preview": { + "input_cost_per_image": 0.0011, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "gemini", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 65536, + "mode": "image_generation", + "output_cost_per_image": 0.134, + "output_cost_per_token": 1.2e-05, + "rpm": 1000, + "tpm": 4000000, + "output_cost_per_token_batches": 6e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true + }, "gemini/gemini-2.5-flash-lite": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 5e-07, @@ -12274,6 +13678,55 @@ "supports_web_search": true, "tpm": 800000 }, + "gemini/gemini-3-pro-preview": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_above_200k_tokens": 1.8e-05, + "output_cost_per_token_batches": 6e-06, + "rpm": 2000, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 800000 + }, "gemini/gemini-2.5-pro-exp-03-25": { "cache_read_input_token_cost": 0.0, "input_cost_per_token": 0.0, @@ -12617,7 +14070,7 @@ "supports_audio_output": false, "supports_function_calling": true, "supports_response_schema": true, - "supports_system_messages": true, + "supports_system_messages": false, "supports_tool_choice": true, "supports_vision": true }, @@ -12826,12 +14279,13 @@ "input_cost_per_token": 1.1e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", "supports_assistant_prefill": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -15593,20 +17047,23 @@ "cache_creation_input_token_cost": 1.375e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, - "litellm_provider": "bedrock", + "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", "supports_assistant_prefill": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 }, "lambda_ai/deepseek-llama3.3-70b": { "input_cost_per_token": 2e-07, @@ -18671,6 +20128,53 @@ "supports_tool_choice": true, "supports_vision": true }, + "openrouter/google/gemini-3-pro-preview": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "openrouter", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_above_200k_tokens": 1.8e-05, + "output_cost_per_token_batches": 6e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true + }, "openrouter/google/gemini-pro-1.5": { "input_cost_per_image": 0.00265, "input_cost_per_token": 2.5e-06, @@ -21260,6 +22764,20 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true }, + "together_ai/zai-org/GLM-4.6": { + "input_cost_per_token": 0.6e-06, + "litellm_provider": "together_ai", + "max_input_tokens": 200000, + "max_output_tokens": 200000, + "max_tokens": 200000, + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "source": "https://www.together.ai/models/glm-4-6", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "together_ai/moonshotai/Kimi-K2-Instruct-0905": { "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", @@ -21384,20 +22902,23 @@ "cache_creation_input_token_cost": 1.375e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, - "litellm_provider": "bedrock", + "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", "supports_assistant_prefill": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 }, "us.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -21555,14 +23076,16 @@ "input_cost_per_token": 1.1e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "supports_assistant_prefill": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, @@ -23189,6 +24712,68 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "vertex_ai/gemini-2.5-flash-image": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "vertex_ai-language-models", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "max_pdf_size_mb": 30, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "image_generation", + "output_cost_per_image": 0.039, + "output_cost_per_reasoning_token": 2.5e-06, + "output_cost_per_token": 2.5e-06, + "rpm": 100000, + "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/image-generation#edit-an-image", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": false, + "tpm": 8000000 + }, + "vertex_ai/gemini-3-pro-image-preview": { + "input_cost_per_image": 0.0011, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 65536, + "mode": "image_generation", + "output_cost_per_image": 0.134, + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_batches": 6e-06, + "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image" + }, "vertex_ai/imagegeneration@006": { "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", @@ -23213,6 +24798,12 @@ "output_cost_per_image": 0.04, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, + "vertex_ai/imagen-3.0-capability-001": { + "litellm_provider": "vertex_ai-image-models", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/image/edit-insert-objects" + }, "vertex_ai/imagen-4.0-fast-generate-001": { "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", @@ -23708,7 +25299,7 @@ "max_input_tokens": 1024, "max_tokens": 1024, "mode": "video_generation", - "output_cost_per_second": 0.4, + "output_cost_per_second": 0.15, "source": "https://ai.google.dev/gemini-api/docs/video", "supported_modalities": [ "text" @@ -23722,7 +25313,35 @@ "max_input_tokens": 1024, "max_tokens": 1024, "mode": "video_generation", - "output_cost_per_second": 0.75, + "output_cost_per_second": 0.4, + "source": "https://ai.google.dev/gemini-api/docs/video", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "vertex_ai/veo-3.0-fast-generate-001": { + "litellm_provider": "vertex_ai-video-models", + "max_input_tokens": 1024, + "max_tokens": 1024, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "source": "https://ai.google.dev/gemini-api/docs/video", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "vertex_ai/veo-3.0-generate-001": { + "litellm_provider": "vertex_ai-video-models", + "max_input_tokens": 1024, + "max_tokens": 1024, + "mode": "video_generation", + "output_cost_per_second": 0.4, "source": "https://ai.google.dev/gemini-api/docs/video", "supported_modalities": [ "text" @@ -24697,6 +26316,104 @@ "supports_tool_choice": true, "supports_web_search": true }, + "xai/grok-4-1-fast": { + "cache_read_input_token_cost": 0.05e-06, + "input_cost_per_token": 0.2e-06, + "input_cost_per_token_above_128k_tokens": 0.4e-06, + "litellm_provider": "xai", + "max_input_tokens": 2e6, + "max_output_tokens": 2e6, + "max_tokens": 2e6, + "mode": "chat", + "output_cost_per_token": 0.5e-06, + "output_cost_per_token_above_128k_tokens": 1e-06, + "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", + "supports_audio_input": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "xai/grok-4-1-fast-reasoning": { + "cache_read_input_token_cost": 0.05e-06, + "input_cost_per_token": 0.2e-06, + "input_cost_per_token_above_128k_tokens": 0.4e-06, + "litellm_provider": "xai", + "max_input_tokens": 2e6, + "max_output_tokens": 2e6, + "max_tokens": 2e6, + "mode": "chat", + "output_cost_per_token": 0.5e-06, + "output_cost_per_token_above_128k_tokens": 1e-06, + "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", + "supports_audio_input": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "xai/grok-4-1-fast-reasoning-latest": { + "cache_read_input_token_cost": 0.05e-06, + "input_cost_per_token": 0.2e-06, + "input_cost_per_token_above_128k_tokens": 0.4e-06, + "litellm_provider": "xai", + "max_input_tokens": 2e6, + "max_output_tokens": 2e6, + "max_tokens": 2e6, + "mode": "chat", + "output_cost_per_token": 0.5e-06, + "output_cost_per_token_above_128k_tokens": 1e-06, + "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", + "supports_audio_input": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "xai/grok-4-1-fast-non-reasoning": { + "cache_read_input_token_cost": 0.05e-06, + "input_cost_per_token": 0.2e-06, + "input_cost_per_token_above_128k_tokens": 0.4e-06, + "litellm_provider": "xai", + "max_input_tokens": 2e6, + "max_output_tokens": 2e6, + "max_tokens": 2e6, + "mode": "chat", + "output_cost_per_token": 0.5e-06, + "output_cost_per_token_above_128k_tokens": 1e-06, + "source": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", + "supports_audio_input": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "xai/grok-4-1-fast-non-reasoning-latest": { + "cache_read_input_token_cost": 0.05e-06, + "input_cost_per_token": 0.2e-06, + "input_cost_per_token_above_128k_tokens": 0.4e-06, + "litellm_provider": "xai", + "max_input_tokens": 2e6, + "max_output_tokens": 2e6, + "max_tokens": 2e6, + "mode": "chat", + "output_cost_per_token": 0.5e-06, + "output_cost_per_token_above_128k_tokens": 1e-06, + "source": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", + "supports_audio_input": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "xai/grok-beta": { "input_cost_per_token": 5e-06, "litellm_provider": "xai", diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 5e2badb7bbe..320565f7f66 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -16,10 +16,20 @@ from urllib.parse import urlparse from fastapi import HTTPException from httpx import HTTPStatusError -from mcp.types import CallToolRequestParams as MCPCallToolRequestParams +from mcp import ReadResourceResult, Resource +from mcp.types import ( + CallToolRequestParams as MCPCallToolRequestParams, + GetPromptRequestParams, + GetPromptResult, + Prompt, + ResourceTemplate, +) from mcp.types import CallToolResult from mcp.types import Tool as MCPTool +from pydantic import AnyUrl + +import litellm from litellm._logging import verbose_logger from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.experimental_mcp_client.client import MCPClient @@ -28,11 +38,11 @@ from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) from litellm.proxy._experimental.mcp_server.utils import ( - add_server_prefix_to_tool_name, - get_server_name_prefix_tool_mcp, + add_server_prefix_to_name, get_server_prefix, is_tool_name_prefixed, normalize_server_name, + split_server_prefix_from_name, validate_mcp_server_name, ) from litellm.proxy._types import ( @@ -356,7 +366,7 @@ class MCPServerManager: base_tool_name = operation_id.replace(" ", "_").lower() # Add server prefix to tool name - prefixed_tool_name = add_server_prefix_to_tool_name( + prefixed_tool_name = add_server_prefix_to_name( base_tool_name, server_prefix ) @@ -714,12 +724,190 @@ class MCPServerManager: f"Failed to get tools from server {server.name}: {str(e)}" ) return [] - finally: - if client: - try: - await client.disconnect() - except Exception: - pass + + async def get_prompts_from_server( + self, + server: MCPServer, + mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, + extra_headers: Optional[Dict[str, str]] = None, + add_prefix: bool = True, + ) -> List[Prompt]: + """ + Helper method to get prompts from a single MCP server with prefixed names. + + Args: + server (MCPServer): The server to query prompts from + mcp_auth_header: Optional auth header for MCP server + + Returns: + List[Prompt]: List of prompts available on the server with prefixed names + """ + + verbose_logger.debug(f"Connecting to url: {server.url}") + verbose_logger.info(f"get_prompts_from_server for {server.name}...") + + client = None + + try: + if server.static_headers: + if extra_headers is None: + extra_headers = {} + extra_headers.update(server.static_headers) + + client = self._create_mcp_client( + server=server, + mcp_auth_header=mcp_auth_header, + extra_headers=extra_headers, + ) + + prompts = await client.list_prompts() + + prefixed_or_original_prompts = self._create_prefixed_prompts( + prompts, server, add_prefix=add_prefix + ) + + return prefixed_or_original_prompts + + except Exception as e: + verbose_logger.warning( + f"Failed to get prompts from server {server.name}: {str(e)}" + ) + return [] + + async def get_resources_from_server( + self, + server: MCPServer, + mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, + extra_headers: Optional[Dict[str, str]] = None, + add_prefix: bool = True, + ) -> List[Resource]: + """Fetch available resources from a single MCP server.""" + + verbose_logger.debug(f"Connecting to url: {server.url}") + verbose_logger.info(f"get_resources_from_server for {server.name}...") + + client = None + + try: + if server.static_headers: + if extra_headers is None: + extra_headers = {} + extra_headers.update(server.static_headers) + + client = self._create_mcp_client( + server=server, + mcp_auth_header=mcp_auth_header, + extra_headers=extra_headers, + ) + + resources = await client.list_resources() + + prefixed_resources = self._create_prefixed_resources( + resources, server, add_prefix=add_prefix + ) + + return prefixed_resources + + except Exception as e: + verbose_logger.warning( + f"Failed to get resources from server {server.name}: {str(e)}" + ) + return [] + + async def get_resource_templates_from_server( + self, + server: MCPServer, + mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, + extra_headers: Optional[Dict[str, str]] = None, + add_prefix: bool = True, + ) -> List[ResourceTemplate]: + """Fetch available resource templates from a single MCP server.""" + + verbose_logger.debug(f"Connecting to url: {server.url}") + verbose_logger.info(f"get_resource_templates_from_server for {server.name}...") + + client = None + + try: + if server.static_headers: + if extra_headers is None: + extra_headers = {} + extra_headers.update(server.static_headers) + + client = self._create_mcp_client( + server=server, + mcp_auth_header=mcp_auth_header, + extra_headers=extra_headers, + ) + + resource_templates = await client.list_resource_templates() + + prefixed_templates = self._create_prefixed_resource_templates( + resource_templates, server, add_prefix=add_prefix + ) + + return prefixed_templates + + except Exception as e: + verbose_logger.warning( + f"Failed to get resource templates from server {server.name}: {str(e)}" + ) + return [] + + async def read_resource_from_server( + self, + server: MCPServer, + url: AnyUrl, + mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, + extra_headers: Optional[Dict[str, str]] = None, + ) -> ReadResourceResult: + """Read resource contents from a specific MCP server.""" + + verbose_logger.debug(f"Connecting to url: {server.url}") + verbose_logger.info(f"read_resource_from_server for {server.name}...") + + if server.static_headers: + if extra_headers is None: + extra_headers = {} + extra_headers.update(server.static_headers) + + client = self._create_mcp_client( + server=server, + mcp_auth_header=mcp_auth_header, + extra_headers=extra_headers, + ) + + return await client.read_resource(url) + + async def get_prompt_from_server( + self, + server: MCPServer, + prompt_name: str, + arguments: Optional[Dict[str, Any]] = None, + mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, + extra_headers: Optional[Dict[str, str]] = None, + ) -> GetPromptResult: + """Fetch a specific prompt definition from a single MCP server.""" + + verbose_logger.debug(f"Connecting to url: {server.url}") + verbose_logger.info(f"get_prompt_from_server for {server.name}...") + + if server.static_headers: + if extra_headers is None: + extra_headers = {} + extra_headers.update(server.static_headers) + + client = self._create_mcp_client( + server=server, + mcp_auth_header=mcp_auth_header, + extra_headers=extra_headers, + ) + + get_prompt_request_params = GetPromptRequestParams( + name=prompt_name, + arguments=arguments, + ) + return await client.get_prompt(get_prompt_request_params) async def _descovery_metadata( self, @@ -829,7 +1017,7 @@ class MCPServerManager: try: client = get_async_httpx_client( llm_provider=httpxSpecialProvider.MCP, - params={"timeout": 10.0, "follow_redirects": True}, + params={"timeout": 10.0}, ) response = await client.get(resource_metadata_url) response.raise_for_status() @@ -925,7 +1113,7 @@ class MCPServerManager: try: client = get_async_httpx_client( llm_provider=httpxSpecialProvider.MCP, - params={"timeout": 10.0, "follow_redirects": True}, + params={"timeout": 10.0}, ) response = await client.get(url) response.raise_for_status() @@ -983,8 +1171,6 @@ class MCPServerManager: async def _list_tools_task(): try: - await client.connect() - tools = await client.list_tools() verbose_logger.debug(f"Tools from {server_name}: {tools}") return tools @@ -1033,7 +1219,7 @@ class MCPServerManager: prefix = get_server_prefix(server) for tool in tools: - prefixed_name = add_server_prefix_to_tool_name(tool.name, prefix) + prefixed_name = add_server_prefix_to_name(tool.name, prefix) name_to_use = prefixed_name if add_prefix else tool.name @@ -1053,6 +1239,82 @@ class MCPServerManager: ) return prefixed_tools + def _create_prefixed_prompts( + self, prompts: List[Prompt], server: MCPServer, add_prefix: bool = True + ) -> List[Prompt]: + """ + Create prefixed prompts and update prompt mapping. + + Args: + prompts: List of original prompts from server + server: Server instance + + Returns: + List of prompts with prefixed names + """ + prefixed_prompts = [] + prefix = get_server_prefix(server) + + for prompt in prompts: + prefixed_name = add_server_prefix_to_name(prompt.name, prefix) + + name_to_use = prefixed_name if add_prefix else prompt.name + + prompt.name = name_to_use + prefixed_prompts.append(prompt) + + verbose_logger.info( + f"Successfully fetched {len(prefixed_prompts)} prompts from server {server.name}" + ) + return prefixed_prompts + + def _create_prefixed_resources( + self, resources: List[Resource], server: MCPServer, add_prefix: bool = True + ) -> List[Resource]: + """Prefix resource names and track origin server for read requests.""" + + prefixed_resources: List[Resource] = [] + prefix = get_server_prefix(server) + + for resource in resources: + name_to_use = ( + add_server_prefix_to_name(resource.name, prefix) + if add_prefix + else resource.name + ) + resource.name = name_to_use + prefixed_resources.append(resource) + + verbose_logger.info( + f"Successfully fetched {len(prefixed_resources)} resources from server {server.name}" + ) + return prefixed_resources + + def _create_prefixed_resource_templates( + self, + resource_templates: List[ResourceTemplate], + server: MCPServer, + add_prefix: bool = True, + ) -> List[ResourceTemplate]: + """Prefix resource template names for multi-server scenarios.""" + + prefixed_templates: List[ResourceTemplate] = [] + prefix = get_server_prefix(server) + + for resource_template in resource_templates: + name_to_use = ( + add_server_prefix_to_name(resource_template.name, prefix) + if add_prefix + else resource_template.name + ) + resource_template.name = name_to_use + prefixed_templates.append(resource_template) + + verbose_logger.info( + f"Successfully fetched {len(prefixed_templates)} resource templates from server {server.name}" + ) + return prefixed_templates + def check_allowed_or_banned_tools(self, tool_name: str, server: MCPServer) -> bool: """ Check if the tool is allowed or banned for the given server @@ -1087,7 +1349,7 @@ class MCPServerManager: HTTPException: If allowed_params is configured for this tool but arguments contain disallowed params """ from litellm.proxy._experimental.mcp_server.utils import ( - get_server_name_prefix_tool_mcp, + split_server_prefix_from_name, ) # If no allowed_params configured, return all arguments @@ -1095,7 +1357,7 @@ class MCPServerManager: return # Get the unprefixed tool name to match against config - unprefixed_tool_name, _ = get_server_name_prefix_tool_mcp(tool_name) + unprefixed_tool_name, _ = split_server_prefix_from_name(tool_name) # Check both prefixed and unprefixed tool names allowed_params_list = server.allowed_params.get( @@ -1439,14 +1701,12 @@ class MCPServerManager: ) async def _call_tool_via_client(client, params): - async with client: - return await client.call_tool(params) + return await client.call_tool(params) tasks.append( asyncio.create_task(_call_tool_via_client(client, call_tool_params)) ) - # IMPORTANT: Must await tasks INSIDE the context manager to keep connection alive try: mcp_responses = await asyncio.gather(*tasks) except ( @@ -1498,7 +1758,7 @@ class MCPServerManager: start_time = datetime.datetime.now() # Get the MCP server - prefixed_tool_name = add_server_prefix_to_tool_name(name, server_name) + prefixed_tool_name = add_server_prefix_to_name(name, server_name) mcp_server = self._get_mcp_server_from_tool_name(prefixed_tool_name) if mcp_server is None: raise ValueError(f"Tool {name} not found") @@ -1604,7 +1864,7 @@ class MCPServerManager: for tool in tools: # The tool.name here is already prefixed from _get_tools_from_server # Extract original name for mapping - original_name, _ = get_server_name_prefix_tool_mcp(tool.name) + original_name, _ = split_server_prefix_from_name(tool.name) self.tool_name_to_mcp_server_name_mapping[original_name] = server.name self.tool_name_to_mcp_server_name_mapping[tool.name] = server.name @@ -1632,7 +1892,7 @@ class MCPServerManager: ( original_tool_name, server_name_from_prefix, - ) = get_server_name_prefix_tool_mcp(tool_name) + ) = split_server_prefix_from_name(tool_name) if original_tool_name in self.tool_name_to_mcp_server_name_mapping: for server in self.get_registry().values(): if normalize_server_name(server.name) == normalize_server_name( @@ -1668,6 +1928,14 @@ class MCPServerManager: f"Registry now contains {len(self.get_registry())} servers" ) + def get_mcp_servers_from_ids(self, server_ids: List[str]) -> List[MCPServer]: + servers = [] + registry = self.get_registry() + for server in registry.values(): + if server.server_id in server_ids: + servers.append(server) + return servers + def get_mcp_server_by_id(self, server_id: str) -> Optional[MCPServer]: """ Get the MCP Server from the server id @@ -1678,11 +1946,16 @@ class MCPServerManager: return server return None - def get_mcp_servers_from_ids(self, server_ids: List[str]) -> List[MCPServer]: - servers = [] - registry = self.get_registry() - for server in registry.values(): - if server.server_id in server_ids: + def get_public_mcp_servers(self) -> List[MCPServer]: + """ + Get the public MCP servers + """ + servers: List[MCPServer] = [] + if litellm.public_mcp_servers is None: + return servers + for server_id in litellm.public_mcp_servers: + server = self.get_mcp_server_by_id(server_id) + if server: servers.append(server) return servers diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index e427507a4b8..4288f25740c 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -76,12 +76,12 @@ if MCP_AVAILABLE: mcp_auth_header=server_auth_header, add_prefix=False, ) - + # Filter tools based on allowed_tools configuration # Only filter if allowed_tools is explicitly configured (not None and not empty) if server.allowed_tools is not None and len(server.allowed_tools) > 0: tools = filter_tools_by_allowed_tools(tools, server) - + return _create_tool_response_objects(tools, server.mcp_info) ######################################################## @@ -212,7 +212,9 @@ if MCP_AVAILABLE: from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.proxy.proxy_server import add_litellm_data_to_request, proxy_config - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + MCPRequestHandler, + ) try: data = await request.json() @@ -222,21 +224,27 @@ if MCP_AVAILABLE: user_api_key_dict=user_api_key_dict, proxy_config=proxy_config, ) - + # FIX: Extract MCP auth headers from request # The UI sends bearer token in x-mcp-auth header and server-specific headers, # but they weren't being extracted and passed to call_mcp_tool. # This fix ensures auth headers are properly extracted from the HTTP request # and passed through to the MCP server for authentication. - mcp_auth_header = MCPRequestHandler._get_mcp_auth_header_from_headers(request.headers) - mcp_server_auth_headers = MCPRequestHandler._get_mcp_server_auth_headers_from_headers(request.headers) - + mcp_auth_header = MCPRequestHandler._get_mcp_auth_header_from_headers( + request.headers + ) + mcp_server_auth_headers = ( + MCPRequestHandler._get_mcp_server_auth_headers_from_headers( + request.headers + ) + ) + # Add extracted headers to data dict to pass to call_mcp_tool if mcp_auth_header: data["mcp_auth_header"] = mcp_auth_header if mcp_server_auth_headers: data["mcp_server_auth_headers"] = mcp_server_auth_headers - + result = await call_mcp_tool(**data) return result except BlockedPiiEntityError as e: @@ -296,7 +304,6 @@ if MCP_AVAILABLE: Returns: Operation result or error response """ - client = None try: client = global_mcp_server_manager._create_mcp_client( server=MCPServer( @@ -315,13 +322,6 @@ if MCP_AVAILABLE: except Exception as e: verbose_logger.error(f"Error in MCP operation: {e}", exc_info=True) return {"status": "error", "message": "An internal error has occurred."} - finally: - # Ensure client is properly disconnected before response is sent - if client is not None: - try: - await client.disconnect() - except Exception as e: - verbose_logger.warning(f"Error disconnecting MCP client: {e}") @router.post("/test/connection") async def test_connection( @@ -332,7 +332,10 @@ if MCP_AVAILABLE: """ async def _test_connection_operation(client): - await client.connect() + async def _noop(session): + return "ok" + + await client.run_with_session(_noop) return {"status": "ok"} return await _execute_with_mcp_client(request, _test_connection_operation) @@ -347,7 +350,13 @@ if MCP_AVAILABLE: """ async def _list_tools_operation(client): - list_tools_result: List[MCPTool] = await client.list_tools() + async def _list_tools_session_operation(session): + return await session.list_tools() + + list_tools_response = await client.run_with_session( + _list_tools_session_operation + ) + list_tools_result: List[MCPTool] = list_tools_response.tools model_dumped_tools: List[dict] = [ tool.model_dump() for tool in list_tools_result ] diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 019e55b9104..b6b36622d60 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -5,10 +5,10 @@ LiteLLM MCP Server Routes import asyncio import contextlib from datetime import datetime -from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, List, Optional, Tuple, Union from fastapi import FastAPI, HTTPException -from pydantic import ConfigDict +from pydantic import AnyUrl, ConfigDict from starlette.types import Receive, Scope, Send from litellm._logging import verbose_logger @@ -33,10 +33,29 @@ from litellm.utils import client # TODO: Make this a util function for litellm client usage MCP_AVAILABLE: bool = True try: + from mcp import ReadResourceResult, Resource from mcp.server import Server + from mcp.server.lowlevel.helper_types import ReadResourceContents + from mcp.types import ( + BlobResourceContents, + GetPromptResult, + ResourceTemplate, + TextResourceContents, + ) except ImportError as e: verbose_logger.debug(f"MCP module not found: {e}") MCP_AVAILABLE = False + # For type checking only - these will never be accessed at runtime when MCP is unavailable + # because all code using them is guarded by `if MCP_AVAILABLE:` + if TYPE_CHECKING: + from typing import Any as BlobResourceContents # type: ignore + from typing import Any as GetPromptResult + from typing import Any as ReadResourceContents + from typing import Any as ReadResourceResult + from typing import Any as Resource + from typing import Any as ResourceTemplate + from typing import Any as Server + from typing import Any as TextResourceContents # Global variables to track initialization @@ -52,7 +71,7 @@ if MCP_AVAILABLE: auth_context_var, ) from mcp.server.streamable_http_manager import StreamableHTTPSessionManager - from mcp.types import EmbeddedResource, ImageContent, TextContent + from mcp.types import EmbeddedResource, ImageContent, Prompt, TextContent from mcp.types import Tool as MCPTool from litellm.proxy._experimental.mcp_server.auth.litellm_auth_handler import ( @@ -66,7 +85,7 @@ if MCP_AVAILABLE: global_mcp_tool_registry, ) from litellm.proxy._experimental.mcp_server.utils import ( - get_server_name_prefix_tool_mcp, + split_server_prefix_from_name, ) ###################################################### @@ -303,6 +322,210 @@ if MCP_AVAILABLE: return response + @server.list_prompts() + async def list_prompts() -> List[Prompt]: + """ + List all available prompts + """ + try: + # Get user authentication from context variable + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + ) = get_auth_context() + verbose_logger.debug( + f"MCP list_prompts - User API Key Auth from context: {user_api_key_auth}" + ) + verbose_logger.debug( + f"MCP list_prompts - MCP servers from context: {mcp_servers}" + ) + verbose_logger.debug( + f"MCP list_prompts - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + ) + # Get mcp_servers from context variable + verbose_logger.debug("MCP list_prompts - Calling _list_prompts") + prompts = await _list_mcp_prompts( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + verbose_logger.info( + f"MCP list_prompts - Successfully returned {len(prompts)} prompts" + ) + return prompts + except Exception as e: + verbose_logger.exception(f"Error in list_prompts endpoint: {str(e)}") + # Return empty list instead of failing completely + # This prevents the HTTP stream from failing and allows the client to get a response + return [] + + @server.get_prompt() + async def get_prompt( + name: str, arguments: dict[str, str] | None + ) -> GetPromptResult: + """ + Get a specific prompt with the provided arguments + + Args: + name (str): Name of the prompt to get + arguments (Dict[str, Any] | None): Arguments to pass to the prompt + + Returns: + GetPromptResult: Getting prompt execution results + """ + + # Validate arguments + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + ) = get_auth_context() + + verbose_logger.debug( + f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}" + ) + return await mcp_get_prompt( + name=name, + arguments=arguments, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + @server.list_resources() + async def list_resources() -> List[Resource]: + """List all available resources.""" + try: + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + ) = get_auth_context() + verbose_logger.debug( + f"MCP list_resources - User API Key Auth from context: {user_api_key_auth}" + ) + verbose_logger.debug( + f"MCP list_resources - MCP servers from context: {mcp_servers}" + ) + verbose_logger.debug( + f"MCP list_resources - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + ) + + resources = await _list_mcp_resources( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + verbose_logger.info( + f"MCP list_resources - Successfully returned {len(resources)} resources" + ) + return resources + except Exception as e: + verbose_logger.exception(f"Error in list_resources endpoint: {str(e)}") + return [] + + @server.list_resource_templates() + async def list_resource_templates() -> List[ResourceTemplate]: + """List all available resource templates.""" + try: + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + ) = get_auth_context() + verbose_logger.debug( + f"MCP list_resource_templates - User API Key Auth from context: {user_api_key_auth}" + ) + verbose_logger.debug( + f"MCP list_resource_templates - MCP servers from context: {mcp_servers}" + ) + verbose_logger.debug( + f"MCP list_resource_templates - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + ) + + resource_templates = await _list_mcp_resource_templates( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + verbose_logger.info( + "MCP list_resource_templates - Successfully returned " + f"{len(resource_templates)} resource templates" + ) + return resource_templates + except Exception as e: + verbose_logger.exception( + f"Error in list_resource_templates endpoint: {str(e)}" + ) + return [] + + @server.read_resource() + async def read_resource(url: AnyUrl) -> list[ReadResourceContents]: + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + ) = get_auth_context() + + read_resource_result = await mcp_read_resource( + url=url, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + normalized_contents: List[ReadResourceContents] = [] + for content in read_resource_result.contents: + if isinstance(content, TextResourceContents): + text_content: TextResourceContents = content + normalized_contents.append( + ReadResourceContents( + content=text_content.text, + mime_type=text_content.mimeType, + ) + ) + elif isinstance(content, BlobResourceContents): + blob_content: BlobResourceContents = content + normalized_contents.append( + ReadResourceContents( + content=blob_content.blob, + mime_type=None, + ) + ) + + return normalized_contents + ######################################################## ############ End of MCP Server Routes ################## ######################################################## @@ -379,7 +602,7 @@ if MCP_AVAILABLE: True if the tool name (prefixed or unprefixed) is in the filter list """ from litellm.proxy._experimental.mcp_server.utils import ( - get_server_name_prefix_tool_mcp, + split_server_prefix_from_name, ) # Check if the full name is in the list @@ -387,7 +610,7 @@ if MCP_AVAILABLE: return True # Check if the unprefixed name is in the list - unprefixed_name, _ = get_server_name_prefix_tool_mcp(tool_name) + unprefixed_name, _ = split_server_prefix_from_name(tool_name) return unprefixed_name in filter_list def filter_tools_by_allowed_tools( @@ -428,6 +651,60 @@ if MCP_AVAILABLE: return tools_to_return + async def _get_allowed_mcp_servers( + user_api_key_auth: Optional[UserAPIKeyAuth], + mcp_servers: Optional[List[str]], + ) -> List[MCPServer]: + """Return allowed MCP servers for a request after applying filters.""" + allowed_mcp_server_ids = ( + await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth) + ) + allowed_mcp_servers: List[MCPServer] = [] + for allowed_mcp_server_id in allowed_mcp_server_ids: + mcp_server = global_mcp_server_manager.get_mcp_server_by_id( + allowed_mcp_server_id + ) + if mcp_server is not None: + allowed_mcp_servers.append(mcp_server) + + if mcp_servers is not None: + allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names( + mcp_servers=mcp_servers, + allowed_mcp_servers=allowed_mcp_servers, + ) + + return allowed_mcp_servers + + def _prepare_mcp_server_headers( + server: MCPServer, + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]], + mcp_auth_header: Optional[str], + oauth2_headers: Optional[Dict[str, str]], + raw_headers: Optional[Dict[str, str]], + ) -> Tuple[Optional[Union[Dict[str, str], str]], Optional[Dict[str, str]]]: + """Build auth and extra headers for a server.""" + server_auth_header: Optional[Union[Dict[str, str], str]] = None + if mcp_server_auth_headers and server.alias is not None: + server_auth_header = mcp_server_auth_headers.get(server.alias) + elif mcp_server_auth_headers and server.server_name is not None: + server_auth_header = mcp_server_auth_headers.get(server.server_name) + + extra_headers: Optional[Dict[str, str]] = None + if server.auth_type == MCPAuth.oauth2: + extra_headers = oauth2_headers + + if server.extra_headers and raw_headers: + if extra_headers is None: + extra_headers = {} + for header in server.extra_headers: + if header in raw_headers: + extra_headers[header] = raw_headers[header] + + if server_auth_header is None: + server_auth_header = mcp_auth_header + + return server_auth_header, extra_headers + async def _get_tools_from_mcp_servers( user_api_key_auth: Optional[UserAPIKeyAuth], mcp_auth_header: Optional[str], @@ -452,19 +729,10 @@ if MCP_AVAILABLE: if not MCP_AVAILABLE: return [] - # Get allowed MCP servers based on user permissions - allowed_mcp_server_ids = ( - await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth) + allowed_mcp_servers = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, ) - allowed_mcp_servers = global_mcp_server_manager.get_mcp_servers_from_ids( - allowed_mcp_server_ids - ) - - if mcp_servers is not None: - allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names( - mcp_servers=mcp_servers, - allowed_mcp_servers=allowed_mcp_servers, - ) # Decide whether to add prefix based on number of allowed servers add_prefix = not (len(allowed_mcp_servers) == 1) @@ -475,27 +743,13 @@ if MCP_AVAILABLE: if server is None: continue - # Get server-specific auth header if available - server_auth_header: Optional[Union[Dict[str, str], str]] = None - if mcp_server_auth_headers and server.alias is not None: - server_auth_header = mcp_server_auth_headers.get(server.alias) - elif mcp_server_auth_headers and server.server_name is not None: - server_auth_header = mcp_server_auth_headers.get(server.server_name) - - extra_headers: Optional[Dict[str, str]] = None - if server.auth_type == MCPAuth.oauth2: - extra_headers = oauth2_headers - - if server.extra_headers and raw_headers: - if extra_headers is None: - extra_headers = {} - for header in server.extra_headers: - if header in raw_headers: - extra_headers[header] = raw_headers[header] - - # Fall back to deprecated mcp_auth_header if no server-specific header found - if server_auth_header is None: - server_auth_header = mcp_auth_header + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) try: tools = await global_mcp_server_manager._get_tools_from_server( @@ -530,6 +784,195 @@ if MCP_AVAILABLE: return all_tools + async def _get_prompts_from_mcp_servers( + user_api_key_auth: Optional[UserAPIKeyAuth], + mcp_auth_header: Optional[str], + mcp_servers: Optional[List[str]], + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + ) -> List[Prompt]: + """ + Helper method to fetch prompt from MCP servers based on server filtering criteria. + + Args: + user_api_key_auth: User authentication info for access control + mcp_auth_header: Optional auth header for MCP server (deprecated) + mcp_servers: Optional list of server names/aliases to filter by + mcp_server_auth_headers: Optional dict of server-specific auth headers + oauth2_headers: Optional dict of oauth2 headers + + Returns: + List[Prompt]: Combined list of prompts from filtered servers + """ + if not MCP_AVAILABLE: + return [] + + allowed_mcp_servers = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, + ) + + # Decide whether to add prefix based on number of allowed servers + add_prefix = not (len(allowed_mcp_servers) == 1) + + # Get prompts from each allowed server + all_prompts = [] + for server in allowed_mcp_servers: + if server is None: + continue + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + try: + prompts = await global_mcp_server_manager.get_prompts_from_server( + server=server, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + add_prefix=add_prefix, + ) + + all_prompts.extend(prompts) + + verbose_logger.debug( + f"Successfully fetched {len(prompts)} prompts from server {server.name}" + ) + except Exception as e: + verbose_logger.exception( + f"Error getting prompts from server {server.name}: {str(e)}" + ) + # Continue with other servers instead of failing completely + + verbose_logger.info( + f"Successfully fetched {len(all_prompts)} prompts total from all MCP servers" + ) + + return all_prompts + + async def _get_resources_from_mcp_servers( + user_api_key_auth: Optional[UserAPIKeyAuth], + mcp_auth_header: Optional[str], + mcp_servers: Optional[List[str]], + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + ) -> List[Resource]: + """Fetch resources from allowed MCP servers.""" + + if not MCP_AVAILABLE: + return [] + + allowed_mcp_servers = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, + ) + + add_prefix = not (len(allowed_mcp_servers) == 1) + + all_resources: List[Resource] = [] + for server in allowed_mcp_servers: + if server is None: + continue + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + try: + resources = await global_mcp_server_manager.get_resources_from_server( + server=server, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + add_prefix=add_prefix, + ) + all_resources.extend(resources) + + verbose_logger.debug( + f"Successfully fetched {len(resources)} resources from server {server.name}" + ) + except Exception as e: + verbose_logger.exception( + f"Error getting resources from server {server.name}: {str(e)}" + ) + + verbose_logger.info( + f"Successfully fetched {len(all_resources)} resources total from all MCP servers" + ) + + return all_resources + + async def _get_resource_templates_from_mcp_servers( + user_api_key_auth: Optional[UserAPIKeyAuth], + mcp_auth_header: Optional[str], + mcp_servers: Optional[List[str]], + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + ) -> List[ResourceTemplate]: + """Fetch resource templates from allowed MCP servers.""" + + if not MCP_AVAILABLE: + return [] + + allowed_mcp_servers = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, + ) + + add_prefix = not (len(allowed_mcp_servers) == 1) + + all_resource_templates: List[ResourceTemplate] = [] + for server in allowed_mcp_servers: + if server is None: + continue + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + try: + resource_templates = ( + await global_mcp_server_manager.get_resource_templates_from_server( + server=server, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + add_prefix=add_prefix, + ) + ) + all_resource_templates.extend(resource_templates) + verbose_logger.debug( + "Successfully fetched %s resource templates from server %s", + len(resource_templates), + server.name, + ) + except Exception as e: + verbose_logger.exception( + "Error getting resource templates from server %s: %s", + server.name, + str(e), + ) + + verbose_logger.info( + "Successfully fetched %s resource templates total from all MCP servers", + len(all_resource_templates), + ) + + return all_resource_templates + async def filter_tools_by_key_team_permissions( tools: List[MCPTool], server_id: str, @@ -553,7 +996,7 @@ if MCP_AVAILABLE: filtered_tools = [] for t in tools: # Get tool name without server prefix - unprefixed_tool_name, _ = get_server_name_prefix_tool_mcp(t.name) + unprefixed_tool_name, _ = split_server_prefix_from_name(t.name) if unprefixed_tool_name in allowed_tool_names: filtered_tools.append(t) else: @@ -606,6 +1049,118 @@ if MCP_AVAILABLE: return managed_tools + async def _list_mcp_prompts( + user_api_key_auth: Optional[UserAPIKeyAuth] = None, + mcp_auth_header: Optional[str] = None, + mcp_servers: Optional[List[str]] = None, + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + ) -> List[Prompt]: + """ + List all available MCP prompts. + + Args: + user_api_key_auth: User authentication info for access control + mcp_auth_header: Optional auth header for MCP server (deprecated) + mcp_servers: Optional list of server names/aliases to filter by + mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value} + + Returns: + List[Prompt]: Combined list of tools from all accessible servers + """ + if not MCP_AVAILABLE: + return [] + # Get tools from managed MCP servers with error handling + managed_prompts = [] + try: + managed_prompts = await _get_prompts_from_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + verbose_logger.debug( + f"Successfully fetched {len(managed_prompts)} prompts from managed MCP servers" + ) + except Exception as e: + verbose_logger.exception( + f"Error getting tools from managed MCP servers: {str(e)}" + ) + # Continue with empty managed tools list instead of failing completely + + return managed_prompts + + async def _list_mcp_resources( + user_api_key_auth: Optional[UserAPIKeyAuth] = None, + mcp_auth_header: Optional[str] = None, + mcp_servers: Optional[List[str]] = None, + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + ) -> List[Resource]: + """List all available MCP resources.""" + + if not MCP_AVAILABLE: + return [] + + managed_resources: List[Resource] = [] + try: + managed_resources = await _get_resources_from_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + verbose_logger.debug( + f"Successfully fetched {len(managed_resources)} resources from managed MCP servers" + ) + except Exception as e: + verbose_logger.exception( + f"Error getting resources from managed MCP servers: {str(e)}" + ) + + return managed_resources + + async def _list_mcp_resource_templates( + user_api_key_auth: Optional[UserAPIKeyAuth] = None, + mcp_auth_header: Optional[str] = None, + mcp_servers: Optional[List[str]] = None, + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + ) -> List[ResourceTemplate]: + """List all available MCP resource templates.""" + + if not MCP_AVAILABLE: + return [] + + managed_resource_templates: List[ResourceTemplate] = [] + try: + managed_resource_templates = await _get_resource_templates_from_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + verbose_logger.debug( + "Successfully fetched %s resource templates from managed MCP servers", + len(managed_resource_templates), + ) + except Exception as e: + verbose_logger.exception( + "Error getting resource templates from managed MCP servers: %s", + str(e), + ) + + return managed_resource_templates + @client async def call_mcp_tool( name: str, @@ -634,9 +1189,13 @@ if MCP_AVAILABLE: ) ) - allowed_mcp_servers = global_mcp_server_manager.get_mcp_servers_from_ids( - allowed_mcp_server_ids - ) + allowed_mcp_servers: List[MCPServer] = [] + for allowed_mcp_server_id in allowed_mcp_server_ids: + mcp_server = global_mcp_server_manager.get_mcp_server_by_id( + allowed_mcp_server_id + ) + if mcp_server is not None: + allowed_mcp_servers.append(mcp_server) allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names( mcp_servers=mcp_servers, @@ -647,7 +1206,7 @@ if MCP_AVAILABLE: mcp_server: Optional[MCPServer] = None # Remove prefix from tool name for logging and processing - original_tool_name, server_name = get_server_name_prefix_tool_mcp(name) + original_tool_name, server_name = split_server_prefix_from_name(name) # If tool name is unprefixed, resolve its server so we can enforce permissions if not server_name: @@ -735,6 +1294,110 @@ if MCP_AVAILABLE: ) return response + async def mcp_get_prompt( + name: str, + arguments: Optional[Dict[str, Any]] = None, + user_api_key_auth: Optional[UserAPIKeyAuth] = None, + mcp_auth_header: Optional[str] = None, + mcp_servers: Optional[List[str]] = None, + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + ) -> GetPromptResult: + """ + Fetch a specific MCP prompt, handling both prefixed and unprefixed names. + """ + allowed_mcp_servers = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, + ) + + if not allowed_mcp_servers: + raise HTTPException( + status_code=403, + detail="User not allowed to get this prompt.", + ) + + # Decide whether to add prefix based on number of allowed servers + add_prefix = not (len(allowed_mcp_servers) == 1) + + if add_prefix: + original_prompt_name, server_name = split_server_prefix_from_name(name) + else: + original_prompt_name = name + server_name = allowed_mcp_servers[0].name + + server = next((s for s in allowed_mcp_servers if s.name == server_name), None) + if server is None: + raise HTTPException( + status_code=403, + detail="User not allowed to get this prompt.", + ) + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + return await global_mcp_server_manager.get_prompt_from_server( + server=server, + prompt_name=original_prompt_name, + arguments=arguments, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + ) + + async def mcp_read_resource( + url: AnyUrl, + user_api_key_auth: Optional[UserAPIKeyAuth] = None, + mcp_auth_header: Optional[str] = None, + mcp_servers: Optional[List[str]] = None, + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + ) -> ReadResourceResult: + """Read resource contents from upstream MCP servers.""" + + allowed_mcp_servers = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, + ) + + if not allowed_mcp_servers: + raise HTTPException( + status_code=403, + detail="User not allowed to read this resource.", + ) + + if len(allowed_mcp_servers) != 1: + raise HTTPException( + status_code=400, + detail=( + "Multiple MCP servers configured; read_resource currently " + "supports exactly one allowed server." + ), + ) + + server = allowed_mcp_servers[0] + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + return await global_mcp_server_manager.read_resource_from_server( + server=server, + url=url, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + ) + def _get_standard_logging_mcp_tool_call( name: str, arguments: Dict[str, Any], diff --git a/litellm/proxy/_experimental/mcp_server/utils.py b/litellm/proxy/_experimental/mcp_server/utils.py index fb28eaf8cf2..d801b312aac 100644 --- a/litellm/proxy/_experimental/mcp_server/utils.py +++ b/litellm/proxy/_experimental/mcp_server/utils.py @@ -13,6 +13,7 @@ LITELLM_MCP_SERVER_DESCRIPTION = "MCP Server for LiteLLM" MCP_TOOL_PREFIX_SEPARATOR = os.environ.get("MCP_TOOL_PREFIX_SEPARATOR", "-") MCP_TOOL_PREFIX_FORMAT = "{server_name}{separator}{tool_name}" + def is_mcp_available() -> bool: """ Returns True if the MCP module is available, False otherwise @@ -23,92 +24,81 @@ def is_mcp_available() -> bool: except ImportError: return False + def normalize_server_name(server_name: str) -> str: """ Normalize server name by replacing spaces with underscores """ return server_name.replace(" ", "_") + def validate_and_normalize_mcp_server_payload(payload: Any) -> None: """ Validate and normalize MCP server payload fields (server_name and alias). - + This function: 1. Validates that server_name and alias don't contain the MCP_TOOL_PREFIX_SEPARATOR 2. Normalizes alias by replacing spaces with underscores 3. Sets default alias if not provided (using server_name as base) - + Args: payload: The payload object containing server_name and alias fields - + Raises: HTTPException: If validation fails """ # Server name validation: disallow '-' - if hasattr(payload, 'server_name') and payload.server_name: + if hasattr(payload, "server_name") and payload.server_name: validate_mcp_server_name(payload.server_name, raise_http_exception=True) - + # Alias validation: disallow '-' - if hasattr(payload, 'alias') and payload.alias: + if hasattr(payload, "alias") and payload.alias: validate_mcp_server_name(payload.alias, raise_http_exception=True) - + # Alias normalization and defaulting - alias = getattr(payload, 'alias', None) - server_name = getattr(payload, 'server_name', None) - + alias = getattr(payload, "alias", None) + server_name = getattr(payload, "server_name", None) + if not alias and server_name: alias = normalize_server_name(server_name) elif alias: alias = normalize_server_name(alias) - + # Update the payload with normalized alias - if hasattr(payload, 'alias'): + if hasattr(payload, "alias"): payload.alias = alias -def add_server_prefix_to_tool_name(tool_name: str, server_name: str) -> str: - """ - Add server name prefix to tool name - Args: - tool_name: Original tool name - server_name: MCP server name - - Returns: - Prefixed tool name in format: server_name::tool_name - """ +def add_server_prefix_to_name(name: str, server_name: str) -> str: + """Add server name prefix to any MCP resource name.""" formatted_server_name = normalize_server_name(server_name) return MCP_TOOL_PREFIX_FORMAT.format( server_name=formatted_server_name, separator=MCP_TOOL_PREFIX_SEPARATOR, - tool_name=tool_name + tool_name=name, ) + def get_server_prefix(server: Any) -> str: """Return the prefix for a server: alias if present, else server_name, else server_id""" - if hasattr(server, 'alias') and server.alias: + if hasattr(server, "alias") and server.alias: return server.alias - if hasattr(server, 'server_name') and server.server_name: + if hasattr(server, "server_name") and server.server_name: return server.server_name - if hasattr(server, 'server_id'): + if hasattr(server, "server_id"): return server.server_id return "" -def get_server_name_prefix_tool_mcp(prefixed_tool_name: str) -> Tuple[str, str]: - """ - Remove server name prefix from tool name - Args: - prefixed_tool_name: Tool name with server prefix - - Returns: - Tuple of (original_tool_name, server_name) - """ - if MCP_TOOL_PREFIX_SEPARATOR in prefixed_tool_name: - parts = prefixed_tool_name.split(MCP_TOOL_PREFIX_SEPARATOR, 1) +def split_server_prefix_from_name(prefixed_name: str) -> Tuple[str, str]: + """Return the unprefixed name plus the server name used as prefix.""" + if MCP_TOOL_PREFIX_SEPARATOR in prefixed_name: + parts = prefixed_name.split(MCP_TOOL_PREFIX_SEPARATOR, 1) if len(parts) == 2: - return parts[1], parts[0] # tool_name, server_name - return prefixed_tool_name, "" # No prefix found, return original name + return parts[1], parts[0] + return prefixed_name, "" + def is_tool_name_prefixed(tool_name: str) -> bool: """ @@ -122,14 +112,17 @@ def is_tool_name_prefixed(tool_name: str) -> bool: """ return MCP_TOOL_PREFIX_SEPARATOR in tool_name -def validate_mcp_server_name(server_name: str, raise_http_exception: bool = False) -> None: + +def validate_mcp_server_name( + server_name: str, raise_http_exception: bool = False +) -> None: """ Validate that MCP server name does not contain 'MCP_TOOL_PREFIX_SEPARATOR'. - + Args: server_name: The server name to validate raise_http_exception: If True, raises HTTPException instead of generic Exception - + Raises: Exception or HTTPException: If server name contains 'MCP_TOOL_PREFIX_SEPARATOR' """ @@ -138,9 +131,9 @@ def validate_mcp_server_name(server_name: str, raise_http_exception: bool = Fals if raise_http_exception: from fastapi import HTTPException from starlette import status + raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail={"error": error_message} + status_code=status.HTTP_400_BAD_REQUEST, detail={"error": error_message} ) else: raise Exception(error_message) diff --git a/litellm/proxy/_experimental/out/_next/static/zzKcMfj4Db-ZZ7hcspdhR/_buildManifest.js b/litellm/proxy/_experimental/out/_next/static/TkaZwJ2-CB-TmqPtTfFGx/_buildManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/zzKcMfj4Db-ZZ7hcspdhR/_buildManifest.js rename to litellm/proxy/_experimental/out/_next/static/TkaZwJ2-CB-TmqPtTfFGx/_buildManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/zzKcMfj4Db-ZZ7hcspdhR/_ssgManifest.js b/litellm/proxy/_experimental/out/_next/static/TkaZwJ2-CB-TmqPtTfFGx/_ssgManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/zzKcMfj4Db-ZZ7hcspdhR/_ssgManifest.js rename to litellm/proxy/_experimental/out/_next/static/TkaZwJ2-CB-TmqPtTfFGx/_ssgManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/1051-64b6f8c98af90027.js b/litellm/proxy/_experimental/out/_next/static/chunks/1051-64b6f8c98af90027.js new file mode 100644 index 00000000000..e83ac5816ce --- /dev/null +++ b/litellm/proxy/_experimental/out/_next/static/chunks/1051-64b6f8c98af90027.js @@ -0,0 +1 @@ +(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[1051],{79276:function(e,t,n){"use strict";n.d(t,{Z:function(){return a}});var r=n(1119),l=n(2265),i={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M868 545.5L536.1 163a31.96 31.96 0 00-48.3 0L156 545.5a7.97 7.97 0 006 13.2h81c4.6 0 9-2 12.1-5.5L474 300.9V864c0 4.4 3.6 8 8 8h60c4.4 0 8-3.6 8-8V300.9l218.9 252.3c3 3.5 7.4 5.5 12.1 5.5h81c6.8 0 10.5-8 6-13.2z"}}]},name:"arrow-up",theme:"outlined"},o=n(55015),a=l.forwardRef(function(e,t){return l.createElement(o.Z,(0,r.Z)({},e,{ref:t,icon:i}))})},83322:function(e,t,n){"use strict";n.d(t,{Z:function(){return a}});var r=n(1119),l=n(2265),i={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M632 888H392c-4.4 0-8 3.6-8 8v32c0 17.7 14.3 32 32 32h192c17.7 0 32-14.3 32-32v-32c0-4.4-3.6-8-8-8zM512 64c-181.1 0-328 146.9-328 328 0 121.4 66 227.4 164 284.1V792c0 17.7 14.3 32 32 32h264c17.7 0 32-14.3 32-32V676.1c98-56.7 164-162.7 164-284.1 0-181.1-146.9-328-328-328zm127.9 549.8L604 634.6V752H420V634.6l-35.9-20.8C305.4 568.3 256 484.5 256 392c0-141.4 114.6-256 256-256s256 114.6 256 256c0 92.5-49.4 176.3-128.1 221.8z"}}]},name:"bulb",theme:"outlined"},o=n(55015),a=l.forwardRef(function(e,t){return l.createElement(o.Z,(0,r.Z)({},e,{ref:t,icon:i}))})},26430:function(e,t,n){"use strict";n.d(t,{Z:function(){return a}});var r=n(1119),l=n(2265),i={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"defs",attrs:{},children:[{tag:"style",attrs:{}}]},{tag:"path",attrs:{d:"M899.1 869.6l-53-305.6H864c14.4 0 26-11.6 26-26V346c0-14.4-11.6-26-26-26H618V138c0-14.4-11.6-26-26-26H432c-14.4 0-26 11.6-26 26v182H160c-14.4 0-26 11.6-26 26v192c0 14.4 11.6 26 26 26h17.9l-53 305.6a25.95 25.95 0 0025.6 30.4h723c1.5 0 3-.1 4.4-.4a25.88 25.88 0 0021.2-30zM204 390h272V182h72v208h272v104H204V390zm468 440V674c0-4.4-3.6-8-8-8h-48c-4.4 0-8 3.6-8 8v156H416V674c0-4.4-3.6-8-8-8h-48c-4.4 0-8 3.6-8 8v156H202.8l45.1-260H776l45.1 260H672z"}}]},name:"clear",theme:"outlined"},o=n(55015),a=l.forwardRef(function(e,t){return l.createElement(o.Z,(0,r.Z)({},e,{ref:t,icon:i}))})},5540:function(e,t,n){"use strict";n.d(t,{Z:function(){return a}});var r=n(1119),l=n(2265),i={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M512 64C264.6 64 64 264.6 64 512s200.6 448 448 448 448-200.6 448-448S759.4 64 512 64zm0 820c-205.4 0-372-166.6-372-372s166.6-372 372-372 372 166.6 372 372-166.6 372-372 372z"}},{tag:"path",attrs:{d:"M686.7 638.6L544.1 535.5V288c0-4.4-3.6-8-8-8H488c-4.4 0-8 3.6-8 8v275.4c0 2.6 1.2 5 3.3 6.5l165.4 120.6c3.6 2.6 8.6 1.8 11.2-1.7l28.6-39c2.6-3.7 1.8-8.7-1.8-11.2z"}}]},name:"clock-circle",theme:"outlined"},o=n(55015),a=l.forwardRef(function(e,t){return l.createElement(o.Z,(0,r.Z)({},e,{ref:t,icon:i}))})},11894:function(e,t,n){"use strict";n.d(t,{Z:function(){return a}});var r=n(1119),l=n(2265),i={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M516 673c0 4.4 3.4 8 7.5 8h185c4.1 0 7.5-3.6 7.5-8v-48c0-4.4-3.4-8-7.5-8h-185c-4.1 0-7.5 3.6-7.5 8v48zm-194.9 6.1l192-161c3.8-3.2 3.8-9.1 0-12.3l-192-160.9A7.95 7.95 0 00308 351v62.7c0 2.4 1 4.6 2.9 6.1L420.7 512l-109.8 92.2a8.1 8.1 0 00-2.9 6.1V673c0 6.8 7.9 10.5 13.1 6.1zM880 112H144c-17.7 0-32 14.3-32 32v736c0 17.7 14.3 32 32 32h736c17.7 0 32-14.3 32-32V144c0-17.7-14.3-32-32-32zm-40 728H184V184h656v656z"}}]},name:"code",theme:"outlined"},o=n(55015),a=l.forwardRef(function(e,t){return l.createElement(o.Z,(0,r.Z)({},e,{ref:t,icon:i}))})},62670:function(e,t,n){"use strict";n.d(t,{Z:function(){return a}});var r=n(1119),l=n(2265),i={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M512 64C264.6 64 64 264.6 64 512s200.6 448 448 448 448-200.6 448-448S759.4 64 512 64zm0 820c-205.4 0-372-166.6-372-372s166.6-372 372-372 372 166.6 372 372-166.6 372-372 372zm47.7-395.2l-25.4-5.9V348.6c38 5.2 61.5 29 65.5 58.2.5 4 3.9 6.9 7.9 6.9h44.9c4.7 0 8.4-4.1 8-8.8-6.1-62.3-57.4-102.3-125.9-109.2V263c0-4.4-3.6-8-8-8h-28.1c-4.4 0-8 3.6-8 8v33c-70.8 6.9-126.2 46-126.2 119 0 67.6 49.8 100.2 102.1 112.7l24.7 6.3v142.7c-44.2-5.9-69-29.5-74.1-61.3-.6-3.8-4-6.6-7.9-6.6H363c-4.7 0-8.4 4-8 8.7 4.5 55 46.2 105.6 135.2 112.1V761c0 4.4 3.6 8 8 8h28.4c4.4 0 8-3.6 8-8.1l-.2-31.7c78.3-6.9 134.3-48.8 134.3-124-.1-69.4-44.2-100.4-109-116.4zm-68.6-16.2c-5.6-1.6-10.3-3.1-15-5-33.8-12.2-49.5-31.9-49.5-57.3 0-36.3 27.5-57 64.5-61.7v124zM534.3 677V543.3c3.1.9 5.9 1.6 8.8 2.2 47.3 14.4 63.2 34.4 63.2 65.1 0 39.1-29.4 62.6-72 66.4z"}}]},name:"dollar",theme:"outlined"},o=n(55015),a=l.forwardRef(function(e,t){return l.createElement(o.Z,(0,r.Z)({},e,{ref:t,icon:i}))})},11741:function(e,t,n){"use strict";n.d(t,{Z:function(){return a}});var r=n(1119),l=n(2265),i={icon:{tag:"svg",attrs:{"fill-rule":"evenodd",viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M880 912H144c-17.7 0-32-14.3-32-32V144c0-17.7 14.3-32 32-32h360c4.4 0 8 3.6 8 8v56c0 4.4-3.6 8-8 8H184v656h656V520c0-4.4 3.6-8 8-8h56c4.4 0 8 3.6 8 8v360c0 17.7-14.3 32-32 32zM770.87 199.13l-52.2-52.2a8.01 8.01 0 014.7-13.6l179.4-21c5.1-.6 9.5 3.7 8.9 8.9l-21 179.4c-.8 6.6-8.9 9.4-13.6 4.7l-52.4-52.4-256.2 256.2a8.03 8.03 0 01-11.3 0l-42.4-42.4a8.03 8.03 0 010-11.3l256.1-256.3z"}}]},name:"export",theme:"outlined"},o=n(55015),a=l.forwardRef(function(e,t){return l.createElement(o.Z,(0,r.Z)({},e,{ref:t,icon:i}))})},71282:function(e,t,n){"use strict";n.d(t,{Z:function(){return a}});var r=n(1119),l=n(2265),i={icon:{tag:"svg",attrs:{"fill-rule":"evenodd",viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M880 912H144c-17.7 0-32-14.3-32-32V144c0-17.7 14.3-32 32-32h360c4.4 0 8 3.6 8 8v56c0 4.4-3.6 8-8 8H184v656h656V520c0-4.4 3.6-8 8-8h56c4.4 0 8 3.6 8 8v360c0 17.7-14.3 32-32 32zM653.3 424.6l52.2 52.2a8.01 8.01 0 01-4.7 13.6l-179.4 21c-5.1.6-9.5-3.7-8.9-8.9l21-179.4c.8-6.6 8.9-9.4 13.6-4.7l52.4 52.4 256.2-256.2c3.1-3.1 8.2-3.1 11.3 0l42.4 42.4c3.1 3.1 3.1 8.2 0 11.3L653.3 424.6z"}}]},name:"import",theme:"outlined"},o=n(55015),a=l.forwardRef(function(e,t){return l.createElement(o.Z,(0,r.Z)({},e,{ref:t,icon:i}))})},16601:function(e,t,n){"use strict";n.d(t,{Z:function(){return a}});var r=n(1119),l=n(2265),i={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M872 394c4.4 0 8-3.6 8-8v-60c0-4.4-3.6-8-8-8H708V152c0-4.4-3.6-8-8-8h-64c-4.4 0-8 3.6-8 8v166H400V152c0-4.4-3.6-8-8-8h-64c-4.4 0-8 3.6-8 8v166H152c-4.4 0-8 3.6-8 8v60c0 4.4 3.6 8 8 8h168v236H152c-4.4 0-8 3.6-8 8v60c0 4.4 3.6 8 8 8h168v166c0 4.4 3.6 8 8 8h64c4.4 0 8-3.6 8-8V706h228v166c0 4.4 3.6 8 8 8h64c4.4 0 8-3.6 8-8V706h164c4.4 0 8-3.6 8-8v-60c0-4.4-3.6-8-8-8H708V394h164zM628 630H400V394h228v236z"}}]},name:"number",theme:"outlined"},o=n(55015),a=l.forwardRef(function(e,t){return l.createElement(o.Z,(0,r.Z)({},e,{ref:t,icon:i}))})},69993:function(e,t,n){"use strict";n.d(t,{Z:function(){return a}});var r=n(1119),l=n(2265),i={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M300 328a60 60 0 10120 0 60 60 0 10-120 0zM852 64H172c-17.7 0-32 14.3-32 32v660c0 17.7 14.3 32 32 32h680c17.7 0 32-14.3 32-32V96c0-17.7-14.3-32-32-32zm-32 660H204V128h616v596zM604 328a60 60 0 10120 0 60 60 0 10-120 0zm250.2 556H169.8c-16.5 0-29.8 14.3-29.8 32v36c0 4.4 3.3 8 7.4 8h729.1c4.1 0 7.4-3.6 7.4-8v-36c.1-17.7-13.2-32-29.7-32zM664 508H360c-4.4 0-8 3.6-8 8v60c0 4.4 3.6 8 8 8h304c4.4 0 8-3.6 8-8v-60c0-4.4-3.6-8-8-8z"}}]},name:"robot",theme:"outlined"},o=n(55015),a=l.forwardRef(function(e,t){return l.createElement(o.Z,(0,r.Z)({},e,{ref:t,icon:i}))})},58630:function(e,t,n){"use strict";n.d(t,{Z:function(){return a}});var r=n(1119),l=n(2265),i={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M876.6 239.5c-.5-.9-1.2-1.8-2-2.5-5-5-13.1-5-18.1 0L684.2 409.3l-67.9-67.9L788.7 169c.8-.8 1.4-1.6 2-2.5 3.6-6.1 1.6-13.9-4.5-17.5-98.2-58-226.8-44.7-311.3 39.7-67 67-89.2 162-66.5 247.4l-293 293c-3 3-2.8 7.9.3 11l169.7 169.7c3.1 3.1 8.1 3.3 11 .3l292.9-292.9c85.5 22.8 180.5.7 247.6-66.4 84.4-84.5 97.7-213.1 39.7-311.3zM786 499.8c-58.1 58.1-145.3 69.3-214.6 33.6l-8.8 8.8-.1-.1-274 274.1-79.2-79.2 230.1-230.1s0 .1.1.1l52.8-52.8c-35.7-69.3-24.5-156.5 33.6-214.6a184.2 184.2 0 01144-53.5L537 318.9a32.05 32.05 0 000 45.3l124.5 124.5a32.05 32.05 0 0045.3 0l132.8-132.8c3.7 51.8-14.4 104.8-53.6 143.9z"}}]},name:"tool",theme:"outlined"},o=n(55015),a=l.forwardRef(function(e,t){return l.createElement(o.Z,(0,r.Z)({},e,{ref:t,icon:i}))})},92570:function(e,t,n){"use strict";n.d(t,{Z:function(){return r}});let r=e=>e?"function"==typeof e?e():e:null},20435:function(e,t,n){"use strict";var r=n(2265),l=n(36760),i=n.n(l),o=n(5769),a=n(92570),u=n(71744),c=n(72262),s=function(e,t){var n={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(n[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var l=0,r=Object.getOwnPropertySymbols(e);lt.indexOf(r[l])&&Object.prototype.propertyIsEnumerable.call(e,r[l])&&(n[r[l]]=e[r[l]]);return n};let f=(e,t,n)=>t||n?r.createElement(r.Fragment,null,t&&r.createElement("div",{className:"".concat(e,"-title")},(0,a.Z)(t)),r.createElement("div",{className:"".concat(e,"-inner-content")},(0,a.Z)(n))):null,p=e=>{let{hashId:t,prefixCls:n,className:l,style:a,placement:u="top",title:c,content:s,children:p}=e;return r.createElement("div",{className:i()(t,n,"".concat(n,"-pure"),"".concat(n,"-placement-").concat(u),l),style:a},r.createElement("div",{className:"".concat(n,"-arrow")}),r.createElement(o.G,Object.assign({},e,{className:t,prefixCls:n}),p||f(n,c,s)))};t.ZP=e=>{let{prefixCls:t,className:n}=e,l=s(e,["prefixCls","className"]),{getPrefixCls:o}=r.useContext(u.E_),a=o("popover",t),[f,d,h]=(0,c.Z)(a);return f(r.createElement(p,Object.assign({},l,{prefixCls:a,hashId:d,className:i()(n,h)})))}},79326:function(e,t,n){"use strict";var r=n(2265),l=n(36760),i=n.n(l),o=n(92570),a=n(68710),u=n(71744),c=n(89970),s=n(20435),f=n(72262),p=function(e,t){var n={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(n[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var l=0,r=Object.getOwnPropertySymbols(e);lt.indexOf(r[l])&&Object.prototype.propertyIsEnumerable.call(e,r[l])&&(n[r[l]]=e[r[l]]);return n};let d=e=>{let{title:t,content:n,prefixCls:l}=e;return r.createElement(r.Fragment,null,t&&r.createElement("div",{className:"".concat(l,"-title")},(0,o.Z)(t)),r.createElement("div",{className:"".concat(l,"-inner-content")},(0,o.Z)(n)))},h=r.forwardRef((e,t)=>{let{prefixCls:n,title:l,content:o,overlayClassName:s,placement:h="top",trigger:m="hover",mouseEnterDelay:g=.1,mouseLeaveDelay:y=.1,overlayStyle:v={}}=e,x=p(e,["prefixCls","title","content","overlayClassName","placement","trigger","mouseEnterDelay","mouseLeaveDelay","overlayStyle"]),{getPrefixCls:k}=r.useContext(u.E_),b=k("popover",n),[w,S,E]=(0,f.Z)(b),C=k(),P=i()(s,S,E);return w(r.createElement(c.Z,Object.assign({placement:h,trigger:m,mouseEnterDelay:g,mouseLeaveDelay:y,overlayStyle:v},x,{prefixCls:b,overlayClassName:P,ref:t,overlay:l||o?r.createElement(d,{prefixCls:b,title:l,content:o}):null,transitionName:(0,a.m)(C,"zoom-big",x.transitionName),"data-popover-inject":!0})))});h._InternalPanelDoNotUseOrYouWillBeFired=s.ZP,t.Z=h},72262:function(e,t,n){"use strict";var r=n(12918),l=n(691),i=n(88260),o=n(53454),a=n(80669),u=n(3104),c=n(34442);let s=e=>{let{componentCls:t,popoverColor:n,titleMinWidth:l,fontWeightStrong:o,innerPadding:a,boxShadowSecondary:u,colorTextHeading:c,borderRadiusLG:s,zIndexPopup:f,titleMarginBottom:p,colorBgElevated:d,popoverBg:h,titleBorderBottom:m,innerContentPadding:g,titlePadding:y}=e;return[{[t]:Object.assign(Object.assign({},(0,r.Wf)(e)),{position:"absolute",top:0,left:{_skip_check_:!0,value:0},zIndex:f,fontWeight:"normal",whiteSpace:"normal",textAlign:"start",cursor:"auto",userSelect:"text",transformOrigin:"var(--arrow-x, 50%) var(--arrow-y, 50%)","--antd-arrow-background-color":d,"&-rtl":{direction:"rtl"},"&-hidden":{display:"none"},["".concat(t,"-content")]:{position:"relative"},["".concat(t,"-inner")]:{backgroundColor:h,backgroundClip:"padding-box",borderRadius:s,boxShadow:u,padding:a},["".concat(t,"-title")]:{minWidth:l,marginBottom:p,color:c,fontWeight:o,borderBottom:m,padding:y},["".concat(t,"-inner-content")]:{color:n,padding:g}})},(0,i.ZP)(e,"var(--antd-arrow-background-color)"),{["".concat(t,"-pure")]:{position:"relative",maxWidth:"none",margin:e.sizePopupArrow,display:"inline-block",["".concat(t,"-content")]:{display:"inline-block"}}}]},f=e=>{let{componentCls:t}=e;return{[t]:o.i.map(n=>{let r=e["".concat(n,"6")];return{["&".concat(t,"-").concat(n)]:{"--antd-arrow-background-color":r,["".concat(t,"-inner")]:{backgroundColor:r},["".concat(t,"-arrow")]:{background:"transparent"}}}})}};t.Z=(0,a.I$)("Popover",e=>{let{colorBgElevated:t,colorText:n}=e,r=(0,u.TS)(e,{popoverBg:t,popoverColor:n});return[s(r),f(r),(0,l._y)(r,"zoom-big")]},e=>{let{lineWidth:t,controlHeight:n,fontHeight:r,padding:l,wireframe:o,zIndexPopupBase:a,borderRadiusLG:u,marginXS:s,lineType:f,colorSplit:p,paddingSM:d}=e,h=n-r;return Object.assign(Object.assign(Object.assign({titleMinWidth:177,zIndexPopup:a+30},(0,c.w)(e)),(0,i.wZ)({contentRadius:u,limitVerticalRadius:!0})),{innerPadding:o?0:12,titleMarginBottom:o?0:s,titlePadding:o?"".concat(h/2,"px ").concat(l,"px ").concat(h/2-t,"px"):0,titleBorderBottom:o?"".concat(t,"px ").concat(f," ").concat(p):"none",innerContentPadding:o?"".concat(d,"px ").concat(l,"px"):0})},{resetStyle:!1,deprecatedTokens:[["width","titleMinWidth"],["minWidth","titleMinWidth"]]})},6500:function(e){"use strict";var t=Object.prototype.hasOwnProperty,n=Object.prototype.toString,r=Object.defineProperty,l=Object.getOwnPropertyDescriptor,i=function(e){return"function"==typeof Array.isArray?Array.isArray(e):"[object Array]"===n.call(e)},o=function(e){if(!e||"[object Object]"!==n.call(e))return!1;var r,l=t.call(e,"constructor"),i=e.constructor&&e.constructor.prototype&&t.call(e.constructor.prototype,"isPrototypeOf");if(e.constructor&&!l&&!i)return!1;for(r in e);return void 0===r||t.call(e,r)},a=function(e,t){r&&"__proto__"===t.name?r(e,t.name,{enumerable:!0,configurable:!0,value:t.newValue,writable:!0}):e[t.name]=t.newValue},u=function(e,n){if("__proto__"===n){if(!t.call(e,n))return;if(l)return l(e,n).value}return e[n]};e.exports=function e(){var t,n,r,l,c,s,f=arguments[0],p=1,d=arguments.length,h=!1;for("boolean"==typeof f&&(h=f,f=arguments[1]||{},p=2),(null==f||"object"!=typeof f&&"function"!=typeof f)&&(f={});p code':{position:"relative",zIndex:"1",borderLeft:"10px solid #358ccb",boxShadow:"-1px 0px 0px 0px #358ccb, 0px 0px 0px 1px #dfdfdf",backgroundColor:"#fdfdfd",backgroundImage:"linear-gradient(transparent 50%, rgba(69, 142, 209, 0.04) 50%)",backgroundSize:"3em 3em",backgroundOrigin:"content-box",backgroundAttachment:"local"},':not(pre) > code[class*="language-"]':{backgroundColor:"#fdfdfd",WebkitBoxSizing:"border-box",MozBoxSizing:"border-box",boxSizing:"border-box",marginBottom:"1em",position:"relative",padding:".2em",borderRadius:"0.3em",color:"#c92c2c",border:"1px solid rgba(0, 0, 0, 0.1)",display:"inline",whiteSpace:"normal"},'pre[class*="language-"]:before':{content:"''",display:"block",position:"absolute",bottom:"0.75em",left:"0.18em",width:"40%",height:"20%",maxHeight:"13em",boxShadow:"0px 13px 8px #979797",WebkitTransform:"rotate(-2deg)",MozTransform:"rotate(-2deg)",msTransform:"rotate(-2deg)",OTransform:"rotate(-2deg)",transform:"rotate(-2deg)"},'pre[class*="language-"]:after':{content:"''",display:"block",position:"absolute",bottom:"0.75em",left:"auto",width:"40%",height:"20%",maxHeight:"13em",boxShadow:"0px 13px 8px #979797",WebkitTransform:"rotate(2deg)",MozTransform:"rotate(2deg)",msTransform:"rotate(2deg)",OTransform:"rotate(2deg)",transform:"rotate(2deg)",right:"0.75em"},comment:{color:"#7D8B99"},"block-comment":{color:"#7D8B99"},prolog:{color:"#7D8B99"},doctype:{color:"#7D8B99"},cdata:{color:"#7D8B99"},punctuation:{color:"#5F6364"},property:{color:"#c92c2c"},tag:{color:"#c92c2c"},boolean:{color:"#c92c2c"},number:{color:"#c92c2c"},"function-name":{color:"#c92c2c"},constant:{color:"#c92c2c"},symbol:{color:"#c92c2c"},deleted:{color:"#c92c2c"},selector:{color:"#2f9c0a"},"attr-name":{color:"#2f9c0a"},string:{color:"#2f9c0a"},char:{color:"#2f9c0a"},function:{color:"#2f9c0a"},builtin:{color:"#2f9c0a"},inserted:{color:"#2f9c0a"},operator:{color:"#a67f59",background:"rgba(255, 255, 255, 0.5)"},entity:{color:"#a67f59",background:"rgba(255, 255, 255, 0.5)",cursor:"help"},url:{color:"#a67f59",background:"rgba(255, 255, 255, 0.5)"},variable:{color:"#a67f59",background:"rgba(255, 255, 255, 0.5)"},atrule:{color:"#1990b8"},"attr-value":{color:"#1990b8"},keyword:{color:"#1990b8"},"class-name":{color:"#1990b8"},regex:{color:"#e90"},important:{color:"#e90",fontWeight:"normal"},".language-css .token.string":{color:"#a67f59",background:"rgba(255, 255, 255, 0.5)"},".style .token.string":{color:"#a67f59",background:"rgba(255, 255, 255, 0.5)"},bold:{fontWeight:"bold"},italic:{fontStyle:"italic"},namespace:{Opacity:".7"},'pre[class*="language-"].line-numbers.line-numbers':{paddingLeft:"0"},'pre[class*="language-"].line-numbers.line-numbers code':{paddingLeft:"3.8em"},'pre[class*="language-"].line-numbers.line-numbers .line-numbers-rows':{left:"0"},'pre[class*="language-"][data-line]':{paddingTop:"0",paddingBottom:"0",paddingLeft:"0"},"pre[data-line] code":{position:"relative",paddingLeft:"4em"},"pre .line-highlight":{marginTop:"0"}}},52744:function(e,t,n){"use strict";var r=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(e,t){var n=null;if(!e||"string"!=typeof e)return n;var r=(0,l.default)(e),i="function"==typeof t;return r.forEach(function(e){if("declaration"===e.type){var r=e.property,l=e.value;i?t(r,l,e):l&&((n=n||{})[r]=l)}}),n};var l=r(n(80662))},243:function(e,t,n){"use strict";n.d(t,{U:function(){return nA}});var r={};n.r(r),n.d(r,{boolean:function(){return g},booleanish:function(){return y},commaOrSpaceSeparated:function(){return w},commaSeparated:function(){return b},number:function(){return x},overloadedBoolean:function(){return v},spaceSeparated:function(){return k}});var l={};n.r(l),n.d(l,{attentionMarkers:function(){return tT},contentInitial:function(){return tS},disable:function(){return tI},document:function(){return tw},flow:function(){return tC},flowInitial:function(){return tE},insideSpan:function(){return tz},string:function(){return tP},text:function(){return tO}});let i=/^[$_\p{ID_Start}][$_\u{200C}\u{200D}\p{ID_Continue}]*$/u,o=/^[$_\p{ID_Start}][-$_\u{200C}\u{200D}\p{ID_Continue}]*$/u,a={};function u(e,t){return((t||a).jsx?o:i).test(e)}let c=/[ \t\n\f\r]/g;function s(e){return""===e.replace(c,"")}class f{constructor(e,t,n){this.property=e,this.normal=t,n&&(this.space=n)}}function p(e,t){let n={},r={},l=-1;for(;++l"xlink:"+t.slice(5).toLowerCase(),properties:{xLinkActuate:null,xLinkArcRole:null,xLinkHref:null,xLinkRole:null,xLinkShow:null,xLinkTitle:null,xLinkType:null}}),T=O({space:"xml",transform:(e,t)=>"xml:"+t.slice(3).toLowerCase(),properties:{xmlLang:null,xmlBase:null,xmlSpace:null}});function I(e,t){return t in e?e[t]:t}function A(e,t){return I(e,t.toLowerCase())}let M=O({space:"xmlns",attributes:{xmlnsxlink:"xmlns:xlink"},transform:A,properties:{xmlns:null,xmlnsXLink:null}}),L=O({transform:(e,t)=>"role"===t?t:"aria-"+t.slice(4).toLowerCase(),properties:{ariaActiveDescendant:null,ariaAtomic:y,ariaAutoComplete:null,ariaBusy:y,ariaChecked:y,ariaColCount:x,ariaColIndex:x,ariaColSpan:x,ariaControls:k,ariaCurrent:null,ariaDescribedBy:k,ariaDetails:null,ariaDisabled:y,ariaDropEffect:k,ariaErrorMessage:null,ariaExpanded:y,ariaFlowTo:k,ariaGrabbed:y,ariaHasPopup:null,ariaHidden:y,ariaInvalid:null,ariaKeyShortcuts:null,ariaLabel:null,ariaLabelledBy:k,ariaLevel:x,ariaLive:null,ariaModal:y,ariaMultiLine:y,ariaMultiSelectable:y,ariaOrientation:null,ariaOwns:k,ariaPlaceholder:null,ariaPosInSet:x,ariaPressed:y,ariaReadOnly:y,ariaRelevant:null,ariaRequired:y,ariaRoleDescription:k,ariaRowCount:x,ariaRowIndex:x,ariaRowSpan:x,ariaSelected:y,ariaSetSize:x,ariaSort:null,ariaValueMax:x,ariaValueMin:x,ariaValueNow:x,ariaValueText:null,role:null}}),D=O({space:"html",attributes:{acceptcharset:"accept-charset",classname:"class",htmlfor:"for",httpequiv:"http-equiv"},transform:A,mustUseProperty:["checked","multiple","muted","selected"],properties:{abbr:null,accept:b,acceptCharset:k,accessKey:k,action:null,allow:null,allowFullScreen:g,allowPaymentRequest:g,allowUserMedia:g,alt:null,as:null,async:g,autoCapitalize:null,autoComplete:k,autoFocus:g,autoPlay:g,blocking:k,capture:null,charSet:null,checked:g,cite:null,className:k,cols:x,colSpan:null,content:null,contentEditable:y,controls:g,controlsList:k,coords:x|b,crossOrigin:null,data:null,dateTime:null,decoding:null,default:g,defer:g,dir:null,dirName:null,disabled:g,download:v,draggable:y,encType:null,enterKeyHint:null,fetchPriority:null,form:null,formAction:null,formEncType:null,formMethod:null,formNoValidate:g,formTarget:null,headers:k,height:x,hidden:g,high:x,href:null,hrefLang:null,htmlFor:k,httpEquiv:k,id:null,imageSizes:null,imageSrcSet:null,inert:g,inputMode:null,integrity:null,is:null,isMap:g,itemId:null,itemProp:k,itemRef:k,itemScope:g,itemType:k,kind:null,label:null,lang:null,language:null,list:null,loading:null,loop:g,low:x,manifest:null,max:null,maxLength:x,media:null,method:null,min:null,minLength:x,multiple:g,muted:g,name:null,nonce:null,noModule:g,noValidate:g,onAbort:null,onAfterPrint:null,onAuxClick:null,onBeforeMatch:null,onBeforePrint:null,onBeforeToggle:null,onBeforeUnload:null,onBlur:null,onCancel:null,onCanPlay:null,onCanPlayThrough:null,onChange:null,onClick:null,onClose:null,onContextLost:null,onContextMenu:null,onContextRestored:null,onCopy:null,onCueChange:null,onCut:null,onDblClick:null,onDrag:null,onDragEnd:null,onDragEnter:null,onDragExit:null,onDragLeave:null,onDragOver:null,onDragStart:null,onDrop:null,onDurationChange:null,onEmptied:null,onEnded:null,onError:null,onFocus:null,onFormData:null,onHashChange:null,onInput:null,onInvalid:null,onKeyDown:null,onKeyPress:null,onKeyUp:null,onLanguageChange:null,onLoad:null,onLoadedData:null,onLoadedMetadata:null,onLoadEnd:null,onLoadStart:null,onMessage:null,onMessageError:null,onMouseDown:null,onMouseEnter:null,onMouseLeave:null,onMouseMove:null,onMouseOut:null,onMouseOver:null,onMouseUp:null,onOffline:null,onOnline:null,onPageHide:null,onPageShow:null,onPaste:null,onPause:null,onPlay:null,onPlaying:null,onPopState:null,onProgress:null,onRateChange:null,onRejectionHandled:null,onReset:null,onResize:null,onScroll:null,onScrollEnd:null,onSecurityPolicyViolation:null,onSeeked:null,onSeeking:null,onSelect:null,onSlotChange:null,onStalled:null,onStorage:null,onSubmit:null,onSuspend:null,onTimeUpdate:null,onToggle:null,onUnhandledRejection:null,onUnload:null,onVolumeChange:null,onWaiting:null,onWheel:null,open:g,optimum:x,pattern:null,ping:k,placeholder:null,playsInline:g,popover:null,popoverTarget:null,popoverTargetAction:null,poster:null,preload:null,readOnly:g,referrerPolicy:null,rel:k,required:g,reversed:g,rows:x,rowSpan:x,sandbox:k,scope:null,scoped:g,seamless:g,selected:g,shadowRootDelegatesFocus:g,shadowRootMode:null,shape:null,size:x,sizes:null,slot:null,span:x,spellCheck:y,src:null,srcDoc:null,srcLang:null,srcSet:null,start:x,step:null,style:null,tabIndex:x,target:null,title:null,translate:null,type:null,typeMustMatch:g,useMap:null,value:y,width:x,wrap:null,align:null,aLink:null,archive:k,axis:null,background:null,bgColor:null,border:x,borderColor:null,bottomMargin:x,cellPadding:null,cellSpacing:null,char:null,charOff:null,classId:null,clear:null,code:null,codeBase:null,codeType:null,color:null,compact:g,declare:g,event:null,face:null,frame:null,frameBorder:null,hSpace:x,leftMargin:x,link:null,longDesc:null,lowSrc:null,marginHeight:x,marginWidth:x,noResize:g,noHref:g,noShade:g,noWrap:g,object:null,profile:null,prompt:null,rev:null,rightMargin:x,rules:null,scheme:null,scrolling:y,standby:null,summary:null,text:null,topMargin:x,valueType:null,version:null,vAlign:null,vLink:null,vSpace:x,allowTransparency:null,autoCorrect:null,autoSave:null,disablePictureInPicture:g,disableRemotePlayback:g,prefix:null,property:null,results:x,security:null,unselectable:null}}),j=O({space:"svg",attributes:{accentHeight:"accent-height",alignmentBaseline:"alignment-baseline",arabicForm:"arabic-form",baselineShift:"baseline-shift",capHeight:"cap-height",className:"class",clipPath:"clip-path",clipRule:"clip-rule",colorInterpolation:"color-interpolation",colorInterpolationFilters:"color-interpolation-filters",colorProfile:"color-profile",colorRendering:"color-rendering",crossOrigin:"crossorigin",dataType:"datatype",dominantBaseline:"dominant-baseline",enableBackground:"enable-background",fillOpacity:"fill-opacity",fillRule:"fill-rule",floodColor:"flood-color",floodOpacity:"flood-opacity",fontFamily:"font-family",fontSize:"font-size",fontSizeAdjust:"font-size-adjust",fontStretch:"font-stretch",fontStyle:"font-style",fontVariant:"font-variant",fontWeight:"font-weight",glyphName:"glyph-name",glyphOrientationHorizontal:"glyph-orientation-horizontal",glyphOrientationVertical:"glyph-orientation-vertical",hrefLang:"hreflang",horizAdvX:"horiz-adv-x",horizOriginX:"horiz-origin-x",horizOriginY:"horiz-origin-y",imageRendering:"image-rendering",letterSpacing:"letter-spacing",lightingColor:"lighting-color",markerEnd:"marker-end",markerMid:"marker-mid",markerStart:"marker-start",navDown:"nav-down",navDownLeft:"nav-down-left",navDownRight:"nav-down-right",navLeft:"nav-left",navNext:"nav-next",navPrev:"nav-prev",navRight:"nav-right",navUp:"nav-up",navUpLeft:"nav-up-left",navUpRight:"nav-up-right",onAbort:"onabort",onActivate:"onactivate",onAfterPrint:"onafterprint",onBeforePrint:"onbeforeprint",onBegin:"onbegin",onCancel:"oncancel",onCanPlay:"oncanplay",onCanPlayThrough:"oncanplaythrough",onChange:"onchange",onClick:"onclick",onClose:"onclose",onCopy:"oncopy",onCueChange:"oncuechange",onCut:"oncut",onDblClick:"ondblclick",onDrag:"ondrag",onDragEnd:"ondragend",onDragEnter:"ondragenter",onDragExit:"ondragexit",onDragLeave:"ondragleave",onDragOver:"ondragover",onDragStart:"ondragstart",onDrop:"ondrop",onDurationChange:"ondurationchange",onEmptied:"onemptied",onEnd:"onend",onEnded:"onended",onError:"onerror",onFocus:"onfocus",onFocusIn:"onfocusin",onFocusOut:"onfocusout",onHashChange:"onhashchange",onInput:"oninput",onInvalid:"oninvalid",onKeyDown:"onkeydown",onKeyPress:"onkeypress",onKeyUp:"onkeyup",onLoad:"onload",onLoadedData:"onloadeddata",onLoadedMetadata:"onloadedmetadata",onLoadStart:"onloadstart",onMessage:"onmessage",onMouseDown:"onmousedown",onMouseEnter:"onmouseenter",onMouseLeave:"onmouseleave",onMouseMove:"onmousemove",onMouseOut:"onmouseout",onMouseOver:"onmouseover",onMouseUp:"onmouseup",onMouseWheel:"onmousewheel",onOffline:"onoffline",onOnline:"ononline",onPageHide:"onpagehide",onPageShow:"onpageshow",onPaste:"onpaste",onPause:"onpause",onPlay:"onplay",onPlaying:"onplaying",onPopState:"onpopstate",onProgress:"onprogress",onRateChange:"onratechange",onRepeat:"onrepeat",onReset:"onreset",onResize:"onresize",onScroll:"onscroll",onSeeked:"onseeked",onSeeking:"onseeking",onSelect:"onselect",onShow:"onshow",onStalled:"onstalled",onStorage:"onstorage",onSubmit:"onsubmit",onSuspend:"onsuspend",onTimeUpdate:"ontimeupdate",onToggle:"ontoggle",onUnload:"onunload",onVolumeChange:"onvolumechange",onWaiting:"onwaiting",onZoom:"onzoom",overlinePosition:"overline-position",overlineThickness:"overline-thickness",paintOrder:"paint-order",panose1:"panose-1",pointerEvents:"pointer-events",referrerPolicy:"referrerpolicy",renderingIntent:"rendering-intent",shapeRendering:"shape-rendering",stopColor:"stop-color",stopOpacity:"stop-opacity",strikethroughPosition:"strikethrough-position",strikethroughThickness:"strikethrough-thickness",strokeDashArray:"stroke-dasharray",strokeDashOffset:"stroke-dashoffset",strokeLineCap:"stroke-linecap",strokeLineJoin:"stroke-linejoin",strokeMiterLimit:"stroke-miterlimit",strokeOpacity:"stroke-opacity",strokeWidth:"stroke-width",tabIndex:"tabindex",textAnchor:"text-anchor",textDecoration:"text-decoration",textRendering:"text-rendering",transformOrigin:"transform-origin",typeOf:"typeof",underlinePosition:"underline-position",underlineThickness:"underline-thickness",unicodeBidi:"unicode-bidi",unicodeRange:"unicode-range",unitsPerEm:"units-per-em",vAlphabetic:"v-alphabetic",vHanging:"v-hanging",vIdeographic:"v-ideographic",vMathematical:"v-mathematical",vectorEffect:"vector-effect",vertAdvY:"vert-adv-y",vertOriginX:"vert-origin-x",vertOriginY:"vert-origin-y",wordSpacing:"word-spacing",writingMode:"writing-mode",xHeight:"x-height",playbackOrder:"playbackorder",timelineBegin:"timelinebegin"},transform:I,properties:{about:w,accentHeight:x,accumulate:null,additive:null,alignmentBaseline:null,alphabetic:x,amplitude:x,arabicForm:null,ascent:x,attributeName:null,attributeType:null,azimuth:x,bandwidth:null,baselineShift:null,baseFrequency:null,baseProfile:null,bbox:null,begin:null,bias:x,by:null,calcMode:null,capHeight:x,className:k,clip:null,clipPath:null,clipPathUnits:null,clipRule:null,color:null,colorInterpolation:null,colorInterpolationFilters:null,colorProfile:null,colorRendering:null,content:null,contentScriptType:null,contentStyleType:null,crossOrigin:null,cursor:null,cx:null,cy:null,d:null,dataType:null,defaultAction:null,descent:x,diffuseConstant:x,direction:null,display:null,dur:null,divisor:x,dominantBaseline:null,download:g,dx:null,dy:null,edgeMode:null,editable:null,elevation:x,enableBackground:null,end:null,event:null,exponent:x,externalResourcesRequired:null,fill:null,fillOpacity:x,fillRule:null,filter:null,filterRes:null,filterUnits:null,floodColor:null,floodOpacity:null,focusable:null,focusHighlight:null,fontFamily:null,fontSize:null,fontSizeAdjust:null,fontStretch:null,fontStyle:null,fontVariant:null,fontWeight:null,format:null,fr:null,from:null,fx:null,fy:null,g1:b,g2:b,glyphName:b,glyphOrientationHorizontal:null,glyphOrientationVertical:null,glyphRef:null,gradientTransform:null,gradientUnits:null,handler:null,hanging:x,hatchContentUnits:null,hatchUnits:null,height:null,href:null,hrefLang:null,horizAdvX:x,horizOriginX:x,horizOriginY:x,id:null,ideographic:x,imageRendering:null,initialVisibility:null,in:null,in2:null,intercept:x,k:x,k1:x,k2:x,k3:x,k4:x,kernelMatrix:w,kernelUnitLength:null,keyPoints:null,keySplines:null,keyTimes:null,kerning:null,lang:null,lengthAdjust:null,letterSpacing:null,lightingColor:null,limitingConeAngle:x,local:null,markerEnd:null,markerMid:null,markerStart:null,markerHeight:null,markerUnits:null,markerWidth:null,mask:null,maskContentUnits:null,maskUnits:null,mathematical:null,max:null,media:null,mediaCharacterEncoding:null,mediaContentEncodings:null,mediaSize:x,mediaTime:null,method:null,min:null,mode:null,name:null,navDown:null,navDownLeft:null,navDownRight:null,navLeft:null,navNext:null,navPrev:null,navRight:null,navUp:null,navUpLeft:null,navUpRight:null,numOctaves:null,observer:null,offset:null,onAbort:null,onActivate:null,onAfterPrint:null,onBeforePrint:null,onBegin:null,onCancel:null,onCanPlay:null,onCanPlayThrough:null,onChange:null,onClick:null,onClose:null,onCopy:null,onCueChange:null,onCut:null,onDblClick:null,onDrag:null,onDragEnd:null,onDragEnter:null,onDragExit:null,onDragLeave:null,onDragOver:null,onDragStart:null,onDrop:null,onDurationChange:null,onEmptied:null,onEnd:null,onEnded:null,onError:null,onFocus:null,onFocusIn:null,onFocusOut:null,onHashChange:null,onInput:null,onInvalid:null,onKeyDown:null,onKeyPress:null,onKeyUp:null,onLoad:null,onLoadedData:null,onLoadedMetadata:null,onLoadStart:null,onMessage:null,onMouseDown:null,onMouseEnter:null,onMouseLeave:null,onMouseMove:null,onMouseOut:null,onMouseOver:null,onMouseUp:null,onMouseWheel:null,onOffline:null,onOnline:null,onPageHide:null,onPageShow:null,onPaste:null,onPause:null,onPlay:null,onPlaying:null,onPopState:null,onProgress:null,onRateChange:null,onRepeat:null,onReset:null,onResize:null,onScroll:null,onSeeked:null,onSeeking:null,onSelect:null,onShow:null,onStalled:null,onStorage:null,onSubmit:null,onSuspend:null,onTimeUpdate:null,onToggle:null,onUnload:null,onVolumeChange:null,onWaiting:null,onZoom:null,opacity:null,operator:null,order:null,orient:null,orientation:null,origin:null,overflow:null,overlay:null,overlinePosition:x,overlineThickness:x,paintOrder:null,panose1:null,path:null,pathLength:x,patternContentUnits:null,patternTransform:null,patternUnits:null,phase:null,ping:k,pitch:null,playbackOrder:null,pointerEvents:null,points:null,pointsAtX:x,pointsAtY:x,pointsAtZ:x,preserveAlpha:null,preserveAspectRatio:null,primitiveUnits:null,propagate:null,property:w,r:null,radius:null,referrerPolicy:null,refX:null,refY:null,rel:w,rev:w,renderingIntent:null,repeatCount:null,repeatDur:null,requiredExtensions:w,requiredFeatures:w,requiredFonts:w,requiredFormats:w,resource:null,restart:null,result:null,rotate:null,rx:null,ry:null,scale:null,seed:null,shapeRendering:null,side:null,slope:null,snapshotTime:null,specularConstant:x,specularExponent:x,spreadMethod:null,spacing:null,startOffset:null,stdDeviation:null,stemh:null,stemv:null,stitchTiles:null,stopColor:null,stopOpacity:null,strikethroughPosition:x,strikethroughThickness:x,string:null,stroke:null,strokeDashArray:w,strokeDashOffset:null,strokeLineCap:null,strokeLineJoin:null,strokeMiterLimit:x,strokeOpacity:x,strokeWidth:null,style:null,surfaceScale:x,syncBehavior:null,syncBehaviorDefault:null,syncMaster:null,syncTolerance:null,syncToleranceDefault:null,systemLanguage:w,tabIndex:x,tableValues:null,target:null,targetX:x,targetY:x,textAnchor:null,textDecoration:null,textRendering:null,textLength:null,timelineBegin:null,title:null,transformBehavior:null,type:null,typeOf:w,to:null,transform:null,transformOrigin:null,u1:null,u2:null,underlinePosition:x,underlineThickness:x,unicode:null,unicodeBidi:null,unicodeRange:null,unitsPerEm:x,values:null,vAlphabetic:x,vMathematical:x,vectorEffect:null,vHanging:x,vIdeographic:x,version:null,vertAdvY:x,vertOriginX:x,vertOriginY:x,viewBox:null,viewTarget:null,visibility:null,width:null,widths:null,wordSpacing:null,writingMode:null,x:null,x1:null,x2:null,xChannelSelector:null,xHeight:x,y:null,y1:null,y2:null,yChannelSelector:null,z:null,zoomAndPan:null}}),F=p([T,z,M,L,D],"html"),R=p([T,z,M,L,j],"svg"),N=/^data[-\w.:]+$/i,_=/-[a-z]/g,B=/[A-Z]/g;function H(e){return"-"+e.toLowerCase()}function V(e){return e.charAt(1).toUpperCase()}let U={classId:"classID",dataType:"datatype",itemId:"itemID",strokeDashArray:"strokeDasharray",strokeDashOffset:"strokeDashoffset",strokeLineCap:"strokeLinecap",strokeLineJoin:"strokeLinejoin",strokeMiterLimit:"strokeMiterlimit",typeOf:"typeof",xLinkActuate:"xlinkActuate",xLinkArcRole:"xlinkArcrole",xLinkHref:"xlinkHref",xLinkRole:"xlinkRole",xLinkShow:"xlinkShow",xLinkTitle:"xlinkTitle",xLinkType:"xlinkType",xmlnsXLink:"xmlnsXlink"};var Z=n(52744),q=Z.default||Z;let W=Q("end"),K=Q("start");function Q(e){return function(t){let n=t&&t.position&&t.position[e]||{};if("number"==typeof n.line&&n.line>0&&"number"==typeof n.column&&n.column>0)return{line:n.line,column:n.column,offset:"number"==typeof n.offset&&n.offset>-1?n.offset:void 0}}}function Y(e){return e&&"object"==typeof e?"position"in e||"type"in e?X(e.position):"start"in e||"end"in e?X(e):"line"in e||"column"in e?$(e):"":""}function $(e){return J(e&&e.line)+":"+J(e&&e.column)}function X(e){return $(e&&e.start)+"-"+$(e&&e.end)}function J(e){return e&&"number"==typeof e?e:1}class G extends Error{constructor(e,t,n){super(),"string"==typeof t&&(n=t,t=void 0);let r="",l={},i=!1;if(t&&(l="line"in t&&"column"in t?{place:t}:"start"in t&&"end"in t?{place:t}:"type"in t?{ancestors:[t],place:t.position}:{...t}),"string"==typeof e?r=e:!l.cause&&e&&(i=!0,r=e.message,l.cause=e),!l.ruleId&&!l.source&&"string"==typeof n){let e=n.indexOf(":");-1===e?l.ruleId=n:(l.source=n.slice(0,e),l.ruleId=n.slice(e+1))}if(!l.place&&l.ancestors&&l.ancestors){let e=l.ancestors[l.ancestors.length-1];e&&(l.place=e.position)}let o=l.place&&"start"in l.place?l.place.start:l.place;this.ancestors=l.ancestors||void 0,this.cause=l.cause||void 0,this.column=o?o.column:void 0,this.fatal=void 0,this.file,this.message=r,this.line=o?o.line:void 0,this.name=Y(l.place)||"1:1",this.place=l.place||void 0,this.reason=this.message,this.ruleId=l.ruleId||void 0,this.source=l.source||void 0,this.stack=i&&l.cause&&"string"==typeof l.cause.stack?l.cause.stack:"",this.actual,this.expected,this.note,this.url}}G.prototype.file="",G.prototype.name="",G.prototype.reason="",G.prototype.message="",G.prototype.stack="",G.prototype.column=void 0,G.prototype.line=void 0,G.prototype.ancestors=void 0,G.prototype.cause=void 0,G.prototype.fatal=void 0,G.prototype.place=void 0,G.prototype.ruleId=void 0,G.prototype.source=void 0;let ee={}.hasOwnProperty,et=new Map,en=/[A-Z]/g,er=/-([a-z])/g,el=new Set(["table","tbody","thead","tfoot","tr"]),ei=new Set(["td","th"]),eo="https://github.com/syntax-tree/hast-util-to-jsx-runtime";function ea(e,t,n){return"element"===t.type?function(e,t,n){let r=e.schema,l=r;"svg"===t.tagName.toLowerCase()&&"html"===r.space&&(l=R,e.schema=l),e.ancestors.push(t);let i=ef(e,t.tagName,!1),o=function(e,t){let n,r;let l={};for(r in t.properties)if("children"!==r&&ee.call(t.properties,r)){let i=function(e,t,n){let r=function(e,t){let n=d(t),r=t,l=h;if(n in e.normal)return e.property[e.normal[n]];if(n.length>4&&"data"===n.slice(0,4)&&N.test(t)){if("-"===t.charAt(4)){let e=t.slice(5).replace(_,V);r="data"+e.charAt(0).toUpperCase()+e.slice(1)}else{let e=t.slice(4);if(!_.test(e)){let n=e.replace(B,H);"-"!==n.charAt(0)&&(n="-"+n),t="data"+n}}l=C}return new l(r,t)}(e.schema,t);if(!(null==n||"number"==typeof n&&Number.isNaN(n))){if(Array.isArray(n)&&(n=r.commaSeparated?function(e,t){let n={};return(""===e[e.length-1]?[...e,""]:e).join((n.padRight?" ":"")+","+(!1===n.padLeft?"":" ")).trim()}(n):n.join(" ").trim()),"style"===r.property){let t="object"==typeof n?n:function(e,t){let n={};try{q(t,function(e,t){let r=e;"--"!==r.slice(0,2)&&("-ms-"===r.slice(0,4)&&(r="ms-"+r.slice(4)),r=r.replace(er,ed)),n[r]=t})}catch(t){if(!e.ignoreInvalidStyle){let n=new G("Cannot parse `style` attribute",{ancestors:e.ancestors,cause:t,ruleId:"style",source:"hast-util-to-jsx-runtime"});throw n.file=e.filePath||void 0,n.url=eo+"#cannot-parse-style-attribute",n}}return n}(e,String(n));return"css"===e.stylePropertyNameCase&&(t=function(e){let t;let n={};for(t in e)ee.call(e,t)&&(n[function(e){let t=e.replace(en,eh);return"ms-"===t.slice(0,3)&&(t="-"+t),t}(t)]=e[t]);return n}(t)),["style",t]}return["react"===e.elementAttributeNameCase&&r.space?U[r.property]||r.property:r.attribute,n]}}(e,r,t.properties[r]);if(i){let[r,o]=i;e.tableCellAlignToStyle&&"align"===r&&"string"==typeof o&&ei.has(t.tagName)?n=o:l[r]=o}}return n&&((l.style||(l.style={}))["css"===e.stylePropertyNameCase?"text-align":"textAlign"]=n),l}(e,t),a=es(e,t);return el.has(t.tagName)&&(a=a.filter(function(e){return"string"!=typeof e||!("object"==typeof e?"text"===e.type&&s(e.value):s(e))})),eu(e,o,i,t),ec(o,a),e.ancestors.pop(),e.schema=r,e.create(t,i,o,n)}(e,t,n):"mdxFlowExpression"===t.type||"mdxTextExpression"===t.type?function(e,t){if(t.data&&t.data.estree&&e.evaluater){let n=t.data.estree.body[0];return n.type,e.evaluater.evaluateExpression(n.expression)}ep(e,t.position)}(e,t):"mdxJsxFlowElement"===t.type||"mdxJsxTextElement"===t.type?function(e,t,n){let r=e.schema,l=r;"svg"===t.name&&"html"===r.space&&(l=R,e.schema=l),e.ancestors.push(t);let i=null===t.name?e.Fragment:ef(e,t.name,!0),o=function(e,t){let n={};for(let r of t.attributes)if("mdxJsxExpressionAttribute"===r.type){if(r.data&&r.data.estree&&e.evaluater){let t=r.data.estree.body[0];t.type;let l=t.expression;l.type;let i=l.properties[0];i.type,Object.assign(n,e.evaluater.evaluateExpression(i.argument))}else ep(e,t.position)}else{let l;let i=r.name;if(r.value&&"object"==typeof r.value){if(r.value.data&&r.value.data.estree&&e.evaluater){let t=r.value.data.estree.body[0];t.type,l=e.evaluater.evaluateExpression(t.expression)}else ep(e,t.position)}else l=null===r.value||r.value;n[i]=l}return n}(e,t),a=es(e,t);return eu(e,o,i,t),ec(o,a),e.ancestors.pop(),e.schema=r,e.create(t,i,o,n)}(e,t,n):"mdxjsEsm"===t.type?function(e,t){if(t.data&&t.data.estree&&e.evaluater)return e.evaluater.evaluateProgram(t.data.estree);ep(e,t.position)}(e,t):"root"===t.type?function(e,t,n){let r={};return ec(r,es(e,t)),e.create(t,e.Fragment,r,n)}(e,t,n):"text"===t.type?t.value:void 0}function eu(e,t,n,r){"string"!=typeof n&&n!==e.Fragment&&e.passNode&&(t.node=r)}function ec(e,t){if(t.length>0){let n=t.length>1?t:t[0];n&&(e.children=n)}}function es(e,t){let n=[],r=-1,l=e.passKeys?new Map:et;for(;++ri?0:i+t:t>i?i:t,n=n>0?n:0,r.length<1e4)(l=Array.from(r)).unshift(t,n),e.splice(...l);else for(n&&e.splice(t,n);o0?(ek(e,e.length,0,t),e):t}function ew(e){let t,n,r,l,i,o,a;let u={},c=-1;for(;++c-1&&e.test(String.fromCharCode(t))}}function eR(e,t,n,r){let l=r?r-1:Number.POSITIVE_INFINITY,i=0;return function(r){return eL(r)?(e.enter(n),function r(o){return eL(o)&&i++r))return;let a=l.events.length,u=a;for(;u--;)if("exit"===l.events[u][0]&&"chunkFlow"===l.events[u][1].type){if(e){n=l.events[u][1].end;break}e=!0}for(g(o),i=a;it;){let t=i[n];l.containerState=t[1],t[0].exit.call(l,e)}i.length=t}function y(){t.write([null]),n=void 0,t=void 0,l.containerState._closeFlow=void 0}}},eB={tokenize:function(e,t,n){return eR(e,e.attempt(this.parser.constructs.document,t,n),"linePrefix",this.parser.constructs.disable.null.includes("codeIndented")?void 0:4)}},eH={tokenize:function(e,t,n){return function(t){return eL(t)?eR(e,r,"linePrefix")(t):r(t)};function r(e){return null===e||eA(e)?t(e):n(e)}},partial:!0},eV={tokenize:function(e,t){let n;return function(t){return e.enter("content"),n=e.enter("chunkContent",{contentType:"content"}),r(t)};function r(t){return null===t?l(t):eA(t)?e.check(eU,i,l)(t):(e.consume(t),r)}function l(n){return e.exit("chunkContent"),e.exit("content"),t(n)}function i(t){return e.consume(t),e.exit("chunkContent"),n.next=e.enter("chunkContent",{contentType:"content",previous:n}),n=n.next,r}},resolve:function(e){return ew(e),e}},eU={tokenize:function(e,t,n){let r=this;return function(t){return e.exit("chunkContent"),e.enter("lineEnding"),e.consume(t),e.exit("lineEnding"),eR(e,l,"linePrefix")};function l(l){if(null===l||eA(l))return n(l);let i=r.events[r.events.length-1];return!r.parser.constructs.disable.null.includes("codeIndented")&&i&&"linePrefix"===i[1].type&&i[2].sliceSerialize(i[1],!0).length>=4?t(l):e.interrupt(r.parser.constructs.flow,n,t)(l)}},partial:!0},eZ={tokenize:function(e){let t=this,n=e.attempt(eH,function(r){if(null===r){e.consume(r);return}return e.enter("lineEndingBlank"),e.consume(r),e.exit("lineEndingBlank"),t.currentConstruct=void 0,n},e.attempt(this.parser.constructs.flowInitial,r,eR(e,e.attempt(this.parser.constructs.flow,r,e.attempt(eV,r)),"linePrefix")));return n;function r(r){if(null===r){e.consume(r);return}return e.enter("lineEnding"),e.consume(r),e.exit("lineEnding"),t.currentConstruct=void 0,n}}},eq={resolveAll:eY()},eW=eQ("string"),eK=eQ("text");function eQ(e){return{tokenize:function(t){let n=this,r=this.parser.constructs[e],l=t.attempt(r,i,o);return i;function i(e){return u(e)?l(e):o(e)}function o(e){if(null===e){t.consume(e);return}return t.enter("data"),t.consume(e),a}function a(e){return u(e)?(t.exit("data"),l(e)):(t.consume(e),a)}function u(e){if(null===e)return!0;let t=r[e],l=-1;if(t)for(;++l=3&&(null===o||eA(o))?(e.exit("thematicBreak"),t(o)):n(o)}(i)}}},eG={name:"list",tokenize:function(e,t,n){let r=this,l=r.events[r.events.length-1],i=l&&"linePrefix"===l[1].type?l[2].sliceSerialize(l[1],!0).length:0,o=0;return function(t){let l=r.containerState.type||(42===t||43===t||45===t?"listUnordered":"listOrdered");if("listUnordered"===l?!r.containerState.marker||t===r.containerState.marker:ez(t)){if(r.containerState.type||(r.containerState.type=l,e.enter(l,{_container:!0})),"listUnordered"===l)return e.enter("listItemPrefix"),42===t||45===t?e.check(eJ,n,a)(t):a(t);if(!r.interrupt||49===t)return e.enter("listItemPrefix"),e.enter("listItemValue"),function t(l){return ez(l)&&++o<10?(e.consume(l),t):(!r.interrupt||o<2)&&(r.containerState.marker?l===r.containerState.marker:41===l||46===l)?(e.exit("listItemValue"),a(l)):n(l)}(t)}return n(t)};function a(t){return e.enter("listItemMarker"),e.consume(t),e.exit("listItemMarker"),r.containerState.marker=r.containerState.marker||t,e.check(eH,r.interrupt?n:u,e.attempt(e1,s,c))}function u(e){return r.containerState.initialBlankLine=!0,i++,s(e)}function c(t){return eL(t)?(e.enter("listItemPrefixWhitespace"),e.consume(t),e.exit("listItemPrefixWhitespace"),s):n(t)}function s(n){return r.containerState.size=i+r.sliceSerialize(e.exit("listItemPrefix"),!0).length,t(n)}},continuation:{tokenize:function(e,t,n){let r=this;return r.containerState._closeFlow=void 0,e.check(eH,function(n){return r.containerState.furtherBlankLines=r.containerState.furtherBlankLines||r.containerState.initialBlankLine,eR(e,t,"listItemIndent",r.containerState.size+1)(n)},function(n){return r.containerState.furtherBlankLines||!eL(n)?(r.containerState.furtherBlankLines=void 0,r.containerState.initialBlankLine=void 0,l(n)):(r.containerState.furtherBlankLines=void 0,r.containerState.initialBlankLine=void 0,e.attempt(e0,t,l)(n))});function l(l){return r.containerState._closeFlow=!0,r.interrupt=void 0,eR(e,e.attempt(eG,t,n),"linePrefix",r.parser.constructs.disable.null.includes("codeIndented")?void 0:4)(l)}}},exit:function(e){e.exit(this.containerState.type)}},e1={tokenize:function(e,t,n){let r=this;return eR(e,function(e){let l=r.events[r.events.length-1];return!eL(e)&&l&&"listItemPrefixWhitespace"===l[1].type?t(e):n(e)},"listItemPrefixWhitespace",r.parser.constructs.disable.null.includes("codeIndented")?void 0:5)},partial:!0},e0={tokenize:function(e,t,n){let r=this;return eR(e,function(e){let l=r.events[r.events.length-1];return l&&"listItemIndent"===l[1].type&&l[2].sliceSerialize(l[1],!0).length===r.containerState.size?t(e):n(e)},"listItemIndent",r.containerState.size+1)},partial:!0},e2={name:"blockQuote",tokenize:function(e,t,n){let r=this;return function(t){if(62===t){let n=r.containerState;return n.open||(e.enter("blockQuote",{_container:!0}),n.open=!0),e.enter("blockQuotePrefix"),e.enter("blockQuoteMarker"),e.consume(t),e.exit("blockQuoteMarker"),l}return n(t)};function l(n){return eL(n)?(e.enter("blockQuotePrefixWhitespace"),e.consume(n),e.exit("blockQuotePrefixWhitespace"),e.exit("blockQuotePrefix"),t):(e.exit("blockQuotePrefix"),t(n))}},continuation:{tokenize:function(e,t,n){let r=this;return function(t){return eL(t)?eR(e,l,"linePrefix",r.parser.constructs.disable.null.includes("codeIndented")?void 0:4)(t):l(t)};function l(r){return e.attempt(e2,t,n)(r)}}},exit:function(e){e.exit("blockQuote")}};function e4(e,t,n,r,l,i,o,a,u){let c=u||Number.POSITIVE_INFINITY,s=0;return function(t){return 60===t?(e.enter(r),e.enter(l),e.enter(i),e.consume(t),e.exit(i),f):null===t||32===t||41===t||eO(t)?n(t):(e.enter(r),e.enter(o),e.enter(a),e.enter("chunkString",{contentType:"string"}),h(t))};function f(n){return 62===n?(e.enter(i),e.consume(n),e.exit(i),e.exit(l),e.exit(r),t):(e.enter(a),e.enter("chunkString",{contentType:"string"}),p(n))}function p(t){return 62===t?(e.exit("chunkString"),e.exit(a),f(t)):null===t||60===t||eA(t)?n(t):(e.consume(t),92===t?d:p)}function d(t){return 60===t||62===t||92===t?(e.consume(t),p):p(t)}function h(l){return!s&&(null===l||41===l||eM(l))?(e.exit("chunkString"),e.exit(a),e.exit(o),e.exit(r),t(l)):s999||null===f||91===f||93===f&&!o||94===f&&!u&&"_hiddenFootnoteSupport"in a.parser.constructs?n(f):93===f?(e.exit(i),e.enter(l),e.consume(f),e.exit(l),e.exit(r),t):eA(f)?(e.enter("lineEnding"),e.consume(f),e.exit("lineEnding"),c):(e.enter("chunkString",{contentType:"string"}),s(f))}function s(t){return null===t||91===t||93===t||eA(t)||u++>999?(e.exit("chunkString"),c(t)):(e.consume(t),o||(o=!eL(t)),92===t?f:s)}function f(t){return 91===t||92===t||93===t?(e.consume(t),u++,s):s(t)}}function e3(e,t,n,r,l,i){let o;return function(t){return 34===t||39===t||40===t?(e.enter(r),e.enter(l),e.consume(t),e.exit(l),o=40===t?41:t,a):n(t)};function a(n){return n===o?(e.enter(l),e.consume(n),e.exit(l),e.exit(r),t):(e.enter(i),u(n))}function u(t){return t===o?(e.exit(i),a(o)):null===t?n(t):eA(t)?(e.enter("lineEnding"),e.consume(t),e.exit("lineEnding"),eR(e,u,"linePrefix")):(e.enter("chunkString",{contentType:"string"}),c(t))}function c(t){return t===o||null===t||eA(t)?(e.exit("chunkString"),u(t)):(e.consume(t),92===t?s:c)}function s(t){return t===o||92===t?(e.consume(t),c):c(t)}}function e5(e,t){let n;return function r(l){return eA(l)?(e.enter("lineEnding"),e.consume(l),e.exit("lineEnding"),n=!0,r):eL(l)?eR(e,r,n?"linePrefix":"lineSuffix")(l):t(l)}}function e8(e){return e.replace(/[\t\n\r ]+/g," ").replace(/^ | $/g,"").toLowerCase().toUpperCase()}let e9={tokenize:function(e,t,n){return function(t){return eM(t)?e5(e,r)(t):n(t)};function r(t){return e3(e,l,n,"definitionTitle","definitionTitleMarker","definitionTitleString")(t)}function l(t){return eL(t)?eR(e,i,"whitespace")(t):i(t)}function i(e){return null===e||eA(e)?t(e):n(e)}},partial:!0},e7={name:"codeIndented",tokenize:function(e,t,n){let r=this;return function(t){return e.enter("codeIndented"),eR(e,l,"linePrefix",5)(t)};function l(t){let l=r.events[r.events.length-1];return l&&"linePrefix"===l[1].type&&l[2].sliceSerialize(l[1],!0).length>=4?function t(n){return null===n?i(n):eA(n)?e.attempt(te,t,i)(n):(e.enter("codeFlowValue"),function n(r){return null===r||eA(r)?(e.exit("codeFlowValue"),t(r)):(e.consume(r),n)}(n))}(t):n(t)}function i(n){return e.exit("codeIndented"),t(n)}}},te={tokenize:function(e,t,n){let r=this;return l;function l(t){return r.parser.lazy[r.now().line]?n(t):eA(t)?(e.enter("lineEnding"),e.consume(t),e.exit("lineEnding"),l):eR(e,i,"linePrefix",5)(t)}function i(e){let i=r.events[r.events.length-1];return i&&"linePrefix"===i[1].type&&i[2].sliceSerialize(i[1],!0).length>=4?t(e):eA(e)?l(e):n(e)}},partial:!0},tt={name:"setextUnderline",tokenize:function(e,t,n){let r;let l=this;return function(t){let o,a=l.events.length;for(;a--;)if("lineEnding"!==l.events[a][1].type&&"linePrefix"!==l.events[a][1].type&&"content"!==l.events[a][1].type){o="paragraph"===l.events[a][1].type;break}return!l.parser.lazy[l.now().line]&&(l.interrupt||o)?(e.enter("setextHeadingLine"),r=t,e.enter("setextHeadingLineSequence"),function t(n){return n===r?(e.consume(n),t):(e.exit("setextHeadingLineSequence"),eL(n)?eR(e,i,"lineSuffix")(n):i(n))}(t)):n(t)};function i(r){return null===r||eA(r)?(e.exit("setextHeadingLine"),t(r)):n(r)}},resolveTo:function(e,t){let n,r,l,i=e.length;for(;i--;)if("enter"===e[i][0]){if("content"===e[i][1].type){n=i;break}"paragraph"===e[i][1].type&&(r=i)}else"content"===e[i][1].type&&e.splice(i,1),l||"definition"!==e[i][1].type||(l=i);let o={type:"setextHeading",start:Object.assign({},e[r][1].start),end:Object.assign({},e[e.length-1][1].end)};return e[r][1].type="setextHeadingText",l?(e.splice(r,0,["enter",o,t]),e.splice(l+1,0,["exit",e[n][1],t]),e[n][1].end=Object.assign({},e[l][1].end)):e[n][1]=o,e.push(["exit",o,t]),e}},tn=["address","article","aside","base","basefont","blockquote","body","caption","center","col","colgroup","dd","details","dialog","dir","div","dl","dt","fieldset","figcaption","figure","footer","form","frame","frameset","h1","h2","h3","h4","h5","h6","head","header","hr","html","iframe","legend","li","link","main","menu","menuitem","nav","noframes","ol","optgroup","option","p","param","search","section","summary","table","tbody","td","tfoot","th","thead","title","tr","track","ul"],tr=["pre","script","style","textarea"],tl={tokenize:function(e,t,n){return function(r){return e.enter("lineEnding"),e.consume(r),e.exit("lineEnding"),e.attempt(eH,t,n)}},partial:!0},ti={tokenize:function(e,t,n){let r=this;return function(t){return eA(t)?(e.enter("lineEnding"),e.consume(t),e.exit("lineEnding"),l):n(t)};function l(e){return r.parser.lazy[r.now().line]?n(e):t(e)}},partial:!0},to={tokenize:function(e,t,n){let r=this;return function(t){return null===t?n(t):(e.enter("lineEnding"),e.consume(t),e.exit("lineEnding"),l)};function l(e){return r.parser.lazy[r.now().line]?n(e):t(e)}},partial:!0},ta={name:"codeFenced",tokenize:function(e,t,n){let r;let l=this,i={tokenize:function(e,t,n){let i=0;return function(t){return e.enter("lineEnding"),e.consume(t),e.exit("lineEnding"),o};function o(t){return e.enter("codeFencedFence"),eL(t)?eR(e,u,"linePrefix",l.parser.constructs.disable.null.includes("codeIndented")?void 0:4)(t):u(t)}function u(t){return t===r?(e.enter("codeFencedFenceSequence"),function t(l){return l===r?(i++,e.consume(l),t):i>=a?(e.exit("codeFencedFenceSequence"),eL(l)?eR(e,c,"whitespace")(l):c(l)):n(l)}(t)):n(t)}function c(r){return null===r||eA(r)?(e.exit("codeFencedFence"),t(r)):n(r)}},partial:!0},o=0,a=0;return function(t){return function(t){let i=l.events[l.events.length-1];return o=i&&"linePrefix"===i[1].type?i[2].sliceSerialize(i[1],!0).length:0,r=t,e.enter("codeFenced"),e.enter("codeFencedFence"),e.enter("codeFencedFenceSequence"),function t(l){return l===r?(a++,e.consume(l),t):a<3?n(l):(e.exit("codeFencedFenceSequence"),eL(l)?eR(e,u,"whitespace")(l):u(l))}(t)}(t)};function u(i){return null===i||eA(i)?(e.exit("codeFencedFence"),l.interrupt?t(i):e.check(to,s,h)(i)):(e.enter("codeFencedFenceInfo"),e.enter("chunkString",{contentType:"string"}),function t(l){return null===l||eA(l)?(e.exit("chunkString"),e.exit("codeFencedFenceInfo"),u(l)):eL(l)?(e.exit("chunkString"),e.exit("codeFencedFenceInfo"),eR(e,c,"whitespace")(l)):96===l&&l===r?n(l):(e.consume(l),t)}(i))}function c(t){return null===t||eA(t)?u(t):(e.enter("codeFencedFenceMeta"),e.enter("chunkString",{contentType:"string"}),function t(l){return null===l||eA(l)?(e.exit("chunkString"),e.exit("codeFencedFenceMeta"),u(l)):96===l&&l===r?n(l):(e.consume(l),t)}(t))}function s(t){return e.attempt(i,h,f)(t)}function f(t){return e.enter("lineEnding"),e.consume(t),e.exit("lineEnding"),p}function p(t){return o>0&&eL(t)?eR(e,d,"linePrefix",o+1)(t):d(t)}function d(t){return null===t||eA(t)?e.check(to,s,h)(t):(e.enter("codeFlowValue"),function t(n){return null===n||eA(n)?(e.exit("codeFlowValue"),d(n)):(e.consume(n),t)}(t))}function h(n){return e.exit("codeFenced"),t(n)}},concrete:!0},tu=document.createElement("i");function tc(e){let t="&"+e+";";tu.innerHTML=t;let n=tu.textContent;return(59!==n.charCodeAt(n.length-1)||"semi"===e)&&n!==t&&n}let ts={name:"characterReference",tokenize:function(e,t,n){let r,l;let i=this,o=0;return function(t){return e.enter("characterReference"),e.enter("characterReferenceMarker"),e.consume(t),e.exit("characterReferenceMarker"),a};function a(t){return 35===t?(e.enter("characterReferenceMarkerNumeric"),e.consume(t),e.exit("characterReferenceMarkerNumeric"),u):(e.enter("characterReferenceValue"),r=31,l=eC,c(t))}function u(t){return 88===t||120===t?(e.enter("characterReferenceMarkerHexadecimal"),e.consume(t),e.exit("characterReferenceMarkerHexadecimal"),e.enter("characterReferenceValue"),r=6,l=eT,c):(e.enter("characterReferenceValue"),r=7,l=ez,c(t))}function c(a){if(59===a&&o){let r=e.exit("characterReferenceValue");return l!==eC||tc(i.sliceSerialize(r))?(e.enter("characterReferenceMarker"),e.consume(a),e.exit("characterReferenceMarker"),e.exit("characterReference"),t):n(a)}return l(a)&&o++1&&e[s][1].end.offset-e[s][1].start.offset>1?2:1;let f=Object.assign({},e[n][1].end),p=Object.assign({},e[s][1].start);tk(f,-a),tk(p,a),i={type:a>1?"strongSequence":"emphasisSequence",start:f,end:Object.assign({},e[n][1].end)},o={type:a>1?"strongSequence":"emphasisSequence",start:Object.assign({},e[s][1].start),end:p},l={type:a>1?"strongText":"emphasisText",start:Object.assign({},e[n][1].end),end:Object.assign({},e[s][1].start)},r={type:a>1?"strong":"emphasis",start:Object.assign({},i.start),end:Object.assign({},o.end)},e[n][1].end=Object.assign({},i.start),e[s][1].start=Object.assign({},o.end),u=[],e[n][1].end.offset-e[n][1].start.offset&&(u=eb(u,[["enter",e[n][1],t],["exit",e[n][1],t]])),u=eb(u,[["enter",r,t],["enter",i,t],["exit",i,t],["enter",l,t]]),u=eb(u,eX(t.parser.constructs.insideSpan.null,e.slice(n+1,s),t)),u=eb(u,[["exit",l,t],["enter",o,t],["exit",o,t],["exit",r,t]]),e[s][1].end.offset-e[s][1].start.offset?(c=2,u=eb(u,[["enter",e[s][1],t],["exit",e[s][1],t]])):c=0,ek(e,n-1,s-n+3,u),s=n+u.length-c-2;break}}for(s=-1;++si&&"whitespace"===e[l][1].type&&(l-=2),"atxHeadingSequence"===e[l][1].type&&(i===l-1||l-4>i&&"whitespace"===e[l-2][1].type)&&(l-=i+1===l?2:4),l>i&&(n={type:"atxHeadingText",start:e[i][1].start,end:e[l][1].end},r={type:"chunkText",start:e[i][1].start,end:e[l][1].end,contentType:"text"},ek(e,i,l-i+1,[["enter",n,t],["enter",r,t],["exit",r,t],["exit",n,t]])),e}},42:eJ,45:[tt,eJ],60:{name:"htmlFlow",tokenize:function(e,t,n){let r,l,i,o,a;let u=this;return function(t){return e.enter("htmlFlow"),e.enter("htmlFlowData"),e.consume(t),c};function c(o){return 33===o?(e.consume(o),s):47===o?(e.consume(o),l=!0,d):63===o?(e.consume(o),r=3,u.interrupt?t:M):eE(o)?(e.consume(o),i=String.fromCharCode(o),h):n(o)}function s(l){return 45===l?(e.consume(l),r=2,f):91===l?(e.consume(l),r=5,o=0,p):eE(l)?(e.consume(l),r=4,u.interrupt?t:M):n(l)}function f(r){return 45===r?(e.consume(r),u.interrupt?t:M):n(r)}function p(r){let l="CDATA[";return r===l.charCodeAt(o++)?(e.consume(r),o===l.length)?u.interrupt?t:E:p:n(r)}function d(t){return eE(t)?(e.consume(t),i=String.fromCharCode(t),h):n(t)}function h(o){if(null===o||47===o||62===o||eM(o)){let a=47===o,c=i.toLowerCase();return!a&&!l&&tr.includes(c)?(r=1,u.interrupt?t(o):E(o)):tn.includes(i.toLowerCase())?(r=6,a)?(e.consume(o),m):u.interrupt?t(o):E(o):(r=7,u.interrupt&&!u.parser.lazy[u.now().line]?n(o):l?function t(n){return eL(n)?(e.consume(n),t):w(n)}(o):g(o))}return 45===o||eC(o)?(e.consume(o),i+=String.fromCharCode(o),h):n(o)}function m(r){return 62===r?(e.consume(r),u.interrupt?t:E):n(r)}function g(t){return 47===t?(e.consume(t),w):58===t||95===t||eE(t)?(e.consume(t),y):eL(t)?(e.consume(t),g):w(t)}function y(t){return 45===t||46===t||58===t||95===t||eC(t)?(e.consume(t),y):v(t)}function v(t){return 61===t?(e.consume(t),x):eL(t)?(e.consume(t),v):g(t)}function x(t){return null===t||60===t||61===t||62===t||96===t?n(t):34===t||39===t?(e.consume(t),a=t,k):eL(t)?(e.consume(t),x):function t(n){return null===n||34===n||39===n||47===n||60===n||61===n||62===n||96===n||eM(n)?v(n):(e.consume(n),t)}(t)}function k(t){return t===a?(e.consume(t),a=null,b):null===t||eA(t)?n(t):(e.consume(t),k)}function b(e){return 47===e||62===e||eL(e)?g(e):n(e)}function w(t){return 62===t?(e.consume(t),S):n(t)}function S(t){return null===t||eA(t)?E(t):eL(t)?(e.consume(t),S):n(t)}function E(t){return 45===t&&2===r?(e.consume(t),z):60===t&&1===r?(e.consume(t),T):62===t&&4===r?(e.consume(t),L):63===t&&3===r?(e.consume(t),M):93===t&&5===r?(e.consume(t),A):eA(t)&&(6===r||7===r)?(e.exit("htmlFlowData"),e.check(tl,D,C)(t)):null===t||eA(t)?(e.exit("htmlFlowData"),C(t)):(e.consume(t),E)}function C(t){return e.check(ti,P,D)(t)}function P(t){return e.enter("lineEnding"),e.consume(t),e.exit("lineEnding"),O}function O(t){return null===t||eA(t)?C(t):(e.enter("htmlFlowData"),E(t))}function z(t){return 45===t?(e.consume(t),M):E(t)}function T(t){return 47===t?(e.consume(t),i="",I):E(t)}function I(t){if(62===t){let n=i.toLowerCase();return tr.includes(n)?(e.consume(t),L):E(t)}return eE(t)&&i.length<8?(e.consume(t),i+=String.fromCharCode(t),I):E(t)}function A(t){return 93===t?(e.consume(t),M):E(t)}function M(t){return 62===t?(e.consume(t),L):45===t&&2===r?(e.consume(t),M):E(t)}function L(t){return null===t||eA(t)?(e.exit("htmlFlowData"),D(t)):(e.consume(t),L)}function D(n){return e.exit("htmlFlow"),t(n)}},resolveTo:function(e){let t=e.length;for(;t--&&("enter"!==e[t][0]||"htmlFlow"!==e[t][1].type););return t>1&&"linePrefix"===e[t-2][1].type&&(e[t][1].start=e[t-2][1].start,e[t+1][1].start=e[t-2][1].start,e.splice(t-2,2)),e},concrete:!0},61:tt,95:eJ,96:ta,126:ta},tP={38:ts,92:tf},tO={[-5]:tp,[-4]:tp,[-3]:tp,33:ty,38:ts,42:tx,60:[{name:"autolink",tokenize:function(e,t,n){let r=0;return function(t){return e.enter("autolink"),e.enter("autolinkMarker"),e.consume(t),e.exit("autolinkMarker"),e.enter("autolinkProtocol"),l};function l(t){return eE(t)?(e.consume(t),i):a(t)}function i(t){return 43===t||45===t||46===t||eC(t)?(r=1,function t(n){return 58===n?(e.consume(n),r=0,o):(43===n||45===n||46===n||eC(n))&&r++<32?(e.consume(n),t):(r=0,a(n))}(t)):a(t)}function o(r){return 62===r?(e.exit("autolinkProtocol"),e.enter("autolinkMarker"),e.consume(r),e.exit("autolinkMarker"),e.exit("autolink"),t):null===r||32===r||60===r||eO(r)?n(r):(e.consume(r),o)}function a(t){return 64===t?(e.consume(t),u):eP(t)?(e.consume(t),a):n(t)}function u(l){return eC(l)?function l(i){return 46===i?(e.consume(i),r=0,u):62===i?(e.exit("autolinkProtocol").type="autolinkEmail",e.enter("autolinkMarker"),e.consume(i),e.exit("autolinkMarker"),e.exit("autolink"),t):function t(i){if((45===i||eC(i))&&r++<63){let n=45===i?t:l;return e.consume(i),n}return n(i)}(i)}(l):n(l)}}},{name:"htmlText",tokenize:function(e,t,n){let r,l,i;let o=this;return function(t){return e.enter("htmlText"),e.enter("htmlTextData"),e.consume(t),a};function a(t){return 33===t?(e.consume(t),u):47===t?(e.consume(t),k):63===t?(e.consume(t),v):eE(t)?(e.consume(t),w):n(t)}function u(t){return 45===t?(e.consume(t),c):91===t?(e.consume(t),l=0,d):eE(t)?(e.consume(t),y):n(t)}function c(t){return 45===t?(e.consume(t),p):n(t)}function s(t){return null===t?n(t):45===t?(e.consume(t),f):eA(t)?(i=s,I(t)):(e.consume(t),s)}function f(t){return 45===t?(e.consume(t),p):s(t)}function p(e){return 62===e?T(e):45===e?f(e):s(e)}function d(t){let r="CDATA[";return t===r.charCodeAt(l++)?(e.consume(t),l===r.length?h:d):n(t)}function h(t){return null===t?n(t):93===t?(e.consume(t),m):eA(t)?(i=h,I(t)):(e.consume(t),h)}function m(t){return 93===t?(e.consume(t),g):h(t)}function g(t){return 62===t?T(t):93===t?(e.consume(t),g):h(t)}function y(t){return null===t||62===t?T(t):eA(t)?(i=y,I(t)):(e.consume(t),y)}function v(t){return null===t?n(t):63===t?(e.consume(t),x):eA(t)?(i=v,I(t)):(e.consume(t),v)}function x(e){return 62===e?T(e):v(e)}function k(t){return eE(t)?(e.consume(t),b):n(t)}function b(t){return 45===t||eC(t)?(e.consume(t),b):function t(n){return eA(n)?(i=t,I(n)):eL(n)?(e.consume(n),t):T(n)}(t)}function w(t){return 45===t||eC(t)?(e.consume(t),w):47===t||62===t||eM(t)?S(t):n(t)}function S(t){return 47===t?(e.consume(t),T):58===t||95===t||eE(t)?(e.consume(t),E):eA(t)?(i=S,I(t)):eL(t)?(e.consume(t),S):T(t)}function E(t){return 45===t||46===t||58===t||95===t||eC(t)?(e.consume(t),E):function t(n){return 61===n?(e.consume(n),C):eA(n)?(i=t,I(n)):eL(n)?(e.consume(n),t):S(n)}(t)}function C(t){return null===t||60===t||61===t||62===t||96===t?n(t):34===t||39===t?(e.consume(t),r=t,P):eA(t)?(i=C,I(t)):eL(t)?(e.consume(t),C):(e.consume(t),O)}function P(t){return t===r?(e.consume(t),r=void 0,z):null===t?n(t):eA(t)?(i=P,I(t)):(e.consume(t),P)}function O(t){return null===t||34===t||39===t||60===t||61===t||96===t?n(t):47===t||62===t||eM(t)?S(t):(e.consume(t),O)}function z(e){return 47===e||62===e||eM(e)?S(e):n(e)}function T(r){return 62===r?(e.consume(r),e.exit("htmlTextData"),e.exit("htmlText"),t):n(r)}function I(t){return e.exit("htmlTextData"),e.enter("lineEnding"),e.consume(t),e.exit("lineEnding"),A}function A(t){return eL(t)?eR(e,M,"linePrefix",o.parser.constructs.disable.null.includes("codeIndented")?void 0:4)(t):M(t)}function M(t){return e.enter("htmlTextData"),i(t)}}}],91:tb,92:[{name:"hardBreakEscape",tokenize:function(e,t,n){return function(t){return e.enter("hardBreakEscape"),e.consume(t),r};function r(r){return eA(r)?(e.exit("hardBreakEscape"),t(r)):n(r)}}},tf],93:td,95:tx,96:{name:"codeText",tokenize:function(e,t,n){let r,l,i=0;return function(t){return e.enter("codeText"),e.enter("codeTextSequence"),function t(n){return 96===n?(e.consume(n),i++,t):(e.exit("codeTextSequence"),o(n))}(t)};function o(u){return null===u?n(u):32===u?(e.enter("space"),e.consume(u),e.exit("space"),o):96===u?(l=e.enter("codeTextSequence"),r=0,function n(o){return 96===o?(e.consume(o),r++,n):r===i?(e.exit("codeTextSequence"),e.exit("codeText"),t(o)):(l.type="codeTextData",a(o))}(u)):eA(u)?(e.enter("lineEnding"),e.consume(u),e.exit("lineEnding"),o):(e.enter("codeTextData"),a(u))}function a(t){return null===t||32===t||96===t||eA(t)?(e.exit("codeTextData"),o(t)):(e.consume(t),a)}},resolve:function(e){let t,n,r=e.length-4,l=3;if(("lineEnding"===e[3][1].type||"space"===e[l][1].type)&&("lineEnding"===e[r][1].type||"space"===e[r][1].type)){for(t=l;++t13&&n<32||n>126&&n<160||n>55295&&n<57344||n>64975&&n<65008||(65535&n)==65535||(65535&n)==65534||n>1114111?"�":String.fromCodePoint(n)}let tL=/\\([!-/:-@[-`{-~])|&(#(?:\d{1,7}|x[\da-f]{1,6})|[\da-z]{1,31});/gi;function tD(e,t,n){if(t)return t;if(35===n.charCodeAt(0)){let e=n.charCodeAt(1),t=120===e||88===e;return tM(n.slice(t?2:1),t?16:10)}return tc(n)||e}let tj={}.hasOwnProperty;function tF(e){return{line:e.line,column:e.column,offset:e.offset}}function tR(e,t){if(e)throw Error("Cannot close `"+e.type+"` ("+Y({start:e.start,end:e.end})+"): a different token (`"+t.type+"`, "+Y({start:t.start,end:t.end})+") is open");throw Error("Cannot close document, a token (`"+t.type+"`, "+Y({start:t.start,end:t.end})+") is still open")}function tN(e){let t=this;t.parser=function(n){var r,i;let o,a,u,c;return"string"!=typeof(r={...t.data("settings"),...e,extensions:t.data("micromarkExtensions")||[],mdastExtensions:t.data("fromMarkdownExtensions")||[]})&&(i=r,r=void 0),(function(e){let t={transforms:[],canContainEols:["emphasis","fragment","heading","paragraph","strong"],enter:{autolink:r(y),autolinkProtocol:c,autolinkEmail:c,atxHeading:r(h),blockQuote:r(function(){return{type:"blockquote",children:[]}}),characterEscape:c,characterReference:c,codeFenced:r(d),codeFencedFenceInfo:l,codeFencedFenceMeta:l,codeIndented:r(d,l),codeText:r(function(){return{type:"inlineCode",value:""}},l),codeTextData:c,data:c,codeFlowValue:c,definition:r(function(){return{type:"definition",identifier:"",label:null,title:null,url:""}}),definitionDestinationString:l,definitionLabelString:l,definitionTitleString:l,emphasis:r(function(){return{type:"emphasis",children:[]}}),hardBreakEscape:r(m),hardBreakTrailing:r(m),htmlFlow:r(g,l),htmlFlowData:c,htmlText:r(g,l),htmlTextData:c,image:r(function(){return{type:"image",title:null,url:"",alt:null}}),label:l,link:r(y),listItem:r(function(e){return{type:"listItem",spread:e._spread,checked:null,children:[]}}),listItemValue:function(e){this.data.expectingFirstListItemValue&&(this.stack[this.stack.length-2].start=Number.parseInt(this.sliceSerialize(e),10),this.data.expectingFirstListItemValue=void 0)},listOrdered:r(v,function(){this.data.expectingFirstListItemValue=!0}),listUnordered:r(v),paragraph:r(function(){return{type:"paragraph",children:[]}}),reference:function(){this.data.referenceType="collapsed"},referenceString:l,resourceDestinationString:l,resourceTitleString:l,setextHeading:r(h),strong:r(function(){return{type:"strong",children:[]}}),thematicBreak:r(function(){return{type:"thematicBreak"}})},exit:{atxHeading:o(),atxHeadingSequence:function(e){let t=this.stack[this.stack.length-1];if(!t.depth){let n=this.sliceSerialize(e).length;t.depth=n}},autolink:o(),autolinkEmail:function(e){s.call(this,e),this.stack[this.stack.length-1].url="mailto:"+this.sliceSerialize(e)},autolinkProtocol:function(e){s.call(this,e),this.stack[this.stack.length-1].url=this.sliceSerialize(e)},blockQuote:o(),characterEscapeValue:s,characterReferenceMarkerHexadecimal:p,characterReferenceMarkerNumeric:p,characterReferenceValue:function(e){let t;let n=this.sliceSerialize(e),r=this.data.characterReferenceType;r?(t=tM(n,"characterReferenceMarkerNumeric"===r?10:16),this.data.characterReferenceType=void 0):t=tc(n);let l=this.stack.pop();l.value+=t,l.position.end=tF(e.end)},codeFenced:o(function(){let e=this.resume();this.stack[this.stack.length-1].value=e.replace(/^(\r?\n|\r)|(\r?\n|\r)$/g,""),this.data.flowCodeInside=void 0}),codeFencedFence:function(){this.data.flowCodeInside||(this.buffer(),this.data.flowCodeInside=!0)},codeFencedFenceInfo:function(){let e=this.resume();this.stack[this.stack.length-1].lang=e},codeFencedFenceMeta:function(){let e=this.resume();this.stack[this.stack.length-1].meta=e},codeFlowValue:s,codeIndented:o(function(){let e=this.resume();this.stack[this.stack.length-1].value=e.replace(/(\r?\n|\r)$/g,"")}),codeText:o(function(){let e=this.resume();this.stack[this.stack.length-1].value=e}),codeTextData:s,data:s,definition:o(),definitionDestinationString:function(){let e=this.resume();this.stack[this.stack.length-1].url=e},definitionLabelString:function(e){let t=this.resume(),n=this.stack[this.stack.length-1];n.label=t,n.identifier=e8(this.sliceSerialize(e)).toLowerCase()},definitionTitleString:function(){let e=this.resume();this.stack[this.stack.length-1].title=e},emphasis:o(),hardBreakEscape:o(f),hardBreakTrailing:o(f),htmlFlow:o(function(){let e=this.resume();this.stack[this.stack.length-1].value=e}),htmlFlowData:s,htmlText:o(function(){let e=this.resume();this.stack[this.stack.length-1].value=e}),htmlTextData:s,image:o(function(){let e=this.stack[this.stack.length-1];if(this.data.inReference){let t=this.data.referenceType||"shortcut";e.type+="Reference",e.referenceType=t,delete e.url,delete e.title}else delete e.identifier,delete e.label;this.data.referenceType=void 0}),label:function(){let e=this.stack[this.stack.length-1],t=this.resume(),n=this.stack[this.stack.length-1];if(this.data.inReference=!0,"link"===n.type){let t=e.children;n.children=t}else n.alt=t},labelText:function(e){let t=this.sliceSerialize(e),n=this.stack[this.stack.length-2];n.label=t.replace(tL,tD),n.identifier=e8(t).toLowerCase()},lineEnding:function(e){let n=this.stack[this.stack.length-1];if(this.data.atHardBreak){n.children[n.children.length-1].position.end=tF(e.end),this.data.atHardBreak=void 0;return}!this.data.setextHeadingSlurpLineEnding&&t.canContainEols.includes(n.type)&&(c.call(this,e),s.call(this,e))},link:o(function(){let e=this.stack[this.stack.length-1];if(this.data.inReference){let t=this.data.referenceType||"shortcut";e.type+="Reference",e.referenceType=t,delete e.url,delete e.title}else delete e.identifier,delete e.label;this.data.referenceType=void 0}),listItem:o(),listOrdered:o(),listUnordered:o(),paragraph:o(),referenceString:function(e){let t=this.resume(),n=this.stack[this.stack.length-1];n.label=t,n.identifier=e8(this.sliceSerialize(e)).toLowerCase(),this.data.referenceType="full"},resourceDestinationString:function(){let e=this.resume();this.stack[this.stack.length-1].url=e},resourceTitleString:function(){let e=this.resume();this.stack[this.stack.length-1].title=e},resource:function(){this.data.inReference=void 0},setextHeading:o(function(){this.data.setextHeadingSlurpLineEnding=void 0}),setextHeadingLineSequence:function(e){this.stack[this.stack.length-1].depth=61===this.sliceSerialize(e).codePointAt(0)?1:2},setextHeadingText:function(){this.data.setextHeadingSlurpLineEnding=!0},strong:o(),thematicBreak:o()}};(function e(t,n){let r=-1;for(;++r0){let e=o.tokenStack[o.tokenStack.length-1];(e[1]||tR).call(o,void 0,e[0])}for(r.position={start:tF(e.length>0?e[0][1].start:{line:1,column:1,offset:0}),end:tF(e.length>0?e[e.length-2][1].end:{line:1,column:1,offset:0})},s=-1;++s-1){let e=n[0];"string"==typeof e?n[0]=e.slice(l):n.shift()}o>0&&n.push(e[i].slice(0,o))}return n}(o,e)}function p(){let{line:e,column:t,offset:n,_index:l,_bufferIndex:i}=r;return{line:e,column:t,offset:n,_index:l,_bufferIndex:i}}function d(e,t){t.restore()}function h(e,t){return function(n,l,i){let o,s,f,d;return Array.isArray(n)?h(n):"tokenize"in n?h([n]):function(e){let t=null!==e&&n[e],r=null!==e&&n.null;return h([...Array.isArray(t)?t:t?[t]:[],...Array.isArray(r)?r:r?[r]:[]])(e)};function h(e){return(o=e,s=0,0===e.length)?i:m(e[s])}function m(e){return function(n){return(d=function(){let e=p(),t=c.previous,n=c.currentConstruct,l=c.events.length,i=Array.from(a);return{restore:function(){r=e,c.previous=t,c.currentConstruct=n,c.events.length=l,a=i,g()},from:l}}(),f=e,e.partial||(c.currentConstruct=e),e.name&&c.parser.constructs.disable.null.includes(e.name))?v(n):e.tokenize.call(t?Object.assign(Object.create(c),t):c,u,y,v)(n)}}function y(t){return e(f,d),l}function v(e){return(d.restore(),++s{let n=(t,n)=>(e.set(n,t),t),r=l=>{if(e.has(l))return e.get(l);let[i,o]=t[l];switch(i){case 0:case -1:return n(o,l);case 1:{let e=n([],l);for(let t of o)e.push(r(t));return e}case 2:{let e=n({},l);for(let[t,n]of o)e[r(t)]=r(n);return e}case 3:return n(new Date(o),l);case 4:{let{source:e,flags:t}=o;return n(new RegExp(e,t),l)}case 5:{let e=n(new Map,l);for(let[t,n]of o)e.set(r(t),r(n));return e}case 6:{let e=n(new Set,l);for(let t of o)e.add(r(t));return e}case 7:{let{name:e,message:t}=o;return n(new t_[e](t),l)}case 8:return n(BigInt(o),l);case"BigInt":return n(Object(BigInt(o)),l)}return n(new t_[i](o),l)};return r},tH=e=>tB(new Map,e)(0),{toString:tV}={},{keys:tU}=Object,tZ=e=>{let t=typeof e;if("object"!==t||!e)return[0,t];let n=tV.call(e).slice(8,-1);switch(n){case"Array":return[1,""];case"Object":return[2,""];case"Date":return[3,""];case"RegExp":return[4,""];case"Map":return[5,""];case"Set":return[6,""]}return n.includes("Array")?[1,n]:n.includes("Error")?[7,n]:[2,n]},tq=([e,t])=>0===e&&("function"===t||"symbol"===t),tW=(e,t,n,r)=>{let l=(e,t)=>{let l=r.push(e)-1;return n.set(t,l),l},i=r=>{if(n.has(r))return n.get(r);let[o,a]=tZ(r);switch(o){case 0:{let t=r;switch(a){case"bigint":o=8,t=r.toString();break;case"function":case"symbol":if(e)throw TypeError("unable to serialize "+a);t=null;break;case"undefined":return l([-1],r)}return l([o,t],r)}case 1:{if(a)return l([a,[...r]],r);let e=[],t=l([o,e],r);for(let t of r)e.push(i(t));return t}case 2:{if(a)switch(a){case"BigInt":return l([a,r.toString()],r);case"Boolean":case"Number":case"String":return l([a,r.valueOf()],r)}if(t&&"toJSON"in r)return i(r.toJSON());let n=[],u=l([o,n],r);for(let t of tU(r))(e||!tq(tZ(r[t])))&&n.push([i(t),i(r[t])]);return u}case 3:return l([o,r.toISOString()],r);case 4:{let{source:e,flags:t}=r;return l([o,{source:e,flags:t}],r)}case 5:{let t=[],n=l([o,t],r);for(let[n,l]of r)(e||!(tq(tZ(n))||tq(tZ(l))))&&t.push([i(n),i(l)]);return n}case 6:{let t=[],n=l([o,t],r);for(let n of r)(e||!tq(tZ(n)))&&t.push(i(n));return n}}let{message:u}=r;return l([o,{name:a,message:u}],r)};return i},tK=(e,{json:t,lossy:n}={})=>{let r=[];return tW(!(t||n),!!t,new Map,r)(e),r};var tQ="function"==typeof structuredClone?(e,t)=>t&&("json"in t||"lossy"in t)?tH(tK(e,t)):structuredClone(e):(e,t)=>tH(tK(e,t));function tY(e){let t=[],n=-1,r=0,l=0;for(;++n55295&&i<57344){let t=e.charCodeAt(n+1);i<56320&&t>56319&&t<57344?(o=String.fromCharCode(i,t),l=1):o="�"}else o=String.fromCharCode(i);o&&(t.push(e.slice(r,n),encodeURIComponent(o)),r=n+l+1,o=""),l&&(n+=l,l=0)}return t.join("")+e.slice(r)}function t$(e,t){let n=[{type:"text",value:"↩"}];return t>1&&n.push({type:"element",tagName:"sup",properties:{},children:[{type:"text",value:String(t)}]}),n}function tX(e,t){return"Back to reference "+(e+1)+(t>1?"-"+t:"")}let tJ=function(e){if(null==e)return t1;if("function"==typeof e)return tG(e);if("object"==typeof e)return Array.isArray(e)?function(e){let t=[],n=-1;for(;++n":"")+")"})}return s;function s(){var c;let s,f,p,d=t0;if((!t||i(l,a,u[u.length-1]||void 0))&&!1===(d=Array.isArray(c=n(l,u))?c:"number"==typeof c?[!0,c]:null==c?t0:[c])[0])return d;if("children"in l&&l.children&&l.children&&"skip"!==d[0])for(f=(r?l.children.length:-1)+o,p=u.concat(l);f>-1&&f1:t}function t3(e,t,n){let r=0,l=e.length;if(t){let t=e.codePointAt(r);for(;9===t||32===t;)r++,t=e.codePointAt(r)}if(n){let t=e.codePointAt(l-1);for(;9===t||32===t;)l--,t=e.codePointAt(l-1)}return l>r?e.slice(r,l):""}let t5={blockquote:function(e,t){let n={type:"element",tagName:"blockquote",properties:{},children:e.wrap(e.all(t),!0)};return e.patch(t,n),e.applyData(t,n)},break:function(e,t){let n={type:"element",tagName:"br",properties:{},children:[]};return e.patch(t,n),[e.applyData(t,n),{type:"text",value:"\n"}]},code:function(e,t){let n=t.value?t.value+"\n":"",r={};t.lang&&(r.className=["language-"+t.lang]);let l={type:"element",tagName:"code",properties:r,children:[{type:"text",value:n}]};return t.meta&&(l.data={meta:t.meta}),e.patch(t,l),l={type:"element",tagName:"pre",properties:{},children:[l=e.applyData(t,l)]},e.patch(t,l),l},delete:function(e,t){let n={type:"element",tagName:"del",properties:{},children:e.all(t)};return e.patch(t,n),e.applyData(t,n)},emphasis:function(e,t){let n={type:"element",tagName:"em",properties:{},children:e.all(t)};return e.patch(t,n),e.applyData(t,n)},footnoteReference:function(e,t){let n;let r="string"==typeof e.options.clobberPrefix?e.options.clobberPrefix:"user-content-",l=String(t.identifier).toUpperCase(),i=tY(l.toLowerCase()),o=e.footnoteOrder.indexOf(l),a=e.footnoteCounts.get(l);void 0===a?(a=0,e.footnoteOrder.push(l),n=e.footnoteOrder.length):n=o+1,a+=1,e.footnoteCounts.set(l,a);let u={type:"element",tagName:"a",properties:{href:"#"+r+"fn-"+i,id:r+"fnref-"+i+(a>1?"-"+a:""),dataFootnoteRef:!0,ariaDescribedBy:["footnote-label"]},children:[{type:"text",value:String(n)}]};e.patch(t,u);let c={type:"element",tagName:"sup",properties:{},children:[u]};return e.patch(t,c),e.applyData(t,c)},heading:function(e,t){let n={type:"element",tagName:"h"+t.depth,properties:{},children:e.all(t)};return e.patch(t,n),e.applyData(t,n)},html:function(e,t){if(e.options.allowDangerousHtml){let n={type:"raw",value:t.value};return e.patch(t,n),e.applyData(t,n)}},imageReference:function(e,t){let n=String(t.identifier).toUpperCase(),r=e.definitionById.get(n);if(!r)return t4(e,t);let l={src:tY(r.url||""),alt:t.alt};null!==r.title&&void 0!==r.title&&(l.title=r.title);let i={type:"element",tagName:"img",properties:l,children:[]};return e.patch(t,i),e.applyData(t,i)},image:function(e,t){let n={src:tY(t.url)};null!==t.alt&&void 0!==t.alt&&(n.alt=t.alt),null!==t.title&&void 0!==t.title&&(n.title=t.title);let r={type:"element",tagName:"img",properties:n,children:[]};return e.patch(t,r),e.applyData(t,r)},inlineCode:function(e,t){let n={type:"text",value:t.value.replace(/\r?\n|\r/g," ")};e.patch(t,n);let r={type:"element",tagName:"code",properties:{},children:[n]};return e.patch(t,r),e.applyData(t,r)},linkReference:function(e,t){let n=String(t.identifier).toUpperCase(),r=e.definitionById.get(n);if(!r)return t4(e,t);let l={href:tY(r.url||"")};null!==r.title&&void 0!==r.title&&(l.title=r.title);let i={type:"element",tagName:"a",properties:l,children:e.all(t)};return e.patch(t,i),e.applyData(t,i)},link:function(e,t){let n={href:tY(t.url)};null!==t.title&&void 0!==t.title&&(n.title=t.title);let r={type:"element",tagName:"a",properties:n,children:e.all(t)};return e.patch(t,r),e.applyData(t,r)},listItem:function(e,t,n){let r=e.all(t),l=n?function(e){let t=!1;if("list"===e.type){t=e.spread||!1;let n=e.children,r=-1;for(;!t&&++r0&&e.children.unshift({type:"text",value:" "}),e.children.unshift({type:"element",tagName:"input",properties:{type:"checkbox",checked:t.checked,disabled:!0},children:[]}),i.className=["task-list-item"]}let a=-1;for(;++a0){let r={type:"element",tagName:"tbody",properties:{},children:e.wrap(n,!0)},i=K(t.children[1]),o=W(t.children[t.children.length-1]);i&&o&&(r.position={start:i,end:o}),l.push(r)}let i={type:"element",tagName:"table",properties:{},children:e.wrap(l,!0)};return e.patch(t,i),e.applyData(t,i)},tableCell:function(e,t){let n={type:"element",tagName:"td",properties:{},children:e.all(t)};return e.patch(t,n),e.applyData(t,n)},tableRow:function(e,t,n){let r=n?n.children:void 0,l=0===(r?r.indexOf(t):1)?"th":"td",i=n&&"table"===n.type?n.align:void 0,o=i?i.length:t.children.length,a=-1,u=[];for(;++a0,!0),r[0]),l=r.index+r[0].length,r=n.exec(t);return i.push(t3(t.slice(l),l>0,!1)),i.join("")}(String(t.value))};return e.patch(t,n),e.applyData(t,n)},thematicBreak:function(e,t){let n={type:"element",tagName:"hr",properties:{},children:[]};return e.patch(t,n),e.applyData(t,n)},toml:t8,yaml:t8,definition:t8,footnoteDefinition:t8};function t8(){}let t9={}.hasOwnProperty,t7={};function ne(e,t){e.position&&(t.position=function(e){let t=K(e),n=W(e);if(t&&n)return{start:t,end:n}}(e))}function nt(e,t){let n=t;if(e&&e.data){let t=e.data.hName,r=e.data.hChildren,l=e.data.hProperties;"string"==typeof t&&("element"===n.type?n.tagName=t:n={type:"element",tagName:t,properties:{},children:"children"in n?n.children:[n]}),"element"===n.type&&l&&Object.assign(n.properties,tQ(l)),"children"in n&&n.children&&null!=r&&(n.children=r)}return n}function nn(e,t){let n=[],r=-1;for(t&&n.push({type:"text",value:"\n"});++r0&&n.push({type:"text",value:"\n"}),n}function nr(e){let t=0,n=e.charCodeAt(t);for(;9===n||32===n;)t++,n=e.charCodeAt(t);return e.slice(t)}function nl(e,t){let n=function(e,t){let n=t||t7,r=new Map,l=new Map,i={all:function(e){let t=[];if("children"in e){let n=e.children,r=-1;for(;++r0&&f.push({type:"text",value:" "});let e="string"==typeof n?n:n(u,s);"string"==typeof e&&(e={type:"text",value:e}),f.push({type:"element",tagName:"a",properties:{href:"#"+t+"fnref-"+c+(s>1?"-"+s:""),dataFootnoteBackref:"",ariaLabel:"string"==typeof r?r:r(u,s),className:["data-footnote-backref"]},children:Array.isArray(e)?e:[e]})}let d=i[i.length-1];if(d&&"element"===d.type&&"p"===d.tagName){let e=d.children[d.children.length-1];e&&"text"===e.type?e.value+=" ":d.children.push({type:"text",value:" "}),d.children.push(...f)}else i.push(...f);let h={type:"element",tagName:"li",properties:{id:t+"fn-"+c},children:e.wrap(i,!0)};e.patch(l,h),a.push(h)}if(0!==a.length)return{type:"element",tagName:"section",properties:{dataFootnotes:!0,className:["footnotes"]},children:[{type:"element",tagName:i,properties:{...tQ(o),id:"footnote-label"},children:[{type:"text",value:l}]},{type:"text",value:"\n"},{type:"element",tagName:"ol",properties:{},children:e.wrap(a,!0)},{type:"text",value:"\n"}]}}(n),i=Array.isArray(r)?{type:"root",children:r}:r||{type:"root",children:[]};return l&&i.children.push({type:"text",value:"\n"},l),i}function ni(e,t){return e&&"run"in e?async function(n,r){let l=nl(n,{file:r,...t});await e.run(l,r)}:function(n,r){return nl(n,{file:r,...t||e})}}function no(e){if(e)throw e}var na=n(6500);function nu(e){if("object"!=typeof e||null===e)return!1;let t=Object.getPrototypeOf(e);return(null===t||t===Object.prototype||null===Object.getPrototypeOf(t))&&!(Symbol.toStringTag in e)&&!(Symbol.iterator in e)}let nc={basename:function(e,t){let n;if(void 0!==t&&"string"!=typeof t)throw TypeError('"ext" argument must be a string');ns(e);let r=0,l=-1,i=e.length;if(void 0===t||0===t.length||t.length>e.length){for(;i--;)if(47===e.codePointAt(i)){if(n){r=i+1;break}}else l<0&&(n=!0,l=i+1);return l<0?"":e.slice(r,l)}if(t===e)return"";let o=-1,a=t.length-1;for(;i--;)if(47===e.codePointAt(i)){if(n){r=i+1;break}}else o<0&&(n=!0,o=i+1),a>-1&&(e.codePointAt(i)===t.codePointAt(a--)?a<0&&(l=i):(a=-1,l=o));return r===l?l=o:l<0&&(l=e.length),e.slice(r,l)},dirname:function(e){let t;if(ns(e),0===e.length)return".";let n=-1,r=e.length;for(;--r;)if(47===e.codePointAt(r)){if(t){n=r;break}}else t||(t=!0);return n<0?47===e.codePointAt(0)?"/":".":1===n&&47===e.codePointAt(0)?"//":e.slice(0,n)},extname:function(e){let t;ns(e);let n=e.length,r=-1,l=0,i=-1,o=0;for(;n--;){let a=e.codePointAt(n);if(47===a){if(t){l=n+1;break}continue}r<0&&(t=!0,r=n+1),46===a?i<0?i=n:1!==o&&(o=1):i>-1&&(o=-1)}return i<0||r<0||0===o||1===o&&i===r-1&&i===l+1?"":e.slice(i,r)},join:function(...e){let t,n=-1;for(;++n2){if((r=l.lastIndexOf("/"))!==l.length-1){r<0?(l="",i=0):i=(l=l.slice(0,r)).length-1-l.lastIndexOf("/"),o=u,a=0;continue}}else if(l.length>0){l="",i=0,o=u,a=0;continue}}t&&(l=l.length>0?l+"/..":"..",i=2)}else l.length>0?l+="/"+e.slice(o+1,u):l=e.slice(o+1,u),i=u-o-1;o=u,a=0}else 46===n&&a>-1?a++:a=-1}return l}(e,!t);return 0!==n.length||t||(n="."),n.length>0&&47===e.codePointAt(e.length-1)&&(n+="/"),t?"/"+n:n}(t)},sep:"/"};function ns(e){if("string"!=typeof e)throw TypeError("Path must be a string. Received "+JSON.stringify(e))}function nf(e){return!!(null!==e&&"object"==typeof e&&"href"in e&&e.href&&"protocol"in e&&e.protocol&&void 0===e.auth)}let np=["history","path","basename","stem","extname","dirname"];class nd{constructor(e){let t,n;t=e?nf(e)?{path:e}:"string"==typeof e||e&&"object"==typeof e&&"byteLength"in e&&"byteOffset"in e?{value:e}:e:{},this.cwd="/",this.data={},this.history=[],this.messages=[],this.value,this.map,this.result,this.stored;let r=-1;for(;++rt.length;o&&t.push(r);try{i=e.apply(this,t)}catch(e){if(o&&n)throw e;return r(e)}o||(i&&i.then&&"function"==typeof i.then?i.then(l,r):i instanceof Error?r(i):l(i))};function r(e,...l){n||(n=!0,t(e,...l))}function l(e){r(null,e)}})(a,l)(...o):r(null,...o)})(null,...t)},use:function(n){if("function"!=typeof n)throw TypeError("Expected `middelware` to be a function, not "+n);return e.push(n),t}};return t}()}copy(){let e=new nx,t=-1;for(;++t0){let[r,...i]=t,o=n[l][1];nu(o)&&nu(r)&&(r=na(!0,o,r)),n[l]=[e,r,...i]}}}}let nk=new nx().freeze();function nb(e,t){if("function"!=typeof t)throw TypeError("Cannot `"+e+"` without `parser`")}function nw(e,t){if("function"!=typeof t)throw TypeError("Cannot `"+e+"` without `compiler`")}function nS(e,t){if(t)throw Error("Cannot call `"+e+"` on a frozen processor.\nCreate a new processor first, by calling it: use `processor()` instead of `processor`.")}function nE(e){if(!nu(e)||"string"!=typeof e.type)throw TypeError("Expected node, got `"+e+"`")}function nC(e,t,n){if(!n)throw Error("`"+e+"` finished async. Use `"+t+"` instead")}function nP(e){return e&&"object"==typeof e&&"message"in e&&"messages"in e?e:new nd(e)}let nO=[],nz={allowDangerousHtml:!0},nT=/^(https?|ircs?|mailto|xmpp)$/i,nI=[{from:"astPlugins",id:"remove-buggy-html-in-markdown-parser"},{from:"allowDangerousHtml",id:"remove-buggy-html-in-markdown-parser"},{from:"allowNode",id:"replace-allownode-allowedtypes-and-disallowedtypes",to:"allowElement"},{from:"allowedTypes",id:"replace-allownode-allowedtypes-and-disallowedtypes",to:"allowedElements"},{from:"disallowedTypes",id:"replace-allownode-allowedtypes-and-disallowedtypes",to:"disallowedElements"},{from:"escapeHtml",id:"remove-buggy-html-in-markdown-parser"},{from:"includeElementIndex",id:"#remove-includeelementindex"},{from:"includeNodeIndex",id:"change-includenodeindex-to-includeelementindex"},{from:"linkTarget",id:"remove-linktarget"},{from:"plugins",id:"change-plugins-to-remarkplugins",to:"remarkPlugins"},{from:"rawSourcePos",id:"#remove-rawsourcepos"},{from:"renderers",id:"change-renderers-to-components",to:"components"},{from:"source",id:"change-source-to-children",to:"children"},{from:"sourcePos",id:"#remove-sourcepos"},{from:"transformImageUri",id:"#add-urltransform",to:"urlTransform"},{from:"transformLinkUri",id:"#add-urltransform",to:"urlTransform"}];function nA(e){let t=e.allowedElements,n=e.allowElement,r=e.children||"",l=e.className,i=e.components,o=e.disallowedElements,a=e.rehypePlugins||nO,u=e.remarkPlugins||nO,c=e.remarkRehypeOptions?{...e.remarkRehypeOptions,...nz}:nz,s=e.skipHtml,f=e.unwrapDisallowed,p=e.urlTransform||nM,d=nk().use(tN).use(u).use(ni,c).use(a),h=new nd;for(let t of("string"==typeof r&&(h.value=r),nI))Object.hasOwn(e,t.from)&&(t.from,t.to&&t.to,t.id);let m=d.parse(h),g=d.runSync(m,h);return l&&(g={type:"element",tagName:"div",properties:{className:l},children:"root"===g.type?g.children:[g]}),t2(g,function(e,r,l){if("raw"===e.type&&l&&"number"==typeof r)return s?l.children.splice(r,1):l.children[r]={type:"text",value:e.value},r;if("element"===e.type){let t;for(t in em)if(Object.hasOwn(em,t)&&Object.hasOwn(e.properties,t)){let n=e.properties[t],r=em[t];(null===r||r.includes(e.tagName))&&(e.properties[t]=p(String(n||""),t,e))}}if("element"===e.type){let i=t?!t.includes(e.tagName):!!o&&o.includes(e.tagName);if(!i&&n&&"number"==typeof r&&(i=!n(e,r,l)),i&&l&&"number"==typeof r)return f&&e.children?l.children.splice(r,1,...e.children):l.children.splice(r,1),r}}),function(e,t){var n,r,l;let i;if(!t||void 0===t.Fragment)throw TypeError("Expected `Fragment` in options");let o=t.filePath||void 0;if(t.development){if("function"!=typeof t.jsxDEV)throw TypeError("Expected `jsxDEV` in options when `development: true`");n=t.jsxDEV,i=function(e,t,r,l){let i=Array.isArray(r.children),a=K(e);return n(t,r,l,i,{columnNumber:a?a.column-1:void 0,fileName:o,lineNumber:a?a.line:void 0},void 0)}}else{if("function"!=typeof t.jsx)throw TypeError("Expected `jsx` in production options");if("function"!=typeof t.jsxs)throw TypeError("Expected `jsxs` in production options");r=t.jsx,l=t.jsxs,i=function(e,t,n,i){let o=Array.isArray(n.children)?l:r;return i?o(t,n,i):o(t,n)}}let a={Fragment:t.Fragment,ancestors:[],components:t.components||{},create:i,elementAttributeNameCase:t.elementAttributeNameCase||"react",evaluater:t.createEvaluater?t.createEvaluater():void 0,filePath:o,ignoreInvalidStyle:t.ignoreInvalidStyle||!1,passKeys:!1!==t.passKeys,passNode:t.passNode||!1,schema:"svg"===t.space?R:F,stylePropertyNameCase:t.stylePropertyNameCase||"dom",tableCellAlignToStyle:!1!==t.tableCellAlignToStyle},u=ea(a,e,void 0);return u&&"string"!=typeof u?u:a.create(e,a.Fragment,{children:u||void 0},void 0)}(g,{Fragment:eg.Fragment,components:i,ignoreInvalidStyle:!0,jsx:eg.jsx,jsxs:eg.jsxs,passKeys:!0,passNode:!0})}function nM(e){let t=e.indexOf(":"),n=e.indexOf("?"),r=e.indexOf("#"),l=e.indexOf("/");return t<0||l>-1&&t>l||n>-1&&t>n||r>-1&&t>r||nT.test(e.slice(0,t))?e:""}}}]); \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/1116-2d5ec30ef7d86f0e.js b/litellm/proxy/_experimental/out/_next/static/chunks/1116-2d5ec30ef7d86f0e.js deleted file mode 100644 index 7c33ceca243..00000000000 --- a/litellm/proxy/_experimental/out/_next/static/chunks/1116-2d5ec30ef7d86f0e.js +++ /dev/null @@ -1 +0,0 @@ -"use strict";(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[1116],{69993:function(e,t,r){r.d(t,{Z:function(){return s}});var n=r(1119),o=r(2265),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M300 328a60 60 0 10120 0 60 60 0 10-120 0zM852 64H172c-17.7 0-32 14.3-32 32v660c0 17.7 14.3 32 32 32h680c17.7 0 32-14.3 32-32V96c0-17.7-14.3-32-32-32zm-32 660H204V128h616v596zM604 328a60 60 0 10120 0 60 60 0 10-120 0zm250.2 556H169.8c-16.5 0-29.8 14.3-29.8 32v36c0 4.4 3.3 8 7.4 8h729.1c4.1 0 7.4-3.6 7.4-8v-36c.1-17.7-13.2-32-29.7-32zM664 508H360c-4.4 0-8 3.6-8 8v60c0 4.4 3.6 8 8 8h304c4.4 0 8-3.6 8-8v-60c0-4.4-3.6-8-8-8z"}}]},name:"robot",theme:"outlined"},i=r(55015),s=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,n.Z)({},e,{ref:t,icon:a}))})},92858:function(e,t,r){r.d(t,{Z:function(){return S}});var n=r(5853),o=r(2265),a=r(62963),i=r(90945),s=r(13323),l=r(17684),c=r(80004),u=r(93689),d=r(38198),f=r(47634),m=r(56314),h=r(27847),p=r(64518);let g=(0,o.createContext)(null),v=Object.assign((0,h.yV)(function(e,t){let r=(0,l.M)(),{id:n="headlessui-description-".concat(r),...a}=e,i=function e(){let t=(0,o.useContext)(g);if(null===t){let t=Error("You used a component, but it is not inside a relevant parent.");throw Error.captureStackTrace&&Error.captureStackTrace(t,e),t}return t}(),s=(0,u.T)(t);(0,p.e)(()=>i.register(n),[n,i.register]);let c={ref:s,...i.props,id:n};return(0,h.sY)({ourProps:c,theirProps:a,slot:i.slot||{},defaultTag:"p",name:i.name||"Description"})}),{});var w=r(37388);let k=(0,o.createContext)(null),b=Object.assign((0,h.yV)(function(e,t){let r=(0,l.M)(),{id:n="headlessui-label-".concat(r),passive:a=!1,...i}=e,s=function e(){let t=(0,o.useContext)(k);if(null===t){let t=Error("You used a