diff --git a/.circleci/config.yml b/.circleci/config.yml index 62c12c1cf92..6dd1177f79c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3339,7 +3339,7 @@ jobs: python -m build twine upload --verbose dist/* - e2e_ui_testing: + ui_build: machine: image: ubuntu-2204:2023.10.1 resource_class: xlarge @@ -3366,6 +3366,48 @@ jobs: # Now source the build script source ./build_ui.sh + - persist_to_workspace: + root: . + paths: + - litellm/proxy/_experimental/out + + ui_unit_tests: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Run UI unit tests (Vitest) + command: | + # Use Node 20 (several deps require >=20) + export NVM_DIR="/opt/circleci/.nvm" + source "$NVM_DIR/nvm.sh" + nvm install 20 + nvm use 20 + + cd ui/litellm-dashboard + npm ci || npm install + + # CI run, with both LCOV (Codecov) and HTML (artifact you can click) + CI=true npm run test -- --run --coverage \ + --coverage.provider=v8 \ + --coverage.reporter=lcov \ + --coverage.reporter=html \ + --coverage.reportsDirectory=coverage/html + + e2e_ui_testing: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - attach_workspace: + at: ~/project - run: name: Upgrade Docker to v24.x (API 1.44+) command: | @@ -3411,24 +3453,6 @@ jobs: name: Install Playwright Browsers command: | npx playwright install - - run: - name: Run UI unit tests (Vitest) - command: | - # Use Node 20 (several deps require >=20) - export NVM_DIR="/opt/circleci/.nvm" - source "$NVM_DIR/nvm.sh" - nvm install 20 - nvm use 20 - - cd ui/litellm-dashboard - npm ci || npm install - - # CI run, with both LCOV (Codecov) and HTML (artifact you can click) - CI=true npm run test -- --run --coverage \ - --coverage.provider=v8 \ - --coverage.reporter=lcov \ - --coverage.reporter=html \ - --coverage.reportsDirectory=coverage/html - run: name: Build Docker image @@ -3633,6 +3657,20 @@ workflows: only: - main - /litellm_.*/ + - ui_build: + filters: + branches: + only: + - main + - /litellm_.*/ + - ui_unit_tests: + requires: + - ui_build + filters: + branches: + only: + - main + - /litellm_.*/ - auth_ui_unit_tests: filters: branches: @@ -3640,6 +3678,8 @@ workflows: - main - /litellm_.*/ - e2e_ui_testing: + requires: + - ui_build filters: branches: only: diff --git a/Makefile b/Makefile index a79a397f945..1614a58fc7d 100644 --- a/Makefile +++ b/Makefile @@ -34,13 +34,13 @@ install-proxy-dev: # CI-compatible installations (matches GitHub workflows exactly) install-dev-ci: - pip install openai==1.99.5 + pip install openai==2.8.0 poetry install --with dev - pip install openai==1.99.5 + pip install openai==2.8.0 install-proxy-dev-ci: poetry install --with dev,proxy-dev --extras proxy - pip install openai==1.99.5 + pip install openai==2.8.0 install-test-deps: install-proxy-dev poetry run pip install "pytest-retry==1.6.3" diff --git a/cookbook/LiteLLM_CometAPI.ipynb b/cookbook/LiteLLM_CometAPI.ipynb index bdd916c5bfe..0a7ab581ae3 100644 --- a/cookbook/LiteLLM_CometAPI.ipynb +++ b/cookbook/LiteLLM_CometAPI.ipynb @@ -28,7 +28,7 @@ "Requirement already satisfied: importlib-metadata>=6.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (8.6.1)\n", "Requirement already satisfied: jinja2<4.0.0,>=3.1.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (3.1.6)\n", "Requirement already satisfied: jsonschema<5.0.0,>=4.22.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (4.25.1)\n", - "Requirement already satisfied: openai>=1.99.5 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.109.1)\n", + "Requirement already satisfied: openai>=2.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.109.1)\n", "Requirement already satisfied: pydantic<3.0.0,>=2.5.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (2.11.10)\n", "Requirement already satisfied: python-dotenv>=0.2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.1.1)\n", "Requirement already satisfied: tiktoken>=0.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.12.0)\n", @@ -50,11 +50,11 @@ "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (2025.9.1)\n", "Requirement already satisfied: referencing>=0.28.4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.36.2)\n", "Requirement already satisfied: rpds-py>=0.7.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.27.1)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (1.9.0)\n", - "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (0.11.0)\n", - "Requirement already satisfied: sniffio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (1.3.1)\n", - "Requirement already satisfied: tqdm>4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (4.67.1)\n", - "Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (4.15.0)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.9.0)\n", + "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (0.11.0)\n", + "Requirement already satisfied: sniffio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.3.1)\n", + "Requirement already satisfied: tqdm>4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.67.1)\n", + "Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.15.0)\n", "Requirement already satisfied: annotated-types>=0.6.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.7.0)\n", "Requirement already satisfied: pydantic-core==2.33.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (2.33.2)\n", "Requirement already satisfied: typing-inspection>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.4.2)\n", diff --git a/deploy/charts/litellm-helm/Chart.yaml b/deploy/charts/litellm-helm/Chart.yaml index aa81e4efecc..eedadebaa8e 100644 --- a/deploy/charts/litellm-helm/Chart.yaml +++ b/deploy/charts/litellm-helm/Chart.yaml @@ -18,7 +18,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.7 +version: 0.4.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml index 243a4ba7d48..f8893a47afe 100644 --- a/deploy/charts/litellm-helm/templates/migrations-job.yaml +++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml @@ -22,6 +22,9 @@ spec: metadata: labels: {{- include "litellm.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} annotations: {{- with .Values.migrationJob.annotations }} {{- toYaml . | nindent 8 }} diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 351c4f6bc48..09b5265191b 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -12,7 +12,10 @@ WORKDIR /app USER root # Install build dependencies -RUN apk add --no-cache gcc python3-dev openssl openssl-dev +RUN apk add --no-cache \ + build-base \ + python3-dev \ + openssl-dev RUN pip install --upgrade pip && \ diff --git a/docs/my-website/blog/authors.yml b/docs/my-website/blog/authors.yml new file mode 100644 index 00000000000..2a49a736333 --- /dev/null +++ b/docs/my-website/blog/authors.yml @@ -0,0 +1,24 @@ +litellm: + name: LiteLLM Team + title: LiteLLM Core Team + url: https://github.com/BerriAI/litellm + image_url: https://github.com/BerriAI.png + +krrish: + name: Krrish Dholakia + title: CEO, LiteLLM + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + +ishaan: + name: Ishaan Jaffer + title: CTO, LiteLLM + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + +# Alias for typo in name +ishaan-alt: + name: Ishaan Jaff + title: CTO, LiteLLM + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg diff --git a/docs/my-website/blog/gemini_3/index.md b/docs/my-website/blog/gemini_3/index.md new file mode 100644 index 00000000000..2f4cc5bb0f8 --- /dev/null +++ b/docs/my-website/blog/gemini_3/index.md @@ -0,0 +1,700 @@ +--- +slug: gemini_3 +title: "DAY 0 Support: Gemini 3 on LiteLLM" +date: 2025-11-19T10:00:00 +authors: + - name: Sameer Kankute + title: "SWE @ LiteLLM (LLM Translation)" + url: https://in.linkedin.com/in/sameer-kankute + image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1765411200&v=beta&t=c8396f--_lH6Fb_pVvx_jGholPfcl0bvwmNynbNdnII + - name: Krrish Dholakia + title: "CEO, LiteLLM" + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +tags: [gemini, day 0 support, llms] +hide_table_of_contents: false +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +:::info + +This guide covers common questions and best practices for using `gemini-3-pro-preview` with LiteLLM Proxy and SDK. + +::: + +## Quick Start + + + + +```python +from litellm import completion +import os + +os.environ["GEMINI_API_KEY"] = "your-api-key" + +response = completion( + model="gemini/gemini-3-pro-preview", + messages=[{"role": "user", "content": "Hello!"}], + reasoning_effort="low" +) + +print(response.choices[0].message.content) +``` + + + + +**1. Add to config.yaml:** + +```yaml +model_list: + - model_name: gemini-3-pro-preview + litellm_params: + model: gemini/gemini-3-pro-preview + api_key: os.environ/GEMINI_API_KEY +``` + +**2. Start proxy:** + +```bash +litellm --config /path/to/config.yaml +``` + +**3. Make request:** + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gemini-3-pro-preview", + "messages": [{"role": "user", "content": "Hello!"}], + "reasoning_effort": "low" + }' +``` + + + + +## Supported Endpoints + +LiteLLM provides **full end-to-end support** for Gemini 3 Pro Preview on: + +- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint +- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint + +Both endpoints support: +- Streaming and non-streaming responses +- Function calling with thought signatures +- Multi-turn conversations +- All Gemini 3-specific features + +## Thought Signatures + +#### What are Thought Signatures? + +Thought signatures are encrypted representations of the model's internal reasoning process. They're essential for maintaining context across multi-turn conversations, especially with function calling. + +#### How Thought Signatures Work + +1. **Automatic Extraction**: When Gemini 3 returns a function call, LiteLLM automatically extracts the `thought_signature` from the response +2. **Storage**: Thought signatures are stored in `provider_specific_fields.thought_signature` of tool calls +3. **Automatic Preservation**: When you include the assistant's message in conversation history, LiteLLM automatically preserves and returns thought signatures to Gemini + +## Example: Multi-Turn Function Calling + +#### Streaming with Thought Signatures + +When using streaming mode with `stream_chunk_builder()`, thought signatures are now automatically preserved: + + + + +```python +import os +import litellm +from litellm import completion + +os.environ["GEMINI_API_KEY"] = "your-api-key" + +MODEL = "gemini/gemini-3-pro-preview" + +messages = [ + {"role": "system", "content": "You are a helpful assistant. Use the calculate tool."}, + {"role": "user", "content": "What is 2+2?"}, +] + +tools = [{ + "type": "function", + "function": { + "name": "calculate", + "description": "Calculate a mathematical expression", + "parameters": { + "type": "object", + "properties": {"expression": {"type": "string"}}, + "required": ["expression"], + }, + }, +}] + +print("Step 1: Sending request with stream=True...") +response = completion( + model=MODEL, + messages=messages, + stream=True, + tools=tools, + reasoning_effort="low" +) + +# Collect all chunks +chunks = [] +for part in response: + chunks.append(part) + +# Reconstruct message using stream_chunk_builder +# Thought signatures are now preserved automatically! +full_response = litellm.stream_chunk_builder(chunks, messages=messages) +print(f"Full response: {full_response}") + +assistant_msg = full_response.choices[0].message + +# ✅ Thought signature is now preserved in provider_specific_fields +if assistant_msg.tool_calls and assistant_msg.tool_calls[0].provider_specific_fields: + thought_sig = assistant_msg.tool_calls[0].provider_specific_fields.get("thought_signature") + print(f"Thought signature preserved: {thought_sig is not None}") + +# Append assistant message (includes thought signatures automatically) +messages.append(assistant_msg) + +# Mock tool execution +messages.append({ + "role": "tool", + "content": "4", + "tool_call_id": assistant_msg.tool_calls[0].id +}) + +print("\nStep 2: Sending tool result back to model...") +response_2 = completion( + model=MODEL, + messages=messages, + stream=True, + tools=tools, + reasoning_effort="low" +) + +for part in response_2: + if part.choices[0].delta.content: + print(part.choices[0].delta.content, end="") +print() # New line +``` + +**Key Points:** +- ✅ `stream_chunk_builder()` now preserves `provider_specific_fields` including thought signatures +- ✅ Thought signatures are automatically included when appending `assistant_msg` to conversation history +- ✅ Multi-turn conversations work seamlessly with streaming + + + + +```python +from openai import OpenAI +import json + +client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000") + +# Define tools +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + } + } +] + +# Step 1: Initial request +messages = [{"role": "user", "content": "What's the weather in Tokyo?"}] + +response = client.chat.completions.create( + model="gemini-3-pro-preview", + messages=messages, + tools=tools, + reasoning_effort="low" +) + +# Step 2: Append assistant message (thought signatures automatically preserved) +messages.append(response.choices[0].message) + +# Step 3: Execute tool and append result +for tool_call in response.choices[0].message.tool_calls: + if tool_call.function.name == "get_weather": + result = {"temperature": 30, "unit": "celsius"} + messages.append({ + "role": "tool", + "content": json.dumps(result), + "tool_call_id": tool_call.id + }) + +# Step 4: Follow-up request (thought signatures automatically included) +response2 = client.chat.completions.create( + model="gemini-3-pro-preview", + messages=messages, + tools=tools, + reasoning_effort="low" +) + +print(response2.choices[0].message.content) +``` + +**Key Points:** +- ✅ Thought signatures are automatically extracted from `response.choices[0].message.tool_calls[].provider_specific_fields.thought_signature` +- ✅ When you append `response.choices[0].message` to your conversation history, thought signatures are automatically preserved +- ✅ You don't need to manually extract or manage thought signatures + + + + +```bash +# Step 1: Initial request +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gemini-3-pro-preview", + "messages": [ + {"role": "user", "content": "What'\''s the weather in Tokyo?"} + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + } + } + ], + "reasoning_effort": "low" + }' +``` + +**Response includes thought signature:** + +```json +{ + "choices": [{ + "message": { + "role": "assistant", + "tool_calls": [{ + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"location\": \"Tokyo\"}" + }, + "provider_specific_fields": { + "thought_signature": "CpcHAdHtim9+q4rstcbvQC0ic4x1/vqQlCJWgE+UZ6dTLYGHMMBkF/AxqL5UmP6SY46uYC8t4BTFiXG5zkw6EMJ..." + } + }] + } + }] +} +``` + +```bash +# Step 2: Follow-up request (include assistant message with thought signature) +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gemini-3-pro-preview", + "messages": [ + {"role": "user", "content": "What'\''s the weather in Tokyo?"}, + { + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"location\": \"Tokyo\"}" + }, + "provider_specific_fields": { + "thought_signature": "CpcHAdHtim9+q4rstcbvQC0ic4x1/vqQlCJWgE+UZ6dTLYGHMMBkF/AxqL5UmP6SY46uYC8t4BTFiXG5zkw6EMJ..." + } + }] + }, + { + "role": "tool", + "content": "{\"temperature\": 30, \"unit\": \"celsius\"}", + "tool_call_id": "call_abc123" + } + ], + "tools": [...], + "reasoning_effort": "low" + }' +``` + + + + +#### Important Notes on Thought Signatures + +1. **Automatic Handling**: LiteLLM automatically extracts and preserves thought signatures. You don't need to manually manage them. + +2. **Parallel Function Calls**: When the model makes parallel function calls, only the **first function call** has a thought signature. + +3. **Sequential Function Calls**: In multi-step function calling, each step's first function call has its own thought signature that must be preserved. + +4. **Required for Context**: Thought signatures are essential for maintaining reasoning context. Without them, the model may lose context of its previous reasoning. + +## Conversation History: Switching from Non-Gemini-3 Models + +#### Common Question: Will switching from a non-Gemini-3 model to Gemini-3 break conversation history? + +**Answer: No!** LiteLLM automatically handles this by adding dummy thought signatures when needed. + +#### How It Works + +When you switch from a model that doesn't use thought signatures (e.g., `gemini-2.5-flash`) to Gemini 3, LiteLLM: + +1. **Detects missing signatures**: Identifies assistant messages with tool calls that lack thought signatures +2. **Adds dummy signature**: Automatically injects a dummy thought signature (`skip_thought_signature_validator`) for compatibility +3. **Maintains conversation flow**: Your conversation history continues to work seamlessly + +#### Example: Switching Models Mid-Conversation + + + + +```python +from openai import OpenAI + +client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000") + +# Step 1: Start with gemini-2.5-flash (no thought signatures) +messages = [{"role": "user", "content": "What's the weather?"}] + +response1 = client.chat.completions.create( + model="gemini-2.5-flash", + messages=messages, + tools=[...], + reasoning_effort="low" +) + +# Append assistant message (no tool call thought signature from gemini-2.5-flash) +messages.append(response1.choices[0].message) + +# Step 2: Switch to gemini-3-pro-preview +# LiteLLM automatically adds dummy thought signature to the previous assistant message +response2 = client.chat.completions.create( + model="gemini-3-pro-preview", # 👈 Switched model + messages=messages, # 👈 Same conversation history + tools=[...], + reasoning_effort="low" +) + +# ✅ Works seamlessly! No errors, no breaking changes +print(response2.choices[0].message.content) +``` + + + + +```bash +# Step 1: Start with gemini-2.5-flash +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gemini-2.5-flash", + "messages": [{"role": "user", "content": "What'\''s the weather?"}], + "tools": [...], + "reasoning_effort": "low" + }' + +# Step 2: Switch to gemini-3-pro-preview with same conversation history +# LiteLLM automatically handles the missing thought signature +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gemini-3-pro-preview", # 👈 Switched model + "messages": [ + {"role": "user", "content": "What'\''s the weather?"}, + { + "role": "assistant", + "tool_calls": [...] # 👈 No thought_signature from gemini-2.5-flash + } + ], + "tools": [...], + "reasoning_effort": "low" + }' +# ✅ Works! LiteLLM adds dummy signature automatically +``` + + + + +#### Dummy Signature Details + +The dummy signature used is: `base64("skip_thought_signature_validator")` + +This is the recommended approach by Google for handling conversation history from models that don't support thought signatures. It allows Gemini 3 to: +- Accept the conversation history without validation errors +- Continue the conversation seamlessly +- Maintain context across model switches + +## Thinking Level Parameter + +#### How `reasoning_effort` Maps to `thinking_level` + +For Gemini 3 Pro Preview, LiteLLM automatically maps `reasoning_effort` to the new `thinking_level` parameter: + +| `reasoning_effort` | `thinking_level` | Notes | +|-------------------|------------------|-------| +| `"minimal"` | `"low"` | Maps to low thinking level | +| `"low"` | `"low"` | Default for most use cases | +| `"medium"` | `"high"` | Medium not available yet, maps to high | +| `"high"` | `"high"` | Maximum reasoning depth | +| `"disable"` | `"low"` | Gemini 3 cannot fully disable thinking | +| `"none"` | `"low"` | Gemini 3 cannot fully disable thinking | + +#### Default Behavior + +If you don't specify `reasoning_effort`, LiteLLM automatically sets `thinking_level="low"` for Gemini 3 models, to avoid high costs. + +### Example Usage + + + + +```python +from litellm import completion + +# Low thinking level (faster, lower cost) +response = completion( + model="gemini/gemini-3-pro-preview", + messages=[{"role": "user", "content": "What's the weather?"}], + reasoning_effort="low" # Maps to thinking_level="low" +) + +# High thinking level (deeper reasoning, higher cost) +response = completion( + model="gemini/gemini-3-pro-preview", + messages=[{"role": "user", "content": "Solve this complex math problem step by step."}], + reasoning_effort="high" # Maps to thinking_level="high" +) +``` + + + + +```bash +# Low thinking level +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gemini-3-pro-preview", + "messages": [{"role": "user", "content": "What'\''s the weather?"}], + "reasoning_effort": "low" + }' + +# High thinking level +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gemini-3-pro-preview", + "messages": [{"role": "user", "content": "Solve this complex problem."}], + "reasoning_effort": "high" + }' +``` + + + + +## Important Notes + +1. **Gemini 3 Cannot Disable Thinking**: Unlike Gemini 2.5 models, Gemini 3 cannot fully disable thinking. Even when you set `reasoning_effort="none"` or `"disable"`, it maps to `thinking_level="low"`. + +2. **Temperature Recommendation**: For Gemini 3 models, LiteLLM defaults `temperature` to `1.0` and strongly recommends keeping it at this default. Setting `temperature < 1.0` can cause: + - Infinite loops + - Degraded reasoning performance + - Failure on complex tasks + +3. **Automatic Defaults**: If you don't specify `reasoning_effort`, LiteLLM automatically sets `thinking_level="low"` for optimal performance. + +## Using with Claude Code CLI + +You can use `gemini-3-pro-preview` with **Claude Code CLI** - Anthropic's command-line interface. This allows you to use Gemini 3 Pro Preview with Claude Code's native syntax and workflows. + +### Setup + +**1. Add Gemini 3 Pro Preview to your `config.yaml`:** + +```yaml +model_list: + - model_name: gemini-3-pro-preview + litellm_params: + model: gemini/gemini-3-pro-preview + api_key: os.environ/GEMINI_API_KEY + +litellm_settings: + master_key: os.environ/LITELLM_MASTER_KEY +``` + +**2. Set environment variables:** + +```bash +export GEMINI_API_KEY="your-gemini-api-key" +export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key +``` + +**3. Start LiteLLM Proxy:** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +**4. Configure Claude Code to use LiteLLM Proxy:** + +```bash +export ANTHROPIC_BASE_URL="http://0.0.0.0:4000" +export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY" +``` + +**5. Use Gemini 3 Pro Preview with Claude Code:** + +```bash +# Claude Code will use gemini-3-pro-preview from your LiteLLM proxy +claude --model gemini-3-pro-preview + +``` + +### Example Usage + +Once configured, you can interact with Gemini 3 Pro Preview using Claude Code's native interface: + +```bash +$ claude --model gemini-3-pro-preview +> Explain how thought signatures work in multi-turn conversations. + +# Gemini 3 Pro Preview responds through Claude Code interface +``` + +### Benefits + +- ✅ **Native Claude Code Experience**: Use Gemini 3 Pro Preview with Claude Code's familiar CLI interface +- ✅ **Unified Authentication**: Single API key for all models through LiteLLM proxy +- ✅ **Cost Tracking**: All usage tracked through LiteLLM's centralized logging +- ✅ **Seamless Model Switching**: Easily switch between Claude and Gemini models +- ✅ **Full Feature Support**: All Gemini 3 features (thought signatures, function calling, etc.) work through Claude Code + +### Troubleshooting + +**Claude Code not finding the model:** +- Ensure the model name in Claude Code matches exactly: `gemini-3-pro-preview` +- Verify your proxy is running: `curl http://0.0.0.0:4000/health` +- Check that `ANTHROPIC_BASE_URL` points to your LiteLLM proxy + +**Authentication errors:** +- Verify `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key +- Ensure `GEMINI_API_KEY` is set correctly +- Check LiteLLM proxy logs for detailed error messages + +## Best Practices + +#### 1. Always Include Thought Signatures in Conversation History + +When building multi-turn conversations with function calling: + +✅ **Do:** +```python +# Append the full assistant message (includes thought signatures) +messages.append(response.choices[0].message) +``` + +❌ **Don't:** +```python +# Don't manually construct assistant messages without thought signatures +messages.append({ + "role": "assistant", + "tool_calls": [...] # Missing thought signatures! +}) +``` + +#### 2. Use Appropriate Thinking Levels + +- **`reasoning_effort="low"`**: For simple queries, quick responses, cost optimization +- **`reasoning_effort="high"`**: For complex problems requiring deep reasoning + +#### 3. Keep Temperature at Default + +For Gemini 3 models, always use `temperature=1.0` (default). Lower temperatures can cause issues. + +#### 4. Handle Model Switches Gracefully + +When switching from non-Gemini-3 to Gemini-3: +- ✅ LiteLLM automatically handles missing thought signatures +- ✅ No manual intervention needed +- ✅ Conversation history continues seamlessly + +## Troubleshooting + +#### Issue: Missing Thought Signatures + +**Symptom**: Error when including assistant messages in conversation history + +**Solution**: Ensure you're appending the full assistant message from the response: +```python +messages.append(response.choices[0].message) # ✅ Includes thought signatures +``` + +#### Issue: Conversation Breaks When Switching Models + +**Symptom**: Errors when switching from gemini-2.5-flash to gemini-3-pro-preview + +**Solution**: This should work automatically! LiteLLM adds dummy signatures. If you see errors, ensure you're using the latest LiteLLM version. + +#### Issue: Infinite Loops or Poor Performance + +**Symptom**: Model gets stuck or produces poor results + +**Solution**: +- Ensure `temperature=1.0` (default for Gemini 3) +- Check that `reasoning_effort` is set appropriately +- Verify you're using the correct model name: `gemini/gemini-3-pro-preview` + +## Additional Resources + +- [Gemini Provider Documentation](../gemini.md) +- [Thought Signatures Guide](../gemini.md#thought-signatures) +- [Reasoning Content Documentation](../../reasoning_content.md) +- [Function Calling Guide](../../function_calling.md) + diff --git a/docs/my-website/docs/image_edits.md b/docs/my-website/docs/image_edits.md index 9a53da510f7..5a108aabf3a 100644 --- a/docs/my-website/docs/image_edits.md +++ b/docs/my-website/docs/image_edits.md @@ -16,7 +16,7 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit | Supported operations | Create image edits | Single and multiple images supported | | Supported LiteLLM SDK Versions | 1.63.8+ | Gemini support requires 1.79.3+ | | Supported LiteLLM Proxy Versions | 1.71.1+ | Gemini support requires 1.79.3+ | -| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)** | Gemini supports the new `gemini-2.5-flash-image` family | +| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. | #### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/) @@ -197,6 +197,53 @@ for idx, image_obj in enumerate(response.data): f.write(base64.b64decode(image_obj.b64_json)) ``` + + + + +#### Basic Image Edit (Gemini) +```python showLineNumbers title="Vertex AI Gemini Image Edit" +import os +import litellm + +# Set Vertex AI credentials +os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id" +os.environ["VERTEXAI_LOCATION"] = "us-central1" +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/path/to/service-account.json" + +response = litellm.image_edit( + model="vertex_ai/gemini-2.5-flash", + image=open("original_image.png", "rb"), + prompt="Add neon lights in the background", + size="1024x1024", +) + +print(response) +``` + +#### Image Edit with Imagen (Supports Masks) +```python showLineNumbers title="Vertex AI Imagen Image Edit" +import os +import litellm + +# Set Vertex AI credentials +os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id" +os.environ["VERTEXAI_LOCATION"] = "us-central1" +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/path/to/service-account.json" + +# Imagen supports mask for inpainting +response = litellm.image_edit( + model="vertex_ai/imagen-3.0-capability-001", + image=open("original_image.png", "rb"), + mask=open("mask_image.png", "rb"), # Optional: for inpainting + prompt="Turn this into watercolor style scenery", + n=2, # Number of variations + size="1024x1024", +) + +print(response) +``` + @@ -302,6 +349,55 @@ curl -X POST "http://0.0.0.0:4000/v1/images/edits" \ -F "size=1024x1024" ``` + + + + +1. Add Vertex AI image edit models to your `config.yaml`: +```yaml showLineNumbers title="Vertex AI Proxy Configuration" +model_list: + - model_name: vertex-gemini-image-edit + litellm_params: + model: vertex_ai/gemini-2.5-flash + vertex_project: os.environ/VERTEXAI_PROJECT + vertex_location: os.environ/VERTEXAI_LOCATION + vertex_credentials: os.environ/GOOGLE_APPLICATION_CREDENTIALS + + - model_name: vertex-imagen-image-edit + litellm_params: + model: vertex_ai/imagen-3.0-capability-001 + vertex_project: os.environ/VERTEXAI_PROJECT + vertex_location: os.environ/VERTEXAI_LOCATION + vertex_credentials: os.environ/GOOGLE_APPLICATION_CREDENTIALS +``` + +2. Start the LiteLLM proxy server: +```bash showLineNumbers title="Start LiteLLM Proxy Server" +litellm --config /path/to/config.yaml +``` + +3. Make an image edit request: +```bash showLineNumbers title="Vertex AI Gemini Proxy Image Edit" +curl -X POST "http://0.0.0.0:4000/v1/images/edits" \ + -H "Authorization: Bearer " \ + -F "model=vertex-gemini-image-edit" \ + -F "image=@original_image.png" \ + -F "prompt=Add neon lights in the background" \ + -F "size=1024x1024" +``` + +4. Imagen image edit with mask: +```bash showLineNumbers title="Vertex AI Imagen Proxy Image Edit with Mask" +curl -X POST "http://0.0.0.0:4000/v1/images/edits" \ + -H "Authorization: Bearer " \ + -F "model=vertex-imagen-image-edit" \ + -F "image=@original_image.png" \ + -F "mask=@mask_image.png" \ + -F "prompt=Turn this into watercolor style scenery" \ + -F "n=2" \ + -F "size=1024x1024" +``` + diff --git a/docs/my-website/docs/mcp.md b/docs/my-website/docs/mcp.md index 408560e5c0a..9a1e25a516c 100644 --- a/docs/my-website/docs/mcp.md +++ b/docs/my-website/docs/mcp.md @@ -657,7 +657,7 @@ LiteLLM Proxy provides two methods for controlling access to specific MCP server ### Method 1: URL-based Namespacing -LiteLLM Proxy supports URL-based namespacing for MCP servers using the format `/mcp/`. This allows you to: +LiteLLM Proxy supports URL-based namespacing for MCP servers using the format `//mcp`. This allows you to: - **Direct URL Access**: Point MCP clients directly to specific servers or access groups via URL - **Simplified Configuration**: Use URLs instead of headers for server selection @@ -666,14 +666,14 @@ LiteLLM Proxy supports URL-based namespacing for MCP servers using the format `/ #### URL Format ``` -/mcp/ +//mcp ``` **Examples:** -- `/mcp/github` - Access tools from the "github" MCP server -- `/mcp/zapier` - Access tools from the "zapier" MCP server -- `/mcp/dev_group` - Access tools from all servers in the "dev_group" access group -- `/mcp/github,zapier` - Access tools from multiple specific servers +- `/github_mcp/mcp` - Access tools from the "github_mcp" MCP server +- `/zapier/mcp` - Access tools from the "zapier" MCP server +- `/dev_group/mcp` - Access tools from all servers in the "dev_group" access group +- `/github_mcp,zapier/mcp` - Access tools from multiple specific servers #### Usage Examples @@ -690,7 +690,7 @@ curl --location 'https://api.openai.com/v1/responses' \ { "type": "mcp", "server_label": "litellm", - "server_url": "/mcp/github", + "server_url": "/github_mcp/mcp", "require_approval": "never", "headers": { "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY" @@ -718,7 +718,7 @@ curl --location '/v1/responses' \ { "type": "mcp", "server_label": "litellm", - "server_url": "/mcp/dev_group", + "server_url": "/dev_group/mcp", "require_approval": "never", "headers": { "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY" @@ -740,7 +740,7 @@ This example uses URL namespacing to access all servers in the "dev_group" acces { "mcpServers": { "LiteLLM": { - "url": "/mcp/github,zapier", + "url": "/github_mcp,zapier/mcp", "headers": { "x-litellm-api-key": "Bearer $LITELLM_API_KEY" } @@ -862,8 +862,8 @@ This configuration in Cursor IDE settings will limit tool access to only the spe | Feature | Header Namespacing | URL Namespacing | |---------|-------------------|-----------------| -| **Method** | Uses `x-mcp-servers` header | Uses URL path `/mcp/` | -| **Endpoint** | Standard `litellm_proxy` endpoint | Custom `/mcp/` endpoint | +| **Method** | Uses `x-mcp-servers` header | Uses URL path `//mcp` | +| **Endpoint** | Standard `litellm_proxy` endpoint | Custom `//mcp` endpoint | | **Configuration** | Requires additional header | Self-contained in URL | | **Multiple Servers** | Comma-separated in header | Comma-separated in URL path | | **Access Groups** | Supported via header | Supported via URL path | diff --git a/docs/my-website/docs/providers/docker_model_runner.md b/docs/my-website/docs/providers/docker_model_runner.md new file mode 100644 index 00000000000..fcd4c74f8f4 --- /dev/null +++ b/docs/my-website/docs/providers/docker_model_runner.md @@ -0,0 +1,277 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Docker Model Runner + +## Overview + +| Property | Details | +|-------|-------| +| Description | Docker Model Runner allows you to run large language models locally using Docker Desktop. | +| Provider Route on LiteLLM | `docker_model_runner/` | +| Link to Provider Doc | [Docker Model Runner ↗](https://docs.docker.com/ai/model-runner/) | +| Base URL | `http://localhost:22088` | +| Supported Operations | [`/chat/completions`](#sample-usage) | + +
+
+ +https://docs.docker.com/ai/model-runner/ + +**We support ALL Docker Model Runner models, just set `docker_model_runner/` as a prefix when sending completion requests** + +## Quick Start + +Docker Model Runner is a Docker Desktop feature that lets you run AI models locally. It provides better performance than other local solutions while maintaining OpenAI compatibility. + +### Installation + +1. Install [Docker Desktop](https://www.docker.com/products/docker-desktop/) +2. Enable Docker Model Runner in Docker Desktop settings +3. Download your preferred model through Docker Desktop + +## Environment Variables + +```python showLineNumbers title="Environment Variables" +os.environ["DOCKER_MODEL_RUNNER_API_BASE"] = "http://localhost:22088/engines/llama.cpp" # Optional - defaults to this +os.environ["DOCKER_MODEL_RUNNER_API_KEY"] = "dummy-key" # Optional - Docker Model Runner may not require auth for local instances +``` + +**Note:** +- Docker Model Runner typically runs locally and may not require authentication. LiteLLM will use a dummy key by default if no key is provided. +- The API base should include the engine path (e.g., `/engines/llama.cpp`) + +## API Base Structure + +Docker Model Runner uses a unique URL structure: + +``` +http://model-runner.docker.internal/engines/{engine}/v1/chat/completions +``` + +Where `{engine}` is the engine you want to use (typically `llama.cpp`). + +**Important:** Specify the engine in your `api_base` URL, not in the model name: +- ✅ Correct: `api_base="http://localhost:22088/engines/llama.cpp"`, `model="docker_model_runner/llama-3.1"` +- ❌ Incorrect: `api_base="http://localhost:22088"`, `model="docker_model_runner/llama.cpp/llama-3.1"` + +## Usage - LiteLLM Python SDK + +### Non-streaming + +```python showLineNumbers title="Docker Model Runner Non-streaming Completion" +import os +import litellm +from litellm import completion + +# Specify the engine in the api_base URL +os.environ["DOCKER_MODEL_RUNNER_API_BASE"] = "http://localhost:22088/engines/llama.cpp" + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Docker Model Runner call +response = completion( + model="docker_model_runner/llama-3.1", + messages=messages +) + +print(response) +``` + +### Streaming + +```python showLineNumbers title="Docker Model Runner Streaming Completion" +import os +import litellm +from litellm import completion + +# Specify the engine in the api_base URL +os.environ["DOCKER_MODEL_RUNNER_API_BASE"] = "http://localhost:22088/engines/llama.cpp" + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Docker Model Runner call with streaming +response = completion( + model="docker_model_runner/llama-3.1", + messages=messages, + stream=True +) + +for chunk in response: + print(chunk) +``` + +### Custom API Base and Engine + +```python showLineNumbers title="Custom API Base with Different Engine" +import litellm +from litellm import completion + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Specify the engine in the api_base URL +# Using a different host and engine +response = completion( + model="docker_model_runner/llama-3.1", + messages=messages, + api_base="http://model-runner.docker.internal/engines/llama.cpp" +) + +print(response) +``` + +### Using Different Engines + +```python showLineNumbers title="Using a Different Engine" +import litellm +from litellm import completion + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# To use a different engine, specify it in the api_base +# For example, if Docker Model Runner supports other engines: +response = completion( + model="docker_model_runner/mistral-7b", + messages=messages, + api_base="http://localhost:22088/engines/custom-engine" +) + +print(response) +``` + +## Usage - LiteLLM Proxy + +Add the following to your LiteLLM Proxy configuration file: + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: llama-3.1 + litellm_params: + model: docker_model_runner/llama-3.1 + api_base: http://localhost:22088/engines/llama.cpp + + - model_name: mistral-7b + litellm_params: + model: docker_model_runner/mistral-7b + api_base: http://localhost:22088/engines/llama.cpp +``` + +Start your LiteLLM Proxy server: + +```bash showLineNumbers title="Start LiteLLM Proxy" +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + + + + +```python showLineNumbers title="Docker Model Runner via Proxy - Non-streaming" +from openai import OpenAI + +# Initialize client with your proxy URL +client = OpenAI( + base_url="http://localhost:4000", # Your proxy URL + api_key="your-proxy-api-key" # Your proxy API key +) + +# Non-streaming response +response = client.chat.completions.create( + model="llama-3.1", + messages=[{"role": "user", "content": "hello from litellm"}] +) + +print(response.choices[0].message.content) +``` + +```python showLineNumbers title="Docker Model Runner via Proxy - Streaming" +from openai import OpenAI + +# Initialize client with your proxy URL +client = OpenAI( + base_url="http://localhost:4000", # Your proxy URL + api_key="your-proxy-api-key" # Your proxy API key +) + +# Streaming response +response = client.chat.completions.create( + model="llama-3.1", + messages=[{"role": "user", "content": "hello from litellm"}], + stream=True +) + +for chunk in response: + if chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="") +``` + + + + + +```python showLineNumbers title="Docker Model Runner via Proxy - LiteLLM SDK" +import litellm + +# Configure LiteLLM to use your proxy +response = litellm.completion( + model="litellm_proxy/llama-3.1", + messages=[{"role": "user", "content": "hello from litellm"}], + api_base="http://localhost:4000", + api_key="your-proxy-api-key" +) + +print(response.choices[0].message.content) +``` + +```python showLineNumbers title="Docker Model Runner via Proxy - LiteLLM SDK Streaming" +import litellm + +# Configure LiteLLM to use your proxy with streaming +response = litellm.completion( + model="litellm_proxy/llama-3.1", + messages=[{"role": "user", "content": "hello from litellm"}], + api_base="http://localhost:4000", + api_key="your-proxy-api-key", + stream=True +) + +for chunk in response: + if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="") +``` + + + + + +```bash showLineNumbers title="Docker Model Runner via Proxy - cURL" +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "llama-3.1", + "messages": [{"role": "user", "content": "hello from litellm"}] + }' +``` + +```bash showLineNumbers title="Docker Model Runner via Proxy - cURL Streaming" +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "llama-3.1", + "messages": [{"role": "user", "content": "hello from litellm"}], + "stream": true + }' +``` + + + + +For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy). + +## API Reference + +For detailed API information, see the [Docker Model Runner API Reference](https://docs.docker.com/ai/model-runner/api-reference/). + diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md index fd20e907d3b..e04225e1f85 100644 --- a/docs/my-website/docs/providers/gemini.md +++ b/docs/my-website/docs/providers/gemini.md @@ -1308,6 +1308,8 @@ curl --location 'http://localhost:4000/v1/chat/completions' \ 5. **Format**: Thought signatures are stored in `provider_specific_fields.thought_signature` of tool calls in the response, and are automatically included when you append the assistant message to your conversation history. +6. **Chat Completions Clients**: With chat completions clients where you cannot control whether or not the previous assistant message is included as-is (ex langchain's ChatOpenAI), LiteLLM also preserves the thought signature by appending it to the tool call id (`call_123__thought__`) and extracting it back out before sending the outbound request to Gemini. + ## JSON Mode diff --git a/docs/my-website/docs/providers/xai.md b/docs/my-website/docs/providers/xai.md index 49a3640991d..afeecc21528 100644 --- a/docs/my-website/docs/providers/xai.md +++ b/docs/my-website/docs/providers/xai.md @@ -11,6 +11,68 @@ https://docs.x.ai/docs ::: +## Supported Models + + + +**Latest Release** - Grok 4.1 Fast: Optimized for high-performance agentic tool calling with 2M context and prompt caching. + +| Model | Context | Features | +|-------|---------|----------| +| `xai/grok-4-1-fast-reasoning` | 2M tokens | **Reasoning**, Function calling, Vision, Audio, Web search, Caching | +| `xai/grok-4-1-fast-non-reasoning` | 2M tokens | Function calling, Vision, Audio, Web search, Caching | + +**When to use:** +- ✅ **Reasoning model**: Complex analysis, planning, multi-step reasoning problems +- ✅ **Non-reasoning model**: Simple queries, faster responses, lower token usage + +**Example:** +```python +from litellm import completion + +# With reasoning +response = completion( + model="xai/grok-4-1-fast-reasoning", + messages=[{"role": "user", "content": "Analyze this problem step by step..."}] +) + +# Without reasoning +response = completion( + model="xai/grok-4-1-fast-non-reasoning", + messages=[{"role": "user", "content": "What's 2+2?"}] +) +``` + +--- + +### All Available Models + +| Model Family | Model | Context | Features | +|--------------|-------|---------|----------| +| **Grok 4.1** | `xai/grok-4-1-fast-reasoning` | 2M | **Reasoning**, Tools, Vision, Audio, Web search, Caching | +| | `xai/grok-4-1-fast-non-reasoning` | 2M | Tools, Vision, Audio, Web search, Caching | +| **Grok 4** | `xai/grok-4` | 256K | Tools, Web search | +| | `xai/grok-4-0709` | 256K | Tools, Web search | +| | `xai/grok-4-fast-reasoning` | 2M | **Reasoning**, Tools, Web search | +| | `xai/grok-4-fast-non-reasoning` | 2M | Tools, Web search | +| **Grok 3** | `xai/grok-3` | 131K | Tools, Web search | +| | `xai/grok-3-mini` | 131K | Tools, Web search | +| | `xai/grok-3-fast-beta` | 131K | Tools, Web search | +| **Grok Code** | `xai/grok-code-fast` | 256K | **Reasoning**, Tools, Code generation, Caching | +| **Grok 2** | `xai/grok-2` | 131K | Tools, **Vision** | +| | `xai/grok-2-vision-latest` | 32K | Tools, **Vision** | + +**Features:** +- **Reasoning** = Chain-of-thought reasoning with reasoning tokens +- **Tools** = Function calling / Tool use +- **Web search** = Live internet search +- **Vision** = Image understanding +- **Audio** = Audio input support +- **Caching** = Prompt caching for cost savings +- **Code generation** = Optimized for code tasks + +**Pricing:** See [xAI's pricing page](https://docs.x.ai/docs/models) for current rates. + ## API Key ```python # env variable diff --git a/docs/my-website/docs/proxy/admin_ui_sso.md b/docs/my-website/docs/proxy/admin_ui_sso.md index ae082848b6b..0438c264685 100644 --- a/docs/my-website/docs/proxy/admin_ui_sso.md +++ b/docs/my-website/docs/proxy/admin_ui_sso.md @@ -380,3 +380,54 @@ If you need to inspect the JWT fields received from your SSO provider by LiteLLM Once redirected, you should see a page called "SSO Debug Information". This page displays the JWT fields received from your SSO provider (as shown in the image above) + +## Advanced + +### Manage User Roles via Azure App Roles + +Centralize role management by defining user permissions in Azure Entra ID. LiteLLM will automatically assign roles based on your Azure configuration when users sign in—no need to manually manage roles in LiteLLM. + +#### Step 1: Create App Roles on Azure App Registration + +1. Navigate to your App Registration on https://portal.azure.com/ +2. Go to **App roles** > **Create app role** +3. Configure the app role using one of the [supported LiteLLM roles](./access_control.md#global-proxy-roles): + - **Display name**: Admin Viewer (or your preferred display name) + - **Value**: `proxy_admin_viewer` (must match one of the LiteLLM role values exactly) +4. Click **Apply** to save the role +5. Repeat for each LiteLLM role you want to use + + +**Supported LiteLLM role values** (see [full role documentation](./access_control.md#global-proxy-roles)): +- `proxy_admin` - Full admin access +- `proxy_admin_viewer` - Read-only admin access +- `internal_user` - Can create/view/delete own keys +- `internal_user_viewer` - Can view own keys (read-only) + + + +--- + +#### Step 2: Assign Users to App Roles + +1. Navigate to **Enterprise Applications** on https://portal.azure.com/ +2. Select your LiteLLM application +3. Go to **Users and groups** > **Add user/group** +4. Select the user +5. Under **Select a role**, choose the app role you created (e.g., `proxy_admin_viewer`) +6. Click **Assign** to save + + + +--- + +#### Step 3: Sign in and verify + +1. Sign in to the LiteLLM UI via SSO +2. LiteLLM will automatically extract the app role from the JWT token +3. The user will be assigned the corresponding role (you can verify this in the UI by checking the user profile dropdown) + + + +**Note:** The role from Entra ID will take precedence over any existing role in the LiteLLM database. This ensures your SSO provider is the authoritative source for user roles. + diff --git a/docs/my-website/docs/proxy/guardrails/ibm_guardrails.md b/docs/my-website/docs/proxy/guardrails/ibm_guardrails.md index 0c13d2dcea9..43ba6622078 100644 --- a/docs/my-website/docs/proxy/guardrails/ibm_guardrails.md +++ b/docs/my-website/docs/proxy/guardrails/ibm_guardrails.md @@ -95,6 +95,7 @@ curl -i http://localhost:4000/v1/chat/completions \ These go under `optional_params`: - `detector_params` - dict - Parameters to pass to your detector +- `extra_headers` - dict - Additional headers to inject into requests to IBM Guardrails, as a key-value dict. - `score_threshold` - float - Only count detections above this score (0.0 to 1.0) - `block_on_detection` - bool - Block the request when violations found. Default: `true` diff --git a/docs/my-website/docs/proxy/guardrails/tool_permission.md b/docs/my-website/docs/proxy/guardrails/tool_permission.md index 9ed05ed46a8..22ecdd2251e 100644 --- a/docs/my-website/docs/proxy/guardrails/tool_permission.md +++ b/docs/my-website/docs/proxy/guardrails/tool_permission.md @@ -46,6 +46,43 @@ guardrails: - `pre_call` Run **before** LLM call, on **input** - `post_call` Run **after** LLM call, on **input & output** +### `on_disallowed_action` behavior + +| Value | What happens | +| --- | --- | +| `block` | The request is immediately rejected. Pre-call checks raise a `400` HTTP error. Post-call checks raise `GuardrailRaisedException`, so the proxy responds with an error instead of the model output. Use when invoking the forbidden tool must halt the workflow. | +| `rewrite` | LiteLLM silently strips disallowed tools from the payload before it reaches the model (pre-call) or rewrites the model response/tool calls after the fact. The guardrail inserts error text into `message.content`/`tool_result` entries so the client learns the tool was blocked while the rest of the completion continues. Use when you want graceful degradation instead of hard failures. | + +### Custom denial message + +Set `violation_message_template` when you want the guardrail to return a branded error (e.g., “this violates our org policy…”). LiteLLM replaces placeholders from the denied tool: + +- `{tool_name}` – the tool/function name (e.g., `Read`) +- `{rule_id}` – the matching rule ID (or `None` when the default action kicks in) +- `{default_message}` – the original LiteLLM message if you need to append it + +Example: + +```yaml +guardrails: + - guardrail_name: "tool-permission-guardrail" + litellm_params: + guardrail: tool_permission + mode: "post_call" + violation_message_template: "this violates our org policy, we don't support executing {tool_name} commands" + rules: + - id: "allow_bash" + tool_name: "Bash" + decision: "allow" + - id: "deny_read" + tool_name: "Read" + decision: "deny" + default_action: "deny" + on_disallowed_action: "block" +``` + +If a request tries to invoke `Read`, the proxy now returns “this violates our org policy, we don't support executing Read commands” instead of the stock error text. Omit the field to keep the default messaging. + ### 2. Start the Proxy ```shell @@ -57,7 +94,7 @@ litellm --config config.yaml --port 4000 -**Block requset** +**Block request (`on_disallowed_action: block`)** ```bash # Test @@ -96,7 +133,7 @@ curl -X POST "http://localhost:4000/v1/chat/completions" \ -**Rewrite requset** +**Rewrite request (`on_disallowed_action: rewrite`)** ```bash # Test @@ -118,7 +155,7 @@ curl -X POST "http://localhost:4000/v1/chat/completions" \ }' ``` -**Expected response:** +**Expected response (tool removed, completion continues):** ```json { diff --git a/docs/my-website/docs/proxy/litellm_managed_files.md b/docs/my-website/docs/proxy/litellm_managed_files.md index c63b1218892..7aba173f35b 100644 --- a/docs/my-website/docs/proxy/litellm_managed_files.md +++ b/docs/my-website/docs/proxy/litellm_managed_files.md @@ -424,4 +424,4 @@ No, as of `v1.71.2` users can only view/edit/delete files they have created. ## See Also - [Managed Files w/ Finetuning APIs](../../docs/proxy/managed_finetuning) -- [Managed Files w/ Batch APIs](../../docs/proxy/managed_batch) \ No newline at end of file +- [Managed Files w/ Batch APIs](../../docs/proxy/managed_batches) \ No newline at end of file diff --git a/docs/my-website/docs/secret_managers/aws_secret_manager.md b/docs/my-website/docs/secret_managers/aws_secret_manager.md index 44fa23a4ae5..5b7ab1e3e7b 100644 --- a/docs/my-website/docs/secret_managers/aws_secret_manager.md +++ b/docs/my-website/docs/secret_managers/aws_secret_manager.md @@ -110,3 +110,57 @@ The `primary_secret_name` allows you to read multiple keys from a single AWS Sec This reduces the number of AWS Secrets you need to manage. +## IAM Role Assumption + +Use IAM roles instead of static AWS credentials for better security. + +### Basic IAM Role + +```yaml +general_settings: + key_management_system: "aws_secret_manager" + key_management_settings: + store_virtual_keys: true + aws_region_name: "us-east-1" + aws_role_name: "arn:aws:iam::123456789012:role/LiteLLMSecretManagerRole" + aws_session_name: "litellm-session" +``` + +### Cross-Account Access + +```yaml +general_settings: + key_management_system: "aws_secret_manager" + key_management_settings: + store_virtual_keys: true + aws_region_name: "us-east-1" + aws_role_name: "arn:aws:iam::999999999999:role/CrossAccountRole" + aws_external_id: "unique-external-id" +``` + +### EKS with IRSA + +```yaml +general_settings: + key_management_system: "aws_secret_manager" + key_management_settings: + store_virtual_keys: true + aws_region_name: "us-east-1" + aws_role_name: "arn:aws:iam::123456789012:role/LiteLLMServiceAccountRole" + aws_web_identity_token: "os.environ/AWS_WEB_IDENTITY_TOKEN_FILE" +``` + +### Configuration Parameters + +| Parameter | Description | +|-----------|-------------| +| `aws_region_name` | AWS region | +| `aws_role_name` | IAM role ARN to assume | +| `aws_session_name` | Session name (optional) | +| `aws_external_id` | External ID for cross-account | +| `aws_profile_name` | AWS profile from `~/.aws/credentials` | +| `aws_web_identity_token` | OIDC token path for IRSA | +| `aws_sts_endpoint` | Custom STS endpoint for VPC | + + + diff --git a/docs/my-website/docs/tutorials/claude_responses_api.md b/docs/my-website/docs/tutorials/claude_responses_api.md index 0dbb4a2f1e7..aafeccceaf5 100644 --- a/docs/my-website/docs/tutorials/claude_responses_api.md +++ b/docs/my-website/docs/tutorials/claude_responses_api.md @@ -105,7 +105,7 @@ LITELLM_MASTER_KEY gives claude access to all proxy models, whereas a virtual ke Alternatively, use the Anthropic pass-through endpoint: ```bash -export ANTHROPIC_BASE_URL="http://0.0.0.0:4000" +export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic" export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY" ``` @@ -221,7 +221,6 @@ You can also connect MCP servers to Claude Code via LiteLLM Proxy. Limitations: - Currently, only HTTP MCP servers are supported -- Does not work in Cursor IDE yet. ::: diff --git a/docs/my-website/docusaurus.config.js b/docs/my-website/docusaurus.config.js index 4ec74b07c3c..32d5d800b71 100644 --- a/docs/my-website/docusaurus.config.js +++ b/docs/my-website/docusaurus.config.js @@ -101,6 +101,21 @@ const config = { include: ['**/*.{md,mdx}'], }, ], + [ + '@docusaurus/plugin-content-blog', + { + id: 'blog', + path: './blog', + routeBasePath: 'blog', + blogTitle: 'Blog', + blogSidebarTitle: 'All Posts', + blogSidebarCount: 'ALL', + postsPerPage: 10, + showReadingTime: false, + sortPosts: 'descending', + include: ['**/index.{md,mdx}'], + }, + ], () => ({ name: 'cripchat', @@ -129,6 +144,7 @@ const config = { docs: { sidebarPath: require.resolve('./sidebars.js'), }, + blog: false, // Disable the default blog plugin from preset-classic theme: { customCss: require.resolve('./src/css/custom.css'), }, @@ -177,6 +193,7 @@ const config = { to: "docs/enterprise" }, { to: '/release_notes', label: 'Release Notes', position: 'left' }, + { to: '/blog', label: 'Blog', position: 'left' }, { href: 'https://models.litellm.ai/', label: '💸 LLM Model Cost Map', diff --git a/docs/my-website/img/app_role2.png b/docs/my-website/img/app_role2.png new file mode 100644 index 00000000000..81eaf8f96ae Binary files /dev/null and b/docs/my-website/img/app_role2.png differ diff --git a/docs/my-website/img/app_role3.png b/docs/my-website/img/app_role3.png new file mode 100644 index 00000000000..e11d73ccc21 Binary files /dev/null and b/docs/my-website/img/app_role3.png differ diff --git a/docs/my-website/img/app_roles.png b/docs/my-website/img/app_roles.png new file mode 100644 index 00000000000..4587ab3a058 Binary files /dev/null and b/docs/my-website/img/app_roles.png differ diff --git a/docs/my-website/img/favicon_converted.ico b/docs/my-website/img/favicon_converted.ico new file mode 100644 index 00000000000..7c45601d5c3 Binary files /dev/null and b/docs/my-website/img/favicon_converted.ico differ diff --git a/docs/my-website/release_notes/authors.yml b/docs/my-website/release_notes/authors.yml new file mode 100644 index 00000000000..aaa3d51ec97 --- /dev/null +++ b/docs/my-website/release_notes/authors.yml @@ -0,0 +1,18 @@ +krrish: + name: Krrish Dholakia + title: CEO, LiteLLM + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + +ishaan: + name: Ishaan Jaffer + title: CTO, LiteLLM + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + +# Alias for typo in name +ishaan-alt: + name: Ishaan Jaff + title: CTO, LiteLLM + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index dbd33e05371..376bcfd5357 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -146,13 +146,13 @@ const sidebars = { type: "category", label: "Admin UI", items: [ + "proxy/ui", "proxy/admin_ui_sso", "proxy/custom_root_ui", "proxy/custom_sso", "proxy/ai_hub", "proxy/public_teams", "proxy/self_serve", - "proxy/ui", "proxy/ui/bulk_edit_users", "proxy/ui_credentials", "tutorials/scim_litellm", @@ -530,13 +530,39 @@ const sidebars = { "providers/bedrock_vector_store", ] }, - "providers/milvus_vector_stores", "providers/litellm_proxy", - "providers/meta_llama", - "providers/mistral", + "providers/ai21", + "providers/aiml", + "providers/aleph_alpha", + "providers/anyscale", + "providers/baseten", + "providers/bytez", + "providers/cerebras", + "providers/clarifai", + "providers/cloudflare_workers", "providers/codestral", "providers/cohere", - "providers/anyscale", + "providers/cometapi", + "providers/compactifai", + "providers/custom_llm_server", + "providers/dashscope", + "providers/databricks", + "providers/datarobot", + "providers/deepgram", + "providers/deepinfra", + "providers/deepseek", + "providers/docker_model_runner", + "providers/elevenlabs", + "providers/fal_ai", + "providers/featherless_ai", + "providers/fireworks_ai", + "providers/friendliai", + "providers/galadriel", + "providers/github", + "providers/github_copilot", + "providers/gradient_ai", + "providers/groq", + "providers/heroku", { type: "category", label: "HuggingFace", @@ -546,10 +572,21 @@ const sidebars = { ] }, "providers/hyperbolic", - "providers/databricks", - "providers/deepgram", - "providers/watsonx", - "providers/predibase", + "providers/infinity", + "providers/jina_ai", + "providers/lambda_ai", + "providers/lemonade", + "providers/llamafile", + "providers/lm_studio", + "providers/meta_llama", + "providers/milvus_vector_stores", + "providers/mistral", + "providers/moonshot", + "providers/morph", + "providers/nebius", + "providers/nlp_cloud", + "providers/novita", + { type: "doc", id: "providers/nscale", label: "Nscale (EU Sovereign)" }, { type: "category", label: "Nvidia NIM", @@ -558,37 +595,13 @@ const sidebars = { "providers/nvidia_nim_rerank", ] }, - { type: "doc", id: "providers/nscale", label: "Nscale (EU Sovereign)" }, - "providers/xai", - "providers/moonshot", - "providers/lm_studio", - "providers/cerebras", - "providers/volcano", - "providers/triton-inference-server", + "providers/oci", "providers/ollama", + "providers/openrouter", + "providers/ovhcloud", "providers/perplexity", - "providers/friendliai", - "providers/galadriel", - "providers/topaz", - "providers/groq", - "providers/deepseek", - "providers/elevenlabs", - "providers/fal_ai", - "providers/fireworks_ai", - "providers/clarifai", - "providers/compactifai", - "providers/lemonade", - "providers/vllm", - "providers/llamafile", - "providers/infinity", - "providers/xinference", - "providers/aiml", - "providers/cloudflare_workers", - "providers/deepinfra", - "providers/github", - "providers/github_copilot", - "providers/ai21", - "providers/nlp_cloud", + "providers/petals", + "providers/predibase", "providers/recraft", "providers/replicate", { @@ -599,32 +612,20 @@ const sidebars = { "providers/runwayml/videos", ] }, + "providers/sambanova", + "providers/snowflake", "providers/togetherai", + "providers/topaz", + "providers/triton-inference-server", "providers/v0", "providers/vercel_ai_gateway", - "providers/morph", - "providers/lambda_ai", - "providers/novita", + "providers/vllm", + "providers/volcano", "providers/voyage", - "providers/jina_ai", - "providers/aleph_alpha", - "providers/baseten", - "providers/openrouter", - "providers/sambanova", - "providers/custom_llm_server", - "providers/petals", - "providers/snowflake", - "providers/gradient_ai", - "providers/featherless_ai", - "providers/nebius", - "providers/dashscope", - "providers/bytez", - "providers/heroku", - "providers/oci", - "providers/datarobot", - "providers/ovhcloud", "providers/wandb_inference", - "providers/cometapi", + "providers/watsonx", + "providers/xai", + "providers/xinference", ], }, { diff --git a/docs/my-website/static/img/favicon.ico b/docs/my-website/static/img/favicon.ico index 88caa2b8315..7c45601d5c3 100644 Binary files a/docs/my-website/static/img/favicon.ico and b/docs/my-website/static/img/favicon.ico differ diff --git a/enterprise/dist/litellm_enterprise-0.1.22-py3-none-any.whl b/enterprise/dist/litellm_enterprise-0.1.22-py3-none-any.whl new file mode 100644 index 00000000000..6ad5b7041c5 Binary files /dev/null and b/enterprise/dist/litellm_enterprise-0.1.22-py3-none-any.whl differ diff --git a/enterprise/dist/litellm_enterprise-0.1.22.tar.gz b/enterprise/dist/litellm_enterprise-0.1.22.tar.gz new file mode 100644 index 00000000000..9db2c14b12f Binary files /dev/null and b/enterprise/dist/litellm_enterprise-0.1.22.tar.gz differ diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 20b850191dd..608bb495885 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -529,7 +529,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): for file_id in file_ids: ## CHECK IF FILE ID IS MANAGED BY LITELM is_base64_unified_file_id = _is_base64_encoded_unified_file_id(file_id) - if is_base64_unified_file_id: litellm_managed_file_ids.append(file_id) @@ -540,6 +539,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): unified_file_object = await self.get_unified_file_id( file_id, litellm_parent_otel_span ) + if unified_file_object: file_id_mapping[file_id] = unified_file_object.model_mappings @@ -815,18 +815,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): llm_router: Router, **data: Dict, ) -> OpenAIFileObject: - file_id = convert_b64_uid_to_unified_uid(file_id) + + # file_id = convert_b64_uid_to_unified_uid(file_id) model_file_id_mapping = await self.get_model_file_id_mapping( [file_id], litellm_parent_otel_span ) + specific_model_file_id_mapping = model_file_id_mapping.get(file_id) if specific_model_file_id_mapping: - for model_id, file_id in specific_model_file_id_mapping.items(): - await llm_router.afile_delete(model=model_id, file_id=file_id, **data) # type: ignore + for model_id, model_file_id in specific_model_file_id_mapping.items(): + await llm_router.afile_delete(model=model_id, file_id=model_file_id, **data) # type: ignore stored_file_object = await self.delete_unified_file_id( file_id, litellm_parent_otel_span ) + if stored_file_object: return stored_file_object else: @@ -847,6 +850,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_file_id_mapping or await self.get_model_file_id_mapping([file_id], litellm_parent_otel_span) ) + specific_model_file_id_mapping = model_file_id_mapping.get(file_id) if specific_model_file_id_mapping: diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index aec888ddc94..2c1fa9945bb 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-enterprise" -version = "0.1.21" +version = "0.1.22" description = "Package for LiteLLM Enterprise features" authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.1.21" +version = "0.1.22" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-enterprise==", diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.6-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.6-py3-none-any.whl new file mode 100644 index 00000000000..346c07b06ea Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.6-py3-none-any.whl differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.6.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.6.tar.gz new file mode 100644 index 00000000000..3a25d44425d Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.6.tar.gz differ diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251119131227_add_prompt_versioning/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251119131227_add_prompt_versioning/migration.sql new file mode 100644 index 00000000000..a9d9528bd24 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251119131227_add_prompt_versioning/migration.sql @@ -0,0 +1,12 @@ +-- DropIndex +DROP INDEX "LiteLLM_PromptTable_prompt_id_key"; + +-- AlterTable +ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "version" INTEGER NOT NULL DEFAULT 1; + +-- CreateIndex +CREATE INDEX "LiteLLM_PromptTable_prompt_id_idx" ON "LiteLLM_PromptTable"("prompt_id"); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_PromptTable_prompt_id_version_key" ON "LiteLLM_PromptTable"("prompt_id", "version"); + diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index d6b7cebbd14..6cfbb90c362 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -561,11 +561,15 @@ model LiteLLM_GuardrailsTable { // Prompt table for storing prompt configurations model LiteLLM_PromptTable { id String @id @default(uuid()) - prompt_id String @unique + prompt_id String + version Int @default(1) litellm_params Json prompt_info Json? created_at DateTime @default(now()) updated_at DateTime @updatedAt + + @@unique([prompt_id, version]) + @@index([prompt_id]) } model LiteLLM_HealthCheckTable { diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index c29492558a3..78e34ccd01a 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.4.5" +version = "0.4.6" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.4.5" +version = "0.4.6" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/litellm/__init__.py b/litellm/__init__.py index c86768490f3..51be5ee2e29 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -388,6 +388,7 @@ disable_add_prefix_to_prompt: bool = ( disable_copilot_system_to_assistant: bool = ( False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. ) +public_mcp_servers: Optional[List[str]] = None public_model_groups: Optional[List[str]] = None public_agent_groups: Optional[List[str]] = None public_model_groups_links: Dict[str, str] = {} @@ -562,6 +563,7 @@ wandb_models: Set = set(WANDB_MODELS) ovhcloud_models: Set = set() ovhcloud_embedding_models: Set = set() lemonade_models: Set = set() +docker_model_runner_models: Set = set() def is_bedrock_pricing_only_model(key: str) -> bool: @@ -796,6 +798,8 @@ def add_known_models(): ovhcloud_embedding_models.add(key) elif value.get("litellm_provider") == "lemonade": lemonade_models.add(key) + elif value.get("litellm_provider") == "docker_model_runner": + docker_model_runner_models.add(key) add_known_models() @@ -899,6 +903,7 @@ model_list = list( | wandb_models | ovhcloud_models | lemonade_models + | docker_model_runner_models | set(clarifai_models) ) @@ -1342,10 +1347,14 @@ from .llms.watsonx.completion.transformation import IBMWatsonXAIConfig from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig from .llms.github_copilot.chat.transformation import GithubCopilotConfig +from .llms.github_copilot.responses.transformation import ( + GithubCopilotResponsesAPIConfig, +) from .llms.nebius.chat.transformation import NebiusConfig from .llms.wandb.chat.transformation import WandbConfig from .llms.dashscope.chat.transformation import DashScopeChatConfig from .llms.moonshot.chat.transformation import MoonshotChatConfig +from .llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig from .llms.v0.chat.transformation import V0ChatConfig from .llms.oci.chat.transformation import OCIChatConfig from .llms.morph.chat.transformation import MorphChatConfig diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 48521e5fba0..838ee95b2b5 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -17,6 +17,7 @@ from functools import partial from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast import httpx +from openai.types.batch import BatchRequestCounts import litellm from litellm._logging import verbose_logger @@ -223,10 +224,12 @@ def create_batch( api_key=optional_params.api_key, logging_obj=litellm_logging_obj, _is_async=_is_async, - client=client - if client is not None - and isinstance(client, (HTTPHandler, AsyncHTTPHandler)) - else None, + client=( + client + if client is not None + and isinstance(client, (HTTPHandler, AsyncHTTPHandler)) + else None + ), timeout=timeout, model=model, ) @@ -609,10 +612,12 @@ def retrieve_batch( function_id="batch_retrieve", ), _is_async=_is_async, - client=client - if client is not None - and isinstance(client, (HTTPHandler, AsyncHTTPHandler)) - else None, + client=( + client + if client is not None + and isinstance(client, (HTTPHandler, AsyncHTTPHandler)) + else None + ), timeout=timeout, model=model, ) @@ -799,6 +804,7 @@ def list_batches( async def acancel_batch( batch_id: str, + model: Optional[str] = None, custom_llm_provider: Literal["openai", "azure"] = "openai", metadata: Optional[Dict[str, str]] = None, extra_headers: Optional[Dict[str, str]] = None, @@ -813,11 +819,13 @@ async def acancel_batch( try: loop = asyncio.get_event_loop() kwargs["acancel_batch"] = True + model = kwargs.pop("model", None) # Use a partial function to pass your keyword arguments func = partial( cancel_batch, batch_id, + model, custom_llm_provider, metadata, extra_headers, @@ -840,7 +848,8 @@ async def acancel_batch( def cancel_batch( batch_id: str, - custom_llm_provider: Literal["openai", "azure"] = "openai", + model: Optional[str] = None, + custom_llm_provider: Union[Literal["openai", "azure"], str] = "openai", metadata: Optional[Dict[str, str]] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, @@ -852,6 +861,17 @@ def cancel_batch( LiteLLM Equivalent of POST https://api.openai.com/v1/batches/{batch_id}/cancel """ try: + + try: + if model is not None: + _, custom_llm_provider, _, _ = get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + ) + except Exception as e: + verbose_logger.exception( + f"litellm.batches.main.py::cancel_batch() - Error inferring custom_llm_provider - {str(e)}" + ) optional_params = GenericLiteLLMParams(**kwargs) litellm_params = get_litellm_params( custom_llm_provider=custom_llm_provider, @@ -1005,21 +1025,28 @@ def _handle_async_invoke_status( created_at=status_response["submitTime"], in_progress_at=status_response["lastModifiedTime"], completed_at=status_response.get("endTime"), - failed_at=status_response.get("endTime") - if status_response["status"] == "failed" - else None, - request_counts={ - "total": 1, - "completed": 1 if status_response["status"] == "completed" else 0, - "failed": 1 if status_response["status"] == "failed" else 0, - }, - metadata={ - "output_file_id": status_response["outputDataConfig"][ - "s3OutputDataConfig" - ]["s3Uri"], - "failure_message": status_response.get("failureMessage"), - "model_arn": status_response["modelArn"], - }, + failed_at=( + status_response.get("endTime") + if status_response["status"] == "failed" + else None + ), + request_counts=BatchRequestCounts( + total=1, + completed=1 if status_response["status"] == "completed" else 0, + failed=1 if status_response["status"] == "failed" else 0, + ), + metadata=dict( + **{ + "output_file_id": status_response["outputDataConfig"][ + "s3OutputDataConfig" + ]["s3Uri"], + "failure_message": status_response.get("failureMessage") or "", + "model_arn": status_response["modelArn"], + } + ), + completion_window="24h", + endpoint="/v1/embeddings", + input_file_id="", ) return result diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 8c3ebd51036..3ac5a28f900 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -26,7 +26,12 @@ from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.bridges.completion_transformation import ( CompletionTransformationBridge, ) -from litellm.types.llms.openai import ChatCompletionToolParamFunctionChunk, Reasoning +from litellm.types.llms.openai import ( + ChatCompletionToolParamFunctionChunk, + Reasoning, + ResponsesAPIOptionalRequestParams, + ResponsesAPIStreamEvents, +) if TYPE_CHECKING: from openai.types.responses import ResponseInputImageParam @@ -165,13 +170,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): litellm_logging_obj: "LiteLLMLoggingObj", client: Optional[Any] = None, ) -> dict: - from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams - ( input_items, instructions, ) = self.convert_chat_completion_messages_to_responses_api(messages) + optional_params = self._extract_extra_body_params(optional_params) + # Build responses API request using the reverse transformation logic responses_api_request = ResponsesAPIOptionalRequestParams() @@ -194,9 +199,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys(): responses_api_request[key] = value # type: ignore - elif key in ("metadata"): + elif key == "metadata": responses_api_request["metadata"] = value - elif key in ("previous_response_id"): + elif key == "previous_response_id": responses_api_request["previous_response_id"] = value elif key == "reasoning_effort": responses_api_request["reasoning"] = self._map_reasoning_effort(value) @@ -538,6 +543,35 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return cast(List["ALL_RESPONSES_API_TOOL_PARAMS"], responses_tools) + def _extract_extra_body_params(self, optional_params: dict): + """ + Extract extra_body from optional_params and separate supported Responses API params + from unsupported ones. Supported params are moved to top-level optional_params, + unsupported params remain in extra_body. + """ + # Extract extra_body and separate supported params from unsupported ones + extra_body = optional_params.pop("extra_body", None) or {} + if not extra_body: + return optional_params + + supported_responses_api_params = set( + ResponsesAPIOptionalRequestParams.__annotations__.keys() + ) + # Also include params we handle specially + supported_responses_api_params.update({ + "previous_response_id", + "reasoning_effort", # We map this to "reasoning" + }) + + # Extract supported params from extra_body and merge into optional_params + extra_body_copy = extra_body.copy() + for key, value in extra_body_copy.items(): + if key in supported_responses_api_params: + # Prefer extra_body value if it exists (may have more complete info like summary in reasoning_effort) + optional_params[key] = extra_body.pop(key) + + return optional_params + def _map_reasoning_effort(self, reasoning_effort: Union[str, Dict[str, Any]]) -> Optional[Reasoning]: # If dict is passed, convert it directly to Reasoning object if isinstance(reasoning_effort, dict): @@ -619,6 +653,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): # Handle different event types from responses API event_type = parsed_chunk.get("type") + if isinstance(event_type, ResponsesAPIStreamEvents): + event_type = event_type.value verbose_logger.debug(f"Chat provider: Processing event type: {event_type}") if event_type == "response.created": @@ -638,7 +674,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): index=0, type="function", function=ChatCompletionToolCallFunctionChunk( - name=parsed_chunk.get("name", None), + name=output_item.get("name", None), arguments=parsed_chunk.get("arguments", ""), ), ), @@ -684,7 +720,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): index=0, type="function", function=ChatCompletionToolCallFunctionChunk( - name=parsed_chunk.get("name", None), + name=output_item.get("name", None), arguments="", # responses API sends everything again, we don't ), ), diff --git a/litellm/constants.py b/litellm/constants.py index b90f36ae96f..b312a15892b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -381,6 +381,7 @@ LITELLM_CHAT_PROVIDERS = [ "wandb", "ovhcloud", "lemonade", + "docker_model_runner", ] LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [ @@ -567,6 +568,7 @@ openai_compatible_providers: List = [ "wandb", "cometapi", "clarifai", + "docker_model_runner", ] openai_text_completion_compatible_providers: List = ( [ # providers that support `/v1/completions` @@ -1118,6 +1120,7 @@ SECRET_MANAGER_REFRESH_INTERVAL = int( ) LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [ "default_internal_user_params", + "public_mcp_servers", "public_agent_groups", "public_model_groups", "public_model_groups_links", diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index d1c7ede6552..0f5195e31af 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -133,6 +133,25 @@ def _cost_per_token_custom_pricing_helper( return None +def _transcription_usage_has_token_details( + usage_block: Optional[Usage], +) -> bool: + if usage_block is None: + return False + + prompt_tokens_val = getattr(usage_block, "prompt_tokens", 0) or 0 + completion_tokens_val = getattr(usage_block, "completion_tokens", 0) or 0 + prompt_details = getattr(usage_block, "prompt_tokens_details", None) + + if prompt_details is not None: + audio_token_count = getattr(prompt_details, "audio_tokens", 0) or 0 + text_token_count = getattr(prompt_details, "text_tokens", 0) or 0 + if audio_token_count > 0 or text_token_count > 0: + return True + + return (prompt_tokens_val > 0) or (completion_tokens_val > 0) + + def cost_per_token( # noqa: PLR0915 model: str = "", prompt_tokens: int = 0, @@ -324,19 +343,18 @@ def cost_per_token( # noqa: PLR0915 usage=usage_block, model=model, custom_llm_provider=custom_llm_provider ) elif call_type == "atranscription" or call_type == "transcription": - - if model == "gpt-4o-mini-transcribe": + if _transcription_usage_has_token_details(usage_block): return openai_cost_per_token( - model=model, + model=model_without_prefix, usage=usage_block, service_tier=service_tier, ) - else: - return openai_cost_per_second( - model=model, - custom_llm_provider=custom_llm_provider, - duration=audio_transcription_file_duration, - ) + + return openai_cost_per_second( + model=model_without_prefix, + custom_llm_provider=custom_llm_provider, + duration=audio_transcription_file_duration, + ) elif call_type == "search" or call_type == "asearch": # Search providers use per-query pricing from litellm.search import search_provider_cost_per_query diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index fe11349b62b..943cc6b2d53 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -8,14 +8,21 @@ from datetime import timedelta from typing import Awaitable, Callable, Dict, List, Optional, TypeVar, Union import httpx -from mcp import ClientSession, StdioServerParameters +from mcp import ClientSession, ReadResourceResult, Resource, StdioServerParameters from mcp.client.sse import sse_client from mcp.client.stdio import stdio_client from mcp.client.streamable_http import streamablehttp_client -from mcp.types import CallToolRequestParams as MCPCallToolRequestParams +from mcp.types import ( + CallToolRequestParams as MCPCallToolRequestParams, + GetPromptRequestParams, + GetPromptResult, + Prompt, + ResourceTemplate, +) from mcp.types import CallToolResult as MCPCallToolResult from mcp.types import TextContent from mcp.types import Tool as MCPTool +from pydantic import AnyUrl from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import get_ssl_configuration @@ -214,7 +221,7 @@ class MCPClient: raise except Exception as e: error_type = type(e).__name__ - verbose_logger.error( + verbose_logger.exception( f"MCP client list_tools failed - " f"Error Type: {error_type}, " f"Error: {str(e)}, " @@ -289,3 +296,218 @@ class MCPClient: ], # Empty content for error case isError=True, ) + + async def list_prompts(self) -> List[Prompt]: + """List available prompts from the server.""" + verbose_logger.debug( + f"MCP client listing tools from {self.server_url or 'stdio'}" + ) + + async def _list_prompts_operation(session: ClientSession): + return await session.list_prompts() + + try: + result = await self.run_with_session(_list_prompts_operation) + prompt_count = len(result.prompts) + prompt_names = [prompt.name for prompt in result.prompts] + verbose_logger.info( + f"MCP client listed {prompt_count} tools from {self.server_url or 'stdio'}: {prompt_names}" + ) + return result.prompts + except asyncio.CancelledError: + verbose_logger.warning("MCP client list_prompts was cancelled") + raise + except Exception as e: + error_type = type(e).__name__ + verbose_logger.error( + f"MCP client list_prompts failed - " + f"Error Type: {error_type}, " + f"Error: {str(e)}, " + f"Server: {self.server_url or 'stdio'}, " + f"Transport: {self.transport_type}" + ) + + # Check if it's a stream/connection error + if "BrokenResourceError" in error_type or "Broken" in error_type: + verbose_logger.error( + "MCP client detected broken connection/stream during list_tools - " + "the MCP server may have crashed, disconnected, or timed out" + ) + + # Return empty list instead of raising to allow graceful degradation + return [] + + async def get_prompt( + self, get_prompt_request_params: GetPromptRequestParams + ) -> GetPromptResult: + """Fetch a prompt definition from the MCP server.""" + verbose_logger.info( + f"MCP client fetching prompt '{get_prompt_request_params.name}' with arguments: {get_prompt_request_params.arguments}" + ) + + async def _get_prompt_operation(session: ClientSession): + verbose_logger.debug("MCP client sending get_prompt request to session") + return await session.get_prompt( + name=get_prompt_request_params.name, + arguments=get_prompt_request_params.arguments, + ) + + try: + get_prompt_result = await self.run_with_session(_get_prompt_operation) + verbose_logger.info( + f"MCP client get_prompt '{get_prompt_request_params.name}' completed successfully" + ) + return get_prompt_result + except asyncio.CancelledError: + verbose_logger.warning("MCP client get_prompt was cancelled") + raise + except Exception as e: + import traceback + + error_trace = traceback.format_exc() + verbose_logger.debug(f"MCP client get_prompt traceback:\n{error_trace}") + + # Log detailed error information + error_type = type(e).__name__ + verbose_logger.error( + f"MCP client get_prompt failed - " + f"Error Type: {error_type}, " + f"Error: {str(e)}, " + f"Prompt: {get_prompt_request_params.name}, " + f"Server: {self.server_url or 'stdio'}, " + f"Transport: {self.transport_type}" + ) + + # Check if it's a stream/connection error + if "BrokenResourceError" in error_type or "Broken" in error_type: + verbose_logger.error( + "MCP client detected broken connection/stream during get_prompt - " + "the MCP server may have crashed, disconnected, or timed out." + ) + + raise + + async def list_resources(self) -> list[Resource]: + """List available resources from the server.""" + verbose_logger.debug( + f"MCP client listing resources from {self.server_url or 'stdio'}" + ) + + async def _list_resources_operation(session: ClientSession): + return await session.list_resources() + + try: + result = await self.run_with_session(_list_resources_operation) + resource_count = len(result.resources) + resource_names = [resource.name for resource in result.resources] + verbose_logger.info( + f"MCP client listed {resource_count} resources from {self.server_url or 'stdio'}: {resource_names}" + ) + return result.resources + except asyncio.CancelledError: + verbose_logger.warning("MCP client list_resources was cancelled") + raise + except Exception as e: + error_type = type(e).__name__ + verbose_logger.error( + f"MCP client list_resources failed - " + f"Error Type: {error_type}, " + f"Error: {str(e)}, " + f"Server: {self.server_url or 'stdio'}, " + f"Transport: {self.transport_type}" + ) + + # Check if it's a stream/connection error + if "BrokenResourceError" in error_type or "Broken" in error_type: + verbose_logger.error( + "MCP client detected broken connection/stream during list_resources - " + "the MCP server may have crashed, disconnected, or timed out" + ) + + # Return empty list instead of raising to allow graceful degradation + return [] + + async def list_resource_templates(self) -> list[ResourceTemplate]: + """List available resource templates from the server.""" + verbose_logger.debug( + f"MCP client listing resource templates from {self.server_url or 'stdio'}" + ) + + async def _list_resource_templates_operation(session: ClientSession): + return await session.list_resource_templates() + + try: + result = await self.run_with_session(_list_resource_templates_operation) + resource_template_count = len(result.resourceTemplates) + resource_template_names = [ + resourceTemplate.name for resourceTemplate in result.resourceTemplates + ] + verbose_logger.info( + f"MCP client listed {resource_template_count} resource templates from {self.server_url or 'stdio'}: {resource_template_names}" + ) + return result.resourceTemplates + except asyncio.CancelledError: + verbose_logger.warning("MCP client list_resource_templates was cancelled") + raise + except Exception as e: + error_type = type(e).__name__ + verbose_logger.error( + f"MCP client list_resource_templates failed - " + f"Error Type: {error_type}, " + f"Error: {str(e)}, " + f"Server: {self.server_url or 'stdio'}, " + f"Transport: {self.transport_type}" + ) + + # Check if it's a stream/connection error + if "BrokenResourceError" in error_type or "Broken" in error_type: + verbose_logger.error( + "MCP client detected broken connection/stream during list_resource_templates - " + "the MCP server may have crashed, disconnected, or timed out" + ) + + # Return empty list instead of raising to allow graceful degradation + return [] + + async def read_resource(self, url: AnyUrl) -> ReadResourceResult: + """Fetch resource contents from the MCP server.""" + verbose_logger.info(f"MCP client fetching resource '{url}'") + + async def _read_resource_operation(session: ClientSession): + verbose_logger.debug("MCP client sending read_resource request to session") + return await session.read_resource(url) + + try: + read_resource_result = await self.run_with_session(_read_resource_operation) + verbose_logger.info( + f"MCP client read_resource '{url}' completed successfully" + ) + return read_resource_result + except asyncio.CancelledError: + verbose_logger.warning("MCP client read_resource was cancelled") + raise + except Exception as e: + import traceback + + error_trace = traceback.format_exc() + verbose_logger.debug(f"MCP client read_resource traceback:\n{error_trace}") + + # Log detailed error information + error_type = type(e).__name__ + verbose_logger.error( + f"MCP client read_resource failed - " + f"Error Type: {error_type}, " + f"Error: {str(e)}, " + f"Url: {url}, " + f"Server: {self.server_url or 'stdio'}, " + f"Transport: {self.transport_type}" + ) + + # Check if it's a stream/connection error + if "BrokenResourceError" in error_type or "Broken" in error_type: + verbose_logger.error( + "MCP client detected broken connection/stream during read_resource - " + "the MCP server may have crashed, disconnected, or timed out." + ) + + raise diff --git a/litellm/files/main.py b/litellm/files/main.py index 9c85fa10565..535772fa42c 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -95,7 +95,9 @@ async def acreate_file( def create_file( file: FileTypes, purpose: Literal["assistants", "batch", "fine-tune"], - custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai", "bedrock"]] = None, + custom_llm_provider: Optional[ + Literal["openai", "azure", "vertex_ai", "bedrock"] + ] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -155,10 +157,12 @@ def create_file( api_key=optional_params.api_key, logging_obj=logging_obj, _is_async=_is_async, - client=client - if client is not None - and isinstance(client, (HTTPHandler, AsyncHTTPHandler)) - else None, + client=( + client + if client is not None + and isinstance(client, (HTTPHandler, AsyncHTTPHandler)) + else None + ), timeout=timeout, ) elif custom_llm_provider == "openai": @@ -441,12 +445,14 @@ async def afile_delete( """ try: loop = asyncio.get_event_loop() + model = kwargs.pop("model", None) kwargs["is_async"] = True # Use a partial function to pass your keyword arguments func = partial( file_delete, file_id, + model, custom_llm_provider, extra_headers, extra_body, @@ -470,7 +476,8 @@ async def afile_delete( @client def file_delete( file_id: str, - custom_llm_provider: Literal["openai", "azure"] = "openai", + model: Optional[str] = None, + custom_llm_provider: Union[Literal["openai", "azure"], str] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -481,6 +488,13 @@ def file_delete( LiteLLM Equivalent of DELETE https://api.openai.com/v1/files """ try: + try: + if model is not None: + _, custom_llm_provider, _, _ = get_llm_provider( + model, custom_llm_provider + ) + except Exception: + pass optional_params = GenericLiteLLMParams(**kwargs) litellm_params_dict = get_litellm_params(**kwargs) ### TIMEOUT LOGIC ### @@ -566,7 +580,7 @@ def file_delete( ) else: raise litellm.exceptions.BadRequestError( - message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format( + message="LiteLLM doesn't support {} for 'delete_batch'. Only 'openai' is supported.".format( custom_llm_provider ), model="n/a", diff --git a/litellm/integrations/arize/_utils.py b/litellm/integrations/arize/_utils.py index 10597d6e713..c9a1531b5d4 100644 --- a/litellm/integrations/arize/_utils.py +++ b/litellm/integrations/arize/_utils.py @@ -208,7 +208,10 @@ def set_attributes( ) try: + # Remove secret_fields to prevent leaking sensitive data (e.g., authorization headers) optional_params = kwargs.get("optional_params", {}) + if isinstance(optional_params, dict): + optional_params.pop("secret_fields", None) litellm_params = kwargs.get("litellm_params", {}) standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get( "standard_logging_object" diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index b50d05ed2ec..b52f1b3095e 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -36,6 +36,7 @@ class CustomGuardrail(CustomLogger): default_on: bool = False, mask_request_content: bool = False, mask_response_content: bool = False, + violation_message_template: Optional[str] = None, **kwargs, ): """ @@ -57,12 +58,34 @@ class CustomGuardrail(CustomLogger): self.default_on: bool = default_on self.mask_request_content: bool = mask_request_content self.mask_response_content: bool = mask_response_content + self.violation_message_template: Optional[str] = violation_message_template if supported_event_hooks: ## validate event_hook is in supported_event_hooks self._validate_event_hook(event_hook, supported_event_hooks) super().__init__(**kwargs) + def render_violation_message( + self, default: str, context: Optional[Dict[str, Any]] = None + ) -> str: + """Return a custom violation message if template is configured.""" + + if not self.violation_message_template: + return default + + format_context: Dict[str, Any] = {"default_message": default} + if context: + format_context.update(context) + try: + return self.violation_message_template.format(**format_context) + except Exception as e: + verbose_logger.warning( + "Failed to format violation message template for guardrail %s: %s", + self.guardrail_name, + e, + ) + return default + @staticmethod def get_config_model() -> Optional[Type["GuardrailConfigModel"]]: """ @@ -279,7 +302,7 @@ class CustomGuardrail(CustomLogger): data, self.event_hook ) if result is not None: - return result + return result return True def _event_hook_is_event_type(self, event_type: GuardrailEventHooks) -> bool: diff --git a/litellm/integrations/dotprompt/__init__.py b/litellm/integrations/dotprompt/__init__.py index 3af7fbf6dd3..3847c8fa192 100644 --- a/litellm/integrations/dotprompt/__init__.py +++ b/litellm/integrations/dotprompt/__init__.py @@ -25,6 +25,23 @@ def set_global_prompt_directory(directory: str) -> None: litellm.global_prompt_directory = directory # type: ignore +def _get_prompt_data_from_dotprompt_content(dotprompt_content: str) -> dict: + """ + Get the prompt data from the dotprompt content. + + The UI stores prompts under `dotprompt_content` in the database. This function parses the content and returns the prompt data in the format expected by the prompt manager. + """ + from .prompt_manager import PromptManager + + # Parse the dotprompt content to extract frontmatter and content + temp_manager = PromptManager() + metadata, content = temp_manager._parse_frontmatter(dotprompt_content) + + # Convert to prompt_data format + return { + "content": content.strip(), + "metadata": metadata + } def prompt_initializer( litellm_params: "PromptLiteLLMParams", prompt_spec: "PromptSpec" @@ -41,6 +58,11 @@ def prompt_initializer( ) prompt_file = getattr(litellm_params, "prompt_file", None) + + # Handle dotprompt_content from database + dotprompt_content = getattr(litellm_params, "dotprompt_content", None) + if dotprompt_content and not prompt_data and not prompt_file: + prompt_data = _get_prompt_data_from_dotprompt_content(dotprompt_content) try: dot_prompt_manager = DotpromptManager( diff --git a/litellm/integrations/dotprompt/dotprompt_manager.py b/litellm/integrations/dotprompt/dotprompt_manager.py index 0f0d7b938f3..7aaa6cc9628 100644 --- a/litellm/integrations/dotprompt/dotprompt_manager.py +++ b/litellm/integrations/dotprompt/dotprompt_manager.py @@ -108,7 +108,7 @@ class DotpromptManager(CustomPromptManagement): Compile a .prompt file into a PromptManagementClient structure. This method: - 1. Loads the prompt template from the .prompt file + 1. Loads the prompt template from the .prompt file (with optional version) 2. Renders it with the provided variables 3. Converts the rendered text into chat messages 4. Extracts model and optional parameters from metadata @@ -116,13 +116,22 @@ class DotpromptManager(CustomPromptManagement): try: - # Get the prompt template - template = self.prompt_manager.get_prompt(prompt_id) + # Get the prompt template (versioned or base) + template = self.prompt_manager.get_prompt( + prompt_id=prompt_id, version=prompt_version + ) if template is None: - raise ValueError(f"Prompt '{prompt_id}' not found in prompt directory") + version_str = f" (version {prompt_version})" if prompt_version else "" + raise ValueError( + f"Prompt '{prompt_id}'{version_str} not found in prompt directory" + ) - # Render the template with variables - rendered_content = self.prompt_manager.render(prompt_id, prompt_variables) + # Render the template with variables (pass version for proper lookup) + rendered_content = self.prompt_manager.render( + prompt_id=prompt_id, + prompt_variables=prompt_variables, + version=prompt_version, + ) # Convert rendered content to chat messages messages = self._convert_to_messages(rendered_content) diff --git a/litellm/integrations/dotprompt/prompt_manager.py b/litellm/integrations/dotprompt/prompt_manager.py index 9623ddab5fb..fc5a325ffe1 100644 --- a/litellm/integrations/dotprompt/prompt_manager.py +++ b/litellm/integrations/dotprompt/prompt_manager.py @@ -183,7 +183,10 @@ class PromptManager: return frontmatter, template_content def render( - self, prompt_id: str, prompt_variables: Optional[Dict[str, Any]] = None + self, + prompt_id: str, + prompt_variables: Optional[Dict[str, Any]] = None, + version: Optional[int] = None, ) -> str: """ Render a prompt template with the given variables. @@ -191,6 +194,7 @@ class PromptManager: Args: prompt_id: The ID of the prompt template to render prompt_variables: Variables to substitute in the template + version: Optional version number. If provided, looks for {prompt_id}.v{version} Returns: The rendered prompt string @@ -199,13 +203,16 @@ class PromptManager: KeyError: If prompt_id is not found ValueError: If template rendering fails """ - if prompt_id not in self.prompts: + # Get the template (versioned or base) + template = self.get_prompt(prompt_id=prompt_id, version=version) + + if template is None: available_prompts = list(self.prompts.keys()) + version_str = f" (version {version})" if version else "" raise KeyError( - f"Prompt '{prompt_id}' not found. Available prompts: {available_prompts}" + f"Prompt '{prompt_id}'{version_str} not found. Available prompts: {available_prompts}" ) - template = self.prompts[prompt_id] variables = prompt_variables or {} # Validate input variables against schema if defined @@ -254,8 +261,26 @@ class PromptManager: return type_mapping.get(schema_type.lower(), str) # type: ignore - def get_prompt(self, prompt_id: str) -> Optional[PromptTemplate]: - """Get a prompt template by ID.""" + def get_prompt( + self, prompt_id: str, version: Optional[int] = None + ) -> Optional[PromptTemplate]: + """ + Get a prompt template by ID and optional version. + + Args: + prompt_id: The base prompt ID + version: Optional version number. If provided, looks for {prompt_id}.v{version} + + Returns: + The prompt template if found, None otherwise + """ + if version is not None: + # Try versioned prompt first: prompt_id.v{version} + versioned_id = f"{prompt_id}.v{version}" + if versioned_id in self.prompts: + return self.prompts[versioned_id] + + # Fall back to base prompt_id return self.prompts.get(prompt_id) def list_prompts(self) -> List[str]: diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index c2a2cc77950..12eb00efa9e 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -228,6 +228,8 @@ class LangFuseLogger: functions = optional_params.pop("functions", None) tools = optional_params.pop("tools", None) + # Remove secret_fields to prevent leaking sensitive data (e.g., authorization headers) + optional_params.pop("secret_fields", None) if functions is not None: prompt["functions"] = functions if tools is not None: diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index ef0ebe074d7..eefe680217d 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -741,6 +741,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.MoonshotChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "docker_model_runner": + ( + api_base, + dynamic_api_key, + ) = litellm.DockerModelRunnerChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) elif custom_llm_provider == "v0": ( api_base, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 6bad7ee29e2..e5c52ce48ae 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -585,7 +585,10 @@ class Logging(LiteLLMLoggingBaseClass): custom_logger = ( prompt_management_logger or self.get_custom_logger_for_prompt_management( - model=model, non_default_params=non_default_params + model=model, + non_default_params=non_default_params, + prompt_id=prompt_id, + dynamic_callback_params=self.standard_callback_dynamic_params, ) ) @@ -622,7 +625,11 @@ class Logging(LiteLLMLoggingBaseClass): custom_logger = ( prompt_management_logger or self.get_custom_logger_for_prompt_management( - model=model, tools=tools, non_default_params=non_default_params + model=model, + tools=tools, + non_default_params=non_default_params, + prompt_id=prompt_id, + dynamic_callback_params=self.standard_callback_dynamic_params, ) ) @@ -646,19 +653,69 @@ class Logging(LiteLLMLoggingBaseClass): self.messages = messages return model, messages, non_default_params + def _auto_detect_prompt_management_logger( + self, + prompt_id: str, + dynamic_callback_params: StandardCallbackDynamicParams, + ) -> Optional[CustomLogger]: + """ + Auto-detect which prompt management system owns the given prompt_id. + + This allows a user to just pass prompt_id in the completion call and it will be auto-detected which system owns this prompt. + + Args: + prompt_id: The prompt ID to check + dynamic_callback_params: Dynamic callback parameters for should_run_prompt_management checks + + Returns: + A CustomLogger instance if a matching prompt management system is found, None otherwise + """ + prompt_management_loggers = ( + litellm.logging_callback_manager.get_custom_loggers_for_type( + callback_type=CustomPromptManagement + ) + ) + + for logger in prompt_management_loggers: + if isinstance(logger, CustomPromptManagement): + try: + if logger.should_run_prompt_management( + prompt_id=prompt_id, + dynamic_callback_params=dynamic_callback_params, + ): + self.model_call_details["prompt_integration"] = ( + logger.__class__.__name__ + ) + return logger + except Exception: + # If check fails, continue to next logger + continue + + return None + def get_custom_logger_for_prompt_management( - self, model: str, non_default_params: Dict, tools: Optional[List[Dict]] = None + self, + model: str, + non_default_params: Dict, + tools: Optional[List[Dict]] = None, + prompt_id: Optional[str] = None, + dynamic_callback_params: Optional[StandardCallbackDynamicParams] = None, ) -> Optional[CustomLogger]: """ Get a custom logger for prompt management based on model name or available callbacks. Args: model: The model name to check for prompt management integration + non_default_params: Non-default parameters passed to the completion call + tools: Optional tools passed to the completion call + prompt_id: Optional prompt ID to auto-detect which system owns this prompt + dynamic_callback_params: Dynamic callback parameters for should_run_prompt_management checks Returns: A CustomLogger instance if one is found, None otherwise """ # First check if model starts with a known custom logger compatible callback + # This takes precedence for backward compatibility for callback_name in litellm._known_custom_logger_compatible_callbacks: if model.startswith(callback_name): custom_logger = _init_custom_logger_compatible_class( @@ -670,7 +727,16 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["prompt_integration"] = model.split("/")[0] return custom_logger - # Then check for any registered CustomPromptManagement loggers + # If prompt_id is provided, try to auto-detect which system has this prompt + if prompt_id and dynamic_callback_params is not None: + auto_detected_logger = self._auto_detect_prompt_management_logger( + prompt_id=prompt_id, + dynamic_callback_params=dynamic_callback_params, + ) + if auto_detected_logger is not None: + return auto_detected_logger + + # Then check for any registered CustomPromptManagement loggers (fallback) prompt_management_loggers = ( litellm.logging_callback_manager.get_custom_loggers_for_type( callback_type=CustomPromptManagement diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 9b1cbbd5773..0f4a159975a 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -58,6 +58,10 @@ def prompt_injection_detection_default_pt(): BAD_MESSAGE_ERROR_STR = "Invalid Message " +# Separator used to embed Gemini thought signatures in tool call IDs +# See: https://ai.google.dev/gemini-api/docs/thought-signatures +THOUGHT_SIGNATURE_SEPARATOR = "__thought__" + # used to interweave user messages, to ensure user/assistant alternating DEFAULT_USER_CONTINUE_MESSAGE = { "role": "user", @@ -1162,16 +1166,94 @@ def _gemini_tool_call_invoke_helper( return function_call -def _get_thought_signature_from_tool(tool: dict) -> Optional[str]: - """Extract thought signature from tool call's provider_specific_fields""" +def _encode_tool_call_id_with_signature( + tool_call_id: str, thought_signature: Optional[str] +) -> str: + """ + Embed thought signature into tool call ID for OpenAI client compatibility. + + Args: + tool_call_id: The tool call ID (e.g., "call_abc123...") + thought_signature: Base64-encoded signature from Gemini response + + Returns: + Tool call ID with embedded signature if present, otherwise original ID + Format: call___thought__ + + See: https://ai.google.dev/gemini-api/docs/thought-signatures + """ + if thought_signature: + return f"{tool_call_id}{THOUGHT_SIGNATURE_SEPARATOR}{thought_signature}" + return tool_call_id + + +def _get_thought_signature_from_tool( + tool: dict, model: Optional[str] = None +) -> Optional[str]: + """Extract thought signature from tool call's provider_specific_fields. + + If not provided try to extract thought signature from tool call id + + Checks both tool.provider_specific_fields and tool.function.provider_specific_fields. + If no signature is found and model is gemini-3, returns a dummy signature. + """ + # First check tool's provider_specific_fields provider_fields = tool.get("provider_specific_fields") or {} if isinstance(provider_fields, dict): - return provider_fields.get("thought_signature") + signature = provider_fields.get("thought_signature") + if signature: + return signature + + # Then check function's provider_specific_fields + function = tool.get("function") + if function: + if isinstance(function, dict): + func_provider_fields = function.get("provider_specific_fields") or {} + if isinstance(func_provider_fields, dict): + signature = func_provider_fields.get("thought_signature") + if signature: + return signature + elif ( + hasattr(function, "provider_specific_fields") + and function.provider_specific_fields + ): + if isinstance(function.provider_specific_fields, dict): + signature = function.provider_specific_fields.get("thought_signature") + if signature: + return signature + # Check if thought signature is embedded in tool call ID + tool_call_id = tool.get("id") + if tool_call_id and THOUGHT_SIGNATURE_SEPARATOR in tool_call_id: + parts = tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1) + if len(parts) == 2: + _, signature = parts + return signature + # If no signature found and model is gemini-3, return dummy signature + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + if model and VertexGeminiConfig._is_gemini_3_or_newer(model): + return _get_dummy_thought_signature() return None +def _get_dummy_thought_signature() -> str: + """Generate a dummy thought signature for models that require it. + + This is used when transferring conversation history from older models + (like gemini-2.5-flash) to gemini-3, which requires thought_signature + for strict validation. + """ + # Return a base64-encoded dummy signature string + # Below dummy signature is recommended by google - https://ai.google.dev/gemini-api/docs/thought-signatures#faqs + dummy_data = b"skip_thought_signature_validator" + return base64.b64encode(dummy_data).decode("utf-8") + + def convert_to_gemini_tool_call_invoke( message: ChatCompletionAssistantMessage, + model: Optional[str] = None, ) -> List[VertexPartType]: """ OpenAI tool invokes: @@ -1216,23 +1298,25 @@ def convert_to_gemini_tool_call_invoke( _parts_list: List[VertexPartType] = [] tool_calls = message.get("tool_calls", None) function_call = message.get("function_call", None) - + if tool_calls is not None: for idx, tool in enumerate(tool_calls): if "function" in tool: - gemini_function_call: Optional[VertexFunctionCall] = ( - _gemini_tool_call_invoke_helper( - function_call_params=tool["function"] - ) + gemini_function_call: Optional[ + VertexFunctionCall + ] = _gemini_tool_call_invoke_helper( + function_call_params=tool["function"] ) if gemini_function_call is not None: part_dict: VertexPartType = { "function_call": gemini_function_call } - thought_signature = _get_thought_signature_from_tool(dict(tool)) + thought_signature = _get_thought_signature_from_tool( + dict(tool), model=model + ) if thought_signature: part_dict["thoughtSignature"] = thought_signature - + _parts_list.append(part_dict) else: # don't silently drop params. Make it clear to user what's happening. raise Exception( @@ -1248,14 +1332,32 @@ def convert_to_gemini_tool_call_invoke( part_dict_function: VertexPartType = { "function_call": gemini_function_call } - + # Extract thought signature from function_call's provider_specific_fields - provider_fields = function_call.get("provider_specific_fields") if isinstance(function_call, dict) else {} + thought_signature = None + provider_fields = ( + function_call.get("provider_specific_fields") + if isinstance(function_call, dict) + else {} + ) if isinstance(provider_fields, dict): thought_signature = provider_fields.get("thought_signature") - if thought_signature: - part_dict_function["thoughtSignature"] = thought_signature - + + # If no signature found and model is gemini-3, use dummy signature + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + if ( + not thought_signature + and model + and VertexGeminiConfig._is_gemini_3_or_newer(model) + ): + thought_signature = _get_dummy_thought_signature() + + if thought_signature: + part_dict_function["thoughtSignature"] = thought_signature + _parts_list.append(part_dict_function) else: # don't silently drop params. Make it clear to user what's happening. raise Exception( @@ -1758,9 +1860,9 @@ def anthropic_messages_pt( # noqa: PLR0915 ) if "cache_control" in _content_element: - _anthropic_content_element["cache_control"] = ( - _content_element["cache_control"] - ) + _anthropic_content_element[ + "cache_control" + ] = _content_element["cache_control"] user_content.append(_anthropic_content_element) elif m.get("type", "") == "text": m = cast(ChatCompletionTextObject, m) @@ -1798,9 +1900,9 @@ def anthropic_messages_pt( # noqa: PLR0915 ) if "cache_control" in _content_element: - _anthropic_content_text_element["cache_control"] = ( - _content_element["cache_control"] - ) + _anthropic_content_text_element[ + "cache_control" + ] = _content_element["cache_control"] user_content.append(_anthropic_content_text_element) @@ -2566,17 +2668,19 @@ class BedrockImageProcessor: """Handles both sync and async image processing for Bedrock conversations.""" @staticmethod - def _post_call_image_processing(response: httpx.Response, image_url: str = "") -> Tuple[str, str]: + def _post_call_image_processing( + response: httpx.Response, image_url: str = "" + ) -> Tuple[str, str]: # Check the response's content type to ensure it is an image content_type = response.headers.get("content-type") - + # Use helper function to infer content type with fallback logic content_type = infer_content_type_from_url_and_content( url=image_url, content=response.content, current_content_type=content_type, ) - + content_type = _parse_content_type(content_type) # Convert the image content to base64 bytes @@ -2595,7 +2699,9 @@ class BedrockImageProcessor: response = await client.get(image_url, follow_redirects=True) response.raise_for_status() # Raise an exception for HTTP errors - return BedrockImageProcessor._post_call_image_processing(response, image_url) + return BedrockImageProcessor._post_call_image_processing( + response, image_url + ) except Exception as e: raise e @@ -2608,7 +2714,9 @@ class BedrockImageProcessor: response = client.get(image_url, follow_redirects=True) response.raise_for_status() # Raise an exception for HTTP errors - return BedrockImageProcessor._post_call_image_processing(response, image_url) + return BedrockImageProcessor._post_call_image_processing( + response, image_url + ) except Exception as e: raise e @@ -2939,21 +3047,33 @@ def _convert_to_bedrock_tool_call_result( """ - """ - content_str: str = "" + tool_result_content_blocks:List[BedrockToolResultContentBlock] = [] if isinstance(message["content"], str): - content_str = message["content"] + tool_result_content_blocks.append(BedrockToolResultContentBlock(text=message["content"])) elif isinstance(message["content"], List): content_list = message["content"] for content in content_list: if content["type"] == "text": - content_str += content["text"] + tool_result_content_blocks.append(BedrockToolResultContentBlock(text=content["text"])) + elif content["type"] == "image_url": + format: Optional[str] = None + if isinstance(content["image_url"], dict): + image_url = content["image_url"]["url"] + format = content["image_url"].get("format") + else: + image_url = content["image_url"] + _block:BedrockContentBlock = BedrockImageProcessor.process_image_sync( + image_url=image_url, + format=format, + ) + if "image" in _block: + tool_result_content_blocks.append(BedrockToolResultContentBlock(image=_block["image"])) message.get("name", "") id = str(message.get("tool_call_id", str(uuid.uuid4()))) - tool_result_content_block = BedrockToolResultContentBlock(text=content_str) tool_result = BedrockToolResultBlock( - content=[tool_result_content_block], + content=tool_result_content_blocks, toolUseId=id, ) @@ -3865,7 +3985,9 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 ) elif element["type"] == "text": # AWS Bedrock doesn't allow empty or whitespace-only text content, so use placeholder for empty strings - text_content = element["text"] if element["text"].strip() else "." + text_content = ( + element["text"] if element["text"].strip() else "." + ) assistants_part = BedrockContentBlock(text=text_content) assistants_parts.append(assistants_part) elif element["type"] == "image_url": diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 2f85c7aef60..ddcf81b5ba5 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -137,6 +137,7 @@ class ChunkProcessor: "name": None, "type": None, "arguments": [], + "provider_specific_fields": None, } if hasattr(tool_call, "id") and tool_call.id: @@ -156,22 +157,48 @@ class ChunkProcessor: tool_call_map[index]["arguments"].append( tool_call.function.arguments ) + + # Preserve provider_specific_fields from streaming chunks + provider_fields = None + if hasattr(tool_call, "provider_specific_fields") and tool_call.provider_specific_fields: + provider_fields = tool_call.provider_specific_fields + elif hasattr(tool_call, "function") and hasattr(tool_call.function, "provider_specific_fields") and tool_call.function.provider_specific_fields: + provider_fields = tool_call.function.provider_specific_fields + + if provider_fields: + # Merge provider_specific_fields if multiple chunks have them + if tool_call_map[index]["provider_specific_fields"] is None: + tool_call_map[index]["provider_specific_fields"] = {} + if isinstance(provider_fields, dict): + tool_call_map[index]["provider_specific_fields"].update( + provider_fields + ) # Convert the map to a list of tool calls for index in sorted(tool_call_map.keys()): tool_call_data = tool_call_map[index] if tool_call_data["id"] and tool_call_data["name"]: combined_arguments = "".join(tool_call_data["arguments"]) or "{}" - tool_calls_list.append( - ChatCompletionMessageToolCall( - id=tool_call_data["id"], - function=Function( - arguments=combined_arguments, - name=tool_call_data["name"], - ), - type=tool_call_data["type"] or "function", - ) + + # Build function - provider_specific_fields should be on tool_call level, not function level + function = Function( + arguments=combined_arguments, + name=tool_call_data["name"], ) + + # Prepare params for ChatCompletionMessageToolCall + tool_call_params = { + "id": tool_call_data["id"], + "function": function, + "type": tool_call_data["type"] or "function", + } + + # Add provider_specific_fields if present (for thought signatures in Gemini 3) + if tool_call_data.get("provider_specific_fields"): + tool_call_params["provider_specific_fields"] = tool_call_data["provider_specific_fields"] + + tool_call = ChatCompletionMessageToolCall(**tool_call_params) + tool_calls_list.append(tool_call) return tool_calls_list diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index a786f06921f..0e905014fe2 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -3,6 +3,7 @@ from typing import ( TYPE_CHECKING, Any, AsyncIterator, + Dict, List, Literal, Optional, @@ -129,6 +130,39 @@ class LiteLLMAnthropicMessagesAdapter: ### FOR [BETA] `/v1/messages` endpoint support + def _extract_signature_from_tool_call( + self, tool_call: Any + ) -> Optional[str]: + """ + Extract signature from a tool call's provider_specific_fields. + Only checks provider_specific_fields, not thinking blocks. + """ + signature = None + + if hasattr(tool_call, "provider_specific_fields") and tool_call.provider_specific_fields: + if "thought_signature" in tool_call.provider_specific_fields: + signature = tool_call.provider_specific_fields["thought_signature"] + elif ( + hasattr(tool_call.function, "provider_specific_fields") + and tool_call.function.provider_specific_fields + ): + if "thought_signature" in tool_call.function.provider_specific_fields: + signature = tool_call.function.provider_specific_fields["thought_signature"] + + return signature + + def _extract_signature_from_tool_use_content( + self, content: Dict[str, Any] + ) -> Optional[str]: + """ + Extract signature from a tool_use content block's provider_specific_fields. + """ + provider_specific_fields = content.get("provider_specific_fields", {}) + if provider_specific_fields: + return provider_specific_fields.get("signature") + return None + + def translatable_anthropic_params(self) -> List: """ Which anthropic params, we need to translate to the openai format. @@ -263,10 +297,18 @@ class LiteLLMAnthropicMessagesAdapter: else: assistant_message_str += content.get("text", "") elif content.get("type") == "tool_use": - function_chunk = ChatCompletionToolCallFunctionChunk( - name=content.get("name", ""), - arguments=json.dumps(content.get("input", {})), - ) + function_chunk: ChatCompletionToolCallFunctionChunk = { + "name": content.get("name", ""), + "arguments": json.dumps(content.get("input", {})), + } + signature = self._extract_signature_from_tool_use_content(content) + + if signature: + provider_specific_fields: Dict[str, Any] = ( + function_chunk.get("provider_specific_fields") or {} + ) + provider_specific_fields["thought_signature"] = signature + function_chunk["provider_specific_fields"] = provider_specific_fields tool_calls.append( ChatCompletionAssistantToolCall( @@ -512,18 +554,27 @@ class LiteLLMAnthropicMessagesAdapter: and len(choice.message.tool_calls) > 0 ): for tool_call in choice.message.tool_calls: - new_content.append( - AnthropicResponseContentBlockToolUse( - type="tool_use", - id=tool_call.id, - name=tool_call.function.name or "", - input=( - json.loads(tool_call.function.arguments) - if tool_call.function.arguments - else {} - ), - ) + # Extract signature from provider_specific_fields only + signature = self._extract_signature_from_tool_call(tool_call) + + provider_specific_fields = {} + if signature: + provider_specific_fields["signature"] = signature + + tool_use_block = AnthropicResponseContentBlockToolUse( + type="tool_use", + id=tool_call.id, + name=tool_call.function.name or "", + input=( + json.loads(tool_call.function.arguments) + if tool_call.function.arguments + else {} + ), ) + # Add provider_specific_fields if signature is present + if provider_specific_fields: + tool_use_block.provider_specific_fields = provider_specific_fields + new_content.append(tool_use_block) # Handle text content elif choice.message.content is not None: new_content.append( diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 53cbafcbe6a..b35e86cabd2 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -51,7 +51,11 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallFunctionChunk, ChatCompletionUsageBlock, ) -from litellm.types.utils import ChatCompletionMessageToolCall, Choices, Delta +from litellm.types.utils import ( + ChatCompletionMessageToolCall, + Choices, + Delta, +) from litellm.types.utils import GenericStreamingChunk as GChunk from litellm.types.utils import ( ModelResponse, @@ -493,9 +497,9 @@ class BedrockLLM(BaseAWSLLM): content=None, ) model_response.choices[0].message = _message # type: ignore - model_response._hidden_params["original_response"] = ( - outputText # allow user to access raw anthropic tool calling response - ) + model_response._hidden_params[ + "original_response" + ] = outputText # allow user to access raw anthropic tool calling response if ( _is_function_call is True and stream is not None @@ -793,9 +797,9 @@ class BedrockLLM(BaseAWSLLM): ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in inference_params[k] = v if stream is True: - inference_params["stream"] = ( - True # cohere requires stream = True in inference params - ) + inference_params[ + "stream" + ] = True # cohere requires stream = True in inference params data = json.dumps({"prompt": prompt, **inference_params}) elif provider == "anthropic": if model.startswith("anthropic.claude-3"): @@ -1184,6 +1188,7 @@ class AWSEventStreamDecoder: self.parser = EventStreamJSONParser() self.content_blocks: List[ContentBlockDeltaEvent] = [] self.tool_calls_index: Optional[int] = None + self.response_id: Optional[str] = None def check_empty_tool_call_args(self) -> bool: """ @@ -1245,8 +1250,169 @@ class AWSEventStreamDecoder: thinking_blocks_list.append(_thinking_block) return thinking_blocks_list + def _initialize_converse_response_id(self, chunk_data: dict): + """Initialize response_id from chunk data if not already set.""" + if self.response_id is None: + if "messageStart" in chunk_data: + conversation_id = chunk_data["messageStart"].get("conversationId") + if conversation_id: + self.response_id = f"chatcmpl-{conversation_id}" + else: + # Fallback to generating a UUID if the first chunk is not messageStart + self.response_id = f"chatcmpl-{uuid.uuid4()}" + + def _handle_converse_start_event( + self, + start_obj: ContentBlockStartEvent, + ) -> Tuple[ + Optional[ChatCompletionToolCallChunk], + dict, + Optional[ + List[ + Union[ + ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock + ] + ] + ], + ]: + """Handle 'start' event in converse chunk parsing.""" + tool_use: Optional[ChatCompletionToolCallChunk] = None + provider_specific_fields: dict = {} + thinking_blocks: Optional[ + List[ + Union[ + ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock + ] + ] + ] = None + + self.content_blocks = [] # reset + if start_obj is not None: + if "toolUse" in start_obj and start_obj["toolUse"] is not None: + ## check tool name was formatted by litellm + _response_tool_name = start_obj["toolUse"]["name"] + response_tool_name = get_bedrock_tool_name( + response_tool_name=_response_tool_name + ) + self.tool_calls_index = ( + 0 + if self.tool_calls_index is None + else self.tool_calls_index + 1 + ) + tool_use = { + "id": start_obj["toolUse"]["toolUseId"], + "type": "function", + "function": { + "name": response_tool_name, + "arguments": "", + }, + "index": self.tool_calls_index, + } + elif ( + "reasoningContent" in start_obj + and start_obj["reasoningContent"] is not None + ): # redacted thinking can be in start object + thinking_blocks = self.translate_thinking_blocks( + start_obj["reasoningContent"] + ) + provider_specific_fields = { + "reasoningContent": start_obj["reasoningContent"], + } + return tool_use, provider_specific_fields, thinking_blocks + + def _handle_converse_delta_event( + self, + delta_obj: ContentBlockDeltaEvent, + index: int, + ) -> Tuple[ + str, + Optional[ChatCompletionToolCallChunk], + dict, + Optional[str], + Optional[ + List[ + Union[ + ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock + ] + ] + ], + ]: + """Handle 'delta' event in converse chunk parsing.""" + text = "" + tool_use: Optional[ChatCompletionToolCallChunk] = None + provider_specific_fields: dict = {} + reasoning_content: Optional[str] = None + thinking_blocks: Optional[ + List[ + Union[ + ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock + ] + ] + ] = None + + self.content_blocks.append(delta_obj) + if "text" in delta_obj: + text = delta_obj["text"] + elif "toolUse" in delta_obj: + tool_use = { + "id": None, + "type": "function", + "function": { + "name": None, + "arguments": delta_obj["toolUse"]["input"], + }, + "index": ( + self.tool_calls_index + if self.tool_calls_index is not None + else index + ), + } + elif "reasoningContent" in delta_obj: + provider_specific_fields = { + "reasoningContent": delta_obj["reasoningContent"], + } + reasoning_content = self.extract_reasoning_content_str( + delta_obj["reasoningContent"] + ) + thinking_blocks = self.translate_thinking_blocks( + delta_obj["reasoningContent"] + ) + if ( + thinking_blocks + and len(thinking_blocks) > 0 + and reasoning_content is None + ): + reasoning_content = "" # set to non-empty string to ensure consistency with Anthropic + return text, tool_use, provider_specific_fields, reasoning_content, thinking_blocks + + def _handle_converse_stop_event( + self, index: int + ) -> Optional[ChatCompletionToolCallChunk]: + """Handle stop/contentBlockIndex event in converse chunk parsing.""" + tool_use: Optional[ChatCompletionToolCallChunk] = None + is_empty = self.check_empty_tool_call_args() + if is_empty: + tool_use = { + "id": None, + "type": "function", + "function": { + "name": None, + "arguments": "{}", + }, + "index": ( + self.tool_calls_index + if self.tool_calls_index is not None + else index + ), + } + return tool_use + def converse_chunk_parser(self, chunk_data: dict) -> ModelResponseStream: try: + # Capture the conversationId from the first messageStart event + # and use it as the consistent ID for all subsequent chunks. + self._initialize_converse_response_id(chunk_data) + verbose_logger.debug("\n\nRaw Chunk: {}\n\n".format(chunk_data)) text = "" tool_use: Optional[ChatCompletionToolCallChunk] = None @@ -1265,91 +1431,22 @@ class AWSEventStreamDecoder: index = int(chunk_data.get("contentBlockIndex", 0)) if "start" in chunk_data: start_obj = ContentBlockStartEvent(**chunk_data["start"]) - self.content_blocks = [] # reset - if start_obj is not None: - if "toolUse" in start_obj and start_obj["toolUse"] is not None: - ## check tool name was formatted by litellm - _response_tool_name = start_obj["toolUse"]["name"] - response_tool_name = get_bedrock_tool_name( - response_tool_name=_response_tool_name - ) - self.tool_calls_index = ( - 0 - if self.tool_calls_index is None - else self.tool_calls_index + 1 - ) - tool_use = { - "id": start_obj["toolUse"]["toolUseId"], - "type": "function", - "function": { - "name": response_tool_name, - "arguments": "", - }, - "index": self.tool_calls_index, - } - elif ( - "reasoningContent" in start_obj - and start_obj["reasoningContent"] is not None - ): # redacted thinking can be in start object - thinking_blocks = self.translate_thinking_blocks( - start_obj["reasoningContent"] - ) - provider_specific_fields = { - "reasoningContent": start_obj["reasoningContent"], - } + tool_use, provider_specific_fields, thinking_blocks = ( + self._handle_converse_start_event(start_obj) + ) elif "delta" in chunk_data: delta_obj = ContentBlockDeltaEvent(**chunk_data["delta"]) - self.content_blocks.append(delta_obj) - if "text" in delta_obj: - text = delta_obj["text"] - elif "toolUse" in delta_obj: - tool_use = { - "id": None, - "type": "function", - "function": { - "name": None, - "arguments": delta_obj["toolUse"]["input"], - }, - "index": ( - self.tool_calls_index - if self.tool_calls_index is not None - else index - ), - } - elif "reasoningContent" in delta_obj: - provider_specific_fields = { - "reasoningContent": delta_obj["reasoningContent"], - } - reasoning_content = self.extract_reasoning_content_str( - delta_obj["reasoningContent"] - ) - thinking_blocks = self.translate_thinking_blocks( - delta_obj["reasoningContent"] - ) - if ( - thinking_blocks - and len(thinking_blocks) > 0 - and reasoning_content is None - ): - reasoning_content = "" # set to non-empty string to ensure consistency with Anthropic + ( + text, + tool_use, + provider_specific_fields, + reasoning_content, + thinking_blocks, + ) = self._handle_converse_delta_event(delta_obj, index) elif ( "contentBlockIndex" in chunk_data ): # stop block, no 'start' or 'delta' object - is_empty = self.check_empty_tool_call_args() - if is_empty: - tool_use = { - "id": None, - "type": "function", - "function": { - "name": None, - "arguments": "{}", - }, - "index": ( - self.tool_calls_index - if self.tool_calls_index is not None - else index - ), - } + tool_use = self._handle_converse_stop_event(index) elif "stopReason" in chunk_data: finish_reason = map_finish_reason(chunk_data.get("stopReason", "stop")) elif "usage" in chunk_data: @@ -1378,6 +1475,7 @@ class AWSEventStreamDecoder: ), ) ], + id=self.response_id, usage=usage, provider_specific_fields=model_response_provider_specific_fields, ) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 30c07ebe7f0..6b0aef31ff6 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1140,6 +1140,7 @@ class BaseLLMHTTPHandler: atranscription: bool = False, headers: Optional[Dict[str, Any]] = None, provider_config: Optional[BaseAudioTranscriptionConfig] = None, + shared_session: Optional["ClientSession"] = None, ) -> Union[TranscriptionResponse, Coroutine[Any, Any, TranscriptionResponse]]: if provider_config is None: raise ValueError( @@ -1162,6 +1163,7 @@ class BaseLLMHTTPHandler: client=client, headers=headers, provider_config=provider_config, + shared_session=shared_session, ) # Prepare the request @@ -1226,6 +1228,7 @@ class BaseLLMHTTPHandler: client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, headers: Optional[Dict[str, Any]] = None, provider_config: Optional[BaseAudioTranscriptionConfig] = None, + shared_session: Optional["ClientSession"] = None, ) -> TranscriptionResponse: if provider_config is None: raise ValueError( @@ -1254,6 +1257,7 @@ class BaseLLMHTTPHandler: async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + shared_session=shared_session, ) else: async_httpx_client = client diff --git a/litellm/llms/docker_model_runner/chat/transformation.py b/litellm/llms/docker_model_runner/chat/transformation.py new file mode 100644 index 00000000000..3d84b24a01c --- /dev/null +++ b/litellm/llms/docker_model_runner/chat/transformation.py @@ -0,0 +1,144 @@ +""" +Translates from OpenAI's `/v1/chat/completions` to Docker Model Runner's `/engines/{engine}/v1/chat/completions` + +Docker Model Runner API Reference: https://docs.docker.com/ai/model-runner/api-reference/ +""" + +from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload + +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + handle_messages_with_content_list_to_str_conversion, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues + +from ...openai.chat.gpt_transformation import OpenAIGPTConfig + + +class DockerModelRunnerChatConfig(OpenAIGPTConfig): + """ + Configuration for Docker Model Runner API. + + Docker Model Runner uses URLs in the format: /engines/{engine}/v1/chat/completions + The engine name (e.g., "llama.cpp") is part of the API endpoint path. + """ + + @overload + def _transform_messages( + self, messages: List[AllMessageValues], model: str, is_async: Literal[True] + ) -> Coroutine[Any, Any, List[AllMessageValues]]: + ... + + @overload + def _transform_messages( + self, + messages: List[AllMessageValues], + model: str, + is_async: Literal[False] = False, + ) -> List[AllMessageValues]: + ... + + def _transform_messages( + self, messages: List[AllMessageValues], model: str, is_async: bool = False + ) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]: + """ + Docker Model Runner is OpenAI-compatible, so we use standard message transformation. + """ + messages = handle_messages_with_content_list_to_str_conversion(messages) + if is_async: + return super()._transform_messages( + messages=messages, model=model, is_async=True + ) + else: + return super()._transform_messages( + messages=messages, model=model, is_async=False + ) + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + """ + Get API base and key for Docker Model Runner. + + Default API base: http://localhost:22088/engines/llama.cpp + The engine path should be included in the api_base. + """ + api_base = ( + api_base + or get_secret_str("DOCKER_MODEL_RUNNER_API_BASE") + or "http://localhost:22088/engines/llama.cpp" + ) # type: ignore + # Docker Model Runner may not require authentication for local instances + dynamic_api_key = api_key or get_secret_str("DOCKER_MODEL_RUNNER_API_KEY") or "dummy-key" + return api_base, dynamic_api_key + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Build the complete URL for Docker Model Runner API. + + Docker Model Runner uses URLs in the format: /engines/{engine}/v1/chat/completions + + The engine name should be specified in the api_base: + - api_base="http://model-runner.docker.internal/engines/llama.cpp" + - Default: "http://localhost:22088/engines/llama.cpp" + + Args: + api_base: Base URL for the Docker Model Runner instance including engine path + api_key: API key (may not be required for local instances) + model: Model name (e.g., "llama-3.1") + optional_params: Optional parameters + litellm_params: LiteLLM parameters + stream: Whether streaming is enabled + + Returns: + Complete URL for the API call + """ + if not api_base: + api_base = "http://localhost:22088/engines/llama.cpp" + + # Remove trailing slashes from api_base + api_base = api_base.rstrip("/") + + # Build the URL: {api_base}/v1/chat/completions + # api_base is expected to already contain the engine path + complete_url = f"{api_base}/v1/chat/completions" + + return complete_url + + def get_supported_openai_params(self, model: str) -> list: + """ + Get the supported OpenAI params for Docker Model Runner. + + Docker Model Runner is OpenAI-compatible and supports standard parameters. + """ + return super().get_supported_openai_params(model=model) + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Map OpenAI parameters to Docker Model Runner parameters. + + Docker Model Runner is OpenAI-compatible, so most parameters map directly. + """ + supported_openai_params = self.get_supported_openai_params(model) + for param, value in non_default_params.items(): + if param == "max_completion_tokens": + optional_params["max_tokens"] = value + elif param in supported_openai_params: + optional_params[param] = value + + return optional_params + diff --git a/litellm/llms/gemini/chat/transformation.py b/litellm/llms/gemini/chat/transformation.py index f7ce03a34d1..c5e2d8b3dac 100644 --- a/litellm/llms/gemini/chat/transformation.py +++ b/litellm/llms/gemini/chat/transformation.py @@ -140,4 +140,4 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig): except Exception: # If conversion fails, leave as is and let the API handle it pass - return _gemini_convert_messages_with_history(messages=messages) + return _gemini_convert_messages_with_history(messages=messages, model=model) diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py index d1ae47af269..ce2519e9177 100644 --- a/litellm/llms/gemini/videos/transformation.py +++ b/litellm/llms/gemini/videos/transformation.py @@ -15,17 +15,16 @@ from litellm.images.utils import ImageEditRequestUtils import litellm from litellm.types.llms.gemini import GeminiLongRunningOperationResponse, GeminiVideoGenerationInstance, GeminiVideoGenerationParameters, GeminiVideoGenerationRequest from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS +from litellm.llms.base_llm.videos.transformation import BaseVideoConfig + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from ...base_llm.videos.transformation import BaseVideoConfig as _BaseVideoConfig from ...base_llm.chat.transformation import BaseLLMException as _BaseLLMException LiteLLMLoggingObj = _LiteLLMLoggingObj - BaseVideoConfig = _BaseVideoConfig BaseLLMException = _BaseLLMException else: LiteLLMLoggingObj = Any - BaseVideoConfig = Any BaseLLMException = Any diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py new file mode 100644 index 00000000000..cc96e3415f3 --- /dev/null +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -0,0 +1,317 @@ +""" +GitHub Copilot Responses API Configuration. + +This module provides the configuration for GitHub Copilot's Responses API, +which is required for models like gpt-5.1-codex that only support the /responses endpoint. + +Implementation based on analysis of the copilot-api project by caozhiyuan: +https://github.com/caozhiyuan/copilot-api +""" +from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from uuid import uuid4 + +from litellm._logging import verbose_logger +from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.exceptions import AuthenticationError +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.types.llms.openai import ( + ResponseInputParam, + ResponsesAPIOptionalRequestParams, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + +from ..authenticator import Authenticator +from ..common_utils import GetAPIKeyError + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + +# GitHub Copilot API Constants (from copilot-api) +COPILOT_VERSION = "0.26.7" +EDITOR_PLUGIN_VERSION = f"copilot-chat/{COPILOT_VERSION}" +USER_AGENT = f"GitHubCopilotChat/{COPILOT_VERSION}" +API_VERSION = "2025-04-01" + + +class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): + """ + Configuration for GitHub Copilot's Responses API. + + Inherits from OpenAIResponsesAPIConfig since GitHub Copilot's Responses API + is compatible with OpenAI's Responses API specification. + + Key differences from OpenAI: + - Uses OAuth Device Flow authentication (handled by Authenticator) + - Uses api.githubcopilot.com as the API base + - Requires specific headers for VSCode/Copilot integration + - Supports vision requests with special header + - Requires X-Initiator header based on input analysis + + Reference: https://api.githubcopilot.com/ + """ + + GITHUB_COPILOT_API_BASE = "https://api.githubcopilot.com" + + def __init__(self) -> None: + super().__init__() + self.authenticator = Authenticator() + + @property + def custom_llm_provider(self) -> LlmProviders: + """Return the GitHub Copilot provider identifier.""" + return LlmProviders.GITHUB_COPILOT + + def get_supported_openai_params(self, model: str) -> list: + """ + Get supported parameters for GitHub Copilot Responses API. + + GitHub Copilot supports all standard OpenAI Responses API parameters. + """ + return super().get_supported_openai_params(model) + + def map_openai_params( + self, + response_api_optional_params: ResponsesAPIOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict: + """ + Map parameters for GitHub Copilot Responses API. + + GitHub Copilot uses the same parameter format as OpenAI, + so no transformation is needed. + """ + return dict(response_api_optional_params) + + def validate_environment( + self, + headers: dict, + model: str, + litellm_params: Optional[GenericLiteLLMParams], + ) -> dict: + """ + Validate environment and set up headers for GitHub Copilot API. + + Uses the Authenticator to obtain GitHub Copilot API key via OAuth Device Flow, + then configures all required headers for the Responses API. + + Headers include: + - Authorization with API key + - Standard GitHub Copilot headers (editor-version, user-agent, etc.) + - X-Initiator based on input analysis + - copilot-vision-request if vision content detected + - User-provided extra_headers (merged with priority) + """ + try: + # Get GitHub Copilot API key via OAuth + api_key = self.authenticator.get_api_key() + + if not api_key: + raise AuthenticationError( + model=model, + llm_provider="github_copilot", + message="GitHub Copilot API key is required. Please authenticate via OAuth Device Flow.", + ) + + # Get default headers (from copilot-api configuration) + default_headers = self._get_default_headers(api_key) + + # Merge with existing headers (user's extra_headers take priority) + merged_headers = {**default_headers, **headers} + + # Analyze input to determine additional headers + input_param = self._get_input_from_params(litellm_params) + + # Add X-Initiator header based on input analysis + if input_param is not None: + initiator = self._get_initiator(input_param) + merged_headers["X-Initiator"] = initiator + verbose_logger.debug( + f"GitHub Copilot Responses API: Set X-Initiator={initiator}" + ) + + # Add vision header if input contains images + if self._has_vision_input(input_param): + merged_headers["copilot-vision-request"] = "true" + verbose_logger.debug( + "GitHub Copilot Responses API: Enabled vision request" + ) + + verbose_logger.debug( + f"GitHub Copilot Responses API: Successfully configured headers for model {model}" + ) + + return merged_headers + + except GetAPIKeyError as e: + raise AuthenticationError( + model=model, + llm_provider="github_copilot", + message=str(e), + ) + + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Get the complete URL for GitHub Copilot Responses API endpoint. + + Returns: https://api.githubcopilot.com/responses + + Note: Currently only supports individual accounts. + Business/enterprise accounts (api.business.githubcopilot.com) can be + added in the future by detecting account type. + """ + # Use provided api_base or fall back to authenticator's base or default + api_base = ( + api_base + or self.authenticator.get_api_base() + or self.GITHUB_COPILOT_API_BASE + ) + + # Remove trailing slashes + api_base = api_base.rstrip("/") + + # Return the responses endpoint + return f"{api_base}/responses" + + # ==================== Helper Methods ==================== + + def _get_default_headers(self, api_key: str) -> Dict[str, str]: + """ + Get default headers for GitHub Copilot Responses API. + + Based on copilot-api's header configuration. + """ + return { + "Authorization": f"Bearer {api_key}", + "content-type": "application/json", + "copilot-integration-id": "vscode-chat", + "editor-version": "vscode/1.95.0", # Fixed version for stability + "editor-plugin-version": EDITOR_PLUGIN_VERSION, + "user-agent": USER_AGENT, + "openai-intent": "conversation-panel", + "x-github-api-version": API_VERSION, + "x-request-id": str(uuid4()), + "x-vscode-user-agent-library-version": "electron-fetch", + } + + def _get_input_from_params( + self, litellm_params: Optional[GenericLiteLLMParams] + ) -> Optional[Union[str, ResponseInputParam]]: + """ + Extract input parameter from litellm_params. + + The input parameter contains the conversation history and is needed + for vision detection and initiator determination. + """ + if litellm_params is None: + return None + + # Try to get input from litellm_params + # This might be in different locations depending on how LiteLLM structures it + if hasattr(litellm_params, "input"): + return litellm_params.input + + # If not found, return None and let the API handle it + return None + + def _get_initiator(self, input_param: Union[str, ResponseInputParam]) -> str: + """ + Determine X-Initiator header value based on input analysis. + + Based on copilot-api's hasAgentInitiator logic: + - Returns "agent" if input contains assistant role or items without role + - Returns "user" otherwise + + Args: + input_param: The input parameter (string or list of input items) + + Returns: + "agent" or "user" + """ + # If input is a string, it's user-initiated + if isinstance(input_param, str): + return "user" + + # If input is a list, analyze items + if isinstance(input_param, list): + for item in input_param: + if not isinstance(item, dict): + continue + + # Check if item has no role (agent-initiated) + if "role" not in item or not item.get("role"): + return "agent" + + # Check if role is assistant (agent-initiated) + role = item.get("role") + if isinstance(role, str) and role.lower() == "assistant": + return "agent" + + # Default to user-initiated + return "user" + + def _has_vision_input(self, input_param: Union[str, ResponseInputParam]) -> bool: + """ + Check if input contains vision content (images). + + Based on copilot-api's hasVisionInput and containsVisionContent logic. + Recursively searches for input_image type in the input structure. + + Args: + input_param: The input parameter to analyze + + Returns: + True if input contains image content, False otherwise + """ + return self._contains_vision_content(input_param) + + def _contains_vision_content( + self, value: Any, depth: int = 0, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH + ) -> bool: + """ + Recursively check if a value contains vision content. + + Looks for items with type="input_image" in the structure. + """ + if depth > max_depth: + verbose_logger.warning( + f"[GitHub Copilot] Max recursion depth {max_depth} reached while checking for vision content" + ) + return False + + if value is None: + return False + + # Check arrays + if isinstance(value, list): + return any( + self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) + for item in value + ) + + # Only check dict/object types + if not isinstance(value, dict): + return False + + # Check if this item is an input_image + item_type = value.get("type") + if isinstance(item_type, str) and item_type.lower() == "input_image": + return True + + # Check content field recursively + if "content" in value and isinstance(value["content"], list): + return any( + self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) + for item in value["content"] + ) + + return False diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index 4d60b8a8310..e241d2c1c7d 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -1,10 +1,13 @@ -from typing import Optional, Union, cast +from typing import TYPE_CHECKING, Optional, Union, cast import httpx from openai import AsyncOpenAI, OpenAI from pydantic import BaseModel import litellm + +if TYPE_CHECKING: + from aiohttp import ClientSession from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.audio_transcription.transformation import ( @@ -89,6 +92,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): client=None, atranscription: bool = False, provider_config: Optional[BaseAudioTranscriptionConfig] = None, + shared_session: Optional["ClientSession"] = None, ) -> TranscriptionResponse: """ Handle audio transcription request @@ -116,6 +120,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): client=client, max_retries=max_retries, logging_obj=logging_obj, + shared_session=shared_session, ) openai_client: OpenAI = self._get_openai_client( # type: ignore @@ -170,6 +175,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): api_base: Optional[str] = None, client=None, max_retries=None, + shared_session: Optional["ClientSession"] = None, ): try: openai_aclient: AsyncOpenAI = self._get_openai_client( # type: ignore @@ -179,6 +185,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): timeout=timeout, max_retries=max_retries, client=client, + shared_session=shared_session, ) ## LOGGING diff --git a/litellm/llms/vertex_ai/context_caching/transformation.py b/litellm/llms/vertex_ai/context_caching/transformation.py index bb40b7665c1..bc5c1b451f1 100644 --- a/litellm/llms/vertex_ai/context_caching/transformation.py +++ b/litellm/llms/vertex_ai/context_caching/transformation.py @@ -173,7 +173,7 @@ def transform_openai_messages_to_gemini_context_caching( supports_system_message=supports_system_message, messages=messages ) - transformed_messages = _gemini_convert_messages_with_history(messages=new_messages) + transformed_messages = _gemini_convert_messages_with_history(messages=new_messages, model=model) model_name = "models/{}".format(model) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index a971dab9426..e4fcd35b954 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -195,6 +195,7 @@ def check_if_part_exists_in_parts( def _gemini_convert_messages_with_history( # noqa: PLR0915 messages: List[AllMessageValues], + model: Optional[str] = None, ) -> List[ContentType]: """ Converts given messages from OpenAI format to Gemini format @@ -379,7 +380,7 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 or assistant_msg.get("function_call") is not None ): # support assistant tool invoke conversion gemini_tool_call_parts = convert_to_gemini_tool_call_invoke( - assistant_msg + assistant_msg, model=model ) ## check if gemini_tool_call already exists in assistant_content for gemini_tool_call_part in gemini_tool_call_parts: diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index d83096b26b0..f1f0a67b9a1 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -81,6 +81,9 @@ from litellm.types.utils import ( TopLogprob, Usage, ) +from litellm.litellm_core_utils.prompt_templates.factory import ( + _encode_tool_call_id_with_signature, +) from litellm.utils import ( CustomStreamWrapper, ModelResponse, @@ -217,12 +220,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): @classmethod def get_config(cls): return super().get_config() - + @staticmethod def _is_gemini_3_or_newer(model: str) -> bool: """ Check if the model is Gemini 3 Pro or newer. - + Gemini 3 models include: - gemini-3-pro-preview - Any future Gemini 3.x models @@ -230,7 +233,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): # Check for Gemini 3 models if "gemini-3" in model: return True - + return False def _supports_penalty_parameters(self, model: str) -> bool: @@ -260,11 +263,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "parallel_tool_calls", "web_search_options", ] - + # Add penalty parameters only for non-preview models if self._supports_penalty_parameters(model): supported_params.extend(["frequency_penalty", "presence_penalty"]) - + if supports_reasoning(model): supported_params.append("reasoning_effort") supported_params.append("thinking") @@ -308,14 +311,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) -> Tuple[dict, Optional[dict]]: """ Extract location configuration from googleMaps tool for Vertex AI toolConfig. - + Supports two interface styles: 1. Nested (recommended): {"enableWidget": "...", "retrievalConfig": {"latitude": ..., "longitude": ...}} 2. Flat (backward compat): {"enableWidget": "...", "latitude": ..., "longitude": ...} - + Args: google_maps_config: The googleMaps tool configuration from LiteLLM - + Returns: Tuple of (cleaned_google_maps_config, retrieval_config): - cleaned_google_maps_config: googleMaps config without location fields @@ -325,7 +328,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): latitude = google_maps_config.get("latitude") longitude = google_maps_config.get("longitude") language_code = google_maps_config.get("languageCode") - + if latitude is not None and longitude is not None: retrieval_config = { "latLng": { @@ -335,21 +338,17 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): } if language_code is not None: retrieval_config["languageCode"] = language_code - + # Remove location fields from tool definition cleaned_config = { k: v for k, v in google_maps_config.items() if k not in ["latitude", "longitude", "languageCode"] } - + return cleaned_config, retrieval_config - - def get_tool_value( - self, - tool: dict, - tool_name: str - ) -> Optional[dict]: + + def get_tool_value(self, tool: dict, tool_name: str) -> Optional[dict]: """ Helper function to get tool value handling both camelCase and underscore_case variants @@ -373,19 +372,19 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): else: return None - def _map_function( # noqa: PLR0915 + def _map_function( # noqa: PLR0915 self, value: List[dict], optional_params: dict ) -> List[Tools]: """ Map OpenAI-style tools/functions to Vertex AI format. - + Args: value: List of tool definitions optional_params: Request-scoped parameters to store retrieval config - + Returns: List of mapped tools in Vertex AI format - + Side effects: May add 'toolConfig' with 'retrievalConfig' to optional_params if googleMaps tools contain location data @@ -432,25 +431,43 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): tool_name = list(tool.keys())[0] if len(tool.keys()) == 1 else None if tool_name and ( - tool_name == "codeExecution" or tool_name == VertexToolName.CODE_EXECUTION.value + tool_name == "codeExecution" + or tool_name == VertexToolName.CODE_EXECUTION.value ): # code_execution maintained for backwards compatibility code_execution = self.get_tool_value(tool, "codeExecution") elif tool_name and tool_name == VertexToolName.GOOGLE_SEARCH.value: - googleSearch = self.get_tool_value(tool, VertexToolName.GOOGLE_SEARCH.value) - elif tool_name and tool_name == VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value: - googleSearchRetrieval = self.get_tool_value(tool, VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value) + googleSearch = self.get_tool_value( + tool, VertexToolName.GOOGLE_SEARCH.value + ) + elif ( + tool_name and tool_name == VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value + ): + googleSearchRetrieval = self.get_tool_value( + tool, VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value + ) elif tool_name and tool_name == VertexToolName.ENTERPRISE_WEB_SEARCH.value: - enterpriseWebSearch = self.get_tool_value(tool, VertexToolName.ENTERPRISE_WEB_SEARCH.value) - elif tool_name and (tool_name == VertexToolName.URL_CONTEXT.value or tool_name == "urlContext"): + enterpriseWebSearch = self.get_tool_value( + tool, VertexToolName.ENTERPRISE_WEB_SEARCH.value + ) + elif tool_name and ( + tool_name == VertexToolName.URL_CONTEXT.value + or tool_name == "urlContext" + ): urlContext = self.get_tool_value(tool, tool_name) elif tool_name and ( - tool_name == VertexToolName.GOOGLE_MAPS.value or tool_name == "google_maps" + tool_name == VertexToolName.GOOGLE_MAPS.value + or tool_name == "google_maps" ): - google_maps_value = self.get_tool_value(tool, VertexToolName.GOOGLE_MAPS.value) - + google_maps_value = self.get_tool_value( + tool, VertexToolName.GOOGLE_MAPS.value + ) + # Extract and transform location configuration for toolConfig if google_maps_value is not None: - googleMaps, google_maps_retrieval_config = self._extract_google_maps_retrieval_config( + ( + googleMaps, + google_maps_retrieval_config, + ) = self._extract_google_maps_retrieval_config( google_maps_config=google_maps_value ) elif openai_function_object is not None: @@ -490,13 +507,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): _tools[VertexToolName.URL_CONTEXT.value] = urlContext if googleMaps is not None: _tools[VertexToolName.GOOGLE_MAPS.value] = googleMaps - + # Add retrieval config to toolConfig if googleMaps has location data if google_maps_retrieval_config is not None: if "toolConfig" not in optional_params: optional_params["toolConfig"] = {} - optional_params["toolConfig"]["retrievalConfig"] = google_maps_retrieval_config - + optional_params["toolConfig"][ + "retrievalConfig" + ] = google_maps_retrieval_config + return [_tools] def _map_response_schema(self, value: dict) -> dict: @@ -599,23 +618,27 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): Map reasoning_effort to thinking_level for Gemini 3+ models. Args: reasoning_effort: The reasoning effort value - model: The model name (for validation, currently unused but kept for consistency) - + model: The model name + Returns: - GeminiThinkingConfig with thinkingLevel set + GeminiThinkingConfig with thinkingLevel and includeThoughts """ if reasoning_effort == "minimal": - return {"thinkingLevel": "low"} + return {"thinkingLevel": "low", "includeThoughts": True} elif reasoning_effort == "low": - return {"thinkingLevel": "low"} + return {"thinkingLevel": "low", "includeThoughts": True} elif reasoning_effort == "medium": - return {"thinkingLevel": "high"} # medium is not out yet + return { + "thinkingLevel": "high", + "includeThoughts": True, + } # medium is not out yet elif reasoning_effort == "high": - return {"thinkingLevel": "high"} + return {"thinkingLevel": "high", "includeThoughts": True} elif reasoning_effort == "disable": - return {"thinkingLevel": "low"} # gemini 3 cannot fully disable thinking, so we use "low" + # Gemini 3 cannot fully disable thinking, so we use "low" but hide thoughts + return {"thinkingLevel": "low", "includeThoughts": False} elif reasoning_effort == "none": - return {"thinkingLevel": "low"} # gemini 3 cannot fully disable thinking, so we use "low" + return {"thinkingLevel": "low", "includeThoughts": False} else: raise ValueError(f"Invalid reasoning effort: {reasoning_effort}") @@ -663,7 +686,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): status_code=400, ) - @staticmethod def _map_thinking_param( thinking_param: AnthropicThinkingParam, @@ -835,9 +857,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if VertexGeminiConfig._is_gemini_3_or_newer(model): optional_params[ "thinkingConfig" - ] = VertexGeminiConfig._map_reasoning_effort_to_thinking_level( - value, model - ) + ] = VertexGeminiConfig._map_reasoning_effort_to_thinking_level( + value, model + ) else: optional_params[ "thinkingConfig" @@ -879,8 +901,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if VertexGeminiConfig._is_gemini_3_or_newer(model): if "temperature" not in optional_params: optional_params["temperature"] = 1.0 - if "thinkingConfig" not in optional_params or "thinkingLevel" not in optional_params.get("thinkingConfig", {}): - thinking_config = optional_params.get("thinkingConfig", {}) + thinking_config = optional_params.get("thinkingConfig", {}) + if ( + "thinkingLevel" not in thinking_config + and "thinkingBudget" not in thinking_config + ): thinking_config["thinkingLevel"] = "low" optional_params["thinkingConfig"] = thinking_config @@ -1147,17 +1172,21 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if "functionCall" in part: _function_chunk: ChatCompletionToolCallFunctionChunk = { "name": part["functionCall"]["name"], - "arguments": json.dumps(part["functionCall"]["args"], ensure_ascii=False), + "arguments": json.dumps( + part["functionCall"]["args"], ensure_ascii=False + ), } # Extract thought signature if present thought_signature = part.get("thoughtSignature") - + if is_function_call is True: function_dict: Dict[str, Any] = dict(_function_chunk) if thought_signature: if "provider_specific_fields" not in function_dict: function_dict["provider_specific_fields"] = {} - function_dict["provider_specific_fields"]["thought_signature"] = thought_signature + function_dict["provider_specific_fields"][ + "thought_signature" + ] = thought_signature function = cast(ChatCompletionToolCallFunctionChunk, function_dict) else: _tool_response_chunk: ChatCompletionToolCallChunk = { @@ -1166,7 +1195,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "function": _function_chunk, "index": cumulative_tool_call_idx, } + # Embed thought signature in ID for OpenAI client compatibility if thought_signature: + _tool_response_chunk[ + "id" + ] = _encode_tool_call_id_with_signature( + _tool_response_chunk["id"], thought_signature + ) _tool_response_chunk["provider_specific_fields"] = { # type: ignore "thought_signature": thought_signature } @@ -1506,7 +1541,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): annotations: List[ChatCompletionAnnotation] = [] - for metadata in grounding_metadata: # Extract groundingSupports - these map text segments to sources grounding_supports = metadata.get("groundingSupports", []) @@ -1527,23 +1561,23 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): segment = support.get("segment", {}) start_index = segment.get("startIndex") end_index = segment.get("endIndex") - + # Get the chunk indices for this support chunk_indices = support.get("groundingChunkIndices", []) - + if start_index is not None and end_index is not None and chunk_indices: # Use the first chunk's URL for the annotation first_chunk_idx = chunk_indices[0] if first_chunk_idx in chunk_to_uri_map: uri_info = chunk_to_uri_map[first_chunk_idx] - + url_citation: ChatCompletionAnnotationURLCitation = { "start_index": start_index, "end_index": end_index, "url": uri_info["url"], "title": uri_info["title"], } - + annotation: ChatCompletionAnnotation = { "type": "url_citation", "url_citation": url_citation, @@ -1643,9 +1677,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): chat_completion_message["reasoning_content"] = reasoning_content if candidate_grounding_metadata: - annotations = VertexGeminiConfig._convert_grounding_metadata_to_annotations( - grounding_metadata=candidate_grounding_metadata, - content_text=content, + annotations = ( + VertexGeminiConfig._convert_grounding_metadata_to_annotations( + grounding_metadata=candidate_grounding_metadata, + content_text=content, + ) ) if annotations: chat_completion_message["annotations"] = annotations # type: ignore @@ -1673,6 +1709,22 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if thinking_blocks is not None: chat_completion_message["thinking_blocks"] = thinking_blocks # type: ignore + # Convert thinking_blocks to reasoning_content for streaming + # This ensures reasoning_content is available in streaming responses + if ( + isinstance(model_response, ModelResponseStream) + and reasoning_content is None + ): + reasoning_content_parts = [] + for block in thinking_blocks: + thinking_text = block.get("thinking") + if thinking_text: + reasoning_content_parts.append(thinking_text) + + if reasoning_content_parts: + reasoning_content = "\n".join(reasoning_content_parts) + chat_completion_message["reasoning_content"] = reasoning_content + if isinstance(model_response, ModelResponseStream): choice = VertexGeminiConfig._create_streaming_choice( chat_completion_message=chat_completion_message, @@ -1852,7 +1904,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): def _transform_messages( self, messages: List[AllMessageValues], model: Optional[str] = None ) -> List[ContentType]: - return _gemini_convert_messages_with_history(messages=messages) + return _gemini_convert_messages_with_history(messages=messages, model=model) def get_error_class( self, error_message: str, status_code: int, headers: Union[Dict, httpx.Headers] @@ -1911,7 +1963,9 @@ async def make_call( ) try: - response = await client.post(api_base, headers=headers, data=data, stream=True, logging_obj=logging_obj) + response = await client.post( + api_base, headers=headers, data=data, stream=True, logging_obj=logging_obj + ) response.raise_for_status() except httpx.HTTPStatusError as e: exception_string = str(await e.response.aread()) @@ -1958,7 +2012,9 @@ def make_sync_call( if client is None: client = HTTPHandler() # Create a new client if none provided - response = client.post(api_base, headers=headers, data=data, stream=True, logging_obj=logging_obj) + response = client.post( + api_base, headers=headers, data=data, stream=True, logging_obj=logging_obj + ) if response.status_code != 200 and response.status_code != 201: raise VertexAIError( @@ -2013,7 +2069,6 @@ class VertexLLM(VertexBase): gemini_api_key: Optional[str] = None, extra_headers: Optional[dict] = None, ) -> CustomStreamWrapper: - should_use_v1beta1_features = self.is_using_v1beta1_features( optional_params=optional_params ) @@ -2050,8 +2105,8 @@ class VertexLLM(VertexBase): **data, vertex_project=vertex_project, vertex_location=vertex_location, - vertex_auth_header=auth_header) # type: ignore - + vertex_auth_header=auth_header, + ) # type: ignore ## LOGGING logging_obj.pre_call( @@ -2144,7 +2199,8 @@ class VertexLLM(VertexBase): **data, vertex_project=vertex_project, vertex_location=vertex_location, - vertex_auth_header=auth_header) # type: ignore + vertex_auth_header=auth_header, + ) # type: ignore _async_client_params = {} if timeout: @@ -2168,7 +2224,10 @@ class VertexLLM(VertexBase): try: response = await client.post( - api_base, headers=headers, json=cast(dict, request_body), logging_obj=logging_obj + api_base, + headers=headers, + json=cast(dict, request_body), + logging_obj=logging_obj, ) # type: ignore response.raise_for_status() except httpx.HTTPStatusError as err: @@ -2322,9 +2381,10 @@ class VertexLLM(VertexBase): ## TRANSFORMATION ## data = sync_transform_request_body( **transform_request_params, - vertex_project=vertex_project, + vertex_project=vertex_project, vertex_location=vertex_location, - vertex_auth_header=auth_header) + vertex_auth_header=auth_header, + ) ## LOGGING logging_obj.pre_call( diff --git a/litellm/llms/vertex_ai/image_edit/__init__.py b/litellm/llms/vertex_ai/image_edit/__init__.py new file mode 100644 index 00000000000..44914e861a7 --- /dev/null +++ b/litellm/llms/vertex_ai/image_edit/__init__.py @@ -0,0 +1,39 @@ +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.vertex_ai.common_utils import VertexAIModelRoute, get_vertex_ai_model_route + +from .cost_calculator import cost_calculator +from .vertex_gemini_transformation import VertexAIGeminiImageEditConfig +from .vertex_imagen_transformation import VertexAIImagenImageEditConfig + +__all__ = [ + "VertexAIGeminiImageEditConfig", + "VertexAIImagenImageEditConfig", + "get_vertex_ai_image_edit_config", + "cost_calculator" +] + + +def get_vertex_ai_image_edit_config(model: str) -> BaseImageEditConfig: + """ + Get the appropriate image edit config for a Vertex AI model. + + Routes to the correct transformation class based on the model type: + - Gemini models use generateContent API (VertexAIGeminiImageEditConfig) + - Imagen models use predict API (VertexAIImagenImageEditConfig) + + Args: + model: The model name (e.g., "gemini-2.5-flash", "imagegeneration@006") + + Returns: + BaseImageEditConfig: The appropriate configuration class + """ + # Determine the model route + model_route = get_vertex_ai_model_route(model) + + if model_route == VertexAIModelRoute.GEMINI: + # Gemini models use generateContent API + return VertexAIGeminiImageEditConfig() + else: + # Default to Imagen for other models (imagegeneration, etc.) + # This includes NON_GEMINI models like imagegeneration@006 + return VertexAIImagenImageEditConfig() diff --git a/litellm/llms/vertex_ai/image_edit/cost_calculator.py b/litellm/llms/vertex_ai/image_edit/cost_calculator.py new file mode 100644 index 00000000000..b346622a336 --- /dev/null +++ b/litellm/llms/vertex_ai/image_edit/cost_calculator.py @@ -0,0 +1,34 @@ +""" +Vertex AI Image Edit Cost Calculator +""" + +from typing import Any + +import litellm +from litellm.types.utils import ImageResponse + + +def cost_calculator( + model: str, + image_response: Any, +) -> float: + """ + Vertex AI image edit cost calculator. + + Mirrors image generation pricing: charge per returned image based on + model metadata (`output_cost_per_image`). + """ + model_info = litellm.get_model_info( + model=model, + custom_llm_provider="vertex_ai", + ) + + output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0 + + if not isinstance(image_response, ImageResponse): + raise ValueError( + f"image_response must be of type ImageResponse got type={type(image_response)}" + ) + + num_images = len(image_response.data or []) + return output_cost_per_image * num_images diff --git a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py new file mode 100644 index 00000000000..469340f6bba --- /dev/null +++ b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py @@ -0,0 +1,263 @@ +import base64 +import json +import os +from io import BufferedReader, BytesIO +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast + +import httpx +from httpx._types import RequestFiles + +import litellm + +from litellm.images.utils import ImageEditRequestUtils +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM +from litellm.secret_managers.main import get_secret_str +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import FileTypes, ImageObject, ImageResponse, OpenAIImage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM): + """ + Vertex AI Gemini Image Edit Configuration + + Uses generateContent API for Gemini models on Vertex AI + """ + SUPPORTED_PARAMS: List[str] = ["size"] + + def __init__(self) -> None: + BaseImageEditConfig.__init__(self) + VertexLLM.__init__(self) + + def get_supported_openai_params(self, model: str) -> List[str]: + return list(self.SUPPORTED_PARAMS) + + def map_openai_params( + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict[str, Any]: + supported_params = self.get_supported_openai_params(model) + filtered_params = { + key: value + for key, value in image_edit_optional_params.items() + if key in supported_params + } + + mapped_params: Dict[str, Any] = {} + + if "size" in filtered_params: + mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio( + filtered_params["size"] # type: ignore[arg-type] + ) + + return mapped_params + + def _resolve_vertex_project(self) -> Optional[str]: + return ( + getattr(self, "_vertex_project", None) + or os.environ.get("VERTEXAI_PROJECT") + or getattr(litellm, "vertex_project", None) + or get_secret_str("VERTEXAI_PROJECT") + ) + + def _resolve_vertex_location(self) -> Optional[str]: + return ( + getattr(self, "_vertex_location", None) + or os.environ.get("VERTEXAI_LOCATION") + or os.environ.get("VERTEX_LOCATION") + or getattr(litellm, "vertex_location", None) + or get_secret_str("VERTEXAI_LOCATION") + or get_secret_str("VERTEX_LOCATION") + ) + + def _resolve_vertex_credentials(self) -> Optional[str]: + return ( + getattr(self, "_vertex_credentials", None) + or os.environ.get("VERTEXAI_CREDENTIALS") + or getattr(litellm, "vertex_credentials", None) + or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + or get_secret_str("VERTEXAI_CREDENTIALS") + ) + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + headers = headers or {} + vertex_project = self._resolve_vertex_project() + vertex_credentials = self._resolve_vertex_credentials() + access_token, _ = self._ensure_access_token( + credentials=vertex_credentials, + project_id=vertex_project, + custom_llm_provider="vertex_ai", + ) + return self.set_headers(access_token, headers) + + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Get the complete URL for Vertex AI Gemini generateContent API + """ + vertex_project = self._resolve_vertex_project() + vertex_location = self._resolve_vertex_location() + + if not vertex_project or not vertex_location: + raise ValueError("vertex_project and vertex_location are required for Vertex AI") + + # Use the model name as provided, handling vertex_ai prefix + model_name = model + if model.startswith("vertex_ai/"): + model_name = model.replace("vertex_ai/", "") + + if api_base: + base_url = api_base.rstrip("/") + else: + base_url = f"https://{vertex_location}-aiplatform.googleapis.com" + + return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:generateContent" + + def transform_image_edit_request( # type: ignore[override] + self, + model: str, + prompt: str, + image: FileTypes, + image_edit_optional_request_params: Dict[str, Any], + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[Dict[str, Any], Optional[RequestFiles]]: + inline_parts = self._prepare_inline_image_parts(image) + if not inline_parts: + raise ValueError("Vertex AI Gemini image edit requires at least one image.") + + # Correct format for Vertex AI Gemini image editing + contents = { + "role": "USER", + "parts": inline_parts + [{"text": prompt}] + } + + request_body: Dict[str, Any] = {"contents": contents} + + # Generation config with proper structure for image editing + generation_config: Dict[str, Any] = { + "response_modalities": ["IMAGE"] + } + + # Add image-specific configuration + image_config: Dict[str, Any] = {} + if "aspectRatio" in image_edit_optional_request_params: + image_config["aspect_ratio"] = image_edit_optional_request_params["aspectRatio"] + + if image_config: + generation_config["image_config"] = image_config + + request_body["generationConfig"] = generation_config + + payload: Any = json.dumps(request_body) + empty_files = cast(RequestFiles, []) + return cast(Tuple[Dict[str, Any], Optional[RequestFiles]], (payload, empty_files)) + + def transform_image_edit_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: Any, + ) -> ImageResponse: + model_response = ImageResponse() + try: + response_json = raw_response.json() + except Exception as exc: + raise self.get_error_class( + error_message=f"Error transforming image edit response: {exc}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + candidates = response_json.get("candidates", []) + data_list: List[ImageObject] = [] + + for candidate in candidates: + content = candidate.get("content", {}) + parts = content.get("parts", []) + for part in parts: + inline_data = part.get("inlineData") + if inline_data and inline_data.get("data"): + data_list.append( + ImageObject( + b64_json=inline_data["data"], + url=None, + ) + ) + + model_response.data = cast(List[OpenAIImage], data_list) + return model_response + + def _map_size_to_aspect_ratio(self, size: str) -> str: + """Map OpenAI size format to Gemini aspect ratio format""" + aspect_ratio_map = { + "1024x1024": "1:1", + "1792x1024": "16:9", + "1024x1792": "9:16", + "1280x896": "4:3", + "896x1280": "3:4", + } + return aspect_ratio_map.get(size, "1:1") + + def _prepare_inline_image_parts( + self, image: Union[FileTypes, List[FileTypes]] + ) -> List[Dict[str, Any]]: + images: List[FileTypes] + if isinstance(image, list): + images = image + else: + images = [image] + + inline_parts: List[Dict[str, Any]] = [] + for img in images: + if img is None: + continue + + mime_type = ImageEditRequestUtils.get_image_content_type(img) + image_bytes = self._read_all_bytes(img) + inline_parts.append( + { + "inlineData": { + "mimeType": mime_type, + "data": base64.b64encode(image_bytes).decode("utf-8"), + } + } + ) + + return inline_parts + + def _read_all_bytes(self, image: FileTypes) -> bytes: + if isinstance(image, bytes): + return image + if isinstance(image, BytesIO): + current_pos = image.tell() + image.seek(0) + data = image.read() + image.seek(current_pos) + return data + if isinstance(image, BufferedReader): + current_pos = image.tell() + image.seek(0) + data = image.read() + image.seek(current_pos) + return data + raise ValueError("Unsupported image type for Vertex AI Gemini image edit.") diff --git a/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py new file mode 100644 index 00000000000..ad650e38499 --- /dev/null +++ b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py @@ -0,0 +1,353 @@ +import base64 +import json +import os +from io import BufferedRandom, BufferedReader, BytesIO +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast + +import httpx +from httpx._types import RequestFiles + +import litellm + +from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM +from litellm.secret_managers.main import get_secret_str +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import FileTypes, ImageObject, ImageResponse, OpenAIImage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): + """ + Vertex AI Imagen Image Edit Configuration + + Uses predict API for Imagen models on Vertex AI + """ + SUPPORTED_PARAMS: List[str] = ["n", "size", "mask"] + + def __init__(self) -> None: + BaseImageEditConfig.__init__(self) + VertexLLM.__init__(self) + + def get_supported_openai_params(self, model: str) -> List[str]: + return list(self.SUPPORTED_PARAMS) + + def map_openai_params( + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict[str, Any]: + supported_params = self.get_supported_openai_params(model) + filtered_params = { + key: value + for key, value in image_edit_optional_params.items() + if key in supported_params + } + + mapped_params: Dict[str, Any] = {} + + # Map OpenAI parameters to Imagen format + if "n" in filtered_params: + mapped_params["sampleCount"] = filtered_params["n"] + + if "size" in filtered_params: + mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio( + filtered_params["size"] # type: ignore[arg-type] + ) + + if "mask" in filtered_params: + mapped_params["mask"] = filtered_params["mask"] + + return mapped_params + + def _resolve_vertex_project(self) -> Optional[str]: + return ( + getattr(self, "_vertex_project", None) + or os.environ.get("VERTEXAI_PROJECT") + or getattr(litellm, "vertex_project", None) + or get_secret_str("VERTEXAI_PROJECT") + ) + + def _resolve_vertex_location(self) -> Optional[str]: + return ( + getattr(self, "_vertex_location", None) + or os.environ.get("VERTEXAI_LOCATION") + or os.environ.get("VERTEX_LOCATION") + or getattr(litellm, "vertex_location", None) + or get_secret_str("VERTEXAI_LOCATION") + or get_secret_str("VERTEX_LOCATION") + ) + + def _resolve_vertex_credentials(self) -> Optional[str]: + return ( + getattr(self, "_vertex_credentials", None) + or os.environ.get("VERTEXAI_CREDENTIALS") + or getattr(litellm, "vertex_credentials", None) + or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + or get_secret_str("VERTEXAI_CREDENTIALS") + ) + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + headers = headers or {} + vertex_project = self._resolve_vertex_project() + vertex_credentials = self._resolve_vertex_credentials() + access_token, _ = self._ensure_access_token( + credentials=vertex_credentials, + project_id=vertex_project, + custom_llm_provider="vertex_ai", + ) + return self.set_headers(access_token, headers) + + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Get the complete URL for Vertex AI Imagen predict API + """ + vertex_project = self._resolve_vertex_project() + vertex_location = self._resolve_vertex_location() + + if not vertex_project or not vertex_location: + raise ValueError("vertex_project and vertex_location are required for Vertex AI") + + # Use the model name as provided, handling vertex_ai prefix + model_name = model + if model.startswith("vertex_ai/"): + model_name = model.replace("vertex_ai/", "") + + if api_base: + base_url = api_base.rstrip("/") + else: + base_url = f"https://{vertex_location}-aiplatform.googleapis.com" + + return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:predict" + + def transform_image_edit_request( # type: ignore[override] + self, + model: str, + prompt: str, + image: FileTypes, + image_edit_optional_request_params: Dict[str, Any], + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[Dict[str, Any], Optional[RequestFiles]]: + # Prepare reference images in the correct Imagen format + reference_images = self._prepare_reference_images(image, image_edit_optional_request_params) + if not reference_images: + raise ValueError("Vertex AI Imagen image edit requires at least one reference image.") + + # Correct Imagen instances format + instances = [ + { + "prompt": prompt, + "referenceImages": reference_images + } + ] + + # Extract OpenAI parameters and set sensible defaults for Vertex AI-specific parameters + sample_count = image_edit_optional_request_params.get("sampleCount", 1) + # Use sensible defaults for Vertex AI-specific parameters (not exposed to users) + edit_mode = "EDIT_MODE_INPAINT_INSERTION" # Default edit mode + base_steps = 50 # Default number of steps + + # Imagen parameters with correct structure + parameters = { + "sampleCount": sample_count, + "editMode": edit_mode, + "editConfig": { + "baseSteps": base_steps + } + } + + # Set default values for Vertex AI-specific parameters (not configurable by users via OpenAI API) + parameters["guidanceScale"] = 7.5 # Default guidance scale + parameters["seed"] = None # Let Vertex AI choose random seed + + request_body: Dict[str, Any] = { + "instances": instances, + "parameters": parameters + } + + payload: Any = json.dumps(request_body) + empty_files = cast(RequestFiles, []) + return cast(Tuple[Dict[str, Any], Optional[RequestFiles]], (payload, empty_files)) + + def transform_image_edit_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: Any, + ) -> ImageResponse: + model_response = ImageResponse() + try: + response_json = raw_response.json() + except Exception as exc: + raise self.get_error_class( + error_message=f"Error transforming image edit response: {exc}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + predictions = response_json.get("predictions", []) + data_list: List[ImageObject] = [] + + for prediction in predictions: + # Imagen returns images as bytesBase64Encoded + if "bytesBase64Encoded" in prediction: + data_list.append( + ImageObject( + b64_json=prediction["bytesBase64Encoded"], + url=None, + ) + ) + + model_response.data = cast(List[OpenAIImage], data_list) + return model_response + + def _map_size_to_aspect_ratio(self, size: str) -> str: + """Map OpenAI size format to Imagen aspect ratio format""" + aspect_ratio_map = { + "1024x1024": "1:1", + "1792x1024": "16:9", + "1024x1792": "9:16", + "1280x896": "4:3", + "896x1280": "3:4", + } + return aspect_ratio_map.get(size, "1:1") + + def _prepare_reference_images( + self, image: Union[FileTypes, List[FileTypes]], + image_edit_optional_request_params: Dict[str, Any] + ) -> List[Dict[str, Any]]: + """ + Prepare reference images in the correct Imagen API format + """ + images: List[FileTypes] + if isinstance(image, list): + images = image + else: + images = [image] + + reference_images: List[Dict[str, Any]] = [] + + for idx, img in enumerate(images): + if img is None: + continue + + image_bytes = self._read_all_bytes(img) + base64_data = base64.b64encode(image_bytes).decode("utf-8") + + # Create reference image structure + reference_image = { + "referenceType": "REFERENCE_TYPE_RAW", + "referenceId": idx + 1, + "referenceImage": { + "bytesBase64Encoded": base64_data + } + } + + reference_images.append(reference_image) + + # Handle mask image if provided (for inpainting) + mask_image = image_edit_optional_request_params.get("mask") + if mask_image is not None: + mask_bytes = self._read_all_bytes(mask_image) + mask_base64 = base64.b64encode(mask_bytes).decode("utf-8") + + mask_reference = { + "referenceType": "REFERENCE_TYPE_MASK", + "referenceId": len(reference_images) + 1, + "referenceImage": { + "bytesBase64Encoded": mask_base64 + }, + "maskImageConfig": { + "maskMode": "MASK_MODE_USER_PROVIDED", + "dilation": 0.03 # Default dilation value (not configurable via OpenAI API) + } + } + reference_images.append(mask_reference) + + return reference_images + + def _read_all_bytes( + self, image: Any, depth: int = 0, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH + ) -> bytes: + if depth > max_depth: + raise ValueError( + f"Max recursion depth {max_depth} reached while reading image bytes for Vertex AI Imagen image edit." + ) + + if isinstance(image, (list, tuple)): + for item in image: + if item is not None: + return self._read_all_bytes(item, depth=depth + 1, max_depth=max_depth) + raise ValueError("Unsupported image type for Vertex AI Imagen image edit.") + + if isinstance(image, dict): + for key in ("data", "bytes", "content"): + if key in image and image[key] is not None: + value = image[key] + if isinstance(value, str): + try: + return base64.b64decode(value) + except Exception: + continue + return self._read_all_bytes(value, depth=depth + 1, max_depth=max_depth) + if "path" in image: + return self._read_all_bytes(image["path"], depth=depth + 1, max_depth=max_depth) + + if isinstance(image, bytes): + return image + if isinstance(image, bytearray): + return bytes(image) + if isinstance(image, BytesIO): + current_pos = image.tell() + image.seek(0) + data = image.read() + image.seek(current_pos) + return data + if isinstance(image, (BufferedReader, BufferedRandom)): + stream_pos: Optional[int] = None + try: + stream_pos = image.tell() + except Exception: + stream_pos = None + if stream_pos is not None: + image.seek(0) + data = image.read() + if stream_pos is not None: + image.seek(stream_pos) + return data + if isinstance(image, (str, Path)): + path_obj = Path(image) + if not path_obj.exists(): + raise ValueError( + f"Mask/image path does not exist for Vertex AI Imagen image edit: {path_obj}" + ) + return path_obj.read_bytes() + if hasattr(image, "read"): + data = image.read() + if isinstance(data, str): + data = data.encode("utf-8") + return data + raise ValueError( + f"Unsupported image type for Vertex AI Imagen image edit. Got type={type(image)}" + ) diff --git a/litellm/main.py b/litellm/main.py index 88c3f7bc55b..1e3826b9a60 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5516,6 +5516,7 @@ def transcription( atranscription = kwargs.pop("atranscription", False) litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore extra_headers = kwargs.get("extra_headers", None) + shared_session = kwargs.get("shared_session", None) kwargs.pop("tags", []) non_default_params = get_non_default_transcription_params(kwargs) @@ -5653,6 +5654,7 @@ def transcription( api_key=api_key, provider_config=provider_config, litellm_params=litellm_params_dict, + shared_session=shared_session, ) elif provider_config is not None: response = base_llm_http_handler.audio_transcriptions( @@ -5679,6 +5681,7 @@ def transcription( custom_llm_provider=custom_llm_provider, headers={}, provider_config=provider_config, + shared_session=shared_session, ) # Calculate and add duration if response is missing it diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index fa13c2f6a0b..fb3d4c91710 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1297,6 +1297,132 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/eu/gpt-5.1": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.38e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/eu/gpt-5.1-chat": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.38e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/eu/gpt-5.1-codex": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.38e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 1.1e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/eu/gpt-5.1-codex-mini": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 2.75e-07, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 2.2e-06, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/eu/gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5.5e-09, "input_cost_per_token": 5.5e-08, @@ -1471,6 +1597,132 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/global/gpt-5.1": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/global/gpt-5.1-chat": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/global/gpt-5.1-codex": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/global/gpt-5.1-codex-mini": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_token": 2.5e-07, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 2e-06, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/gpt-3.5-turbo": { "input_cost_per_token": 5e-07, "litellm_provider": "azure", @@ -1987,6 +2239,68 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/gpt-audio-2025-08-28": { + "input_cost_per_audio_token": 4e-05, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_audio_token": 8e-05, + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "azure/gpt-audio-mini-2025-10-06": { + "input_cost_per_audio_token": 1e-05, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, "azure/gpt-4o-audio-preview-2024-12-17": { "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 2.5e-06, @@ -2100,6 +2414,70 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "azure/gpt-realtime-2025-08-28": { + "cache_creation_input_audio_token_cost": 4e-06, + "cache_read_input_token_cost": 4e-06, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1.6e-05, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "azure/gpt-realtime-mini-2025-10-06": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "azure/gpt-4o-mini-transcribe": { "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 1.25e-06, @@ -2187,6 +2565,155 @@ "/v1/audio/transcriptions" ] }, + "azure/gpt-4o-transcribe-diarize": { + "input_cost_per_audio_token": 6e-06, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "azure", + "max_input_tokens": 16000, + "max_output_tokens": 2000, + "mode": "audio_transcription", + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, + "azure/gpt-5.1-2025-11-13": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true + }, + "azure/gpt-5.1-chat-2025-11-13": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_native_streaming": true, + "supports_parallel_function_calling": false, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": false, + "supports_vision": true + }, + "azure/gpt-5.1-codex-2025-11-13": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/gpt-5.1-codex-mini-2025-11-13": { + "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_priority": 4.5e-08, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_priority": 4.5e-07, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 2e-06, + "output_cost_per_token_priority": 3.6e-06, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/gpt-5": { "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, @@ -2503,6 +3030,132 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/gpt-5.1": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/gpt-5.1-chat": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/gpt-5.1-codex": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/gpt-5.1-codex-mini": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_token": 2.5e-07, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 2e-06, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/gpt-image-1": { "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", @@ -3420,6 +4073,132 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/us/gpt-5.1": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.38e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/us/gpt-5.1-chat": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.38e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/us/gpt-5.1-codex": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.38e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 1.1e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/us/gpt-5.1-codex-mini": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 2.75e-07, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 2.2e-06, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/us/o1-2024-12-17": { "cache_read_input_token_cost": 8.25e-06, "input_cost_per_token": 1.65e-05, @@ -5127,7 +5906,7 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "cerebras/openai/gpt-oss-120b": { + "cerebras/gpt-oss-120b": { "input_cost_per_token": 2.5e-07, "litellm_provider": "cerebras", "max_input_tokens": 131072, @@ -10588,6 +11367,39 @@ "supports_web_search": true, "tpm": 8000000 }, + "gemini-3-pro-image-preview": { + "input_cost_per_image": 0.0011, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 65536, + "mode": "image_generation", + "output_cost_per_image": 0.134, + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_batches": 6e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true + }, "gemini-2.5-flash-lite": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 5e-07, @@ -10994,10 +11806,12 @@ "supports_web_search": true }, "gemini-3-pro-preview": { - "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "vertex_ai-language-models", "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, @@ -11011,10 +11825,60 @@ "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, + "output_cost_per_token_batches": 6e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", - "/v1/completions" + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true + }, + "vertex_ai/gemini-3-pro-preview": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "vertex_ai", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_above_200k_tokens": 1.8e-05, + "output_cost_per_token_batches": 6e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" ], "supported_modalities": [ "text", @@ -12240,6 +13104,39 @@ "supports_web_search": true, "tpm": 8000000 }, + "gemini/gemini-3-pro-image-preview": { + "input_cost_per_image": 0.0011, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "gemini", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 65536, + "mode": "image_generation", + "output_cost_per_image": 0.134, + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_batches": 6e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true + }, "gemini/gemini-2.5-flash-lite": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 5e-07, @@ -12697,9 +13594,11 @@ "tpm": 800000 }, "gemini/gemini-3-pro-preview": { - "cache_read_input_token_cost": 3.125e-07, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "gemini", "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, @@ -12713,11 +13612,13 @@ "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, + "output_cost_per_token_batches": 6e-06, "rpm": 2000, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", - "/v1/completions" + "/v1/completions", + "/v1/batch" ], "supported_modalities": [ "text", @@ -13084,7 +13985,7 @@ "supports_audio_output": false, "supports_function_calling": true, "supports_response_schema": true, - "supports_system_messages": true, + "supports_system_messages": false, "supports_tool_choice": true, "supports_vision": true }, @@ -19142,6 +20043,53 @@ "supports_tool_choice": true, "supports_vision": true }, + "openrouter/google/gemini-3-pro-preview": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "openrouter", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_above_200k_tokens": 1.8e-05, + "output_cost_per_token_batches": 6e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true + }, "openrouter/google/gemini-pro-1.5": { "input_cost_per_image": 0.00265, "input_cost_per_token": 2.5e-06, @@ -21721,6 +22669,20 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true }, + "together_ai/zai-org/GLM-4.6": { + "input_cost_per_token": 0.6e-06, + "litellm_provider": "together_ai", + "max_input_tokens": 200000, + "max_output_tokens": 200000, + "max_tokens": 200000, + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "source": "https://www.together.ai/models/glm-4-6", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "together_ai/moonshotai/Kimi-K2-Instruct-0905": { "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", @@ -23655,6 +24617,26 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "vertex_ai/gemini-2.5-flash-image": { + "litellm_provider": "vertex_ai-language-models", + "mode": "image_generation", + "output_cost_per_image": 0.039, + "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/image-generation#edit-an-image" + }, + "vertex_ai/gemini-3-pro-image-preview": { + "input_cost_per_image": 0.0011, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 65536, + "mode": "image_generation", + "output_cost_per_image": 0.134, + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_batches": 6e-06, + "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image" + }, "vertex_ai/imagegeneration@006": { "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", @@ -23679,6 +24661,12 @@ "output_cost_per_image": 0.04, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, + "vertex_ai/imagen-3.0-capability-001": { + "litellm_provider": "vertex_ai-image-models", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/image/edit-insert-objects" + }, "vertex_ai/imagen-4.0-fast-generate-001": { "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", @@ -25191,6 +26179,104 @@ "supports_tool_choice": true, "supports_web_search": true }, + "xai/grok-4-1-fast": { + "cache_read_input_token_cost": 0.05e-06, + "input_cost_per_token": 0.2e-06, + "input_cost_per_token_above_128k_tokens": 0.4e-06, + "litellm_provider": "xai", + "max_input_tokens": 2e6, + "max_output_tokens": 2e6, + "max_tokens": 2e6, + "mode": "chat", + "output_cost_per_token": 0.5e-06, + "output_cost_per_token_above_128k_tokens": 1e-06, + "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", + "supports_audio_input": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "xai/grok-4-1-fast-reasoning": { + "cache_read_input_token_cost": 0.05e-06, + "input_cost_per_token": 0.2e-06, + "input_cost_per_token_above_128k_tokens": 0.4e-06, + "litellm_provider": "xai", + "max_input_tokens": 2e6, + "max_output_tokens": 2e6, + "max_tokens": 2e6, + "mode": "chat", + "output_cost_per_token": 0.5e-06, + "output_cost_per_token_above_128k_tokens": 1e-06, + "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", + "supports_audio_input": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "xai/grok-4-1-fast-reasoning-latest": { + "cache_read_input_token_cost": 0.05e-06, + "input_cost_per_token": 0.2e-06, + "input_cost_per_token_above_128k_tokens": 0.4e-06, + "litellm_provider": "xai", + "max_input_tokens": 2e6, + "max_output_tokens": 2e6, + "max_tokens": 2e6, + "mode": "chat", + "output_cost_per_token": 0.5e-06, + "output_cost_per_token_above_128k_tokens": 1e-06, + "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", + "supports_audio_input": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "xai/grok-4-1-fast-non-reasoning": { + "cache_read_input_token_cost": 0.05e-06, + "input_cost_per_token": 0.2e-06, + "input_cost_per_token_above_128k_tokens": 0.4e-06, + "litellm_provider": "xai", + "max_input_tokens": 2e6, + "max_output_tokens": 2e6, + "max_tokens": 2e6, + "mode": "chat", + "output_cost_per_token": 0.5e-06, + "output_cost_per_token_above_128k_tokens": 1e-06, + "source": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", + "supports_audio_input": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "xai/grok-4-1-fast-non-reasoning-latest": { + "cache_read_input_token_cost": 0.05e-06, + "input_cost_per_token": 0.2e-06, + "input_cost_per_token_above_128k_tokens": 0.4e-06, + "litellm_provider": "xai", + "max_input_tokens": 2e6, + "max_output_tokens": 2e6, + "max_tokens": 2e6, + "mode": "chat", + "output_cost_per_token": 0.5e-06, + "output_cost_per_token_above_128k_tokens": 1e-06, + "source": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", + "supports_audio_input": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "xai/grok-beta": { "input_cost_per_token": 5e-06, "litellm_provider": "xai", diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 94bfb9a5002..f28c9b5acbe 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -16,10 +16,20 @@ from urllib.parse import urlparse from fastapi import HTTPException from httpx import HTTPStatusError -from mcp.types import CallToolRequestParams as MCPCallToolRequestParams +from mcp import ReadResourceResult, Resource +from mcp.types import ( + CallToolRequestParams as MCPCallToolRequestParams, + GetPromptRequestParams, + GetPromptResult, + Prompt, + ResourceTemplate, +) from mcp.types import CallToolResult from mcp.types import Tool as MCPTool +from pydantic import AnyUrl + +import litellm from litellm._logging import verbose_logger from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.experimental_mcp_client.client import MCPClient @@ -28,11 +38,11 @@ from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) from litellm.proxy._experimental.mcp_server.utils import ( - add_server_prefix_to_tool_name, - get_server_name_prefix_tool_mcp, + add_server_prefix_to_name, get_server_prefix, is_tool_name_prefixed, normalize_server_name, + split_server_prefix_from_name, validate_mcp_server_name, ) from litellm.proxy._types import ( @@ -356,7 +366,7 @@ class MCPServerManager: base_tool_name = operation_id.replace(" ", "_").lower() # Add server prefix to tool name - prefixed_tool_name = add_server_prefix_to_tool_name( + prefixed_tool_name = add_server_prefix_to_name( base_tool_name, server_prefix ) @@ -385,12 +395,12 @@ class MCPServerManager: ) # Update tool name to server name mapping (for both prefixed and base names) - self.tool_name_to_mcp_server_name_mapping[ - base_tool_name - ] = server_prefix - self.tool_name_to_mcp_server_name_mapping[ - prefixed_tool_name - ] = server_prefix + self.tool_name_to_mcp_server_name_mapping[base_tool_name] = ( + server_prefix + ) + self.tool_name_to_mcp_server_name_mapping[prefixed_tool_name] = ( + server_prefix + ) registered_count += 1 verbose_logger.debug( @@ -715,6 +725,190 @@ class MCPServerManager: ) return [] + async def get_prompts_from_server( + self, + server: MCPServer, + mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, + extra_headers: Optional[Dict[str, str]] = None, + add_prefix: bool = True, + ) -> List[Prompt]: + """ + Helper method to get prompts from a single MCP server with prefixed names. + + Args: + server (MCPServer): The server to query prompts from + mcp_auth_header: Optional auth header for MCP server + + Returns: + List[Prompt]: List of prompts available on the server with prefixed names + """ + + verbose_logger.debug(f"Connecting to url: {server.url}") + verbose_logger.info(f"get_prompts_from_server for {server.name}...") + + client = None + + try: + if server.static_headers: + if extra_headers is None: + extra_headers = {} + extra_headers.update(server.static_headers) + + client = self._create_mcp_client( + server=server, + mcp_auth_header=mcp_auth_header, + extra_headers=extra_headers, + ) + + prompts = await client.list_prompts() + + prefixed_or_original_prompts = self._create_prefixed_prompts( + prompts, server, add_prefix=add_prefix + ) + + return prefixed_or_original_prompts + + except Exception as e: + verbose_logger.warning( + f"Failed to get prompts from server {server.name}: {str(e)}" + ) + return [] + + async def get_resources_from_server( + self, + server: MCPServer, + mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, + extra_headers: Optional[Dict[str, str]] = None, + add_prefix: bool = True, + ) -> List[Resource]: + """Fetch available resources from a single MCP server.""" + + verbose_logger.debug(f"Connecting to url: {server.url}") + verbose_logger.info(f"get_resources_from_server for {server.name}...") + + client = None + + try: + if server.static_headers: + if extra_headers is None: + extra_headers = {} + extra_headers.update(server.static_headers) + + client = self._create_mcp_client( + server=server, + mcp_auth_header=mcp_auth_header, + extra_headers=extra_headers, + ) + + resources = await client.list_resources() + + prefixed_resources = self._create_prefixed_resources( + resources, server, add_prefix=add_prefix + ) + + return prefixed_resources + + except Exception as e: + verbose_logger.warning( + f"Failed to get resources from server {server.name}: {str(e)}" + ) + return [] + + async def get_resource_templates_from_server( + self, + server: MCPServer, + mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, + extra_headers: Optional[Dict[str, str]] = None, + add_prefix: bool = True, + ) -> List[ResourceTemplate]: + """Fetch available resource templates from a single MCP server.""" + + verbose_logger.debug(f"Connecting to url: {server.url}") + verbose_logger.info(f"get_resource_templates_from_server for {server.name}...") + + client = None + + try: + if server.static_headers: + if extra_headers is None: + extra_headers = {} + extra_headers.update(server.static_headers) + + client = self._create_mcp_client( + server=server, + mcp_auth_header=mcp_auth_header, + extra_headers=extra_headers, + ) + + resource_templates = await client.list_resource_templates() + + prefixed_templates = self._create_prefixed_resource_templates( + resource_templates, server, add_prefix=add_prefix + ) + + return prefixed_templates + + except Exception as e: + verbose_logger.warning( + f"Failed to get resource templates from server {server.name}: {str(e)}" + ) + return [] + + async def read_resource_from_server( + self, + server: MCPServer, + url: AnyUrl, + mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, + extra_headers: Optional[Dict[str, str]] = None, + ) -> ReadResourceResult: + """Read resource contents from a specific MCP server.""" + + verbose_logger.debug(f"Connecting to url: {server.url}") + verbose_logger.info(f"read_resource_from_server for {server.name}...") + + if server.static_headers: + if extra_headers is None: + extra_headers = {} + extra_headers.update(server.static_headers) + + client = self._create_mcp_client( + server=server, + mcp_auth_header=mcp_auth_header, + extra_headers=extra_headers, + ) + + return await client.read_resource(url) + + async def get_prompt_from_server( + self, + server: MCPServer, + prompt_name: str, + arguments: Optional[Dict[str, Any]] = None, + mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, + extra_headers: Optional[Dict[str, str]] = None, + ) -> GetPromptResult: + """Fetch a specific prompt definition from a single MCP server.""" + + verbose_logger.debug(f"Connecting to url: {server.url}") + verbose_logger.info(f"get_prompt_from_server for {server.name}...") + + if server.static_headers: + if extra_headers is None: + extra_headers = {} + extra_headers.update(server.static_headers) + + client = self._create_mcp_client( + server=server, + mcp_auth_header=mcp_auth_header, + extra_headers=extra_headers, + ) + + get_prompt_request_params = GetPromptRequestParams( + name=prompt_name, + arguments=arguments, + ) + return await client.get_prompt(get_prompt_request_params) + async def _descovery_metadata( self, server_url: str, @@ -1025,7 +1219,7 @@ class MCPServerManager: prefix = get_server_prefix(server) for tool in tools: - prefixed_name = add_server_prefix_to_tool_name(tool.name, prefix) + prefixed_name = add_server_prefix_to_name(tool.name, prefix) name_to_use = prefixed_name if add_prefix else tool.name @@ -1045,6 +1239,82 @@ class MCPServerManager: ) return prefixed_tools + def _create_prefixed_prompts( + self, prompts: List[Prompt], server: MCPServer, add_prefix: bool = True + ) -> List[Prompt]: + """ + Create prefixed prompts and update prompt mapping. + + Args: + prompts: List of original prompts from server + server: Server instance + + Returns: + List of prompts with prefixed names + """ + prefixed_prompts = [] + prefix = get_server_prefix(server) + + for prompt in prompts: + prefixed_name = add_server_prefix_to_name(prompt.name, prefix) + + name_to_use = prefixed_name if add_prefix else prompt.name + + prompt.name = name_to_use + prefixed_prompts.append(prompt) + + verbose_logger.info( + f"Successfully fetched {len(prefixed_prompts)} prompts from server {server.name}" + ) + return prefixed_prompts + + def _create_prefixed_resources( + self, resources: List[Resource], server: MCPServer, add_prefix: bool = True + ) -> List[Resource]: + """Prefix resource names and track origin server for read requests.""" + + prefixed_resources: List[Resource] = [] + prefix = get_server_prefix(server) + + for resource in resources: + name_to_use = ( + add_server_prefix_to_name(resource.name, prefix) + if add_prefix + else resource.name + ) + resource.name = name_to_use + prefixed_resources.append(resource) + + verbose_logger.info( + f"Successfully fetched {len(prefixed_resources)} resources from server {server.name}" + ) + return prefixed_resources + + def _create_prefixed_resource_templates( + self, + resource_templates: List[ResourceTemplate], + server: MCPServer, + add_prefix: bool = True, + ) -> List[ResourceTemplate]: + """Prefix resource template names for multi-server scenarios.""" + + prefixed_templates: List[ResourceTemplate] = [] + prefix = get_server_prefix(server) + + for resource_template in resource_templates: + name_to_use = ( + add_server_prefix_to_name(resource_template.name, prefix) + if add_prefix + else resource_template.name + ) + resource_template.name = name_to_use + prefixed_templates.append(resource_template) + + verbose_logger.info( + f"Successfully fetched {len(prefixed_templates)} resource templates from server {server.name}" + ) + return prefixed_templates + def check_allowed_or_banned_tools(self, tool_name: str, server: MCPServer) -> bool: """ Check if the tool is allowed or banned for the given server @@ -1079,7 +1349,7 @@ class MCPServerManager: HTTPException: If allowed_params is configured for this tool but arguments contain disallowed params """ from litellm.proxy._experimental.mcp_server.utils import ( - get_server_name_prefix_tool_mcp, + split_server_prefix_from_name, ) # If no allowed_params configured, return all arguments @@ -1087,7 +1357,7 @@ class MCPServerManager: return # Get the unprefixed tool name to match against config - unprefixed_tool_name, _ = get_server_name_prefix_tool_mcp(tool_name) + unprefixed_tool_name, _ = split_server_prefix_from_name(tool_name) # Check both prefixed and unprefixed tool names allowed_params_list = server.allowed_params.get( @@ -1488,7 +1758,7 @@ class MCPServerManager: start_time = datetime.datetime.now() # Get the MCP server - prefixed_tool_name = add_server_prefix_to_tool_name(name, server_name) + prefixed_tool_name = add_server_prefix_to_name(name, server_name) mcp_server = self._get_mcp_server_from_tool_name(prefixed_tool_name) if mcp_server is None: raise ValueError(f"Tool {name} not found") @@ -1594,7 +1864,7 @@ class MCPServerManager: for tool in tools: # The tool.name here is already prefixed from _get_tools_from_server # Extract original name for mapping - original_name, _ = get_server_name_prefix_tool_mcp(tool.name) + original_name, _ = split_server_prefix_from_name(tool.name) self.tool_name_to_mcp_server_name_mapping[original_name] = server.name self.tool_name_to_mcp_server_name_mapping[tool.name] = server.name @@ -1622,7 +1892,7 @@ class MCPServerManager: ( original_tool_name, server_name_from_prefix, - ) = get_server_name_prefix_tool_mcp(tool_name) + ) = split_server_prefix_from_name(tool_name) if original_tool_name in self.tool_name_to_mcp_server_name_mapping: for server in self.get_registry().values(): if normalize_server_name(server.name) == normalize_server_name( @@ -1668,11 +1938,16 @@ class MCPServerManager: return server return None - def get_mcp_servers_from_ids(self, server_ids: List[str]) -> List[MCPServer]: - servers = [] - registry = self.get_registry() - for server in registry.values(): - if server.server_id in server_ids: + def get_public_mcp_servers(self) -> List[MCPServer]: + """ + Get the public MCP servers + """ + servers: List[MCPServer] = [] + if litellm.public_mcp_servers is None: + return servers + for server_id in litellm.public_mcp_servers: + server = self.get_mcp_server_by_id(server_id) + if server: servers.append(server) return servers diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 019e55b9104..4e0a3e258cb 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -8,7 +8,15 @@ from datetime import datetime from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Union from fastapi import FastAPI, HTTPException -from pydantic import ConfigDict +from mcp import ReadResourceResult, Resource +from mcp.server.lowlevel.helper_types import ReadResourceContents +from mcp.types import ( + BlobResourceContents, + GetPromptResult, + ResourceTemplate, + TextResourceContents, +) +from pydantic import AnyUrl, ConfigDict from starlette.types import Receive, Scope, Send from litellm._logging import verbose_logger @@ -54,6 +62,7 @@ if MCP_AVAILABLE: from mcp.server.streamable_http_manager import StreamableHTTPSessionManager from mcp.types import EmbeddedResource, ImageContent, TextContent from mcp.types import Tool as MCPTool + from mcp.types import Prompt from litellm.proxy._experimental.mcp_server.auth.litellm_auth_handler import ( MCPAuthenticatedUser, @@ -66,7 +75,7 @@ if MCP_AVAILABLE: global_mcp_tool_registry, ) from litellm.proxy._experimental.mcp_server.utils import ( - get_server_name_prefix_tool_mcp, + split_server_prefix_from_name, ) ###################################################### @@ -303,6 +312,208 @@ if MCP_AVAILABLE: return response + @server.list_prompts() + async def list_prompts() -> List[Prompt]: + """ + List all available prompts + """ + try: + # Get user authentication from context variable + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + ) = get_auth_context() + verbose_logger.debug( + f"MCP list_prompts - User API Key Auth from context: {user_api_key_auth}" + ) + verbose_logger.debug( + f"MCP list_prompts - MCP servers from context: {mcp_servers}" + ) + verbose_logger.debug( + f"MCP list_prompts - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + ) + # Get mcp_servers from context variable + verbose_logger.debug("MCP list_prompts - Calling _list_prompts") + prompts = await _list_mcp_prompts( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + verbose_logger.info( + f"MCP list_prompts - Successfully returned {len(prompts)} prompts" + ) + return prompts + except Exception as e: + verbose_logger.exception(f"Error in list_prompts endpoint: {str(e)}") + # Return empty list instead of failing completely + # This prevents the HTTP stream from failing and allows the client to get a response + return [] + + @server.get_prompt() + async def get_prompt( + name: str, arguments: dict[str, str] | None + ) -> GetPromptResult: + """ + Get a specific prompt with the provided arguments + + Args: + name (str): Name of the prompt to get + arguments (Dict[str, Any] | None): Arguments to pass to the prompt + + Returns: + GetPromptResult: Getting prompt execution results + """ + + # Validate arguments + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + ) = get_auth_context() + + verbose_logger.debug( + f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}" + ) + return await mcp_get_prompt( + name=name, + arguments=arguments, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + @server.list_resources() + async def list_resources() -> List[Resource]: + """List all available resources.""" + try: + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + ) = get_auth_context() + verbose_logger.debug( + f"MCP list_resources - User API Key Auth from context: {user_api_key_auth}" + ) + verbose_logger.debug( + f"MCP list_resources - MCP servers from context: {mcp_servers}" + ) + verbose_logger.debug( + f"MCP list_resources - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + ) + + resources = await _list_mcp_resources( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + verbose_logger.info( + f"MCP list_resources - Successfully returned {len(resources)} resources" + ) + return resources + except Exception as e: + verbose_logger.exception(f"Error in list_resources endpoint: {str(e)}") + return [] + + @server.list_resource_templates() + async def list_resource_templates() -> List[ResourceTemplate]: + """List all available resource templates.""" + try: + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + ) = get_auth_context() + verbose_logger.debug( + f"MCP list_resource_templates - User API Key Auth from context: {user_api_key_auth}" + ) + verbose_logger.debug( + f"MCP list_resource_templates - MCP servers from context: {mcp_servers}" + ) + verbose_logger.debug( + f"MCP list_resource_templates - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" + ) + + resource_templates = await _list_mcp_resource_templates( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + verbose_logger.info( + "MCP list_resource_templates - Successfully returned " + f"{len(resource_templates)} resource templates" + ) + return resource_templates + except Exception as e: + verbose_logger.exception( + f"Error in list_resource_templates endpoint: {str(e)}" + ) + return [] + + @server.read_resource() + async def read_resource(url: AnyUrl) -> list[ReadResourceContents]: + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + ) = get_auth_context() + + read_resource_result = await mcp_read_resource( + url=url, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + normalized_contents: List[ReadResourceContents] = [] + for content in read_resource_result.contents: + if isinstance(content, TextResourceContents): + normalized_contents.append( + ReadResourceContents( + content=content.text, + mime_type=content.mimeType, + ) + ) + elif isinstance(content, BlobResourceContents): + normalized_contents.append( + ReadResourceContents( + content=content.blob, + mime_type=None, + ) + ) + + return normalized_contents + ######################################################## ############ End of MCP Server Routes ################## ######################################################## @@ -379,7 +590,7 @@ if MCP_AVAILABLE: True if the tool name (prefixed or unprefixed) is in the filter list """ from litellm.proxy._experimental.mcp_server.utils import ( - get_server_name_prefix_tool_mcp, + split_server_prefix_from_name, ) # Check if the full name is in the list @@ -387,7 +598,7 @@ if MCP_AVAILABLE: return True # Check if the unprefixed name is in the list - unprefixed_name, _ = get_server_name_prefix_tool_mcp(tool_name) + unprefixed_name, _ = split_server_prefix_from_name(tool_name) return unprefixed_name in filter_list def filter_tools_by_allowed_tools( @@ -428,6 +639,56 @@ if MCP_AVAILABLE: return tools_to_return + async def _get_allowed_mcp_servers( + user_api_key_auth: Optional[UserAPIKeyAuth], + mcp_servers: Optional[List[str]], + ) -> List[MCPServer]: + """Return allowed MCP servers for a request after applying filters.""" + allowed_mcp_server_ids = ( + await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth) + ) + allowed_mcp_servers = global_mcp_server_manager.get_mcp_servers_from_ids( # type: ignore[attr-defined] + allowed_mcp_server_ids + ) + + if mcp_servers is not None: + allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names( + mcp_servers=mcp_servers, + allowed_mcp_servers=allowed_mcp_servers, + ) + + return allowed_mcp_servers + + def _prepare_mcp_server_headers( + server: MCPServer, + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]], + mcp_auth_header: Optional[str], + oauth2_headers: Optional[Dict[str, str]], + raw_headers: Optional[Dict[str, str]], + ) -> Tuple[Optional[Union[Dict[str, str], str]], Optional[Dict[str, str]]]: + """Build auth and extra headers for a server.""" + server_auth_header: Optional[Union[Dict[str, str], str]] = None + if mcp_server_auth_headers and server.alias is not None: + server_auth_header = mcp_server_auth_headers.get(server.alias) + elif mcp_server_auth_headers and server.server_name is not None: + server_auth_header = mcp_server_auth_headers.get(server.server_name) + + extra_headers: Optional[Dict[str, str]] = None + if server.auth_type == MCPAuth.oauth2: + extra_headers = oauth2_headers + + if server.extra_headers and raw_headers: + if extra_headers is None: + extra_headers = {} + for header in server.extra_headers: + if header in raw_headers: + extra_headers[header] = raw_headers[header] + + if server_auth_header is None: + server_auth_header = mcp_auth_header + + return server_auth_header, extra_headers + async def _get_tools_from_mcp_servers( user_api_key_auth: Optional[UserAPIKeyAuth], mcp_auth_header: Optional[str], @@ -452,19 +713,10 @@ if MCP_AVAILABLE: if not MCP_AVAILABLE: return [] - # Get allowed MCP servers based on user permissions - allowed_mcp_server_ids = ( - await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth) + allowed_mcp_servers = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, ) - allowed_mcp_servers = global_mcp_server_manager.get_mcp_servers_from_ids( - allowed_mcp_server_ids - ) - - if mcp_servers is not None: - allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names( - mcp_servers=mcp_servers, - allowed_mcp_servers=allowed_mcp_servers, - ) # Decide whether to add prefix based on number of allowed servers add_prefix = not (len(allowed_mcp_servers) == 1) @@ -475,27 +727,13 @@ if MCP_AVAILABLE: if server is None: continue - # Get server-specific auth header if available - server_auth_header: Optional[Union[Dict[str, str], str]] = None - if mcp_server_auth_headers and server.alias is not None: - server_auth_header = mcp_server_auth_headers.get(server.alias) - elif mcp_server_auth_headers and server.server_name is not None: - server_auth_header = mcp_server_auth_headers.get(server.server_name) - - extra_headers: Optional[Dict[str, str]] = None - if server.auth_type == MCPAuth.oauth2: - extra_headers = oauth2_headers - - if server.extra_headers and raw_headers: - if extra_headers is None: - extra_headers = {} - for header in server.extra_headers: - if header in raw_headers: - extra_headers[header] = raw_headers[header] - - # Fall back to deprecated mcp_auth_header if no server-specific header found - if server_auth_header is None: - server_auth_header = mcp_auth_header + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) try: tools = await global_mcp_server_manager._get_tools_from_server( @@ -530,6 +768,195 @@ if MCP_AVAILABLE: return all_tools + async def _get_prompts_from_mcp_servers( + user_api_key_auth: Optional[UserAPIKeyAuth], + mcp_auth_header: Optional[str], + mcp_servers: Optional[List[str]], + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + ) -> List[Prompt]: + """ + Helper method to fetch prompt from MCP servers based on server filtering criteria. + + Args: + user_api_key_auth: User authentication info for access control + mcp_auth_header: Optional auth header for MCP server (deprecated) + mcp_servers: Optional list of server names/aliases to filter by + mcp_server_auth_headers: Optional dict of server-specific auth headers + oauth2_headers: Optional dict of oauth2 headers + + Returns: + List[Prompt]: Combined list of prompts from filtered servers + """ + if not MCP_AVAILABLE: + return [] + + allowed_mcp_servers = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, + ) + + # Decide whether to add prefix based on number of allowed servers + add_prefix = not (len(allowed_mcp_servers) == 1) + + # Get prompts from each allowed server + all_prompts = [] + for server in allowed_mcp_servers: + if server is None: + continue + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + try: + prompts = await global_mcp_server_manager.get_prompts_from_server( + server=server, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + add_prefix=add_prefix, + ) + + all_prompts.extend(prompts) + + verbose_logger.debug( + f"Successfully fetched {len(prompts)} prompts from server {server.name}" + ) + except Exception as e: + verbose_logger.exception( + f"Error getting prompts from server {server.name}: {str(e)}" + ) + # Continue with other servers instead of failing completely + + verbose_logger.info( + f"Successfully fetched {len(all_prompts)} prompts total from all MCP servers" + ) + + return all_prompts + + async def _get_resources_from_mcp_servers( + user_api_key_auth: Optional[UserAPIKeyAuth], + mcp_auth_header: Optional[str], + mcp_servers: Optional[List[str]], + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + ) -> List[Resource]: + """Fetch resources from allowed MCP servers.""" + + if not MCP_AVAILABLE: + return [] + + allowed_mcp_servers = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, + ) + + add_prefix = not (len(allowed_mcp_servers) == 1) + + all_resources: List[Resource] = [] + for server in allowed_mcp_servers: + if server is None: + continue + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + try: + resources = await global_mcp_server_manager.get_resources_from_server( + server=server, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + add_prefix=add_prefix, + ) + all_resources.extend(resources) + + verbose_logger.debug( + f"Successfully fetched {len(resources)} resources from server {server.name}" + ) + except Exception as e: + verbose_logger.exception( + f"Error getting resources from server {server.name}: {str(e)}" + ) + + verbose_logger.info( + f"Successfully fetched {len(all_resources)} resources total from all MCP servers" + ) + + return all_resources + + async def _get_resource_templates_from_mcp_servers( + user_api_key_auth: Optional[UserAPIKeyAuth], + mcp_auth_header: Optional[str], + mcp_servers: Optional[List[str]], + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + ) -> List[ResourceTemplate]: + """Fetch resource templates from allowed MCP servers.""" + + if not MCP_AVAILABLE: + return [] + + allowed_mcp_servers = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, + ) + + add_prefix = not (len(allowed_mcp_servers) == 1) + + all_resource_templates: List[ResourceTemplate] = [] + for server in allowed_mcp_servers: + if server is None: + continue + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + try: + resource_templates = ( + await global_mcp_server_manager.get_resource_templates_from_server( + server=server, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + add_prefix=add_prefix, + ) + ) + all_resource_templates.extend(resource_templates) + verbose_logger.debug( + "Successfully fetched %s resource templates from server %s", + len(resource_templates), + server.name, + ) + except Exception as e: + verbose_logger.exception( + "Error getting resource templates from server %s: %s", + server.name, + str(e), + ) + + verbose_logger.info( + "Successfully fetched %s resource templates total from all MCP servers", + len(all_resource_templates), + ) + + return all_resource_templates + async def filter_tools_by_key_team_permissions( tools: List[MCPTool], server_id: str, @@ -553,7 +980,7 @@ if MCP_AVAILABLE: filtered_tools = [] for t in tools: # Get tool name without server prefix - unprefixed_tool_name, _ = get_server_name_prefix_tool_mcp(t.name) + unprefixed_tool_name, _ = split_server_prefix_from_name(t.name) if unprefixed_tool_name in allowed_tool_names: filtered_tools.append(t) else: @@ -606,6 +1033,118 @@ if MCP_AVAILABLE: return managed_tools + async def _list_mcp_prompts( + user_api_key_auth: Optional[UserAPIKeyAuth] = None, + mcp_auth_header: Optional[str] = None, + mcp_servers: Optional[List[str]] = None, + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + ) -> List[Prompt]: + """ + List all available MCP prompts. + + Args: + user_api_key_auth: User authentication info for access control + mcp_auth_header: Optional auth header for MCP server (deprecated) + mcp_servers: Optional list of server names/aliases to filter by + mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value} + + Returns: + List[Prompt]: Combined list of tools from all accessible servers + """ + if not MCP_AVAILABLE: + return [] + # Get tools from managed MCP servers with error handling + managed_prompts = [] + try: + managed_prompts = await _get_prompts_from_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + verbose_logger.debug( + f"Successfully fetched {len(managed_prompts)} prompts from managed MCP servers" + ) + except Exception as e: + verbose_logger.exception( + f"Error getting tools from managed MCP servers: {str(e)}" + ) + # Continue with empty managed tools list instead of failing completely + + return managed_prompts + + async def _list_mcp_resources( + user_api_key_auth: Optional[UserAPIKeyAuth] = None, + mcp_auth_header: Optional[str] = None, + mcp_servers: Optional[List[str]] = None, + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + ) -> List[Resource]: + """List all available MCP resources.""" + + if not MCP_AVAILABLE: + return [] + + managed_resources: List[Resource] = [] + try: + managed_resources = await _get_resources_from_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + verbose_logger.debug( + f"Successfully fetched {len(managed_resources)} resources from managed MCP servers" + ) + except Exception as e: + verbose_logger.exception( + f"Error getting resources from managed MCP servers: {str(e)}" + ) + + return managed_resources + + async def _list_mcp_resource_templates( + user_api_key_auth: Optional[UserAPIKeyAuth] = None, + mcp_auth_header: Optional[str] = None, + mcp_servers: Optional[List[str]] = None, + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + ) -> List[ResourceTemplate]: + """List all available MCP resource templates.""" + + if not MCP_AVAILABLE: + return [] + + managed_resource_templates: List[ResourceTemplate] = [] + try: + managed_resource_templates = await _get_resource_templates_from_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + verbose_logger.debug( + "Successfully fetched %s resource templates from managed MCP servers", + len(managed_resource_templates), + ) + except Exception as e: + verbose_logger.exception( + "Error getting resource templates from managed MCP servers: %s", + str(e), + ) + + return managed_resource_templates + @client async def call_mcp_tool( name: str, @@ -634,7 +1173,7 @@ if MCP_AVAILABLE: ) ) - allowed_mcp_servers = global_mcp_server_manager.get_mcp_servers_from_ids( + allowed_mcp_servers = global_mcp_server_manager.get_mcp_servers_from_ids( # type: ignore[attr-defined] allowed_mcp_server_ids ) @@ -647,7 +1186,7 @@ if MCP_AVAILABLE: mcp_server: Optional[MCPServer] = None # Remove prefix from tool name for logging and processing - original_tool_name, server_name = get_server_name_prefix_tool_mcp(name) + original_tool_name, server_name = split_server_prefix_from_name(name) # If tool name is unprefixed, resolve its server so we can enforce permissions if not server_name: @@ -735,6 +1274,110 @@ if MCP_AVAILABLE: ) return response + async def mcp_get_prompt( + name: str, + arguments: Optional[Dict[str, Any]] = None, + user_api_key_auth: Optional[UserAPIKeyAuth] = None, + mcp_auth_header: Optional[str] = None, + mcp_servers: Optional[List[str]] = None, + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + ) -> GetPromptResult: + """ + Fetch a specific MCP prompt, handling both prefixed and unprefixed names. + """ + allowed_mcp_servers = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, + ) + + if not allowed_mcp_servers: + raise HTTPException( + status_code=403, + detail="User not allowed to get this prompt.", + ) + + # Decide whether to add prefix based on number of allowed servers + add_prefix = not (len(allowed_mcp_servers) == 1) + + if add_prefix: + original_prompt_name, server_name = split_server_prefix_from_name(name) + else: + original_prompt_name = name + server_name = allowed_mcp_servers[0].name + + server = next((s for s in allowed_mcp_servers if s.name == server_name), None) + if server is None: + raise HTTPException( + status_code=403, + detail="User not allowed to get this prompt.", + ) + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + return await global_mcp_server_manager.get_prompt_from_server( + server=server, + prompt_name=original_prompt_name, + arguments=arguments, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + ) + + async def mcp_read_resource( + url: AnyUrl, + user_api_key_auth: Optional[UserAPIKeyAuth] = None, + mcp_auth_header: Optional[str] = None, + mcp_servers: Optional[List[str]] = None, + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + ) -> ReadResourceResult: + """Read resource contents from upstream MCP servers.""" + + allowed_mcp_servers = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, + ) + + if not allowed_mcp_servers: + raise HTTPException( + status_code=403, + detail="User not allowed to read this resource.", + ) + + if len(allowed_mcp_servers) != 1: + raise HTTPException( + status_code=400, + detail=( + "Multiple MCP servers configured; read_resource currently " + "supports exactly one allowed server." + ), + ) + + server = allowed_mcp_servers[0] + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + return await global_mcp_server_manager.read_resource_from_server( + server=server, + url=url, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + ) + def _get_standard_logging_mcp_tool_call( name: str, arguments: Dict[str, Any], diff --git a/litellm/proxy/_experimental/mcp_server/utils.py b/litellm/proxy/_experimental/mcp_server/utils.py index fb28eaf8cf2..d801b312aac 100644 --- a/litellm/proxy/_experimental/mcp_server/utils.py +++ b/litellm/proxy/_experimental/mcp_server/utils.py @@ -13,6 +13,7 @@ LITELLM_MCP_SERVER_DESCRIPTION = "MCP Server for LiteLLM" MCP_TOOL_PREFIX_SEPARATOR = os.environ.get("MCP_TOOL_PREFIX_SEPARATOR", "-") MCP_TOOL_PREFIX_FORMAT = "{server_name}{separator}{tool_name}" + def is_mcp_available() -> bool: """ Returns True if the MCP module is available, False otherwise @@ -23,92 +24,81 @@ def is_mcp_available() -> bool: except ImportError: return False + def normalize_server_name(server_name: str) -> str: """ Normalize server name by replacing spaces with underscores """ return server_name.replace(" ", "_") + def validate_and_normalize_mcp_server_payload(payload: Any) -> None: """ Validate and normalize MCP server payload fields (server_name and alias). - + This function: 1. Validates that server_name and alias don't contain the MCP_TOOL_PREFIX_SEPARATOR 2. Normalizes alias by replacing spaces with underscores 3. Sets default alias if not provided (using server_name as base) - + Args: payload: The payload object containing server_name and alias fields - + Raises: HTTPException: If validation fails """ # Server name validation: disallow '-' - if hasattr(payload, 'server_name') and payload.server_name: + if hasattr(payload, "server_name") and payload.server_name: validate_mcp_server_name(payload.server_name, raise_http_exception=True) - + # Alias validation: disallow '-' - if hasattr(payload, 'alias') and payload.alias: + if hasattr(payload, "alias") and payload.alias: validate_mcp_server_name(payload.alias, raise_http_exception=True) - + # Alias normalization and defaulting - alias = getattr(payload, 'alias', None) - server_name = getattr(payload, 'server_name', None) - + alias = getattr(payload, "alias", None) + server_name = getattr(payload, "server_name", None) + if not alias and server_name: alias = normalize_server_name(server_name) elif alias: alias = normalize_server_name(alias) - + # Update the payload with normalized alias - if hasattr(payload, 'alias'): + if hasattr(payload, "alias"): payload.alias = alias -def add_server_prefix_to_tool_name(tool_name: str, server_name: str) -> str: - """ - Add server name prefix to tool name - Args: - tool_name: Original tool name - server_name: MCP server name - - Returns: - Prefixed tool name in format: server_name::tool_name - """ +def add_server_prefix_to_name(name: str, server_name: str) -> str: + """Add server name prefix to any MCP resource name.""" formatted_server_name = normalize_server_name(server_name) return MCP_TOOL_PREFIX_FORMAT.format( server_name=formatted_server_name, separator=MCP_TOOL_PREFIX_SEPARATOR, - tool_name=tool_name + tool_name=name, ) + def get_server_prefix(server: Any) -> str: """Return the prefix for a server: alias if present, else server_name, else server_id""" - if hasattr(server, 'alias') and server.alias: + if hasattr(server, "alias") and server.alias: return server.alias - if hasattr(server, 'server_name') and server.server_name: + if hasattr(server, "server_name") and server.server_name: return server.server_name - if hasattr(server, 'server_id'): + if hasattr(server, "server_id"): return server.server_id return "" -def get_server_name_prefix_tool_mcp(prefixed_tool_name: str) -> Tuple[str, str]: - """ - Remove server name prefix from tool name - Args: - prefixed_tool_name: Tool name with server prefix - - Returns: - Tuple of (original_tool_name, server_name) - """ - if MCP_TOOL_PREFIX_SEPARATOR in prefixed_tool_name: - parts = prefixed_tool_name.split(MCP_TOOL_PREFIX_SEPARATOR, 1) +def split_server_prefix_from_name(prefixed_name: str) -> Tuple[str, str]: + """Return the unprefixed name plus the server name used as prefix.""" + if MCP_TOOL_PREFIX_SEPARATOR in prefixed_name: + parts = prefixed_name.split(MCP_TOOL_PREFIX_SEPARATOR, 1) if len(parts) == 2: - return parts[1], parts[0] # tool_name, server_name - return prefixed_tool_name, "" # No prefix found, return original name + return parts[1], parts[0] + return prefixed_name, "" + def is_tool_name_prefixed(tool_name: str) -> bool: """ @@ -122,14 +112,17 @@ def is_tool_name_prefixed(tool_name: str) -> bool: """ return MCP_TOOL_PREFIX_SEPARATOR in tool_name -def validate_mcp_server_name(server_name: str, raise_http_exception: bool = False) -> None: + +def validate_mcp_server_name( + server_name: str, raise_http_exception: bool = False +) -> None: """ Validate that MCP server name does not contain 'MCP_TOOL_PREFIX_SEPARATOR'. - + Args: server_name: The server name to validate raise_http_exception: If True, raises HTTPException instead of generic Exception - + Raises: Exception or HTTPException: If server name contains 'MCP_TOOL_PREFIX_SEPARATOR' """ @@ -138,9 +131,9 @@ def validate_mcp_server_name(server_name: str, raise_http_exception: bool = Fals if raise_http_exception: from fastapi import HTTPException from starlette import status + raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail={"error": error_message} + status_code=status.HTTP_400_BAD_REQUEST, detail={"error": error_message} ) else: raise Exception(error_message) diff --git a/litellm/proxy/_experimental/out/_next/static/zzKcMfj4Db-ZZ7hcspdhR/_buildManifest.js b/litellm/proxy/_experimental/out/_next/static/RP6qFLO9Sa0mS2DSQFL5i/_buildManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/zzKcMfj4Db-ZZ7hcspdhR/_buildManifest.js rename to litellm/proxy/_experimental/out/_next/static/RP6qFLO9Sa0mS2DSQFL5i/_buildManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/zzKcMfj4Db-ZZ7hcspdhR/_ssgManifest.js b/litellm/proxy/_experimental/out/_next/static/RP6qFLO9Sa0mS2DSQFL5i/_ssgManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/zzKcMfj4Db-ZZ7hcspdhR/_ssgManifest.js rename to litellm/proxy/_experimental/out/_next/static/RP6qFLO9Sa0mS2DSQFL5i/_ssgManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/1116-2d5ec30ef7d86f0e.js b/litellm/proxy/_experimental/out/_next/static/chunks/1116-2d5ec30ef7d86f0e.js deleted file mode 100644 index 7c33ceca243..00000000000 --- a/litellm/proxy/_experimental/out/_next/static/chunks/1116-2d5ec30ef7d86f0e.js +++ /dev/null @@ -1 +0,0 @@ -"use strict";(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[1116],{69993:function(e,t,r){r.d(t,{Z:function(){return s}});var n=r(1119),o=r(2265),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M300 328a60 60 0 10120 0 60 60 0 10-120 0zM852 64H172c-17.7 0-32 14.3-32 32v660c0 17.7 14.3 32 32 32h680c17.7 0 32-14.3 32-32V96c0-17.7-14.3-32-32-32zm-32 660H204V128h616v596zM604 328a60 60 0 10120 0 60 60 0 10-120 0zm250.2 556H169.8c-16.5 0-29.8 14.3-29.8 32v36c0 4.4 3.3 8 7.4 8h729.1c4.1 0 7.4-3.6 7.4-8v-36c.1-17.7-13.2-32-29.7-32zM664 508H360c-4.4 0-8 3.6-8 8v60c0 4.4 3.6 8 8 8h304c4.4 0 8-3.6 8-8v-60c0-4.4-3.6-8-8-8z"}}]},name:"robot",theme:"outlined"},i=r(55015),s=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,n.Z)({},e,{ref:t,icon:a}))})},92858:function(e,t,r){r.d(t,{Z:function(){return S}});var n=r(5853),o=r(2265),a=r(62963),i=r(90945),s=r(13323),l=r(17684),c=r(80004),u=r(93689),d=r(38198),f=r(47634),m=r(56314),h=r(27847),p=r(64518);let g=(0,o.createContext)(null),v=Object.assign((0,h.yV)(function(e,t){let r=(0,l.M)(),{id:n="headlessui-description-".concat(r),...a}=e,i=function e(){let t=(0,o.useContext)(g);if(null===t){let t=Error("You used a component, but it is not inside a relevant parent.");throw Error.captureStackTrace&&Error.captureStackTrace(t,e),t}return t}(),s=(0,u.T)(t);(0,p.e)(()=>i.register(n),[n,i.register]);let c={ref:s,...i.props,id:n};return(0,h.sY)({ourProps:c,theirProps:a,slot:i.slot||{},defaultTag:"p",name:i.name||"Description"})}),{});var w=r(37388);let k=(0,o.createContext)(null),b=Object.assign((0,h.yV)(function(e,t){let r=(0,l.M)(),{id:n="headlessui-label-".concat(r),passive:a=!1,...i}=e,s=function e(){let t=(0,o.useContext)(k);if(null===t){let t=Error("You used a