mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
Merge remote-tracking branch 'origin/main' into litellm_chore-migrate-router-settings-page-off-of-tremor
This commit is contained in:
Ryan Crabbe
commit
545db8c4e1
642 changed files with 13222 additions and 5649 deletions
|
|
@ -638,7 +638,7 @@ jobs:
|
|||
username: ${DOCKERHUB_USERNAME}
|
||||
password: ${DOCKERHUB_PASSWORD}
|
||||
working_directory: ~/project
|
||||
resource_class: large
|
||||
resource_class: xlarge
|
||||
|
||||
steps:
|
||||
- checkout
|
||||
|
|
@ -682,7 +682,7 @@ jobs:
|
|||
for dir in "${IGNORE_DIRS[@]}"; do
|
||||
IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir"
|
||||
done
|
||||
uv run --no-sync python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 8 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5
|
||||
uv run --no-sync python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 --max-worker-restart=5
|
||||
no_output_timeout: 15m
|
||||
|
||||
# Store test results
|
||||
|
|
@ -2916,90 +2916,6 @@ jobs:
|
|||
- codecov/upload:
|
||||
file: ./coverage.xml
|
||||
|
||||
publish_proxy_extras:
|
||||
docker:
|
||||
- image: cimg/python:3.12
|
||||
working_directory: ~/project/litellm-proxy-extras
|
||||
environment:
|
||||
TWINE_USERNAME: __token__
|
||||
|
||||
steps:
|
||||
- checkout:
|
||||
path: ~/project
|
||||
|
||||
- run:
|
||||
name: Check if litellm-proxy-extras dir or pyproject.toml was modified
|
||||
command: |
|
||||
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
||||
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
||||
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
||||
rm -f /tmp/uv-install.sh
|
||||
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
||||
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
# Get current version from pyproject.toml
|
||||
CURRENT_VERSION=$(python -c 'import tomllib; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); print(data["project"]["version"])')
|
||||
|
||||
# Get last published version from PyPI
|
||||
LAST_VERSION=$(curl -s https://pypi.org/pypi/litellm-proxy-extras/json | python -c "import json, sys; print(json.load(sys.stdin)['info']['version'])")
|
||||
|
||||
echo "Current version: $CURRENT_VERSION"
|
||||
echo "Last published version: $LAST_VERSION"
|
||||
|
||||
# Compare versions using Python's packaging.version
|
||||
VERSION_COMPARE=$(uv run --with 'packaging==25.0' python -c "from packaging import version; print(1 if version.parse('$CURRENT_VERSION') < version.parse('$LAST_VERSION') else 0)")
|
||||
|
||||
echo "Version compare: $VERSION_COMPARE"
|
||||
if [ "$VERSION_COMPARE" = "1" ]; then
|
||||
echo "Error: Current version ($CURRENT_VERSION) is less than last published version ($LAST_VERSION)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# If versions are equal or current is greater, compare against the published package contents.
|
||||
EXTRACTED_DIR=$(uv run --with "litellm-proxy-extras==$LAST_VERSION" python -c 'import importlib.util; from pathlib import Path; spec = importlib.util.find_spec("litellm_proxy_extras"); assert spec is not None and spec.origin is not None, "litellm_proxy_extras not found in uv-run environment"; print(Path(spec.origin).resolve().parent)')
|
||||
|
||||
# Compare contents
|
||||
if ! diff -r "$EXTRACTED_DIR" ./litellm_proxy_extras; then
|
||||
if [ "$CURRENT_VERSION" = "$LAST_VERSION" ]; then
|
||||
echo "Error: Changes detected in litellm-proxy-extras but version was not bumped"
|
||||
echo "Current version: $CURRENT_VERSION"
|
||||
echo "Last published version: $LAST_VERSION"
|
||||
echo "Changes:"
|
||||
diff -r "$EXTRACTED_DIR" ./litellm_proxy_extras
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "No changes detected in litellm-proxy-extras. Skipping PyPI publish."
|
||||
circleci step halt
|
||||
fi
|
||||
|
||||
- run:
|
||||
name: Get new version
|
||||
command: |
|
||||
NEW_VERSION=$(python -c 'import tomllib; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); print(data["project"]["version"])')
|
||||
echo "export NEW_VERSION=$NEW_VERSION" >> $BASH_ENV
|
||||
|
||||
- run:
|
||||
name: Check if versions match
|
||||
command: |
|
||||
cd ~/project
|
||||
# Check pyproject.toml
|
||||
CURRENT_VERSION=$(uv run --with 'packaging==25.0' python -c 'import tomllib; from packaging.requirements import Requirement; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); matches = [spec.version for requirement in data["project"]["optional-dependencies"]["proxy"] for parsed in [Requirement(requirement)] if parsed.name == "litellm-proxy-extras" and parsed.specifier for spec in parsed.specifier if spec.operator == "=="]; print(matches[0] if matches else (_ for _ in ()).throw(SystemExit("Could not find exact litellm-proxy-extras pin in project.optional-dependencies.proxy")))')
|
||||
if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then
|
||||
echo "Error: Version in pyproject.toml ($CURRENT_VERSION) doesn't match new version ($NEW_VERSION)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- run:
|
||||
name: Publish to PyPI
|
||||
command: |
|
||||
echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc
|
||||
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
rm -rf build dist
|
||||
uv build
|
||||
uv tool run --from 'twine==6.2.0' twine upload --verbose dist/*
|
||||
|
||||
ui_build:
|
||||
docker:
|
||||
- image: cimg/node:20.19
|
||||
|
|
@ -3214,60 +3130,6 @@ jobs:
|
|||
- litellm-docker-database.tar.zst
|
||||
|
||||
|
||||
prisma_schema_sync:
|
||||
machine:
|
||||
image: ubuntu-2204:2023.10.1
|
||||
resource_class: medium
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- setup_google_dns
|
||||
- attach_workspace:
|
||||
at: ~/project
|
||||
- run:
|
||||
name: Start PostgreSQL Database
|
||||
command: |
|
||||
docker run -d \
|
||||
--name postgres-db \
|
||||
-e POSTGRES_USER=postgres \
|
||||
-e POSTGRES_PASSWORD=postgres \
|
||||
-e POSTGRES_DB=litellm_schema_sync \
|
||||
-p 5432:5432 \
|
||||
postgres:14
|
||||
- wait_for_service:
|
||||
url: tcp://localhost:5432
|
||||
timeout: "60"
|
||||
- run:
|
||||
name: Load Docker Database Image
|
||||
command: |
|
||||
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
||||
docker images | grep litellm-docker-database
|
||||
- run:
|
||||
name: Run schema sync via prisma db push
|
||||
command: |
|
||||
docker run -d \
|
||||
-p 4000:4000 \
|
||||
-e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_schema_sync" \
|
||||
-e LITELLM_MASTER_KEY="sk-1234" \
|
||||
--name schema-sync \
|
||||
--add-host=host.docker.internal:host-gateway \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--use_prisma_db_push
|
||||
- run:
|
||||
name: Start outputting logs
|
||||
command: docker logs -f schema-sync
|
||||
background: true
|
||||
- wait_for_service:
|
||||
url: http://localhost:4000
|
||||
timeout: "300"
|
||||
- run:
|
||||
name: Stop schema sync container
|
||||
command: docker stop schema-sync
|
||||
|
||||
|
||||
test_bad_database_url:
|
||||
machine:
|
||||
image: ubuntu-2204:2023.10.1
|
||||
|
|
@ -3421,14 +3283,6 @@ workflows:
|
|||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- prisma_schema_sync:
|
||||
requires:
|
||||
- build_docker_database_image
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- e2e_ui_testing:
|
||||
filters:
|
||||
branches:
|
||||
|
|
@ -3688,9 +3542,3 @@ workflows:
|
|||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- publish_proxy_extras:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- main
|
||||
- /litellm_release_day_.*/
|
||||
|
|
|
|||
6
.github/workflows/test-linting.yml
vendored
6
.github/workflows/test-linting.yml
vendored
|
|
@ -2,7 +2,11 @@ name: LiteLLM Linting
|
|||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
6
.github/workflows/test-litellm-ui-build.yml
vendored
6
.github/workflows/test-litellm-ui-build.yml
vendored
|
|
@ -4,7 +4,11 @@ permissions:
|
|||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
|
||||
jobs:
|
||||
build-ui:
|
||||
|
|
|
|||
6
.github/workflows/test-mcp.yml
vendored
6
.github/workflows/test-mcp.yml
vendored
|
|
@ -2,7 +2,11 @@ name: LiteLLM MCP Tests (folder - tests/mcp_tests)
|
|||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
6
.github/workflows/test-model-map.yaml
vendored
6
.github/workflows/test-model-map.yaml
vendored
|
|
@ -2,7 +2,11 @@ name: Validate model_prices_and_context_window.json
|
|||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
6
.github/workflows/test-unit-core-utils.yml
vendored
6
.github/workflows/test-unit-core-utils.yml
vendored
|
|
@ -2,7 +2,11 @@ name: "Unit Tests: Core Utilities"
|
|||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
|
|
@ -2,7 +2,11 @@ name: "Unit Tests: Documentation Validation"
|
|||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
|
|
@ -2,7 +2,11 @@ name: "Unit Tests: Enterprise, Google GenAI & Routing"
|
|||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
6
.github/workflows/test-unit-integrations.yml
vendored
6
.github/workflows/test-unit-integrations.yml
vendored
|
|
@ -2,7 +2,11 @@ name: "Unit Tests: Integrations (Callbacks & Logging)"
|
|||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
|
|
@ -2,7 +2,11 @@ name: "Unit Tests: LLM Provider Transformations"
|
|||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
6
.github/workflows/test-unit-misc.yml
vendored
6
.github/workflows/test-unit-misc.yml
vendored
|
|
@ -2,7 +2,11 @@ name: "Unit Tests: MCP, Secrets, Containers & Misc"
|
|||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
6
.github/workflows/test-unit-proxy-auth.yml
vendored
6
.github/workflows/test-unit-proxy-auth.yml
vendored
|
|
@ -2,7 +2,11 @@ name: "Unit Tests: Proxy Auth & Key Management"
|
|||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
2
.github/workflows/test-unit-proxy-db.yml
vendored
2
.github/workflows/test-unit-proxy-db.yml
vendored
|
|
@ -3,7 +3,7 @@ name: "Unit Tests: Proxy DB Operations"
|
|||
# Uses DATABASE_URL secret — only runs on trusted branches, not PRs.
|
||||
on:
|
||||
push:
|
||||
branches: [main, "litellm_*"]
|
||||
branches: [main, "litellm_**"]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
|
|
@ -2,7 +2,11 @@ name: "Unit Tests: Proxy API Endpoints"
|
|||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
6
.github/workflows/test-unit-proxy-infra.yml
vendored
6
.github/workflows/test-unit-proxy-infra.yml
vendored
|
|
@ -2,7 +2,11 @@ name: "Unit Tests: Proxy Infrastructure"
|
|||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
6
.github/workflows/test-unit-proxy-legacy.yml
vendored
6
.github/workflows/test-unit-proxy-legacy.yml
vendored
|
|
@ -2,7 +2,11 @@ name: "Unit Tests: Proxy Legacy Tests"
|
|||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
|
|
@ -2,7 +2,11 @@ name: "Unit Tests: Responses, Caching & Types"
|
|||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
2
.github/workflows/test-unit-security.yml
vendored
2
.github/workflows/test-unit-security.yml
vendored
|
|
@ -3,7 +3,7 @@ name: "Unit Tests: Security"
|
|||
# Uses DATABASE_URL secret — only runs on trusted branches, not PRs.
|
||||
on:
|
||||
push:
|
||||
branches: [main, "litellm_*"]
|
||||
branches: [main, "litellm_**"]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
6
.github/workflows/test_server_root_path.yml
vendored
6
.github/workflows/test_server_root_path.yml
vendored
|
|
@ -4,7 +4,11 @@ permissions:
|
|||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
|
||||
jobs:
|
||||
test-server-root-path:
|
||||
|
|
|
|||
366
docs/my-website/blog/claude_opus_4_7/index.md
Normal file
366
docs/my-website/blog/claude_opus_4_7/index.md
Normal file
|
|
@ -0,0 +1,366 @@
|
|||
---
|
||||
slug: claude_opus_4_7
|
||||
title: "Day 0 Support: Claude Opus 4.7"
|
||||
date: 2026-04-16T10:00:00
|
||||
authors:
|
||||
- sameer
|
||||
- ishaan-alt
|
||||
- krrish
|
||||
description: "Day 0 support for Claude Opus 4.7 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
|
||||
tags: [anthropic, claude, opus 4.7]
|
||||
hide_table_of_contents: false
|
||||
---
|
||||
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
LiteLLM now supports [Claude Opus 4.7](https://www.anthropic.com/news/claude-opus-4-7) on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
|
||||
|
||||
{/* truncate */}
|
||||
|
||||
## Docker Image
|
||||
|
||||
```bash
|
||||
docker pull ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7
|
||||
```
|
||||
|
||||
## Usage - Anthropic
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="proxy" label="LiteLLM Proxy">
|
||||
|
||||
**1. Setup config.yaml**
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: claude-opus-4-7
|
||||
litellm_params:
|
||||
model: anthropic/claude-opus-4-7
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
```
|
||||
|
||||
**2. Start the proxy**
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
-p 4000:4000 \
|
||||
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
|
||||
-v $(pwd)/config.yaml:/app/config.yaml \
|
||||
ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7 \
|
||||
--config /app/config.yaml
|
||||
```
|
||||
|
||||
**3. Test it!**
|
||||
|
||||
```bash
|
||||
curl --location 'http://0.0.0.0:4000/chat/completions' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--header 'Authorization: Bearer $LITELLM_KEY' \
|
||||
--data '{
|
||||
"model": "claude-opus-4-7",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what llm are you"
|
||||
}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Usage - Azure
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="proxy" label="LiteLLM Proxy">
|
||||
|
||||
**1. Setup config.yaml**
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: claude-opus-4-7
|
||||
litellm_params:
|
||||
model: azure_ai/claude-opus-4-7
|
||||
api_key: os.environ/AZURE_AI_API_KEY
|
||||
api_base: os.environ/AZURE_AI_API_BASE # https://<resource>.services.ai.azure.com
|
||||
```
|
||||
|
||||
**2. Start the proxy**
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
-p 4000:4000 \
|
||||
-e AZURE_AI_API_KEY=$AZURE_AI_API_KEY \
|
||||
-e AZURE_AI_API_BASE=$AZURE_AI_API_BASE \
|
||||
-v $(pwd)/config.yaml:/app/config.yaml \
|
||||
ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7 \
|
||||
--config /app/config.yaml
|
||||
```
|
||||
|
||||
**3. Test it!**
|
||||
|
||||
```bash
|
||||
curl --location 'http://0.0.0.0:4000/chat/completions' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--header 'Authorization: Bearer $LITELLM_KEY' \
|
||||
--data '{
|
||||
"model": "claude-opus-4-7",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what llm are you"
|
||||
}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Usage - Vertex AI
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="proxy" label="LiteLLM Proxy">
|
||||
|
||||
**1. Setup config.yaml**
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: claude-opus-4-7
|
||||
litellm_params:
|
||||
model: vertex_ai/claude-opus-4-7
|
||||
vertex_project: os.environ/VERTEX_PROJECT
|
||||
vertex_location: us-east5
|
||||
```
|
||||
|
||||
**2. Start the proxy**
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
-p 4000:4000 \
|
||||
-e VERTEX_PROJECT=$VERTEX_PROJECT \
|
||||
-e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json \
|
||||
-v $(pwd)/config.yaml:/app/config.yaml \
|
||||
-v $(pwd)/credentials.json:/app/credentials.json \
|
||||
ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7 \
|
||||
--config /app/config.yaml
|
||||
```
|
||||
|
||||
**3. Test it!**
|
||||
|
||||
```bash
|
||||
curl --location 'http://0.0.0.0:4000/chat/completions' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--header 'Authorization: Bearer $LITELLM_KEY' \
|
||||
--data '{
|
||||
"model": "claude-opus-4-7",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what llm are you"
|
||||
}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Usage - Bedrock
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="proxy" label="LiteLLM Proxy">
|
||||
|
||||
**1. Setup config.yaml**
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: claude-opus-4-7
|
||||
litellm_params:
|
||||
model: bedrock/anthropic.claude-opus-4-7
|
||||
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
|
||||
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
|
||||
aws_region_name: us-east-1
|
||||
```
|
||||
|
||||
**2. Start the proxy**
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
-p 4000:4000 \
|
||||
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
||||
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
||||
-v $(pwd)/config.yaml:/app/config.yaml \
|
||||
ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7 \
|
||||
--config /app/config.yaml
|
||||
```
|
||||
|
||||
**3. Test it!**
|
||||
|
||||
```bash
|
||||
curl --location 'http://0.0.0.0:4000/chat/completions' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--header 'Authorization: Bearer $LITELLM_KEY' \
|
||||
--data '{
|
||||
"model": "claude-opus-4-7",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what llm are you"
|
||||
}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Advanced Features
|
||||
|
||||
### Adaptive Thinking
|
||||
|
||||
:::note
|
||||
When using `reasoning_effort` with Claude Opus 4.7, all values (`low`, `medium`, `high`, `xhigh`) are mapped to `thinking: {type: "adaptive"}`. To use explicit thinking budgets with `type: "enabled"`, pass the native `thinking` parameter directly.
|
||||
:::
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="completions" label="/chat/completions">
|
||||
|
||||
LiteLLM supports adaptive thinking through the `reasoning_effort` parameter:
|
||||
|
||||
```bash
|
||||
curl --location 'http://0.0.0.0:4000/chat/completions' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--header 'Authorization: Bearer $LITELLM_KEY' \
|
||||
--data '{
|
||||
"model": "claude-opus-4-7",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Solve this complex problem: What is the optimal strategy for..."
|
||||
}
|
||||
],
|
||||
"reasoning_effort": "high"
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="messages" label="/v1/messages">
|
||||
|
||||
Use the `thinking` parameter with `type: "adaptive"` to enable adaptive thinking mode:
|
||||
|
||||
```bash
|
||||
curl --location 'http://0.0.0.0:4000/v1/messages' \
|
||||
--header 'x-api-key: sk-12345' \
|
||||
--header 'content-type: application/json' \
|
||||
--data '{
|
||||
"model": "claude-opus-4-7",
|
||||
"max_tokens": 16000,
|
||||
"thinking": {
|
||||
"type": "adaptive"
|
||||
},
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Explain why the sum of two even numbers is always even."
|
||||
}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Effort Levels
|
||||
|
||||
Claude Opus 4.7 supports four effort levels: `low`, `medium`, `high` (default), and `xhigh`. These give you finer-grained control over how much reasoning the model applies to a task. Pass the effort level via the `output_config` parameter.
|
||||
|
||||
`xhigh` is a new effort level introduced with Opus 4.7 that sits above `high`. The `max` effort level is Claude Opus 4.6 only and is not available on 4.7.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="completions" label="/chat/completions">
|
||||
|
||||
```bash
|
||||
curl --location 'http://0.0.0.0:4000/chat/completions' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--header 'Authorization: Bearer $LITELLM_KEY' \
|
||||
--data '{
|
||||
"model": "claude-opus-4-7",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Explain quantum computing"
|
||||
}
|
||||
],
|
||||
"output_config": {
|
||||
"effort": "xhigh"
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
**Using OpenAI SDK:**
|
||||
|
||||
```python
|
||||
import openai
|
||||
|
||||
client = openai.OpenAI(
|
||||
api_key="your-litellm-key",
|
||||
base_url="http://0.0.0.0:4000"
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="claude-opus-4-7",
|
||||
messages=[{"role": "user", "content": "Explain quantum computing"}],
|
||||
extra_body={"output_config": {"effort": "xhigh"}}
|
||||
)
|
||||
```
|
||||
|
||||
**Using LiteLLM SDK:**
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="anthropic/claude-opus-4-7",
|
||||
messages=[{"role": "user", "content": "Explain quantum computing"}],
|
||||
output_config={"effort": "xhigh"},
|
||||
)
|
||||
```
|
||||
|
||||
You can combine `reasoning_effort` with `output_config` for even more fine-grained control over the model's behavior.
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="messages" label="/v1/messages">
|
||||
|
||||
```bash
|
||||
curl --location 'http://0.0.0.0:4000/v1/messages' \
|
||||
--header 'x-api-key: sk-12345' \
|
||||
--header 'content-type: application/json' \
|
||||
--data '{
|
||||
"model": "claude-opus-4-7",
|
||||
"max_tokens": 4096,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Explain quantum computing"
|
||||
}
|
||||
],
|
||||
"output_config": {
|
||||
"effort": "xhigh"
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
**Effort level guide:**
|
||||
|
||||
| Effort | When to use |
|
||||
|--------|-------------|
|
||||
| `low` | Short, fast responses — simple lookups, formatting, classification |
|
||||
| `medium` | Balanced tradeoff for everyday Q&A and light reasoning |
|
||||
| `high` (default) | Complex reasoning, code generation, analysis |
|
||||
| `xhigh` | Hardest problems — multi-step math, deep research, agentic planning |
|
||||
|
||||
|
|
@ -9,6 +9,10 @@ import TabItem from '@theme/TabItem';
|
|||
import NavigationCards from '@site/src/components/NavigationCards';
|
||||
import Image from '@theme/IdealImage';
|
||||
|
||||
:::note Security Update
|
||||
The Trivy supply-chain compromise has been contained :tada: . All affected packages have been deleted and current releases are free of the compromised code/component. Please refer to our [Security Townhall](/blog/security-townhall-updates) for a deeper understanding of the problem, and [CI/CD v2](/blog/ci-cd-v2-improvements) for how we're improving moving forward.
|
||||
:::
|
||||
|
||||
<Image style={{padding: '10px', margin: '0 0 2.5rem'}} img={require('../img/hero.png')} />
|
||||
|
||||
**LiteLLM** is an open-source library that gives you a single, unified interface to call 100+ LLMs — OpenAI, Anthropic, Vertex AI, Bedrock, and more — using the OpenAI format.
|
||||
|
|
|
|||
|
|
@ -487,6 +487,7 @@ router_settings:
|
|||
| AZURE_STORAGE_CLIENT_ID | The Application Client ID to use for Authentication to Azure Blob Storage logging
|
||||
| AZURE_STORAGE_CLIENT_SECRET | The Application Client Secret to use for Authentication to Azure Blob Storage logging
|
||||
| AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY | Cost per GB per day for Azure Vector Store service
|
||||
| BACKGROUND_HEALTH_CHECK_MAX_TOKENS | Optional global default for `max_tokens` on proxy background health checks when a model has no `health_check_max_tokens`. If unset, non-wildcard models default to 1. Applies to wildcard routes when set. Default is unset
|
||||
| BATCH_STATUS_POLL_INTERVAL_SECONDS | Interval in seconds for polling batch status. Default is 3600 (1 hour)
|
||||
| BATCH_STATUS_POLL_MAX_ATTEMPTS | Maximum number of attempts for polling batch status. Default is 24 (for 24 hours)
|
||||
| BEDROCK_MAX_POLICY_SIZE | Maximum size for Bedrock policy. Default is 75
|
||||
|
|
@ -804,6 +805,8 @@ router_settings:
|
|||
| LITELLM_ASSETS_PATH | Path to directory for UI assets and logos. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/assets` in Docker.
|
||||
| LITELLM_BLOG_POSTS_URL | Custom URL for fetching LiteLLM blog posts JSON. Default is the GitHub main branch URL
|
||||
| LITELLM_CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours
|
||||
| LITELLM_CORS_ALLOW_CREDENTIALS | Set to `true` to explicitly allow credentials in CORS responses. When not set, credentials are disabled automatically if `LITELLM_CORS_ORIGINS` is `*` (wildcard) to prevent the browser security misconfiguration of reflecting any origin with credentials
|
||||
| LITELLM_CORS_ORIGINS | Comma-separated list of allowed CORS origins (e.g. `https://app.example.com,https://admin.example.com`). Defaults to `*` (all origins) when not set
|
||||
| LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API
|
||||
| LITELLM_DEPLOYMENT_ENVIRONMENT | Environment name for the deployment (e.g., "production", "staging"). Used as a fallback when OTEL_ENVIRONMENT_NAME is not set. Sets the `environment` tag in telemetry data
|
||||
| LITELLM_DETAILED_TIMING | When true, adds detailed per-phase timing headers to responses (`x-litellm-timing-{pre-processing,llm-api,post-processing,message-copy}-ms`). Default is false. See [latency overhead docs](../troubleshoot/latency_overhead.md)
|
||||
|
|
@ -925,6 +928,7 @@ router_settings:
|
|||
| OPENAI_CHATGPT_API_BASE | Alternative to CHATGPT_API_BASE. Base URL for ChatGPT API
|
||||
| OPENAI_FILE_SEARCH_COST_PER_1K_CALLS | Cost per 1000 calls for OpenAI file search. Default is 0.0025
|
||||
| OPENAI_ORGANIZATION | Organization identifier for OpenAI
|
||||
| OPENAPI_URL | The path to the OpenAPI JSON endpoint. **By default this is "/openapi.json"**
|
||||
| OPENID_BASE_URL | Base URL for OpenID Connect services
|
||||
| OPENID_CLIENT_ID | Client ID for OpenID Connect authentication
|
||||
| OPENID_CLIENT_SECRET | Client secret for OpenID Connect authentication
|
||||
|
|
|
|||
|
|
@ -14,6 +14,10 @@ Provider-specific cost tracking (e.g., [Vertex AI PayGo / priority pricing](../p
|
|||
[Sync model pricing data from GitHub](./sync_models_github.md) to ensure accurate cost tracking.
|
||||
:::
|
||||
|
||||
:::info Cost does not match your provider bill?
|
||||
Use the step-by-step workflow in [Debugging a cost discrepancy](../troubleshoot/cost_discrepancy): align time ranges, compare token categories (including cache), then decide whether the gap is ingestion, formula, or model-map pricing.
|
||||
:::
|
||||
|
||||
### How to Track Spend with LiteLLM
|
||||
|
||||
**Step 1**
|
||||
|
|
|
|||
|
|
@ -2,6 +2,16 @@ import Image from '@theme/IdealImage';
|
|||
|
||||
# Team Soft Budget Alerts
|
||||
|
||||
:::info
|
||||
|
||||
✨ This is an Enterprise feature. Email budget alerts require an enterprise license.
|
||||
|
||||
[Enterprise Pricing](https://www.litellm.ai/#pricing)
|
||||
|
||||
[Get free 7-day trial key](https://www.litellm.ai/enterprise#trial)
|
||||
|
||||
:::
|
||||
|
||||
Set a soft budget on a team and get email alerts when spending crosses the threshold — without blocking any requests.
|
||||
|
||||
## Overview
|
||||
|
|
|
|||
205
docs/my-website/docs/troubleshoot/cost_discrepancy.md
Normal file
205
docs/my-website/docs/troubleshoot/cost_discrepancy.md
Normal file
|
|
@ -0,0 +1,205 @@
|
|||
# Debugging a cost discrepancy
|
||||
|
||||
Cost discrepancies between LiteLLM and your provider bill usually come from one of three areas: token ingestion, the cost formula LiteLLM applies, or stale or incorrect pricing in the model map. This page walks through how to tell which case you are in.
|
||||
|
||||
## Step 1: Pick a time range
|
||||
|
||||
Lock down a specific window where the discrepancy is visible.
|
||||
|
||||
- Use at least 7 days of data when you can.
|
||||
- Prefer a window with stable usage so one-off spikes do not dominate the comparison.
|
||||
- Set the **same start and end time** on both your provider dashboard and the LiteLLM UI.
|
||||
|
||||
|
||||
|
||||
## Step 2: Confirm traffic only goes through LiteLLM
|
||||
|
||||
If any requests hit the provider directly (bypassing LiteLLM), the provider will show higher usage. That is expected, not a LiteLLM bug.
|
||||
|
||||
Before continuing, confirm:
|
||||
|
||||
- All clients use your LiteLLM proxy base URL.
|
||||
- No SDK or script uses provider API keys against the provider directly for the models you are comparing.
|
||||
- During the selected period, the models in question are only called via LiteLLM.
|
||||
|
||||
If you are unsure, filter the provider dashboard by the API key or IAM principal LiteLLM uses, rather than comparing to your whole account.
|
||||
|
||||
## Step 3: Compare token categories
|
||||
|
||||
In the LiteLLM UI, open **Model activity** (under Usage analytics) so you can inspect spend and tokens per model.
|
||||
|
||||
|
||||
|
||||
Scroll the **Model** list and select the model you are reconciling with your provider bill.
|
||||
|
||||
|
||||
|
||||
With the same time range on both sides, fill in:
|
||||
|
||||
| Category | LiteLLM | Provider | Delta |
|
||||
| --- | --- | --- | --- |
|
||||
| Total requests | — | — | — |
|
||||
| Input tokens | — | — | — |
|
||||
| Output tokens | — | — | — |
|
||||
| Cache read tokens | — | — | — |
|
||||
| Cache write tokens | — | — | — |
|
||||
|
||||
LiteLLM surfaces per-category token usage for the selected model—for example prompt, completion, and cache-related tokens.
|
||||
|
||||
|
||||
|
||||
Compare these figures with your provider’s usage view (for example AWS billing tools, Azure Monitor, or the OpenAI usage dashboard) for the same period.
|
||||
|
||||
### Cache token reporting
|
||||
|
||||
- **OpenAI:** Cache read tokens are typically included inside the reported input token count.
|
||||
- **Anthropic:** Cache read tokens are often reported separately from non-cached input tokens.
|
||||
|
||||
Compare the correct columns on each side so you are not treating “input” differently between dashboards.
|
||||
|
||||
### Why use a 10% threshold?
|
||||
|
||||
Provider dashboards and LiteLLM do not bucket requests on identical timestamps. A call at 11:59 PM can land in different daily totals on each side. Token counts can also differ slightly due to rounding across SDKs and APIs. A delta **under ~10%** is often explained by boundary effects and rounding. A delta **over ~10%** usually means something is miscounted, dropped, or categorized differently.
|
||||
|
||||
## Step 4: Follow the right path
|
||||
|
||||
<svg width="100%" viewBox="0 0 680 482" role="img" xmlns="http://www.w3.org/2000/svg" style={{ maxWidth: '100%', fontFamily: 'system-ui, sans-serif' }} aria-labelledby="cost-disc-flow-title">
|
||||
<title id="cost-disc-flow-title">Cost discrepancy debugging flowchart</title>
|
||||
<desc>Flowchart branching into Path A (token ingestion) or Path B which splits further into B1 (formula issue) and B2 (model map issue).</desc>
|
||||
<defs>
|
||||
<marker id="cd-arrow" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="6" markerHeight="6" orient="auto-start-reverse">
|
||||
<path d="M2 1L8 5L2 9" fill="none" stroke="#888780" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round" />
|
||||
</marker>
|
||||
</defs>
|
||||
|
||||
<rect x="215" y="24" width="250" height="44" rx="8" fill="#F1EFE8" stroke="#5F5E5A" strokeWidth="0.5" />
|
||||
<text x="340" y="47" textAnchor="middle" dominantBaseline="central" fill="#444441" fontSize="14" fontWeight="500">Compare provider vs LiteLLM</text>
|
||||
|
||||
<line x1="340" y1="68" x2="340" y2="104" stroke="#888780" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
|
||||
|
||||
<rect x="175" y="104" width="330" height="56" rx="8" fill="#F1EFE8" stroke="#5F5E5A" strokeWidth="0.5" />
|
||||
<text x="340" y="126" textAnchor="middle" dominantBaseline="central" fill="#444441" fontSize="14" fontWeight="500">Any category off by > 10%?</text>
|
||||
<text x="340" y="148" textAnchor="middle" dominantBaseline="central" fill="#5F5E5A" fontSize="12">requests, input, output, cache tokens</text>
|
||||
|
||||
<path d="M220 132 L100 132 L100 250" fill="none" stroke="#0F6E56" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
|
||||
<text x="157" y="122" textAnchor="middle" fill="#0F6E56" fontSize="12">YES</text>
|
||||
|
||||
<path d="M505 132 L580 132 L580 250" fill="none" stroke="#993C1D" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
|
||||
<text x="543" y="122" textAnchor="middle" fill="#993C1D" fontSize="12">NO</text>
|
||||
|
||||
<rect x="40" y="250" width="220" height="56" rx="8" fill="#E1F5EE" stroke="#0F6E56" strokeWidth="0.5" />
|
||||
<text x="150" y="271" textAnchor="middle" dominantBaseline="central" fill="#085041" fontSize="14" fontWeight="500">Path A</text>
|
||||
<text x="150" y="291" textAnchor="middle" dominantBaseline="central" fill="#0F6E56" fontSize="12">Token ingestion issue</text>
|
||||
|
||||
<rect x="420" y="250" width="220" height="56" rx="8" fill="#FAECE7" stroke="#993C1D" strokeWidth="0.5" />
|
||||
<text x="530" y="271" textAnchor="middle" dominantBaseline="central" fill="#712B13" fontSize="14" fontWeight="500">Path B</text>
|
||||
<text x="530" y="291" textAnchor="middle" dominantBaseline="central" fill="#993C1D" fontSize="12">Quantities match, cost differs</text>
|
||||
|
||||
<line x1="150" y1="306" x2="150" y2="370" stroke="#0F6E56" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
|
||||
|
||||
<line x1="530" y1="306" x2="530" y2="318" stroke="#854F0B" strokeWidth="1.5" />
|
||||
<line x1="435" y1="318" x2="575" y2="318" stroke="#854F0B" strokeWidth="1.5" />
|
||||
<line x1="435" y1="318" x2="435" y2="370" stroke="#854F0B" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
|
||||
<line x1="575" y1="318" x2="575" y2="370" stroke="#854F0B" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
|
||||
<text x="448" y="312" textAnchor="middle" fill="#854F0B" fontSize="11">B1</text>
|
||||
<text x="562" y="312" textAnchor="middle" fill="#854F0B" fontSize="11">B2</text>
|
||||
|
||||
<rect x="40" y="370" width="220" height="56" rx="8" fill="#E1F5EE" stroke="#0F6E56" strokeWidth="0.5" />
|
||||
<text x="150" y="391" textAnchor="middle" dominantBaseline="central" fill="#085041" fontSize="14" fontWeight="500">Report to LiteLLM team</text>
|
||||
<text x="150" y="411" textAnchor="middle" dominantBaseline="central" fill="#0F6E56" fontSize="12">endpoints + model + screenshots</text>
|
||||
|
||||
<rect x="380" y="370" width="110" height="56" rx="8" fill="#FAEEDA" stroke="#854F0B" strokeWidth="0.5" />
|
||||
<text x="435" y="391" textAnchor="middle" dominantBaseline="central" fill="#633806" fontSize="14" fontWeight="500">B1</text>
|
||||
<text x="435" y="411" textAnchor="middle" dominantBaseline="central" fill="#854F0B" fontSize="12">Fix formula</text>
|
||||
|
||||
<rect x="510" y="370" width="130" height="56" rx="8" fill="#FAEEDA" stroke="#854F0B" strokeWidth="0.5" />
|
||||
<text x="575" y="391" textAnchor="middle" dominantBaseline="central" fill="#633806" fontSize="14" fontWeight="500">B2</text>
|
||||
<text x="575" y="411" textAnchor="middle" dominantBaseline="central" fill="#854F0B" fontSize="12">Fix model map</text>
|
||||
|
||||
<path d="M150 426 L150 442 L340 442" fill="none" stroke="#888780" strokeWidth="0.5" strokeDasharray="4 3" />
|
||||
<path d="M340 442 L435 442 L435 428" fill="none" stroke="#888780" strokeWidth="0.5" strokeDasharray="4 3" />
|
||||
<path d="M340 442 L575 442 L575 428" fill="none" stroke="#888780" strokeWidth="0.5" strokeDasharray="4 3" />
|
||||
<text x="340" y="454" textAnchor="middle" fill="#5F5E5A" fontSize="11">if neither path resolves it,</text>
|
||||
<text x="340" y="470" textAnchor="middle" fill="#5F5E5A" fontSize="11">Open a github issue backing up with all your data</text>
|
||||
</svg>
|
||||
|
||||
## Path A: Token quantity mismatch
|
||||
|
||||
If any category is off by more than about 10%, LiteLLM may not be ingesting that category correctly (or the provider dashboard is categorizing tokens differently—recheck Step 3 first).
|
||||
|
||||
**What to send the LiteLLM team:**
|
||||
|
||||
1. Screenshots of both dashboards with the date range visible.
|
||||
2. Which category is off (input, output, cache reads, cache writes, or request count).
|
||||
3. Endpoints used (for example `/chat/completions`, `/responses`, `/embeddings`).
|
||||
4. Model names as sent in the request (for example `anthropic.claude-opus-4-5`, `gpt-4o`).
|
||||
|
||||
### For maintainers debugging ingestion
|
||||
|
||||
1. Start the proxy with verbose logging, for example:
|
||||
```bash
|
||||
litellm --config config.yaml --detailed_debug
|
||||
```
|
||||
2. Reproduce a single request with the reported endpoint and model.
|
||||
3. Inspect the raw `usage` object in each streamed chunk (if streaming) or in the final response body.
|
||||
4. Compare that to the standard logging object (or the UI request log for that call).
|
||||
5. Any gap between raw provider usage and what LiteLLM logs or aggregates is where ingestion may be wrong.
|
||||
|
||||
## Path B: Quantities match but cost is wrong
|
||||
|
||||
If token and request counts agree within ~10% but dollar amounts differ, focus on how cost is computed.
|
||||
|
||||
### B1: Formula issue
|
||||
|
||||
Manually compute expected cost using the provider’s token breakdown and published rates (per million tokens or per token).
|
||||
|
||||
Add other billed dimensions your provider applies (for example cache creation, audio, or tier surcharges). If your hand calculation matches the provider bill but not LiteLLM, the implementation in LiteLLM for that provider or modality may be wrong.
|
||||
|
||||
### B2: Model map issue
|
||||
|
||||
If the formula structure matches how the provider bills, the values in LiteLLM’s model map may be stale or incorrect. Cross-check:
|
||||
|
||||
- [`model_prices_and_context_window.json`](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json)
|
||||
- The provider’s current public pricing
|
||||
|
||||
Inspect `input_cost_per_token`, `output_cost_per_token`, and any cache-related pricing fields for your exact model id (including provider prefix).
|
||||
|
||||
### For maintainers
|
||||
|
||||
1. Take authoritative token quantities from the user’s provider report.
|
||||
2. Derive the formula that reproduces the provider’s line item.
|
||||
3. Diff that against LiteLLM’s cost path for the same provider and response shape.
|
||||
4. If the formula matches but numbers differ, update pricing in `model_prices_and_context_window.json` (and follow the project’s sync / backup rules for that file).
|
||||
5. If the formula in code is wrong, fix the calculation and add a regression test using the user’s token breakdown.
|
||||
|
||||
## Still stuck?
|
||||
|
||||
1. Open a GitHub issue on [BerriAI/litellm](https://github.com/BerriAI/litellm) with your Step 3 comparison table, endpoints, and model names.
|
||||
|
||||
|
||||
On the issue, it helps to clarify:
|
||||
|
||||
- Reproducible on demand or intermittent?
|
||||
- Single model or many?
|
||||
- Steady over time, or starting from a specific release date or config change?
|
||||
|
||||
### For LiteLLM maintainers
|
||||
|
||||
If Path A and Path B do not close the case after triage, **you** should reach out and **schedule a call with the customer** (support or engineering), with the Step 3 table and screenshots—before treating the issue.
|
||||
|
||||
## Checklist
|
||||
|
||||
```
|
||||
□ Same time range on both dashboards
|
||||
□ Confirmed no direct-to-provider traffic for those models
|
||||
□ Compared: requests, input tokens, output tokens, cache tokens
|
||||
□ Noted cache reporting differences (OpenAI vs Anthropic, and so on)
|
||||
□ If > ~10% delta on quantities → Path A: report with screenshots, endpoints, model names
|
||||
□ If quantities match → Path B: verify formula (B1) and model map pricing (B2)
|
||||
□ If neither path fits → open a GitHub issue.
|
||||
```
|
||||
|
||||
## See also
|
||||
|
||||
- [Spend tracking](../proxy/cost_tracking)
|
||||
- [Sync model pricing from GitHub](../proxy/sync_models_github)
|
||||
|
|
@ -1149,6 +1149,7 @@ const sidebars = {
|
|||
label: "Troubleshooting",
|
||||
items: [
|
||||
"troubleshoot/ui_issues",
|
||||
"troubleshoot/cost_discrepancy",
|
||||
"mcp_troubleshoot",
|
||||
{
|
||||
type: "category",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,10 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
:::note Security Update
|
||||
The Trivy supply-chain compromise has been contained :tada: . All affected packages have been deleted and current releases are free of the compromised code/component. Please refer to our [Security Townhall](/blog/security-townhall-updates) for a deeper understanding of the problem, and [CI/CD v2](/blog/ci-cd-v2-improvements) for how we're improving moving forward.
|
||||
:::
|
||||
|
||||
# LiteLLM - Getting Started
|
||||
|
||||
https://github.com/BerriAI/litellm
|
||||
|
|
|
|||
Binary file not shown.
|
After Width: | Height: | Size: 509 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 445 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 296 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 281 KiB |
|
|
@ -0,0 +1,2 @@
|
|||
-- AlterTable
|
||||
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "instructions" TEXT;
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
-- CreateIndex (CONCURRENTLY)
|
||||
--
|
||||
-- Disclaimer:
|
||||
-- - CREATE INDEX CONCURRENTLY cannot run inside a transaction. This migration must stay a
|
||||
-- single statement so Prisma Migrate on PostgreSQL can apply it outside a transaction.
|
||||
-- - Builds are slower and use more I/O than a blocking CREATE INDEX; if the build is
|
||||
-- interrupted, Postgres may leave an INVALID index that must be dropped and recreated.
|
||||
-- - Do not edit this file after it has been applied to any database: Prisma checksums
|
||||
-- migrations; add a new migration instead.
|
||||
-- - Requires PostgreSQL that supports CONCURRENTLY with IF NOT EXISTS (use a new migration
|
||||
-- without IF NOT EXISTS if you must support older versions).
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_HealthCheckTable_model_id_model_name_checked_at_idx" ON "LiteLLM_HealthCheckTable"("model_id", "model_name", "checked_at" DESC);
|
||||
|
|
@ -289,6 +289,7 @@ model LiteLLM_MCPServerTable {
|
|||
server_name String?
|
||||
alias String?
|
||||
description String?
|
||||
instructions String?
|
||||
url String?
|
||||
spec_path String?
|
||||
transport String @default("sse")
|
||||
|
|
@ -1045,6 +1046,7 @@ model LiteLLM_HealthCheckTable {
|
|||
@@index([model_name])
|
||||
@@index([checked_at])
|
||||
@@index([status])
|
||||
@@index([model_id, model_name, checked_at(sort: Desc)], map: "LiteLLM_HealthCheckTable_model_id_model_name_checked_at_idx")
|
||||
}
|
||||
|
||||
// Search Tools table for storing search tool configurations
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.65"
|
||||
version = "0.4.66"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -25,7 +25,7 @@ required-version = "==0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.65"
|
||||
version = "0.4.66"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-proxy-extras==",
|
||||
|
|
|
|||
13
litellm/_internal_context.py
Normal file
13
litellm/_internal_context.py
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
"""
|
||||
Internal request context for LiteLLM.
|
||||
|
||||
Provides a ContextVar-based mechanism for internal signals that must not
|
||||
be settable from user input. Context variables are scoped to the current
|
||||
asyncio task and cannot be injected via HTTP request bodies.
|
||||
"""
|
||||
|
||||
from contextvars import ContextVar
|
||||
|
||||
# When True, suppresses async logging and billing for internal sub-calls
|
||||
# (e.g., emulated file-search steps that make nested LLM calls).
|
||||
is_internal_call: ContextVar[bool] = ContextVar("is_internal_call", default=False)
|
||||
|
|
@ -86,6 +86,8 @@ _SECRET_RE = _build_secret_patterns()
|
|||
|
||||
|
||||
def _redact_string(value: str) -> str:
|
||||
if not _ENABLE_SECRET_REDACTION:
|
||||
return value
|
||||
return _SECRET_RE.sub(_REDACTED, value)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -71,12 +71,12 @@
|
|||
"computer-use-2025-01-24": "computer-use-2025-01-24",
|
||||
"computer-use-2025-11-24": "computer-use-2025-11-24",
|
||||
"context-1m-2025-08-07": "context-1m-2025-08-07",
|
||||
"context-management-2025-06-27": "context-management-2025-06-27",
|
||||
"context-management-2025-06-27": null,
|
||||
"effort-2025-11-24": null,
|
||||
"fast-mode-2026-02-01": null,
|
||||
"files-api-2025-04-14": null,
|
||||
"fine-grained-tool-streaming-2025-05-14": null,
|
||||
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
|
||||
"interleaved-thinking-2025-05-14": null,
|
||||
"mcp-client-2025-11-20": null,
|
||||
"mcp-client-2025-04-04": null,
|
||||
"mcp-servers-2025-12-04": null,
|
||||
|
|
@ -102,12 +102,12 @@
|
|||
"computer-use-2025-01-24": "computer-use-2025-01-24",
|
||||
"computer-use-2025-11-24": "computer-use-2025-11-24",
|
||||
"context-1m-2025-08-07": "context-1m-2025-08-07",
|
||||
"context-management-2025-06-27": "context-management-2025-06-27",
|
||||
"context-management-2025-06-27": null,
|
||||
"effort-2025-11-24": null,
|
||||
"fast-mode-2026-02-01": null,
|
||||
"files-api-2025-04-14": null,
|
||||
"fine-grained-tool-streaming-2025-05-14": null,
|
||||
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
|
||||
"interleaved-thinking-2025-05-14": null,
|
||||
"mcp-client-2025-11-20": null,
|
||||
"mcp-client-2025-04-04": null,
|
||||
"mcp-servers-2025-12-04": null,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import os
|
||||
import sys
|
||||
from typing import List, Literal
|
||||
from typing import List, Literal, Optional
|
||||
|
||||
from litellm.litellm_core_utils.env_utils import get_env_int
|
||||
|
||||
|
|
@ -413,7 +413,20 @@ MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int(
|
|||
)
|
||||
DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000))
|
||||
#### Networking settings ####
|
||||
request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", 6000)) # time in seconds
|
||||
# Sentinel used when `REQUEST_TIMEOUT` is unset: `litellm.request_timeout` keeps this
|
||||
# value so longer-running surfaces (Router `timeout or litellm.request_timeout`,
|
||||
# speech/TTS, responses, vector stores, etc.) get a long HTTP deadline. Chat
|
||||
# `completion()` maps this sentinel down to 600s when the caller did not set a
|
||||
# per-request/model timeout—see ``CompletionTimeout.resolve`` in completion_timeout.py. MCP uses
|
||||
# dedicated timeouts (e.g. `MCP_CLIENT_TIMEOUT`), not `request_timeout`.
|
||||
DEFAULT_REQUEST_TIMEOUT_SECONDS: float = 6000.0
|
||||
# Pair used for default httpx clients when no custom timeout is passed: read/write
|
||||
# deadline and connect handshake (see ``http_handler`` cached handler paths).
|
||||
COMPLETION_HTTP_FALLBACK_SECONDS: float = 600.0
|
||||
HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS: float = 5.0
|
||||
request_timeout: float = float(
|
||||
os.getenv("REQUEST_TIMEOUT", str(int(DEFAULT_REQUEST_TIMEOUT_SECONDS)))
|
||||
)
|
||||
DEFAULT_A2A_AGENT_TIMEOUT: float = float(
|
||||
os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000)
|
||||
) # 10 minutes
|
||||
|
|
@ -1113,6 +1126,7 @@ BEDROCK_CONVERSE_MODELS = [
|
|||
"openai.gpt-oss-120b-1:0",
|
||||
"anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
"anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
"anthropic.claude-opus-4-7",
|
||||
"anthropic.claude-opus-4-6-v1:0",
|
||||
"anthropic.claude-opus-4-6-v1",
|
||||
"anthropic.claude-sonnet-4-6",
|
||||
|
|
@ -1330,6 +1344,22 @@ BATCH_STATUS_POLL_MAX_ATTEMPTS = int(
|
|||
HEALTH_CHECK_TIMEOUT_SECONDS = int(
|
||||
os.getenv("HEALTH_CHECK_TIMEOUT_SECONDS", 60)
|
||||
) # 60 seconds
|
||||
_background_health_check_max_tokens_env = os.getenv(
|
||||
"BACKGROUND_HEALTH_CHECK_MAX_TOKENS"
|
||||
)
|
||||
try:
|
||||
_raw_background_health_check_max_tokens = (
|
||||
_background_health_check_max_tokens_env.strip()
|
||||
if _background_health_check_max_tokens_env is not None
|
||||
else ""
|
||||
)
|
||||
BACKGROUND_HEALTH_CHECK_MAX_TOKENS: Optional[int] = (
|
||||
int(_raw_background_health_check_max_tokens)
|
||||
if _raw_background_health_check_max_tokens
|
||||
else None
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
BACKGROUND_HEALTH_CHECK_MAX_TOKENS = None
|
||||
LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME = "litellm-internal-health-check"
|
||||
LITTELM_CLI_SERVICE_ACCOUNT_NAME = "litellm-cli"
|
||||
LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME = "litellm_internal_jobs"
|
||||
|
|
|
|||
|
|
@ -966,6 +966,8 @@ def _store_cost_breakdown_in_logging_obj(
|
|||
margin_percent: Optional[float] = None,
|
||||
margin_fixed_amount: Optional[float] = None,
|
||||
margin_total_amount: Optional[float] = None,
|
||||
cache_read_cost: Optional[float] = None,
|
||||
cache_creation_cost: Optional[float] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Helper function to store cost breakdown in the logging object.
|
||||
|
|
@ -1001,6 +1003,8 @@ def _store_cost_breakdown_in_logging_obj(
|
|||
margin_percent=margin_percent,
|
||||
margin_fixed_amount=margin_fixed_amount,
|
||||
margin_total_amount=margin_total_amount,
|
||||
cache_read_cost=cache_read_cost,
|
||||
cache_creation_cost=cache_creation_cost,
|
||||
)
|
||||
|
||||
except Exception as breakdown_error:
|
||||
|
|
@ -1599,6 +1603,22 @@ def completion_cost( # noqa: PLR0915
|
|||
|
||||
# Store cost breakdown in logging object if available
|
||||
if litellm_logging_obj is not None:
|
||||
_cache_read_cost: Optional[float] = None
|
||||
_cache_creation_cost: Optional[float] = None
|
||||
if cost_per_token_usage_object is not None:
|
||||
_cr = getattr(cost_per_token_usage_object, "cache_read_input_tokens", None) or (cost_per_token_usage_object.model_extra or {}).get("cache_read_input_tokens")
|
||||
_cc = getattr(cost_per_token_usage_object, "cache_creation_input_tokens", None) or (cost_per_token_usage_object.model_extra or {}).get("cache_creation_input_tokens")
|
||||
if (_cr or _cc) and model:
|
||||
try:
|
||||
_mi = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
_cr_rate = _mi.get("cache_read_input_token_cost")
|
||||
if _cr and _cr_rate is not None:
|
||||
_cache_read_cost = float(_cr) * float(_cr_rate)
|
||||
_cc_rate = _mi.get("cache_creation_input_token_cost")
|
||||
if _cc and _cc_rate is not None:
|
||||
_cache_creation_cost = float(_cc) * float(_cc_rate)
|
||||
except Exception:
|
||||
pass
|
||||
_store_cost_breakdown_in_logging_obj(
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
prompt_tokens_cost_usd_dollar=prompt_tokens_cost_usd_dollar,
|
||||
|
|
@ -1612,6 +1632,8 @@ def completion_cost( # noqa: PLR0915
|
|||
margin_percent=margin_percent,
|
||||
margin_fixed_amount=margin_fixed_amount,
|
||||
margin_total_amount=margin_total_amount,
|
||||
cache_read_cost=_cache_read_cost,
|
||||
cache_creation_cost=_cache_creation_cost,
|
||||
)
|
||||
|
||||
return _final_cost
|
||||
|
|
|
|||
|
|
@ -221,6 +221,7 @@ class MCPClient:
|
|||
self.extra_headers: Optional[Dict[str, str]] = extra_headers
|
||||
self.ssl_verify: Optional[VerifyTypes] = ssl_verify
|
||||
self._aws_auth: Optional[httpx.Auth] = aws_auth
|
||||
self._last_initialize_instructions: Optional[str] = None
|
||||
# handle the basic auth value if provided
|
||||
if auth_value:
|
||||
self.update_auth_value(auth_value)
|
||||
|
|
@ -296,7 +297,12 @@ class MCPClient:
|
|||
session_ctx = ClientSession(read_stream, write_stream)
|
||||
session = await session_ctx.__aenter__()
|
||||
try:
|
||||
await session.initialize()
|
||||
init_result = await session.initialize()
|
||||
self._last_initialize_instructions = None
|
||||
if init_result is not None:
|
||||
ins = getattr(init_result, "instructions", None)
|
||||
if isinstance(ins, str) and ins.strip():
|
||||
self._last_initialize_instructions = ins.strip()
|
||||
return await operation(session)
|
||||
finally:
|
||||
try:
|
||||
|
|
@ -315,6 +321,7 @@ class MCPClient:
|
|||
"""Open a session, run the provided coroutine, and clean up."""
|
||||
http_client: Optional[httpx.AsyncClient] = None
|
||||
try:
|
||||
self._last_initialize_instructions = None
|
||||
transport_ctx, http_client = self._create_transport_context()
|
||||
return await self._execute_session_operation(transport_ctx, operation)
|
||||
except Exception:
|
||||
|
|
|
|||
83
litellm/litellm_core_utils/completion_timeout.py
Normal file
83
litellm/litellm_core_utils/completion_timeout.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
"""Completion HTTP timeout resolution (kept out of ``main.py`` to limit import cycles)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Callable, Optional, Union
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.constants import (
|
||||
COMPLETION_HTTP_FALLBACK_SECONDS,
|
||||
DEFAULT_REQUEST_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
class CompletionTimeout:
|
||||
"""Resolves HTTP timeout for ``completion()`` from model vs global settings."""
|
||||
|
||||
@staticmethod
|
||||
def _fallback_when_no_explicit_timeout(
|
||||
global_timeout: Optional[Union[float, str]],
|
||||
) -> float:
|
||||
"""
|
||||
Used when ``model_timeout`` and kwargs timeouts are all unset.
|
||||
|
||||
``global_timeout`` is :attr:`litellm.request_timeout` (numeric / string), not
|
||||
:class:`httpx.Timeout`.
|
||||
|
||||
If it equals :data:`~litellm.constants.DEFAULT_REQUEST_TIMEOUT_SECONDS` (6000),
|
||||
return :data:`~litellm.constants.COMPLETION_HTTP_FALLBACK_SECONDS`. Same if
|
||||
``None``. Otherwise return ``float(global_timeout)``.
|
||||
"""
|
||||
if global_timeout is None:
|
||||
return COMPLETION_HTTP_FALLBACK_SECONDS
|
||||
if float(global_timeout) == float(DEFAULT_REQUEST_TIMEOUT_SECONDS):
|
||||
return COMPLETION_HTTP_FALLBACK_SECONDS
|
||||
return float(global_timeout)
|
||||
|
||||
@staticmethod
|
||||
def resolve(
|
||||
model_timeout: Optional[Union[float, str, httpx.Timeout]],
|
||||
kwargs: dict,
|
||||
custom_llm_provider: str,
|
||||
*,
|
||||
global_timeout: Optional[Union[float, str]],
|
||||
supports_httpx_timeout: Callable[[str], bool],
|
||||
) -> Union[float, httpx.Timeout]:
|
||||
"""
|
||||
Resolution order (first non-None wins):
|
||||
|
||||
1. ``model_timeout`` (call argument / merged ``litellm_params``)
|
||||
2. ``kwargs["timeout"]``
|
||||
3. ``kwargs["request_timeout"]``
|
||||
4. Fallback from ``global_timeout`` (:attr:`litellm.request_timeout`) — if it is
|
||||
the package default (6000), use 600 instead.
|
||||
|
||||
Coerce :class:`httpx.Timeout` when the provider does not support it.
|
||||
Explicit ``6000`` on the model or in kwargs is kept as ``6000``.
|
||||
"""
|
||||
resolved: Union[float, str, httpx.Timeout]
|
||||
if model_timeout is not None:
|
||||
resolved = model_timeout
|
||||
elif kwargs.get("timeout") is not None:
|
||||
resolved = kwargs["timeout"]
|
||||
elif kwargs.get("request_timeout") is not None:
|
||||
resolved = kwargs["request_timeout"]
|
||||
else:
|
||||
resolved = CompletionTimeout._fallback_when_no_explicit_timeout(
|
||||
global_timeout
|
||||
)
|
||||
|
||||
if isinstance(resolved, httpx.Timeout) and not supports_httpx_timeout(
|
||||
custom_llm_provider
|
||||
):
|
||||
read_timeout = resolved.read
|
||||
resolved = (
|
||||
float(read_timeout)
|
||||
if read_timeout is not None
|
||||
else COMPLETION_HTTP_FALLBACK_SECONDS
|
||||
) # default 10 min timeout
|
||||
elif not isinstance(resolved, httpx.Timeout):
|
||||
resolved = float(resolved) # type: ignore
|
||||
|
||||
return resolved
|
||||
|
|
@ -6,7 +6,7 @@ from typing import Any, Optional
|
|||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm._logging import _redact_string, verbose_logger
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
from ..exceptions import (
|
||||
|
|
@ -2304,7 +2304,7 @@ def exception_type( # type: ignore # noqa: PLR0915
|
|||
else:
|
||||
# if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
|
||||
raise APIConnectionError(
|
||||
message=f"{exception_provider} APIConnectionError - {message}\n{traceback.format_exc()}",
|
||||
message=f"{exception_provider} APIConnectionError - {message}\n{_redact_string(traceback.format_exc())}",
|
||||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
|
|
@ -2431,7 +2431,7 @@ def exception_type( # type: ignore # noqa: PLR0915
|
|||
else:
|
||||
raise APIConnectionError(
|
||||
message="{}\n{}".format(
|
||||
str(original_exception), traceback.format_exc()
|
||||
str(original_exception), _redact_string(traceback.format_exc())
|
||||
),
|
||||
llm_provider=custom_llm_provider,
|
||||
model=model,
|
||||
|
|
@ -2460,7 +2460,7 @@ def exception_type( # type: ignore # noqa: PLR0915
|
|||
setattr(e, "litellm_response_headers", litellm_response_headers)
|
||||
raise e # it's already mapped
|
||||
raised_exc = APIConnectionError(
|
||||
message="{}\n{}".format(original_exception, traceback.format_exc()),
|
||||
message="{}\n{}".format(original_exception, _redact_string(traceback.format_exc())),
|
||||
llm_provider="",
|
||||
model="",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ from litellm import (
|
|||
log_raw_request_response,
|
||||
turn_off_message_logging,
|
||||
)
|
||||
from litellm._logging import _is_debugging_on, verbose_logger
|
||||
from litellm._logging import _is_debugging_on, _redact_string, verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.batches.batch_utils import _handle_completed_batch
|
||||
from litellm.caching.caching import DualCache, InMemoryCache
|
||||
|
|
@ -354,9 +354,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
self.function_id = function_id
|
||||
self.streaming_chunks: List[Any] = [] # for generating complete stream response
|
||||
self.sync_streaming_chunks: List[
|
||||
Any
|
||||
] = [] # for generating complete stream response
|
||||
self.sync_streaming_chunks: List[Any] = (
|
||||
[]
|
||||
) # for generating complete stream response
|
||||
self.log_raw_request_response = log_raw_request_response
|
||||
|
||||
# Initialize dynamic callbacks
|
||||
|
|
@ -811,9 +811,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
prompt_spec=prompt_spec,
|
||||
dynamic_callback_params=dynamic_callback_params,
|
||||
):
|
||||
self.model_call_details[
|
||||
"prompt_integration"
|
||||
] = logger.__class__.__name__
|
||||
self.model_call_details["prompt_integration"] = (
|
||||
logger.__class__.__name__
|
||||
)
|
||||
return logger
|
||||
except Exception:
|
||||
# If check fails, continue to next logger
|
||||
|
|
@ -881,9 +881,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if anthropic_cache_control_logger := AnthropicCacheControlHook.get_custom_logger_for_anthropic_cache_control_hook(
|
||||
non_default_params
|
||||
):
|
||||
self.model_call_details[
|
||||
"prompt_integration"
|
||||
] = anthropic_cache_control_logger.__class__.__name__
|
||||
self.model_call_details["prompt_integration"] = (
|
||||
anthropic_cache_control_logger.__class__.__name__
|
||||
)
|
||||
return anthropic_cache_control_logger
|
||||
|
||||
#########################################################
|
||||
|
|
@ -895,9 +895,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
internal_usage_cache=None,
|
||||
llm_router=None,
|
||||
)
|
||||
self.model_call_details[
|
||||
"prompt_integration"
|
||||
] = vector_store_custom_logger.__class__.__name__
|
||||
self.model_call_details["prompt_integration"] = (
|
||||
vector_store_custom_logger.__class__.__name__
|
||||
)
|
||||
# Add to global callbacks so post-call hooks are invoked
|
||||
if (
|
||||
vector_store_custom_logger
|
||||
|
|
@ -957,9 +957,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
model
|
||||
): # if model name was changes pre-call, overwrite the initial model call name with the new one
|
||||
self.model_call_details["model"] = model
|
||||
self.model_call_details["litellm_params"][
|
||||
"api_base"
|
||||
] = self._get_masked_api_base(additional_args.get("api_base", ""))
|
||||
self.model_call_details["litellm_params"]["api_base"] = (
|
||||
self._get_masked_api_base(additional_args.get("api_base", ""))
|
||||
)
|
||||
|
||||
def pre_call(self, input, api_key, model=None, additional_args={}): # noqa: PLR0915
|
||||
# Log the exact input to the LLM API
|
||||
|
|
@ -988,10 +988,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
try:
|
||||
# [Non-blocking Extra Debug Information in metadata]
|
||||
if turn_off_message_logging is True:
|
||||
_metadata[
|
||||
"raw_request"
|
||||
] = "redacted by litellm. \
|
||||
_metadata["raw_request"] = (
|
||||
"redacted by litellm. \
|
||||
'litellm.turn_off_message_logging=True'"
|
||||
)
|
||||
else:
|
||||
curl_command = self._get_request_curl_command(
|
||||
api_base=additional_args.get("api_base", ""),
|
||||
|
|
@ -1002,34 +1002,34 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
_metadata["raw_request"] = str(curl_command)
|
||||
# split up, so it's easier to parse in the UI
|
||||
self.model_call_details[
|
||||
"raw_request_typed_dict"
|
||||
] = RawRequestTypedDict(
|
||||
raw_request_api_base=str(
|
||||
additional_args.get("api_base") or ""
|
||||
),
|
||||
raw_request_body=self._get_raw_request_body(
|
||||
additional_args.get("complete_input_dict", {})
|
||||
),
|
||||
# NOTE: setting ignore_sensitive_headers to True will cause
|
||||
# the Authorization header to be leaked when calls to the health
|
||||
# endpoint are made and fail.
|
||||
raw_request_headers=self._get_masked_headers(
|
||||
additional_args.get("headers", {}) or {},
|
||||
),
|
||||
error=None,
|
||||
self.model_call_details["raw_request_typed_dict"] = (
|
||||
RawRequestTypedDict(
|
||||
raw_request_api_base=str(
|
||||
additional_args.get("api_base") or ""
|
||||
),
|
||||
raw_request_body=self._get_raw_request_body(
|
||||
additional_args.get("complete_input_dict", {})
|
||||
),
|
||||
# NOTE: setting ignore_sensitive_headers to True will cause
|
||||
# the Authorization header to be leaked when calls to the health
|
||||
# endpoint are made and fail.
|
||||
raw_request_headers=self._get_masked_headers(
|
||||
additional_args.get("headers", {}) or {},
|
||||
),
|
||||
error=None,
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
self.model_call_details[
|
||||
"raw_request_typed_dict"
|
||||
] = RawRequestTypedDict(
|
||||
error=str(e),
|
||||
self.model_call_details["raw_request_typed_dict"] = (
|
||||
RawRequestTypedDict(
|
||||
error=str(e),
|
||||
)
|
||||
)
|
||||
_metadata[
|
||||
"raw_request"
|
||||
] = "Unable to Log \
|
||||
_metadata["raw_request"] = (
|
||||
"Unable to Log \
|
||||
raw request: {}".format(
|
||||
str(e)
|
||||
str(e)
|
||||
)
|
||||
)
|
||||
if getattr(self, "logger_fn", None) and callable(self.logger_fn):
|
||||
try:
|
||||
|
|
@ -1330,13 +1330,13 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
for callback in callbacks:
|
||||
try:
|
||||
if isinstance(callback, CustomLogger):
|
||||
response: Optional[
|
||||
MCPPostCallResponseObject
|
||||
] = await callback.async_post_mcp_tool_call_hook(
|
||||
kwargs=kwargs,
|
||||
response_obj=post_mcp_tool_call_response_obj,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
response: Optional[MCPPostCallResponseObject] = (
|
||||
await callback.async_post_mcp_tool_call_hook(
|
||||
kwargs=kwargs,
|
||||
response_obj=post_mcp_tool_call_response_obj,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
)
|
||||
######################################################################
|
||||
# if any of the callbacks modify the response, use the modified response
|
||||
|
|
@ -1387,6 +1387,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
margin_percent: Optional[float] = None,
|
||||
margin_fixed_amount: Optional[float] = None,
|
||||
margin_total_amount: Optional[float] = None,
|
||||
cache_read_cost: Optional[float] = None,
|
||||
cache_creation_cost: Optional[float] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Helper method to store cost breakdown in the logging object.
|
||||
|
|
@ -1411,6 +1413,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
total_cost=total_cost,
|
||||
tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar,
|
||||
)
|
||||
if cache_read_cost is not None and cache_read_cost > 0:
|
||||
self.cost_breakdown["cache_read_cost"] = cache_read_cost
|
||||
if cache_creation_cost is not None and cache_creation_cost > 0:
|
||||
self.cost_breakdown["cache_creation_cost"] = cache_creation_cost
|
||||
|
||||
# Store additional costs if provided (free-form dict for extensibility)
|
||||
if (
|
||||
|
|
@ -1537,9 +1543,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
verbose_logger.debug(
|
||||
f"response_cost_failure_debug_information: {debug_info}"
|
||||
)
|
||||
self.model_call_details[
|
||||
"response_cost_failure_debug_information"
|
||||
] = debug_info
|
||||
self.model_call_details["response_cost_failure_debug_information"] = (
|
||||
debug_info
|
||||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
|
|
@ -1565,9 +1571,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
verbose_logger.debug(
|
||||
f"response_cost_failure_debug_information: {debug_info}"
|
||||
)
|
||||
self.model_call_details[
|
||||
"response_cost_failure_debug_information"
|
||||
] = debug_info
|
||||
self.model_call_details["response_cost_failure_debug_information"] = (
|
||||
debug_info
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
|
@ -1716,9 +1722,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.model_call_details["litellm_params"].setdefault("metadata", {})
|
||||
if self.model_call_details["litellm_params"]["metadata"] is None:
|
||||
self.model_call_details["litellm_params"]["metadata"] = {}
|
||||
self.model_call_details["litellm_params"]["metadata"][
|
||||
"hidden_params"
|
||||
] = getattr(logging_result, "_hidden_params", {})
|
||||
self.model_call_details["litellm_params"]["metadata"]["hidden_params"] = (
|
||||
getattr(logging_result, "_hidden_params", {})
|
||||
)
|
||||
|
||||
def _process_hidden_params_and_response_cost(
|
||||
self,
|
||||
|
|
@ -1747,9 +1753,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
result=logging_result
|
||||
)
|
||||
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = self._build_standard_logging_payload(logging_result, start_time, end_time)
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
self._build_standard_logging_payload(logging_result, start_time, end_time)
|
||||
)
|
||||
|
||||
if (
|
||||
standard_logging_payload := self.model_call_details.get(
|
||||
|
|
@ -1827,9 +1833,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
end_time = datetime.datetime.now()
|
||||
if self.completion_start_time is None:
|
||||
self.completion_start_time = end_time
|
||||
self.model_call_details[
|
||||
"completion_start_time"
|
||||
] = self.completion_start_time
|
||||
self.model_call_details["completion_start_time"] = (
|
||||
self.completion_start_time
|
||||
)
|
||||
|
||||
self.model_call_details["log_event_type"] = "successful_api_call"
|
||||
self.model_call_details["end_time"] = end_time
|
||||
|
|
@ -1866,10 +1872,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
end_time=end_time,
|
||||
)
|
||||
elif isinstance(result, dict) or isinstance(result, list):
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = self._build_standard_logging_payload(
|
||||
result, start_time, end_time
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
self._build_standard_logging_payload(
|
||||
result, start_time, end_time
|
||||
)
|
||||
)
|
||||
if (
|
||||
standard_logging_payload := self.model_call_details.get(
|
||||
|
|
@ -1878,9 +1884,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
) is not None:
|
||||
emit_standard_logging_payload(standard_logging_payload)
|
||||
elif standard_logging_object is not None:
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = standard_logging_object
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
standard_logging_object
|
||||
)
|
||||
else:
|
||||
self.model_call_details["response_cost"] = None
|
||||
|
||||
|
|
@ -2038,20 +2044,20 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
verbose_logger.debug(
|
||||
"Logging Details LiteLLM-Success Call streaming complete"
|
||||
)
|
||||
self.model_call_details[
|
||||
"complete_streaming_response"
|
||||
] = complete_streaming_response
|
||||
self.model_call_details[
|
||||
"response_cost"
|
||||
] = self._response_cost_calculator(result=complete_streaming_response)
|
||||
self.model_call_details["complete_streaming_response"] = (
|
||||
complete_streaming_response
|
||||
)
|
||||
self.model_call_details["response_cost"] = (
|
||||
self._response_cost_calculator(result=complete_streaming_response)
|
||||
)
|
||||
self._merge_hidden_params_from_response_into_metadata(
|
||||
complete_streaming_response
|
||||
)
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = self._build_standard_logging_payload(
|
||||
complete_streaming_response, start_time, end_time
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
self._build_standard_logging_payload(
|
||||
complete_streaming_response, start_time, end_time
|
||||
)
|
||||
)
|
||||
if (
|
||||
standard_logging_payload := self.model_call_details.get(
|
||||
|
|
@ -2385,10 +2391,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
else:
|
||||
if self.stream and complete_streaming_response:
|
||||
self.model_call_details[
|
||||
"complete_response"
|
||||
] = self.model_call_details.get(
|
||||
"complete_streaming_response", {}
|
||||
self.model_call_details["complete_response"] = (
|
||||
self.model_call_details.get(
|
||||
"complete_streaming_response", {}
|
||||
)
|
||||
)
|
||||
result = self.model_call_details["complete_response"]
|
||||
openMeterLogger.log_success_event(
|
||||
|
|
@ -2412,10 +2418,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
else:
|
||||
if self.stream and complete_streaming_response:
|
||||
self.model_call_details[
|
||||
"complete_response"
|
||||
] = self.model_call_details.get(
|
||||
"complete_streaming_response", {}
|
||||
self.model_call_details["complete_response"] = (
|
||||
self.model_call_details.get(
|
||||
"complete_streaming_response", {}
|
||||
)
|
||||
)
|
||||
result = self.model_call_details["complete_response"]
|
||||
|
||||
|
|
@ -2554,9 +2560,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if complete_streaming_response is not None:
|
||||
print_verbose("Async success callbacks: Got a complete streaming response")
|
||||
|
||||
self.model_call_details[
|
||||
"async_complete_streaming_response"
|
||||
] = complete_streaming_response
|
||||
self.model_call_details["async_complete_streaming_response"] = (
|
||||
complete_streaming_response
|
||||
)
|
||||
|
||||
try:
|
||||
if self.model_call_details.get("cache_hit", False) is True:
|
||||
|
|
@ -2567,10 +2573,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
model_call_details=self.model_call_details
|
||||
)
|
||||
# base_model defaults to None if not set on model_info
|
||||
self.model_call_details[
|
||||
"response_cost"
|
||||
] = self._response_cost_calculator(
|
||||
result=complete_streaming_response
|
||||
self.model_call_details["response_cost"] = (
|
||||
self._response_cost_calculator(
|
||||
result=complete_streaming_response
|
||||
)
|
||||
)
|
||||
|
||||
verbose_logger.debug(
|
||||
|
|
@ -2587,10 +2593,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = self._build_standard_logging_payload(
|
||||
complete_streaming_response, start_time, end_time
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
self._build_standard_logging_payload(
|
||||
complete_streaming_response, start_time, end_time
|
||||
)
|
||||
)
|
||||
|
||||
# print standard logging payload
|
||||
|
|
@ -2617,9 +2623,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
# _success_handler_helper_fn
|
||||
if self.model_call_details.get("standard_logging_object") is None:
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = self._build_standard_logging_payload(result, start_time, end_time)
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
self._build_standard_logging_payload(result, start_time, end_time)
|
||||
)
|
||||
|
||||
# print standard logging payload
|
||||
if (
|
||||
|
|
@ -2848,7 +2854,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
self.model_call_details["log_event_type"] = "failed_api_call"
|
||||
self.model_call_details["exception"] = exception
|
||||
self.model_call_details["traceback_exception"] = traceback_exception
|
||||
self.model_call_details["traceback_exception"] = (
|
||||
_redact_string(traceback_exception)
|
||||
if isinstance(traceback_exception, str)
|
||||
else traceback_exception
|
||||
)
|
||||
self.model_call_details["end_time"] = end_time
|
||||
self.model_call_details.setdefault("original_response", None)
|
||||
self.model_call_details["response_cost"] = 0
|
||||
|
|
@ -2862,18 +2872,18 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = get_standard_logging_object_payload(
|
||||
kwargs=self.model_call_details,
|
||||
init_response_obj={},
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
logging_obj=self,
|
||||
status="failure",
|
||||
error_str=str(exception),
|
||||
original_exception=exception,
|
||||
standard_built_in_tools_params=self.standard_built_in_tools_params,
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
get_standard_logging_object_payload(
|
||||
kwargs=self.model_call_details,
|
||||
init_response_obj={},
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
logging_obj=self,
|
||||
status="failure",
|
||||
error_str=_redact_string(str(exception)),
|
||||
original_exception=exception,
|
||||
standard_built_in_tools_params=self.standard_built_in_tools_params,
|
||||
)
|
||||
)
|
||||
return start_time, end_time
|
||||
|
||||
|
|
@ -3843,9 +3853,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
|
|||
service_name=arize_config.project_name,
|
||||
)
|
||||
|
||||
os.environ[
|
||||
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
|
||||
] = f"space_id={arize_config.space_key or arize_config.space_id},api_key={arize_config.api_key}"
|
||||
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
|
||||
f"space_id={arize_config.space_key or arize_config.space_id},api_key={arize_config.api_key}"
|
||||
)
|
||||
for callback in _in_memory_loggers:
|
||||
if (
|
||||
isinstance(callback, ArizeLogger)
|
||||
|
|
@ -3871,13 +3881,13 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
|
|||
existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "")
|
||||
# Add openinference.project.name attribute
|
||||
if existing_attrs:
|
||||
os.environ[
|
||||
"OTEL_RESOURCE_ATTRIBUTES"
|
||||
] = f"{existing_attrs},openinference.project.name={arize_phoenix_config.project_name}"
|
||||
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
|
||||
f"{existing_attrs},openinference.project.name={arize_phoenix_config.project_name}"
|
||||
)
|
||||
else:
|
||||
os.environ[
|
||||
"OTEL_RESOURCE_ATTRIBUTES"
|
||||
] = f"openinference.project.name={arize_phoenix_config.project_name}"
|
||||
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
|
||||
f"openinference.project.name={arize_phoenix_config.project_name}"
|
||||
)
|
||||
|
||||
# Set Phoenix project name from environment variable
|
||||
phoenix_project_name = os.environ.get("PHOENIX_PROJECT_NAME", None)
|
||||
|
|
@ -3885,19 +3895,19 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
|
|||
existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "")
|
||||
# Add openinference.project.name attribute
|
||||
if existing_attrs:
|
||||
os.environ[
|
||||
"OTEL_RESOURCE_ATTRIBUTES"
|
||||
] = f"{existing_attrs},openinference.project.name={phoenix_project_name}"
|
||||
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
|
||||
f"{existing_attrs},openinference.project.name={phoenix_project_name}"
|
||||
)
|
||||
else:
|
||||
os.environ[
|
||||
"OTEL_RESOURCE_ATTRIBUTES"
|
||||
] = f"openinference.project.name={phoenix_project_name}"
|
||||
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
|
||||
f"openinference.project.name={phoenix_project_name}"
|
||||
)
|
||||
|
||||
# auth can be disabled on local deployments of arize phoenix
|
||||
if arize_phoenix_config.otlp_auth_headers is not None:
|
||||
os.environ[
|
||||
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
|
||||
] = arize_phoenix_config.otlp_auth_headers
|
||||
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
|
||||
arize_phoenix_config.otlp_auth_headers
|
||||
)
|
||||
|
||||
for callback in _in_memory_loggers:
|
||||
if (
|
||||
|
|
@ -4084,9 +4094,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
|
|||
exporter="otlp_http",
|
||||
endpoint="https://langtrace.ai/api/trace",
|
||||
)
|
||||
os.environ[
|
||||
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
|
||||
] = f"api_key={os.getenv('LANGTRACE_API_KEY')}"
|
||||
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
|
||||
f"api_key={os.getenv('LANGTRACE_API_KEY')}"
|
||||
)
|
||||
for callback in _in_memory_loggers:
|
||||
if (
|
||||
isinstance(callback, OpenTelemetry)
|
||||
|
|
@ -4981,16 +4991,22 @@ class StandardLoggingPayloadSetup:
|
|||
|
||||
additional_logging_headers: StandardLoggingAdditionalHeaders = {}
|
||||
|
||||
# Populate well-known typed fields with int/str coercion where needed
|
||||
typed_keys: dict = {}
|
||||
for key in StandardLoggingAdditionalHeaders.__annotations__.keys():
|
||||
_key = key.lower()
|
||||
_key = _key.replace("_", "-")
|
||||
_key = key.lower().replace("_", "-")
|
||||
typed_keys[_key] = key
|
||||
if _key in additiona_headers:
|
||||
try:
|
||||
additional_logging_headers[key] = int(additiona_headers[_key]) # type: ignore
|
||||
except (ValueError, TypeError):
|
||||
verbose_logger.debug(
|
||||
f"Could not convert {additiona_headers[_key]} to int for key {key}."
|
||||
)
|
||||
additional_logging_headers[key] = additiona_headers[_key] # type: ignore
|
||||
|
||||
# Preserve all remaining headers verbatim (e.g. llm_provider-x-request-id)
|
||||
for k, v in additiona_headers.items():
|
||||
if k.lower() not in typed_keys:
|
||||
additional_logging_headers[k] = v # type: ignore
|
||||
|
||||
return additional_logging_headers
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -5012,10 +5028,10 @@ class StandardLoggingPayloadSetup:
|
|||
for key in StandardLoggingHiddenParams.__annotations__.keys():
|
||||
if key in hidden_params:
|
||||
if key == "additional_headers":
|
||||
clean_hidden_params[
|
||||
"additional_headers"
|
||||
] = StandardLoggingPayloadSetup.get_additional_headers(
|
||||
hidden_params[key]
|
||||
clean_hidden_params["additional_headers"] = (
|
||||
StandardLoggingPayloadSetup.get_additional_headers(
|
||||
hidden_params[key]
|
||||
)
|
||||
)
|
||||
else:
|
||||
clean_hidden_params[key] = hidden_params[key] # type: ignore
|
||||
|
|
@ -5656,9 +5672,9 @@ def scrub_sensitive_keys_in_metadata(litellm_params: Optional[dict]):
|
|||
):
|
||||
for k, v in metadata["user_api_key_metadata"].items():
|
||||
if k == "logging": # prevent logging user logging keys
|
||||
cleaned_user_api_key_metadata[
|
||||
k
|
||||
] = "scrubbed_by_litellm_for_sensitive_keys"
|
||||
cleaned_user_api_key_metadata[k] = (
|
||||
"scrubbed_by_litellm_for_sensitive_keys"
|
||||
)
|
||||
else:
|
||||
cleaned_user_api_key_metadata[k] = v
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,10 @@ from openai.types.completion_create_params import (
|
|||
CompletionCreateParamsStreaming as TextCompletionCreateParamsStreaming,
|
||||
)
|
||||
from openai.types.embedding_create_params import EmbeddingCreateParams
|
||||
from openai.types.responses.response_create_params import (
|
||||
ResponseCreateParamsNonStreaming,
|
||||
ResponseCreateParamsStreaming,
|
||||
)
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.types.rerank import RerankRequest
|
||||
|
|
@ -65,6 +69,9 @@ class ModelParamHelper:
|
|||
ModelParamHelper._get_litellm_supported_transcription_kwargs()
|
||||
)
|
||||
rerank_kwargs = ModelParamHelper._get_litellm_supported_rerank_kwargs()
|
||||
responses_api_kwargs = (
|
||||
ModelParamHelper._get_litellm_supported_responses_api_kwargs()
|
||||
)
|
||||
exclude_kwargs = ModelParamHelper._get_exclude_kwargs()
|
||||
|
||||
combined_kwargs = chat_completion_kwargs.union(
|
||||
|
|
@ -72,6 +79,7 @@ class ModelParamHelper:
|
|||
embedding_kwargs,
|
||||
transcription_kwargs,
|
||||
rerank_kwargs,
|
||||
responses_api_kwargs,
|
||||
)
|
||||
combined_kwargs = combined_kwargs.difference(exclude_kwargs)
|
||||
return combined_kwargs
|
||||
|
|
@ -93,9 +101,9 @@ class ModelParamHelper:
|
|||
streaming_params: Set[str] = set(
|
||||
getattr(CompletionCreateParamsStreaming, "__annotations__", {}).keys()
|
||||
)
|
||||
litellm_provider_specific_params: Set[
|
||||
str
|
||||
] = ModelParamHelper.get_litellm_provider_specific_params_for_chat_params()
|
||||
litellm_provider_specific_params: Set[str] = (
|
||||
ModelParamHelper.get_litellm_provider_specific_params_for_chat_params()
|
||||
)
|
||||
all_chat_completion_kwargs: Set[str] = non_streaming_params.union(
|
||||
streaming_params
|
||||
).union(litellm_provider_specific_params)
|
||||
|
|
@ -167,6 +175,21 @@ class ModelParamHelper:
|
|||
verbose_logger.debug("Error getting transcription kwargs %s", str(e))
|
||||
return set()
|
||||
|
||||
@staticmethod
|
||||
def _get_litellm_supported_responses_api_kwargs() -> Set[str]:
|
||||
"""
|
||||
Get the litellm supported responses API kwargs
|
||||
|
||||
This follows the OpenAI API Spec
|
||||
"""
|
||||
non_streaming_params: Set[str] = set(
|
||||
getattr(ResponseCreateParamsNonStreaming, "__annotations__", {}).keys()
|
||||
)
|
||||
streaming_params: Set[str] = set(
|
||||
getattr(ResponseCreateParamsStreaming, "__annotations__", {}).keys()
|
||||
)
|
||||
return non_streaming_params.union(streaming_params)
|
||||
|
||||
@staticmethod
|
||||
def _get_exclude_kwargs() -> Set[str]:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ from litellm.types.utils import (
|
|||
from litellm.utils import (
|
||||
ModelResponse,
|
||||
Usage,
|
||||
_supports_factory,
|
||||
add_dummy_tool,
|
||||
any_assistant_message_has_thinking_blocks,
|
||||
get_max_tokens,
|
||||
|
|
@ -189,6 +190,30 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
v in model_lower for v in ("opus-4-6", "opus_4_6", "opus-4.6", "opus_4.6")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_opus_4_7_model(model: str) -> bool:
|
||||
"""Check if the model is specifically Claude Opus 4.7."""
|
||||
model_lower = model.lower()
|
||||
return any(
|
||||
v in model_lower for v in ("opus-4-7", "opus_4_7", "opus-4.7", "opus_4.7")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_effort_level(model: str, level: str) -> bool:
|
||||
"""Check ``supports_{level}_reasoning_effort`` in the model map.
|
||||
|
||||
Mirrors the pattern used in ``openai/chat/gpt_5_transformation.py`` so
|
||||
that adding support for a new effort level is a pure model-map change.
|
||||
"""
|
||||
try:
|
||||
return _supports_factory(
|
||||
model=model,
|
||||
custom_llm_provider="anthropic",
|
||||
key=f"supports_{level}_reasoning_effort",
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def get_supported_openai_params(self, model: str):
|
||||
params = [
|
||||
"stream",
|
||||
|
|
@ -212,6 +237,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
if (
|
||||
"claude-3-7-sonnet" in model
|
||||
or AnthropicConfig._is_claude_4_6_model(model)
|
||||
or AnthropicConfig._is_claude_4_7_model(model)
|
||||
or supports_reasoning(
|
||||
model=model,
|
||||
custom_llm_provider=self.custom_llm_provider,
|
||||
|
|
@ -771,7 +797,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
) -> Optional[AnthropicThinkingParam]:
|
||||
if reasoning_effort is None or reasoning_effort == "none":
|
||||
return None
|
||||
if AnthropicConfig._is_claude_4_6_model(model):
|
||||
if AnthropicConfig._is_claude_4_6_model(
|
||||
model
|
||||
) or AnthropicConfig._is_claude_4_7_model(model):
|
||||
return AnthropicThinkingParam(
|
||||
type="adaptive",
|
||||
)
|
||||
|
|
@ -1020,6 +1048,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
"opus-4-5",
|
||||
"opus-4.6",
|
||||
"opus-4-6",
|
||||
"opus-4.7",
|
||||
"opus-4-7",
|
||||
"sonnet-4.6",
|
||||
"sonnet-4-6",
|
||||
"sonnet_4.6",
|
||||
|
|
@ -1061,14 +1091,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
|
||||
reasoning_effort=value, model=model
|
||||
)
|
||||
# For Claude 4.6 models, effort is controlled via output_config,
|
||||
# For Claude 4.6+ models, effort is controlled via output_config,
|
||||
# not thinking budget_tokens. Map reasoning_effort to output_config.
|
||||
if AnthropicConfig._is_claude_4_6_model(model):
|
||||
if AnthropicConfig._is_claude_4_6_model(
|
||||
model
|
||||
) or AnthropicConfig._is_claude_4_7_model(model):
|
||||
effort_map = {
|
||||
"low": "low",
|
||||
"minimal": "low",
|
||||
"medium": "medium",
|
||||
"high": "high",
|
||||
"xhigh": "xhigh",
|
||||
"max": "max",
|
||||
}
|
||||
mapped_effort = effort_map.get(value, value)
|
||||
|
|
@ -1495,13 +1528,24 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
if not output_config or not isinstance(output_config, dict):
|
||||
return
|
||||
effort = output_config.get("effort")
|
||||
if effort and effort not in ["high", "medium", "low", "max"]:
|
||||
valid_efforts = ["high", "medium", "low", "xhigh", "max"]
|
||||
if effort and effort not in valid_efforts:
|
||||
raise ValueError(
|
||||
f"Invalid effort value: {effort}. Must be one of: 'high', 'medium', 'low', 'max'"
|
||||
f"Invalid effort value: {effort}. Must be one of: "
|
||||
f"'high', 'medium', 'low', 'xhigh', 'max'"
|
||||
)
|
||||
# ``max`` is Claude Opus 4.6 only (not Sonnet 4.6, not Opus 4.5/4.7).
|
||||
# Keep this hardcoded so the error message is specific and stable.
|
||||
if effort == "max" and not self._is_opus_4_6_model(model):
|
||||
raise ValueError(
|
||||
f"effort='max' is only supported by Claude Opus 4.6. Got model: {model}"
|
||||
f"effort='max' is only supported by Claude Opus 4.6. "
|
||||
f"Got model: {model}"
|
||||
)
|
||||
# ``xhigh`` is data-driven via ``supports_xhigh_reasoning_effort`` so
|
||||
# enabling it for a new model is a pure model-map change.
|
||||
if effort == "xhigh" and not self._supports_effort_level(model, "xhigh"):
|
||||
raise ValueError(
|
||||
f"effort='xhigh' is not supported by this model. Got model: {model}"
|
||||
)
|
||||
data["output_config"] = output_config
|
||||
|
||||
|
|
@ -1702,10 +1746,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
),
|
||||
)
|
||||
|
||||
raw_input_tokens = usage_object.get("input_tokens", 0) or 0
|
||||
prompt_tokens_details = PromptTokensDetailsWrapper(
|
||||
cached_tokens=cache_read_input_tokens,
|
||||
cache_creation_tokens=cache_creation_input_tokens,
|
||||
cache_creation_token_details=cache_creation_token_details,
|
||||
text_tokens=raw_input_tokens,
|
||||
)
|
||||
# Always populate completion_token_details, not just when there's reasoning_content
|
||||
reasoning_tokens = (
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
This file contains common utils for anthropic calls.
|
||||
"""
|
||||
|
||||
import copy
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import httpx
|
||||
|
|
@ -256,6 +257,27 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_claude_4_7_model(model: str) -> bool:
|
||||
"""Check if the model is a Claude 4.7 model (Opus 4.7)."""
|
||||
model_lower = model.lower()
|
||||
return any(
|
||||
v in model_lower
|
||||
for v in (
|
||||
"opus-4-7",
|
||||
"opus_4_7",
|
||||
"opus-4.7",
|
||||
"opus_4.7",
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_adaptive_thinking_model(model: str) -> bool:
|
||||
"""Claude 4.6+ models use adaptive thinking with output_config effort."""
|
||||
return AnthropicModelInfo._is_claude_4_6_model(
|
||||
model
|
||||
) or AnthropicModelInfo._is_claude_4_7_model(model)
|
||||
|
||||
def is_effort_used(
|
||||
self, optional_params: Optional[dict], model: Optional[str] = None
|
||||
) -> bool:
|
||||
|
|
@ -263,14 +285,14 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
Check if effort parameter is being used and requires a beta header.
|
||||
|
||||
Returns True if effort-related parameters are present and
|
||||
the model requires the effort beta header. Claude 4.6 models
|
||||
the model requires the effort beta header. Claude 4.6+ models
|
||||
use output_config as a stable API feature — no beta header needed.
|
||||
"""
|
||||
if not optional_params:
|
||||
return False
|
||||
|
||||
# Claude 4.6 models use output_config as a stable API feature — no beta header needed
|
||||
if model and self._is_claude_4_6_model(model):
|
||||
# Claude 4.6+ models use output_config as a stable API feature — no beta header needed
|
||||
if model and self._is_adaptive_thinking_model(model):
|
||||
return False
|
||||
|
||||
# Check if reasoning_effort is provided for Claude Opus 4.5
|
||||
|
|
@ -736,6 +758,69 @@ def strip_advisor_blocks_from_messages(
|
|||
return messages
|
||||
|
||||
|
||||
def is_anthropic_invalid_thinking_signature_error(error_text: str) -> bool:
|
||||
"""
|
||||
Detect Anthropic 400 when encrypted thinking signatures in history do not match
|
||||
the current deployment (e.g. user rotated API key or switched model endpoint).
|
||||
|
||||
Example API message:
|
||||
messages.N.content.M: Invalid `signature` in `thinking` block
|
||||
"""
|
||||
if not error_text:
|
||||
return False
|
||||
lower = error_text.lower()
|
||||
return (
|
||||
"invalid" in lower
|
||||
and "signature" in lower
|
||||
and "thinking" in lower
|
||||
and "block" in lower
|
||||
)
|
||||
|
||||
|
||||
def strip_thinking_blocks_from_anthropic_messages(messages: List[Any]) -> List[Any]:
|
||||
"""
|
||||
Return a new message list with thinking / redacted_thinking content blocks removed
|
||||
from each message. Used to recover from invalid thinking signatures on retry.
|
||||
|
||||
Messages whose content is a list and becomes empty after stripping are omitted,
|
||||
since Anthropic rejects empty content arrays.
|
||||
"""
|
||||
out: List[Any] = []
|
||||
for m in messages:
|
||||
if not isinstance(m, dict):
|
||||
out.append(m)
|
||||
continue
|
||||
mm = copy.deepcopy(m)
|
||||
content = mm.get("content")
|
||||
if isinstance(content, list):
|
||||
filtered = [
|
||||
b
|
||||
for b in content
|
||||
if not (
|
||||
isinstance(b, dict)
|
||||
and b.get("type") in ("thinking", "redacted_thinking")
|
||||
)
|
||||
]
|
||||
if not filtered:
|
||||
continue
|
||||
mm["content"] = filtered
|
||||
out.append(mm)
|
||||
return out
|
||||
|
||||
|
||||
def strip_thinking_blocks_from_anthropic_messages_request_dict(
|
||||
data: Dict[str, Any],
|
||||
) -> None:
|
||||
"""
|
||||
Mutate an Anthropic Messages-style request dict: strip thinking blocks from
|
||||
``messages`` and remove the top-level ``thinking`` extended-thinking param.
|
||||
"""
|
||||
msgs = data.get("messages")
|
||||
if isinstance(msgs, list):
|
||||
data["messages"] = strip_thinking_blocks_from_anthropic_messages(msgs)
|
||||
data.pop("thinking", None)
|
||||
|
||||
|
||||
def process_anthropic_headers(headers: Union[httpx.Headers, dict]) -> dict:
|
||||
openai_headers = {}
|
||||
if "anthropic-ratelimit-requests-limit" in headers:
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ class BaseAnthropicMessagesStreamingIterator:
|
|||
self.litellm_logging_obj = litellm_logging_obj
|
||||
self.request_body = request_body
|
||||
self.start_time = datetime.now()
|
||||
self.completion_start_time: datetime | None = None
|
||||
|
||||
async def _handle_streaming_logging(self, collected_chunks: List[bytes]):
|
||||
"""Handle the logging after all chunks have been collected."""
|
||||
|
|
@ -35,6 +36,15 @@ class BaseAnthropicMessagesStreamingIterator:
|
|||
)
|
||||
|
||||
end_time = datetime.now()
|
||||
# Set completion_start_time so TTFT is calculated from the first
|
||||
# chunk rather than falling back to end_time in async_success_handler.
|
||||
if self.completion_start_time is not None:
|
||||
self.litellm_logging_obj.completion_start_time = (
|
||||
self.completion_start_time
|
||||
)
|
||||
self.litellm_logging_obj.model_call_details[
|
||||
"completion_start_time"
|
||||
] = self.completion_start_time
|
||||
asyncio.create_task(
|
||||
PassThroughStreamingHandler._route_streaming_logging_to_handler(
|
||||
litellm_logging_obj=self.litellm_logging_obj,
|
||||
|
|
@ -100,6 +110,8 @@ class BaseAnthropicMessagesStreamingIterator:
|
|||
collected_chunks = []
|
||||
|
||||
async for chunk in completion_stream:
|
||||
if self.completion_start_time is None:
|
||||
self.completion_start_time = datetime.now()
|
||||
encoded_chunk = self._convert_chunk_to_sse_format(chunk)
|
||||
collected_chunks.append(encoded_chunk)
|
||||
yield encoded_chunk
|
||||
|
|
|
|||
|
|
@ -166,6 +166,36 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
|
||||
return headers, api_base
|
||||
|
||||
@staticmethod
|
||||
def _translate_legacy_thinking_for_adaptive_model(
|
||||
model: str, optional_params: Dict
|
||||
) -> None:
|
||||
"""Translate legacy ``thinking.type=enabled`` to adaptive for 4.6/4.7.
|
||||
Caller-provided ``output_config.effort`` is never overridden.
|
||||
"""
|
||||
if not AnthropicModelInfo._is_adaptive_thinking_model(model):
|
||||
return
|
||||
thinking = optional_params.get("thinking")
|
||||
if not isinstance(thinking, dict) or thinking.get("type") != "enabled":
|
||||
return
|
||||
|
||||
budget = int(thinking.get("budget_tokens") or 0)
|
||||
if budget >= 24000:
|
||||
effort = "xhigh"
|
||||
elif budget >= 10000:
|
||||
effort = "high"
|
||||
elif budget >= 5000:
|
||||
effort = "medium"
|
||||
else:
|
||||
effort = "low"
|
||||
|
||||
optional_params["thinking"] = {"type": "adaptive"}
|
||||
existing_output_config = optional_params.get("output_config")
|
||||
if not isinstance(existing_output_config, dict):
|
||||
existing_output_config = {}
|
||||
existing_output_config.setdefault("effort", effort)
|
||||
optional_params["output_config"] = existing_output_config
|
||||
|
||||
def transform_anthropic_messages_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -187,6 +217,11 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
status_code=400,
|
||||
)
|
||||
|
||||
self._translate_legacy_thinking_for_adaptive_model(
|
||||
model=model,
|
||||
optional_params=anthropic_messages_optional_request_params,
|
||||
)
|
||||
|
||||
# Filter out x-anthropic-billing-header from system messages
|
||||
system_param = anthropic_messages_optional_request_params.get("system")
|
||||
if system_param is not None:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
from typing import TYPE_CHECKING, List, Optional, Tuple
|
||||
|
||||
import httpx
|
||||
from httpx import Response
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM
|
||||
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
|
@ -11,6 +13,8 @@ from litellm.types.router import GenericLiteLLMParams
|
|||
if TYPE_CHECKING:
|
||||
from httpx import URL
|
||||
|
||||
from litellm.types.utils import CostResponseTypes
|
||||
|
||||
|
||||
class AzurePassthroughConfig(BasePassthroughConfig):
|
||||
def is_streaming_request(self, endpoint: str, request_data: dict) -> bool:
|
||||
|
|
@ -83,3 +87,36 @@ class AzurePassthroughConfig(BasePassthroughConfig):
|
|||
self, api_key: Optional[str] = None, api_base: Optional[str] = None
|
||||
) -> List[str]:
|
||||
return super().get_models(api_key, api_base)
|
||||
|
||||
def logging_non_streaming_response(
|
||||
self,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
httpx_response: Response,
|
||||
request_data: dict,
|
||||
logging_obj: Logging,
|
||||
endpoint: str,
|
||||
) -> Optional["CostResponseTypes"]:
|
||||
from litellm import encoding
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
if "chat/completions" not in endpoint:
|
||||
return None
|
||||
|
||||
openai_chat_config = OpenAIGPTConfig()
|
||||
|
||||
litellm_model_response: ModelResponse = openai_chat_config.transform_response(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "no-message-pass-through-endpoint"}],
|
||||
raw_response=httpx_response,
|
||||
model_response=ModelResponse(),
|
||||
logging_obj=logging_obj,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
api_key="",
|
||||
request_data=request_data,
|
||||
encoding=encoding,
|
||||
)
|
||||
|
||||
return litellm_model_response
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy.
|
|||
|
||||
from typing import Any, Optional, cast
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._logging import _redact_string, verbose_proxy_logger
|
||||
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
|
||||
|
||||
from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
|
|
@ -118,7 +118,7 @@ class AzureOpenAIRealtime(AzureChatCompletion):
|
|||
await realtime_streaming.bidirectional_forward()
|
||||
|
||||
except websockets.exceptions.InvalidStatusCode as e: # type: ignore
|
||||
await websocket.close(code=e.status_code, reason=str(e))
|
||||
await websocket.close(code=e.status_code, reason=_redact_string(str(e)))
|
||||
except Exception:
|
||||
verbose_proxy_logger.exception(
|
||||
"Error in AzureOpenAIRealtime.async_realtime"
|
||||
|
|
|
|||
|
|
@ -120,3 +120,46 @@ class BaseAnthropicMessagesConfig(ABC):
|
|||
return BaseLLMException(
|
||||
message=error_message, status_code=status_code, headers=headers
|
||||
)
|
||||
|
||||
@property
|
||||
def max_retry_on_anthropic_messages_http_error(self) -> int:
|
||||
"""
|
||||
Max HTTP attempts for /v1/messages when the handler may mutate the body and
|
||||
retry (e.g. strip invalid encrypted thinking signatures after a deployment or
|
||||
credential change).
|
||||
"""
|
||||
return 2
|
||||
|
||||
def should_retry_anthropic_messages_on_http_error(
|
||||
self, e: httpx.HTTPStatusError, litellm_params: dict
|
||||
) -> bool:
|
||||
"""
|
||||
When True, async_anthropic_messages_handler will transform the request body
|
||||
and issue one more attempt (bounded by max_retry_on_anthropic_messages_http_error).
|
||||
"""
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
is_anthropic_invalid_thinking_signature_error,
|
||||
)
|
||||
|
||||
return (
|
||||
e.response.status_code == 400
|
||||
and is_anthropic_invalid_thinking_signature_error(e.response.text)
|
||||
)
|
||||
|
||||
def transform_anthropic_messages_request_on_http_error(
|
||||
self, e: httpx.HTTPStatusError, request_data: dict
|
||||
) -> dict:
|
||||
"""
|
||||
Mutates request_data in place when retrying after a recoverable HTTP error.
|
||||
"""
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
is_anthropic_invalid_thinking_signature_error,
|
||||
strip_thinking_blocks_from_anthropic_messages_request_dict,
|
||||
)
|
||||
|
||||
if (
|
||||
e.response.status_code == 400
|
||||
and is_anthropic_invalid_thinking_signature_error(e.response.text)
|
||||
):
|
||||
strip_thinking_blocks_from_anthropic_messages_request_dict(request_data)
|
||||
return request_data
|
||||
|
|
|
|||
|
|
@ -1003,7 +1003,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
description=description,
|
||||
)
|
||||
optional_params["outputConfig"] = output_config
|
||||
else:
|
||||
elif json_schema is not None:
|
||||
# Fallback: translate to a synthetic tool call
|
||||
# https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode
|
||||
_tool = self._create_json_tool_call_for_response_format(
|
||||
|
|
@ -1025,6 +1025,12 @@ class AmazonConverseConfig(BaseConfig):
|
|||
)
|
||||
if non_default_params.get("stream", False) is True:
|
||||
optional_params["fake_stream"] = True
|
||||
# else: response_format=json_object with no schema.
|
||||
# Don't inject the synthetic json_tool_call tool here. When no
|
||||
# schema is given, _create_json_tool_call_for_response_format
|
||||
# produces an empty schema (properties: {}), and the model
|
||||
# returns {} instead of the requested JSON. The model already
|
||||
# returns JSON when the prompt asks for it.
|
||||
|
||||
optional_params["json_mode"] = True
|
||||
return optional_params
|
||||
|
|
@ -1298,12 +1304,16 @@ class AmazonConverseConfig(BaseConfig):
|
|||
# Add computer use tools and anthropic_beta if needed (only when computer use tools are present)
|
||||
if computer_use_tools:
|
||||
# Determine the correct computer-use beta header based on model
|
||||
# "computer-use-2025-11-24" for Claude Opus 4.6, Claude Opus 4.5
|
||||
# "computer-use-2025-11-24" for Claude Opus 4.7, Opus 4.6, and Opus 4.5
|
||||
# "computer-use-2025-01-24" for Claude Sonnet 4.5, Haiku 4.5, Opus 4.1, Sonnet 4, Opus 4, and Sonnet 3.7
|
||||
# "computer-use-2024-10-22" for older models
|
||||
model_lower = model.lower()
|
||||
if (
|
||||
"opus-4.6" in model_lower
|
||||
"opus-4.7" in model_lower
|
||||
or "opus_4.7" in model_lower
|
||||
or "opus-4-7" in model_lower
|
||||
or "opus_4_7" in model_lower
|
||||
or "opus-4.6" in model_lower
|
||||
or "opus_4.6" in model_lower
|
||||
or "opus-4-6" in model_lower
|
||||
or "opus_4_6" in model_lower
|
||||
|
|
@ -1651,6 +1661,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
cache_creation_input_tokens: int = 0
|
||||
cache_read_input_tokens: int = 0
|
||||
|
||||
raw_input_tokens = input_tokens # capture before inflation
|
||||
if "cacheReadInputTokens" in usage:
|
||||
cache_read_input_tokens = usage["cacheReadInputTokens"]
|
||||
input_tokens += cache_read_input_tokens
|
||||
|
|
@ -1659,7 +1670,9 @@ class AmazonConverseConfig(BaseConfig):
|
|||
input_tokens += cache_creation_input_tokens
|
||||
|
||||
prompt_tokens_details = PromptTokensDetailsWrapper(
|
||||
cached_tokens=cache_read_input_tokens
|
||||
cached_tokens=cache_read_input_tokens,
|
||||
cache_creation_tokens=cache_creation_input_tokens,
|
||||
text_tokens=raw_input_tokens,
|
||||
)
|
||||
reasoning_tokens = (
|
||||
token_counter(text=reasoning_content, count_response_tokens=True)
|
||||
|
|
@ -2027,6 +2040,12 @@ class AmazonConverseConfig(BaseConfig):
|
|||
_message = Message(**chat_completion_message)
|
||||
initial_finish_reason = map_finish_reason(completion_response["stopReason"])
|
||||
|
||||
# When json_mode filtered out all synthetic tool calls the response
|
||||
# is plain content, not a pending tool invocation. Fix finish_reason
|
||||
# so callers (e.g. OpenAI SDK) don't misinterpret it.
|
||||
if json_mode and not filtered_tools and tools:
|
||||
initial_finish_reason = "stop"
|
||||
|
||||
(
|
||||
returned_message,
|
||||
returned_finish_reason,
|
||||
|
|
|
|||
|
|
@ -589,6 +589,10 @@ def is_claude_4_5_on_bedrock(model: str) -> bool:
|
|||
"opus_4.6",
|
||||
"opus-4-6",
|
||||
"opus_4_6",
|
||||
"opus-4.7",
|
||||
"opus_4.7",
|
||||
"opus-4-7",
|
||||
"opus_4_7",
|
||||
]
|
||||
return any(pattern in model_lower for pattern in claude_4_5_patterns)
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ from typing import (
|
|||
import httpx
|
||||
|
||||
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
|
||||
from litellm.constants import BEDROCK_MIN_THINKING_BUDGET_TOKENS
|
||||
from litellm.litellm_core_utils.litellm_logging import verbose_logger
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
|
|
@ -205,10 +207,78 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
"opus_4.6",
|
||||
"opus-4-6",
|
||||
"opus_4_6",
|
||||
"opus-4.7",
|
||||
"opus_4.7",
|
||||
"opus-4-7",
|
||||
"opus_4_7",
|
||||
]
|
||||
|
||||
return any(pattern in model_lower for pattern in supported_patterns)
|
||||
|
||||
def _ensure_thinking_for_clear_thinking_context_management(
|
||||
self,
|
||||
anthropic_messages_request: Dict,
|
||||
model: str,
|
||||
) -> bool:
|
||||
"""
|
||||
Bedrock rejects ``clear_thinking_20251015`` context-management edits unless
|
||||
extended thinking is ``enabled`` or ``adaptive``. Claude Code often sends
|
||||
context management without a top-level ``thinking`` field.
|
||||
|
||||
When we detect that edit type on a model that supports extended thinking on
|
||||
Bedrock, inject a minimal ``thinking`` config so the request succeeds.
|
||||
|
||||
Returns:
|
||||
True if ``thinking`` was added or upgraded for this fix (caller may
|
||||
need to add the interleaved-thinking beta header).
|
||||
"""
|
||||
cm = anthropic_messages_request.get("context_management")
|
||||
if not isinstance(cm, dict):
|
||||
return False
|
||||
edits = cm.get("edits")
|
||||
if not isinstance(edits, list):
|
||||
return False
|
||||
needs_thinking = any(
|
||||
isinstance(e, dict) and e.get("type") == "clear_thinking_20251015"
|
||||
for e in edits
|
||||
)
|
||||
if not needs_thinking:
|
||||
return False
|
||||
if not self._supports_extended_thinking_on_bedrock(model):
|
||||
return False
|
||||
|
||||
thinking = anthropic_messages_request.get("thinking")
|
||||
if isinstance(thinking, dict):
|
||||
t = thinking.get("type")
|
||||
if t in ("enabled", "adaptive"):
|
||||
return False
|
||||
# ``disabled`` or unknown — replace with enabled so clear_thinking is valid
|
||||
verbose_logger.debug(
|
||||
"Bedrock clear_thinking_20251015: replacing thinking=%s with minimal enabled thinking",
|
||||
thinking,
|
||||
)
|
||||
|
||||
max_tokens = anthropic_messages_request.get("max_tokens")
|
||||
budget = BEDROCK_MIN_THINKING_BUDGET_TOKENS
|
||||
if isinstance(max_tokens, int) and max_tokens <= budget:
|
||||
verbose_logger.warning(
|
||||
"Bedrock clear_thinking_20251015: max_tokens=%s is not greater than "
|
||||
"minimum thinking budget (%s); cannot inject thinking safely",
|
||||
max_tokens,
|
||||
budget,
|
||||
)
|
||||
return False
|
||||
|
||||
anthropic_messages_request["thinking"] = {
|
||||
"type": "enabled",
|
||||
"budget_tokens": budget,
|
||||
}
|
||||
verbose_logger.debug(
|
||||
"Bedrock clear_thinking_20251015: injected thinking with budget_tokens=%s",
|
||||
budget,
|
||||
)
|
||||
return True
|
||||
|
||||
def _is_claude_opus_4_5(self, model: str) -> bool:
|
||||
"""
|
||||
Check if the model is Claude Opus 4.5.
|
||||
|
|
@ -281,6 +351,10 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
"sonnet_4.6",
|
||||
"sonnet-4-6",
|
||||
"sonnet_4_6",
|
||||
# NOTE: Opus 4.7 on Bedrock does not support server-side tool search
|
||||
# as of launch (2026-04-16). Bedrock rejects the tool type with:
|
||||
# "tool type 'tool_search_tool_..._20251119' is not supported for this model".
|
||||
# Re-add the opus-4.7 patterns here once AWS announces support.
|
||||
]
|
||||
|
||||
return any(pattern in model_lower for pattern in supported_patterns)
|
||||
|
|
@ -406,6 +480,13 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
if "model" in anthropic_messages_request:
|
||||
anthropic_messages_request.pop("model", None)
|
||||
|
||||
injected_thinking_for_clear_thinking = (
|
||||
self._ensure_thinking_for_clear_thinking_context_management(
|
||||
anthropic_messages_request=anthropic_messages_request,
|
||||
model=model,
|
||||
)
|
||||
)
|
||||
|
||||
# 4. Remove `ttl` field from cache_control in messages (Bedrock doesn't support it for older models)
|
||||
self._remove_ttl_from_cache_control(
|
||||
anthropic_messages_request=anthropic_messages_request, model=model
|
||||
|
|
@ -455,6 +536,9 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
)
|
||||
beta_set.update(auto_betas)
|
||||
|
||||
if injected_thinking_for_clear_thinking:
|
||||
beta_set.add("interleaved-thinking-2025-05-14")
|
||||
|
||||
self._get_tool_search_beta_header_for_bedrock(
|
||||
model=model,
|
||||
tool_search_used=tool_search_used,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import asyncio
|
|||
import json
|
||||
from typing import Any, Optional
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._logging import _redact_string, verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
|
||||
from ..base_aws_llm import BaseAWSLLM
|
||||
|
|
@ -152,7 +152,7 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
f"Error in BedrockRealtime.async_realtime: {e}"
|
||||
)
|
||||
try:
|
||||
await websocket.close(code=1011, reason=f"Internal error: {str(e)}")
|
||||
await websocket.close(code=1011, reason=_redact_string(f"Internal error: {str(e)}"))
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -30,7 +30,9 @@ from litellm.constants import (
|
|||
AIOHTTP_KEEPALIVE_TIMEOUT,
|
||||
AIOHTTP_NEEDS_CLEANUP_CLOSED,
|
||||
AIOHTTP_TTL_DNS_CACHE,
|
||||
COMPLETION_HTTP_FALLBACK_SECONDS,
|
||||
DEFAULT_SSL_CIPHERS,
|
||||
HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
|
||||
)
|
||||
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
|
||||
from litellm.types.llms.custom_http import *
|
||||
|
|
@ -70,7 +72,10 @@ def get_default_headers() -> dict:
|
|||
headers = get_default_headers()
|
||||
|
||||
# https://www.python-httpx.org/advanced/timeouts
|
||||
_DEFAULT_TIMEOUT = httpx.Timeout(timeout=5.0, connect=5.0)
|
||||
_DEFAULT_TIMEOUT = httpx.Timeout(
|
||||
timeout=COMPLETION_HTTP_FALLBACK_SECONDS,
|
||||
connect=HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
def _prepare_request_data_and_content(
|
||||
|
|
@ -316,30 +321,95 @@ def mask_sensitive_info(error_message):
|
|||
return error_message
|
||||
|
||||
|
||||
def _safe_get_response_text(response: httpx.Response) -> str:
|
||||
"""Safely read response text, falling back to empty string on decoding errors."""
|
||||
try:
|
||||
return response.text
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
async def _safe_aread_response(response: httpx.Response) -> bytes:
|
||||
"""Safely read async response body, falling back to empty bytes on errors."""
|
||||
try:
|
||||
return await response.aread()
|
||||
except Exception:
|
||||
return b""
|
||||
|
||||
|
||||
def _safe_read_response(response: httpx.Response) -> bytes:
|
||||
"""Safely read sync response body, falling back to empty bytes on errors."""
|
||||
try:
|
||||
return response.read()
|
||||
except Exception:
|
||||
return b""
|
||||
|
||||
|
||||
def _raise_masked_sync_error(e: httpx.HTTPStatusError, stream: bool) -> None:
|
||||
"""Raise a MaskedHTTPStatusError for sync HTTP handlers."""
|
||||
if stream:
|
||||
_body = mask_sensitive_info(_safe_read_response(e.response))
|
||||
raise MaskedHTTPStatusError(e, message=_body, text=_body) from None
|
||||
_text = mask_sensitive_info(_safe_get_response_text(e.response))
|
||||
raise MaskedHTTPStatusError(e, message=_text, text=_text) from None
|
||||
|
||||
|
||||
async def _raise_masked_async_error(e: httpx.HTTPStatusError, stream: bool) -> None:
|
||||
"""Raise a MaskedHTTPStatusError for async HTTP handlers."""
|
||||
if stream:
|
||||
_body = mask_sensitive_info(await _safe_aread_response(e.response))
|
||||
raise MaskedHTTPStatusError(e, message=_body, text=_body) from None
|
||||
_text = mask_sensitive_info(_safe_get_response_text(e.response))
|
||||
raise MaskedHTTPStatusError(e, message=_text, text=_text) from None
|
||||
|
||||
|
||||
class MaskedHTTPStatusError(httpx.HTTPStatusError):
|
||||
def __init__(
|
||||
self, original_error, message: Optional[str] = None, text: Optional[str] = None
|
||||
):
|
||||
# Create a new error with the masked URL
|
||||
masked_url = mask_sensitive_info(str(original_error.request.url))
|
||||
# Create a new error that looks like the original, but with a masked URL
|
||||
# Mask the original exception message too (it contains the full URL)
|
||||
masked_original_message = mask_sensitive_info(str(original_error))
|
||||
|
||||
# Safely access response content — decompression can fail (e.g. zlib error).
|
||||
# `.content` returns already-decoded bytes, so we must strip transport
|
||||
# encoding headers before rebuilding the Response (otherwise httpx will
|
||||
# try to decode the bytes a second time and raise DecodingError).
|
||||
try:
|
||||
response_content = original_error.response.content
|
||||
except Exception:
|
||||
response_content = b""
|
||||
|
||||
response_headers = {
|
||||
k: v
|
||||
for k, v in original_error.response.headers.items()
|
||||
if k.lower() not in ("content-encoding", "content-length")
|
||||
}
|
||||
|
||||
masked_request = httpx.Request(
|
||||
method=original_error.request.method,
|
||||
url=masked_url,
|
||||
headers=original_error.request.headers,
|
||||
content=original_error.request.content,
|
||||
)
|
||||
|
||||
super().__init__(
|
||||
message=original_error.message,
|
||||
request=httpx.Request(
|
||||
method=original_error.request.method,
|
||||
url=masked_url,
|
||||
headers=original_error.request.headers,
|
||||
content=original_error.request.content,
|
||||
),
|
||||
message=masked_original_message,
|
||||
request=masked_request,
|
||||
# Attach the masked request so `response.request` is set — otherwise
|
||||
# downstream code that inspects err.response.request (e.g.
|
||||
# exception_mapping_utils) hits `RuntimeError: .request not set`.
|
||||
response=httpx.Response(
|
||||
status_code=original_error.response.status_code,
|
||||
content=original_error.response.content,
|
||||
headers=original_error.response.headers,
|
||||
content=response_content,
|
||||
headers=response_headers,
|
||||
request=masked_request,
|
||||
),
|
||||
)
|
||||
self.message = message
|
||||
self.text = text
|
||||
self.status_code = original_error.response.status_code
|
||||
|
||||
|
||||
class AsyncHTTPHandler:
|
||||
|
|
@ -501,16 +571,7 @@ class AsyncHTTPHandler:
|
|||
headers=headers,
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
if stream is True:
|
||||
setattr(e, "message", await e.response.aread())
|
||||
setattr(e, "text", await e.response.aread())
|
||||
else:
|
||||
setattr(e, "message", mask_sensitive_info(e.response.text))
|
||||
setattr(e, "text", mask_sensitive_info(e.response.text))
|
||||
|
||||
setattr(e, "status_code", e.response.status_code)
|
||||
|
||||
raise e
|
||||
await _raise_masked_async_error(e, stream)
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
|
|
@ -571,12 +632,7 @@ class AsyncHTTPHandler:
|
|||
headers=headers,
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
setattr(e, "status_code", e.response.status_code)
|
||||
if stream is True:
|
||||
setattr(e, "message", await e.response.aread())
|
||||
else:
|
||||
setattr(e, "message", e.response.text)
|
||||
raise e
|
||||
await _raise_masked_async_error(e, stream)
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
|
|
@ -637,12 +693,7 @@ class AsyncHTTPHandler:
|
|||
headers=headers,
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
setattr(e, "status_code", e.response.status_code)
|
||||
if stream is True:
|
||||
setattr(e, "message", await e.response.aread())
|
||||
else:
|
||||
setattr(e, "message", e.response.text)
|
||||
raise e
|
||||
await _raise_masked_async_error(e, stream)
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
|
|
@ -690,12 +741,7 @@ class AsyncHTTPHandler:
|
|||
finally:
|
||||
await new_client.aclose()
|
||||
except httpx.HTTPStatusError as e:
|
||||
setattr(e, "status_code", e.response.status_code)
|
||||
if stream is True:
|
||||
setattr(e, "message", await e.response.aread())
|
||||
else:
|
||||
setattr(e, "message", e.response.text)
|
||||
raise e
|
||||
await _raise_masked_async_error(e, stream)
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
|
|
@ -886,9 +932,9 @@ class AsyncHTTPHandler:
|
|||
if AIOHTTP_CONNECTOR_LIMIT > 0:
|
||||
transport_connector_kwargs["limit"] = AIOHTTP_CONNECTOR_LIMIT
|
||||
if AIOHTTP_CONNECTOR_LIMIT_PER_HOST > 0:
|
||||
transport_connector_kwargs[
|
||||
"limit_per_host"
|
||||
] = AIOHTTP_CONNECTOR_LIMIT_PER_HOST
|
||||
transport_connector_kwargs["limit_per_host"] = (
|
||||
AIOHTTP_CONNECTOR_LIMIT_PER_HOST
|
||||
)
|
||||
|
||||
return LiteLLMAiohttpTransport(
|
||||
client=lambda: ClientSession(
|
||||
|
|
@ -1035,16 +1081,7 @@ class HTTPHandler:
|
|||
llm_provider="litellm-httpx-handler",
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
if stream is True:
|
||||
setattr(e, "message", mask_sensitive_info(e.response.read()))
|
||||
setattr(e, "text", mask_sensitive_info(e.response.read()))
|
||||
else:
|
||||
error_text = mask_sensitive_info(e.response.text)
|
||||
setattr(e, "message", error_text)
|
||||
setattr(e, "text", error_text)
|
||||
|
||||
setattr(e, "status_code", e.response.status_code)
|
||||
raise e
|
||||
_raise_masked_sync_error(e, stream)
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
|
|
@ -1083,17 +1120,7 @@ class HTTPHandler:
|
|||
llm_provider="litellm-httpx-handler",
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
if stream is True:
|
||||
setattr(e, "message", mask_sensitive_info(e.response.read()))
|
||||
setattr(e, "text", mask_sensitive_info(e.response.read()))
|
||||
else:
|
||||
error_text = mask_sensitive_info(e.response.text)
|
||||
setattr(e, "message", error_text)
|
||||
setattr(e, "text", error_text)
|
||||
|
||||
setattr(e, "status_code", e.response.status_code)
|
||||
|
||||
raise e
|
||||
_raise_masked_sync_error(e, stream)
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
|
|
@ -1130,6 +1157,8 @@ class HTTPHandler:
|
|||
model="default-model-name",
|
||||
llm_provider="litellm-httpx-handler",
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
_raise_masked_sync_error(e, stream)
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
|
|
@ -1168,17 +1197,7 @@ class HTTPHandler:
|
|||
llm_provider="litellm-httpx-handler",
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
if stream is True:
|
||||
setattr(e, "message", mask_sensitive_info(e.response.read()))
|
||||
setattr(e, "text", mask_sensitive_info(e.response.read()))
|
||||
else:
|
||||
error_text = mask_sensitive_info(e.response.text)
|
||||
setattr(e, "message", error_text)
|
||||
setattr(e, "text", error_text)
|
||||
|
||||
setattr(e, "status_code", e.response.status_code)
|
||||
|
||||
raise e
|
||||
_raise_masked_sync_error(e, stream)
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
|
|
@ -1244,7 +1263,7 @@ def get_async_httpx_client(
|
|||
_new_client = AsyncHTTPHandler(**handler_params)
|
||||
else:
|
||||
_new_client = AsyncHTTPHandler(
|
||||
timeout=httpx.Timeout(timeout=600.0, connect=5.0),
|
||||
timeout=_DEFAULT_TIMEOUT,
|
||||
shared_session=shared_session,
|
||||
)
|
||||
|
||||
|
|
@ -1293,7 +1312,7 @@ def _get_httpx_client(params: Optional[dict] = None) -> HTTPHandler:
|
|||
}
|
||||
_new_client = HTTPHandler(**handler_params)
|
||||
else:
|
||||
_new_client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0))
|
||||
_new_client = HTTPHandler(timeout=_DEFAULT_TIMEOUT)
|
||||
|
||||
cache.set_cache(
|
||||
key=_cache_key_name,
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import litellm
|
|||
import litellm.litellm_core_utils
|
||||
import litellm.types
|
||||
import litellm.types.utils
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm._logging import _redact_string, verbose_logger
|
||||
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
|
||||
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
|
||||
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
|
||||
|
|
@ -1816,6 +1816,73 @@ class BaseLLMHTTPHandler:
|
|||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
async def _async_post_anthropic_messages_with_http_error_retry(
|
||||
self,
|
||||
async_httpx_client: AsyncHTTPHandler,
|
||||
request_url: str,
|
||||
headers: dict,
|
||||
signed_json_body: Optional[bytes],
|
||||
request_body: dict,
|
||||
stream: bool,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
provider_config: BaseAnthropicMessagesConfig,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
api_key: Optional[str],
|
||||
model: str,
|
||||
) -> httpx.Response:
|
||||
max_attempts = max(
|
||||
provider_config.max_retry_on_anthropic_messages_http_error, 1
|
||||
)
|
||||
litellm_params_dict = dict(litellm_params)
|
||||
optional_params_dict = dict(litellm_params)
|
||||
for attempt_idx in range(max_attempts):
|
||||
try:
|
||||
response = await async_httpx_client.post(
|
||||
url=request_url,
|
||||
headers=headers,
|
||||
data=signed_json_body or json.dumps(request_body),
|
||||
stream=stream or False,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except httpx.HTTPStatusError as e:
|
||||
hit_max_attempt = attempt_idx + 1 == max_attempts
|
||||
should_retry = (
|
||||
provider_config.should_retry_anthropic_messages_on_http_error(
|
||||
e=e, litellm_params=litellm_params_dict
|
||||
)
|
||||
)
|
||||
if should_retry and not hit_max_attempt:
|
||||
verbose_logger.debug(
|
||||
"Anthropic /v1/messages: invalid thinking signature; "
|
||||
"stripping thinking blocks and retrying (attempt %s/%s).",
|
||||
attempt_idx + 2,
|
||||
max_attempts,
|
||||
)
|
||||
provider_config.transform_anthropic_messages_request_on_http_error(
|
||||
e=e, request_data=request_body
|
||||
)
|
||||
headers, signed_json_body = provider_config.sign_request(
|
||||
headers=headers,
|
||||
optional_params=optional_params_dict,
|
||||
request_data=request_body,
|
||||
api_base=request_url,
|
||||
api_key=api_key,
|
||||
stream=stream,
|
||||
fake_stream=False,
|
||||
model=model,
|
||||
)
|
||||
logging_obj.model_call_details.update(request_body)
|
||||
continue
|
||||
raise self._handle_error(e=e, provider_config=provider_config)
|
||||
except Exception as e:
|
||||
raise self._handle_error(e=e, provider_config=provider_config)
|
||||
|
||||
raise RuntimeError(
|
||||
"unreachable: anthropic messages HTTP retry loop exited without return"
|
||||
)
|
||||
|
||||
async def async_anthropic_messages_handler(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -1955,19 +2022,19 @@ class BaseLLMHTTPHandler:
|
|||
},
|
||||
)
|
||||
|
||||
try:
|
||||
response = await async_httpx_client.post(
|
||||
url=request_url,
|
||||
headers=headers,
|
||||
data=signed_json_body or json.dumps(request_body),
|
||||
stream=stream or False,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
response.raise_for_status()
|
||||
except Exception as e:
|
||||
raise self._handle_error(
|
||||
e=e, provider_config=anthropic_messages_provider_config
|
||||
)
|
||||
response = await self._async_post_anthropic_messages_with_http_error_retry(
|
||||
async_httpx_client=async_httpx_client,
|
||||
request_url=request_url,
|
||||
headers=headers,
|
||||
signed_json_body=signed_json_body,
|
||||
request_body=request_body,
|
||||
stream=stream or False,
|
||||
logging_obj=logging_obj,
|
||||
provider_config=anthropic_messages_provider_config,
|
||||
litellm_params=litellm_params,
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
)
|
||||
|
||||
# used for logging + cost tracking
|
||||
logging_obj.model_call_details["httpx_response"] = response
|
||||
|
|
@ -4496,9 +4563,9 @@ class BaseLLMHTTPHandler:
|
|||
# Second: Execute agentic loop
|
||||
# Add custom_llm_provider to kwargs so the agentic loop can reconstruct the full model name
|
||||
kwargs_with_provider = kwargs.copy() if kwargs else {}
|
||||
kwargs_with_provider[
|
||||
"custom_llm_provider"
|
||||
] = custom_llm_provider
|
||||
kwargs_with_provider["custom_llm_provider"] = (
|
||||
custom_llm_provider
|
||||
)
|
||||
agentic_response = await callback.async_run_agentic_loop(
|
||||
tools=tool_calls,
|
||||
model=model,
|
||||
|
|
@ -4614,9 +4681,9 @@ class BaseLLMHTTPHandler:
|
|||
# Second: Execute agentic loop
|
||||
# Add custom_llm_provider to kwargs so the agentic loop can reconstruct the full model name
|
||||
kwargs_with_provider = kwargs.copy() if kwargs else {}
|
||||
kwargs_with_provider[
|
||||
"custom_llm_provider"
|
||||
] = custom_llm_provider
|
||||
kwargs_with_provider["custom_llm_provider"] = (
|
||||
custom_llm_provider
|
||||
)
|
||||
agentic_response = (
|
||||
await callback.async_run_chat_completion_agentic_loop(
|
||||
tools=tool_calls,
|
||||
|
|
@ -4789,12 +4856,12 @@ class BaseLLMHTTPHandler:
|
|||
|
||||
except websockets.exceptions.InvalidStatusCode as e: # type: ignore
|
||||
verbose_logger.exception(f"Error connecting to backend: {e}")
|
||||
await websocket.close(code=e.status_code, reason=str(e))
|
||||
await websocket.close(code=e.status_code, reason=_redact_string(str(e)))
|
||||
except Exception as e:
|
||||
verbose_logger.exception(f"Error connecting to backend: {e}")
|
||||
try:
|
||||
await websocket.close(
|
||||
code=1011, reason=f"Internal server error: {str(e)}"
|
||||
code=1011, reason=_redact_string(f"Internal server error: {str(e)}")
|
||||
)
|
||||
except RuntimeError as close_error:
|
||||
if "already completed" in str(close_error) or "websocket.close" in str(
|
||||
|
|
@ -5076,12 +5143,12 @@ class BaseLLMHTTPHandler:
|
|||
|
||||
except websockets.exceptions.InvalidStatusCode as e: # type: ignore
|
||||
verbose_logger.exception(f"Error connecting to responses WS backend: {e}")
|
||||
await websocket.close(code=e.status_code, reason=str(e))
|
||||
await websocket.close(code=e.status_code, reason=_redact_string(str(e)))
|
||||
except Exception as e:
|
||||
verbose_logger.exception(f"Error in responses WS: {e}")
|
||||
try:
|
||||
await websocket.close(
|
||||
code=1011, reason=f"Internal server error: {str(e)}"
|
||||
code=1011, reason=_redact_string(f"Internal server error: {str(e)}")
|
||||
)
|
||||
except RuntimeError as close_error:
|
||||
if "already completed" in str(close_error) or "websocket.close" in str(
|
||||
|
|
@ -5110,7 +5177,10 @@ class BaseLLMHTTPHandler:
|
|||
_is_async: bool = False,
|
||||
fake_stream: bool = False,
|
||||
litellm_metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> Union[ImageResponse, Coroutine[Any, Any, ImageResponse],]:
|
||||
) -> Union[
|
||||
ImageResponse,
|
||||
Coroutine[Any, Any, ImageResponse],
|
||||
]:
|
||||
"""
|
||||
|
||||
Handles image edit requests.
|
||||
|
|
@ -5322,7 +5392,10 @@ class BaseLLMHTTPHandler:
|
|||
fake_stream: bool = False,
|
||||
litellm_metadata: Optional[Dict[str, Any]] = None,
|
||||
api_key: Optional[str] = None,
|
||||
) -> Union[ImageResponse, Coroutine[Any, Any, ImageResponse],]:
|
||||
) -> Union[
|
||||
ImageResponse,
|
||||
Coroutine[Any, Any, ImageResponse],
|
||||
]:
|
||||
"""
|
||||
Handles image generation requests.
|
||||
When _is_async=True, returns a coroutine instead of making the call directly.
|
||||
|
|
@ -5562,7 +5635,10 @@ class BaseLLMHTTPHandler:
|
|||
fake_stream: bool = False,
|
||||
litellm_metadata: Optional[Dict[str, Any]] = None,
|
||||
api_key: Optional[str] = None,
|
||||
) -> Union[VideoObject, Coroutine[Any, Any, VideoObject],]:
|
||||
) -> Union[
|
||||
VideoObject,
|
||||
Coroutine[Any, Any, VideoObject],
|
||||
]:
|
||||
"""
|
||||
Handles video generation requests.
|
||||
When _is_async=True, returns a coroutine instead of making the call directly.
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ class GeminiModelInfo(BaseLLMModelInfo):
|
|||
api_key: Optional[str] = None,
|
||||
api_base: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""Google AI Studio sends api key in query params"""
|
||||
"""Google AI Studio sends api key via x-goog-api-key header"""
|
||||
return headers
|
||||
|
||||
@property
|
||||
|
|
@ -75,7 +75,8 @@ class GeminiModelInfo(BaseLLMModelInfo):
|
|||
)
|
||||
|
||||
response = litellm.module_level_client.get(
|
||||
url=f"{api_base}{endpoint}?key={api_key}",
|
||||
url=f"{api_base}{endpoint}",
|
||||
headers={"x-goog-api-key": api_key},
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
|
|||
if not final_api_key:
|
||||
raise ValueError("api_key is required")
|
||||
|
||||
url = "{}/{}?key={}".format(api_base, endpoint, final_api_key)
|
||||
url = "{}/{}".format(api_base, endpoint)
|
||||
return url
|
||||
|
||||
def get_supported_openai_params(
|
||||
|
|
@ -231,9 +231,9 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
|
|||
)
|
||||
api_base = api_base.rstrip("/")
|
||||
|
||||
url = f"{api_base}/v1beta/{file_part}?key={api_key}"
|
||||
url = f"{api_base}/v1beta/{file_part}"
|
||||
|
||||
# Return empty params dict - API key is already in URL, no query params needed
|
||||
# API key is passed via x-goog-api-key header (set in validate_environment)
|
||||
return url, {}
|
||||
|
||||
def _normalize_gemini_file_id(self, file_id: str) -> str:
|
||||
|
|
|
|||
|
|
@ -75,9 +75,13 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
|
|||
model: str,
|
||||
litellm_params: Optional[GenericLiteLLMParams],
|
||||
) -> dict:
|
||||
"""Google AI Studio uses API key in query params, not headers."""
|
||||
"""Google AI Studio uses x-goog-api-key header for authentication."""
|
||||
headers = headers or {}
|
||||
headers["Content-Type"] = "application/json"
|
||||
if litellm_params:
|
||||
api_key = GeminiModelInfo.get_api_key(litellm_params.get("api_key"))
|
||||
if api_key:
|
||||
headers["x-goog-api-key"] = api_key
|
||||
return headers
|
||||
|
||||
def get_complete_url(
|
||||
|
|
@ -98,11 +102,10 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
|
|||
"Google API key is required. Set GOOGLE_API_KEY or GEMINI_API_KEY environment variable."
|
||||
)
|
||||
|
||||
query_params = f"key={api_key}"
|
||||
if stream:
|
||||
query_params += "&alt=sse"
|
||||
return f"{api_base}/{self.api_version}/interactions?alt=sse"
|
||||
|
||||
return f"{api_base}/{self.api_version}/interactions?{query_params}"
|
||||
return f"{api_base}/{self.api_version}/interactions"
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
|
|
@ -200,11 +203,10 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
|
|||
) -> Tuple[str, Dict]:
|
||||
"""GET /{api_version}/interactions/{interaction_id}"""
|
||||
resolved_api_base = GeminiModelInfo.get_api_base(api_base)
|
||||
api_key = GeminiModelInfo.get_api_key(litellm_params.api_key)
|
||||
if not api_key:
|
||||
if not GeminiModelInfo.get_api_key(litellm_params.api_key):
|
||||
raise ValueError("Google API key is required")
|
||||
return (
|
||||
f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}?key={api_key}",
|
||||
f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}",
|
||||
{},
|
||||
)
|
||||
|
||||
|
|
@ -234,11 +236,10 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
|
|||
) -> Tuple[str, Dict]:
|
||||
"""DELETE /{api_version}/interactions/{interaction_id}"""
|
||||
resolved_api_base = GeminiModelInfo.get_api_base(api_base)
|
||||
api_key = GeminiModelInfo.get_api_key(litellm_params.api_key)
|
||||
if not api_key:
|
||||
if not GeminiModelInfo.get_api_key(litellm_params.api_key):
|
||||
raise ValueError("Google API key is required")
|
||||
return (
|
||||
f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}?key={api_key}",
|
||||
f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}",
|
||||
{},
|
||||
)
|
||||
|
||||
|
|
@ -265,11 +266,10 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
|
|||
) -> Tuple[str, Dict]:
|
||||
"""POST /{api_version}/interactions/{interaction_id}:cancel (if supported)"""
|
||||
resolved_api_base = GeminiModelInfo.get_api_base(api_base)
|
||||
api_key = GeminiModelInfo.get_api_key(litellm_params.api_key)
|
||||
if not api_key:
|
||||
if not GeminiModelInfo.get_api_key(litellm_params.api_key):
|
||||
raise ValueError("Google API key is required")
|
||||
return (
|
||||
f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}:cancel?key={api_key}",
|
||||
f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}:cancel",
|
||||
{},
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -85,6 +85,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
raise ValueError("api_key is required for Gemini API calls")
|
||||
api_base = api_base.replace("https://", "wss://")
|
||||
api_base = api_base.replace("http://", "ws://")
|
||||
# WebSocket connections do not support custom HTTP headers in all clients,
|
||||
# so the API key must remain as a query parameter here. This is an accepted
|
||||
# limitation; httpx is not used for WebSocket so MaskedHTTPStatusError
|
||||
# already covers the main leak vector.
|
||||
return f"{api_base}/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key={api_key}"
|
||||
|
||||
def map_model_turn_event(
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig):
|
|||
def get_auth_credentials(
|
||||
self, litellm_params: dict
|
||||
) -> BaseVectorStoreAuthCredentials:
|
||||
"""Gemini uses API key in query params, not headers."""
|
||||
"""Gemini uses x-goog-api-key header for authentication."""
|
||||
return {}
|
||||
|
||||
def get_vector_store_endpoints_by_type(self) -> VectorStoreIndexEndpoints:
|
||||
|
|
@ -79,6 +79,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig):
|
|||
api_key = litellm_params.get("api_key") or get_api_key_from_env()
|
||||
if api_key:
|
||||
self._cached_api_key = api_key
|
||||
headers["x-goog-api-key"] = api_key
|
||||
|
||||
return headers
|
||||
|
||||
|
|
@ -133,13 +134,10 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig):
|
|||
if model and model.startswith("gemini/"):
|
||||
model = model.replace("gemini/", "")
|
||||
|
||||
# Get API key - Gemini requires it as a query parameter
|
||||
api_key = litellm_params.get("api_key") or GeminiModelInfo.get_api_key()
|
||||
if not api_key:
|
||||
raise ValueError("GEMINI_API_KEY or GOOGLE_API_KEY is required")
|
||||
|
||||
# Build the URL for generateContent with API key
|
||||
url = f"{api_base}/models/{model}:generateContent?key={api_key}"
|
||||
url = f"{api_base}/models/{model}:generateContent"
|
||||
|
||||
# Build file_search tool configuration (using snake_case as per Gemini docs)
|
||||
file_search_config: Dict[str, Any] = {
|
||||
|
|
@ -286,10 +284,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig):
|
|||
"""
|
||||
url = f"{api_base}/fileSearchStores"
|
||||
|
||||
# Append API key as query parameter (required by Gemini)
|
||||
api_key = self._cached_api_key or get_api_key_from_env()
|
||||
if api_key:
|
||||
url = f"{url}?key={api_key}"
|
||||
# API key is passed via x-goog-api-key header (set in validate_environment)
|
||||
|
||||
request_body: Dict[str, Any] = {}
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from httpx._models import Headers, Response
|
|||
from pydantic import BaseModel
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
_extract_reasoning_content,
|
||||
convert_content_list_to_str,
|
||||
|
|
@ -349,7 +350,8 @@ class OllamaChatConfig(BaseConfig):
|
|||
response_json = raw_response.json()
|
||||
|
||||
## RESPONSE OBJECT
|
||||
model_response.choices[0].finish_reason = "stop"
|
||||
_done_reason = map_finish_reason(response_json.get("done_reason") or "stop")
|
||||
model_response.choices[0].finish_reason = _done_reason
|
||||
response_json_message = response_json.get("message")
|
||||
if response_json_message is not None:
|
||||
if "thinking" in response_json_message:
|
||||
|
|
@ -535,7 +537,7 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
|
|||
)
|
||||
|
||||
if chunk["done"] is True:
|
||||
finish_reason = chunk.get("done_reason", "stop")
|
||||
finish_reason = chunk.get("done_reason") or "stop"
|
||||
# Override finish_reason when tool_calls are present
|
||||
# Fixes: https://github.com/BerriAI/litellm/issues/18922
|
||||
if tool_calls is not None:
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy.
|
|||
|
||||
from typing import Any, Optional, cast
|
||||
|
||||
from litellm._logging import _redact_string
|
||||
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
|
||||
from litellm.types.realtime import RealtimeQueryParams
|
||||
|
||||
|
|
@ -148,11 +149,11 @@ class OpenAIRealtime(OpenAIChatCompletion):
|
|||
await realtime_streaming.bidirectional_forward()
|
||||
|
||||
except websockets.exceptions.InvalidStatusCode as e: # type: ignore
|
||||
await websocket.close(code=e.status_code, reason=str(e))
|
||||
await websocket.close(code=e.status_code, reason=_redact_string(str(e)))
|
||||
except Exception as e:
|
||||
try:
|
||||
await websocket.close(
|
||||
code=1011, reason=f"Internal server error: {str(e)}"
|
||||
code=1011, reason=_redact_string(f"Internal server error: {str(e)}")
|
||||
)
|
||||
except RuntimeError as close_error:
|
||||
if "already completed" in str(close_error) or "websocket.close" in str(
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ Docs: https://docs.together.ai/reference/completions-1
|
|||
|
||||
from typing import Optional
|
||||
|
||||
from litellm.utils import get_model_info
|
||||
from litellm.utils import supports_function_calling
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
from ..openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
|
@ -21,18 +21,23 @@ class TogetherAIConfig(OpenAIGPTConfig):
|
|||
|
||||
Docs: https://docs.together.ai/docs/json-mode
|
||||
"""
|
||||
supports_function_calling: Optional[bool] = None
|
||||
# Use supports_function_calling() — which reads _get_model_info_helper
|
||||
# directly — instead of get_model_info(). get_model_info() calls
|
||||
# get_supported_openai_params() as its first step, which routes back
|
||||
# into this method for together_ai models, creating a recursion that
|
||||
# only terminates when Python's recursion limit or the "not mapped"
|
||||
# exception in _get_model_info_helper is hit (~332 deep calls).
|
||||
supports_fc: Optional[bool] = None
|
||||
try:
|
||||
model_info = get_model_info(model, custom_llm_provider="together_ai")
|
||||
supports_function_calling = model_info.get(
|
||||
"supports_function_calling", False
|
||||
supports_fc = supports_function_calling(
|
||||
model, custom_llm_provider="together_ai"
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.debug(f"Error getting supported openai params: {e}")
|
||||
pass
|
||||
|
||||
optional_params = super().get_supported_openai_params(model)
|
||||
if supports_function_calling is not True:
|
||||
if supports_fc is not True:
|
||||
verbose_logger.debug(
|
||||
"Only some together models support function calling/response_format. Docs - https://docs.together.ai/docs/function-calling"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -337,8 +337,13 @@ def _get_gemini_url(
|
|||
mode: all_gemini_url_modes,
|
||||
model: str,
|
||||
stream: Optional[bool],
|
||||
gemini_api_key: Optional[str],
|
||||
) -> Tuple[str, str]:
|
||||
"""Build the Gemini API URL for the given mode.
|
||||
|
||||
The API key is NOT included in the URL. Callers must pass it via the
|
||||
``x-goog-api-key`` header instead to avoid leaking credentials in
|
||||
error tracebacks.
|
||||
"""
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
VertexGeminiConfig,
|
||||
)
|
||||
|
|
@ -352,27 +357,27 @@ def _get_gemini_url(
|
|||
endpoint = "generateContent"
|
||||
if stream is True:
|
||||
endpoint = "streamGenerateContent"
|
||||
url = "https://generativelanguage.googleapis.com/{}/{}:{}?key={}&alt=sse".format(
|
||||
api_version, _gemini_model_name, endpoint, gemini_api_key
|
||||
url = "https://generativelanguage.googleapis.com/{}/{}:{}?alt=sse".format(
|
||||
api_version, _gemini_model_name, endpoint
|
||||
)
|
||||
else:
|
||||
url = "https://generativelanguage.googleapis.com/{}/{}:{}?key={}".format(
|
||||
api_version, _gemini_model_name, endpoint, gemini_api_key
|
||||
url = "https://generativelanguage.googleapis.com/{}/{}:{}".format(
|
||||
api_version, _gemini_model_name, endpoint
|
||||
)
|
||||
elif mode == "embedding":
|
||||
endpoint = "embedContent"
|
||||
url = "https://generativelanguage.googleapis.com/v1beta/{}:{}?key={}".format(
|
||||
_gemini_model_name, endpoint, gemini_api_key
|
||||
url = "https://generativelanguage.googleapis.com/v1beta/{}:{}".format(
|
||||
_gemini_model_name, endpoint
|
||||
)
|
||||
elif mode == "batch_embedding":
|
||||
endpoint = "batchEmbedContents"
|
||||
url = "https://generativelanguage.googleapis.com/v1beta/{}:{}?key={}".format(
|
||||
_gemini_model_name, endpoint, gemini_api_key
|
||||
url = "https://generativelanguage.googleapis.com/v1beta/{}:{}".format(
|
||||
_gemini_model_name, endpoint
|
||||
)
|
||||
elif mode == "count_tokens":
|
||||
endpoint = "countTokens"
|
||||
url = "https://generativelanguage.googleapis.com/v1beta/{}:{}?key={}".format(
|
||||
_gemini_model_name, endpoint, gemini_api_key
|
||||
url = "https://generativelanguage.googleapis.com/v1beta/{}:{}".format(
|
||||
_gemini_model_name, endpoint
|
||||
)
|
||||
elif mode == "image_generation":
|
||||
raise ValueError(
|
||||
|
|
|
|||
|
|
@ -61,12 +61,11 @@ class ContextCachingEndpoints(VertexBase):
|
|||
Returns
|
||||
token, url
|
||||
"""
|
||||
auth_header: Optional[str]
|
||||
if custom_llm_provider == "gemini":
|
||||
auth_header = None
|
||||
auth_header = {"x-goog-api-key": gemini_api_key} # type: ignore[assignment]
|
||||
endpoint = "cachedContents"
|
||||
url = "https://generativelanguage.googleapis.com/v1beta/{}?key={}".format(
|
||||
endpoint, gemini_api_key
|
||||
)
|
||||
url = "https://generativelanguage.googleapis.com/v1beta/{}".format(endpoint)
|
||||
elif custom_llm_provider == "vertex_ai":
|
||||
auth_header = vertex_auth_header
|
||||
endpoint = "cachedContents"
|
||||
|
|
@ -93,9 +92,9 @@ class ContextCachingEndpoints(VertexBase):
|
|||
model=model,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_api_version="v1beta1"
|
||||
if custom_llm_provider == "vertex_ai_beta"
|
||||
else "v1",
|
||||
vertex_api_version=(
|
||||
"v1beta1" if custom_llm_provider == "vertex_ai_beta" else "v1"
|
||||
),
|
||||
)
|
||||
|
||||
def check_cache(
|
||||
|
|
@ -353,7 +352,9 @@ class ContextCachingEndpoints(VertexBase):
|
|||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
if token is not None:
|
||||
if isinstance(token, dict):
|
||||
headers.update(token)
|
||||
elif token is not None:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
if extra_headers is not None:
|
||||
headers.update(extra_headers)
|
||||
|
|
@ -501,7 +502,9 @@ class ContextCachingEndpoints(VertexBase):
|
|||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
if token is not None:
|
||||
if isinstance(token, dict):
|
||||
headers.update(token)
|
||||
elif token is not None:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
if extra_headers is not None:
|
||||
headers.update(extra_headers)
|
||||
|
|
|
|||
|
|
@ -78,6 +78,19 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
|
|||
|
||||
return endpoint
|
||||
|
||||
@staticmethod
|
||||
def _strip_version_suffix(model: str) -> str:
|
||||
"""
|
||||
Strip version suffixes (e.g. @default, @20251001) from model names.
|
||||
|
||||
The Vertex AI count-tokens endpoint rejects model names that include
|
||||
version suffixes — for example, "claude-sonnet-4-6@default" returns
|
||||
"not supported for token counting" while "claude-sonnet-4-6" works.
|
||||
"""
|
||||
if "@" in model:
|
||||
return model.split("@")[0]
|
||||
return model
|
||||
|
||||
async def handle_count_tokens_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -98,6 +111,15 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
|
|||
Raises:
|
||||
ValueError: If required parameters are missing or invalid
|
||||
"""
|
||||
# Strip version suffixes (@default, @20251001, etc.) — the Vertex AI
|
||||
# count-tokens endpoint does not accept versioned model names.
|
||||
model = self._strip_version_suffix(model)
|
||||
if "model" in request_data:
|
||||
request_data = {
|
||||
**request_data,
|
||||
"model": self._strip_version_suffix(request_data["model"]),
|
||||
}
|
||||
|
||||
# Validate request
|
||||
if "messages" not in request_data:
|
||||
raise ValueError("messages required for token counting")
|
||||
|
|
|
|||
|
|
@ -412,7 +412,7 @@ class VertexBase:
|
|||
url = "{}/models/{}:{}".format(api_base, model, endpoint)
|
||||
if gemini_api_key is None:
|
||||
raise ValueError(
|
||||
"Missing gemini_api_key, please set `GEMINI_API_KEY`"
|
||||
"Missing Gemini API key. Set the GEMINI_API_KEY or GOOGLE_API_KEY environment variable."
|
||||
)
|
||||
if gemini_api_key is not None:
|
||||
auth_header = {"x-goog-api-key": gemini_api_key} # type: ignore[assignment]
|
||||
|
|
@ -469,13 +469,16 @@ class VertexBase:
|
|||
"""
|
||||
version: Optional[Literal["v1beta1", "v1"]] = None
|
||||
if custom_llm_provider == "gemini":
|
||||
if not gemini_api_key:
|
||||
raise ValueError(
|
||||
"Missing Gemini API key. Set the GEMINI_API_KEY or GOOGLE_API_KEY environment variable."
|
||||
)
|
||||
url, endpoint = _get_gemini_url(
|
||||
mode=mode,
|
||||
model=model,
|
||||
stream=stream,
|
||||
gemini_api_key=gemini_api_key,
|
||||
)
|
||||
auth_header = None # this field is not used for gemin
|
||||
auth_header = {"x-goog-api-key": gemini_api_key} # type: ignore[assignment]
|
||||
else:
|
||||
vertex_location = self.get_vertex_region(
|
||||
vertex_region=vertex_location,
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ from typing import (
|
|||
get_args,
|
||||
)
|
||||
|
||||
from litellm._logging import _redact_string
|
||||
from litellm._uuid import uuid
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -76,6 +77,7 @@ from litellm.litellm_core_utils.audio_utils.utils import (
|
|||
calculate_request_duration,
|
||||
get_audio_file_for_health_check,
|
||||
)
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.dd_tracing import tracer
|
||||
from litellm.litellm_core_utils.get_provider_specific_headers import (
|
||||
ProviderSpecificHeaderUtils,
|
||||
|
|
@ -1400,14 +1402,13 @@ def completion( # type: ignore # noqa: PLR0915
|
|||
) # support region-based pricing for bedrock
|
||||
|
||||
### TIMEOUT LOGIC ###
|
||||
timeout = timeout or kwargs.get("request_timeout", 600) or 600
|
||||
# set timeout for 10 minutes by default
|
||||
if isinstance(timeout, httpx.Timeout) and not supports_httpx_timeout(
|
||||
custom_llm_provider
|
||||
):
|
||||
timeout = timeout.read or 600 # default 10 min timeout
|
||||
elif not isinstance(timeout, httpx.Timeout):
|
||||
timeout = float(timeout) # type: ignore
|
||||
timeout = CompletionTimeout.resolve(
|
||||
timeout,
|
||||
kwargs,
|
||||
custom_llm_provider,
|
||||
global_timeout=getattr(litellm, "request_timeout", None),
|
||||
supports_httpx_timeout=supports_httpx_timeout,
|
||||
)
|
||||
|
||||
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
|
||||
if (
|
||||
|
|
@ -7244,7 +7245,7 @@ async def ahealth_check(
|
|||
f"Mode {mode} not supported. See modes here: https://docs.litellm.ai/docs/proxy/health"
|
||||
)
|
||||
except Exception as e:
|
||||
stack_trace = traceback.format_exc()
|
||||
stack_trace = _redact_string(traceback.format_exc())
|
||||
if isinstance(stack_trace, str):
|
||||
stack_trace = stack_trace[:1000]
|
||||
|
||||
|
|
|
|||
|
|
@ -1005,7 +1005,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"tool_use_system_prompt_tokens": 346,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true
|
||||
},
|
||||
"global.anthropic.claude-opus-4-6-v1": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
|
|
@ -1032,7 +1033,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"tool_use_system_prompt_tokens": 346,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true
|
||||
},
|
||||
"us.anthropic.claude-opus-4-6-v1": {
|
||||
"cache_creation_input_token_cost": 6.875e-06,
|
||||
|
|
@ -1059,7 +1061,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"tool_use_system_prompt_tokens": 346,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true
|
||||
},
|
||||
"eu.anthropic.claude-opus-4-6-v1": {
|
||||
"cache_creation_input_token_cost": 6.875e-06,
|
||||
|
|
@ -1086,7 +1089,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"tool_use_system_prompt_tokens": 346,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true
|
||||
},
|
||||
"au.anthropic.claude-opus-4-6-v1": {
|
||||
"cache_creation_input_token_cost": 6.875e-06,
|
||||
|
|
@ -1113,6 +1117,147 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"tool_use_system_prompt_tokens": 346,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true
|
||||
},
|
||||
"anthropic.claude-opus-4-7": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-05,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"tool_use_system_prompt_tokens": 346,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"global.anthropic.claude-opus-4-7": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-05,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"tool_use_system_prompt_tokens": 346,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"us.anthropic.claude-opus-4-7": {
|
||||
"cache_creation_input_token_cost": 6.875e-06,
|
||||
"cache_read_input_token_cost": 5.5e-07,
|
||||
"input_cost_per_token": 5.5e-06,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.75e-05,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"tool_use_system_prompt_tokens": 346,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"eu.anthropic.claude-opus-4-7": {
|
||||
"cache_creation_input_token_cost": 6.875e-06,
|
||||
"cache_read_input_token_cost": 5.5e-07,
|
||||
"input_cost_per_token": 5.5e-06,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.75e-05,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"tool_use_system_prompt_tokens": 346,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"au.anthropic.claude-opus-4-7": {
|
||||
"cache_creation_input_token_cost": 6.875e-06,
|
||||
"cache_read_input_token_cost": 5.5e-07,
|
||||
"input_cost_per_token": 5.5e-06,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.75e-05,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"tool_use_system_prompt_tokens": 346,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"anthropic.claude-sonnet-4-6": {
|
||||
|
|
@ -1765,6 +1910,35 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"tool_use_system_prompt_tokens": 159,
|
||||
"supports_max_reasoning_effort": true
|
||||
},
|
||||
"azure_ai/claude-opus-4-7": {
|
||||
"input_cost_per_token": 5e-06,
|
||||
"output_cost_per_token": 2.5e-05,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"tool_use_system_prompt_tokens": 159
|
||||
},
|
||||
"azure_ai/claude-opus-4-1": {
|
||||
|
|
@ -8928,7 +9102,8 @@
|
|||
"provider_specific_entry": {
|
||||
"us": 1.1,
|
||||
"fast": 6.0
|
||||
}
|
||||
},
|
||||
"supports_max_reasoning_effort": true
|
||||
},
|
||||
"claude-opus-4-6-20260205": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
|
|
@ -8956,6 +9131,71 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"tool_use_system_prompt_tokens": 346,
|
||||
"provider_specific_entry": {
|
||||
"us": 1.1,
|
||||
"fast": 6.0
|
||||
},
|
||||
"supports_max_reasoning_effort": true
|
||||
},
|
||||
"claude-opus-4-7": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "anthropic",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-05,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"tool_use_system_prompt_tokens": 346,
|
||||
"provider_specific_entry": {
|
||||
"us": 1.1,
|
||||
"fast": 6.0
|
||||
}
|
||||
},
|
||||
"claude-opus-4-7-20260416": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "anthropic",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-05,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"tool_use_system_prompt_tokens": 346,
|
||||
"provider_specific_entry": {
|
||||
"us": 1.1,
|
||||
"fast": 6.0
|
||||
|
|
@ -18991,13 +19231,11 @@
|
|||
"cache_read_input_token_cost_above_272k_tokens": 5e-07,
|
||||
"cache_read_input_token_cost_flex": 1.3e-07,
|
||||
"cache_read_input_token_cost_priority": 5e-07,
|
||||
"cache_read_input_token_cost_above_272k_tokens_priority": 1e-06,
|
||||
"input_cost_per_token": 2.5e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 5e-06,
|
||||
"input_cost_per_token_flex": 1.25e-06,
|
||||
"input_cost_per_token_batches": 1.25e-06,
|
||||
"input_cost_per_token_priority": 5e-06,
|
||||
"input_cost_per_token_above_272k_tokens_priority": 1e-05,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 1050000,
|
||||
"max_output_tokens": 128000,
|
||||
|
|
@ -19007,8 +19245,7 @@
|
|||
"output_cost_per_token_above_272k_tokens": 2.25e-05,
|
||||
"output_cost_per_token_flex": 7.5e-06,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"output_cost_per_token_priority": 2.25e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 3.375e-05,
|
||||
"output_cost_per_token_priority": 3e-05,
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/batch",
|
||||
|
|
@ -19041,13 +19278,11 @@
|
|||
"cache_read_input_token_cost_above_272k_tokens": 5e-07,
|
||||
"cache_read_input_token_cost_flex": 1.3e-07,
|
||||
"cache_read_input_token_cost_priority": 5e-07,
|
||||
"cache_read_input_token_cost_above_272k_tokens_priority": 1e-06,
|
||||
"input_cost_per_token": 2.5e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 5e-06,
|
||||
"input_cost_per_token_flex": 1.25e-06,
|
||||
"input_cost_per_token_batches": 1.25e-06,
|
||||
"input_cost_per_token_priority": 5e-06,
|
||||
"input_cost_per_token_above_272k_tokens_priority": 1e-05,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 1050000,
|
||||
"max_output_tokens": 128000,
|
||||
|
|
@ -19057,8 +19292,7 @@
|
|||
"output_cost_per_token_above_272k_tokens": 2.25e-05,
|
||||
"output_cost_per_token_flex": 7.5e-06,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"output_cost_per_token_priority": 2.25e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 3.375e-05,
|
||||
"output_cost_per_token_priority": 3e-05,
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/batch",
|
||||
|
|
@ -19086,14 +19320,10 @@
|
|||
"gpt-5.4-pro": {
|
||||
"cache_read_input_token_cost": 3e-06,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 6e-06,
|
||||
"cache_read_input_token_cost_priority": 6e-06,
|
||||
"cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05,
|
||||
"input_cost_per_token": 3e-05,
|
||||
"input_cost_per_token_above_272k_tokens": 6e-05,
|
||||
"input_cost_per_token_flex": 1.5e-05,
|
||||
"input_cost_per_token_batches": 1.5e-05,
|
||||
"input_cost_per_token_priority": 6e-05,
|
||||
"input_cost_per_token_above_272k_tokens_priority": 0.00012,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 1050000,
|
||||
"max_output_tokens": 128000,
|
||||
|
|
@ -19103,8 +19333,6 @@
|
|||
"output_cost_per_token_above_272k_tokens": 0.00027,
|
||||
"output_cost_per_token_flex": 9e-05,
|
||||
"output_cost_per_token_batches": 9e-05,
|
||||
"output_cost_per_token_priority": 0.00027,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 0.000405,
|
||||
"supported_endpoints": [
|
||||
"/v1/responses",
|
||||
"/v1/batch"
|
||||
|
|
@ -19135,14 +19363,10 @@
|
|||
"gpt-5.4-pro-2026-03-05": {
|
||||
"cache_read_input_token_cost": 3e-06,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 6e-06,
|
||||
"cache_read_input_token_cost_priority": 6e-06,
|
||||
"cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05,
|
||||
"input_cost_per_token": 3e-05,
|
||||
"input_cost_per_token_above_272k_tokens": 6e-05,
|
||||
"input_cost_per_token_flex": 1.5e-05,
|
||||
"input_cost_per_token_batches": 1.5e-05,
|
||||
"input_cost_per_token_priority": 6e-05,
|
||||
"input_cost_per_token_above_272k_tokens_priority": 0.00012,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 1050000,
|
||||
"max_output_tokens": 128000,
|
||||
|
|
@ -19152,8 +19376,6 @@
|
|||
"output_cost_per_token_above_272k_tokens": 0.00027,
|
||||
"output_cost_per_token_flex": 9e-05,
|
||||
"output_cost_per_token_batches": 9e-05,
|
||||
"output_cost_per_token_priority": 0.00027,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 0.000405,
|
||||
"supported_endpoints": [
|
||||
"/v1/responses",
|
||||
"/v1/batch"
|
||||
|
|
@ -19183,11 +19405,13 @@
|
|||
},
|
||||
"gpt-5.4-mini": {
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"cache_read_input_token_cost_batches": 3.8e-08,
|
||||
"cache_read_input_token_cost_flex": 3.75e-08,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08,
|
||||
"cache_read_input_token_cost_priority": 1.5e-07,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
"input_cost_per_token_flex": 3.75e-07,
|
||||
"input_cost_per_token_batches": 3.75e-07,
|
||||
"input_cost_per_token_priority": 1.5e-06,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 272000,
|
||||
"max_output_tokens": 128000,
|
||||
|
|
@ -19196,6 +19420,7 @@
|
|||
"output_cost_per_token": 4.5e-06,
|
||||
"output_cost_per_token_flex": 2.25e-06,
|
||||
"output_cost_per_token_batches": 2.25e-06,
|
||||
"output_cost_per_token_priority": 9e-06,
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/batch",
|
||||
|
|
@ -25179,6 +25404,58 @@
|
|||
"supports_web_search": true,
|
||||
"tpm": 800000
|
||||
},
|
||||
"openrouter/google/gemini-3.1-flash-lite-preview": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_read_input_token_cost_per_audio_token": 5e-08,
|
||||
"input_cost_per_audio_token": 5e-07,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"litellm_provider": "openrouter",
|
||||
"max_audio_length_hours": 8.4,
|
||||
"max_audio_per_prompt": 1,
|
||||
"max_images_per_prompt": 3000,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 65536,
|
||||
"max_pdf_size_mb": 30,
|
||||
"max_tokens": 65536,
|
||||
"max_video_length": 1,
|
||||
"max_videos_per_prompt": 10,
|
||||
"mode": "chat",
|
||||
"output_cost_per_reasoning_token": 1.5e-06,
|
||||
"output_cost_per_token": 1.5e-06,
|
||||
"rpm": 2000,
|
||||
"source": "https://ai.google.dev/pricing/gemini-3",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/completions",
|
||||
"/v1/batch"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image",
|
||||
"audio",
|
||||
"video"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": false,
|
||||
"supports_code_execution": true,
|
||||
"supports_file_search": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
"supports_video_input": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"tpm": 800000
|
||||
},
|
||||
"openrouter/google/gemini-3.1-pro-preview": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
|
||||
|
|
@ -26698,6 +26975,13 @@
|
|||
"supports_reasoning": false,
|
||||
"supports_function_calling": true
|
||||
},
|
||||
"perplexity/anthropic/claude-opus-4-7": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_web_search": true,
|
||||
"supports_reasoning": false,
|
||||
"supports_function_calling": true
|
||||
},
|
||||
"perplexity/anthropic/claude-opus-4-5": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
|
|
@ -31060,7 +31344,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"tool_use_system_prompt_tokens": 346
|
||||
"tool_use_system_prompt_tokens": 346,
|
||||
"supports_max_reasoning_effort": true
|
||||
},
|
||||
"vertex_ai/claude-opus-4-6@default": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
|
|
@ -31086,6 +31371,61 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"tool_use_system_prompt_tokens": 346,
|
||||
"supports_max_reasoning_effort": true
|
||||
},
|
||||
"vertex_ai/claude-opus-4-7": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-05,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"tool_use_system_prompt_tokens": 346
|
||||
},
|
||||
"vertex_ai/claude-opus-4-7@default": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-05,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"tool_use_system_prompt_tokens": 346
|
||||
},
|
||||
"vertex_ai/claude-sonnet-4-5": {
|
||||
|
|
@ -38251,4 +38591,4 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_pdf_input": true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,3 +14,8 @@ from typing import Optional
|
|||
_mcp_active_toolset_id: ContextVar[Optional[str]] = ContextVar(
|
||||
"_mcp_active_toolset_id", default=None
|
||||
)
|
||||
|
||||
# Per-request merged InitializeResult.instructions; set in MCP HTTP/SSE handlers.
|
||||
_mcp_gateway_initialize_instructions: ContextVar[Optional[str]] = ContextVar(
|
||||
"_mcp_gateway_initialize_instructions", default=None
|
||||
)
|
||||
|
|
|
|||
|
|
@ -184,6 +184,16 @@ class MCPServerManager:
|
|||
"gmail_send_email": "zapier_mcp_server",
|
||||
}
|
||||
"""
|
||||
self._upstream_initialize_instructions_by_server_id: Dict[str, str] = {}
|
||||
|
||||
def _remember_upstream_initialize_instructions(
|
||||
self, server: MCPServer, client: MCPClient
|
||||
) -> None:
|
||||
raw = getattr(client, "_last_initialize_instructions", None)
|
||||
if raw and str(raw).strip():
|
||||
self._upstream_initialize_instructions_by_server_id[server.server_id] = str(
|
||||
raw
|
||||
).strip()
|
||||
|
||||
def get_registry(self) -> Dict[str, MCPServer]:
|
||||
"""
|
||||
|
|
@ -204,6 +214,7 @@ class MCPServerManager:
|
|||
mcp_aliases: Optional dictionary mapping aliases to server names from litellm_settings
|
||||
"""
|
||||
verbose_logger.debug("Loading MCP Servers from config-----")
|
||||
self._upstream_initialize_instructions_by_server_id.clear()
|
||||
|
||||
# Track which aliases have been used to ensure only first occurrence is used
|
||||
used_aliases = set()
|
||||
|
|
@ -351,6 +362,7 @@ class MCPServerManager:
|
|||
aws_service_name=server_config.get("aws_service_name", None),
|
||||
aws_role_name=server_config.get("aws_role_name", None),
|
||||
aws_session_name=server_config.get("aws_session_name", None),
|
||||
instructions=server_config.get("instructions", None),
|
||||
)
|
||||
self.config_mcp_servers[server_id] = new_server
|
||||
|
||||
|
|
@ -693,6 +705,7 @@ class MCPServerManager:
|
|||
aws_service_name=aws_creds.get("aws_service_name"),
|
||||
aws_role_name=aws_creds.get("aws_role_name"),
|
||||
aws_session_name=aws_creds.get("aws_session_name"),
|
||||
instructions=mcp_server.instructions,
|
||||
)
|
||||
return new_server
|
||||
|
||||
|
|
@ -1247,6 +1260,7 @@ class MCPServerManager:
|
|||
return tools
|
||||
else:
|
||||
tools = await self._fetch_tools_with_timeout(client, server.name)
|
||||
self._remember_upstream_initialize_instructions(server, client)
|
||||
|
||||
prefixed_or_original_tools = self._create_prefixed_tools(
|
||||
tools, server, add_prefix=add_prefix
|
||||
|
|
@ -2383,6 +2397,7 @@ class MCPServerManager:
|
|||
# If proxy_logging_obj is not None, the tool call result is at index 1 (after the during hook task)
|
||||
result_index = 1 if proxy_logging_obj else 0
|
||||
result = mcp_responses[result_index]
|
||||
self._remember_upstream_initialize_instructions(mcp_server, client)
|
||||
|
||||
return cast(CallToolResult, result)
|
||||
|
||||
|
|
@ -2627,6 +2642,7 @@ class MCPServerManager:
|
|||
)
|
||||
|
||||
verbose_logger.debug("Loading MCP servers from database into registry...")
|
||||
self._upstream_initialize_instructions_by_server_id.clear()
|
||||
|
||||
# perform authz check to filter the mcp servers user has access to
|
||||
prisma_client = get_prisma_client_or_throw(
|
||||
|
|
@ -2910,6 +2926,7 @@ class MCPServerManager:
|
|||
await asyncio.wait_for(
|
||||
client.run_with_session(_noop), timeout=MCP_HEALTH_CHECK_TIMEOUT
|
||||
)
|
||||
self._remember_upstream_initialize_instructions(server, client)
|
||||
status = "healthy"
|
||||
except asyncio.TimeoutError:
|
||||
health_check_error = (
|
||||
|
|
@ -2951,6 +2968,7 @@ class MCPServerManager:
|
|||
token_url=server.token_url,
|
||||
registration_url=server.registration_url,
|
||||
allow_all_keys=server.allow_all_keys,
|
||||
instructions=server.instructions,
|
||||
)
|
||||
|
||||
async def get_all_mcp_servers_with_health_and_teams(
|
||||
|
|
@ -3046,6 +3064,7 @@ class MCPServerManager:
|
|||
is_byok=server.is_byok,
|
||||
byok_description=server.byok_description,
|
||||
byok_api_key_help_url=server.byok_api_key_help_url,
|
||||
instructions=server.instructions,
|
||||
)
|
||||
|
||||
async def get_all_mcp_servers_unfiltered(self) -> List[LiteLLM_MCPServerTable]:
|
||||
|
|
|
|||
|
|
@ -933,6 +933,7 @@ if MCP_AVAILABLE:
|
|||
authorization_url=request.authorization_url,
|
||||
registration_url=request.registration_url,
|
||||
oauth2_flow=_oauth2_flow,
|
||||
instructions=request.instructions,
|
||||
)
|
||||
|
||||
stdio_env = global_mcp_server_manager._build_stdio_env(
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ LiteLLM MCP Server Routes
|
|||
import asyncio
|
||||
import contextlib
|
||||
import time
|
||||
import types
|
||||
import traceback
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
|
@ -37,7 +38,10 @@ from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
|
|||
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
|
||||
get_request_base_url,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_active_toolset_id
|
||||
from litellm.proxy._experimental.mcp_server.mcp_context import (
|
||||
_mcp_active_toolset_id,
|
||||
_mcp_gateway_initialize_instructions,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.mcp_debug import MCPDebug
|
||||
from litellm.proxy._experimental.mcp_server.utils import (
|
||||
LITELLM_MCP_SERVER_DESCRIPTION,
|
||||
|
|
@ -122,6 +126,8 @@ _INITIALIZATION_LOCK = asyncio.Lock()
|
|||
|
||||
if MCP_AVAILABLE:
|
||||
from mcp.server import Server
|
||||
from mcp.server.lowlevel.server import NotificationOptions
|
||||
from mcp.server.models import InitializationOptions
|
||||
|
||||
# Import auth context variables and middleware
|
||||
from mcp.server.auth.middleware.auth_context import (
|
||||
|
|
@ -200,6 +206,21 @@ if MCP_AVAILABLE:
|
|||
)
|
||||
return normalized
|
||||
|
||||
def _gateway_create_initialization_options(
|
||||
self,
|
||||
notification_options: Optional[NotificationOptions] = None,
|
||||
experimental_capabilities: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
) -> InitializationOptions:
|
||||
opts = Server.create_initialization_options(
|
||||
self,
|
||||
notification_options=notification_options,
|
||||
experimental_capabilities=experimental_capabilities or {},
|
||||
)
|
||||
merged = _mcp_gateway_initialize_instructions.get()
|
||||
if merged is not None:
|
||||
return opts.model_copy(update={"instructions": merged})
|
||||
return opts
|
||||
|
||||
########################################################
|
||||
############ Initialize the MCP Server #################
|
||||
########################################################
|
||||
|
|
@ -207,6 +228,9 @@ if MCP_AVAILABLE:
|
|||
name=LITELLM_MCP_SERVER_NAME,
|
||||
version=LITELLM_MCP_SERVER_VERSION,
|
||||
)
|
||||
server.create_initialization_options = types.MethodType( # type: ignore[method-assign]
|
||||
_gateway_create_initialization_options, server
|
||||
)
|
||||
sse: SseServerTransport = SseServerTransport("/mcp/sse/messages")
|
||||
|
||||
# Create session managers
|
||||
|
|
@ -1021,7 +1045,9 @@ if MCP_AVAILABLE:
|
|||
except (ValueError, TypeError):
|
||||
pass
|
||||
ttl = _compute_per_user_token_ttl(server, raw_expires)
|
||||
await mcp_per_user_token_cache.set(user_id, server_id, access_token, ttl)
|
||||
await mcp_per_user_token_cache.set(
|
||||
user_id, server_id, access_token, ttl
|
||||
)
|
||||
|
||||
return {"Authorization": f"Bearer {access_token}"}
|
||||
except Exception as e:
|
||||
|
|
@ -1103,6 +1129,57 @@ if MCP_AVAILABLE:
|
|||
|
||||
return server_auth_header, extra_headers
|
||||
|
||||
def _merge_gateway_initialize_instructions(
|
||||
allowed_mcp_servers: List[MCPServer],
|
||||
) -> Optional[str]:
|
||||
"""YAML/DB override, else in-memory upstream text from list_tools / health_check / call_tool."""
|
||||
if not allowed_mcp_servers:
|
||||
return None
|
||||
|
||||
texts: List[Tuple[str, str]] = []
|
||||
for server in allowed_mcp_servers:
|
||||
label = (
|
||||
server.alias
|
||||
or server.server_name
|
||||
or server.name
|
||||
or server.server_id
|
||||
or "mcp"
|
||||
)
|
||||
if server.instructions and server.instructions.strip():
|
||||
texts.append((label, server.instructions.strip()))
|
||||
continue
|
||||
if server.spec_path:
|
||||
continue
|
||||
cached = global_mcp_server_manager._upstream_initialize_instructions_by_server_id.get(
|
||||
server.server_id
|
||||
)
|
||||
if cached and cached.strip():
|
||||
texts.append((label, cached.strip()))
|
||||
|
||||
if not texts:
|
||||
return None
|
||||
if len(texts) == 1:
|
||||
return texts[0][1]
|
||||
return "\n\n---\n\n".join(f"[{lbl}]\n{txt}" for lbl, txt in texts)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _gateway_initialize_instructions_request_scope(
|
||||
user_api_key_auth: Optional[UserAPIKeyAuth],
|
||||
mcp_servers: Optional[List[str]],
|
||||
client_ip: Optional[str],
|
||||
) -> AsyncIterator[None]:
|
||||
allowed = await _get_allowed_mcp_servers(
|
||||
user_api_key_auth=user_api_key_auth,
|
||||
mcp_servers=mcp_servers,
|
||||
client_ip=client_ip,
|
||||
)
|
||||
merged = _merge_gateway_initialize_instructions(allowed_mcp_servers=allowed)
|
||||
tok = _mcp_gateway_initialize_instructions.set(merged)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_mcp_gateway_initialize_instructions.reset(tok)
|
||||
|
||||
async def _get_tools_from_mcp_servers( # noqa: PLR0915
|
||||
user_api_key_auth: Optional[UserAPIKeyAuth],
|
||||
mcp_auth_header: Optional[str],
|
||||
|
|
@ -2670,7 +2747,12 @@ if MCP_AVAILABLE:
|
|||
# Request was fully handled (e.g., DELETE on non-existent session)
|
||||
return
|
||||
|
||||
await session_manager.handle_request(scope, receive, send)
|
||||
async with _gateway_initialize_instructions_request_scope(
|
||||
user_api_key_auth,
|
||||
mcp_servers,
|
||||
_client_ip,
|
||||
):
|
||||
await session_manager.handle_request(scope, receive, send)
|
||||
except HTTPException:
|
||||
# Re-raise HTTP exceptions to preserve status codes and details
|
||||
raise
|
||||
|
|
@ -2729,7 +2811,12 @@ if MCP_AVAILABLE:
|
|||
await initialize_session_managers()
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
await sse_session_manager.handle_request(scope, receive, send)
|
||||
async with _gateway_initialize_instructions_request_scope(
|
||||
user_api_key_auth,
|
||||
mcp_servers,
|
||||
_sse_client_ip,
|
||||
):
|
||||
await sse_session_manager.handle_request(scope, receive, send)
|
||||
except Exception as e:
|
||||
verbose_logger.exception(f"Error handling MCP request: {e}")
|
||||
# Instead of re-raising, try to send a graceful error response
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1,28 +1,27 @@
|
|||
1:"$Sreact.fragment"
|
||||
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"]
|
||||
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","/litellm-asset-prefix/_next/static/chunks/df37a0019220a941.js","/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","/litellm-asset-prefix/_next/static/chunks/7834a5efb7b5f959.js","/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/a7113797b37526f0.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/db0ac43a898048e2.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","/litellm-asset-prefix/_next/static/chunks/60b0cadba57cd7f7.js","/litellm-asset-prefix/_next/static/chunks/bb71734679762761.js","/litellm-asset-prefix/_next/static/chunks/1fd9dbe73d002173.js","/litellm-asset-prefix/_next/static/chunks/ed901fab61dc16dc.js","/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/ed4f62880278d987.js","/litellm-asset-prefix/_next/static/chunks/60d899dd52430ef8.js","/litellm-asset-prefix/_next/static/chunks/eaa9f9b9bb3e054b.js","/litellm-asset-prefix/_next/static/chunks/f04f887c803d9e60.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/a02f90f97248b9aa.js","/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","/litellm-asset-prefix/_next/static/chunks/a5774cdb9f28daa1.js","/litellm-asset-prefix/_next/static/chunks/1501e804b4d0f510.js","/litellm-asset-prefix/_next/static/chunks/86819b3a4f820602.js","/litellm-asset-prefix/_next/static/chunks/47a838c67cdd745e.js","/litellm-asset-prefix/_next/static/chunks/be00dd25857a2fb3.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/7dd16a650b98a4c5.js","/litellm-asset-prefix/_next/static/chunks/bd5cc6a7a48eedc7.js","/litellm-asset-prefix/_next/static/chunks/169b34fe8aeee0c7.js","/litellm-asset-prefix/_next/static/chunks/7a2dc852f68481ea.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/53ac95bfa383e1b4.js","/litellm-asset-prefix/_next/static/chunks/0c6c65a34bcde140.js"],"default"]
|
||||
18:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
|
||||
19:"$Sreact.suspense"
|
||||
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/c0b640cc12a2b90e.js","/litellm-asset-prefix/_next/static/chunks/d439b54d089ced2b.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/5855ff7033bd4d2e.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","/litellm-asset-prefix/_next/static/chunks/67faedd7c3f4c2e7.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/47a838c67cdd745e.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/747ea6dd4e127c37.js","/litellm-asset-prefix/_next/static/chunks/bcbc49655bbecdc3.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/60d899dd52430ef8.js","/litellm-asset-prefix/_next/static/chunks/d077cc2c21a37474.js","/litellm-asset-prefix/_next/static/chunks/2aa5ca37f441cf6f.js","/litellm-asset-prefix/_next/static/chunks/f04f887c803d9e60.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/bd29d39cc81d3dc6.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/9edb3e10a3bcd754.js","/litellm-asset-prefix/_next/static/chunks/eaa9f9b9bb3e054b.js","/litellm-asset-prefix/_next/static/chunks/5e4cbfe76f1ba150.js","/litellm-asset-prefix/_next/static/chunks/ca91b0fa4d619698.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/bb71734679762761.js","/litellm-asset-prefix/_next/static/chunks/ed901fab61dc16dc.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/443dce180e4b120d.js","/litellm-asset-prefix/_next/static/chunks/a5ab01e86df55e55.js","/litellm-asset-prefix/_next/static/chunks/7a2dc852f68481ea.js","/litellm-asset-prefix/_next/static/chunks/082a01ae76d64ee1.js","/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","/litellm-asset-prefix/_next/static/chunks/ac92164b24de092b.js","/litellm-asset-prefix/_next/static/chunks/7dd16a650b98a4c5.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/8ad88d515b60dca7.js"],"default"]
|
||||
17:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
|
||||
18:"$Sreact.suspense"
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
|
||||
0:{"buildId":"lRBQFcrGOsyCYLFEalzGW","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/df37a0019220a941.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/7834a5efb7b5f959.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/a7113797b37526f0.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/db0ac43a898048e2.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/60b0cadba57cd7f7.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/bb71734679762761.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/1fd9dbe73d002173.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/ed901fab61dc16dc.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/ed4f62880278d987.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/60d899dd52430ef8.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/eaa9f9b9bb3e054b.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/f04f887c803d9e60.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/a02f90f97248b9aa.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/a5774cdb9f28daa1.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/1501e804b4d0f510.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/86819b3a4f820602.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16"],"$L17"]}],"loading":null,"isPartial":false}
|
||||
0:{"buildId":"ak_B7XGok3Ra_ZXFSQmNR","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/c0b640cc12a2b90e.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/d439b54d089ced2b.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/5855ff7033bd4d2e.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/67faedd7c3f4c2e7.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/47a838c67cdd745e.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/747ea6dd4e127c37.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/bcbc49655bbecdc3.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/60d899dd52430ef8.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/d077cc2c21a37474.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/2aa5ca37f441cf6f.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/f04f887c803d9e60.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/bd29d39cc81d3dc6.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/9edb3e10a3bcd754.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/eaa9f9b9bb3e054b.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15"],"$L16"]}],"loading":null,"isPartial":false}
|
||||
4:{}
|
||||
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
|
||||
6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/47a838c67cdd745e.js","async":true}]
|
||||
7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/be00dd25857a2fb3.js","async":true}]
|
||||
6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/5e4cbfe76f1ba150.js","async":true}]
|
||||
7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/ca91b0fa4d619698.js","async":true}]
|
||||
8:["$","script","script-36",{"src":"/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","async":true}]
|
||||
9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}]
|
||||
a:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true}]
|
||||
b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/7dd16a650b98a4c5.js","async":true}]
|
||||
c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/bd5cc6a7a48eedc7.js","async":true}]
|
||||
d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/169b34fe8aeee0c7.js","async":true}]
|
||||
9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/bb71734679762761.js","async":true}]
|
||||
a:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/ed901fab61dc16dc.js","async":true}]
|
||||
b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}]
|
||||
c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/443dce180e4b120d.js","async":true}]
|
||||
d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/a5ab01e86df55e55.js","async":true}]
|
||||
e:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/7a2dc852f68481ea.js","async":true}]
|
||||
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}]
|
||||
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","async":true}]
|
||||
11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","async":true}]
|
||||
12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","async":true}]
|
||||
13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}]
|
||||
14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}]
|
||||
15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/53ac95bfa383e1b4.js","async":true}]
|
||||
16:["$","script","script-50",{"src":"/litellm-asset-prefix/_next/static/chunks/0c6c65a34bcde140.js","async":true}]
|
||||
17:["$","$L18",null,{"children":["$","$19",null,{"name":"Next.MetadataOutlet","children":"$@1a"}]}]
|
||||
1a:null
|
||||
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/082a01ae76d64ee1.js","async":true}]
|
||||
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","async":true}]
|
||||
11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","async":true}]
|
||||
12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/ac92164b24de092b.js","async":true}]
|
||||
13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/7dd16a650b98a4c5.js","async":true}]
|
||||
14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}]
|
||||
15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/8ad88d515b60dca7.js","async":true}]
|
||||
16:["$","$L17",null,{"children":["$","$18",null,{"name":"Next.MetadataOutlet","children":"$@19"}]}]
|
||||
19:null
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -3,4 +3,4 @@
|
|||
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"]
|
||||
4:"$Sreact.suspense"
|
||||
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
|
||||
0:{"buildId":"lRBQFcrGOsyCYLFEalzGW","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}
|
||||
0:{"buildId":"ak_B7XGok3Ra_ZXFSQmNR","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}
|
||||
|
|
|
|||
|
|
@ -4,5 +4,5 @@
|
|||
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
|
||||
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/c3f387b3358b56db.css","style"]
|
||||
0:{"buildId":"lRBQFcrGOsyCYLFEalzGW","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/c3f387b3358b56db.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/2f5024e5325fd185.css","style"]
|
||||
0:{"buildId":"ak_B7XGok3Ra_ZXFSQmNR","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/2f5024e5325fd185.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/c3f387b3358b56db.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/2f5024e5325fd185.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
|
||||
0:{"buildId":"lRBQFcrGOsyCYLFEalzGW","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}
|
||||
0:{"buildId":"ak_B7XGok3Ra_ZXFSQmNR","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue