mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-11 22:51:28 +00:00
This reverts commit 424e1bb931.
This commit is contained in:
Krish Dholakia
GitHub
parent
424e1bb931
commit
ba5275dc9e
649 changed files with 3515 additions and 27167 deletions
|
|
@ -112,7 +112,7 @@ jobs:
|
|||
python -m mypy .
|
||||
cd ..
|
||||
no_output_timeout: 10m
|
||||
local_testing_part1:
|
||||
local_testing:
|
||||
docker:
|
||||
- image: cimg/python:3.12
|
||||
auth:
|
||||
|
|
@ -205,15 +205,13 @@ jobs:
|
|||
|
||||
# Run pytest and generate JUnit XML report
|
||||
- run:
|
||||
name: Run tests (Part 1 - A-M)
|
||||
name: Run tests
|
||||
command: |
|
||||
mkdir test-results
|
||||
|
||||
# Discover test files (A-M)
|
||||
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py")
|
||||
|
||||
# Discover test files
|
||||
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--split-by=timings \
|
||||
--split-by=filesize \
|
||||
--verbose \
|
||||
--command="xargs python -m pytest \
|
||||
-vv \
|
||||
|
|
@ -229,8 +227,8 @@ jobs:
|
|||
- run:
|
||||
name: Rename the coverage files
|
||||
command: |
|
||||
mv coverage.xml local_testing_part1_coverage.xml
|
||||
mv .coverage local_testing_part1_coverage
|
||||
mv coverage.xml local_testing_coverage.xml
|
||||
mv .coverage local_testing_coverage
|
||||
|
||||
# Store test results
|
||||
- store_test_results:
|
||||
|
|
@ -238,136 +236,8 @@ jobs:
|
|||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
- local_testing_part1_coverage.xml
|
||||
- local_testing_part1_coverage
|
||||
local_testing_part2:
|
||||
docker:
|
||||
- image: cimg/python:3.12
|
||||
auth:
|
||||
username: ${DOCKERHUB_USERNAME}
|
||||
password: ${DOCKERHUB_PASSWORD}
|
||||
working_directory: ~/project
|
||||
parallelism: 4
|
||||
steps:
|
||||
- checkout
|
||||
- setup_google_dns
|
||||
- run:
|
||||
name: Show git commit hash
|
||||
command: |
|
||||
echo "Git commit hash: $CIRCLE_SHA1"
|
||||
|
||||
- restore_cache:
|
||||
keys:
|
||||
- v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
|
||||
- run:
|
||||
name: Install Dependencies
|
||||
command: |
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install -r .circleci/requirements.txt
|
||||
pip install "pytest==7.3.1"
|
||||
pip install "pytest-retry==1.6.3"
|
||||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install "pytest-cov==5.0.0"
|
||||
pip install "mypy==1.18.2"
|
||||
pip install "google-generativeai==0.3.2"
|
||||
pip install "google-cloud-aiplatform==1.43.0"
|
||||
pip install pyarrow
|
||||
pip install "boto3==1.36.0"
|
||||
pip install "aioboto3==13.4.0"
|
||||
pip install langchain
|
||||
pip install lunary==0.2.5
|
||||
pip install "azure-identity==1.16.1"
|
||||
pip install "langfuse==2.59.7"
|
||||
pip install "logfire==0.29.0"
|
||||
pip install numpydoc
|
||||
pip install traceloop-sdk==0.21.1
|
||||
pip install opentelemetry-api==1.25.0
|
||||
pip install opentelemetry-sdk==1.25.0
|
||||
pip install opentelemetry-exporter-otlp==1.25.0
|
||||
pip install openai==1.100.1
|
||||
pip install prisma==0.11.0
|
||||
pip install "detect_secrets==1.5.0"
|
||||
pip install "httpx==0.24.1"
|
||||
pip install "respx==0.22.0"
|
||||
pip install fastapi
|
||||
pip install "gunicorn==21.2.0"
|
||||
pip install "anyio==4.2.0"
|
||||
pip install "aiodynamo==23.10.1"
|
||||
pip install "asyncio==3.4.3"
|
||||
pip install "apscheduler==3.10.4"
|
||||
pip install "PyGithub==1.59.1"
|
||||
pip install argon2-cffi
|
||||
pip install "pytest-mock==3.12.0"
|
||||
pip install python-multipart
|
||||
pip install google-cloud-aiplatform
|
||||
pip install prometheus-client==0.20.0
|
||||
pip install "pydantic==2.10.2"
|
||||
pip install "diskcache==5.6.1"
|
||||
pip install "Pillow==10.3.0"
|
||||
pip install "jsonschema==4.22.0"
|
||||
pip install "pytest-xdist==3.6.1"
|
||||
pip install "pytest-timeout==2.2.0"
|
||||
pip install "websockets==13.1.0"
|
||||
pip install semantic_router --no-deps
|
||||
pip install aurelio_sdk --no-deps
|
||||
pip uninstall posthog -y
|
||||
- setup_litellm_enterprise_pip
|
||||
- save_cache:
|
||||
paths:
|
||||
- ./venv
|
||||
key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
|
||||
- run:
|
||||
name: Run prisma ./docker/entrypoint.sh
|
||||
command: |
|
||||
set +e
|
||||
chmod +x docker/entrypoint.sh
|
||||
./docker/entrypoint.sh
|
||||
set -e
|
||||
- run:
|
||||
name: Black Formatting
|
||||
command: |
|
||||
cd litellm
|
||||
python -m pip install black
|
||||
python -m black .
|
||||
cd ..
|
||||
|
||||
# Run pytest and generate JUnit XML report
|
||||
- run:
|
||||
name: Run tests (Part 2 - N-Z)
|
||||
command: |
|
||||
mkdir test-results
|
||||
|
||||
# Discover test files (N-Z)
|
||||
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py")
|
||||
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--split-by=timings \
|
||||
--verbose \
|
||||
--command="xargs python -m pytest \
|
||||
-vv \
|
||||
--cov=litellm \
|
||||
--cov-report=xml \
|
||||
--junitxml=test-results/junit.xml \
|
||||
--durations=20 \
|
||||
-k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \
|
||||
-n 4 \
|
||||
--timeout=300 \
|
||||
--timeout_method=thread"
|
||||
no_output_timeout: 120m
|
||||
- run:
|
||||
name: Rename the coverage files
|
||||
command: |
|
||||
mv coverage.xml local_testing_part2_coverage.xml
|
||||
mv .coverage local_testing_part2_coverage
|
||||
|
||||
# Store test results
|
||||
- store_test_results:
|
||||
path: test-results
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
- local_testing_part2_coverage.xml
|
||||
- local_testing_part2_coverage
|
||||
- local_testing_coverage.xml
|
||||
- local_testing_coverage
|
||||
langfuse_logging_unit_tests:
|
||||
docker:
|
||||
- image: cimg/python:3.11
|
||||
|
|
@ -639,6 +509,7 @@ jobs:
|
|||
username: ${DOCKERHUB_USERNAME}
|
||||
password: ${DOCKERHUB_PASSWORD}
|
||||
working_directory: ~/project
|
||||
parallelism: 4
|
||||
steps:
|
||||
- checkout
|
||||
- setup_google_dns
|
||||
|
|
@ -660,9 +531,21 @@ jobs:
|
|||
- run:
|
||||
name: Run tests
|
||||
command: |
|
||||
pwd
|
||||
ls
|
||||
python -m pytest tests/local_testing --cov=litellm --cov-report=xml -vv -k "router" -v --junitxml=test-results/junit.xml --durations=5
|
||||
mkdir test-results
|
||||
# Find test files only in local_testing
|
||||
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--split-by=filesize \
|
||||
--verbose \
|
||||
--command="xargs python -m pytest -o junit_family=legacy \
|
||||
-k 'router' \
|
||||
--cov=litellm \
|
||||
--cov-report=xml \
|
||||
-n 4 \
|
||||
--dist=loadscope \
|
||||
--junitxml=test-results/junit.xml \
|
||||
--durations=5 \
|
||||
-vv"
|
||||
no_output_timeout: 120m
|
||||
- run:
|
||||
name: Rename the coverage files
|
||||
|
|
@ -715,8 +598,8 @@ jobs:
|
|||
- run:
|
||||
name: Rename the coverage files
|
||||
command: |
|
||||
mv coverage.xml litellm_router_unit_coverage.xml
|
||||
mv .coverage litellm_router_unit_coverage
|
||||
mv coverage.xml litellm_router_coverage.xml
|
||||
mv .coverage litellm_router_coverage
|
||||
# Store test results
|
||||
- store_test_results:
|
||||
path: test-results
|
||||
|
|
@ -724,8 +607,8 @@ jobs:
|
|||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
- litellm_router_unit_coverage.xml
|
||||
- litellm_router_unit_coverage
|
||||
- litellm_router_coverage.xml
|
||||
- litellm_router_coverage
|
||||
litellm_security_tests:
|
||||
machine:
|
||||
image: ubuntu-2204:2023.10.1
|
||||
|
|
@ -1933,7 +1816,6 @@ jobs:
|
|||
pip install "mlflow==2.17.2"
|
||||
pip install "anthropic==0.52.0"
|
||||
pip install "blockbuster==1.5.24"
|
||||
pip install "pytest-xdist==3.6.1"
|
||||
# Run pytest and generate JUnit XML report
|
||||
- setup_litellm_enterprise_pip
|
||||
- run:
|
||||
|
|
@ -1941,7 +1823,7 @@ jobs:
|
|||
command: |
|
||||
pwd
|
||||
ls
|
||||
python -m pytest -vv tests/logging_callback_tests --cov=litellm -n 4 --cov-report=xml -s -v --junitxml=test-results/junit.xml --durations=5
|
||||
python -m pytest -vv tests/logging_callback_tests --cov=litellm --cov-report=xml -s -v --junitxml=test-results/junit.xml --durations=5
|
||||
no_output_timeout: 120m
|
||||
- run:
|
||||
name: Rename the coverage files
|
||||
|
|
@ -3428,7 +3310,7 @@ jobs:
|
|||
python -m venv venv
|
||||
. venv/bin/activate
|
||||
pip install coverage
|
||||
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage litellm_router_unit_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
|
||||
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
|
||||
coverage xml
|
||||
- codecov/upload:
|
||||
file: ./coverage.xml
|
||||
|
|
@ -3478,22 +3360,8 @@ jobs:
|
|||
ls dist/
|
||||
twine upload --verbose dist/*
|
||||
else
|
||||
echo "Version ${VERSION} of package is already published on PyPI."
|
||||
|
||||
# Check if corresponding Docker nightly image exists
|
||||
NIGHTLY_TAG="v${VERSION}-nightly"
|
||||
echo "Checking for Docker nightly image: litellm/litellm:${NIGHTLY_TAG}"
|
||||
|
||||
# Check Docker Hub for the nightly image
|
||||
if curl -s "https://hub.docker.com/v2/repositories/litellm/litellm/tags/${NIGHTLY_TAG}" | grep -q "name"; then
|
||||
echo "Docker nightly image ${NIGHTLY_TAG} exists. This release was already completed successfully."
|
||||
echo "Skipping PyPI publish and continuing to ensure Docker images are up to date."
|
||||
circleci step halt
|
||||
else
|
||||
echo "ERROR: PyPI package ${VERSION} exists but Docker nightly image ${NIGHTLY_TAG} does not exist!"
|
||||
echo "This indicates an incomplete release. Please investigate."
|
||||
exit 1
|
||||
fi
|
||||
echo "Version ${VERSION} of package is already published on PyPI. Skipping PyPI publish."
|
||||
circleci step halt
|
||||
fi
|
||||
- run:
|
||||
name: Trigger Github Action for new Docker Container + Trigger Load Testing
|
||||
|
|
@ -3502,21 +3370,11 @@ jobs:
|
|||
python3 -m pip install toml
|
||||
VERSION=$(python3 -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
|
||||
echo "LiteLLM Version ${VERSION}"
|
||||
|
||||
# Determine which branch to use for Docker build
|
||||
if [[ "$CIRCLE_BRANCH" =~ ^litellm_release_day_.* ]]; then
|
||||
BUILD_BRANCH="$CIRCLE_BRANCH"
|
||||
echo "Using release branch: $BUILD_BRANCH"
|
||||
else
|
||||
BUILD_BRANCH="main"
|
||||
echo "Using default branch: $BUILD_BRANCH"
|
||||
fi
|
||||
|
||||
curl -X POST \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
-H "Authorization: Bearer $GITHUB_TOKEN" \
|
||||
"https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \
|
||||
-d "{\"ref\":\"${BUILD_BRANCH}\", \"inputs\":{\"tag\":\"v${VERSION}-nightly\", \"commit_hash\":\"$CIRCLE_SHA1\"}}"
|
||||
-d "{\"ref\":\"main\", \"inputs\":{\"tag\":\"v${VERSION}-nightly\", \"commit_hash\":\"$CIRCLE_SHA1\"}}"
|
||||
echo "triggering load testing server for version ${VERSION} and commit ${CIRCLE_SHA1}"
|
||||
curl -X POST "https://proxyloadtester-production.up.railway.app/start/load/test?version=${VERSION}&commit_hash=${CIRCLE_SHA1}&release_type=nightly"
|
||||
|
||||
|
|
@ -3907,13 +3765,7 @@ workflows:
|
|||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- local_testing_part1:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- local_testing_part2:
|
||||
- local_testing:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
|
|
@ -4218,8 +4070,7 @@ workflows:
|
|||
- litellm_proxy_unit_testing_part2
|
||||
- litellm_security_tests
|
||||
- langfuse_logging_unit_tests
|
||||
- local_testing_part1
|
||||
- local_testing_part2
|
||||
- local_testing
|
||||
- litellm_assistants_api_testing
|
||||
- auth_ui_unit_tests
|
||||
- db_migration_disable_update_check:
|
||||
|
|
@ -4259,12 +4110,10 @@ workflows:
|
|||
branches:
|
||||
only:
|
||||
- main
|
||||
- /litellm_release_day_.*/
|
||||
- publish_to_pypi:
|
||||
requires:
|
||||
- mypy_linting
|
||||
- local_testing_part1
|
||||
- local_testing_part2
|
||||
- local_testing
|
||||
- build_and_test
|
||||
- e2e_openai_endpoints
|
||||
- test_bad_database_url
|
||||
|
|
|
|||
|
|
@ -16,5 +16,4 @@ uvloop==0.21.0
|
|||
mcp==1.25.0 # for MCP server
|
||||
semantic_router==0.1.10 # for auto-routing with litellm
|
||||
fastuuid==0.12.0
|
||||
responses==0.25.7 # for proxy client tests
|
||||
pytest-retry==1.6.3 # for automatic test retries
|
||||
responses==0.25.7 # for proxy client tests
|
||||
2
.github/workflows/test-linting.yml
vendored
2
.github/workflows/test-linting.yml
vendored
|
|
@ -73,4 +73,4 @@ jobs:
|
|||
|
||||
- name: Check import safety
|
||||
run: |
|
||||
poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
|
||||
poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
|
||||
2
.github/workflows/test-litellm.yml
vendored
2
.github/workflows/test-litellm.yml
vendored
|
|
@ -34,7 +34,7 @@ jobs:
|
|||
poetry run pip install "google-genai==1.22.0"
|
||||
poetry run pip install "google-cloud-aiplatform>=1.38"
|
||||
poetry run pip install "fastapi-offline==1.7.3"
|
||||
poetry run pip install "python-multipart==0.0.22"
|
||||
poetry run pip install "python-multipart==0.0.18"
|
||||
poetry run pip install "openapi-core"
|
||||
- name: Setup litellm-enterprise as local package
|
||||
run: |
|
||||
|
|
|
|||
15
.github/workflows/test-model-map.yaml
vendored
15
.github/workflows/test-model-map.yaml
vendored
|
|
@ -1,15 +0,0 @@
|
|||
name: Validate model_prices_and_context_window.json
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
|
||||
jobs:
|
||||
validate-model-prices-json:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Validate model_prices_and_context_window.json
|
||||
run: |
|
||||
jq empty model_prices_and_context_window.json
|
||||
8
.gitignore
vendored
8
.gitignore
vendored
|
|
@ -60,6 +60,10 @@ litellm/proxy/_super_secret_config.yaml
|
|||
litellm/proxy/myenv/bin/activate
|
||||
litellm/proxy/myenv/bin/Activate.ps1
|
||||
myenv/*
|
||||
litellm/proxy/_experimental/out/_next/
|
||||
litellm/proxy/_experimental/out/404/index.html
|
||||
litellm/proxy/_experimental/out/model_hub/index.html
|
||||
litellm/proxy/_experimental/out/onboarding/index.html
|
||||
litellm/tests/log.txt
|
||||
litellm/tests/langfuse.log
|
||||
litellm/tests/langfuse.log
|
||||
|
|
@ -72,6 +76,9 @@ tests/local_testing/log.txt
|
|||
litellm/proxy/_new_new_secret_config.yaml
|
||||
litellm/proxy/custom_guardrail.py
|
||||
.mypy_cache/*
|
||||
litellm/proxy/_experimental/out/404.html
|
||||
litellm/proxy/_experimental/out/404.html
|
||||
litellm/proxy/_experimental/out/model_hub.html
|
||||
.mypy_cache/*
|
||||
litellm/proxy/application.log
|
||||
tests/llm_translation/vertex_test_account.json
|
||||
|
|
@ -93,6 +100,7 @@ litellm_config.yaml
|
|||
litellm/proxy/to_delete_loadtest_work/*
|
||||
update_model_cost_map.py
|
||||
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
|
||||
litellm/proxy/_experimental/out/guardrails/index.html
|
||||
scripts/test_vertex_ai_search.py
|
||||
LAZY_LOADING_IMPROVEMENTS.md
|
||||
**/test-results
|
||||
|
|
|
|||
|
|
@ -51,14 +51,12 @@ LiteLLM is a unified interface for 100+ LLMs that:
|
|||
|
||||
### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND)
|
||||
|
||||
1. **Tremor is DEPRECATED, do not use Tremor components in new features/changes**
|
||||
- The only exception is the Tremor Table component and its required Tremor Table sub components.
|
||||
|
||||
2. **Use Common Components as much as possible**:
|
||||
1. **Use Common Components as much as possible**:
|
||||
- These are usually defined in the `common_components` directory
|
||||
- Use these components as much as possible and avoid building new components unless needed
|
||||
- Tremor components are deprecated; prefer using Ant Design (AntD) as much as possible
|
||||
|
||||
3. **Testing**:
|
||||
2. **Testing**:
|
||||
- The codebase uses **Vitest** and **React Testing Library**
|
||||
- **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId`
|
||||
- **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`)
|
||||
|
|
|
|||
15
README.md
15
README.md
|
|
@ -259,17 +259,12 @@ LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https:
|
|||
Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
|
||||
|
||||
## OSS Adopters
|
||||
<img width="250" height="104" alt="Stripe wordmark - Blurple - Small" src="https://github.com/user-attachments/assets/f7296d4f-9fbd-460d-9d05-e4df31697c4b" />
|
||||
|
||||
<img width="250" height="69" alt="download__1_-removebg-preview" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" />
|
||||
|
||||
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td><img height="60" alt="Stripe" src="https://github.com/user-attachments/assets/f7296d4f-9fbd-460d-9d05-e4df31697c4b" /></td>
|
||||
<td><img height="60" alt="Google ADK" src="https://github.com/user-attachments/assets/caf270a2-5aee-45c4-8222-41a2070c4f19" /></td>
|
||||
<td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" /></td>
|
||||
<td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
|
||||
<td><h2>Netflix</h2></td>
|
||||
<td><img height="60" alt="OpenAI Agents SDK" src="https://github.com/user-attachments/assets/c02f7be0-8c2e-4d27-aea7-7c024bfaebc0" /></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
## Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers))
|
||||
|
||||
|
|
|
|||
|
|
@ -38,10 +38,6 @@ spec:
|
|||
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.podSecurityContext | nindent 8 }}
|
||||
{{- with .Values.extraInitContainers }}
|
||||
initContainers:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: {{ include "litellm.name" . }}
|
||||
securityContext:
|
||||
|
|
|
|||
|
|
@ -35,10 +35,6 @@ spec:
|
|||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
|
||||
{{- with .Values.migrationJob.extraInitContainers }}
|
||||
initContainers:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: prisma-migrations
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (printf "main-%s" .Chart.AppVersion) }}"
|
||||
|
|
|
|||
|
|
@ -281,7 +281,6 @@ migrationJob:
|
|||
# cpu: 100m
|
||||
# memory: 100Mi
|
||||
extraContainers: []
|
||||
extraInitContainers: []
|
||||
|
||||
# Hook configuration
|
||||
hooks:
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ Follow [this guide, to add your pydantic ai agent to LiteLLM Agent Gateway](./pr
|
|||
|
||||
## Invoking your Agents
|
||||
|
||||
Use the [A2A Python SDK](https://pypi.org/project/a2a-sdk) to invoke agents through LiteLLM.
|
||||
Use the [A2A Python SDK](https://pypi.org/project/a2a/) to invoke agents through LiteLLM.
|
||||
|
||||
This example shows how to:
|
||||
1. **List available agents** - Query `/v1/agents` to see which agents your key can access
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import TabItem from '@theme/TabItem';
|
|||
LiteLLM Supports logging to the following Datdog Integrations:
|
||||
- `datadog` [Datadog Logs](https://docs.datadoghq.com/logs/)
|
||||
- `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/)
|
||||
- `datadog_cost_management` [Datadog Cloud Cost Management](#datadog-cloud-cost-management)
|
||||
- `ddtrace-run` [Datadog Tracing](#datadog-tracing)
|
||||
|
||||
## Datadog Logs
|
||||
|
|
@ -74,7 +73,7 @@ Send logs through a local DataDog agent (useful for containerized environments):
|
|||
```shell
|
||||
LITELLM_DD_AGENT_HOST="localhost" # hostname or IP of DataDog agent
|
||||
LITELLM_DD_AGENT_PORT="10518" # [OPTIONAL] port of DataDog agent (default: 10518)
|
||||
DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (Agent handles auth for Logs. REQUIRED for LLM Observability)
|
||||
DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (agent handles auth)
|
||||
DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source
|
||||
```
|
||||
|
||||
|
|
@ -85,9 +84,6 @@ When `LITELLM_DD_AGENT_HOST` is set, logs are sent to the agent instead of direc
|
|||
|
||||
**Note:** We use `LITELLM_DD_AGENT_HOST` instead of `DD_AGENT_HOST` to avoid conflicts with `ddtrace` which automatically sets `DD_AGENT_HOST` for APM tracing.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Datadog LLM Observability**: `DD_API_KEY` is **REQUIRED** even when using the Datadog Agent (`LITELLM_DD_AGENT_HOST`). The agent acts as a proxy but the API key header is mandatory for the LLM Observability endpoint.
|
||||
|
||||
**Step 3**: Start the proxy, make a test request
|
||||
|
||||
Start proxy
|
||||
|
|
@ -165,50 +161,6 @@ On the Datadog LLM Observability page, you should see that both input messages a
|
|||
|
||||
|
||||
|
||||
<Image img={require('../../img/dd_llm_obs.png')} />
|
||||
|
||||
|
||||
## Datadog Cloud Cost Management
|
||||
|
||||
| Feature | Details |
|
||||
|---------|---------|
|
||||
| **What is logged** | Aggregated LLM Costs (FOCUS format) |
|
||||
| **Events** | Periodic Uploads of Aggregated Cost Data |
|
||||
| **Product Link** | [Datadog Cloud Cost Management](https://docs.datadoghq.com/cost_management/) |
|
||||
|
||||
We will use the `--config` to set `litellm.callbacks = ["datadog_cost_management"]`. This will periodically upload aggregated LLM cost data to Datadog.
|
||||
|
||||
**Step 1**: Create a `config.yaml` file and set `litellm_settings`: `success_callback`
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-3.5-turbo
|
||||
litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
litellm_settings:
|
||||
callbacks: ["datadog_cost_management"]
|
||||
```
|
||||
|
||||
**Step 2**: Set Required env variables
|
||||
|
||||
```shell
|
||||
DD_API_KEY="your-api-key"
|
||||
DD_APP_KEY="your-app-key" # REQUIRED for Cost Management
|
||||
DD_SITE="us5.datadoghq.com"
|
||||
```
|
||||
|
||||
**Step 3**: Start the proxy
|
||||
|
||||
```shell
|
||||
litellm --config config.yaml
|
||||
```
|
||||
|
||||
**How it works**
|
||||
* LiteLLM aggregates costs in-memory by Provider, Model, Date, and Tags.
|
||||
* Requires `DD_APP_KEY` for the Custom Costs API.
|
||||
* Costs are uploaded periodically (flushed).
|
||||
|
||||
|
||||
### Datadog Tracing
|
||||
|
||||
Use `ddtrace-run` to enable [Datadog Tracing](https://ddtrace.readthedocs.io/en/stable/installation_quickstart.html) on litellm proxy
|
||||
|
|
@ -251,5 +203,5 @@ LiteLLM supports customizing the following Datadog environment variables
|
|||
| `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No |
|
||||
|
||||
\* **Required when using Direct API** (default): `DD_API_KEY` and `DD_SITE` are required
|
||||
\* **Optional when using DataDog Agent**: Set `LITELLM_DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required for **Datadog Logs**. (**Note: `DD_API_KEY` IS REQUIRED for Datadog LLM Observability**)
|
||||
\* **Optional when using DataDog Agent**: Set `LITELLM_DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required
|
||||
|
||||
|
|
|
|||
|
|
@ -1,46 +1,43 @@
|
|||
# Tool Search
|
||||
# Anthropic Tool Search
|
||||
|
||||
Tool search enables Claude to dynamically discover and load tools on-demand from large tool catalogs (10,000+ tools). Instead of loading all tool definitions into the context window upfront, Claude searches your tool catalog and loads only the tools it needs.
|
||||
|
||||
## Supported Providers
|
||||
|
||||
| Provider | Chat Completions API | Messages API |
|
||||
|----------|---------------------|--------------|
|
||||
| **Anthropic API** | ✅ | ✅ |
|
||||
| **Azure Anthropic** (Microsoft Foundry) | ✅ | ✅ |
|
||||
| **Google Cloud Vertex AI** | ✅ | ✅ |
|
||||
| **Amazon Bedrock** | ✅ (Invoke API only, Opus 4.5 only) | ✅ (Invoke API only, Opus 4.5 only) |
|
||||
|
||||
|
||||
## Benefits
|
||||
|
||||
- **Context efficiency**: Avoid consuming massive portions of your context window with tool definitions
|
||||
- **Better tool selection**: Claude's tool selection accuracy degrades with more than 30-50 tools. Tool search maintains accuracy even with thousands of tools
|
||||
- **On-demand loading**: Tools are only loaded when Claude needs them
|
||||
|
||||
## Supported Models
|
||||
|
||||
Tool search is available on:
|
||||
- Claude Opus 4.5
|
||||
- Claude Sonnet 4.5
|
||||
|
||||
## Supported Platforms
|
||||
|
||||
- Anthropic API (direct)
|
||||
- Azure Anthropic (Microsoft Foundry)
|
||||
- Google Cloud Vertex AI
|
||||
- Amazon Bedrock (invoke API only, not converse API)
|
||||
|
||||
## Tool Search Variants
|
||||
|
||||
LiteLLM supports both tool search variants:
|
||||
|
||||
### 1. Regex Tool Search (`tool_search_tool_regex_20251119`)
|
||||
|
||||
Claude constructs regex patterns to search for tools. Best for exact pattern matching (faster).
|
||||
Claude constructs regex patterns to search for tools.
|
||||
|
||||
### 2. BM25 Tool Search (`tool_search_tool_bm25_20251119`)
|
||||
|
||||
Claude uses natural language queries to search for tools using the BM25 algorithm. Best for natural language semantic search.
|
||||
Claude uses natural language queries to search for tools using the BM25 algorithm.
|
||||
|
||||
**Note**: BM25 variant is not supported on Bedrock.
|
||||
## Quick Start
|
||||
|
||||
---
|
||||
### Basic Example with Regex Tool Search
|
||||
|
||||
## Chat Completions API
|
||||
|
||||
### SDK Usage
|
||||
|
||||
#### Basic Example with Regex Tool Search
|
||||
|
||||
```python showLineNumbers title="Basic Tool Search Example"
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
|
|
@ -73,6 +70,26 @@ response = litellm.completion(
|
|||
}
|
||||
},
|
||||
"defer_loading": True # Mark for deferred loading
|
||||
},
|
||||
# Another deferred tool
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search_files",
|
||||
"description": "Search through files in the workspace",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string"},
|
||||
"file_types": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"}
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
}
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
]
|
||||
)
|
||||
|
|
@ -80,9 +97,9 @@ response = litellm.completion(
|
|||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
#### BM25 Tool Search Example
|
||||
### BM25 Tool Search Example
|
||||
|
||||
```python showLineNumbers title="BM25 Tool Search"
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
|
|
@ -117,9 +134,9 @@ response = litellm.completion(
|
|||
)
|
||||
```
|
||||
|
||||
#### Azure Anthropic Example
|
||||
## Using with Azure Anthropic
|
||||
|
||||
```python showLineNumbers title="Azure Anthropic Tool Search"
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
|
|
@ -153,9 +170,9 @@ response = litellm.completion(
|
|||
)
|
||||
```
|
||||
|
||||
#### Vertex AI Example
|
||||
## Using with Vertex AI
|
||||
|
||||
```python showLineNumbers title="Vertex AI Tool Search"
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
|
|
@ -175,9 +192,11 @@ response = litellm.completion(
|
|||
)
|
||||
```
|
||||
|
||||
#### Streaming Support
|
||||
## Streaming Support
|
||||
|
||||
```python showLineNumbers title="Streaming with Tool Search"
|
||||
Tool search works with streaming:
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
|
|
@ -214,13 +233,13 @@ for chunk in response:
|
|||
print(chunk.choices[0].delta.content, end="")
|
||||
```
|
||||
|
||||
### AI Gateway Usage
|
||||
## LiteLLM Proxy
|
||||
|
||||
Tool search works automatically through the LiteLLM proxy.
|
||||
Tool search works automatically through the LiteLLM proxy:
|
||||
|
||||
#### Proxy Configuration
|
||||
### Proxy Config
|
||||
|
||||
```yaml showLineNumbers title="config.yaml"
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: claude-sonnet
|
||||
litellm_params:
|
||||
|
|
@ -228,19 +247,18 @@ model_list:
|
|||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
```
|
||||
|
||||
#### Client Request
|
||||
### Client Request
|
||||
|
||||
```python showLineNumbers title="Client Request via Proxy"
|
||||
from anthropic import Anthropic
|
||||
```python
|
||||
import openai
|
||||
|
||||
client = Anthropic(
|
||||
client = openai.OpenAI(
|
||||
api_key="your-litellm-proxy-key",
|
||||
base_url="http://0.0.0.0:4000"
|
||||
)
|
||||
|
||||
response = client.messages.create(
|
||||
response = client.chat.completions.create(
|
||||
model="claude-sonnet",
|
||||
max_tokens=1024,
|
||||
messages=[
|
||||
{"role": "user", "content": "What's the weather?"}
|
||||
],
|
||||
|
|
@ -250,14 +268,17 @@ response = client.messages.create(
|
|||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
|
|
@ -265,278 +286,127 @@ response = client.messages.create(
|
|||
)
|
||||
```
|
||||
|
||||
---
|
||||
## Important Notes
|
||||
|
||||
## Messages API
|
||||
### Beta Header
|
||||
|
||||
The Messages API provides native Anthropic-style tool search support via the `litellm.anthropic.messages` interface.
|
||||
LiteLLM automatically detects tool search tools and adds the appropriate beta header based on your provider:
|
||||
|
||||
### SDK Usage
|
||||
- **Anthropic API & Microsoft Foundry**: `advanced-tool-use-2025-11-20`
|
||||
- **Google Cloud Vertex AI**: `tool-search-tool-2025-10-19`
|
||||
- **Amazon Bedrock** (Invoke API, Opus 4.5 only): `tool-search-tool-2025-10-19`
|
||||
|
||||
#### Basic Example
|
||||
You don't need to manually specify beta headers—LiteLLM handles this automatically.
|
||||
|
||||
```python showLineNumbers title="Messages API - Basic Tool Search"
|
||||
import litellm
|
||||
### Deferred Loading
|
||||
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
model="anthropic/claude-sonnet-4-20250514",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather in San Francisco?"
|
||||
}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather for a location",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA"
|
||||
}
|
||||
},
|
||||
"required": ["location"]
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
],
|
||||
max_tokens=1024,
|
||||
extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
|
||||
- Tools with `defer_loading: true` are only loaded when Claude discovers them via search
|
||||
- At least one tool must be non-deferred (the tool search tool itself)
|
||||
- Keep your 3-5 most frequently used tools as non-deferred for optimal performance
|
||||
|
||||
### Tool Descriptions
|
||||
|
||||
Write clear, descriptive tool names and descriptions that match how users describe tasks. The search algorithm uses:
|
||||
- Tool names
|
||||
- Tool descriptions
|
||||
- Argument names
|
||||
- Argument descriptions
|
||||
|
||||
### Usage Tracking
|
||||
|
||||
Tool search requests are tracked in the usage object:
|
||||
|
||||
```python
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[{"role": "user", "content": "Search for tools"}],
|
||||
tools=[...]
|
||||
)
|
||||
|
||||
print(response)
|
||||
# Check tool search usage
|
||||
if response.usage.server_tool_use:
|
||||
print(f"Tool search requests: {response.usage.server_tool_use.tool_search_requests}")
|
||||
```
|
||||
|
||||
#### Azure Anthropic Messages Example
|
||||
## Error Handling
|
||||
|
||||
```python showLineNumbers title="Azure Anthropic Messages API"
|
||||
import litellm
|
||||
### All Tools Deferred
|
||||
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
model="azure_anthropic/claude-sonnet-4-20250514",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the stock price of Apple?"
|
||||
}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"name": "get_stock_price",
|
||||
"description": "Get the current stock price for a ticker symbol",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"ticker": {
|
||||
"type": "string",
|
||||
"description": "The stock ticker symbol, e.g. AAPL"
|
||||
}
|
||||
},
|
||||
"required": ["ticker"]
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
],
|
||||
max_tokens=1024,
|
||||
extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
|
||||
)
|
||||
```python
|
||||
# ❌ This will fail - at least one tool must be non-deferred
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {...},
|
||||
"defer_loading": True
|
||||
}
|
||||
]
|
||||
|
||||
# ✅ Correct - tool search tool is non-deferred
|
||||
tools = [
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {...},
|
||||
"defer_loading": True
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
#### Vertex AI Messages Example
|
||||
### Missing Tool Definition
|
||||
|
||||
```python showLineNumbers title="Vertex AI Messages API"
|
||||
import litellm
|
||||
If Claude references a tool that isn't in your deferred tools list, you'll get an error. Make sure all tools that might be discovered are included in the tools parameter with `defer_loading: true`.
|
||||
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
model="vertex_ai/claude-sonnet-4@20250514",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Search the web for information about AI"
|
||||
}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "tool_search_tool_bm25_20251119",
|
||||
"name": "tool_search_tool_bm25"
|
||||
},
|
||||
{
|
||||
"name": "search_web",
|
||||
"description": "Search the web for information",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "The search query"
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
],
|
||||
max_tokens=1024,
|
||||
extra_headers={"anthropic-beta": "tool-search-tool-2025-10-19"}
|
||||
)
|
||||
```
|
||||
## Best Practices
|
||||
|
||||
#### Bedrock Messages Example
|
||||
1. **Keep frequently used tools non-deferred**: Your 3-5 most common tools should not have `defer_loading: true`
|
||||
|
||||
```python showLineNumbers title="Bedrock Messages API (Invoke)"
|
||||
import litellm
|
||||
2. **Use semantic descriptions**: Tool descriptions should use natural language that matches user queries
|
||||
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
model="bedrock/invoke/anthropic.claude-opus-4-20250514-v1:0",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather?"
|
||||
}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
],
|
||||
max_tokens=1024,
|
||||
extra_headers={"anthropic-beta": "tool-search-tool-2025-10-19"}
|
||||
)
|
||||
```
|
||||
3. **Choose the right variant**:
|
||||
- Use **regex** for exact pattern matching (faster)
|
||||
- Use **BM25** for natural language semantic search
|
||||
|
||||
#### Streaming Support
|
||||
4. **Monitor usage**: Track `tool_search_requests` in the usage object to understand search patterns
|
||||
|
||||
```python showLineNumbers title="Messages API - Streaming"
|
||||
import litellm
|
||||
import json
|
||||
5. **Optimize tool catalog**: Remove unused tools and consolidate similar functionality
|
||||
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
model="anthropic/claude-sonnet-4-20250514",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather in Tokyo?"
|
||||
}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
],
|
||||
max_tokens=1024,
|
||||
stream=True,
|
||||
extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
|
||||
)
|
||||
## When to Use Tool Search
|
||||
|
||||
async for chunk in response:
|
||||
if isinstance(chunk, bytes):
|
||||
chunk_str = chunk.decode("utf-8")
|
||||
for line in chunk_str.split("\n"):
|
||||
if line.startswith("data: "):
|
||||
try:
|
||||
json_data = json.loads(line[6:])
|
||||
print(json_data)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
```
|
||||
**Good use cases:**
|
||||
- 10+ tools available in your system
|
||||
- Tool definitions consuming >10K tokens
|
||||
- Experiencing tool selection accuracy issues
|
||||
- Building systems with multiple tool categories
|
||||
- Tool library growing over time
|
||||
|
||||
### AI Gateway Usage
|
||||
**When traditional tool calling is better:**
|
||||
- Less than 10 tools total
|
||||
- All tools are frequently used
|
||||
- Very small tool definitions (\<100 tokens total)
|
||||
|
||||
Configure the proxy to use Messages API endpoints.
|
||||
## Limitations
|
||||
|
||||
#### Proxy Configuration
|
||||
- Not compatible with tool use examples
|
||||
- Requires Claude Opus 4.5 or Sonnet 4.5
|
||||
- On Bedrock, only available via invoke API (not converse API)
|
||||
- On Bedrock, only supported for Claude Opus 4.5 (not Sonnet 4.5)
|
||||
- BM25 variant (`tool_search_tool_bm25_20251119`) is not supported on Bedrock
|
||||
- Maximum 10,000 tools in catalog
|
||||
- Returns 3-5 most relevant tools per search
|
||||
|
||||
```yaml showLineNumbers title="config.yaml"
|
||||
model_list:
|
||||
- model_name: claude-sonnet-messages
|
||||
litellm_params:
|
||||
model: anthropic/claude-sonnet-4-20250514
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
```
|
||||
### Bedrock-Specific Notes
|
||||
|
||||
#### Client Request
|
||||
|
||||
```python showLineNumbers title="Client Request via Proxy (Messages API)"
|
||||
from anthropic import Anthropic
|
||||
|
||||
client = Anthropic(
|
||||
api_key="your-litellm-proxy-key",
|
||||
base_url="http://0.0.0.0:4000"
|
||||
)
|
||||
|
||||
response = client.messages.create(
|
||||
model="claude-sonnet-messages",
|
||||
max_tokens=1024,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather?"
|
||||
}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
],
|
||||
extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
---
|
||||
When using Bedrock's Invoke API:
|
||||
- The regex variant (`tool_search_tool_regex_20251119`) is automatically normalized to `tool_search_tool_regex`
|
||||
- The BM25 variant (`tool_search_tool_bm25_20251119`) is automatically filtered out as it's not supported
|
||||
- Tool search is only available for Claude Opus 4.5 models
|
||||
|
||||
## Additional Resources
|
||||
|
||||
- [Anthropic Tool Search Documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/tool-search)
|
||||
- [LiteLLM Tool Calling Guide](https://docs.litellm.ai/docs/completion/function_call)
|
||||
|
||||
|
|
|
|||
|
|
@ -1840,57 +1840,6 @@ content = response.get('choices', [{}])[0].get('message', {}).get('content')
|
|||
print(content)
|
||||
```
|
||||
|
||||
## gemini-robotics-er-1.5-preview Usage
|
||||
|
||||
```python
|
||||
from litellm import api_base
|
||||
from openai import OpenAI
|
||||
import os
|
||||
import base64
|
||||
|
||||
client = OpenAI(base_url="http://0.0.0.0:4000", api_key="sk-12345")
|
||||
base64_image = base64.b64encode(open("closeup-object-on-table-many-260nw-1216144471.webp", "rb").read()).decode()
|
||||
|
||||
import json
|
||||
import re
|
||||
tools = [{"codeExecution": {}}]
|
||||
response = client.chat.completions.create(
|
||||
model="gemini/gemini-robotics-er-1.5-preview",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Point to no more than 10 items in the image. The label returned should be an identifying name for the object detected. The answer should follow the json format: [{\"point\": [y, x], \"label\": <label1>}, ...]. The points are in [y, x] format normalized to 0-1000."
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
tools=tools
|
||||
)
|
||||
|
||||
# Extract JSON from markdown code block if present
|
||||
content = response.choices[0].message.content
|
||||
# Look for triple-backtick JSON block
|
||||
match = re.search(r'```json\s*(.*?)\s*```', content, re.DOTALL)
|
||||
if match:
|
||||
json_str = match.group(1)
|
||||
else:
|
||||
json_str = content
|
||||
|
||||
try:
|
||||
data = json.loads(json_str)
|
||||
print(json.dumps(data, indent=2))
|
||||
except Exception as e:
|
||||
print("Error parsing response as JSON:", e)
|
||||
print("Response content:", content)
|
||||
```
|
||||
|
||||
## Usage - PDF / Videos / etc. Files
|
||||
|
||||
### Inline Data (e.g. audio stream)
|
||||
|
|
|
|||
|
|
@ -1,89 +0,0 @@
|
|||
# Sarvam.ai
|
||||
|
||||
LiteLLM supports all the text models from [Sarvam ai](https://docs.sarvam.ai/api-reference-docs/chat/chat-completions)
|
||||
|
||||
## Usage
|
||||
|
||||
```python
|
||||
import os
|
||||
from litellm import completion
|
||||
|
||||
# Set your Sarvam API key
|
||||
os.environ["SARVAM_API_KEY"] = ""
|
||||
|
||||
messages = [{"role": "user", "content": "Hello"}]
|
||||
|
||||
response = completion(
|
||||
model="sarvam/sarvam-m",
|
||||
messages=messages,
|
||||
)
|
||||
print(response)
|
||||
```
|
||||
|
||||
## Usage with LiteLLM Proxy Server
|
||||
|
||||
Here's how to call a Sarvam.ai model with the LiteLLM Proxy Server
|
||||
|
||||
1. **Modify the `config.yaml`:**
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: my-model
|
||||
litellm_params:
|
||||
model: sarvam/<your-model-name> # add sarvam/ prefix to route as Sarvam provider
|
||||
api_key: api-key # api key to send your model
|
||||
```
|
||||
|
||||
2. **Start the proxy:**
|
||||
|
||||
```bash
|
||||
$ litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
3. **Send a request to LiteLLM Proxy Server:**
|
||||
|
||||
<Tabs>
|
||||
|
||||
<TabItem value="openai" label="OpenAI Python v1.0.0+">
|
||||
|
||||
```python
|
||||
import openai
|
||||
|
||||
client = openai.OpenAI(
|
||||
api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys
|
||||
base_url="http://0.0.0.0:4000" # litellm-proxy-base url
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="my-model",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what llm are you"
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="curl" label="curl">
|
||||
|
||||
```shell
|
||||
curl --location 'http://0.0.0.0:4000/chat/completions' \
|
||||
--header 'Authorization: Bearer sk-1234' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data '{
|
||||
"model": "my-model",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what llm are you"
|
||||
}
|
||||
]
|
||||
}'
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
</Tabs>
|
||||
|
|
@ -11,7 +11,7 @@ import TabItem from '@theme/TabItem';
|
|||
| Provider Route on LiteLLM | `vercel_ai_gateway/` |
|
||||
| Link to Provider Doc | [Vercel AI Gateway Documentation ↗](https://vercel.com/docs/ai-gateway) |
|
||||
| Base URL | `https://ai-gateway.vercel.sh/v1` |
|
||||
| Supported Operations | `/chat/completions`, `/embeddings`, `/models` |
|
||||
| Supported Operations | `/chat/completions`, `/models` |
|
||||
|
||||
<br />
|
||||
<br />
|
||||
|
|
@ -73,7 +73,7 @@ messages = [{"content": "Hello, how are you?", "role": "user"}]
|
|||
|
||||
# Vercel AI Gateway call with streaming
|
||||
response = completion(
|
||||
model="vercel_ai_gateway/openai/gpt-4o",
|
||||
model="vercel_ai_gateway/openai/gpt-4o",
|
||||
messages=messages,
|
||||
stream=True
|
||||
)
|
||||
|
|
@ -82,33 +82,6 @@ for chunk in response:
|
|||
print(chunk)
|
||||
```
|
||||
|
||||
### Embeddings
|
||||
|
||||
```python showLineNumbers title="Vercel AI Gateway Embeddings"
|
||||
import os
|
||||
from litellm import embedding
|
||||
|
||||
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-api-key"
|
||||
|
||||
# Vercel AI Gateway embedding call
|
||||
response = embedding(
|
||||
model="vercel_ai_gateway/openai/text-embedding-3-small",
|
||||
input="Hello world"
|
||||
)
|
||||
|
||||
print(response.data[0]["embedding"][:5]) # Print first 5 dimensions
|
||||
```
|
||||
|
||||
You can also specify the `dimensions` parameter:
|
||||
|
||||
```python showLineNumbers title="Vercel AI Gateway Embeddings with Dimensions"
|
||||
response = embedding(
|
||||
model="vercel_ai_gateway/openai/text-embedding-3-small",
|
||||
input=["Hello world", "Goodbye world"],
|
||||
dimensions=768
|
||||
)
|
||||
```
|
||||
|
||||
## Usage - LiteLLM Proxy
|
||||
|
||||
Add the following to your LiteLLM Proxy configuration file:
|
||||
|
|
@ -124,11 +97,6 @@ model_list:
|
|||
litellm_params:
|
||||
model: vercel_ai_gateway/anthropic/claude-4-sonnet
|
||||
api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY
|
||||
|
||||
- model_name: text-embedding-3-small-gateway
|
||||
litellm_params:
|
||||
model: vercel_ai_gateway/openai/text-embedding-3-small
|
||||
api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY
|
||||
```
|
||||
|
||||
Start your LiteLLM Proxy server:
|
||||
|
|
|
|||
|
|
@ -28,37 +28,6 @@ EXPERIMENTAL_UI_LOGIN="True" litellm --config config.yaml
|
|||
|
||||
:::
|
||||
|
||||
### Configuration
|
||||
|
||||
#### JWT Token Expiration
|
||||
|
||||
By default, CLI authentication tokens expire after **24 hours**. You can customize this expiration time by setting the `LITELLM_CLI_JWT_EXPIRATION_HOURS` environment variable when starting your LiteLLM Proxy:
|
||||
|
||||
```bash
|
||||
# Set CLI JWT tokens to expire after 48 hours
|
||||
export LITELLM_CLI_JWT_EXPIRATION_HOURS=48
|
||||
export EXPERIMENTAL_UI_LOGIN="True"
|
||||
litellm --config config.yaml
|
||||
```
|
||||
|
||||
Or in a single command:
|
||||
|
||||
```bash
|
||||
LITELLM_CLI_JWT_EXPIRATION_HOURS=48 EXPERIMENTAL_UI_LOGIN="True" litellm --config config.yaml
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
- `LITELLM_CLI_JWT_EXPIRATION_HOURS=12` - Tokens expire after 12 hours
|
||||
- `LITELLM_CLI_JWT_EXPIRATION_HOURS=168` - Tokens expire after 7 days (168 hours)
|
||||
- `LITELLM_CLI_JWT_EXPIRATION_HOURS=720` - Tokens expire after 30 days (720 hours)
|
||||
|
||||
:::tip
|
||||
You can check your current token's age and expiration status using:
|
||||
```bash
|
||||
litellm-proxy whoami
|
||||
```
|
||||
:::
|
||||
|
||||
### Steps
|
||||
|
||||
1. **Install the CLI**
|
||||
|
|
|
|||
|
|
@ -462,7 +462,6 @@ router_settings:
|
|||
| CHATGPT_USER_AGENT_SUFFIX | Suffix to append to the ChatGPT user agent string
|
||||
| CIRCLE_OIDC_TOKEN | OpenID Connect token for CircleCI
|
||||
| CIRCLE_OIDC_TOKEN_V2 | Version 2 of the OpenID Connect token for CircleCI
|
||||
| CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours. Can also be set via LITELLM_CLI_JWT_EXPIRATION_HOURS
|
||||
| CLOUDZERO_API_KEY | CloudZero API key for authentication
|
||||
| CLOUDZERO_CONNECTION_ID | CloudZero connection ID for data submission
|
||||
| CLOUDZERO_EXPORT_INTERVAL_MINUTES | Interval in minutes for CloudZero data export operations
|
||||
|
|
@ -724,7 +723,6 @@ router_settings:
|
|||
| LITERAL_API_URL | API URL for Literal service
|
||||
| LITERAL_BATCH_SIZE | Batch size for Literal operations
|
||||
| LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints
|
||||
| LITELLM_CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours
|
||||
| LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API
|
||||
| LITELLM_DD_AGENT_PORT | Port of DataDog agent for LiteLLM-specific log intake. Default is 10518
|
||||
| LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
|
||||
|
|
|
|||
|
|
@ -200,7 +200,6 @@ Example `requirements.txt`
|
|||
|
||||
```shell
|
||||
litellm[proxy]==1.57.3 # Specify the litellm version you want to use
|
||||
litellm-enterprise
|
||||
prometheus_client
|
||||
langfuse
|
||||
prisma
|
||||
|
|
|
|||
|
|
@ -128,7 +128,6 @@ guardrails:
|
|||
mode: ["pre_call", "post_call", "during_call"] # Run at multiple stages
|
||||
api_key: os.environ/ONYX_API_KEY
|
||||
api_base: os.environ/ONYX_API_BASE
|
||||
timeout: 10.0 # Optional, defaults to 10 seconds
|
||||
```
|
||||
|
||||
### Required Parameters
|
||||
|
|
@ -138,7 +137,6 @@ guardrails:
|
|||
### Optional Parameters
|
||||
|
||||
- **`api_base`**: Onyx API base URL (defaults to `https://ai-guard.onyx.security`)
|
||||
- **`timeout`**: Request timeout in seconds (defaults to `10.0`)
|
||||
|
||||
## Environment Variables
|
||||
|
||||
|
|
@ -147,5 +145,4 @@ You can set these environment variables instead of hardcoding values in your con
|
|||
```shell
|
||||
export ONYX_API_KEY="your-api-key-here"
|
||||
export ONYX_API_BASE="https://ai-guard.onyx.security" # Optional
|
||||
export ONYX_TIMEOUT=10 # Optional, timeout in seconds
|
||||
```
|
||||
|
|
|
|||
|
|
@ -405,10 +405,14 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
|
|||
|
||||
## **Proxy Admin Controls**
|
||||
|
||||
### Monitoring Guardrails
|
||||
### ✨ Monitoring Guardrails
|
||||
|
||||
Monitor which guardrails were executed and whether they passed or failed. e.g. guardrail going rogue and failing requests we don't intend to fail
|
||||
|
||||
:::info
|
||||
|
||||
✨ This is an Enterprise only feature [Get a free trial](https://www.litellm.ai/enterprise#trial)
|
||||
|
||||
:::
|
||||
|
||||
#### Setup
|
||||
|
|
|
|||
|
|
@ -1,150 +0,0 @@
|
|||
import Image from '@theme/IdealImage';
|
||||
|
||||
# UI - Router Settings for Keys and Teams
|
||||
|
||||
Configure router settings at the key and team level to achieve granular control over routing behavior, fallbacks, retries, and other router configurations. This enables you to customize routing behavior for specific keys or teams without affecting global settings.
|
||||
|
||||
## Overview
|
||||
|
||||
Router Settings for Keys and Teams allows you to configure router behavior at different levels of granularity. Previously, router settings could only be configured globally, applying the same routing strategy, fallbacks, timeouts, and retry policies to all requests across your entire proxy instance.
|
||||
|
||||
With key-level and team-level router settings, you can now:
|
||||
|
||||
- **Customize routing strategies** per key or team (e.g., use `least-busy` for high-priority keys, `latency-based-routing` for others)
|
||||
- **Configure different fallback chains** for different keys or teams
|
||||
- **Set key-specific or team-specific timeouts** and retry policies
|
||||
- **Apply different reliability settings** (cooldowns, allowed failures) per key or team
|
||||
- **Override global settings** when needed for specific use cases
|
||||
|
||||
<Image img={require('../../img/ui_granular_router_settings.png')} />
|
||||
|
||||
## Summary
|
||||
|
||||
Router settings follow a **hierarchical resolution order**: **Keys > Teams > Global**. When a request is made:
|
||||
|
||||
1. **Key-level settings** are checked first. If router settings are configured for the API key being used, those settings are applied.
|
||||
2. **Team-level settings** are checked next. If the key belongs to a team and that team has router settings configured, those settings are used (unless key-level settings exist).
|
||||
3. **Global settings** are used as the final fallback. If neither key nor team settings are found, the global router settings from your proxy configuration are applied.
|
||||
|
||||
This hierarchical approach ensures that the most specific settings take precedence, allowing you to fine-tune routing behavior for individual keys or teams while maintaining sensible defaults at the global level.
|
||||
|
||||
## How Router Settings Resolution Works
|
||||
|
||||
Router settings are resolved in the following priority order:
|
||||
|
||||
### Resolution Order: Key > Team > Global
|
||||
|
||||
1. **Key-level router settings** (highest priority)
|
||||
- Applied when router settings are configured directly on an API key
|
||||
- Takes precedence over all other settings
|
||||
- Useful for individual key customization
|
||||
|
||||
2. **Team-level router settings** (medium priority)
|
||||
- Applied when the API key belongs to a team with router settings configured
|
||||
- Only used if no key-level settings exist
|
||||
- Useful for applying consistent settings across multiple keys in a team
|
||||
|
||||
3. **Global router settings** (lowest priority)
|
||||
- Applied from your proxy configuration file or database
|
||||
- Used as the default when no key or team settings are found
|
||||
- Previously, this was the only option available
|
||||
|
||||
## How to Configure Router Settings
|
||||
|
||||
### Configuring Router Settings for Keys
|
||||
|
||||
Follow these steps to configure router settings for an API key:
|
||||
|
||||
1. Navigate to [http://localhost:4000/ui/?login=success](http://localhost:4000/ui/?login=success)
|
||||
|
||||
|
||||
|
||||
2. Click "+ Create New Key" (or edit an existing key)
|
||||
|
||||
|
||||
|
||||
3. Click "Optional Settings"
|
||||
|
||||
|
||||
|
||||
4. Click "Router Settings"
|
||||
|
||||
|
||||
|
||||
5. Configure your desired router settings. For example, click "Fallbacks" to configure fallback models:
|
||||
|
||||
|
||||
|
||||
6. Click "Select a model to begin configuring fallbacks" and configure your fallback chain:
|
||||
|
||||
|
||||
|
||||
### Configuring Router Settings for Teams
|
||||
|
||||
Follow these steps to configure router settings for a team:
|
||||
|
||||
1. Navigate to [http://localhost:4000/ui/?login=success](http://localhost:4000/ui/?login=success)
|
||||
|
||||
|
||||
|
||||
2. Click "Teams"
|
||||
|
||||
|
||||
|
||||
3. Click "+ Create New Team" (or edit an existing team)
|
||||
|
||||
|
||||
|
||||
4. Click "Router Settings"
|
||||
|
||||
|
||||
|
||||
5. Configure your desired router settings. For example, click "Fallbacks" to configure fallback models:
|
||||
|
||||
|
||||
|
||||
6. Click "Select a model to begin configuring fallbacks" and configure your fallback chain:
|
||||
|
||||
|
||||
|
||||
## Use Cases
|
||||
|
||||
### Different Routing Strategies per Key
|
||||
|
||||
Configure different routing strategies for different use cases:
|
||||
|
||||
- **High-priority production keys**: Use `latency-based-routing` for optimal performance
|
||||
- **Development keys**: Use `simple-shuffle` for simplicity
|
||||
- **Cost-sensitive keys**: Use `cost-based-routing` to minimize expenses
|
||||
|
||||
### Team-Level Consistency
|
||||
|
||||
Apply consistent router settings across all keys in a team:
|
||||
|
||||
- Set team-wide fallback chains for reliability
|
||||
- Configure team-specific timeout policies
|
||||
- Apply uniform retry policies across team members
|
||||
|
||||
### Override Global Settings
|
||||
|
||||
Override global settings for specific scenarios:
|
||||
|
||||
- Production keys may need stricter timeout policies than development
|
||||
- Certain teams may require different fallback models
|
||||
- Individual keys may need custom retry policies for specific use cases
|
||||
|
||||
### Gradual Rollout
|
||||
|
||||
Test new router settings on specific keys or teams before applying globally:
|
||||
|
||||
- Configure new routing strategies on a test key first
|
||||
- Validate fallback chains on a small team before global rollout
|
||||
- A/B test different timeout values across different keys
|
||||
|
||||
## Related Features
|
||||
|
||||
- [Router Settings Reference](./config_settings.md#router_settings---reference) - Complete reference of all router settings
|
||||
- [Load Balancing](./load_balancing.md) - Learn about routing strategies and load balancing
|
||||
- [Reliability](./reliability.md) - Configure fallbacks, retries, and error handling
|
||||
- [Keys](./keys.md) - Manage API keys and their settings
|
||||
- [Teams](./teams.md) - Organize keys into teams
|
||||
|
|
@ -11,7 +11,7 @@ import Image from '@theme/IdealImage';
|
|||
|
||||
This is a free LiteLLM Enterprise feature.
|
||||
|
||||
Available via the `litellm` docker image. If you are using the pip package, you must install [`litellm-enterprise`](https://pypi.org/project/litellm-enterprise/).
|
||||
Available via the `litellm[proxy]` package or any `litellm` docker image.
|
||||
|
||||
:::
|
||||
|
||||
|
|
|
|||
|
|
@ -121,8 +121,8 @@ Use this to track overall LiteLLM Proxy usage.
|
|||
|
||||
| Metric Name | Description |
|
||||
|----------------------|--------------------------------------|
|
||||
| `litellm_proxy_failed_requests_metric` | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "user_email", "exception_status", "exception_class", "route", "model_id"` |
|
||||
| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route", "model_id"` |
|
||||
| `litellm_proxy_failed_requests_metric` | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "exception_status", "exception_class", "route"` |
|
||||
| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route"` |
|
||||
|
||||
### Callback Logging Metrics
|
||||
|
||||
|
|
@ -130,12 +130,7 @@ Monitor failures while shipping logs to downstream callbacks like `s3_v3` cold s
|
|||
|
||||
| Metric Name | Description |
|
||||
|----------------------|--------------------------------------|
|
||||
| `litellm_callback_logging_failures_metric` | Total number of failed attempts to emit logs to a configured callback. Labels: `"callback_name"`. Use this to alert on callback delivery issues such as repeated failures when writing to `s3_v3`, `langfuse`, or `langfuse_otel` and other otel providers |
|
||||
|
||||
**Supported Callbacks:**
|
||||
- `S3Logger` - S3 v2 cold storage failures
|
||||
- `langfuse` - Langfuse logging failures
|
||||
- `otel` - OpenTelemetry logging failures
|
||||
| `litellm_callback_logging_failures_metric` | Total number of failed attempts to emit logs to a configured callback. Labels: `"callback_name"`. Use this to alert on callback delivery issues such as repeated failures when writing to `s3_v3`. |
|
||||
|
||||
## LLM Provider Metrics
|
||||
|
||||
|
|
@ -196,10 +191,10 @@ Use this for LLM API Error monitoring and tracking remaining rate limits and tok
|
|||
|
||||
| Metric Name | Description |
|
||||
|----------------------|--------------------------------------|
|
||||
| `litellm_request_total_latency_metric` | Total latency (seconds) for a request to LiteLLM Proxy Server - tracked for labels "end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model", "model_id" |
|
||||
| `litellm_request_total_latency_metric` | Total latency (seconds) for a request to LiteLLM Proxy Server - tracked for labels "end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model" |
|
||||
| `litellm_overhead_latency_metric` | Latency overhead (seconds) added by LiteLLM processing - tracked for labels "model_group", "api_provider", "api_base", "litellm_model_name", "hashed_api_key", "api_key_alias" |
|
||||
| `litellm_llm_api_latency_metric` | Latency (seconds) for just the LLM API call - tracked for labels "model", "hashed_api_key", "api_key_alias", "team", "team_alias", "requested_model", "end_user", "user" |
|
||||
| `litellm_llm_api_time_to_first_token_metric` | Time to first token for LLM API call - tracked for labels `model`, `hashed_api_key`, `api_key_alias`, `team`, `team_alias`, `requested_model`, `end_user`, `user`, `model_id` [Note: only emitted for streaming requests] |
|
||||
| `litellm_llm_api_time_to_first_token_metric` | Time to first token for LLM API call - tracked for labels `model`, `hashed_api_key`, `api_key_alias`, `team`, `team_alias` [Note: only emitted for streaming requests] |
|
||||
|
||||
## Tracking `end_user` on Prometheus
|
||||
|
||||
|
|
|
|||
|
|
@ -1,121 +0,0 @@
|
|||
import Image from '@theme/IdealImage';
|
||||
|
||||
# Control Page Visibility for Internal Users
|
||||
|
||||
Configure which navigation tabs and pages are visible to internal users (non-admin developers) in the LiteLLM UI.
|
||||
|
||||
Use this feature to simplify the UI and control which pages your internal users/developers can see when signing in.
|
||||
|
||||
## Overview
|
||||
|
||||
By default, all pages accessible to internal users are visible in the navigation sidebar. The page visibility control allows admins to restrict which pages internal users can see, creating a more focused and streamlined experience.
|
||||
|
||||
|
||||
## Configure Page Visibility
|
||||
|
||||
### 1. Navigate to Settings
|
||||
|
||||
Click the **Settings** icon in the sidebar.
|
||||
|
||||
|
||||
|
||||
### 2. Go to Admin Settings
|
||||
|
||||
Click **Admin Settings** from the settings menu.
|
||||
|
||||
|
||||
|
||||
### 3. Select UI Settings
|
||||
|
||||
Click **UI Settings** to access the page visibility controls.
|
||||
|
||||
|
||||
|
||||
### 4. Open Page Visibility Configuration
|
||||
|
||||
Click **Configure Page Visibility** to expand the configuration panel.
|
||||
|
||||
|
||||
|
||||
### 5. Select Pages to Make Visible
|
||||
|
||||
Check the boxes for the pages you want internal users to see. Pages are organized by category for easy navigation.
|
||||
|
||||
|
||||
|
||||
**Available pages include:**
|
||||
- Virtual Keys
|
||||
- Playground
|
||||
- Models + Endpoints
|
||||
- Agents
|
||||
- MCP Servers
|
||||
- Search Tools
|
||||
- Vector Stores
|
||||
- Logs
|
||||
- Teams
|
||||
- Organizations
|
||||
- Usage
|
||||
- Budgets
|
||||
- And more...
|
||||
|
||||
### 6. Save Your Configuration
|
||||
|
||||
Click **Save Page Visibility Settings** to apply the changes.
|
||||
|
||||
|
||||
|
||||
### 7. Verify Changes
|
||||
|
||||
Internal users will now only see the selected pages in their navigation sidebar.
|
||||
|
||||
|
||||
|
||||
## Reset to Default
|
||||
|
||||
To restore all pages to internal users:
|
||||
|
||||
1. Open the Page Visibility configuration
|
||||
2. Click **Reset to Default (All Pages)**
|
||||
3. Click **Save Page Visibility Settings**
|
||||
|
||||
This will clear the restriction and show all accessible pages to internal users.
|
||||
|
||||
## API Configuration
|
||||
|
||||
You can also configure page visibility programmatically using the API:
|
||||
|
||||
### Get Current Settings
|
||||
|
||||
```bash
|
||||
curl -X GET 'http://localhost:4000/ui_settings/get' \
|
||||
-H 'Authorization: Bearer <your-admin-key>'
|
||||
```
|
||||
|
||||
### Update Page Visibility
|
||||
|
||||
```bash
|
||||
curl -X PATCH 'http://localhost:4000/ui_settings/update' \
|
||||
-H 'Authorization: Bearer <your-admin-key>' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"enabled_ui_pages_internal_users": [
|
||||
"api-keys",
|
||||
"agents",
|
||||
"mcp-servers",
|
||||
"logs",
|
||||
"teams"
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
### Clear Page Visibility Restrictions
|
||||
|
||||
```bash
|
||||
curl -X PATCH 'http://localhost:4000/ui_settings/update' \
|
||||
-H 'Authorization: Bearer <your-admin-key>' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"enabled_ui_pages_internal_users": null
|
||||
}'
|
||||
```
|
||||
|
||||
|
|
@ -5,7 +5,7 @@ All-in-one document ingestion pipeline: **Upload → Chunk → Embed → Vector
|
|||
| Feature | Supported |
|
||||
|---------|-----------|
|
||||
| Logging | Yes |
|
||||
| Supported Providers | `openai`, `bedrock`, `vertex_ai`, `gemini`, `s3_vectors` |
|
||||
| Supported Providers | `openai`, `bedrock`, `vertex_ai`, `gemini` |
|
||||
|
||||
:::tip
|
||||
After ingesting documents, use [/rag/query](./rag_query.md) to search and generate responses with your ingested content.
|
||||
|
|
@ -75,31 +75,6 @@ curl -X POST "http://localhost:4000/v1/rag/ingest" \
|
|||
}"
|
||||
```
|
||||
|
||||
### AWS S3 Vectors
|
||||
|
||||
```bash showLineNumbers title="Ingest to S3 Vectors"
|
||||
curl -X POST "http://localhost:4000/v1/rag/ingest" \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"file\": {
|
||||
\"filename\": \"document.txt\",
|
||||
\"content\": \"$(base64 -i document.txt)\",
|
||||
\"content_type\": \"text/plain\"
|
||||
},
|
||||
\"ingest_options\": {
|
||||
\"embedding\": {
|
||||
\"model\": \"text-embedding-3-small\"
|
||||
},
|
||||
\"vector_store\": {
|
||||
\"custom_llm_provider\": \"s3_vectors\",
|
||||
\"vector_bucket_name\": \"my-embeddings\",
|
||||
\"aws_region_name\": \"us-west-2\"
|
||||
}
|
||||
}
|
||||
}"
|
||||
```
|
||||
|
||||
## Response
|
||||
|
||||
```json
|
||||
|
|
@ -290,57 +265,6 @@ When `vector_store_id` is omitted, LiteLLM automatically creates:
|
|||
4. Install: `pip install 'google-cloud-aiplatform>=1.60.0'`
|
||||
:::
|
||||
|
||||
### vector_store (AWS S3 Vectors)
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `custom_llm_provider` | string | - | `"s3_vectors"` |
|
||||
| `vector_bucket_name` | string | **required** | S3 vector bucket name |
|
||||
| `index_name` | string | auto-create | Vector index name |
|
||||
| `dimension` | integer | auto-detect | Vector dimension (auto-detected from embedding model) |
|
||||
| `distance_metric` | string | `cosine` | Distance metric: `cosine` or `euclidean` |
|
||||
| `non_filterable_metadata_keys` | array | `["source_text"]` | Metadata keys excluded from filtering |
|
||||
| `aws_region_name` | string | `us-west-2` | AWS region |
|
||||
| `aws_access_key_id` | string | env | AWS access key |
|
||||
| `aws_secret_access_key` | string | env | AWS secret key |
|
||||
|
||||
:::info S3 Vectors Auto-Creation
|
||||
When `index_name` is omitted, LiteLLM automatically creates:
|
||||
- S3 vector bucket (if it doesn't exist)
|
||||
- Vector index with auto-detected dimensions from your embedding model
|
||||
|
||||
**Dimension Auto-Detection**: The vector dimension is automatically detected by making a test embedding request to your specified model. No need to manually specify dimensions!
|
||||
|
||||
**Supported Embedding Models**: Works with any LiteLLM-supported embedding model (OpenAI, Cohere, Bedrock, Azure, etc.)
|
||||
:::
|
||||
|
||||
**Example with auto-detection:**
|
||||
```json
|
||||
{
|
||||
"embedding": {
|
||||
"model": "text-embedding-3-small" // Dimension auto-detected as 1536
|
||||
},
|
||||
"vector_store": {
|
||||
"custom_llm_provider": "s3_vectors",
|
||||
"vector_bucket_name": "my-embeddings"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Example with custom embedding provider:**
|
||||
```json
|
||||
{
|
||||
"embedding": {
|
||||
"model": "cohere/embed-english-v3.0" // Dimension auto-detected as 1024
|
||||
},
|
||||
"vector_store": {
|
||||
"custom_llm_provider": "s3_vectors",
|
||||
"vector_bucket_name": "my-embeddings",
|
||||
"distance_metric": "cosine"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Input Examples
|
||||
|
||||
### File (Base64)
|
||||
|
|
|
|||
|
|
@ -830,12 +830,6 @@ asyncio.run(router_acompletion())
|
|||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Traffic Mirroring / Silent Experiments
|
||||
|
||||
Traffic mirroring allows you to "mimic" production traffic to a secondary (silent) model for evaluation purposes. The silent model's response is gathered in the background and does not affect the latency or result of the primary request.
|
||||
|
||||
[**See detailed guide on A/B Testing - Traffic Mirroring here**](./traffic_mirroring.md)
|
||||
|
||||
## Basic Reliability
|
||||
|
||||
### Deployment Ordering (Priority)
|
||||
|
|
|
|||
|
|
@ -1,83 +0,0 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# A/B Testing - Traffic Mirroring
|
||||
|
||||
Traffic mirroring allows you to "mimic" production traffic to a secondary (silent) model for evaluation purposes. The silent model's response is gathered in the background and does not affect the latency or result of the primary request.
|
||||
|
||||
This is useful for:
|
||||
- Testing a new model's performance on production prompts before switching.
|
||||
- Comparing costs and latency between different providers.
|
||||
- Debugging issues by mirroring traffic to a more verbose model.
|
||||
|
||||
## Quick Start
|
||||
|
||||
To enable traffic mirroring, add `silent_model` to the `litellm_params` of a deployment.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import Router
|
||||
|
||||
model_list = [
|
||||
{
|
||||
"model_name": "gpt-3.5-turbo",
|
||||
"litellm_params": {
|
||||
"model": "azure/chatgpt-v-2",
|
||||
"api_key": "...",
|
||||
"silent_model": "gpt-4" # 👈 Mirror traffic to gpt-4
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4",
|
||||
"api_key": "..."
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
router = Router(model_list=model_list)
|
||||
|
||||
# The request to "gpt-3.5-turbo" will trigger a background call to "gpt-4"
|
||||
response = await router.acompletion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "How does traffic mirroring work?"}]
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="Proxy">
|
||||
|
||||
Add `silent_model` to your `config.yaml`:
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: primary-model
|
||||
litellm_params:
|
||||
model: azure/gpt-35-turbo
|
||||
api_key: os.environ/AZURE_API_KEY
|
||||
silent_model: evaluation-model # 👈 Mirror traffic here
|
||||
- model_name: evaluation-model
|
||||
litellm_params:
|
||||
model: openai/gpt-4o
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## How it works
|
||||
1. **Request Received**: A request is made to a model group (e.g. `primary-model`).
|
||||
2. **Deployment Picked**: LiteLLM picks a deployment from the group.
|
||||
3. **Primary Call**: LiteLLM makes the call to the primary deployment.
|
||||
4. **Mirroring**: If `silent_model` is present, LiteLLM triggers a background call to that model.
|
||||
- For **Sync** calls: Uses a shared thread pool.
|
||||
- For **Async** calls: Uses `asyncio.create_task`.
|
||||
5. **Isolation**: The background call uses a `deepcopy` of the original request parameters and sets `metadata["is_silent_experiment"] = True`. It also strips out logging IDs to prevent collisions in usage tracking.
|
||||
|
||||
## Key Features
|
||||
- **Latency Isolation**: The primary request returns as soon as it's ready. The background (silent) call does not block.
|
||||
- **Unified Logging**: Background calls are processed via the Router, meaning they are automatically logged to your configured observability tools (Langfuse, S3, etc.).
|
||||
- **Evaluation**: Use the `is_silent_experiment: True` flag in your logs to filter and compare results between the primary and mirrored calls.
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 351 KiB |
|
|
@ -1,5 +1,5 @@
|
|||
---
|
||||
title: "v1.81.0-stable - Claude Code - Web Search Across All Providers"
|
||||
title: "v1.81.0 - Claude Code - Web Search Across All Providers"
|
||||
slug: "v1-81-0"
|
||||
date: 2026-01-18T10:00:00
|
||||
authors:
|
||||
|
|
@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
|
|||
docker run \
|
||||
-e STORE_MODEL_IN_DB=True \
|
||||
-p 4000:4000 \
|
||||
docker.litellm.ai/berriai/litellm:v1.81.0-stable
|
||||
docker.litellm.ai/berriai/litellm:v1.81.0.rc.1
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
|
|
|||
|
|
@ -1,423 +0,0 @@
|
|||
---
|
||||
title: "v1.81.3-stable - Performance - 25% CPU Usage Reduction"
|
||||
slug: "v1-81-3"
|
||||
date: 2026-01-26T10:00:00
|
||||
authors:
|
||||
- name: Krrish Dholakia
|
||||
title: CEO, LiteLLM
|
||||
url: https://www.linkedin.com/in/krish-d/
|
||||
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
|
||||
- name: Ishaan Jaff
|
||||
title: CTO, LiteLLM
|
||||
url: https://www.linkedin.com/in/reffajnaahsi/
|
||||
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
|
||||
hide_table_of_contents: false
|
||||
---
|
||||
|
||||
import Image from '@theme/IdealImage';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
## Deploy this version
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="docker" label="Docker">
|
||||
|
||||
``` showLineNumbers title="docker run litellm"
|
||||
docker run \
|
||||
-e STORE_MODEL_IN_DB=True \
|
||||
-p 4000:4000 \
|
||||
docker.litellm.ai/berriai/litellm:v1.81.3.rc.2
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="pip" label="Pip">
|
||||
|
||||
``` showLineNumbers title="pip install litellm"
|
||||
pip install litellm==1.81.3.rc.2
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
---
|
||||
|
||||
## New Models / Updated Models
|
||||
|
||||
### New Model Support
|
||||
|
||||
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Deprecation Date |
|
||||
| -------- | ----- | -------------- | ------------------- | -------------------- | ---------------- |
|
||||
| OpenAI | `gpt-audio`, `gpt-audio-2025-08-28` | 128K | $32/1M audio tokens, $2.5/1M text tokens | $64/1M audio tokens, $10/1M text tokens | - |
|
||||
| OpenAI | `gpt-audio-mini`, `gpt-audio-mini-2025-08-28` | 128K | $10/1M audio tokens, $0.6/1M text tokens | $20/1M audio tokens, $2.4/1M text tokens | - |
|
||||
| Deepinfra, Vertex AI, Google AI Studio, OpenRouter, Vercel AI Gateway | `gemini-2.0-flash-001`, `gemini-2.0-flash` | - | - | - | 2026-03-31 |
|
||||
| Groq | `openai/gpt-oss-120b` | 131K | 0.075/1M cache read | 0.6/1M output tokens | - |
|
||||
| Groq | `groq/openai/gpt-oss-20b` | 131K | 0.0375/1M cache read, $0.075/1M text tokens | 0.3/1M output tokens | - |
|
||||
| Vertex AI | `gemini-2.5-computer-use-preview-10-2025` | 128K | $1.25 | $10 | - |
|
||||
| Azure AI | `claude-haiku-4-5` | $1.25/1M cache read, $2/1M cache read above 1 hr, $0.1/1M text tokens | $5/1M output tokens | - |
|
||||
| Azure AI | `claude-sonnet-4-5` | $3.75/1M cache read, $6/1M cache read above 1 hr, $3/1M text tokens | $15/1M output tokens | - |
|
||||
| Azure AI | `claude-opus-4-5` | $6.25/1M cache read, $10/1M cache read above 1 hr, $0.5/1M text tokens | $25/1M output tokens | - |
|
||||
| Azure AI | `claude-opus-4-1` | $18.75/1M cache read, $30/1M cache read above 1 hr, $1.5/1M text tokens | $75/1M output tokens | - |
|
||||
|
||||
### Features
|
||||
|
||||
- **[OpenAI](../../docs/providers/openai)**
|
||||
- Add gpt-audio and gpt-audio-mini models to pricing - [PR #19509](https://github.com/BerriAI/litellm/pull/19509)
|
||||
- correct audio token costs for gpt-4o-audio-preview models - [PR #19500](https://github.com/BerriAI/litellm/pull/19500)
|
||||
- Limit stop sequence as per openai spec (ensures JetBrains IDE compatibility) - [PR #19562](https://github.com/BerriAI/litellm/pull/19562)
|
||||
|
||||
- **[VertexAI](../../docs/providers/vertex)**
|
||||
- Docs - Google Workload Identity Federation (WIF) support - [PR #19320](https://github.com/BerriAI/litellm/pull/19320)
|
||||
|
||||
- **[Agentcore](../../docs/providers/bedrock_agentcore)**
|
||||
- Fixes streaming issues with AWS Bedrock AgentCore where responses would stop after the first chunk, particularly affecting OAuth-enabled agents - [PR #17141](https://github.com/BerriAI/litellm/pull/17141)
|
||||
|
||||
- **[Chatgpt](../../docs/providers/chatgpt)**
|
||||
- Adds support for calling chatgpt subscription via LiteLLM - [PR #19030](https://github.com/BerriAI/litellm/pull/19030)
|
||||
- Adds responses API bridge support for chatgpt subscription provider - [PR #19030](https://github.com/BerriAI/litellm/pull/19030)
|
||||
|
||||
- **[Bedrock](../../docs/providers/bedrock)**
|
||||
- support for output format for bedrock invoke via v1/messages - [PR #19560](https://github.com/BerriAI/litellm/pull/19560)
|
||||
|
||||
- **[Azure](../../docs/providers/azure/azure)**
|
||||
- Add support for Azure OpenAI v1 API - [PR #19313](https://github.com/BerriAI/litellm/pull/19313)
|
||||
- preserve content_policy_violation details for images (#19328) - [PR #19372](https://github.com/BerriAI/litellm/pull/19372)
|
||||
- Support OpenAI-format nested tool definitions for Responses API - [PR #19526](https://github.com/BerriAI/litellm/pull/19526)
|
||||
|
||||
- **Gemini([Vertex AI](../../docs/providers/vertex), [Google AI Studio](../../docs/providers/gemini))**
|
||||
- use responseJsonSchema for Gemini 2.0+ models - [PR #19314](https://github.com/BerriAI/litellm/pull/19314)
|
||||
|
||||
- **[Volcengine](../../docs/providers/volcano)**
|
||||
- Support Volcengine responses api - [PR #18508](https://github.com/BerriAI/litellm/pull/18508)
|
||||
|
||||
- **[Anthropic](../../docs/providers/anthropic)**
|
||||
- Add Support for calling Claude Code Max subscriptions via LiteLLM - [PR #19453](https://github.com/BerriAI/litellm/pull/19453)
|
||||
- Add Structured output for /v1/messages with Anthropic API, Azure Anthropic API, Bedrock Converse - [PR #19545](https://github.com/BerriAI/litellm/pull/19545)
|
||||
|
||||
- **[Brave Search](../../docs/search/brave)**
|
||||
- New Search provider - [PR #19433](https://github.com/BerriAI/litellm/pull/19433)
|
||||
|
||||
- **Sarvam ai**
|
||||
- Add support for new sarvam models - [PR #19479](https://github.com/BerriAI/litellm/pull/19479)
|
||||
|
||||
- **[GMI](../../docs/providers/gmi)**
|
||||
- add GMI Cloud provider support - [PR #19376](https://github.com/BerriAI/litellm/pull/19376)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **[Anthropic](../../docs/providers/anthropic)**
|
||||
- Fix anthropic-beta sent client side being overridden instead of appended to - [PR #19343](https://github.com/BerriAI/litellm/pull/19343)
|
||||
- Filter out unsupported fields from JSON schema for Anthropic's output_format API - [PR #19482](https://github.com/BerriAI/litellm/pull/19482)
|
||||
|
||||
- **[Bedrock](../../docs/providers/bedrock)**
|
||||
- Expose stability models via /image_edits endpoint and ensure proper request transformation - [PR #19323](https://github.com/BerriAI/litellm/pull/19323)
|
||||
- Claude Code x Bedrock Invoke fails with advanced-tool-use-2025-11-20 - [PR #19373](https://github.com/BerriAI/litellm/pull/19373)
|
||||
- deduplicate tool calls in assistant history - [PR #19324](https://github.com/BerriAI/litellm/pull/19324)
|
||||
- fix: correct us.anthropic.claude-opus-4-5 In-region pricing - [PR #19310](https://github.com/BerriAI/litellm/pull/19310)
|
||||
- Fix request validation errors when using Claude 4 via bedrock invoke - [PR #19381](https://github.com/BerriAI/litellm/pull/19381)
|
||||
- Handle thinking with tool calls for Claude 4 models - [PR #19506](https://github.com/BerriAI/litellm/pull/19506)
|
||||
- correct streaming choice index for tool calls - [PR #19506](https://github.com/BerriAI/litellm/pull/19506)
|
||||
|
||||
- **[Ollama](../../docs/providers/ollama)**
|
||||
- Fix tool call errors due with improved message extraction - [PR #19369](https://github.com/BerriAI/litellm/pull/19369)
|
||||
|
||||
- **[VertexAI](../../docs/providers/vertex)**
|
||||
- Removed optional vertex_count_tokens_location param before request is sent to vertex - [PR #19359](https://github.com/BerriAI/litellm/pull/19359)
|
||||
|
||||
- **Gemini([Vertex AI](../../docs/providers/vertex), [Google AI Studio](../../docs/providers/gemini))**
|
||||
- Supports setting media_resolution and fps parameters on each video file, when using Gemini video understanding - [PR #19273](https://github.com/BerriAI/litellm/pull/19273)
|
||||
- handle reasoning_effort as dict from OpenAI Agents SDK - [PR #19419](https://github.com/BerriAI/litellm/pull/19419)
|
||||
- add file content support in tool results - [PR #19416](https://github.com/BerriAI/litellm/pull/19416)
|
||||
|
||||
- **[Azure](../../docs/providers/azure_ai)**
|
||||
- Fix Azure AI costs for Anthropic models - [PR #19530](https://github.com/BerriAI/litellm/pull/19530)
|
||||
|
||||
- **[Giga Chat](../../docs/providers/gigachat)**
|
||||
- Add tool choice mapping - [PR #19645](https://github.com/BerriAI/litellm/pull/19645)
|
||||
---
|
||||
|
||||
## AI API Endpoints (LLMs, MCP, Agents)
|
||||
|
||||
### Features
|
||||
|
||||
- **[Files API](../../docs/files_endpoints)**
|
||||
- Add managed files support when load_balancing is True - [PR #19338](https://github.com/BerriAI/litellm/pull/19338)
|
||||
|
||||
- **[Claude Plugin Marketplace](../../docs/tutorials/claude_code_plugin_marketplace)**
|
||||
- Add self hosted Claude Code Plugin Marketplace - [PR #19378](https://github.com/BerriAI/litellm/pull/19378)
|
||||
|
||||
- **[MCP](../../docs/mcp)**
|
||||
- Add MCP Protocol version 2025-11-25 support - [PR #19379](https://github.com/BerriAI/litellm/pull/19379)
|
||||
- Log MCP tool calls and list tools in the LiteLLM Spend Logs table for easier debugging - [PR #19469](https://github.com/BerriAI/litellm/pull/19469)
|
||||
|
||||
- **[Vertex AI](../../docs/providers/vertex)**
|
||||
- Ensure only anthropic betas are forwarded down to LLM API (by default) - [PR #19542](https://github.com/BerriAI/litellm/pull/19542)
|
||||
- Allow overriding to support forwarding incoming headers are forwarded down to target - [PR #19524](https://github.com/BerriAI/litellm/pull/19524)
|
||||
|
||||
- **[Chat/Completions](../../docs/completion/input)**
|
||||
- Add MCP tools response to chat completions - [PR #19552](https://github.com/BerriAI/litellm/pull/19552)
|
||||
- Add custom vertex ai finish reasons to the output - [PR #19558](https://github.com/BerriAI/litellm/pull/19558)
|
||||
- Return MCP execution in /chat/completions before model output during streaming - [PR #19623](https://github.com/BerriAI/litellm/pull/19623)
|
||||
|
||||
### Bugs
|
||||
|
||||
- **[Responses API](../../docs/response_api)**
|
||||
- Fix duplicate messages during MCP streaming tool execution - [PR #19317](https://github.com/BerriAI/litellm/pull/19317)
|
||||
- Fix pickle error when using OpenAI's Responses API with stream=True and tool_choice of type allowed_tools (an OpenAI-native parameter) - [PR #17205](https://github.com/BerriAI/litellm/pull/17205)
|
||||
- stream tool call events for non-openai models - [PR #19368](https://github.com/BerriAI/litellm/pull/19368)
|
||||
- preserve tool output ordering for gemini in responses bridge - [PR #19360](https://github.com/BerriAI/litellm/pull/19360)
|
||||
- Add ID caching to prevent ID mismatch text-start and text-delta - [PR #19390](https://github.com/BerriAI/litellm/pull/19390)
|
||||
- Include output_item, reasoning_summary_Text_done and reasoning_summary_part_done events for non-openai models - [PR #19472](https://github.com/BerriAI/litellm/pull/19472)
|
||||
|
||||
- **[Chat/Completions](../../docs/completion/input)**
|
||||
- fix: drop_params not dropping prompt_cache_key for non-OpenAI providers - [PR #19346](https://github.com/BerriAI/litellm/pull/19346)
|
||||
|
||||
- **[Realtime API](../../docs/realtime)**
|
||||
- disable SSL for ws:// WebSocket connections - [PR #19345](https://github.com/BerriAI/litellm/pull/19345)
|
||||
|
||||
- **[Generate Content](../../docs/generateContent)**
|
||||
- Log actual user input when google genai/vertex endpoints are called client-side - [PR #19156](https://github.com/BerriAI/litellm/pull/19156)
|
||||
|
||||
- **[/messages/count_tokens Anthropic Token Counting](../../docs/anthropic_count_tokens)**
|
||||
- ensure it works for Anthropic, Azure AI Anthropic on AI Gateway - [PR #19432](https://github.com/BerriAI/litellm/pull/19432)
|
||||
|
||||
- **[MCP](../../docs/mcp)**
|
||||
- forward static_headers to MCP servers - [PR #19366](https://github.com/BerriAI/litellm/pull/19366)
|
||||
|
||||
- **[Batch API](../../docs/batches)**
|
||||
- Fix: generation config empty for batch - [PR #19556](https://github.com/BerriAI/litellm/pull/19556)
|
||||
|
||||
- **[Pass Through Endpoints](../../docs/proxy/pass_through)**
|
||||
- Always reupdate registry - [PR #19420](https://github.com/BerriAI/litellm/pull/19420)
|
||||
---
|
||||
|
||||
## Management Endpoints / UI
|
||||
|
||||
### Features
|
||||
|
||||
- **Cost Estimator**
|
||||
- Fix model dropdown - [PR #19529](https://github.com/BerriAI/litellm/pull/19529)
|
||||
|
||||
- **Claude Code Plugins**
|
||||
- Allow Adding Claude Code Plugins via UI - [PR #19387](https://github.com/BerriAI/litellm/pull/19387)
|
||||
|
||||
- **Guardrails**
|
||||
- New Policy management UI - [PR #19668](https://github.com/BerriAI/litellm/pull/19668)
|
||||
- Allow adding policies on Keys/Teams + Viewing on Info panels - [PR #19688](https://github.com/BerriAI/litellm/pull/19688)
|
||||
|
||||
- **General**
|
||||
- respects custom authentication header override - [PR #19276](https://github.com/BerriAI/litellm/pull/19276)
|
||||
|
||||
- **Playground**
|
||||
- Button to Fill Custom API Base - [PR #19440](https://github.com/BerriAI/litellm/pull/19440)
|
||||
- display mcp output on the play ground - [PR #19553](https://github.com/BerriAI/litellm/pull/19553)
|
||||
|
||||
- **Models**
|
||||
- Paginate /v2/models/info - [PR #19521](https://github.com/BerriAI/litellm/pull/19521)
|
||||
- All Model Tab Pagination - [PR #19525](https://github.com/BerriAI/litellm/pull/19525)
|
||||
- Adding Optional scope Param to /models - [PR #19539](https://github.com/BerriAI/litellm/pull/19539)
|
||||
- Model Search - [PR #19622](https://github.com/BerriAI/litellm/pull/19622)
|
||||
- Filter by Model ID and Team ID - [PR #19713](https://github.com/BerriAI/litellm/pull/19713)
|
||||
|
||||
- **MCP Servers**
|
||||
- MCP Tools Tab Resetting to Overview - [PR #19468](https://github.com/BerriAI/litellm/pull/19468)
|
||||
|
||||
- **Organizations**
|
||||
- Prevent org admin from creating a new user with proxy_admin permissions - [PR #19296](https://github.com/BerriAI/litellm/pull/19296)
|
||||
- Edit Page: Reusable Model Select - [PR #19601](https://github.com/BerriAI/litellm/pull/19601)
|
||||
|
||||
- **Teams**
|
||||
- Reusable Model Select - [PR #19543](https://github.com/BerriAI/litellm/pull/19543)
|
||||
- [Fix] Team Update with Organization having All Proxy Models - [PR #19604](https://github.com/BerriAI/litellm/pull/19604)
|
||||
|
||||
- **Logs**
|
||||
- Include tool arguments in spend logs table - [PR #19640](https://github.com/BerriAI/litellm/pull/19640)
|
||||
|
||||
- **Fallbacks / Loadbalancing**
|
||||
- New fallbacks modal - [PR #19673](https://github.com/BerriAI/litellm/pull/19673)
|
||||
- Set fallbacks/loadbalancing by team/key - [PR #19686](https://github.com/BerriAI/litellm/pull/19686)
|
||||
|
||||
### Bugs
|
||||
|
||||
- **Playground**
|
||||
- increase model selector width in playground Compare view - [PR #19423](https://github.com/BerriAI/litellm/pull/19423)
|
||||
|
||||
- **Virtual Keys**
|
||||
- Sorting Shows Incorrect Entries - [PR #19534](https://github.com/BerriAI/litellm/pull/19534)
|
||||
|
||||
- **General**
|
||||
- UI 404 error when SERVER_ROOT_PATH is set - [PR #19467](https://github.com/BerriAI/litellm/pull/19467)
|
||||
- Redirect to ui/login on expired JWT - [PR #19687](https://github.com/BerriAI/litellm/pull/19687)
|
||||
|
||||
- **SSO**
|
||||
- Fix SSO user roles not updating for existing users - [PR #19621](https://github.com/BerriAI/litellm/pull/19621)
|
||||
|
||||
- **Guardrails**
|
||||
- ensure guardrail patterns persist on edit and mode toggle - [PR #19265](https://github.com/BerriAI/litellm/pull/19265)
|
||||
---
|
||||
|
||||
## AI Integrations
|
||||
|
||||
### Logging
|
||||
|
||||
- **General Logging**
|
||||
- prevent printing duplicate StandardLoggingPayload logs - [PR #19325](https://github.com/BerriAI/litellm/pull/19325)
|
||||
- Fix: log duplication when json_logs is enabled - [PR #19705](https://github.com/BerriAI/litellm/pull/19705)
|
||||
- **Langfuse OTEL**
|
||||
- ignore service logs and fix callback shadowing - [PR #19298](https://github.com/BerriAI/litellm/pull/19298)
|
||||
- **Langfuse**
|
||||
- Send litellm_trace_id - [PR #19528](https://github.com/BerriAI/litellm/pull/19528)
|
||||
- Add Langfuse mock mode for testing without API calls - [PR #19676](https://github.com/BerriAI/litellm/pull/19676)
|
||||
- **GCS Bucket**
|
||||
- prevent unbounded queue growth due to slow API calls - [PR #19297](https://github.com/BerriAI/litellm/pull/19297)
|
||||
- Add GCS mock mode for testing without API calls - [PR #19683](https://github.com/BerriAI/litellm/pull/19683)
|
||||
- **Responses API Logging**
|
||||
- Fix pydantic serialization error - [PR #19486](https://github.com/BerriAI/litellm/pull/19486)
|
||||
- **Arize Phoenix**
|
||||
- add openinference span kinds to arize phoenix - [PR #19267](https://github.com/BerriAI/litellm/pull/19267)
|
||||
- **Prometheus**
|
||||
- Added new prometheus metrics for user count and team count - [PR #19520](https://github.com/BerriAI/litellm/pull/19520)
|
||||
|
||||
### Guardrails
|
||||
|
||||
- **Bedrock Guardrails**
|
||||
- Ensure post_call guardrail checks input+output - [PR #19151](https://github.com/BerriAI/litellm/pull/19151)
|
||||
- **Prompt Security**
|
||||
- fixing prompt-security's guardrail implementation - [PR #19374](https://github.com/BerriAI/litellm/pull/19374)
|
||||
- **Presidio**
|
||||
- Fixes crash in Presidio Guardrail when running in background threads (logging_hook) - [PR #19714](https://github.com/BerriAI/litellm/pull/19714)
|
||||
- **Pillar Security**
|
||||
- Migrate Pillar Security to Generic Guardrail API - [PR #19364](https://github.com/BerriAI/litellm/pull/19364)
|
||||
- **Policy Engine**
|
||||
- New LiteLLM Policy engine - create policies to manage guardrails, conditions - permissions per Key, Team - [PR #19612](https://github.com/BerriAI/litellm/pull/19612)
|
||||
- **General**
|
||||
- add case-insensitive support for guardrail mode and actions - [PR #19480](https://github.com/BerriAI/litellm/pull/19480)
|
||||
|
||||
### Prompt Management
|
||||
|
||||
- **General**
|
||||
- fix prompt info lookup and delete using correct IDs - [PR #19358](https://github.com/BerriAI/litellm/pull/19358)
|
||||
|
||||
### Secret Manager
|
||||
|
||||
- **AWS Secret Manager**
|
||||
- ensure auto-rotation updates existing AWS secret instead of creating new one - [PR #19455](https://github.com/BerriAI/litellm/pull/19455)
|
||||
- **Hashicorp Vault**
|
||||
- Ensure key rotations work with Vault - [PR #19634](https://github.com/BerriAI/litellm/pull/19634)
|
||||
|
||||
---
|
||||
|
||||
## Spend Tracking, Budgets and Rate Limiting
|
||||
|
||||
- **Pricing Updates**
|
||||
- Add openai/dall-e base pricing entries - [PR #19133](https://github.com/BerriAI/litellm/pull/19133)
|
||||
- Add `input_cost_per_video_per_second` in ModelInfoBase - [PR #19398](https://github.com/BerriAI/litellm/pull/19398)
|
||||
|
||||
---
|
||||
|
||||
## Performance / Loadbalancing / Reliability improvements
|
||||
|
||||
|
||||
- **General**
|
||||
- Fix date overflow/division by zero in proxy utils - [PR #19527](https://github.com/BerriAI/litellm/pull/19527)
|
||||
- Fix in-flight request termination on SIGTERM when health-check runs in a separate process - [PR #19427](https://github.com/BerriAI/litellm/pull/19427)
|
||||
- Fix Pass through routes to work with server root path - [PR #19383](https://github.com/BerriAI/litellm/pull/19383)
|
||||
- Fix logging error for stop iteration - [PR #19649](https://github.com/BerriAI/litellm/pull/19649)
|
||||
- prevent retrying 4xx client errors - [PR #19275](https://github.com/BerriAI/litellm/pull/19275)
|
||||
- add better error handling for misconfig on health check - [PR #19441](https://github.com/BerriAI/litellm/pull/19441)
|
||||
|
||||
- **Router**
|
||||
- Fix Azure RPM calculation formula - [PR #19513](https://github.com/BerriAI/litellm/pull/19513)
|
||||
- Persist scheduler request queue to redis - [PR #19304](https://github.com/BerriAI/litellm/pull/19304)
|
||||
- pass search_tools to Router during DB-triggered initialization - [PR #19388](https://github.com/BerriAI/litellm/pull/19388)
|
||||
- Fixed PromptCachingCache to correctly handle messages where cache_control is a sibling key of string content - [PR #19266](https://github.com/BerriAI/litellm/pull/19266)
|
||||
|
||||
- **Memory Leaks/OOM**
|
||||
- prevent OOM with nested $defs in tool schemas - [PR #19112](https://github.com/BerriAI/litellm/pull/19112)
|
||||
- fix: HTTP client memory leaks in Presidio, OpenAI, and Gemini - [PR #19190](https://github.com/BerriAI/litellm/pull/19190)
|
||||
|
||||
- **Non root**
|
||||
- fix logfile and pidfile of supervisor for non root environment - [PR #17267](https://github.com/BerriAI/litellm/pull/17267)
|
||||
- resolve Read-only file system error in non-root images - [PR #19449](https://github.com/BerriAI/litellm/pull/19449)
|
||||
|
||||
- **Dockerfile**
|
||||
- Redis Semantic Caching - add missing redisvl dependency to requirements.txt - [PR #19417](https://github.com/BerriAI/litellm/pull/19417)
|
||||
- Bump OTEL versions to support a2a dependency - resolves modulenotfounderror for Microsoft Agents by @Harshit28j in #18991
|
||||
|
||||
- **DB**
|
||||
- Handle PostgreSQL cached plan errors during rolling deployments - [PR #19424](https://github.com/BerriAI/litellm/pull/19424)
|
||||
|
||||
- **Timeouts**
|
||||
- Fix: total timeout is not respected - [PR #19389](https://github.com/BerriAI/litellm/pull/19389)
|
||||
|
||||
- **SDK**
|
||||
- Field-Existence Checks to Type Classes to Prevent Attribute Errors - [PR #18321](https://github.com/BerriAI/litellm/pull/18321)
|
||||
- add google-cloud-aiplatform as optional dependency with clear error message - [PR #19437](https://github.com/BerriAI/litellm/pull/19437)
|
||||
- Make grpc dependency optional - [PR #19447](https://github.com/BerriAI/litellm/pull/19447)
|
||||
- Add support for retry policies - [PR #19645](https://github.com/BerriAI/litellm/pull/19645)
|
||||
|
||||
- **Performance**
|
||||
- Cut chat_completion latency by ~21% by reducing pre-call processing time - [PR #19535](https://github.com/BerriAI/litellm/pull/19535)
|
||||
- Optimize strip_trailing_slash with O(1) index check - [PR #19679](https://github.com/BerriAI/litellm/pull/19679)
|
||||
- Optimize use_custom_pricing_for_model with set intersection - [PR #19677](https://github.com/BerriAI/litellm/pull/19677)
|
||||
- perf: skip pattern_router.route() for non-wildcard models - [PR #19664](https://github.com/BerriAI/litellm/pull/19664)
|
||||
- perf: Add LRU caching to get_model_info for faster cost lookups - [PR #19606](https://github.com/BerriAI/litellm/pull/19606)
|
||||
|
||||
---
|
||||
|
||||
## General Proxy Improvements
|
||||
|
||||
### Doc Improvements
|
||||
- new tutorial for adding MCPs to Cursor via LiteLLM - [PR #19317](https://github.com/BerriAI/litellm/pull/19317)
|
||||
- fix vertex_region to vertex_location in Vertex AI pass-through docs - [PR #19380](https://github.com/BerriAI/litellm/pull/19380)
|
||||
- clarify Gemini and Vertex AI model prefix in json file - [PR #19443](https://github.com/BerriAI/litellm/pull/19443)
|
||||
- update Claude Code integration guides - [PR #19415](https://github.com/BerriAI/litellm/pull/19415)
|
||||
- adjust opencode tutorial - [PR #19605](https://github.com/BerriAI/litellm/pull/19605)
|
||||
- add spend-queue-troubleshooting docs - [PR #19659](https://github.com/BerriAI/litellm/pull/19659)
|
||||
- docs: add litellm-enterprise requirement for managed files - [PR #19689](https://github.com/BerriAI/litellm/pull/19689)
|
||||
|
||||
### Helm
|
||||
- Add support for keda in helm chart - [PR #19337](https://github.com/BerriAI/litellm/pull/19337)
|
||||
- sync Helm chart version with LiteLLM release version - [PR #19438](https://github.com/BerriAI/litellm/pull/19438)
|
||||
- Enable PreStop hook configuration in values.yaml - [PR #19613](https://github.com/BerriAI/litellm/pull/19613)
|
||||
|
||||
### General
|
||||
- Add health check scripts and parallel execution support - [PR #19295](https://github.com/BerriAI/litellm/pull/19295)
|
||||
|
||||
|
||||
---
|
||||
|
||||
## New Contributors
|
||||
|
||||
|
||||
* @dushyantzz made their first contribution in [PR #19158](https://github.com/BerriAI/litellm/pull/19158)
|
||||
* @obod-mpw made their first contribution in [PR #19133](https://github.com/BerriAI/litellm/pull/19133)
|
||||
* @msexxeta made their first contribution in [PR #19030](https://github.com/BerriAI/litellm/pull/19030)
|
||||
* @rsicart made their first contribution in [PR #19337](https://github.com/BerriAI/litellm/pull/19337)
|
||||
* @cluebbehusen made their first contribution in [PR #19311](https://github.com/BerriAI/litellm/pull/19311)
|
||||
* @Lucky-Lodhi2004 made their first contribution in [PR #19315](https://github.com/BerriAI/litellm/pull/19315)
|
||||
* @binbandit made their first contribution in [PR #19324](https://github.com/BerriAI/litellm/pull/19324)
|
||||
* @flex-myeonghyeon made their first contribution in [PR #19381](https://github.com/BerriAI/litellm/pull/19381)
|
||||
* @Lrakotoson made their first contribution in [PR #18321](https://github.com/BerriAI/litellm/pull/18321)
|
||||
* @bensi94 made their first contribution in [PR #18787](https://github.com/BerriAI/litellm/pull/18787)
|
||||
* @victorigualada made their first contribution in [PR #19368](https://github.com/BerriAI/litellm/pull/19368)
|
||||
* @VedantMadane made their first contribution in #19266
|
||||
* @stiyyagura0901 made their first contribution in #19276
|
||||
* @kamilio made their first contribution in [PR #19447](https://github.com/BerriAI/litellm/pull/19447)
|
||||
* @jonathansampson made their first contribution in [PR #19433](https://github.com/BerriAI/litellm/pull/19433)
|
||||
* @rynecarbone made their first contribution in [PR #19416](https://github.com/BerriAI/litellm/pull/19416)
|
||||
* @jayy-77 made their first contribution in #19366
|
||||
* @davida-ps made their first contribution in [PR #19374](https://github.com/BerriAI/litellm/pull/19374)
|
||||
* @joaodinissf made their first contribution in [PR #19506](https://github.com/BerriAI/litellm/pull/19506)
|
||||
* @ecao310 made their first contribution in [PR #19520](https://github.com/BerriAI/litellm/pull/19520)
|
||||
* @mpcusack-altos made their first contribution in [PR #19577](https://github.com/BerriAI/litellm/pull/19577)
|
||||
* @milan-berri made their first contribution in [PR #19602](https://github.com/BerriAI/litellm/pull/19602)
|
||||
* @xqe2011 made their first contribution in #19621
|
||||
|
||||
---
|
||||
|
||||
## Full Changelog
|
||||
|
||||
**[View complete changelog on GitHub](https://github.com/BerriAI/litellm/releases/tag/v1.81.3.rc)**
|
||||
|
|
@ -274,19 +274,11 @@ const sidebars = {
|
|||
"proxy/custom_sso",
|
||||
"proxy/ai_hub",
|
||||
"proxy/model_compare_ui",
|
||||
"proxy/public_teams",
|
||||
"proxy/self_serve",
|
||||
"proxy/ui/bulk_edit_users",
|
||||
"proxy/ui_credentials",
|
||||
"tutorials/scim_litellm",
|
||||
{
|
||||
type: "category",
|
||||
label: "UI User/Team Management",
|
||||
items: [
|
||||
"proxy/access_control",
|
||||
"proxy/public_teams",
|
||||
"proxy/self_serve",
|
||||
"proxy/ui/bulk_edit_users",
|
||||
"proxy/ui/page_visibility",
|
||||
]
|
||||
},
|
||||
{
|
||||
type: "category",
|
||||
label: "UI Usage Tracking",
|
||||
|
|
@ -372,7 +364,6 @@ const sidebars = {
|
|||
label: "Load Balancing, Routing, Fallbacks",
|
||||
href: "https://docs.litellm.ai/docs/routing-load-balancing",
|
||||
},
|
||||
"traffic_mirroring",
|
||||
{
|
||||
type: "category",
|
||||
label: "Logging, Alerting, Metrics",
|
||||
|
|
@ -784,7 +775,6 @@ const sidebars = {
|
|||
"providers/oci",
|
||||
"providers/ollama",
|
||||
"providers/openrouter",
|
||||
"providers/sarvam",
|
||||
"providers/ovhcloud",
|
||||
"providers/perplexity",
|
||||
"providers/petals",
|
||||
|
|
@ -853,7 +843,6 @@ const sidebars = {
|
|||
"completion/image_generation_chat",
|
||||
"completion/json_mode",
|
||||
"completion/knowledgebase",
|
||||
"providers/anthropic_tool_search",
|
||||
"guides/code_interpreter",
|
||||
"completion/message_trimming",
|
||||
"completion/model_alias",
|
||||
|
|
@ -890,7 +879,6 @@ const sidebars = {
|
|||
"scheduler",
|
||||
"proxy/auto_routing",
|
||||
"proxy/load_balancing",
|
||||
"proxy/keys_teams_router_settings",
|
||||
"proxy/provider_budget_routing",
|
||||
"proxy/reliability",
|
||||
"proxy/fallback_management",
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ class EnterpriseRouteChecks:
|
|||
if not premium_user:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"🚨🚨🚨 DISABLING ADMIN ENDPOINTS is an Enterprise feature\n🚨 {CommonProxyErrors.not_premium_user.value}",
|
||||
detail=f"🚨🚨🚨 DISABLING LLM API ENDPOINTS is an Enterprise feature\n🚨 {CommonProxyErrors.not_premium_user.value}",
|
||||
)
|
||||
|
||||
return get_secret_bool("DISABLE_ADMIN_ENDPOINTS") is True
|
||||
|
|
|
|||
|
|
@ -244,78 +244,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
return managed_object.created_by == user_id
|
||||
return True # don't raise error if managed object is not found
|
||||
|
||||
async def list_user_batches(
|
||||
self,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
limit: Optional[int] = None,
|
||||
after: Optional[str] = None,
|
||||
provider: Optional[str] = None,
|
||||
target_model_names: Optional[str] = None,
|
||||
llm_router: Optional[Router] = None,
|
||||
) -> Dict[str, Any]:
|
||||
# Provider filtering is not supported for managed batches
|
||||
# This is because the encoded object ids stored in the managed objects table do not contain the provider information
|
||||
# To support provider filtering, we would need to store the provider information in the encoded object ids
|
||||
if provider:
|
||||
raise Exception(
|
||||
"Filtering by 'provider' is not supported when using managed batches."
|
||||
)
|
||||
|
||||
# Model name filtering is not supported for managed batches
|
||||
# This is because the encoded object ids stored in the managed objects table do not contain the model name
|
||||
# A hash of the model name + litellm_params for the model name is encoded as the model id. This is not sufficient to reliably map the target model names to the model ids.
|
||||
if target_model_names:
|
||||
raise Exception(
|
||||
"Filtering by 'target_model_names' is not supported when using managed batches."
|
||||
)
|
||||
|
||||
where_clause: Dict[str, Any] = {"file_purpose": "batch"}
|
||||
|
||||
# Filter by user who created the batch
|
||||
if user_api_key_dict.user_id:
|
||||
where_clause["created_by"] = user_api_key_dict.user_id
|
||||
|
||||
if after:
|
||||
where_clause["id"] = {"gt": after}
|
||||
|
||||
# Fetch more than needed to allow for post-fetch filtering
|
||||
fetch_limit = limit or 20
|
||||
if target_model_names:
|
||||
# Fetch extra to account for filtering
|
||||
fetch_limit = max(fetch_limit * 3, 100)
|
||||
|
||||
batches = await self.prisma_client.db.litellm_managedobjecttable.find_many(
|
||||
where=where_clause,
|
||||
take=fetch_limit,
|
||||
order={"created_at": "desc"},
|
||||
)
|
||||
|
||||
batch_objects: List[LiteLLMBatch] = []
|
||||
for batch in batches:
|
||||
try:
|
||||
# Stop once we have enough after filtering
|
||||
if len(batch_objects) >= (limit or 20):
|
||||
break
|
||||
|
||||
batch_data = json.loads(batch.file_object) if isinstance(batch.file_object, str) else batch.file_object
|
||||
batch_obj = LiteLLMBatch(**batch_data)
|
||||
batch_obj.id = batch.unified_object_id
|
||||
batch_objects.append(batch_obj)
|
||||
|
||||
except Exception as e:
|
||||
verbose_logger.warning(
|
||||
f"Failed to parse batch object {batch.unified_object_id}: {e}"
|
||||
)
|
||||
continue
|
||||
|
||||
return {
|
||||
"object": "list",
|
||||
"data": batch_objects,
|
||||
"first_id": batch_objects[0].id if batch_objects else None,
|
||||
"last_id": batch_objects[-1].id if batch_objects else None,
|
||||
"has_more": len(batch_objects) == (limit or 20),
|
||||
}
|
||||
|
||||
async def get_user_created_file_ids(
|
||||
self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: List[str]
|
||||
) -> List[OpenAIFileObject]:
|
||||
|
|
@ -745,7 +673,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
bytes=file_objects[0].bytes,
|
||||
filename=file_objects[0].filename,
|
||||
status="uploaded",
|
||||
expires_at=file_objects[0].expires_at,
|
||||
)
|
||||
|
||||
return response
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
|
|
@ -1,40 +1,12 @@
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
def formatTime(self, record, datefmt=None):
|
||||
dt = datetime.fromtimestamp(record.created)
|
||||
return dt.isoformat()
|
||||
|
||||
def format(self, record):
|
||||
json_record = {
|
||||
"message": record.getMessage(),
|
||||
"level": record.levelname,
|
||||
"timestamp": self.formatTime(record),
|
||||
}
|
||||
if record.exc_info:
|
||||
json_record["stacktrace"] = self.formatException(record.exc_info)
|
||||
return json.dumps(json_record)
|
||||
|
||||
|
||||
def _is_json_enabled():
|
||||
try:
|
||||
import litellm
|
||||
return getattr(litellm, 'json_logs', False)
|
||||
except (ImportError, AttributeError):
|
||||
return os.getenv("JSON_LOGS", "false").lower() == "true"
|
||||
|
||||
|
||||
# Set up package logger
|
||||
logger = logging.getLogger("litellm_proxy_extras")
|
||||
|
||||
if not logger.handlers:
|
||||
if not logger.handlers: # Only add handler if none exists
|
||||
handler = logging.StreamHandler()
|
||||
if _is_json_enabled():
|
||||
handler.setFormatter(JsonFormatter())
|
||||
else:
|
||||
handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
|
||||
formatter = logging.Formatter(
|
||||
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
||||
)
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
-- DropIndex
|
||||
DROP INDEX IF EXISTS "LiteLLM_PromptTable_prompt_id_key";
|
||||
DROP INDEX "LiteLLM_PromptTable_prompt_id_key";
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "LiteLLM_PromptTable"
|
||||
ADD COLUMN "version" INTEGER NOT NULL DEFAULT 1;
|
||||
ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "version" INTEGER NOT NULL DEFAULT 1;
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "LiteLLM_PromptTable_prompt_id_idx" ON "LiteLLM_PromptTable" ("prompt_id");
|
||||
CREATE INDEX "LiteLLM_PromptTable_prompt_id_idx" ON "LiteLLM_PromptTable"("prompt_id");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "LiteLLM_PromptTable_prompt_id_version_key" ON "LiteLLM_PromptTable" ("prompt_id", "version");
|
||||
CREATE UNIQUE INDEX "LiteLLM_PromptTable_prompt_id_version_key" ON "LiteLLM_PromptTable"("prompt_id", "version");
|
||||
|
||||
|
|
|
|||
|
|
@ -760,11 +760,6 @@ model LiteLLM_ManagedVectorStoresTable {
|
|||
updated_at DateTime @updatedAt
|
||||
litellm_credential_name String?
|
||||
litellm_params Json?
|
||||
team_id String?
|
||||
user_id String?
|
||||
|
||||
@@index([team_id])
|
||||
@@index([user_id])
|
||||
}
|
||||
|
||||
// Guardrails table for storing guardrail configurations
|
||||
|
|
|
|||
|
|
@ -18,15 +18,14 @@ def str_to_bool(value: Optional[str]) -> bool:
|
|||
return value.lower() in ("true", "1", "t", "y", "yes")
|
||||
|
||||
|
||||
|
||||
def _get_prisma_env() -> dict:
|
||||
"""Get environment variables for Prisma, handling offline mode if configured."""
|
||||
prisma_env = os.environ.copy()
|
||||
if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")):
|
||||
# These env vars prevent Prisma from attempting downloads
|
||||
prisma_env["NPM_CONFIG_PREFER_OFFLINE"] = "true"
|
||||
prisma_env["NPM_CONFIG_CACHE"] = os.getenv(
|
||||
"NPM_CONFIG_CACHE", "/app/.cache/npm"
|
||||
)
|
||||
prisma_env["NPM_CONFIG_CACHE"] = os.getenv("NPM_CONFIG_CACHE", "/app/.cache/npm")
|
||||
return prisma_env
|
||||
|
||||
|
||||
|
|
@ -35,28 +34,29 @@ def _get_prisma_command() -> str:
|
|||
if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")):
|
||||
# Primary location where Prisma Python package installs the CLI
|
||||
default_cli_path = "/app/.cache/prisma-python/binaries/node_modules/.bin/prisma"
|
||||
|
||||
|
||||
# Check if custom path is provided (for flexibility)
|
||||
custom_cli_path = os.getenv("PRISMA_CLI_PATH")
|
||||
if custom_cli_path and os.path.exists(custom_cli_path):
|
||||
logger.info(f"Using custom Prisma CLI at {custom_cli_path}")
|
||||
return custom_cli_path
|
||||
|
||||
|
||||
# Check the default location
|
||||
if os.path.exists(default_cli_path):
|
||||
logger.info(f"Using cached Prisma CLI at {default_cli_path}")
|
||||
return default_cli_path
|
||||
|
||||
|
||||
# If not found, log warning and fall back
|
||||
logger.warning(
|
||||
f"Prisma CLI not found at {default_cli_path}. "
|
||||
"Falling back to Python wrapper (may attempt downloads)"
|
||||
)
|
||||
|
||||
|
||||
# Fall back to the Python wrapper (will work in online mode)
|
||||
return "prisma"
|
||||
|
||||
|
||||
|
||||
class ProxyExtrasDBManager:
|
||||
@staticmethod
|
||||
def _get_prisma_dir() -> str:
|
||||
|
|
@ -119,7 +119,7 @@ class ProxyExtrasDBManager:
|
|||
stdout=open(migration_file, "w"),
|
||||
check=True,
|
||||
timeout=30,
|
||||
env=prisma_env,
|
||||
env=prisma_env
|
||||
)
|
||||
|
||||
# 3. Mark the migration as applied since it represents current state
|
||||
|
|
@ -134,7 +134,7 @@ class ProxyExtrasDBManager:
|
|||
],
|
||||
check=True,
|
||||
timeout=30,
|
||||
env=prisma_env,
|
||||
env=prisma_env
|
||||
)
|
||||
|
||||
return True
|
||||
|
|
@ -159,20 +159,14 @@ class ProxyExtrasDBManager:
|
|||
@staticmethod
|
||||
def _roll_back_migration(migration_name: str):
|
||||
"""Mark a specific migration as rolled back"""
|
||||
# Set up environment for offline mode if configured
|
||||
# Set up environment for offline mode if configured
|
||||
prisma_env = _get_prisma_env()
|
||||
subprocess.run(
|
||||
[
|
||||
_get_prisma_command(),
|
||||
"migrate",
|
||||
"resolve",
|
||||
"--rolled-back",
|
||||
migration_name,
|
||||
],
|
||||
[_get_prisma_command(), "migrate", "resolve", "--rolled-back", migration_name],
|
||||
timeout=60,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
env=prisma_env,
|
||||
env=prisma_env
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -184,7 +178,7 @@ class ProxyExtrasDBManager:
|
|||
timeout=60,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
env=prisma_env,
|
||||
env=prisma_env
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -234,8 +228,6 @@ class ProxyExtrasDBManager:
|
|||
r"duplicate key value violates",
|
||||
r"relation .* already exists",
|
||||
r"constraint .* already exists",
|
||||
r"does not exist",
|
||||
r"Can't drop database.* because it doesn't exist",
|
||||
]
|
||||
|
||||
for pattern in idempotent_patterns:
|
||||
|
|
@ -256,7 +248,7 @@ class ProxyExtrasDBManager:
|
|||
if not database_url:
|
||||
logger.error("DATABASE_URL not set")
|
||||
return
|
||||
|
||||
|
||||
diff_dir = (
|
||||
Path(migrations_dir)
|
||||
/ "migrations"
|
||||
|
|
@ -291,7 +283,7 @@ class ProxyExtrasDBManager:
|
|||
check=True,
|
||||
timeout=60,
|
||||
stdout=f,
|
||||
env=_get_prisma_env(),
|
||||
env=_get_prisma_env()
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.warning(f"Failed to generate migration diff: {e.stderr}")
|
||||
|
|
@ -321,7 +313,7 @@ class ProxyExtrasDBManager:
|
|||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=_get_prisma_env(),
|
||||
env=_get_prisma_env()
|
||||
)
|
||||
logger.info(f"prisma db execute stdout: {result.stdout}")
|
||||
logger.info("✅ Migration diff applied successfully")
|
||||
|
|
@ -339,18 +331,12 @@ class ProxyExtrasDBManager:
|
|||
try:
|
||||
logger.info(f"Resolving migration: {migration_name}")
|
||||
subprocess.run(
|
||||
[
|
||||
_get_prisma_command(),
|
||||
"migrate",
|
||||
"resolve",
|
||||
"--applied",
|
||||
migration_name,
|
||||
],
|
||||
[_get_prisma_command(), "migrate", "resolve", "--applied", migration_name],
|
||||
timeout=60,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=_get_prisma_env(),
|
||||
env=_get_prisma_env()
|
||||
)
|
||||
logger.debug(f"Resolved migration: {migration_name}")
|
||||
except subprocess.CalledProcessError as e:
|
||||
|
|
@ -389,7 +375,7 @@ class ProxyExtrasDBManager:
|
|||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=_get_prisma_env(),
|
||||
env=_get_prisma_env()
|
||||
)
|
||||
logger.info(f"prisma migrate deploy stdout: {result.stdout}")
|
||||
|
||||
|
|
@ -411,42 +397,27 @@ class ProxyExtrasDBManager:
|
|||
)
|
||||
if migration_match:
|
||||
failed_migration = migration_match.group(1)
|
||||
if ProxyExtrasDBManager._is_idempotent_error(e.stderr):
|
||||
logger.info(
|
||||
f"Migration {failed_migration} failed due to idempotent error (e.g., column already exists), resolving as applied"
|
||||
)
|
||||
ProxyExtrasDBManager._roll_back_migration(
|
||||
failed_migration
|
||||
)
|
||||
ProxyExtrasDBManager._resolve_specific_migration(
|
||||
failed_migration
|
||||
)
|
||||
logger.info(
|
||||
f"✅ Migration {failed_migration} resolved."
|
||||
)
|
||||
return True
|
||||
else:
|
||||
logger.info(
|
||||
f"Found failed migration: {failed_migration}, marking as rolled back"
|
||||
)
|
||||
# Mark the failed migration as rolled back
|
||||
subprocess.run(
|
||||
[
|
||||
_get_prisma_command(),
|
||||
"migrate",
|
||||
"resolve",
|
||||
"--rolled-back",
|
||||
failed_migration,
|
||||
],
|
||||
timeout=60,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=_get_prisma_env(),
|
||||
)
|
||||
logger.info(
|
||||
f"✅ Migration {failed_migration} marked as rolled back... retrying"
|
||||
)
|
||||
logger.info(
|
||||
f"Found failed migration: {failed_migration}, marking as rolled back"
|
||||
)
|
||||
# Mark the failed migration as rolled back
|
||||
subprocess.run(
|
||||
[
|
||||
_get_prisma_command(),
|
||||
"migrate",
|
||||
"resolve",
|
||||
"--rolled-back",
|
||||
failed_migration,
|
||||
],
|
||||
timeout=60,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=_get_prisma_env()
|
||||
)
|
||||
logger.info(
|
||||
f"✅ Migration {failed_migration} marked as rolled back... retrying"
|
||||
)
|
||||
elif (
|
||||
"P3005" in e.stderr
|
||||
and "database schema is not empty" in e.stderr
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.27"
|
||||
version = "0.4.26"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
authors = ["BerriAI"]
|
||||
readme = "README.md"
|
||||
|
|
@ -22,7 +22,7 @@ requires = ["poetry-core"]
|
|||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.27"
|
||||
version = "0.4.26"
|
||||
version_files = [
|
||||
"pyproject.toml:version",
|
||||
"../requirements.txt:litellm-proxy-extras==",
|
||||
|
|
|
|||
|
|
@ -80,8 +80,6 @@ import dotenv
|
|||
litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV"
|
||||
if litellm_mode == "DEV":
|
||||
dotenv.load_dotenv()
|
||||
|
||||
|
||||
####################################################
|
||||
if set_verbose:
|
||||
_turn_on_debug()
|
||||
|
|
|
|||
|
|
@ -166,66 +166,6 @@ def _initialize_loggers_with_handler(handler: logging.Handler):
|
|||
lg.propagate = False # prevent bubbling to parent/root
|
||||
|
||||
|
||||
def _get_uvicorn_json_log_config():
|
||||
"""
|
||||
Generate a uvicorn log_config dictionary that applies JSON formatting to all loggers.
|
||||
|
||||
This ensures that uvicorn's access logs, error logs, and all application logs
|
||||
are formatted as JSON when json_logs is enabled.
|
||||
"""
|
||||
json_formatter_class = "litellm._logging.JsonFormatter"
|
||||
|
||||
# Use the module-level log_level variable for consistency
|
||||
uvicorn_log_level = log_level.upper()
|
||||
|
||||
log_config = {
|
||||
"version": 1,
|
||||
"disable_existing_loggers": False,
|
||||
"formatters": {
|
||||
"json": {
|
||||
"()": json_formatter_class,
|
||||
},
|
||||
"default": {
|
||||
"()": json_formatter_class,
|
||||
},
|
||||
"access": {
|
||||
"()": json_formatter_class,
|
||||
},
|
||||
},
|
||||
"handlers": {
|
||||
"default": {
|
||||
"formatter": "json",
|
||||
"class": "logging.StreamHandler",
|
||||
"stream": "ext://sys.stdout",
|
||||
},
|
||||
"access": {
|
||||
"formatter": "access",
|
||||
"class": "logging.StreamHandler",
|
||||
"stream": "ext://sys.stdout",
|
||||
},
|
||||
},
|
||||
"loggers": {
|
||||
"uvicorn": {
|
||||
"handlers": ["default"],
|
||||
"level": uvicorn_log_level,
|
||||
"propagate": False,
|
||||
},
|
||||
"uvicorn.error": {
|
||||
"handlers": ["default"],
|
||||
"level": uvicorn_log_level,
|
||||
"propagate": False,
|
||||
},
|
||||
"uvicorn.access": {
|
||||
"handlers": ["access"],
|
||||
"level": uvicorn_log_level,
|
||||
"propagate": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return log_config
|
||||
|
||||
|
||||
def _turn_on_json():
|
||||
"""
|
||||
Turn on JSON logging
|
||||
|
|
|
|||
|
|
@ -1,97 +0,0 @@
|
|||
"""
|
||||
Custom A2A Card Resolver for LiteLLM.
|
||||
|
||||
Extends the A2A SDK's card resolver to support multiple well-known paths.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from a2a.types import AgentCard
|
||||
|
||||
# Runtime imports with availability check
|
||||
_A2ACardResolver: Any = None
|
||||
AGENT_CARD_WELL_KNOWN_PATH: str = "/.well-known/agent-card.json"
|
||||
PREV_AGENT_CARD_WELL_KNOWN_PATH: str = "/.well-known/agent.json"
|
||||
|
||||
try:
|
||||
from a2a.client import A2ACardResolver as _A2ACardResolver # type: ignore[no-redef]
|
||||
from a2a.utils.constants import ( # type: ignore[no-redef]
|
||||
AGENT_CARD_WELL_KNOWN_PATH,
|
||||
PREV_AGENT_CARD_WELL_KNOWN_PATH,
|
||||
)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
class LiteLLMA2ACardResolver(_A2ACardResolver): # type: ignore[misc]
|
||||
"""
|
||||
Custom A2A card resolver that supports multiple well-known paths.
|
||||
|
||||
Extends the base A2ACardResolver to try both:
|
||||
- /.well-known/agent-card.json (standard)
|
||||
- /.well-known/agent.json (previous/alternative)
|
||||
"""
|
||||
|
||||
async def get_agent_card(
|
||||
self,
|
||||
relative_card_path: Optional[str] = None,
|
||||
http_kwargs: Optional[Dict[str, Any]] = None,
|
||||
) -> "AgentCard":
|
||||
"""
|
||||
Fetch the agent card, trying multiple well-known paths.
|
||||
|
||||
First tries the standard path, then falls back to the previous path.
|
||||
|
||||
Args:
|
||||
relative_card_path: Optional path to the agent card endpoint.
|
||||
If None, tries both well-known paths.
|
||||
http_kwargs: Optional dictionary of keyword arguments to pass to httpx.get
|
||||
|
||||
Returns:
|
||||
AgentCard from the A2A agent
|
||||
|
||||
Raises:
|
||||
A2AClientHTTPError or A2AClientJSONError if both paths fail
|
||||
"""
|
||||
# If a specific path is provided, use the parent implementation
|
||||
if relative_card_path is not None:
|
||||
return await super().get_agent_card(
|
||||
relative_card_path=relative_card_path,
|
||||
http_kwargs=http_kwargs,
|
||||
)
|
||||
|
||||
# Try both well-known paths
|
||||
paths = [
|
||||
AGENT_CARD_WELL_KNOWN_PATH,
|
||||
PREV_AGENT_CARD_WELL_KNOWN_PATH,
|
||||
]
|
||||
|
||||
last_error = None
|
||||
for path in paths:
|
||||
try:
|
||||
verbose_logger.debug(
|
||||
f"Attempting to fetch agent card from {self.base_url}{path}"
|
||||
)
|
||||
return await super().get_agent_card(
|
||||
relative_card_path=path,
|
||||
http_kwargs=http_kwargs,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.debug(
|
||||
f"Failed to fetch agent card from {self.base_url}{path}: {e}"
|
||||
)
|
||||
last_error = e
|
||||
continue
|
||||
|
||||
# If we get here, all paths failed - re-raise the last error
|
||||
if last_error is not None:
|
||||
raise last_error
|
||||
|
||||
# This shouldn't happen, but just in case
|
||||
raise Exception(
|
||||
f"Failed to fetch agent card from {self.base_url}. "
|
||||
f"Tried paths: {', '.join(paths)}"
|
||||
)
|
||||
|
|
@ -6,7 +6,6 @@ Provides standalone functions with @client decorator for LiteLLM logging integra
|
|||
|
||||
import asyncio
|
||||
import datetime
|
||||
import uuid
|
||||
from typing import TYPE_CHECKING, Any, AsyncIterator, Coroutine, Dict, Optional, Union
|
||||
|
||||
import litellm
|
||||
|
|
@ -21,6 +20,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
)
|
||||
from litellm.types.agents import LiteLLMSendMessageResponse
|
||||
from litellm.utils import client
|
||||
import uuid
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from a2a.client import A2AClient as A2AClientType
|
||||
|
|
@ -36,18 +36,13 @@ A2ACardResolver: Any = None
|
|||
_A2AClient: Any = None
|
||||
|
||||
try:
|
||||
from a2a.client import A2ACardResolver # type: ignore[no-redef]
|
||||
from a2a.client import A2AClient as _A2AClient # type: ignore[no-redef]
|
||||
|
||||
A2A_SDK_AVAILABLE = True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Import our custom card resolver that supports multiple well-known paths
|
||||
from litellm.a2a_protocol.card_resolver import LiteLLMA2ACardResolver
|
||||
|
||||
# Use our custom resolver instead of the default A2A SDK resolver
|
||||
A2ACardResolver = LiteLLMA2ACardResolver
|
||||
|
||||
|
||||
def _set_usage_on_logging_obj(
|
||||
kwargs: Dict[str, Any],
|
||||
|
|
|
|||
|
|
@ -1165,12 +1165,6 @@ LITELLM_CLI_SOURCE_IDENTIFIER = "litellm-cli"
|
|||
LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token"
|
||||
CLI_SSO_SESSION_CACHE_KEY_PREFIX = "cli_sso_session"
|
||||
CLI_JWT_TOKEN_NAME = "cli-jwt-token"
|
||||
# Support both CLI_JWT_EXPIRATION_HOURS and LITELLM_CLI_JWT_EXPIRATION_HOURS for backwards compatibility
|
||||
CLI_JWT_EXPIRATION_HOURS = int(
|
||||
os.getenv("CLI_JWT_EXPIRATION_HOURS")
|
||||
or os.getenv("LITELLM_CLI_JWT_EXPIRATION_HOURS")
|
||||
or 24
|
||||
)
|
||||
|
||||
########################### DB CRON JOB NAMES ###########################
|
||||
DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job"
|
||||
|
|
@ -1332,13 +1326,6 @@ COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY = int(
|
|||
DEFAULT_CHUNK_SIZE = int(os.getenv("DEFAULT_CHUNK_SIZE", 1000))
|
||||
DEFAULT_CHUNK_OVERLAP = int(os.getenv("DEFAULT_CHUNK_OVERLAP", 200))
|
||||
|
||||
########################### S3 Vectors RAG Constants ###########################
|
||||
S3_VECTORS_DEFAULT_DIMENSION = int(os.getenv("S3_VECTORS_DEFAULT_DIMENSION", 1024))
|
||||
S3_VECTORS_DEFAULT_DISTANCE_METRIC = str(
|
||||
os.getenv("S3_VECTORS_DEFAULT_DISTANCE_METRIC", "cosine")
|
||||
)
|
||||
S3_VECTORS_DEFAULT_NON_FILTERABLE_METADATA_KEYS = ["source_text"]
|
||||
|
||||
########################### Microsoft SSO Constants ###########################
|
||||
MICROSOFT_USER_EMAIL_ATTRIBUTE = str(
|
||||
os.getenv("MICROSOFT_USER_EMAIL_ATTRIBUTE", "userPrincipalName")
|
||||
|
|
|
|||
|
|
@ -23,11 +23,7 @@ from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import
|
|||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
CostCalculatorUtils,
|
||||
_generic_cost_per_character,
|
||||
_get_service_tier_cost_key,
|
||||
_parse_prompt_tokens_details,
|
||||
calculate_cost_component,
|
||||
generic_cost_per_token,
|
||||
get_billable_input_tokens,
|
||||
select_cost_metric_for_model,
|
||||
)
|
||||
from litellm.llms.anthropic.cost_calculation import (
|
||||
|
|
@ -435,18 +431,12 @@ def cost_per_token( # noqa: PLR0915
|
|||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
if (
|
||||
model_info.get("input_cost_per_token", 0) > 0
|
||||
or model_info.get("output_cost_per_token", 0) > 0
|
||||
):
|
||||
return generic_cost_per_token(
|
||||
model=model,
|
||||
usage=usage_block,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
service_tier=service_tier,
|
||||
if model_info["input_cost_per_token"] > 0:
|
||||
## COST PER TOKEN ##
|
||||
prompt_tokens_cost_usd_dollar = (
|
||||
model_info["input_cost_per_token"] * prompt_tokens
|
||||
)
|
||||
|
||||
if (
|
||||
elif (
|
||||
model_info.get("input_cost_per_second", None) is not None
|
||||
and response_time_ms is not None
|
||||
):
|
||||
|
|
@ -461,7 +451,11 @@ def cost_per_token( # noqa: PLR0915
|
|||
model_info["input_cost_per_second"] * response_time_ms / 1000 # type: ignore
|
||||
)
|
||||
|
||||
if (
|
||||
if model_info["output_cost_per_token"] > 0:
|
||||
completion_tokens_cost_usd_dollar = (
|
||||
model_info["output_cost_per_token"] * completion_tokens
|
||||
)
|
||||
elif (
|
||||
model_info.get("output_cost_per_second", None) is not None
|
||||
and response_time_ms is not None
|
||||
):
|
||||
|
|
@ -961,10 +955,7 @@ def completion_cost( # noqa: PLR0915
|
|||
router_model_id=router_model_id,
|
||||
)
|
||||
|
||||
potential_model_names = [
|
||||
selected_model,
|
||||
_get_response_model(completion_response),
|
||||
]
|
||||
potential_model_names = [selected_model, _get_response_model(completion_response)]
|
||||
if model is not None:
|
||||
potential_model_names.append(model)
|
||||
|
||||
|
|
@ -1719,16 +1710,10 @@ def default_image_cost_calculator(
|
|||
)
|
||||
|
||||
# Priority 1: Use per-image pricing if available (for gpt-image-1 and similar models)
|
||||
if (
|
||||
"input_cost_per_image" in cost_info
|
||||
and cost_info["input_cost_per_image"] is not None
|
||||
):
|
||||
if "input_cost_per_image" in cost_info and cost_info["input_cost_per_image"] is not None:
|
||||
return cost_info["input_cost_per_image"] * n
|
||||
# Priority 2: Fall back to per-pixel pricing for backward compatibility
|
||||
elif (
|
||||
"input_cost_per_pixel" in cost_info
|
||||
and cost_info["input_cost_per_pixel"] is not None
|
||||
):
|
||||
elif "input_cost_per_pixel" in cost_info and cost_info["input_cost_per_pixel"] is not None:
|
||||
return cost_info["input_cost_per_pixel"] * height * width * n
|
||||
else:
|
||||
raise Exception(
|
||||
|
|
@ -1848,22 +1833,9 @@ def batch_cost_calculator(
|
|||
if input_cost_per_token_batches:
|
||||
total_prompt_cost = usage.prompt_tokens * input_cost_per_token_batches
|
||||
elif input_cost_per_token:
|
||||
# Subtract cached tokens from prompt_tokens before calculating cost
|
||||
# Fixes issue where cached tokens are being charged again
|
||||
total_prompt_cost = (
|
||||
get_billable_input_tokens(usage) * (input_cost_per_token) / 2
|
||||
usage.prompt_tokens * (input_cost_per_token) / 2
|
||||
) # batch cost is usually half of the regular token cost
|
||||
|
||||
# Add cache read cost if applicable
|
||||
details = _parse_prompt_tokens_details(usage)
|
||||
cache_read_tokens = details["cache_hit_tokens"]
|
||||
cache_read_cost_key = _get_service_tier_cost_key(
|
||||
"cache_read_input_token_cost", None
|
||||
)
|
||||
total_prompt_cost += (
|
||||
calculate_cost_component(model_info, cache_read_cost_key, cache_read_tokens)
|
||||
/ 2
|
||||
)
|
||||
if output_cost_per_token_batches:
|
||||
total_completion_cost = usage.completion_tokens * output_cost_per_token_batches
|
||||
elif output_cost_per_token:
|
||||
|
|
|
|||
|
|
@ -4,26 +4,22 @@ LiteLLM Proxy uses this MCP Client to connnect to other MCP servers.
|
|||
|
||||
import asyncio
|
||||
import base64
|
||||
from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple, TypeVar, Union
|
||||
from typing import Awaitable, Callable, Dict, List, Optional, TypeVar, Union
|
||||
|
||||
import httpx
|
||||
from mcp import ClientSession, ReadResourceResult, Resource, StdioServerParameters
|
||||
from mcp.client.sse import sse_client
|
||||
from mcp.client.stdio import stdio_client
|
||||
|
||||
try:
|
||||
from mcp.client.streamable_http import streamable_http_client # type: ignore
|
||||
except ImportError:
|
||||
streamable_http_client = None
|
||||
from mcp.types import CallToolRequestParams as MCPCallToolRequestParams
|
||||
from mcp.types import CallToolResult as MCPCallToolResult
|
||||
from mcp.client.streamable_http import streamable_http_client
|
||||
from mcp.types import (
|
||||
CallToolRequestParams as MCPCallToolRequestParams,
|
||||
GetPromptRequestParams,
|
||||
GetPromptResult,
|
||||
Prompt,
|
||||
ResourceTemplate,
|
||||
TextContent,
|
||||
)
|
||||
from mcp.types import CallToolResult as MCPCallToolResult
|
||||
from mcp.types import TextContent
|
||||
from mcp.types import Tool as MCPTool
|
||||
from pydantic import AnyUrl
|
||||
|
||||
|
|
@ -78,91 +74,57 @@ class MCPClient:
|
|||
if auth_value:
|
||||
self.update_auth_value(auth_value)
|
||||
|
||||
def _create_transport_context(
|
||||
self,
|
||||
) -> Tuple[Any, Optional[httpx.AsyncClient]]:
|
||||
"""
|
||||
Create the appropriate transport context based on transport type.
|
||||
|
||||
Returns:
|
||||
Tuple of (transport_context, http_client).
|
||||
http_client is only set for HTTP transport and needs cleanup.
|
||||
"""
|
||||
http_client: Optional[httpx.AsyncClient] = None
|
||||
|
||||
if self.transport_type == MCPTransport.stdio:
|
||||
if not self.stdio_config:
|
||||
raise ValueError("stdio_config is required for stdio transport")
|
||||
server_params = StdioServerParameters(
|
||||
command=self.stdio_config.get("command", ""),
|
||||
args=self.stdio_config.get("args", []),
|
||||
env=self.stdio_config.get("env", {}),
|
||||
)
|
||||
return stdio_client(server_params), None
|
||||
|
||||
if self.transport_type == MCPTransport.sse:
|
||||
headers = self._get_auth_headers()
|
||||
httpx_client_factory = self._create_httpx_client_factory()
|
||||
return sse_client(
|
||||
url=self.server_url,
|
||||
timeout=self.timeout,
|
||||
headers=headers,
|
||||
httpx_client_factory=httpx_client_factory,
|
||||
), None
|
||||
|
||||
# HTTP transport (default)
|
||||
headers = self._get_auth_headers()
|
||||
httpx_client_factory = self._create_httpx_client_factory()
|
||||
verbose_logger.debug(
|
||||
"litellm headers for streamable_http_client: %s", headers
|
||||
)
|
||||
http_client = httpx_client_factory(
|
||||
headers=headers,
|
||||
timeout=httpx.Timeout(self.timeout),
|
||||
)
|
||||
transport_ctx = streamable_http_client(
|
||||
url=self.server_url,
|
||||
http_client=http_client,
|
||||
)
|
||||
return transport_ctx, http_client
|
||||
|
||||
async def _execute_session_operation(
|
||||
self,
|
||||
transport_ctx: Any,
|
||||
operation: Callable[[ClientSession], Awaitable[TSessionResult]],
|
||||
) -> TSessionResult:
|
||||
"""
|
||||
Execute an operation within a transport and session context.
|
||||
|
||||
Handles entering/exiting contexts and running the operation.
|
||||
"""
|
||||
transport = await transport_ctx.__aenter__()
|
||||
try:
|
||||
read_stream, write_stream = transport[0], transport[1]
|
||||
session_ctx = ClientSession(read_stream, write_stream)
|
||||
session = await session_ctx.__aenter__()
|
||||
try:
|
||||
await session.initialize()
|
||||
return await operation(session)
|
||||
finally:
|
||||
try:
|
||||
await session_ctx.__aexit__(None, None, None)
|
||||
except BaseException as e:
|
||||
verbose_logger.debug(f"Error during session context exit: {e}")
|
||||
finally:
|
||||
try:
|
||||
await transport_ctx.__aexit__(None, None, None)
|
||||
except BaseException as e:
|
||||
verbose_logger.debug(f"Error during transport context exit: {e}")
|
||||
|
||||
async def run_with_session(
|
||||
self, operation: Callable[[ClientSession], Awaitable[TSessionResult]]
|
||||
) -> TSessionResult:
|
||||
"""Open a session, run the provided coroutine, and clean up."""
|
||||
transport_ctx = None
|
||||
http_client: Optional[httpx.AsyncClient] = None
|
||||
|
||||
try:
|
||||
transport_ctx, http_client = self._create_transport_context()
|
||||
return await self._execute_session_operation(transport_ctx, operation)
|
||||
if self.transport_type == MCPTransport.stdio:
|
||||
if not self.stdio_config:
|
||||
raise ValueError("stdio_config is required for stdio transport")
|
||||
|
||||
server_params = StdioServerParameters(
|
||||
command=self.stdio_config.get("command", ""),
|
||||
args=self.stdio_config.get("args", []),
|
||||
env=self.stdio_config.get("env", {}),
|
||||
)
|
||||
transport_ctx = stdio_client(server_params)
|
||||
elif self.transport_type == MCPTransport.sse:
|
||||
headers = self._get_auth_headers()
|
||||
httpx_client_factory = self._create_httpx_client_factory()
|
||||
transport_ctx = sse_client(
|
||||
url=self.server_url,
|
||||
timeout=self.timeout,
|
||||
headers=headers,
|
||||
httpx_client_factory=httpx_client_factory,
|
||||
)
|
||||
else:
|
||||
headers = self._get_auth_headers()
|
||||
httpx_client_factory = self._create_httpx_client_factory()
|
||||
verbose_logger.debug(
|
||||
"litellm headers for streamable_http_client: %s", headers
|
||||
)
|
||||
http_client = httpx_client_factory(
|
||||
headers=headers,
|
||||
timeout=httpx.Timeout(self.timeout),
|
||||
)
|
||||
transport_ctx = streamable_http_client(
|
||||
url=self.server_url,
|
||||
http_client=http_client,
|
||||
)
|
||||
|
||||
if transport_ctx is None:
|
||||
raise RuntimeError("Failed to create transport context")
|
||||
|
||||
async with transport_ctx as transport:
|
||||
read_stream, write_stream = transport[0], transport[1]
|
||||
session_ctx = ClientSession(read_stream, write_stream)
|
||||
async with session_ctx as session:
|
||||
await session.initialize()
|
||||
return await operation(session)
|
||||
except Exception:
|
||||
verbose_logger.warning(
|
||||
"MCP client run_with_session failed for %s", self.server_url or "stdio"
|
||||
|
|
@ -170,10 +132,7 @@ class MCPClient:
|
|||
raise
|
||||
finally:
|
||||
if http_client is not None:
|
||||
try:
|
||||
await http_client.aclose()
|
||||
except BaseException as e:
|
||||
verbose_logger.debug(f"Error during http_client cleanup: {e}")
|
||||
await http_client.aclose()
|
||||
|
||||
def update_auth_value(self, mcp_auth_value: Union[str, Dict[str, str]]):
|
||||
"""
|
||||
|
|
@ -286,9 +245,7 @@ class MCPClient:
|
|||
return []
|
||||
|
||||
async def call_tool(
|
||||
self,
|
||||
call_tool_request_params: MCPCallToolRequestParams,
|
||||
host_progress_callback: Optional[Callable] = None
|
||||
self, call_tool_request_params: MCPCallToolRequestParams
|
||||
) -> MCPCallToolResult:
|
||||
"""
|
||||
Call an MCP Tool.
|
||||
|
|
@ -297,28 +254,13 @@ class MCPClient:
|
|||
f"MCP client calling tool '{call_tool_request_params.name}' with arguments: {call_tool_request_params.arguments}"
|
||||
)
|
||||
|
||||
async def on_progress(progress: float, total: float | None, message: str | None):
|
||||
percentage = (progress / total * 100) if total else 0
|
||||
verbose_logger.info(
|
||||
f"MCP Tool '{call_tool_request_params.name}' progress: "
|
||||
f"{progress}/{total} ({percentage:.0f}%) - {message or ''}"
|
||||
)
|
||||
|
||||
# Forward to Host if callback provided
|
||||
if host_progress_callback:
|
||||
try:
|
||||
await host_progress_callback(progress, total)
|
||||
except Exception as e:
|
||||
verbose_logger.warning(f"Failed to forward to Host: {e}")
|
||||
|
||||
async def _call_tool_operation(session: ClientSession):
|
||||
verbose_logger.debug("MCP client sending tool call to session")
|
||||
return await session.call_tool(
|
||||
name=call_tool_request_params.name,
|
||||
arguments=call_tool_request_params.arguments,
|
||||
progress_callback=on_progress,
|
||||
|
||||
)
|
||||
|
||||
try:
|
||||
tool_result = await self.run_with_session(_call_tool_operation)
|
||||
verbose_logger.info(
|
||||
|
|
|
|||
|
|
@ -83,33 +83,6 @@
|
|||
},
|
||||
"description": "Datadog Logging Integration"
|
||||
},
|
||||
{
|
||||
"id": "datadog_cost_management",
|
||||
"displayName": "Datadog Cost Management",
|
||||
"logo": "datadog.png",
|
||||
"supports_key_team_logging": false,
|
||||
"dynamic_params": {
|
||||
"dd_api_key": {
|
||||
"type": "password",
|
||||
"ui_name": "API Key",
|
||||
"description": "Datadog API key for authentication",
|
||||
"required": true
|
||||
},
|
||||
"dd_app_key": {
|
||||
"type": "password",
|
||||
"ui_name": "App Key",
|
||||
"description": "Datadog Application Key for Cloud Cost Management",
|
||||
"required": true
|
||||
},
|
||||
"dd_site": {
|
||||
"type": "text",
|
||||
"ui_name": "Site",
|
||||
"description": "Datadog site URL (e.g., us5.datadoghq.com)",
|
||||
"required": true
|
||||
}
|
||||
},
|
||||
"description": "Datadog Cloud Cost Management Integration"
|
||||
},
|
||||
{
|
||||
"id": "lago",
|
||||
"displayName": "Lago",
|
||||
|
|
@ -434,4 +407,4 @@
|
|||
},
|
||||
"description": "SQS Queue (AWS) Logging Integration"
|
||||
}
|
||||
]
|
||||
]
|
||||
|
|
|
|||
|
|
@ -516,9 +516,7 @@ class CustomGuardrail(CustomLogger):
|
|||
from litellm.types.utils import GuardrailMode
|
||||
|
||||
# Use event_type if provided, otherwise fall back to self.event_hook
|
||||
guardrail_mode: Union[
|
||||
GuardrailEventHooks, GuardrailMode, List[GuardrailEventHooks]
|
||||
]
|
||||
guardrail_mode: Union[GuardrailEventHooks, GuardrailMode, List[GuardrailEventHooks]]
|
||||
if event_type is not None:
|
||||
guardrail_mode = event_type
|
||||
elif isinstance(self.event_hook, Mode):
|
||||
|
|
@ -526,21 +524,11 @@ class CustomGuardrail(CustomLogger):
|
|||
else:
|
||||
guardrail_mode = self.event_hook # type: ignore[assignment]
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
filter_exceptions_from_params,
|
||||
)
|
||||
|
||||
# Sanitize the response to ensure it's JSON serializable and free of circular refs
|
||||
# This prevents RecursionErrors in downstream loggers (Langfuse, Datadog, etc.)
|
||||
clean_guardrail_response = filter_exceptions_from_params(
|
||||
guardrail_json_response
|
||||
)
|
||||
|
||||
slg = StandardLoggingGuardrailInformation(
|
||||
guardrail_name=self.guardrail_name,
|
||||
guardrail_provider=guardrail_provider,
|
||||
guardrail_mode=guardrail_mode,
|
||||
guardrail_response=clean_guardrail_response,
|
||||
guardrail_response=guardrail_json_response,
|
||||
guardrail_status=guardrail_status,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
|
|
|
|||
|
|
@ -32,7 +32,6 @@ from litellm.integrations.datadog.datadog_handler import (
|
|||
get_datadog_service,
|
||||
get_datadog_source,
|
||||
get_datadog_tags,
|
||||
get_datadog_base_url_from_env,
|
||||
)
|
||||
from litellm.litellm_core_utils.dd_tracing import tracer
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
|
|
@ -101,9 +100,7 @@ class DataDogLogger(
|
|||
self._configure_dd_direct_api()
|
||||
|
||||
# Optional override for testing
|
||||
dd_base_url = get_datadog_base_url_from_env()
|
||||
if dd_base_url:
|
||||
self.intake_url = f"{dd_base_url}/api/v2/logs"
|
||||
self._apply_dd_base_url_override()
|
||||
self.sync_client = _get_httpx_client()
|
||||
asyncio.create_task(self.periodic_flush())
|
||||
self.flush_lock = asyncio.Lock()
|
||||
|
|
@ -162,6 +159,18 @@ class DataDogLogger(
|
|||
self.DD_API_KEY = os.getenv("DD_API_KEY")
|
||||
self.intake_url = f"https://http-intake.logs.{os.getenv('DD_SITE')}/api/v2/logs"
|
||||
|
||||
def _apply_dd_base_url_override(self) -> None:
|
||||
"""
|
||||
Apply base URL override for testing purposes
|
||||
"""
|
||||
dd_base_url: Optional[str] = (
|
||||
os.getenv("_DATADOG_BASE_URL")
|
||||
or os.getenv("DATADOG_BASE_URL")
|
||||
or os.getenv("DD_BASE_URL")
|
||||
)
|
||||
if dd_base_url is not None:
|
||||
self.intake_url = f"{dd_base_url}/api/v2/logs"
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
"""
|
||||
Async Log success events to Datadog
|
||||
|
|
|
|||
|
|
@ -1,204 +0,0 @@
|
|||
import asyncio
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.custom_batch_logger import CustomBatchLogger
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
)
|
||||
from litellm.types.integrations.datadog_cost_management import (
|
||||
DatadogFOCUSCostEntry,
|
||||
)
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
|
||||
|
||||
class DatadogCostManagementLogger(CustomBatchLogger):
|
||||
def __init__(self, **kwargs):
|
||||
self.dd_api_key = os.getenv("DD_API_KEY")
|
||||
self.dd_app_key = os.getenv("DD_APP_KEY")
|
||||
self.dd_site = os.getenv("DD_SITE", "datadoghq.com")
|
||||
|
||||
if not self.dd_api_key or not self.dd_app_key:
|
||||
verbose_logger.warning(
|
||||
"Datadog Cost Management: DD_API_KEY and DD_APP_KEY are required. Integration will not work."
|
||||
)
|
||||
|
||||
self.upload_url = f"https://api.{self.dd_site}/api/v2/cost/custom_costs"
|
||||
|
||||
self.async_client = get_async_httpx_client(
|
||||
llm_provider=httpxSpecialProvider.LoggingCallback
|
||||
)
|
||||
|
||||
# Initialize lock and start periodic flush task
|
||||
self.flush_lock = asyncio.Lock()
|
||||
asyncio.create_task(self.periodic_flush())
|
||||
|
||||
# Check if flush_lock is already in kwargs to avoid double passing (unlikely but safe)
|
||||
if "flush_lock" not in kwargs:
|
||||
kwargs["flush_lock"] = self.flush_lock
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
try:
|
||||
standard_logging_object: Optional[StandardLoggingPayload] = kwargs.get(
|
||||
"standard_logging_object", None
|
||||
)
|
||||
|
||||
if standard_logging_object is None:
|
||||
return
|
||||
|
||||
# Only log if there is a cost associated
|
||||
if standard_logging_object.get("response_cost", 0) > 0:
|
||||
self.log_queue.append(standard_logging_object)
|
||||
|
||||
if len(self.log_queue) >= self.batch_size:
|
||||
await self.async_send_batch()
|
||||
|
||||
except Exception as e:
|
||||
verbose_logger.exception(
|
||||
f"Datadog Cost Management: Error in async_log_success_event: {str(e)}"
|
||||
)
|
||||
|
||||
async def async_send_batch(self):
|
||||
if not self.log_queue:
|
||||
return
|
||||
|
||||
try:
|
||||
# Aggregate costs from the batch
|
||||
aggregated_entries = self._aggregate_costs(self.log_queue)
|
||||
|
||||
if not aggregated_entries:
|
||||
return
|
||||
|
||||
# Send to Datadog
|
||||
await self._upload_to_datadog(aggregated_entries)
|
||||
|
||||
# Clear queue only on success (or if we decide to drop on failure)
|
||||
# CustomBatchLogger clears queue in flush_queue, so we just process here
|
||||
|
||||
except Exception as e:
|
||||
verbose_logger.exception(
|
||||
f"Datadog Cost Management: Error in async_send_batch: {str(e)}"
|
||||
)
|
||||
|
||||
def _aggregate_costs(
|
||||
self, logs: List[StandardLoggingPayload]
|
||||
) -> List[DatadogFOCUSCostEntry]:
|
||||
"""
|
||||
Aggregates costs by Provider, Model, and Date.
|
||||
Returns a list of DatadogFOCUSCostEntry.
|
||||
"""
|
||||
aggregator: Dict[Tuple[str, str, str, Tuple[Tuple[str, str], ...]], DatadogFOCUSCostEntry] = {}
|
||||
|
||||
for log in logs:
|
||||
try:
|
||||
# Extract keys for aggregation
|
||||
provider = log.get("custom_llm_provider") or "unknown"
|
||||
model = log.get("model") or "unknown"
|
||||
cost = log.get("response_cost", 0)
|
||||
|
||||
if cost == 0:
|
||||
continue
|
||||
|
||||
# Get date strings (FOCUS format requires specific keys, but for aggregation we group by Day)
|
||||
# UTC date
|
||||
# We interpret "ChargePeriod" as the day of the request.
|
||||
ts = log.get("startTime") or time.time()
|
||||
dt = datetime.fromtimestamp(ts)
|
||||
date_str = dt.strftime("%Y-%m-%d")
|
||||
|
||||
# ChargePeriodStart and End
|
||||
# If we want daily granularity, end date is usually same day or next day?
|
||||
# Datadog Custom Costs usually expects periods.
|
||||
# "ChargePeriodStart": "2023-01-01", "ChargePeriodEnd": "2023-12-31" in example.
|
||||
# If we send daily, we can say Start=Date, End=Date.
|
||||
|
||||
# Grouping Key: Provider + Model + Date + Tags?
|
||||
# For simplicity, let's aggregate by Provider + Model + Date first.
|
||||
# If we handle tags, we need to include them in the key.
|
||||
|
||||
tags = self._extract_tags(log)
|
||||
tags_key = tuple(sorted(tags.items())) if tags else ()
|
||||
|
||||
key = (provider, model, date_str, tags_key)
|
||||
|
||||
if key not in aggregator:
|
||||
aggregator[key] = {
|
||||
"ProviderName": provider,
|
||||
"ChargeDescription": f"LLM Usage for {model}",
|
||||
"ChargePeriodStart": date_str,
|
||||
"ChargePeriodEnd": date_str,
|
||||
"BilledCost": 0.0,
|
||||
"BillingCurrency": "USD",
|
||||
"Tags": tags if tags else None,
|
||||
}
|
||||
|
||||
aggregator[key]["BilledCost"] += cost
|
||||
|
||||
except Exception as e:
|
||||
verbose_logger.warning(
|
||||
f"Error processing log for cost aggregation: {e}"
|
||||
)
|
||||
continue
|
||||
|
||||
return list(aggregator.values())
|
||||
|
||||
def _extract_tags(self, log: StandardLoggingPayload) -> Dict[str, str]:
|
||||
from litellm.integrations.datadog.datadog_handler import (
|
||||
get_datadog_env,
|
||||
get_datadog_hostname,
|
||||
get_datadog_pod_name,
|
||||
get_datadog_service,
|
||||
)
|
||||
|
||||
tags = {
|
||||
"env": get_datadog_env(),
|
||||
"service": get_datadog_service(),
|
||||
"host": get_datadog_hostname(),
|
||||
"pod_name": get_datadog_pod_name(),
|
||||
}
|
||||
|
||||
# Add metadata as tags
|
||||
metadata = log.get("metadata", {})
|
||||
if metadata:
|
||||
# Add user info
|
||||
if "user_api_key_alias" in metadata:
|
||||
tags["user"] = str(metadata["user_api_key_alias"])
|
||||
if "user_api_key_team_alias" in metadata:
|
||||
tags["team"] = str(metadata["user_api_key_team_alias"])
|
||||
# model_group is not in StandardLoggingMetadata TypedDict, so we need to access it via dict.get()
|
||||
model_group = metadata.get("model_group") # type: ignore[misc]
|
||||
if model_group:
|
||||
tags["model_group"] = str(model_group)
|
||||
|
||||
return tags
|
||||
|
||||
async def _upload_to_datadog(self, payload: List[Dict]):
|
||||
if not self.dd_api_key or not self.dd_app_key:
|
||||
return
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"DD-API-KEY": self.dd_api_key,
|
||||
"DD-APPLICATION-KEY": self.dd_app_key,
|
||||
}
|
||||
|
||||
# The API endpoint expects a list of objects directly in the body (file content behavior)
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
|
||||
data_json = safe_dumps(payload)
|
||||
|
||||
response = await self.async_client.put(
|
||||
self.upload_url, content=data_json, headers=headers
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
|
||||
verbose_logger.debug(
|
||||
f"Datadog Cost Management: Uploaded {len(payload)} cost entries. Status: {response.status_code}"
|
||||
)
|
||||
|
|
@ -20,14 +20,6 @@ def get_datadog_hostname() -> str:
|
|||
return os.getenv("HOSTNAME", "")
|
||||
|
||||
|
||||
def get_datadog_base_url_from_env() -> Optional[str]:
|
||||
"""
|
||||
Get base URL override from common DD_BASE_URL env var.
|
||||
This is useful for testing or custom endpoints.
|
||||
"""
|
||||
return os.getenv("DD_BASE_URL")
|
||||
|
||||
|
||||
def get_datadog_env() -> str:
|
||||
return os.getenv("DD_ENV", "unknown")
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ from litellm.integrations.custom_batch_logger import CustomBatchLogger
|
|||
from litellm.integrations.datadog.datadog_handler import (
|
||||
get_datadog_service,
|
||||
get_datadog_tags,
|
||||
get_datadog_base_url_from_env,
|
||||
)
|
||||
from litellm.litellm_core_utils.dd_tracing import tracer
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
|
|
@ -44,22 +43,24 @@ class DataDogLLMObsLogger(CustomBatchLogger):
|
|||
def __init__(self, **kwargs):
|
||||
try:
|
||||
verbose_logger.debug("DataDogLLMObs: Initializing logger")
|
||||
# Configure DataDog endpoint (Agent or Direct API)
|
||||
# Use LITELLM_DD_AGENT_HOST to avoid conflicts with ddtrace's DD_AGENT_HOST
|
||||
dd_agent_host = os.getenv("LITELLM_DD_AGENT_HOST")
|
||||
if os.getenv("DD_API_KEY", None) is None:
|
||||
raise Exception("DD_API_KEY is not set, set 'DD_API_KEY=<>'")
|
||||
if os.getenv("DD_SITE", None) is None:
|
||||
raise Exception(
|
||||
"DD_SITE is not set, set 'DD_SITE=<>', example sit = `us5.datadoghq.com`"
|
||||
)
|
||||
|
||||
self.async_client = get_async_httpx_client(
|
||||
llm_provider=httpxSpecialProvider.LoggingCallback
|
||||
)
|
||||
self.DD_API_KEY = os.getenv("DD_API_KEY")
|
||||
self.DD_SITE = os.getenv("DD_SITE")
|
||||
self.intake_url = (
|
||||
f"https://api.{self.DD_SITE}/api/intake/llm-obs/v1/trace/spans"
|
||||
)
|
||||
|
||||
if dd_agent_host:
|
||||
self._configure_dd_agent(dd_agent_host=dd_agent_host)
|
||||
else:
|
||||
self._configure_dd_direct_api()
|
||||
|
||||
# Optional override for testing
|
||||
dd_base_url = get_datadog_base_url_from_env()
|
||||
# testing base url
|
||||
dd_base_url = os.getenv("DD_BASE_URL")
|
||||
if dd_base_url:
|
||||
self.intake_url = f"{dd_base_url}/api/intake/llm-obs/v1/trace/spans"
|
||||
|
||||
|
|
@ -77,38 +78,6 @@ class DataDogLLMObsLogger(CustomBatchLogger):
|
|||
verbose_logger.exception(f"DataDogLLMObs: Error initializing - {str(e)}")
|
||||
raise e
|
||||
|
||||
def _configure_dd_agent(self, dd_agent_host: str):
|
||||
"""
|
||||
Configure the Datadog logger to send traces to the Agent.
|
||||
"""
|
||||
# When using the Agent, LLM Observability Intake does NOT require the API Key
|
||||
# Reference: https://docs.datadoghq.com/llm_observability/setup/sdk/#agent-setup
|
||||
|
||||
# Use specific port for LLM Obs (Trace Agent) to avoid conflict with Logs Agent (10518)
|
||||
agent_port = os.getenv("LITELLM_DD_LLM_OBS_PORT", "8126")
|
||||
self.DD_SITE = "localhost" # Not used for URL construction in agent mode
|
||||
self.intake_url = (
|
||||
f"http://{dd_agent_host}:{agent_port}/api/intake/llm-obs/v1/trace/spans"
|
||||
)
|
||||
verbose_logger.debug(f"DataDogLLMObs: Using DD Agent at {self.intake_url}")
|
||||
|
||||
def _configure_dd_direct_api(self):
|
||||
"""
|
||||
Configure the Datadog logger to send traces directly to the Datadog API.
|
||||
"""
|
||||
if not self.DD_API_KEY:
|
||||
raise Exception("DD_API_KEY is not set, set 'DD_API_KEY=<>'")
|
||||
|
||||
self.DD_SITE = os.getenv("DD_SITE")
|
||||
if not self.DD_SITE:
|
||||
raise Exception(
|
||||
"DD_SITE is not set, set 'DD_SITE=<>', example site = `us5.datadoghq.com`"
|
||||
)
|
||||
|
||||
self.intake_url = (
|
||||
f"https://api.{self.DD_SITE}/api/intake/llm-obs/v1/trace/spans"
|
||||
)
|
||||
|
||||
def _get_datadog_llm_obs_params(self) -> Dict:
|
||||
"""
|
||||
Get the datadog_llm_observability_params from litellm.datadog_llm_observability_params
|
||||
|
|
@ -195,14 +164,13 @@ class DataDogLLMObsLogger(CustomBatchLogger):
|
|||
|
||||
json_payload = safe_dumps(payload)
|
||||
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if self.DD_API_KEY:
|
||||
headers["DD-API-KEY"] = self.DD_API_KEY
|
||||
|
||||
response = await self.async_client.post(
|
||||
url=self.intake_url,
|
||||
content=json_payload,
|
||||
headers=headers,
|
||||
headers={
|
||||
"DD-API-KEY": self.DD_API_KEY,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
|
||||
if response.status_code != 202:
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ from litellm.constants import MAX_LANGFUSE_INITIALIZED_CLIENTS
|
|||
from litellm.litellm_core_utils.core_helpers import (
|
||||
safe_deep_copy,
|
||||
reconstruct_model_name,
|
||||
filter_exceptions_from_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
|
||||
from litellm.integrations.langfuse.langfuse_mock_client import (
|
||||
|
|
@ -76,8 +75,9 @@ def _extract_cache_read_input_tokens(usage_obj) -> int:
|
|||
# Check prompt_tokens_details.cached_tokens (used by Gemini and other providers)
|
||||
if hasattr(usage_obj, "prompt_tokens_details"):
|
||||
prompt_tokens_details = getattr(usage_obj, "prompt_tokens_details", None)
|
||||
if prompt_tokens_details is not None and hasattr(
|
||||
prompt_tokens_details, "cached_tokens"
|
||||
if (
|
||||
prompt_tokens_details is not None
|
||||
and hasattr(prompt_tokens_details, "cached_tokens")
|
||||
):
|
||||
cached_tokens = getattr(prompt_tokens_details, "cached_tokens", None)
|
||||
if (
|
||||
|
|
@ -540,6 +540,7 @@ class LangFuseLogger:
|
|||
verbose_logger.debug("Langfuse Layer Logging - logging to langfuse v2")
|
||||
|
||||
try:
|
||||
metadata = metadata or {}
|
||||
standard_logging_object: Optional[StandardLoggingPayload] = cast(
|
||||
Optional[StandardLoggingPayload],
|
||||
kwargs.get("standard_logging_object", None),
|
||||
|
|
@ -705,10 +706,9 @@ class LangFuseLogger:
|
|||
|
||||
clean_metadata["litellm_response_cost"] = cost
|
||||
if standard_logging_object is not None:
|
||||
hidden_params = standard_logging_object.get("hidden_params", {})
|
||||
clean_metadata["hidden_params"] = filter_exceptions_from_params(
|
||||
hidden_params
|
||||
)
|
||||
clean_metadata["hidden_params"] = standard_logging_object[
|
||||
"hidden_params"
|
||||
]
|
||||
|
||||
if (
|
||||
litellm.langfuse_default_tags is not None
|
||||
|
|
|
|||
|
|
@ -300,59 +300,43 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
|
|||
)
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
try:
|
||||
standard_callback_dynamic_params = kwargs.get(
|
||||
"standard_callback_dynamic_params"
|
||||
)
|
||||
langfuse_logger_to_use = LangFuseHandler.get_langfuse_logger_for_request(
|
||||
globalLangfuseLogger=self,
|
||||
standard_callback_dynamic_params=standard_callback_dynamic_params,
|
||||
in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
|
||||
)
|
||||
langfuse_logger_to_use.log_event_on_langfuse(
|
||||
kwargs=kwargs,
|
||||
response_obj=response_obj,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
user_id=kwargs.get("user", None),
|
||||
)
|
||||
except Exception as e:
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
verbose_logger.exception(
|
||||
f"Langfuse Layer Error - Exception occurred while logging success event: {str(e)}"
|
||||
)
|
||||
self.handle_callback_failure(callback_name="langfuse")
|
||||
standard_callback_dynamic_params = kwargs.get(
|
||||
"standard_callback_dynamic_params"
|
||||
)
|
||||
langfuse_logger_to_use = LangFuseHandler.get_langfuse_logger_for_request(
|
||||
globalLangfuseLogger=self,
|
||||
standard_callback_dynamic_params=standard_callback_dynamic_params,
|
||||
in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
|
||||
)
|
||||
langfuse_logger_to_use.log_event_on_langfuse(
|
||||
kwargs=kwargs,
|
||||
response_obj=response_obj,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
user_id=kwargs.get("user", None),
|
||||
)
|
||||
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
try:
|
||||
standard_callback_dynamic_params = kwargs.get(
|
||||
"standard_callback_dynamic_params"
|
||||
)
|
||||
langfuse_logger_to_use = LangFuseHandler.get_langfuse_logger_for_request(
|
||||
globalLangfuseLogger=self,
|
||||
standard_callback_dynamic_params=standard_callback_dynamic_params,
|
||||
in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
|
||||
)
|
||||
standard_logging_object = cast(
|
||||
Optional[StandardLoggingPayload],
|
||||
kwargs.get("standard_logging_object", None),
|
||||
)
|
||||
if standard_logging_object is None:
|
||||
return
|
||||
langfuse_logger_to_use.log_event_on_langfuse(
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
response_obj=None,
|
||||
user_id=kwargs.get("user", None),
|
||||
status_message=standard_logging_object["error_str"],
|
||||
level="ERROR",
|
||||
kwargs=kwargs,
|
||||
)
|
||||
except Exception as e:
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
verbose_logger.exception(
|
||||
f"Langfuse Layer Error - Exception occurred while logging failure event: {str(e)}"
|
||||
)
|
||||
self.handle_callback_failure(callback_name="langfuse")
|
||||
standard_callback_dynamic_params = kwargs.get(
|
||||
"standard_callback_dynamic_params"
|
||||
)
|
||||
langfuse_logger_to_use = LangFuseHandler.get_langfuse_logger_for_request(
|
||||
globalLangfuseLogger=self,
|
||||
standard_callback_dynamic_params=standard_callback_dynamic_params,
|
||||
in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
|
||||
)
|
||||
standard_logging_object = cast(
|
||||
Optional[StandardLoggingPayload],
|
||||
kwargs.get("standard_logging_object", None),
|
||||
)
|
||||
if standard_logging_object is None:
|
||||
return
|
||||
langfuse_logger_to_use.log_event_on_langfuse(
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
response_obj=None,
|
||||
user_id=kwargs.get("user", None),
|
||||
status_message=standard_logging_object["error_str"],
|
||||
level="ERROR",
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -144,7 +144,6 @@ class OpenTelemetry(CustomLogger):
|
|||
self.OTEL_EXPORTER = self.config.exporter
|
||||
self.OTEL_ENDPOINT = self.config.endpoint
|
||||
self.OTEL_HEADERS = self.config.headers
|
||||
self._tracer_provider_cache: Dict[str, Any] = {}
|
||||
self._init_tracing(tracer_provider)
|
||||
|
||||
_debug_otel = str(os.getenv("DEBUG_OTEL", "False")).lower()
|
||||
|
|
@ -616,20 +615,12 @@ class OpenTelemetry(CustomLogger):
|
|||
"""Create a temporary tracer with dynamic headers for this request only."""
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
|
||||
# Prevents thread exhaustion by reusing providers for the same credential sets (e.g. per-team keys)
|
||||
cache_key = str(sorted(dynamic_headers.items()))
|
||||
if cache_key in self._tracer_provider_cache:
|
||||
return self._tracer_provider_cache[cache_key].get_tracer(LITELLM_TRACER_NAME)
|
||||
|
||||
# Create a temporary tracer provider with dynamic headers
|
||||
temp_provider = TracerProvider(resource=self._get_litellm_resource(self.config))
|
||||
temp_provider.add_span_processor(
|
||||
self._get_span_processor(dynamic_headers=dynamic_headers)
|
||||
)
|
||||
|
||||
# Store in cache for reuse
|
||||
self._tracer_provider_cache[cache_key] = temp_provider
|
||||
|
||||
return temp_provider.get_tracer(LITELLM_TRACER_NAME)
|
||||
|
||||
def construct_dynamic_otel_headers(
|
||||
|
|
@ -1006,7 +997,7 @@ class OpenTelemetry(CustomLogger):
|
|||
try:
|
||||
from opentelemetry.sdk._logs import LogRecord as SdkLogRecord # type: ignore[attr-defined] # OTEL < 1.39.0
|
||||
except ImportError:
|
||||
from opentelemetry.sdk._logs._internal import LogRecord as SdkLogRecord # type: ignore[attr-defined, no-redef] # OTEL >= 1.39.0
|
||||
from opentelemetry.sdk._logs._internal import LogRecord as SdkLogRecord # OTEL >= 1.39.0
|
||||
|
||||
otel_logger = get_logger(LITELLM_LOGGER_NAME)
|
||||
|
||||
|
|
@ -1627,7 +1618,6 @@ class OpenTelemetry(CustomLogger):
|
|||
)
|
||||
|
||||
except Exception as e:
|
||||
self.handle_callback_failure(callback_name= self.callback_name)
|
||||
verbose_logger.exception(
|
||||
"OpenTelemetry logging error in set_attributes %s", str(e)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -229,18 +229,14 @@ class PrometheusLogger(CustomLogger):
|
|||
self.litellm_remaining_api_key_requests_for_model = self._gauge_factory(
|
||||
"litellm_remaining_api_key_requests_for_model",
|
||||
"Remaining Requests API Key can make for model (model based rpm limit on key)",
|
||||
labelnames=self.get_labels_for_metric(
|
||||
"litellm_remaining_api_key_requests_for_model"
|
||||
),
|
||||
labelnames=["hashed_api_key", "api_key_alias", "model"],
|
||||
)
|
||||
|
||||
# Remaining MODEL TPM limit for API Key
|
||||
self.litellm_remaining_api_key_tokens_for_model = self._gauge_factory(
|
||||
"litellm_remaining_api_key_tokens_for_model",
|
||||
"Remaining Tokens API Key can make for model (model based tpm limit on key)",
|
||||
labelnames=self.get_labels_for_metric(
|
||||
"litellm_remaining_api_key_tokens_for_model"
|
||||
),
|
||||
labelnames=["hashed_api_key", "api_key_alias", "model"],
|
||||
)
|
||||
|
||||
########################################
|
||||
|
|
@ -316,18 +312,6 @@ class PrometheusLogger(CustomLogger):
|
|||
labelnames=self.get_labels_for_metric("litellm_deployment_state"),
|
||||
)
|
||||
|
||||
self.litellm_deployment_tpm_limit = self._gauge_factory(
|
||||
"litellm_deployment_tpm_limit",
|
||||
"Deployment TPM limit found in config",
|
||||
labelnames=self.get_labels_for_metric("litellm_deployment_tpm_limit"),
|
||||
)
|
||||
|
||||
self.litellm_deployment_rpm_limit = self._gauge_factory(
|
||||
"litellm_deployment_rpm_limit",
|
||||
"Deployment RPM limit found in config",
|
||||
labelnames=self.get_labels_for_metric("litellm_deployment_rpm_limit"),
|
||||
)
|
||||
|
||||
self.litellm_deployment_cooled_down = self._counter_factory(
|
||||
"litellm_deployment_cooled_down",
|
||||
"LLM Deployment Analytics - Number of times a deployment has been cooled down by LiteLLM load balancing logic. exception_status is the status of the exception that caused the deployment to be cooled down",
|
||||
|
|
@ -389,9 +373,15 @@ class PrometheusLogger(CustomLogger):
|
|||
self.litellm_llm_api_failed_requests_metric = self._counter_factory(
|
||||
name="litellm_llm_api_failed_requests_metric",
|
||||
documentation="deprecated - use litellm_proxy_failed_requests_metric",
|
||||
labelnames=self.get_labels_for_metric(
|
||||
"litellm_llm_api_failed_requests_metric"
|
||||
),
|
||||
labelnames=[
|
||||
"end_user",
|
||||
"hashed_api_key",
|
||||
"api_key_alias",
|
||||
"model",
|
||||
"team",
|
||||
"team_alias",
|
||||
"user",
|
||||
],
|
||||
)
|
||||
|
||||
self.litellm_requests_metric = self._counter_factory(
|
||||
|
|
@ -901,7 +891,7 @@ class PrometheusLogger(CustomLogger):
|
|||
|
||||
model = kwargs.get("model", "")
|
||||
litellm_params = kwargs.get("litellm_params", {}) or {}
|
||||
_metadata = litellm_params.get("metadata") or {}
|
||||
_metadata = litellm_params.get("metadata", {})
|
||||
get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking()
|
||||
|
||||
end_user_id = get_end_user_id_for_cost_tracking(
|
||||
|
|
@ -964,8 +954,6 @@ class PrometheusLogger(CustomLogger):
|
|||
route=standard_logging_payload["metadata"].get(
|
||||
"user_api_key_request_route"
|
||||
),
|
||||
client_ip=standard_logging_payload["metadata"].get("requester_ip_address"),
|
||||
user_agent=standard_logging_payload["metadata"].get("user_agent"),
|
||||
)
|
||||
|
||||
if (
|
||||
|
|
@ -1023,7 +1011,6 @@ class PrometheusLogger(CustomLogger):
|
|||
user_api_key_alias=user_api_key_alias,
|
||||
kwargs=kwargs,
|
||||
metadata=_metadata,
|
||||
model_id=enum_values.model_id,
|
||||
)
|
||||
|
||||
# set latency metrics
|
||||
|
|
@ -1178,15 +1165,26 @@ class PrometheusLogger(CustomLogger):
|
|||
response_cost: float,
|
||||
user_id: Optional[str] = None,
|
||||
):
|
||||
_metadata = litellm_params.get("metadata") or {}
|
||||
_team_spend = _metadata.get("user_api_key_team_spend", None)
|
||||
_team_max_budget = _metadata.get("user_api_key_team_max_budget", None)
|
||||
_team_spend = litellm_params.get("metadata", {}).get(
|
||||
"user_api_key_team_spend", None
|
||||
)
|
||||
_team_max_budget = litellm_params.get("metadata", {}).get(
|
||||
"user_api_key_team_max_budget", None
|
||||
)
|
||||
|
||||
_api_key_spend = _metadata.get("user_api_key_spend", None)
|
||||
_api_key_max_budget = _metadata.get("user_api_key_max_budget", None)
|
||||
_api_key_spend = litellm_params.get("metadata", {}).get(
|
||||
"user_api_key_spend", None
|
||||
)
|
||||
_api_key_max_budget = litellm_params.get("metadata", {}).get(
|
||||
"user_api_key_max_budget", None
|
||||
)
|
||||
|
||||
_user_spend = _metadata.get("user_api_key_user_spend", None)
|
||||
_user_max_budget = _metadata.get("user_api_key_user_max_budget", None)
|
||||
_user_spend = litellm_params.get("metadata", {}).get(
|
||||
"user_api_key_user_spend", None
|
||||
)
|
||||
_user_max_budget = litellm_params.get("metadata", {}).get(
|
||||
"user_api_key_user_max_budget", None
|
||||
)
|
||||
|
||||
await self._set_api_key_budget_metrics_after_api_request(
|
||||
user_api_key=user_api_key,
|
||||
|
|
@ -1247,7 +1245,6 @@ class PrometheusLogger(CustomLogger):
|
|||
user_api_key_alias: Optional[str],
|
||||
kwargs: dict,
|
||||
metadata: dict,
|
||||
model_id: Optional[str] = None,
|
||||
):
|
||||
from litellm.proxy.common_utils.callback_utils import (
|
||||
get_model_group_from_litellm_kwargs,
|
||||
|
|
@ -1269,11 +1266,11 @@ class PrometheusLogger(CustomLogger):
|
|||
)
|
||||
|
||||
self.litellm_remaining_api_key_requests_for_model.labels(
|
||||
user_api_key, user_api_key_alias, model_group, model_id
|
||||
user_api_key, user_api_key_alias, model_group
|
||||
).set(remaining_requests)
|
||||
|
||||
self.litellm_remaining_api_key_tokens_for_model.labels(
|
||||
user_api_key, user_api_key_alias, model_group, model_id
|
||||
user_api_key, user_api_key_alias, model_group
|
||||
).set(remaining_tokens)
|
||||
|
||||
def _set_latency_metrics(
|
||||
|
|
@ -1299,14 +1296,12 @@ class PrometheusLogger(CustomLogger):
|
|||
time_to_first_token_seconds is not None
|
||||
and kwargs.get("stream", False) is True # only emit for streaming requests
|
||||
):
|
||||
_ttft_labels = prometheus_label_factory(
|
||||
supported_enum_labels=self.get_labels_for_metric(
|
||||
metric_name="litellm_llm_api_time_to_first_token_metric"
|
||||
),
|
||||
enum_values=enum_values,
|
||||
)
|
||||
self.litellm_llm_api_time_to_first_token_metric.labels(
|
||||
**_ttft_labels
|
||||
model,
|
||||
user_api_key,
|
||||
user_api_key_alias,
|
||||
user_api_team,
|
||||
user_api_team_alias,
|
||||
).observe(time_to_first_token_seconds)
|
||||
else:
|
||||
verbose_logger.debug(
|
||||
|
|
@ -1346,7 +1341,7 @@ class PrometheusLogger(CustomLogger):
|
|||
|
||||
# request queue time (time from arrival to processing start)
|
||||
_litellm_params = kwargs.get("litellm_params", {}) or {}
|
||||
queue_time_seconds = (_litellm_params.get("metadata") or {}).get(
|
||||
queue_time_seconds = _litellm_params.get("metadata", {}).get(
|
||||
"queue_time_seconds"
|
||||
)
|
||||
if queue_time_seconds is not None and queue_time_seconds >= 0:
|
||||
|
|
@ -1370,14 +1365,14 @@ class PrometheusLogger(CustomLogger):
|
|||
standard_logging_payload: StandardLoggingPayload = kwargs.get(
|
||||
"standard_logging_object", {}
|
||||
)
|
||||
|
||||
|
||||
if self._should_skip_metrics_for_invalid_key(
|
||||
kwargs=kwargs, standard_logging_payload=standard_logging_payload
|
||||
):
|
||||
return
|
||||
|
||||
|
||||
model = kwargs.get("model", "")
|
||||
|
||||
|
||||
litellm_params = kwargs.get("litellm_params", {}) or {}
|
||||
get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking()
|
||||
|
||||
|
|
@ -1401,7 +1396,6 @@ class PrometheusLogger(CustomLogger):
|
|||
user_api_team,
|
||||
user_api_team_alias,
|
||||
user_id,
|
||||
standard_logging_payload.get("model_id", ""),
|
||||
).inc()
|
||||
self.set_llm_deployment_failure_metrics(kwargs)
|
||||
except Exception as e:
|
||||
|
|
@ -1419,57 +1413,49 @@ class PrometheusLogger(CustomLogger):
|
|||
) -> Optional[int]:
|
||||
"""
|
||||
Extract HTTP status code from various input formats for validation.
|
||||
|
||||
|
||||
This is a centralized helper to extract status code from different
|
||||
callback function signatures. Handles both ProxyException (uses 'code')
|
||||
and standard exceptions (uses 'status_code').
|
||||
|
||||
|
||||
Args:
|
||||
kwargs: Dictionary potentially containing 'exception' key
|
||||
enum_values: Object with 'status_code' attribute
|
||||
exception: Exception object to extract status code from directly
|
||||
|
||||
|
||||
Returns:
|
||||
Status code as integer if found, None otherwise
|
||||
"""
|
||||
status_code = None
|
||||
|
||||
|
||||
# Try from enum_values first (most common in our callbacks)
|
||||
if (
|
||||
enum_values
|
||||
and hasattr(enum_values, "status_code")
|
||||
and enum_values.status_code
|
||||
):
|
||||
if enum_values and hasattr(enum_values, "status_code") and enum_values.status_code:
|
||||
try:
|
||||
status_code = int(enum_values.status_code)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
|
||||
if not status_code and exception:
|
||||
# ProxyException uses 'code' attribute, other exceptions may use 'status_code'
|
||||
status_code = getattr(exception, "status_code", None) or getattr(
|
||||
exception, "code", None
|
||||
)
|
||||
status_code = getattr(exception, "status_code", None) or getattr(exception, "code", None)
|
||||
if status_code is not None:
|
||||
try:
|
||||
status_code = int(status_code)
|
||||
except (ValueError, TypeError):
|
||||
status_code = None
|
||||
|
||||
|
||||
if not status_code and kwargs:
|
||||
exception_in_kwargs = kwargs.get("exception")
|
||||
if exception_in_kwargs:
|
||||
status_code = getattr(
|
||||
exception_in_kwargs, "status_code", None
|
||||
) or getattr(exception_in_kwargs, "code", None)
|
||||
status_code = getattr(exception_in_kwargs, "status_code", None) or getattr(exception_in_kwargs, "code", None)
|
||||
if status_code is not None:
|
||||
try:
|
||||
status_code = int(status_code)
|
||||
except (ValueError, TypeError):
|
||||
status_code = None
|
||||
|
||||
|
||||
return status_code
|
||||
|
||||
|
||||
def _is_invalid_api_key_request(
|
||||
self,
|
||||
status_code: Optional[int],
|
||||
|
|
@ -1477,23 +1463,23 @@ class PrometheusLogger(CustomLogger):
|
|||
) -> bool:
|
||||
"""
|
||||
Determine if a request has an invalid API key based on status code and exception.
|
||||
|
||||
|
||||
This method prevents invalid authentication attempts from being recorded in
|
||||
Prometheus metrics. A 401 status code is the definitive indicator of authentication
|
||||
failure. Additionally, we check exception messages for authentication error patterns
|
||||
to catch cases where the exception hasn't been converted to a ProxyException yet.
|
||||
|
||||
|
||||
Args:
|
||||
status_code: HTTP status code (401 indicates authentication error)
|
||||
exception: Exception object to check for auth-related error messages
|
||||
|
||||
|
||||
Returns:
|
||||
True if the request has an invalid API key and metrics should be skipped,
|
||||
False otherwise
|
||||
"""
|
||||
if status_code == 401:
|
||||
return True
|
||||
|
||||
|
||||
# Handle cases where AssertionError is raised before conversion to ProxyException
|
||||
if exception is not None:
|
||||
exception_str = str(exception).lower()
|
||||
|
|
@ -1506,9 +1492,9 @@ class PrometheusLogger(CustomLogger):
|
|||
]
|
||||
if any(pattern in exception_str for pattern in auth_error_patterns):
|
||||
return True
|
||||
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _should_skip_metrics_for_invalid_key(
|
||||
self,
|
||||
kwargs: Optional[dict] = None,
|
||||
|
|
@ -1519,18 +1505,18 @@ class PrometheusLogger(CustomLogger):
|
|||
) -> bool:
|
||||
"""
|
||||
Determine if Prometheus metrics should be skipped for invalid API key requests.
|
||||
|
||||
|
||||
This is a centralized validation method that extracts status code and exception
|
||||
information from various callback function signatures and determines if the request
|
||||
represents an invalid API key attempt that should be filtered from metrics.
|
||||
|
||||
|
||||
Args:
|
||||
kwargs: Dictionary potentially containing exception and other data
|
||||
user_api_key_dict: User API key authentication object (currently unused)
|
||||
enum_values: Object with status_code attribute
|
||||
standard_logging_payload: Standard logging payload dictionary
|
||||
exception: Exception object to check directly
|
||||
|
||||
|
||||
Returns:
|
||||
True if metrics should be skipped (invalid key detected), False otherwise
|
||||
"""
|
||||
|
|
@ -1539,17 +1525,17 @@ class PrometheusLogger(CustomLogger):
|
|||
enum_values=enum_values,
|
||||
exception=exception,
|
||||
)
|
||||
|
||||
|
||||
if exception is None and kwargs:
|
||||
exception = kwargs.get("exception")
|
||||
|
||||
|
||||
if self._is_invalid_api_key_request(status_code, exception=exception):
|
||||
verbose_logger.debug(
|
||||
"Skipping Prometheus metrics for invalid API key request: "
|
||||
f"status_code={status_code}, exception={type(exception).__name__ if exception else None}"
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
return False
|
||||
|
||||
async def async_post_call_failure_hook(
|
||||
|
|
@ -1590,10 +1576,6 @@ class PrometheusLogger(CustomLogger):
|
|||
litellm_params=request_data,
|
||||
proxy_server_request=request_data.get("proxy_server_request", {}),
|
||||
)
|
||||
_metadata = request_data.get("metadata", {}) or {}
|
||||
model_id = _metadata.get("model_info", {}).get("id") or request_data.get(
|
||||
"model_info", {}
|
||||
).get("id")
|
||||
enum_values = UserAPIKeyLabelValues(
|
||||
end_user=user_api_key_dict.end_user_id,
|
||||
user=user_api_key_dict.user_id,
|
||||
|
|
@ -1608,9 +1590,6 @@ class PrometheusLogger(CustomLogger):
|
|||
exception_class=self._get_exception_class_name(original_exception),
|
||||
tags=_tags,
|
||||
route=user_api_key_dict.request_route,
|
||||
client_ip=_metadata.get("requester_ip_address"),
|
||||
user_agent=_metadata.get("user_agent"),
|
||||
model_id=model_id,
|
||||
)
|
||||
_labels = prometheus_label_factory(
|
||||
supported_enum_labels=self.get_labels_for_metric(
|
||||
|
|
@ -1650,7 +1629,6 @@ class PrometheusLogger(CustomLogger):
|
|||
):
|
||||
return
|
||||
|
||||
_metadata = data.get("metadata", {}) or {}
|
||||
enum_values = UserAPIKeyLabelValues(
|
||||
end_user=user_api_key_dict.end_user_id,
|
||||
hashed_api_key=user_api_key_dict.api_key,
|
||||
|
|
@ -1666,8 +1644,6 @@ class PrometheusLogger(CustomLogger):
|
|||
litellm_params=data,
|
||||
proxy_server_request=data.get("proxy_server_request", {}),
|
||||
),
|
||||
client_ip=_metadata.get("requester_ip_address"),
|
||||
user_agent=_metadata.get("user_agent"),
|
||||
)
|
||||
_labels = prometheus_label_factory(
|
||||
supported_enum_labels=self.get_labels_for_metric(
|
||||
|
|
@ -1708,7 +1684,7 @@ class PrometheusLogger(CustomLogger):
|
|||
exception = request_kwargs.get("exception", None)
|
||||
|
||||
llm_provider = _litellm_params.get("custom_llm_provider", None)
|
||||
|
||||
|
||||
if self._should_skip_metrics_for_invalid_key(
|
||||
kwargs=request_kwargs,
|
||||
standard_logging_payload=standard_logging_payload,
|
||||
|
|
@ -1740,10 +1716,6 @@ class PrometheusLogger(CustomLogger):
|
|||
"user_api_key_team_alias"
|
||||
],
|
||||
tags=standard_logging_payload.get("request_tags", []),
|
||||
client_ip=standard_logging_payload["metadata"].get(
|
||||
"requester_ip_address"
|
||||
),
|
||||
user_agent=standard_logging_payload["metadata"].get("user_agent"),
|
||||
)
|
||||
|
||||
"""
|
||||
|
|
@ -1781,49 +1753,6 @@ class PrometheusLogger(CustomLogger):
|
|||
)
|
||||
)
|
||||
|
||||
def _set_deployment_tpm_rpm_limit_metrics(
|
||||
self,
|
||||
model_info: dict,
|
||||
litellm_params: dict,
|
||||
litellm_model_name: Optional[str],
|
||||
model_id: Optional[str],
|
||||
api_base: Optional[str],
|
||||
llm_provider: Optional[str],
|
||||
):
|
||||
"""
|
||||
Set the deployment TPM and RPM limits metrics
|
||||
"""
|
||||
tpm = model_info.get("tpm") or litellm_params.get("tpm")
|
||||
rpm = model_info.get("rpm") or litellm_params.get("rpm")
|
||||
|
||||
if tpm is not None:
|
||||
_labels = prometheus_label_factory(
|
||||
supported_enum_labels=self.get_labels_for_metric(
|
||||
metric_name="litellm_deployment_tpm_limit"
|
||||
),
|
||||
enum_values=UserAPIKeyLabelValues(
|
||||
litellm_model_name=litellm_model_name,
|
||||
model_id=model_id,
|
||||
api_base=api_base,
|
||||
api_provider=llm_provider,
|
||||
),
|
||||
)
|
||||
self.litellm_deployment_tpm_limit.labels(**_labels).set(tpm)
|
||||
|
||||
if rpm is not None:
|
||||
_labels = prometheus_label_factory(
|
||||
supported_enum_labels=self.get_labels_for_metric(
|
||||
metric_name="litellm_deployment_rpm_limit"
|
||||
),
|
||||
enum_values=UserAPIKeyLabelValues(
|
||||
litellm_model_name=litellm_model_name,
|
||||
model_id=model_id,
|
||||
api_base=api_base,
|
||||
api_provider=llm_provider,
|
||||
),
|
||||
)
|
||||
self.litellm_deployment_rpm_limit.labels(**_labels).set(rpm)
|
||||
|
||||
def set_llm_deployment_success_metrics(
|
||||
self,
|
||||
request_kwargs: dict,
|
||||
|
|
@ -1857,16 +1786,6 @@ class PrometheusLogger(CustomLogger):
|
|||
_model_info = _metadata.get("model_info") or {}
|
||||
model_id = _model_info.get("id", None)
|
||||
|
||||
if _model_info or _litellm_params:
|
||||
self._set_deployment_tpm_rpm_limit_metrics(
|
||||
model_info=_model_info,
|
||||
litellm_params=_litellm_params,
|
||||
litellm_model_name=litellm_model_name,
|
||||
model_id=model_id,
|
||||
api_base=api_base,
|
||||
llm_provider=llm_provider,
|
||||
)
|
||||
|
||||
remaining_requests: Optional[int] = None
|
||||
remaining_tokens: Optional[int] = None
|
||||
if additional_headers := standard_logging_payload["hidden_params"][
|
||||
|
|
@ -2344,10 +2263,7 @@ class PrometheusLogger(CustomLogger):
|
|||
|
||||
async def fetch_keys(
|
||||
page_size: int, page: int
|
||||
) -> Tuple[
|
||||
List[Union[str, UserAPIKeyAuth, LiteLLM_DeletedVerificationToken]],
|
||||
Optional[int],
|
||||
]:
|
||||
) -> Tuple[List[Union[str, UserAPIKeyAuth, LiteLLM_DeletedVerificationToken]], Optional[int]]:
|
||||
key_list_response = await _list_key_helper(
|
||||
prisma_client=prisma_client,
|
||||
page=page,
|
||||
|
|
@ -2463,16 +2379,12 @@ class PrometheusLogger(CustomLogger):
|
|||
# Get total user count
|
||||
total_users = await prisma_client.db.litellm_usertable.count()
|
||||
self.litellm_total_users_metric.set(total_users)
|
||||
verbose_logger.debug(
|
||||
f"Prometheus: set litellm_total_users to {total_users}"
|
||||
)
|
||||
verbose_logger.debug(f"Prometheus: set litellm_total_users to {total_users}")
|
||||
|
||||
# Get total team count
|
||||
total_teams = await prisma_client.db.litellm_teamtable.count()
|
||||
self.litellm_teams_count_metric.set(total_teams)
|
||||
verbose_logger.debug(
|
||||
f"Prometheus: set litellm_teams_count to {total_teams}"
|
||||
)
|
||||
verbose_logger.debug(f"Prometheus: set litellm_teams_count to {total_teams}")
|
||||
except Exception as e:
|
||||
verbose_logger.exception(
|
||||
f"Error initializing user/team count metrics: {str(e)}"
|
||||
|
|
@ -2500,8 +2412,8 @@ class PrometheusLogger(CustomLogger):
|
|||
self,
|
||||
user_api_team: Optional[str],
|
||||
user_api_team_alias: Optional[str],
|
||||
team_spend: Optional[float],
|
||||
team_max_budget: Optional[float],
|
||||
team_spend: float,
|
||||
team_max_budget: float,
|
||||
response_cost: float,
|
||||
):
|
||||
"""
|
||||
|
|
@ -2663,7 +2575,7 @@ class PrometheusLogger(CustomLogger):
|
|||
user_api_key: Optional[str],
|
||||
user_api_key_alias: Optional[str],
|
||||
response_cost: float,
|
||||
key_max_budget: Optional[float],
|
||||
key_max_budget: float,
|
||||
key_spend: Optional[float],
|
||||
):
|
||||
if user_api_key:
|
||||
|
|
@ -2680,7 +2592,7 @@ class PrometheusLogger(CustomLogger):
|
|||
self,
|
||||
user_api_key: str,
|
||||
user_api_key_alias: str,
|
||||
key_max_budget: Optional[float],
|
||||
key_max_budget: float,
|
||||
key_spend: Optional[float],
|
||||
response_cost: float,
|
||||
) -> UserAPIKeyAuth:
|
||||
|
|
|
|||
|
|
@ -93,11 +93,8 @@ def get_litellm_params(
|
|||
"text_completion": text_completion,
|
||||
"azure_ad_token_provider": azure_ad_token_provider,
|
||||
"user_continue_message": user_continue_message,
|
||||
"base_model": base_model
|
||||
or (
|
||||
_get_base_model_from_litellm_call_metadata(metadata=metadata)
|
||||
if metadata
|
||||
else None
|
||||
"base_model": base_model or (
|
||||
_get_base_model_from_litellm_call_metadata(metadata=metadata) if metadata else None
|
||||
),
|
||||
"litellm_trace_id": litellm_trace_id,
|
||||
"litellm_session_id": litellm_session_id,
|
||||
|
|
@ -142,7 +139,5 @@ def get_litellm_params(
|
|||
"aws_sts_endpoint": kwargs.get("aws_sts_endpoint"),
|
||||
"aws_external_id": kwargs.get("aws_external_id"),
|
||||
"aws_bedrock_runtime_endpoint": kwargs.get("aws_bedrock_runtime_endpoint"),
|
||||
"tpm": kwargs.get("tpm"),
|
||||
"rpm": kwargs.get("rpm"),
|
||||
}
|
||||
return litellm_params
|
||||
|
|
|
|||
|
|
@ -205,11 +205,6 @@ _in_memory_loggers: List[Any] = []
|
|||
|
||||
### GLOBAL VARIABLES ###
|
||||
|
||||
# Cache custom pricing keys as frozenset for O(1) lookups instead of looping through 49 keys
|
||||
_CUSTOM_PRICING_KEYS: frozenset = frozenset(
|
||||
CustomPricingLiteLLMParams.model_fields.keys()
|
||||
)
|
||||
|
||||
sentry_sdk_instance = None
|
||||
capture_exception = None
|
||||
add_breadcrumb = None
|
||||
|
|
@ -335,9 +330,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.start_time = start_time # log the call start time
|
||||
self.call_type = call_type
|
||||
self.litellm_call_id = litellm_call_id
|
||||
self.litellm_trace_id: str = (
|
||||
litellm_trace_id if litellm_trace_id else str(uuid.uuid4())
|
||||
)
|
||||
self.litellm_trace_id: str = litellm_trace_id if litellm_trace_id else str(uuid.uuid4())
|
||||
self.function_id = function_id
|
||||
self.streaming_chunks: List[Any] = [] # for generating complete stream response
|
||||
self.sync_streaming_chunks: List[
|
||||
|
|
@ -546,11 +539,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if "stream_options" in additional_params:
|
||||
self.stream_options = additional_params["stream_options"]
|
||||
## check if custom pricing set ##
|
||||
if any(
|
||||
litellm_params.get(key) is not None
|
||||
for key in _CUSTOM_PRICING_KEYS & litellm_params.keys()
|
||||
):
|
||||
self.custom_pricing = True
|
||||
custom_pricing_keys = CustomPricingLiteLLMParams.model_fields.keys()
|
||||
for key in custom_pricing_keys:
|
||||
if litellm_params.get(key) is not None:
|
||||
self.custom_pricing = True
|
||||
|
||||
if "custom_llm_provider" in self.model_call_details:
|
||||
self.custom_llm_provider = self.model_call_details["custom_llm_provider"]
|
||||
|
|
@ -1638,19 +1630,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"standard_logging_object"
|
||||
)
|
||||
) is not None:
|
||||
response_dict = (
|
||||
standard_logging_payload["response"] = (
|
||||
result.model_dump()
|
||||
if hasattr(result, "model_dump")
|
||||
else dict(result)
|
||||
)
|
||||
# Ensure usage is properly included with transformed chat format
|
||||
if transformed_usage is not None:
|
||||
response_dict["usage"] = (
|
||||
transformed_usage.model_dump()
|
||||
if hasattr(transformed_usage, "model_dump")
|
||||
else dict(transformed_usage)
|
||||
)
|
||||
standard_logging_payload["response"] = response_dict
|
||||
elif isinstance(result, TranscriptionResponse):
|
||||
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
|
||||
TranscriptionUsageObjectTransformation,
|
||||
|
|
@ -3312,7 +3296,6 @@ def _get_masked_values(
|
|||
"token",
|
||||
"key",
|
||||
"secret",
|
||||
"vertex_credentials",
|
||||
]
|
||||
return {
|
||||
k: (
|
||||
|
|
@ -4265,21 +4248,15 @@ def use_custom_pricing_for_model(litellm_params: Optional[dict]) -> bool:
|
|||
if litellm_params is None:
|
||||
return False
|
||||
|
||||
# Check litellm_params using set intersection (only check keys that exist in both)
|
||||
matching_keys = _CUSTOM_PRICING_KEYS & litellm_params.keys()
|
||||
for key in matching_keys:
|
||||
if litellm_params.get(key) is not None:
|
||||
return True
|
||||
|
||||
# Check model_info
|
||||
metadata: dict = litellm_params.get("metadata", {}) or {}
|
||||
model_info: dict = metadata.get("model_info", {}) or {}
|
||||
|
||||
if model_info:
|
||||
matching_keys = _CUSTOM_PRICING_KEYS & model_info.keys()
|
||||
for key in matching_keys:
|
||||
if model_info.get(key) is not None:
|
||||
return True
|
||||
custom_pricing_keys = CustomPricingLiteLLMParams.model_fields.keys()
|
||||
for key in custom_pricing_keys:
|
||||
if litellm_params.get(key, None) is not None:
|
||||
return True
|
||||
elif model_info.get(key, None) is not None:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
|
@ -4467,7 +4444,6 @@ class StandardLoggingPayloadSetup:
|
|||
user_api_key_request_route=None,
|
||||
spend_logs_metadata=None,
|
||||
requester_ip_address=None,
|
||||
user_agent=None,
|
||||
requester_metadata=None,
|
||||
prompt_management_metadata=prompt_management_metadata,
|
||||
applied_guardrails=applied_guardrails,
|
||||
|
|
@ -4548,10 +4524,6 @@ class StandardLoggingPayloadSetup:
|
|||
)
|
||||
elif isinstance(usage, Usage):
|
||||
return usage
|
||||
elif isinstance(usage, ResponseAPIUsage):
|
||||
return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
|
||||
usage
|
||||
)
|
||||
elif isinstance(usage, dict):
|
||||
if ResponseAPILoggingUtils._is_response_api_usage(usage):
|
||||
return (
|
||||
|
|
@ -4680,10 +4652,7 @@ class StandardLoggingPayloadSetup:
|
|||
@staticmethod
|
||||
def strip_trailing_slash(api_base: Optional[str]) -> Optional[str]:
|
||||
if api_base:
|
||||
if api_base.endswith("//"):
|
||||
return api_base.rstrip("/")
|
||||
if api_base[-1] == "/":
|
||||
return api_base[:-1]
|
||||
return api_base.rstrip("/")
|
||||
return api_base
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -5157,7 +5126,6 @@ def get_standard_logging_object_payload(
|
|||
model_group=_model_group,
|
||||
model_id=_model_id,
|
||||
requester_ip_address=clean_metadata.get("requester_ip_address", None),
|
||||
user_agent=clean_metadata.get("user_agent", None),
|
||||
messages=StandardLoggingPayloadSetup.append_system_prompt_messages(
|
||||
kwargs=kwargs, messages=kwargs.get("messages")
|
||||
),
|
||||
|
|
@ -5223,7 +5191,6 @@ def get_standard_logging_metadata(
|
|||
user_api_key_team_alias=None,
|
||||
spend_logs_metadata=None,
|
||||
requester_ip_address=None,
|
||||
user_agent=None,
|
||||
requester_metadata=None,
|
||||
user_api_key_end_user_id=None,
|
||||
prompt_management_metadata=None,
|
||||
|
|
|
|||
|
|
@ -23,15 +23,6 @@ def _is_above_128k(tokens: float) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def get_billable_input_tokens(usage: Usage) -> int:
|
||||
"""
|
||||
Returns the number of billable input tokens.
|
||||
Subtracts cached tokens from prompt tokens if applicable.
|
||||
"""
|
||||
details = _parse_prompt_tokens_details(usage)
|
||||
return usage.prompt_tokens - details["cache_hit_tokens"]
|
||||
|
||||
|
||||
def select_cost_metric_for_model(
|
||||
model_info: ModelInfo,
|
||||
) -> Literal["cost_per_character", "cost_per_token"]:
|
||||
|
|
@ -199,6 +190,7 @@ def _get_token_base_cost(
|
|||
1000 if "k" in threshold_str else 1
|
||||
)
|
||||
if usage.prompt_tokens > threshold:
|
||||
|
||||
prompt_base_cost = cast(
|
||||
float, _get_cost_per_unit(model_info, key, prompt_base_cost)
|
||||
)
|
||||
|
|
@ -574,28 +566,14 @@ def generic_cost_per_token( # noqa: PLR0915
|
|||
if usage.prompt_tokens_details:
|
||||
prompt_tokens_details = _parse_prompt_tokens_details(usage)
|
||||
|
||||
## EDGE CASE - text tokens not set or includes cached tokens (double-counting)
|
||||
## Some providers (like xAI) report text_tokens = prompt_tokens (including cached)
|
||||
## We detect this when: text_tokens + cached_tokens + other > prompt_tokens
|
||||
## Ref: https://github.com/BerriAI/litellm/issues/19680, #14874, #14875
|
||||
## EDGE CASE - text tokens not set inside PromptTokensDetails
|
||||
|
||||
cache_hit = prompt_tokens_details["cache_hit_tokens"]
|
||||
text_tokens = prompt_tokens_details["text_tokens"]
|
||||
audio_tokens = prompt_tokens_details["audio_tokens"]
|
||||
cache_creation = prompt_tokens_details["cache_creation_tokens"]
|
||||
image_tokens = prompt_tokens_details["image_tokens"]
|
||||
|
||||
# Check for double-counting: sum of details > prompt_tokens means overlap
|
||||
total_details = text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens
|
||||
has_double_counting = cache_hit > 0 and total_details > usage.prompt_tokens
|
||||
|
||||
if text_tokens == 0 or has_double_counting:
|
||||
if prompt_tokens_details["text_tokens"] == 0:
|
||||
text_tokens = (
|
||||
usage.prompt_tokens
|
||||
- cache_hit
|
||||
- audio_tokens
|
||||
- cache_creation
|
||||
- image_tokens
|
||||
- prompt_tokens_details["cache_hit_tokens"]
|
||||
- prompt_tokens_details["audio_tokens"]
|
||||
- prompt_tokens_details["cache_creation_tokens"]
|
||||
)
|
||||
prompt_tokens_details["text_tokens"] = text_tokens
|
||||
|
||||
|
|
@ -641,11 +619,7 @@ def generic_cost_per_token( # noqa: PLR0915
|
|||
# Calculate text tokens as remainder when we have a breakdown
|
||||
# This handles cases like OpenAI's reasoning models where text_tokens isn't provided
|
||||
text_tokens = max(
|
||||
0,
|
||||
usage.completion_tokens
|
||||
- reasoning_tokens
|
||||
- audio_tokens
|
||||
- image_tokens,
|
||||
0, usage.completion_tokens - reasoning_tokens - audio_tokens - image_tokens
|
||||
)
|
||||
else:
|
||||
# No breakdown at all, all tokens are text tokens
|
||||
|
|
|
|||
|
|
@ -21,13 +21,11 @@ from litellm.types.utils import (
|
|||
ChatCompletionMessageToolCall,
|
||||
ChatCompletionRedactedThinkingBlock,
|
||||
Choices,
|
||||
CompletionTokensDetailsWrapper,
|
||||
Delta,
|
||||
EmbeddingResponse,
|
||||
Function,
|
||||
HiddenParams,
|
||||
ImageResponse,
|
||||
PromptTokensDetailsWrapper,
|
||||
)
|
||||
from litellm.types.utils import Logprobs as TextCompletionLogprobs
|
||||
from litellm.types.utils import (
|
||||
|
|
@ -306,22 +304,6 @@ class LiteLLMResponseObjectHandler:
|
|||
"text_tokens": 0,
|
||||
}
|
||||
|
||||
# Map Responses API naming to Chat Completions API naming for cost calculator
|
||||
if usage.get("prompt_tokens") is None:
|
||||
usage["prompt_tokens"] = usage.get("input_tokens", 0)
|
||||
if usage.get("completion_tokens") is None:
|
||||
usage["completion_tokens"] = usage.get("output_tokens", 0)
|
||||
|
||||
# Convert dicts to wrapper objects so getattr() works in cost calculation
|
||||
if isinstance(usage.get("input_tokens_details"), dict):
|
||||
usage["prompt_tokens_details"] = PromptTokensDetailsWrapper(
|
||||
**usage["input_tokens_details"]
|
||||
)
|
||||
if isinstance(usage.get("output_tokens_details"), dict):
|
||||
usage["completion_tokens_details"] = CompletionTokensDetailsWrapper(
|
||||
**usage["output_tokens_details"]
|
||||
)
|
||||
|
||||
if model_response_object is None:
|
||||
model_response_object = ImageResponse(**response_object)
|
||||
return model_response_object
|
||||
|
|
|
|||
|
|
@ -415,28 +415,6 @@ class LoggingWorker:
|
|||
"""
|
||||
Safely log a message during shutdown, suppressing errors if logging is closed.
|
||||
"""
|
||||
# Check if logger has valid handlers before attempting to log
|
||||
# During shutdown, handlers may be closed, causing ValueError when writing
|
||||
if not hasattr(verbose_logger, 'handlers') or not verbose_logger.handlers:
|
||||
return
|
||||
|
||||
# Check if any handler has a valid stream
|
||||
has_valid_handler = False
|
||||
for handler in verbose_logger.handlers:
|
||||
try:
|
||||
if hasattr(handler, 'stream') and handler.stream and not handler.stream.closed:
|
||||
has_valid_handler = True
|
||||
break
|
||||
elif not hasattr(handler, 'stream'):
|
||||
# Non-stream handlers (like NullHandler) are always valid
|
||||
has_valid_handler = True
|
||||
break
|
||||
except (AttributeError, ValueError):
|
||||
continue
|
||||
|
||||
if not has_valid_handler:
|
||||
return
|
||||
|
||||
try:
|
||||
if level == "debug":
|
||||
verbose_logger.debug(message)
|
||||
|
|
|
|||
|
|
@ -1677,16 +1677,13 @@ def convert_to_anthropic_tool_result(
|
|||
] = []
|
||||
for content in content_list:
|
||||
if content["type"] == "text":
|
||||
# Only include cache_control if explicitly set and not None
|
||||
# to avoid sending "cache_control": null which breaks some API channels
|
||||
text_content: AnthropicMessagesToolResultContent = {
|
||||
"type": "text",
|
||||
"text": content["text"],
|
||||
}
|
||||
cache_control_value = content.get("cache_control")
|
||||
if cache_control_value is not None:
|
||||
text_content["cache_control"] = cache_control_value
|
||||
anthropic_content_list.append(text_content)
|
||||
anthropic_content_list.append(
|
||||
AnthropicMessagesToolResultContent(
|
||||
type="text",
|
||||
text=content["text"],
|
||||
cache_control=content.get("cache_control", None),
|
||||
)
|
||||
)
|
||||
elif content["type"] == "image_url":
|
||||
format = (
|
||||
content["image_url"].get("format")
|
||||
|
|
@ -4411,7 +4408,7 @@ def _bedrock_tools_pt(tools: List) -> List[BedrockToolBlock]:
|
|||
]
|
||||
"""
|
||||
"""
|
||||
Bedrock toolConfig looks like:
|
||||
Bedrock toolConfig looks like:
|
||||
"tools": [
|
||||
{
|
||||
"toolSpec": {
|
||||
|
|
@ -4439,7 +4436,6 @@ def _bedrock_tools_pt(tools: List) -> List[BedrockToolBlock]:
|
|||
|
||||
tool_block_list: List[BedrockToolBlock] = []
|
||||
for tool in tools:
|
||||
# Handle regular function tools
|
||||
parameters = tool.get("function", {}).get(
|
||||
"parameters", {"type": "object", "properties": {}}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -31,19 +31,15 @@ def _process_image_response(response: Response, url: str) -> str:
|
|||
f"Error: Image size ({size_mb:.2f}MB) exceeds maximum allowed size ({MAX_IMAGE_URL_DOWNLOAD_SIZE_MB}MB). url={url}"
|
||||
)
|
||||
|
||||
# Stream download with size checking to prevent downloading huge files
|
||||
max_bytes = int(MAX_IMAGE_URL_DOWNLOAD_SIZE_MB * 1024 * 1024)
|
||||
image_bytes = bytearray()
|
||||
bytes_downloaded = 0
|
||||
image_bytes = response.content
|
||||
|
||||
for chunk in response.iter_bytes(chunk_size=8192):
|
||||
bytes_downloaded += len(chunk)
|
||||
if bytes_downloaded > max_bytes:
|
||||
size_mb = bytes_downloaded / (1024 * 1024)
|
||||
# Check actual size after download if Content-Length was not available
|
||||
if content_length is None:
|
||||
size_mb = len(image_bytes) / (1024 * 1024)
|
||||
if size_mb > MAX_IMAGE_URL_DOWNLOAD_SIZE_MB:
|
||||
raise litellm.ImageFetchError(
|
||||
f"Error: Image size ({size_mb:.2f}MB) exceeds maximum allowed size ({MAX_IMAGE_URL_DOWNLOAD_SIZE_MB}MB). url={url}"
|
||||
)
|
||||
image_bytes.extend(chunk)
|
||||
|
||||
base64_image = base64.b64encode(image_bytes).decode("utf-8")
|
||||
|
||||
|
|
|
|||
|
|
@ -290,19 +290,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
elif tool_choice == "none":
|
||||
_tool_choice = AnthropicMessagesToolChoice(type="none")
|
||||
elif isinstance(tool_choice, dict):
|
||||
if "type" in tool_choice and "function" not in tool_choice:
|
||||
tool_type = tool_choice.get("type")
|
||||
if tool_type == "auto":
|
||||
_tool_choice = AnthropicMessagesToolChoice(type="auto")
|
||||
elif tool_type == "required" or tool_type == "any":
|
||||
_tool_choice = AnthropicMessagesToolChoice(type="any")
|
||||
elif tool_type == "none":
|
||||
_tool_choice = AnthropicMessagesToolChoice(type="none")
|
||||
else:
|
||||
_tool_name = tool_choice.get("function", {}).get("name")
|
||||
if _tool_name is not None:
|
||||
_tool_choice = AnthropicMessagesToolChoice(type="tool")
|
||||
_tool_choice["name"] = _tool_name
|
||||
_tool_name = tool_choice.get("function", {}).get("name")
|
||||
_tool_choice = AnthropicMessagesToolChoice(type="tool")
|
||||
if _tool_name is not None:
|
||||
_tool_choice["name"] = _tool_name
|
||||
|
||||
if parallel_tool_use is not None:
|
||||
# Anthropic uses 'disable_parallel_tool_use' flag to determine if parallel tool use is allowed
|
||||
|
|
@ -1378,7 +1369,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
else 0
|
||||
)
|
||||
completion_token_details = CompletionTokensDetailsWrapper(
|
||||
reasoning_tokens=reasoning_tokens if reasoning_tokens > 0 else 0,
|
||||
reasoning_tokens=reasoning_tokens if reasoning_tokens > 0 else None,
|
||||
text_tokens=completion_tokens - reasoning_tokens if reasoning_tokens > 0 else completion_tokens,
|
||||
)
|
||||
total_tokens = prompt_tokens + completion_tokens
|
||||
|
|
|
|||
|
|
@ -168,36 +168,6 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
return provider_specific_fields.get("signature")
|
||||
return None
|
||||
|
||||
def _add_cache_control_if_applicable(
|
||||
self,
|
||||
source: Any,
|
||||
target: Any,
|
||||
model: Optional[str],
|
||||
) -> None:
|
||||
"""
|
||||
Extract cache_control from source and add to target if it should be preserved.
|
||||
|
||||
This method accepts Any type to support both regular dicts and TypedDict objects.
|
||||
TypedDict objects (like ChatCompletionTextObject, ChatCompletionImageObject, etc.)
|
||||
are dicts at runtime but have specific types at type-check time. Using Any allows
|
||||
this method to work with both while maintaining runtime correctness.
|
||||
|
||||
Args:
|
||||
source: Dict or TypedDict containing potential cache_control field
|
||||
target: Dict or TypedDict to add cache_control to
|
||||
model: Model name to check if cache_control should be preserved
|
||||
"""
|
||||
# TypedDict objects are dicts at runtime, so .get() works
|
||||
cache_control = source.get("cache_control") if isinstance(source, dict) else getattr(source, "cache_control", None)
|
||||
if cache_control and model and self.is_anthropic_claude_model(model):
|
||||
# TypedDict objects support dict operations at runtime
|
||||
# Use type ignore consistent with codebase pattern (see anthropic/chat/transformation.py:432)
|
||||
if isinstance(target, dict):
|
||||
target["cache_control"] = cache_control # type: ignore[typeddict-item]
|
||||
else:
|
||||
# Fallback for non-dict objects (shouldn't happen in practice)
|
||||
cast(Dict[str, Any], target)["cache_control"] = cache_control
|
||||
|
||||
def translatable_anthropic_params(self) -> List:
|
||||
"""
|
||||
Which anthropic params, we need to translate to the openai format.
|
||||
|
|
@ -235,8 +205,12 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
text_obj = ChatCompletionTextObject(
|
||||
type="text", text=content.get("text", "")
|
||||
)
|
||||
self._add_cache_control_if_applicable(content, text_obj, model)
|
||||
new_user_content_list.append(text_obj) # type: ignore
|
||||
# Preserve cache_control if present (for prompt caching)
|
||||
# Only for Anthropic models that support prompt caching
|
||||
cache_control = content.get("cache_control")
|
||||
if cache_control and model and self.is_anthropic_claude_model(model):
|
||||
text_obj["cache_control"] = cache_control # type: ignore
|
||||
new_user_content_list.append(text_obj)
|
||||
elif content.get("type") == "image":
|
||||
# Convert Anthropic image format to OpenAI format
|
||||
source = content.get("source", {})
|
||||
|
|
@ -251,24 +225,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
image_obj = ChatCompletionImageObject(
|
||||
type="image_url", image_url=image_url_obj
|
||||
)
|
||||
self._add_cache_control_if_applicable(content, image_obj, model)
|
||||
new_user_content_list.append(image_obj) # type: ignore
|
||||
elif content.get("type") == "document":
|
||||
# Convert Anthropic document format (PDF, etc.) to OpenAI format
|
||||
source = content.get("source", {})
|
||||
openai_image_url = (
|
||||
self._translate_anthropic_image_to_openai(cast(dict, source))
|
||||
)
|
||||
|
||||
if openai_image_url:
|
||||
image_url_obj = ChatCompletionImageUrlObject(
|
||||
url=openai_image_url
|
||||
)
|
||||
doc_obj = ChatCompletionImageObject(
|
||||
type="image_url", image_url=image_url_obj
|
||||
)
|
||||
self._add_cache_control_if_applicable(content, doc_obj, model)
|
||||
new_user_content_list.append(doc_obj) # type: ignore
|
||||
new_user_content_list.append(image_obj)
|
||||
elif content.get("type") == "tool_result":
|
||||
if "content" not in content:
|
||||
tool_result = ChatCompletionToolMessage(
|
||||
|
|
@ -276,16 +233,14 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
tool_call_id=content.get("tool_use_id", ""),
|
||||
content="",
|
||||
)
|
||||
self._add_cache_control_if_applicable(content, tool_result, model)
|
||||
tool_message_list.append(tool_result) # type: ignore[arg-type]
|
||||
tool_message_list.append(tool_result)
|
||||
elif isinstance(content.get("content"), str):
|
||||
tool_result = ChatCompletionToolMessage(
|
||||
role="tool",
|
||||
tool_call_id=content.get("tool_use_id", ""),
|
||||
content=str(content.get("content", "")),
|
||||
)
|
||||
self._add_cache_control_if_applicable(content, tool_result, model)
|
||||
tool_message_list.append(tool_result) # type: ignore[arg-type]
|
||||
tool_message_list.append(tool_result)
|
||||
elif isinstance(content.get("content"), list):
|
||||
# Combine all content items into a single tool message
|
||||
# to avoid creating multiple tool_result blocks with the same ID
|
||||
|
|
@ -301,8 +256,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
tool_call_id=content.get("tool_use_id", ""),
|
||||
content=c,
|
||||
)
|
||||
self._add_cache_control_if_applicable(content, tool_result, model)
|
||||
tool_message_list.append(tool_result) # type: ignore[arg-type]
|
||||
tool_message_list.append(tool_result)
|
||||
elif isinstance(c, dict):
|
||||
if c.get("type") == "text":
|
||||
tool_result = ChatCompletionToolMessage(
|
||||
|
|
@ -312,8 +266,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
),
|
||||
content=c.get("text", ""),
|
||||
)
|
||||
self._add_cache_control_if_applicable(content, tool_result, model)
|
||||
tool_message_list.append(tool_result) # type: ignore[arg-type]
|
||||
tool_message_list.append(tool_result)
|
||||
elif c.get("type") == "image":
|
||||
source = c.get("source", {})
|
||||
openai_image_url = (
|
||||
|
|
@ -329,8 +282,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
),
|
||||
content=openai_image_url,
|
||||
)
|
||||
self._add_cache_control_if_applicable(content, tool_result, model)
|
||||
tool_message_list.append(tool_result) # type: ignore[arg-type]
|
||||
tool_message_list.append(tool_result)
|
||||
else:
|
||||
# For multiple content items, combine into a single tool message
|
||||
# with list content to preserve all items while having one tool_use_id
|
||||
|
|
@ -379,8 +331,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
tool_call_id=content.get("tool_use_id", ""),
|
||||
content=combined_content_parts, # type: ignore
|
||||
)
|
||||
self._add_cache_control_if_applicable(content, tool_result, model)
|
||||
tool_message_list.append(tool_result) # type: ignore[arg-type]
|
||||
tool_message_list.append(tool_result)
|
||||
|
||||
if len(tool_message_list) > 0:
|
||||
new_messages.extend(tool_message_list)
|
||||
|
|
@ -393,8 +344,6 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
|
||||
## ASSISTANT MESSAGE ##
|
||||
assistant_message_str: Optional[str] = None
|
||||
assistant_content_list: List[Dict[str, Any]] = [] # For content blocks with cache_control
|
||||
has_cache_control_in_text = False
|
||||
tool_calls: List[ChatCompletionAssistantToolCall] = []
|
||||
thinking_blocks: List[
|
||||
Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
|
||||
|
|
@ -408,14 +357,10 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
assistant_message_str = str(content)
|
||||
elif isinstance(content, dict):
|
||||
if content.get("type") == "text":
|
||||
text_block: Dict[str, Any] = {
|
||||
"type": "text",
|
||||
"text": content.get("text", ""),
|
||||
}
|
||||
self._add_cache_control_if_applicable(content, text_block, model)
|
||||
if "cache_control" in text_block:
|
||||
has_cache_control_in_text = True
|
||||
assistant_content_list.append(text_block)
|
||||
if assistant_message_str is None:
|
||||
assistant_message_str = content.get("text", "")
|
||||
else:
|
||||
assistant_message_str += content.get("text", "")
|
||||
elif content.get("type") == "tool_use":
|
||||
function_chunk: ChatCompletionToolCallFunctionChunk = {
|
||||
"name": content.get("name", ""),
|
||||
|
|
@ -439,13 +384,13 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
provider_specific_fields
|
||||
)
|
||||
|
||||
tool_call = ChatCompletionAssistantToolCall(
|
||||
id=content.get("id", ""),
|
||||
type="function",
|
||||
function=function_chunk,
|
||||
tool_calls.append(
|
||||
ChatCompletionAssistantToolCall(
|
||||
id=content.get("id", ""),
|
||||
type="function",
|
||||
function=function_chunk,
|
||||
)
|
||||
)
|
||||
self._add_cache_control_if_applicable(content, tool_call, model)
|
||||
tool_calls.append(tool_call)
|
||||
elif content.get("type") == "thinking":
|
||||
thinking_block = ChatCompletionThinkingBlock(
|
||||
type="thinking",
|
||||
|
|
@ -466,30 +411,18 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
|
||||
if (
|
||||
assistant_message_str is not None
|
||||
or len(assistant_content_list) > 0
|
||||
or len(tool_calls) > 0
|
||||
or len(thinking_blocks) > 0
|
||||
):
|
||||
# Use list format if any text block has cache_control, otherwise use string
|
||||
if has_cache_control_in_text and len(assistant_content_list) > 0:
|
||||
assistant_content: Any = assistant_content_list
|
||||
elif len(assistant_content_list) > 0 and not has_cache_control_in_text:
|
||||
# Concatenate text blocks into string when no cache_control
|
||||
assistant_content = "".join(
|
||||
block.get("text", "") for block in assistant_content_list
|
||||
)
|
||||
else:
|
||||
assistant_content = assistant_message_str
|
||||
|
||||
assistant_message = ChatCompletionAssistantMessage(
|
||||
role="assistant",
|
||||
content=assistant_content,
|
||||
content=assistant_message_str,
|
||||
thinking_blocks=(
|
||||
thinking_blocks if len(thinking_blocks) > 0 else None
|
||||
),
|
||||
)
|
||||
if len(tool_calls) > 0:
|
||||
assistant_message["tool_calls"] = tool_calls # type: ignore
|
||||
assistant_message["tool_calls"] = tool_calls
|
||||
if len(thinking_blocks) > 0:
|
||||
assistant_message["thinking_blocks"] = thinking_blocks # type: ignore
|
||||
new_messages.append(assistant_message)
|
||||
|
|
@ -599,10 +532,10 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
)
|
||||
|
||||
def translate_anthropic_tools_to_openai(
|
||||
self, tools: List[AllAnthropicToolsValues], model: Optional[str] = None
|
||||
self, tools: List[AllAnthropicToolsValues]
|
||||
) -> List[ChatCompletionToolParam]:
|
||||
new_tools: List[ChatCompletionToolParam] = []
|
||||
mapped_tool_params = ["name", "input_schema", "description", "cache_control"]
|
||||
mapped_tool_params = ["name", "input_schema", "description"]
|
||||
for tool in tools:
|
||||
function_chunk = ChatCompletionToolParamFunctionChunk(
|
||||
name=tool["name"],
|
||||
|
|
@ -615,11 +548,11 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
for k, v in tool.items():
|
||||
if k not in mapped_tool_params: # pass additional computer kwargs
|
||||
function_chunk.setdefault("parameters", {}).update({k: v})
|
||||
tool_param = ChatCompletionToolParam(type="function", function=function_chunk)
|
||||
self._add_cache_control_if_applicable(tool, tool_param, model)
|
||||
new_tools.append(tool_param) # type: ignore[arg-type]
|
||||
new_tools.append(
|
||||
ChatCompletionToolParam(type="function", function=function_chunk)
|
||||
)
|
||||
|
||||
return new_tools # type: ignore[return-value]
|
||||
return new_tools
|
||||
|
||||
def translate_anthropic_output_format_to_openai(
|
||||
self, output_format: Any
|
||||
|
|
@ -657,41 +590,6 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
},
|
||||
}
|
||||
|
||||
def _add_system_message_to_messages(
|
||||
self,
|
||||
new_messages: List[AllMessageValues],
|
||||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
) -> None:
|
||||
"""Add system message to messages list if present in request."""
|
||||
if "system" not in anthropic_message_request:
|
||||
return
|
||||
system_content = anthropic_message_request["system"]
|
||||
if not system_content:
|
||||
return
|
||||
# Handle system as string or array of content blocks
|
||||
if isinstance(system_content, str):
|
||||
new_messages.insert(
|
||||
0,
|
||||
ChatCompletionSystemMessage(role="system", content=system_content),
|
||||
)
|
||||
elif isinstance(system_content, list):
|
||||
# Convert Anthropic system content blocks to OpenAI format
|
||||
openai_system_content: List[Dict[str, Any]] = []
|
||||
model_name = anthropic_message_request.get("model", "")
|
||||
for block in system_content:
|
||||
if isinstance(block, dict) and block.get("type") == "text":
|
||||
text_block: Dict[str, Any] = {
|
||||
"type": "text",
|
||||
"text": block.get("text", ""),
|
||||
}
|
||||
self._add_cache_control_if_applicable(block, text_block, model_name)
|
||||
openai_system_content.append(text_block)
|
||||
if openai_system_content:
|
||||
new_messages.insert(
|
||||
0,
|
||||
ChatCompletionSystemMessage(role="system", content=openai_system_content), # type: ignore
|
||||
)
|
||||
|
||||
def translate_anthropic_to_openai(
|
||||
self, anthropic_message_request: AnthropicMessagesRequest
|
||||
) -> ChatCompletionRequest:
|
||||
|
|
@ -720,7 +618,13 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
model=anthropic_message_request.get("model"),
|
||||
)
|
||||
## ADD SYSTEM MESSAGE TO MESSAGES
|
||||
self._add_system_message_to_messages(new_messages, anthropic_message_request)
|
||||
if "system" in anthropic_message_request:
|
||||
system_content = anthropic_message_request["system"]
|
||||
if system_content:
|
||||
new_messages.insert(
|
||||
0,
|
||||
ChatCompletionSystemMessage(role="system", content=system_content),
|
||||
)
|
||||
|
||||
new_kwargs: ChatCompletionRequest = {
|
||||
"model": anthropic_message_request["model"],
|
||||
|
|
@ -751,8 +655,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
tools = anthropic_message_request["tools"]
|
||||
if tools:
|
||||
new_kwargs["tools"] = self.translate_anthropic_tools_to_openai(
|
||||
tools=cast(List[AllAnthropicToolsValues], tools),
|
||||
model=new_kwargs.get("model"),
|
||||
tools=cast(List[AllAnthropicToolsValues], tools)
|
||||
)
|
||||
|
||||
## CONVERT THINKING
|
||||
|
|
@ -940,7 +843,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
role="assistant",
|
||||
model=response.model or "unknown-model",
|
||||
stop_sequence=None,
|
||||
usage=anthropic_usage, # type: ignore
|
||||
usage=anthropic_usage,
|
||||
content=anthropic_content, # type: ignore
|
||||
stop_reason=anthropic_finish_reason,
|
||||
)
|
||||
|
|
@ -1089,7 +992,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
else:
|
||||
usage_delta = UsageDelta(input_tokens=0, output_tokens=0)
|
||||
return MessageBlockDelta(
|
||||
type="message_delta", delta=delta, usage=usage_delta # type: ignore
|
||||
type="message_delta", delta=delta, usage=usage_delta
|
||||
)
|
||||
(
|
||||
type_of_content,
|
||||
|
|
|
|||
|
|
@ -22,8 +22,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
|
|||
Accepts both explicit gpt-5 model names and the ``gpt5_series/`` prefix
|
||||
used for manual routing.
|
||||
"""
|
||||
# gpt-5-chat* is a chat model and shouldn't go through GPT-5 reasoning restrictions.
|
||||
return ("gpt-5" in model and "gpt-5-chat" not in model) or "gpt5_series" in model
|
||||
return "gpt-5" in model or "gpt5_series" in model
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> List[str]:
|
||||
"""Get supported parameters for Azure OpenAI GPT-5 models.
|
||||
|
|
@ -38,11 +37,6 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
|
|||
"""
|
||||
params = OpenAIGPT5Config.get_supported_openai_params(self, model=model)
|
||||
|
||||
# Azure supports tool_choice for GPT-5 deployments, but the base GPT-5 config
|
||||
# can drop it when the deployment name isn't in the OpenAI model registry.
|
||||
if "tool_choice" not in params:
|
||||
params.append("tool_choice")
|
||||
|
||||
# Only gpt-5.2 has been verified to support logprobs on Azure
|
||||
if self.is_model_gpt_5_2_model(model):
|
||||
azure_supported_params = ["logprobs", "top_logprobs"]
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
"""
|
||||
Helper util for handling azure openai-specific cost calculation
|
||||
- e.g.: prompt caching, audio tokens
|
||||
- e.g.: prompt caching
|
||||
"""
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.types.utils import Usage
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
|
|
@ -19,15 +18,34 @@ def cost_per_token(
|
|||
|
||||
Input:
|
||||
- model: str, the model name without provider prefix
|
||||
- usage: LiteLLM Usage block, containing caching and audio token information
|
||||
- usage: LiteLLM Usage block, containing anthropic caching information
|
||||
|
||||
Returns:
|
||||
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
|
||||
"""
|
||||
## GET MODEL INFO
|
||||
model_info = get_model_info(model=model, custom_llm_provider="azure")
|
||||
cached_tokens: Optional[int] = None
|
||||
## CALCULATE INPUT COST
|
||||
non_cached_text_tokens = usage.prompt_tokens
|
||||
if usage.prompt_tokens_details and usage.prompt_tokens_details.cached_tokens:
|
||||
cached_tokens = usage.prompt_tokens_details.cached_tokens
|
||||
non_cached_text_tokens = non_cached_text_tokens - cached_tokens
|
||||
prompt_cost: float = non_cached_text_tokens * model_info["input_cost_per_token"]
|
||||
|
||||
## Speech / Audio cost calculation (cost per second for TTS models)
|
||||
## CALCULATE OUTPUT COST
|
||||
completion_cost: float = (
|
||||
usage["completion_tokens"] * model_info["output_cost_per_token"]
|
||||
)
|
||||
|
||||
## Prompt Caching cost calculation
|
||||
if model_info.get("cache_read_input_token_cost") is not None and cached_tokens:
|
||||
# Note: We read ._cache_read_input_tokens from the Usage - since cost_calculator.py standardizes the cache read tokens on usage._cache_read_input_tokens
|
||||
prompt_cost += cached_tokens * (
|
||||
model_info.get("cache_read_input_token_cost", 0) or 0
|
||||
)
|
||||
|
||||
## Speech / Audio cost calculation
|
||||
if (
|
||||
"output_cost_per_second" in model_info
|
||||
and model_info["output_cost_per_second"] is not None
|
||||
|
|
@ -37,14 +55,7 @@ def cost_per_token(
|
|||
f"For model={model} - output_cost_per_second: {model_info.get('output_cost_per_second')}; response time: {response_time_ms}"
|
||||
)
|
||||
## COST PER SECOND ##
|
||||
prompt_cost = 0.0
|
||||
prompt_cost = 0
|
||||
completion_cost = model_info["output_cost_per_second"] * response_time_ms / 1000
|
||||
return prompt_cost, completion_cost
|
||||
|
||||
## Use generic cost calculator for all other cases
|
||||
## This properly handles: text tokens, audio tokens, cached tokens, reasoning tokens, etc.
|
||||
return generic_cost_per_token(
|
||||
model=model,
|
||||
usage=usage,
|
||||
custom_llm_provider="azure",
|
||||
)
|
||||
return prompt_cost, completion_cost
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ import httpx
|
|||
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.vector_stores import (
|
||||
VECTOR_STORE_OPENAI_PARAMS,
|
||||
BaseVectorStoreAuthCredentials,
|
||||
VECTOR_STORE_OPENAI_PARAMS,
|
||||
VectorStoreCreateOptionalRequestParams,
|
||||
VectorStoreCreateResponse,
|
||||
VectorStoreIndexEndpoints,
|
||||
|
|
@ -64,30 +64,6 @@ class BaseVectorStoreConfig:
|
|||
|
||||
pass
|
||||
|
||||
async def atransform_search_vector_store_request(
|
||||
self,
|
||||
vector_store_id: str,
|
||||
query: Union[str, List[str]],
|
||||
vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
|
||||
api_base: str,
|
||||
litellm_logging_obj: LiteLLMLoggingObj,
|
||||
litellm_params: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Optional async version of transform_search_vector_store_request.
|
||||
If not implemented, the handler will fall back to the sync version.
|
||||
Providers that need to make async calls (e.g., generating embeddings) should override this.
|
||||
"""
|
||||
# Default implementation: call the sync version
|
||||
return self.transform_search_vector_store_request(
|
||||
vector_store_id=vector_store_id,
|
||||
query=query,
|
||||
vector_store_search_optional_params=vector_store_search_optional_params,
|
||||
api_base=api_base,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def transform_search_vector_store_response(
|
||||
self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj
|
||||
|
|
|
|||
|
|
@ -1163,7 +1163,7 @@ class BaseAWSLLM:
|
|||
|
||||
def _sign_request(
|
||||
self,
|
||||
service_name: Literal["bedrock", "sagemaker", "bedrock-agentcore", "s3vectors"],
|
||||
service_name: Literal["bedrock", "sagemaker", "bedrock-agentcore"],
|
||||
headers: dict,
|
||||
optional_params: dict,
|
||||
request_data: dict,
|
||||
|
|
|
|||
|
|
@ -298,39 +298,6 @@ class AmazonConverseConfig(BaseConfig):
|
|||
# Check if the model is specifically Nova Lite 2
|
||||
return "nova-2-lite" in model_without_region
|
||||
|
||||
def _map_web_search_options(
|
||||
self,
|
||||
web_search_options: dict,
|
||||
model: str
|
||||
) -> Optional[BedrockToolBlock]:
|
||||
"""
|
||||
Map web_search_options to Nova grounding systemTool.
|
||||
|
||||
Nova grounding (web search) is only supported on Amazon Nova models.
|
||||
Returns None for non-Nova models.
|
||||
|
||||
Args:
|
||||
web_search_options: The web_search_options dict from the request
|
||||
model: The model identifier string
|
||||
|
||||
Returns:
|
||||
BedrockToolBlock with systemTool for Nova models, None otherwise
|
||||
|
||||
Reference: https://docs.aws.amazon.com/nova/latest/userguide/grounding.html
|
||||
"""
|
||||
# Only Nova models support nova_grounding
|
||||
# Model strings can be like: "amazon.nova-pro-v1:0", "us.amazon.nova-pro-v1:0", etc.
|
||||
if "nova" not in model.lower():
|
||||
verbose_logger.debug(
|
||||
f"web_search_options passed but model {model} is not a Nova model. "
|
||||
"Nova grounding is only supported on Amazon Nova models."
|
||||
)
|
||||
return None
|
||||
|
||||
# Nova doesn't support search_context_size or user_location params
|
||||
# (unlike Anthropic), so we just enable grounding with no options
|
||||
return BedrockToolBlock(systemTool={"name": "nova_grounding"})
|
||||
|
||||
def _transform_reasoning_effort_to_reasoning_config(
|
||||
self, reasoning_effort: str
|
||||
) -> dict:
|
||||
|
|
@ -471,10 +438,6 @@ class AmazonConverseConfig(BaseConfig):
|
|||
):
|
||||
supported_params.append("tools")
|
||||
|
||||
# Nova models support web_search_options (mapped to nova_grounding systemTool)
|
||||
if base_model.startswith("amazon.nova"):
|
||||
supported_params.append("web_search_options")
|
||||
|
||||
if litellm.utils.supports_tool_choice(
|
||||
model=model, custom_llm_provider=self.custom_llm_provider
|
||||
) or litellm.utils.supports_tool_choice(
|
||||
|
|
@ -767,13 +730,6 @@ class AmazonConverseConfig(BaseConfig):
|
|||
if bedrock_tier in ("default", "flex", "priority"):
|
||||
optional_params["serviceTier"] = {"type": bedrock_tier}
|
||||
|
||||
if param == "web_search_options" and value and isinstance(value, dict):
|
||||
grounding_tool = self._map_web_search_options(value, model)
|
||||
if grounding_tool is not None:
|
||||
optional_params = self._add_tools_to_optional_params(
|
||||
optional_params=optional_params, tools=[grounding_tool]
|
||||
)
|
||||
|
||||
# Only update thinking tokens for non-GPT-OSS models and non-Nova-Lite-2 models
|
||||
# Nova Lite 2 handles token budgeting differently through reasoningConfig
|
||||
if "gpt-oss" not in model and not self._is_nova_lite_2_model(model):
|
||||
|
|
@ -1432,23 +1388,20 @@ class AmazonConverseConfig(BaseConfig):
|
|||
str,
|
||||
List[ChatCompletionToolCallChunk],
|
||||
Optional[List[BedrockConverseReasoningContentBlock]],
|
||||
Optional[List[CitationsContentBlock]],
|
||||
]:
|
||||
"""
|
||||
Translate the message content to a string and a list of tool calls, reasoning content blocks, and citations.
|
||||
Translate the message content to a string and a list of tool calls and reasoning content blocks
|
||||
|
||||
Returns:
|
||||
content_str: str
|
||||
tools: List[ChatCompletionToolCallChunk]
|
||||
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]]
|
||||
citationsContentBlocks: Optional[List[CitationsContentBlock]] - Citations from Nova grounding
|
||||
"""
|
||||
content_str = ""
|
||||
tools: List[ChatCompletionToolCallChunk] = []
|
||||
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
|
||||
None
|
||||
)
|
||||
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
|
||||
for idx, content in enumerate(content_blocks):
|
||||
"""
|
||||
- Content is either a tool response or text
|
||||
|
|
@ -1493,15 +1446,10 @@ class AmazonConverseConfig(BaseConfig):
|
|||
if reasoningContentBlocks is None:
|
||||
reasoningContentBlocks = []
|
||||
reasoningContentBlocks.append(content["reasoningContent"])
|
||||
# Handle Nova grounding citations content
|
||||
if "citationsContent" in content:
|
||||
if citationsContentBlocks is None:
|
||||
citationsContentBlocks = []
|
||||
citationsContentBlocks.append(content["citationsContent"])
|
||||
|
||||
return content_str, tools, reasoningContentBlocks, citationsContentBlocks
|
||||
return content_str, tools, reasoningContentBlocks
|
||||
|
||||
def _transform_response( # noqa: PLR0915
|
||||
def _transform_response(
|
||||
self,
|
||||
model: str,
|
||||
response: httpx.Response,
|
||||
|
|
@ -1577,27 +1525,18 @@ class AmazonConverseConfig(BaseConfig):
|
|||
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
|
||||
None
|
||||
)
|
||||
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
|
||||
|
||||
if message is not None:
|
||||
(
|
||||
content_str,
|
||||
tools,
|
||||
reasoningContentBlocks,
|
||||
citationsContentBlocks,
|
||||
) = self._translate_message_content(message["content"])
|
||||
|
||||
# Initialize provider_specific_fields if we have any special content blocks
|
||||
provider_specific_fields: dict = {}
|
||||
if reasoningContentBlocks is not None:
|
||||
provider_specific_fields["reasoningContentBlocks"] = reasoningContentBlocks
|
||||
if citationsContentBlocks is not None:
|
||||
provider_specific_fields["citationsContent"] = citationsContentBlocks
|
||||
|
||||
if provider_specific_fields:
|
||||
chat_completion_message["provider_specific_fields"] = provider_specific_fields
|
||||
|
||||
if reasoningContentBlocks is not None:
|
||||
chat_completion_message["provider_specific_fields"] = {
|
||||
"reasoningContentBlocks": reasoningContentBlocks,
|
||||
}
|
||||
chat_completion_message["reasoning_content"] = (
|
||||
self._transform_reasoning_content(reasoningContentBlocks)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1476,11 +1476,6 @@ class AWSEventStreamDecoder:
|
|||
reasoning_content = (
|
||||
"" # set to non-empty string to ensure consistency with Anthropic
|
||||
)
|
||||
elif "citationsContent" in delta_obj:
|
||||
# Handle Nova grounding citations in streaming responses
|
||||
provider_specific_fields = {
|
||||
"citationsContent": delta_obj["citationsContent"],
|
||||
}
|
||||
return (
|
||||
text,
|
||||
tool_use,
|
||||
|
|
|
|||
|
|
@ -53,26 +53,13 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
# Force tool-based structured outputs for Bedrock Invoke
|
||||
# (similar to VertexAI fix in #19201)
|
||||
# Bedrock Invoke doesn't support output_format parameter
|
||||
original_model = model
|
||||
if "response_format" in non_default_params:
|
||||
# Use a model name that forces tool-based approach
|
||||
model = "claude-3-sonnet-20240229"
|
||||
|
||||
optional_params = AnthropicConfig.map_openai_params(
|
||||
return AnthropicConfig.map_openai_params(
|
||||
self,
|
||||
non_default_params,
|
||||
optional_params,
|
||||
model,
|
||||
drop_params,
|
||||
)
|
||||
|
||||
# Restore original model name
|
||||
model = original_model
|
||||
|
||||
return optional_params
|
||||
|
||||
|
||||
def transform_request(
|
||||
|
|
@ -103,8 +90,6 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
|
||||
_anthropic_request.pop("model", None)
|
||||
_anthropic_request.pop("stream", None)
|
||||
# Bedrock Invoke doesn't support output_format parameter
|
||||
_anthropic_request.pop("output_format", None)
|
||||
if "anthropic_version" not in _anthropic_request:
|
||||
_anthropic_request["anthropic_version"] = self.anthropic_version
|
||||
|
||||
|
|
@ -132,26 +117,6 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
if "opus-4" in model.lower() or "opus_4" in model.lower():
|
||||
beta_set.add("tool-search-tool-2025-10-19")
|
||||
|
||||
# Filter out beta headers that Bedrock Invoke doesn't support
|
||||
# AWS Bedrock only supports a specific whitelist of beta flags
|
||||
# Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html
|
||||
BEDROCK_SUPPORTED_BETAS = {
|
||||
"computer-use-2024-10-22", # Legacy computer use
|
||||
"computer-use-2025-01-24", # Current computer use (Claude 3.7 Sonnet)
|
||||
"token-efficient-tools-2025-02-19", # Tool use (Claude 3.7+ and Claude 4+)
|
||||
"interleaved-thinking-2025-05-14", # Interleaved thinking (Claude 4+)
|
||||
"output-128k-2025-02-19", # 128K output tokens (Claude 3.7 Sonnet)
|
||||
"dev-full-thinking-2025-05-14", # Developer mode for raw thinking (Claude 4+)
|
||||
"context-1m-2025-08-07", # 1 million tokens (Claude Sonnet 4)
|
||||
"context-management-2025-06-27", # Context management (Claude Sonnet/Haiku 4.5)
|
||||
"effort-2025-11-24", # Effort parameter (Claude Opus 4.5)
|
||||
"tool-search-tool-2025-10-19", # Tool search (Claude Opus 4.5)
|
||||
"tool-examples-2025-10-29", # Tool use examples (Claude Opus 4.5)
|
||||
}
|
||||
|
||||
# Only keep beta headers that Bedrock supports
|
||||
beta_set = {beta for beta in beta_set if beta in BEDROCK_SUPPORTED_BETAS}
|
||||
|
||||
if beta_set:
|
||||
_anthropic_request["anthropic_beta"] = list(beta_set)
|
||||
|
||||
|
|
|
|||
|
|
@ -797,7 +797,7 @@ class BedrockEventStreamDecoderBase:
|
|||
def get_anthropic_beta_from_headers(headers: dict) -> List[str]:
|
||||
"""
|
||||
Extract anthropic-beta header values and convert them to a list.
|
||||
Supports both JSON array format and comma-separated values from user headers.
|
||||
Supports comma-separated values from user headers.
|
||||
|
||||
Used by both converse and invoke transformations for consistent handling
|
||||
of anthropic-beta headers that should be passed to AWS Bedrock.
|
||||
|
|
@ -812,25 +812,8 @@ def get_anthropic_beta_from_headers(headers: dict) -> List[str]:
|
|||
if not anthropic_beta_header:
|
||||
return []
|
||||
|
||||
# If it's already a list, return it
|
||||
if isinstance(anthropic_beta_header, list):
|
||||
return anthropic_beta_header
|
||||
|
||||
# Try to parse as JSON array first (e.g., '["interleaved-thinking-2025-05-14", "claude-code-20250219"]')
|
||||
if isinstance(anthropic_beta_header, str):
|
||||
anthropic_beta_header = anthropic_beta_header.strip()
|
||||
if anthropic_beta_header.startswith("[") and anthropic_beta_header.endswith("]"):
|
||||
try:
|
||||
parsed = json.loads(anthropic_beta_header)
|
||||
if isinstance(parsed, list):
|
||||
return [str(beta).strip() for beta in parsed]
|
||||
except json.JSONDecodeError:
|
||||
pass # Fall through to comma-separated parsing
|
||||
|
||||
# Fall back to comma-separated values
|
||||
return [beta.strip() for beta in anthropic_beta_header.split(",")]
|
||||
|
||||
return []
|
||||
# Split comma-separated values and strip whitespace
|
||||
return [beta.strip() for beta in anthropic_beta_header.split(",")]
|
||||
|
||||
|
||||
class CommonBatchFilesUtils:
|
||||
|
|
|
|||
|
|
@ -162,49 +162,6 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
|
||||
return any(pattern in model_lower for pattern in supported_patterns)
|
||||
|
||||
def _is_claude_opus_4_5(self, model: str) -> bool:
|
||||
"""
|
||||
Check if the model is Claude Opus 4.5.
|
||||
|
||||
Args:
|
||||
model: The model name
|
||||
|
||||
Returns:
|
||||
True if the model is Claude Opus 4.5
|
||||
"""
|
||||
model_lower = model.lower()
|
||||
opus_4_5_patterns = [
|
||||
"opus-4.5", "opus_4.5", "opus-4-5", "opus_4_5",
|
||||
]
|
||||
return any(pattern in model_lower for pattern in opus_4_5_patterns)
|
||||
|
||||
def _supports_tool_search_on_bedrock(self, model: str) -> bool:
|
||||
"""
|
||||
Check if the model supports tool search on Bedrock.
|
||||
|
||||
On Amazon Bedrock, server-side tool search is supported on Claude Opus 4.5
|
||||
and Claude Sonnet 4.5 with the tool-search-tool-2025-10-19 beta header.
|
||||
|
||||
Ref: https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-search-tool
|
||||
|
||||
Args:
|
||||
model: The model name
|
||||
|
||||
Returns:
|
||||
True if the model supports tool search on Bedrock
|
||||
"""
|
||||
model_lower = model.lower()
|
||||
|
||||
# Supported models for tool search on Bedrock
|
||||
supported_patterns = [
|
||||
# Opus 4.5
|
||||
"opus-4.5", "opus_4.5", "opus-4-5", "opus_4_5",
|
||||
# Sonnet 4.5
|
||||
"sonnet-4.5", "sonnet_4.5", "sonnet-4-5", "sonnet_4_5",
|
||||
]
|
||||
|
||||
return any(pattern in model_lower for pattern in supported_patterns)
|
||||
|
||||
def _filter_unsupported_beta_headers_for_bedrock(
|
||||
self, model: str, beta_set: set
|
||||
) -> None:
|
||||
|
|
@ -212,33 +169,25 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
Remove beta headers that are not supported on Bedrock for the given model.
|
||||
|
||||
Extended thinking beta headers are only supported on specific Claude 4+ models.
|
||||
Advanced tool use headers are not supported on Bedrock Invoke API, but need to be
|
||||
translated to Bedrock-specific headers for models that support tool search
|
||||
(Claude Opus 4.5, Sonnet 4.5).
|
||||
Advanced tool use headers are not supported on Bedrock Invoke API.
|
||||
This prevents 400 "invalid beta flag" errors on Bedrock.
|
||||
|
||||
Note: Bedrock Invoke API fails with a 400 error when unsupported beta headers
|
||||
are sent, returning: {"message":"invalid beta flag"}
|
||||
|
||||
Translation for models supporting tool search (Opus 4.5, Sonnet 4.5):
|
||||
- advanced-tool-use-2025-11-20 -> tool-search-tool-2025-10-19 + tool-examples-2025-10-29
|
||||
|
||||
Args:
|
||||
model: The model name
|
||||
beta_set: The set of beta headers to filter in-place
|
||||
"""
|
||||
beta_headers_to_remove = set()
|
||||
has_advanced_tool_use = False
|
||||
|
||||
# 1. Filter out beta headers that are universally unsupported on Bedrock Invoke and track if advanced-tool-use header is present
|
||||
# 1. Filter out beta headers that are universally unsupported on Bedrock Invoke
|
||||
for beta in beta_set:
|
||||
for unsupported_pattern in self.UNSUPPORTED_BEDROCK_INVOKE_BETA_PATTERNS:
|
||||
if unsupported_pattern in beta.lower():
|
||||
beta_headers_to_remove.add(beta)
|
||||
has_advanced_tool_use = True
|
||||
break
|
||||
|
||||
|
||||
|
||||
# 2. Filter out extended thinking headers for models that don't support them
|
||||
extended_thinking_patterns = [
|
||||
"extended-thinking",
|
||||
|
|
@ -255,14 +204,6 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
for beta in beta_headers_to_remove:
|
||||
beta_set.discard(beta)
|
||||
|
||||
# 3. Translate advanced-tool-use to Bedrock-specific headers for models that support tool search
|
||||
# Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html
|
||||
# Ref: https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-search-tool
|
||||
if has_advanced_tool_use and self._supports_tool_search_on_bedrock(model):
|
||||
beta_set.add("tool-search-tool-2025-10-19")
|
||||
beta_set.add("tool-examples-2025-10-29")
|
||||
|
||||
|
||||
def _get_tool_search_beta_header_for_bedrock(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -315,7 +256,7 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
Ref: https://aws.amazon.com/blogs/machine-learning/structured-data-response-with-amazon-bedrock-prompt-engineering-and-tool-use/
|
||||
"""
|
||||
import json
|
||||
|
||||
|
||||
# Extract schema from output_format
|
||||
schema = output_format.get("schema")
|
||||
if not schema:
|
||||
|
|
|
|||
|
|
@ -7033,31 +7033,17 @@ class BaseLLMHTTPHandler:
|
|||
litellm_params=dict(litellm_params),
|
||||
)
|
||||
|
||||
# Check if provider has async transform method
|
||||
if hasattr(vector_store_provider_config, "atransform_search_vector_store_request"):
|
||||
(
|
||||
url,
|
||||
request_body,
|
||||
) = await vector_store_provider_config.atransform_search_vector_store_request(
|
||||
vector_store_id=vector_store_id,
|
||||
query=query,
|
||||
vector_store_search_optional_params=vector_store_search_optional_params,
|
||||
api_base=api_base,
|
||||
litellm_logging_obj=logging_obj,
|
||||
litellm_params=dict(litellm_params),
|
||||
)
|
||||
else:
|
||||
(
|
||||
url,
|
||||
request_body,
|
||||
) = vector_store_provider_config.transform_search_vector_store_request(
|
||||
vector_store_id=vector_store_id,
|
||||
query=query,
|
||||
vector_store_search_optional_params=vector_store_search_optional_params,
|
||||
api_base=api_base,
|
||||
litellm_logging_obj=logging_obj,
|
||||
litellm_params=dict(litellm_params),
|
||||
)
|
||||
(
|
||||
url,
|
||||
request_body,
|
||||
) = vector_store_provider_config.transform_search_vector_store_request(
|
||||
vector_store_id=vector_store_id,
|
||||
query=query,
|
||||
vector_store_search_optional_params=vector_store_search_optional_params,
|
||||
api_base=api_base,
|
||||
litellm_logging_obj=logging_obj,
|
||||
litellm_params=dict(litellm_params),
|
||||
)
|
||||
all_optional_params: Dict[str, Any] = dict(litellm_params)
|
||||
all_optional_params.update(vector_store_search_optional_params or {})
|
||||
headers, signed_json_body = vector_store_provider_config.sign_request(
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
|
|||
"parallel_tool_calls",
|
||||
"web_search_options",
|
||||
]
|
||||
if supports_reasoning(model, custom_llm_provider="gemini"):
|
||||
if supports_reasoning(model):
|
||||
supported_params.append("reasoning_effort")
|
||||
supported_params.append("thinking")
|
||||
if self.is_model_gemini_audio_model(model):
|
||||
|
|
|
|||
|
|
@ -31,16 +31,6 @@ else:
|
|||
GIGACHAT_BASE_URL = "https://gigachat.devices.sberbank.ru/api/v1"
|
||||
|
||||
|
||||
def is_valid_json(value: str) -> bool:
|
||||
"""Checks whether the value passed is a valid serialized JSON string"""
|
||||
try:
|
||||
json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
|
||||
class GigaChatError(BaseLLMException):
|
||||
"""GigaChat API error."""
|
||||
|
||||
|
|
@ -111,11 +101,7 @@ class GigaChatConfig(BaseConfig):
|
|||
Set up headers with OAuth token.
|
||||
"""
|
||||
# Get access token
|
||||
credentials = (
|
||||
api_key
|
||||
or get_secret_str("GIGACHAT_CREDENTIALS")
|
||||
or get_secret_str("GIGACHAT_API_KEY")
|
||||
)
|
||||
credentials = api_key or get_secret_str("GIGACHAT_CREDENTIALS") or get_secret_str("GIGACHAT_API_KEY")
|
||||
access_token = get_access_token(credentials=credentials)
|
||||
|
||||
# Store credentials for image uploads
|
||||
|
|
@ -207,13 +193,11 @@ class GigaChatConfig(BaseConfig):
|
|||
for tool in tools:
|
||||
if tool.get("type") == "function":
|
||||
func = tool.get("function", {})
|
||||
functions.append(
|
||||
{
|
||||
"name": func.get("name", ""),
|
||||
"description": func.get("description", ""),
|
||||
"parameters": func.get("parameters", {}),
|
||||
}
|
||||
)
|
||||
functions.append({
|
||||
"name": func.get("name", ""),
|
||||
"description": func.get("description", ""),
|
||||
"parameters": func.get("parameters", {}),
|
||||
})
|
||||
return functions
|
||||
|
||||
def _map_tool_choice(
|
||||
|
|
@ -297,14 +281,8 @@ class GigaChatConfig(BaseConfig):
|
|||
}
|
||||
|
||||
# Add optional params
|
||||
for key in [
|
||||
"temperature",
|
||||
"top_p",
|
||||
"max_tokens",
|
||||
"stream",
|
||||
"repetition_penalty",
|
||||
"profanity_check",
|
||||
]:
|
||||
for key in ["temperature", "top_p", "max_tokens", "stream",
|
||||
"repetition_penalty", "profanity_check"]:
|
||||
if key in optional_params:
|
||||
request_data[key] = optional_params[key]
|
||||
|
||||
|
|
@ -336,7 +314,7 @@ class GigaChatConfig(BaseConfig):
|
|||
elif role == "tool":
|
||||
message["role"] = "function"
|
||||
content = message.get("content", "")
|
||||
if not isinstance(content, str) or not is_valid_json(content):
|
||||
if not isinstance(content, str):
|
||||
message["content"] = json.dumps(content, ensure_ascii=False)
|
||||
|
||||
# Handle None content
|
||||
|
|
@ -463,16 +441,14 @@ class GigaChatConfig(BaseConfig):
|
|||
# Convert to tool_calls format
|
||||
if isinstance(args, dict):
|
||||
args = json.dumps(args, ensure_ascii=False)
|
||||
message_data["tool_calls"] = [
|
||||
{
|
||||
"id": f"call_{uuid.uuid4().hex[:24]}",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": func_call.get("name", ""),
|
||||
"arguments": args,
|
||||
},
|
||||
message_data["tool_calls"] = [{
|
||||
"id": f"call_{uuid.uuid4().hex[:24]}",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": func_call.get("name", ""),
|
||||
"arguments": args,
|
||||
}
|
||||
]
|
||||
}]
|
||||
message_data.pop("function_call", None)
|
||||
finish_reason = "tool_calls"
|
||||
|
||||
|
|
|
|||
|
|
@ -323,12 +323,4 @@ class GroqChatCompletionStreamingHandler(OpenAIChatCompletionStreamingHandler):
|
|||
status_code=error.get("code"), message=error.get("message"), body=error
|
||||
)
|
||||
|
||||
# Map Groq's 'reasoning' field to LiteLLM's 'reasoning_content' field
|
||||
# Groq returns delta.reasoning, but LiteLLM expects delta.reasoning_content
|
||||
choices = chunk.get("choices", [])
|
||||
for choice in choices:
|
||||
delta = choice.get("delta", {})
|
||||
if "reasoning" in delta:
|
||||
delta["reasoning_content"] = delta.pop("reasoning")
|
||||
|
||||
return super().chunk_parser(chunk)
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
|||
class HostedVLLMChatConfig(OpenAIGPTConfig):
|
||||
def get_supported_openai_params(self, model: str) -> List[str]:
|
||||
params = super().get_supported_openai_params(model)
|
||||
params.extend(["reasoning_effort", "thinking"])
|
||||
params.append("reasoning_effort")
|
||||
return params
|
||||
|
||||
def map_openai_params(
|
||||
|
|
@ -41,27 +41,6 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
|
|||
_tools = _remove_strict_from_schema(_tools)
|
||||
if _tools is not None:
|
||||
non_default_params["tools"] = _tools
|
||||
|
||||
# Handle thinking parameter - convert Anthropic-style to OpenAI-style reasoning_effort
|
||||
# vLLM is OpenAI-compatible, so it understands reasoning_effort, not thinking
|
||||
# Reference: https://github.com/BerriAI/litellm/issues/19761
|
||||
thinking = non_default_params.pop("thinking", None)
|
||||
if thinking is not None and isinstance(thinking, dict):
|
||||
if thinking.get("type") == "enabled":
|
||||
# Only convert if reasoning_effort not already set
|
||||
if "reasoning_effort" not in non_default_params:
|
||||
budget_tokens = thinking.get("budget_tokens", 0)
|
||||
# Map budget_tokens to reasoning_effort level
|
||||
# Same logic as Anthropic adapter (translate_anthropic_thinking_to_reasoning_effort)
|
||||
if budget_tokens >= 10000:
|
||||
non_default_params["reasoning_effort"] = "high"
|
||||
elif budget_tokens >= 5000:
|
||||
non_default_params["reasoning_effort"] = "medium"
|
||||
elif budget_tokens >= 2000:
|
||||
non_default_params["reasoning_effort"] = "low"
|
||||
else:
|
||||
non_default_params["reasoning_effort"] = "minimal"
|
||||
|
||||
return super().map_openai_params(
|
||||
non_default_params, optional_params, model, drop_params
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
"""
|
||||
MiniMax OpenAI transformation config - extends OpenAI chat config for MiniMax's OpenAI-compatible API
|
||||
"""
|
||||
from typing import List, Optional, Tuple
|
||||
from typing import Optional
|
||||
|
||||
import litellm
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
|
||||
|
||||
|
||||
class MinimaxChatConfig(OpenAIGPTConfig):
|
||||
|
|
@ -74,33 +73,11 @@ class MinimaxChatConfig(OpenAIGPTConfig):
|
|||
else:
|
||||
return f"{base_url}/v1/chat/completions"
|
||||
|
||||
def remove_cache_control_flag_from_messages_and_tools(
|
||||
self,
|
||||
model: str,
|
||||
messages: List[AllMessageValues],
|
||||
tools: Optional[List[ChatCompletionToolParam]] = None,
|
||||
) -> Tuple[List[AllMessageValues], Optional[List[ChatCompletionToolParam]]]:
|
||||
"""
|
||||
Override to preserve cache_control for MiniMax.
|
||||
MiniMax supports cache_control - don't strip it.
|
||||
"""
|
||||
# MiniMax supports cache_control, so return messages and tools unchanged
|
||||
return messages, tools
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
"""
|
||||
Get supported OpenAI parameters for MiniMax.
|
||||
Adds reasoning_split and thinking to the list of supported params.
|
||||
Adds reasoning_split to the list of supported params.
|
||||
"""
|
||||
base_params = super().get_supported_openai_params(model=model)
|
||||
additional_params = ["reasoning_split"]
|
||||
|
||||
# Add thinking parameter if model supports reasoning
|
||||
try:
|
||||
if litellm.supports_reasoning(model=model, custom_llm_provider="minimax"):
|
||||
additional_params.append("thinking")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return base_params + additional_params
|
||||
return base_params + ["reasoning_split"]
|
||||
|
||||
|
|
|
|||
|
|
@ -32,7 +32,6 @@ from litellm.types.llms.oci import (
|
|||
OCICompletionResponse,
|
||||
OCIContentPartUnion,
|
||||
OCIImageContentPart,
|
||||
OCIImageUrl,
|
||||
OCIMessage,
|
||||
OCIRoles,
|
||||
OCIServingMode,
|
||||
|
|
@ -1130,7 +1129,7 @@ def adapt_messages_to_generic_oci_standard_content_message(
|
|||
image_url = image_url.get("url")
|
||||
if not isinstance(image_url, str):
|
||||
raise Exception("Prop `image_url` must be a string or an object with a `url` property")
|
||||
new_content.append(OCIImageContentPart(imageUrl=OCIImageUrl(url=image_url)))
|
||||
new_content.append(OCIImageContentPart(imageUrl=image_url))
|
||||
|
||||
return OCIMessage(
|
||||
role=open_ai_to_generic_oci_role_map[role],
|
||||
|
|
|
|||
|
|
@ -19,9 +19,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
|
|||
|
||||
@classmethod
|
||||
def is_model_gpt_5_model(cls, model: str) -> bool:
|
||||
# gpt-5-chat* behaves like a regular chat model (supports temperature, etc.)
|
||||
# Don't route it through GPT-5 reasoning-specific parameter restrictions.
|
||||
return "gpt-5" in model and "gpt-5-chat" not in model
|
||||
return "gpt-5" in model
|
||||
|
||||
@classmethod
|
||||
def is_model_gpt_5_codex_model(cls, model: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ from typing import Optional
|
|||
|
||||
from litellm import verbose_logger
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.types.utils import ImageResponse, Usage
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.types.utils import ImageResponse
|
||||
|
||||
|
||||
def cost_calculator(
|
||||
|
|
@ -38,18 +39,11 @@ def cost_calculator(
|
|||
)
|
||||
return 0.0
|
||||
|
||||
# If usage is already a Usage object with completion_tokens_details set,
|
||||
# use it directly (it was already transformed in convert_to_image_response)
|
||||
if isinstance(usage, Usage) and usage.completion_tokens_details is not None:
|
||||
chat_usage = usage
|
||||
else:
|
||||
# Transform ImageUsage to Usage using the existing helper
|
||||
# ImageUsage has the same format as ResponseAPIUsage
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
|
||||
chat_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
|
||||
usage
|
||||
)
|
||||
# Transform ImageUsage to Usage using the existing helper
|
||||
# ImageUsage has the same format as ResponseAPIUsage
|
||||
chat_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
|
||||
usage
|
||||
)
|
||||
|
||||
# Use generic_cost_per_token for cost calculation
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
|
|
|
|||
|
|
@ -26,9 +26,6 @@ class CacheControlSupportedModels(str, Enum):
|
|||
"""Models that support cache_control in content blocks."""
|
||||
CLAUDE = "claude"
|
||||
GEMINI = "gemini"
|
||||
MINIMAX = "minimax"
|
||||
GLM = "glm"
|
||||
ZAI = "z-ai"
|
||||
|
||||
|
||||
class OpenrouterConfig(OpenAIGPTConfig):
|
||||
|
|
@ -42,7 +39,6 @@ class OpenrouterConfig(OpenAIGPTConfig):
|
|||
model=model, custom_llm_provider="openrouter"
|
||||
) or litellm.supports_reasoning(model=model):
|
||||
supported_params.append("reasoning_effort")
|
||||
supported_params.append("thinking")
|
||||
except Exception:
|
||||
pass
|
||||
return list(dict.fromkeys(supported_params))
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
# S3 Vectors LLM integration
|
||||
|
|
@ -1 +0,0 @@
|
|||
# S3 Vectors vector store integration
|
||||
|
|
@ -1,254 +0,0 @@
|
|||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.vector_stores import (
|
||||
VECTOR_STORE_OPENAI_PARAMS,
|
||||
BaseVectorStoreAuthCredentials,
|
||||
VectorStoreIndexEndpoints,
|
||||
VectorStoreResultContent,
|
||||
VectorStoreSearchOptionalRequestParams,
|
||||
VectorStoreSearchResponse,
|
||||
VectorStoreSearchResult,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
|
||||
"""Vector store configuration for AWS S3 Vectors."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
BaseVectorStoreConfig.__init__(self)
|
||||
BaseAWSLLM.__init__(self)
|
||||
|
||||
def get_auth_credentials(
|
||||
self, litellm_params: dict
|
||||
) -> BaseVectorStoreAuthCredentials:
|
||||
return {}
|
||||
|
||||
def get_vector_store_endpoints_by_type(self) -> VectorStoreIndexEndpoints:
|
||||
return {
|
||||
"read": [("POST", "/QueryVectors")],
|
||||
"write": [],
|
||||
}
|
||||
|
||||
def get_supported_openai_params(
|
||||
self, model: str
|
||||
) -> List[VECTOR_STORE_OPENAI_PARAMS]:
|
||||
return ["max_num_results"]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
for param, value in non_default_params.items():
|
||||
if param == "max_num_results":
|
||||
optional_params["maxResults"] = value
|
||||
return optional_params
|
||||
|
||||
def validate_environment(
|
||||
self, headers: dict, litellm_params: Optional[GenericLiteLLMParams]
|
||||
) -> dict:
|
||||
headers = headers or {}
|
||||
headers.setdefault("Content-Type", "application/json")
|
||||
return headers
|
||||
|
||||
def get_complete_url(self, api_base: Optional[str], litellm_params: dict) -> str:
|
||||
aws_region_name = litellm_params.get("aws_region_name")
|
||||
if not aws_region_name:
|
||||
raise ValueError("aws_region_name is required for S3 Vectors")
|
||||
return f"https://s3vectors.{aws_region_name}.api.aws"
|
||||
|
||||
def transform_search_vector_store_request(
|
||||
self,
|
||||
vector_store_id: str,
|
||||
query: Union[str, List[str]],
|
||||
vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
|
||||
api_base: str,
|
||||
litellm_logging_obj: LiteLLMLoggingObj,
|
||||
litellm_params: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""Sync version - generates embedding synchronously."""
|
||||
# For S3 Vectors, vector_store_id should be in format: bucket_name:index_name
|
||||
# If not in that format, try to construct it from litellm_params
|
||||
bucket_name: str
|
||||
index_name: str
|
||||
|
||||
if ":" in vector_store_id:
|
||||
bucket_name, index_name = vector_store_id.split(":", 1)
|
||||
else:
|
||||
# Try to get bucket_name from litellm_params
|
||||
bucket_name_from_params = litellm_params.get("vector_bucket_name")
|
||||
if not bucket_name_from_params or not isinstance(bucket_name_from_params, str):
|
||||
raise ValueError(
|
||||
"vector_store_id must be in format 'bucket_name:index_name' for S3 Vectors, "
|
||||
"or vector_bucket_name must be provided in litellm_params"
|
||||
)
|
||||
bucket_name = bucket_name_from_params
|
||||
index_name = vector_store_id
|
||||
|
||||
if isinstance(query, list):
|
||||
query = " ".join(query)
|
||||
|
||||
# Generate embedding for the query
|
||||
embedding_model = litellm_params.get("embedding_model", "text-embedding-3-small")
|
||||
|
||||
import litellm as litellm_module
|
||||
embedding_response = litellm_module.embedding(model=embedding_model, input=[query])
|
||||
query_embedding = embedding_response.data[0]["embedding"]
|
||||
|
||||
url = f"{api_base}/QueryVectors"
|
||||
|
||||
request_body: Dict[str, Any] = {
|
||||
"vectorBucketName": bucket_name,
|
||||
"indexName": index_name,
|
||||
"queryVector": {"float32": query_embedding},
|
||||
"topK": vector_store_search_optional_params.get("max_num_results", 5), # Default to 5
|
||||
"returnDistance": True,
|
||||
"returnMetadata": True,
|
||||
}
|
||||
|
||||
litellm_logging_obj.model_call_details["query"] = query
|
||||
return url, request_body
|
||||
|
||||
async def atransform_search_vector_store_request(
|
||||
self,
|
||||
vector_store_id: str,
|
||||
query: Union[str, List[str]],
|
||||
vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
|
||||
api_base: str,
|
||||
litellm_logging_obj: LiteLLMLoggingObj,
|
||||
litellm_params: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""Async version - generates embedding asynchronously."""
|
||||
# For S3 Vectors, vector_store_id should be in format: bucket_name:index_name
|
||||
# If not in that format, try to construct it from litellm_params
|
||||
bucket_name: str
|
||||
index_name: str
|
||||
|
||||
if ":" in vector_store_id:
|
||||
bucket_name, index_name = vector_store_id.split(":", 1)
|
||||
else:
|
||||
# Try to get bucket_name from litellm_params
|
||||
bucket_name_from_params = litellm_params.get("vector_bucket_name")
|
||||
if not bucket_name_from_params or not isinstance(bucket_name_from_params, str):
|
||||
raise ValueError(
|
||||
"vector_store_id must be in format 'bucket_name:index_name' for S3 Vectors, "
|
||||
"or vector_bucket_name must be provided in litellm_params"
|
||||
)
|
||||
bucket_name = bucket_name_from_params
|
||||
index_name = vector_store_id
|
||||
|
||||
if isinstance(query, list):
|
||||
query = " ".join(query)
|
||||
|
||||
# Generate embedding for the query asynchronously
|
||||
embedding_model = litellm_params.get("embedding_model", "text-embedding-3-small")
|
||||
|
||||
import litellm as litellm_module
|
||||
embedding_response = await litellm_module.aembedding(model=embedding_model, input=[query])
|
||||
query_embedding = embedding_response.data[0]["embedding"]
|
||||
|
||||
url = f"{api_base}/QueryVectors"
|
||||
|
||||
request_body: Dict[str, Any] = {
|
||||
"vectorBucketName": bucket_name,
|
||||
"indexName": index_name,
|
||||
"queryVector": {"float32": query_embedding},
|
||||
"topK": vector_store_search_optional_params.get("max_num_results", 5), # Default to 5
|
||||
"returnDistance": True,
|
||||
"returnMetadata": True,
|
||||
}
|
||||
|
||||
litellm_logging_obj.model_call_details["query"] = query
|
||||
return url, request_body
|
||||
|
||||
def sign_request(
|
||||
self,
|
||||
headers: dict,
|
||||
optional_params: Dict,
|
||||
request_data: Dict,
|
||||
api_base: str,
|
||||
api_key: Optional[str] = None,
|
||||
) -> Tuple[dict, Optional[bytes]]:
|
||||
return self._sign_request(
|
||||
service_name="s3vectors",
|
||||
headers=headers,
|
||||
optional_params=optional_params,
|
||||
request_data=request_data,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
)
|
||||
|
||||
def transform_search_vector_store_response(
|
||||
self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj
|
||||
) -> VectorStoreSearchResponse:
|
||||
try:
|
||||
response_data = response.json()
|
||||
results: List[VectorStoreSearchResult] = []
|
||||
|
||||
for item in response_data.get("vectors", []) or []:
|
||||
metadata = item.get("metadata", {}) or {}
|
||||
source_text = metadata.get("source_text", "")
|
||||
|
||||
if not source_text:
|
||||
continue
|
||||
|
||||
# Extract file information from metadata
|
||||
chunk_index = metadata.get("chunk_index", "0")
|
||||
file_id = f"s3-vectors-chunk-{chunk_index}"
|
||||
filename = metadata.get("filename", f"document-{chunk_index}")
|
||||
|
||||
# S3 Vectors returns distance, convert to similarity score (0-1)
|
||||
# Lower distance = higher similarity
|
||||
# We'll normalize using 1 / (1 + distance) to get a 0-1 score
|
||||
distance = item.get("distance")
|
||||
score = None
|
||||
if distance is not None:
|
||||
# Convert distance to similarity score between 0 and 1
|
||||
# For cosine distance: similarity = 1 - distance
|
||||
# For euclidean: use 1 / (1 + distance)
|
||||
# Assuming cosine distance here
|
||||
score = max(0.0, min(1.0, 1.0 - float(distance)))
|
||||
|
||||
results.append(
|
||||
VectorStoreSearchResult(
|
||||
score=score,
|
||||
content=[VectorStoreResultContent(text=source_text, type="text")],
|
||||
file_id=file_id,
|
||||
filename=filename,
|
||||
attributes=metadata,
|
||||
)
|
||||
)
|
||||
|
||||
return VectorStoreSearchResponse(
|
||||
object="vector_store.search_results.page",
|
||||
search_query=litellm_logging_obj.model_call_details.get("query", ""),
|
||||
data=results,
|
||||
)
|
||||
except Exception as e:
|
||||
raise self.get_error_class(
|
||||
error_message=str(e),
|
||||
status_code=response.status_code,
|
||||
headers=response.headers,
|
||||
)
|
||||
|
||||
# Vector store creation is not yet implemented
|
||||
def transform_create_vector_store_request(
|
||||
self,
|
||||
vector_store_create_optional_params,
|
||||
api_base: str,
|
||||
) -> Tuple[str, Dict]:
|
||||
raise NotImplementedError
|
||||
|
||||
def transform_create_vector_store_response(self, response: httpx.Response):
|
||||
raise NotImplementedError
|
||||
|
|
@ -1,176 +0,0 @@
|
|||
"""
|
||||
Vercel AI Gateway Embedding API Configuration.
|
||||
|
||||
This module provides the configuration for Vercel AI Gateway's Embedding API.
|
||||
Vercel AI Gateway is OpenAI-compatible and supports embeddings via the /v1/embeddings endpoint.
|
||||
|
||||
Docs: https://vercel.com/docs/ai-gateway/openai-compat/embeddings
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllEmbeddingInputValues
|
||||
from litellm.types.utils import EmbeddingResponse
|
||||
from litellm.utils import convert_to_model_response_object
|
||||
|
||||
from ..common_utils import VercelAIGatewayException
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
class VercelAIGatewayEmbeddingConfig(BaseEmbeddingConfig):
|
||||
"""
|
||||
Configuration for Vercel AI Gateway's Embedding API.
|
||||
|
||||
Reference: https://vercel.com/docs/ai-gateway/openai-compat/embeddings
|
||||
"""
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
model: str,
|
||||
messages: list,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
api_key: Optional[str] = None,
|
||||
api_base: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Validate environment and set up headers for Vercel AI Gateway API.
|
||||
|
||||
Vercel AI Gateway requires:
|
||||
- Authorization header with Bearer token (API key or OIDC token)
|
||||
"""
|
||||
vercel_headers = {
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
# Add Authorization header if api_key is provided
|
||||
if api_key:
|
||||
vercel_headers["Authorization"] = f"Bearer {api_key}"
|
||||
|
||||
# Merge with existing headers (user's extra_headers take priority)
|
||||
merged_headers = {**vercel_headers, **headers}
|
||||
|
||||
return merged_headers
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
api_key: Optional[str],
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
stream: Optional[bool] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Get the complete URL for Vercel AI Gateway Embedding API endpoint.
|
||||
"""
|
||||
if api_base:
|
||||
api_base = api_base.rstrip("/")
|
||||
else:
|
||||
api_base = (
|
||||
get_secret_str("VERCEL_AI_GATEWAY_API_BASE")
|
||||
or "https://ai-gateway.vercel.sh/v1"
|
||||
)
|
||||
|
||||
return f"{api_base}/embeddings"
|
||||
|
||||
def transform_embedding_request(
|
||||
self,
|
||||
model: str,
|
||||
input: AllEmbeddingInputValues,
|
||||
optional_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
"""
|
||||
Transform embedding request to Vercel AI Gateway format (OpenAI-compatible).
|
||||
"""
|
||||
# Ensure input is a list
|
||||
if isinstance(input, str):
|
||||
input = [input]
|
||||
|
||||
# Strip 'vercel_ai_gateway/' prefix if present
|
||||
if model.startswith("vercel_ai_gateway/"):
|
||||
model = model.replace("vercel_ai_gateway/", "", 1)
|
||||
|
||||
return {
|
||||
"model": model,
|
||||
"input": input,
|
||||
**optional_params,
|
||||
}
|
||||
|
||||
def transform_embedding_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: EmbeddingResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
api_key: Optional[str],
|
||||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
) -> EmbeddingResponse:
|
||||
"""
|
||||
Transform embedding response from Vercel AI Gateway format (OpenAI-compatible).
|
||||
"""
|
||||
logging_obj.post_call(original_response=raw_response.text)
|
||||
|
||||
# Vercel AI Gateway returns standard OpenAI-compatible embedding response
|
||||
response_json = raw_response.json()
|
||||
|
||||
return convert_to_model_response_object(
|
||||
response_object=response_json,
|
||||
model_response_object=model_response,
|
||||
response_type="embedding",
|
||||
)
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
"""
|
||||
Get list of supported OpenAI parameters for Vercel AI Gateway embeddings.
|
||||
|
||||
Vercel AI Gateway supports the standard OpenAI embeddings parameters
|
||||
and auto-maps 'dimensions' to each provider's expected field.
|
||||
"""
|
||||
return [
|
||||
"timeout",
|
||||
"dimensions",
|
||||
"encoding_format",
|
||||
"user",
|
||||
]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
"""
|
||||
Map OpenAI parameters to Vercel AI Gateway format.
|
||||
"""
|
||||
for param, value in non_default_params.items():
|
||||
if param in self.get_supported_openai_params(model):
|
||||
optional_params[param] = value
|
||||
return optional_params
|
||||
|
||||
def get_error_class(
|
||||
self, error_message: str, status_code: int, headers: Any
|
||||
) -> Any:
|
||||
"""
|
||||
Get the error class for Vercel AI Gateway errors.
|
||||
"""
|
||||
return VercelAIGatewayException(
|
||||
message=error_message,
|
||||
status_code=status_code,
|
||||
headers=headers,
|
||||
)
|
||||
|
|
@ -849,7 +849,7 @@ def get_vertex_model_id_from_url(url: str) -> Optional[str]:
|
|||
|
||||
`https://${LOCATION}-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/publishers/google/models/${MODEL_ID}:streamGenerateContent`
|
||||
"""
|
||||
match = re.search(r"/models/([^:]+)", url)
|
||||
match = re.search(r"/models/([^/:]+)", url)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1657,17 +1657,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
## This is necessary because promptTokensDetails includes both cached and non-cached tokens
|
||||
## See: https://github.com/BerriAI/litellm/issues/18750
|
||||
if cached_text_tokens is not None and prompt_text_tokens is not None:
|
||||
# Explicit caching: subtract cached tokens per modality from cacheTokensDetails
|
||||
prompt_text_tokens = prompt_text_tokens - cached_text_tokens
|
||||
elif (
|
||||
cached_tokens is not None
|
||||
and prompt_text_tokens is not None
|
||||
and cached_text_tokens is None
|
||||
):
|
||||
# Implicit caching: only cachedContentTokenCount is provided (no cacheTokensDetails)
|
||||
# Subtract from text tokens since implicit caching is primarily for text content
|
||||
# See: https://github.com/BerriAI/litellm/issues/16341
|
||||
prompt_text_tokens = prompt_text_tokens - cached_tokens
|
||||
if cached_audio_tokens is not None and prompt_audio_tokens is not None:
|
||||
prompt_audio_tokens = prompt_audio_tokens - cached_audio_tokens
|
||||
if cached_image_tokens is not None and prompt_image_tokens is not None:
|
||||
|
|
|
|||
|
|
@ -1,11 +1,10 @@
|
|||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
|
||||
from litellm.types.llms.xai import XAIWebSearchTool, XAIXSearchTool
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
|
@ -50,85 +49,6 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
|
||||
return supported_params
|
||||
|
||||
def _transform_web_search_tool(self, tool: Dict[str, Any]) -> Union[XAIWebSearchTool, Dict[str, Any]]:
|
||||
"""
|
||||
Transform web_search tool to XAI format.
|
||||
|
||||
XAI supports web_search with specific filters:
|
||||
- allowed_domains (max 5)
|
||||
- excluded_domains (max 5)
|
||||
- enable_image_understanding
|
||||
|
||||
XAI does NOT support search_context_size (OpenAI-specific).
|
||||
"""
|
||||
xai_tool: Dict[str, Any] = {"type": "web_search"}
|
||||
|
||||
# Remove search_context_size if present (not supported by XAI)
|
||||
if "search_context_size" in tool:
|
||||
verbose_logger.info(
|
||||
"XAI does not support 'search_context_size' parameter. Removing it from web_search tool."
|
||||
)
|
||||
|
||||
# Handle filters (XAI-specific structure)
|
||||
filters = {}
|
||||
if "allowed_domains" in tool:
|
||||
allowed_domains = tool["allowed_domains"]
|
||||
filters["allowed_domains"] = allowed_domains
|
||||
|
||||
if "excluded_domains" in tool:
|
||||
excluded_domains = tool["excluded_domains"]
|
||||
filters["excluded_domains"] = excluded_domains
|
||||
|
||||
# Add filters if any were specified
|
||||
if filters:
|
||||
xai_tool["filters"] = filters
|
||||
|
||||
# Handle enable_image_understanding (top-level in XAI format)
|
||||
if "enable_image_understanding" in tool:
|
||||
xai_tool["enable_image_understanding"] = tool["enable_image_understanding"]
|
||||
|
||||
return xai_tool
|
||||
|
||||
def _transform_x_search_tool(self, tool: Dict[str, Any]) -> Union[XAIXSearchTool, Dict[str, Any]]:
|
||||
"""
|
||||
Transform x_search tool to XAI format.
|
||||
|
||||
XAI supports x_search with specific parameters:
|
||||
- allowed_x_handles (max 10)
|
||||
- excluded_x_handles (max 10)
|
||||
- from_date (ISO8601: YYYY-MM-DD)
|
||||
- to_date (ISO8601: YYYY-MM-DD)
|
||||
- enable_image_understanding
|
||||
- enable_video_understanding
|
||||
"""
|
||||
xai_tool: Dict[str, Any] = {"type": "x_search"}
|
||||
|
||||
# Handle allowed_x_handles
|
||||
if "allowed_x_handles" in tool:
|
||||
allowed_handles = tool["allowed_x_handles"]
|
||||
xai_tool["allowed_x_handles"] = allowed_handles
|
||||
|
||||
# Handle excluded_x_handles
|
||||
if "excluded_x_handles" in tool:
|
||||
excluded_handles = tool["excluded_x_handles"]
|
||||
xai_tool["excluded_x_handles"] = excluded_handles
|
||||
|
||||
# Handle date range
|
||||
if "from_date" in tool:
|
||||
xai_tool["from_date"] = tool["from_date"]
|
||||
|
||||
if "to_date" in tool:
|
||||
xai_tool["to_date"] = tool["to_date"]
|
||||
|
||||
# Handle media understanding flags
|
||||
if "enable_image_understanding" in tool:
|
||||
xai_tool["enable_image_understanding"] = tool["enable_image_understanding"]
|
||||
|
||||
if "enable_video_understanding" in tool:
|
||||
xai_tool["enable_video_understanding"] = tool["enable_video_understanding"]
|
||||
|
||||
return xai_tool
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
response_api_optional_params: ResponsesAPIOptionalRequestParams,
|
||||
|
|
@ -141,9 +61,7 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
Handles XAI-specific transformations:
|
||||
1. Drops 'instructions' parameter (not supported)
|
||||
2. Transforms code_interpreter tools to remove 'container' field
|
||||
3. Transforms web_search tools to XAI format (removes search_context_size, adds filters)
|
||||
4. Transforms x_search tools to XAI format
|
||||
5. Sets store=false when images are detected (recommended by XAI)
|
||||
3. Sets store=false when images are detected (recommended by XAI)
|
||||
"""
|
||||
params = dict(response_api_optional_params)
|
||||
|
||||
|
|
@ -154,7 +72,7 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
)
|
||||
params.pop("instructions")
|
||||
|
||||
# Transform tools
|
||||
# Transform code_interpreter tools - remove container field
|
||||
if "tools" in params and params["tools"]:
|
||||
tools_list = params["tools"]
|
||||
# Ensure tools is a list for iteration
|
||||
|
|
@ -163,36 +81,15 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
|
||||
transformed_tools: List[Any] = []
|
||||
for tool in tools_list:
|
||||
if isinstance(tool, dict):
|
||||
tool_type = tool.get("type")
|
||||
|
||||
if tool_type == "code_interpreter":
|
||||
# XAI supports code_interpreter but doesn't use the container field
|
||||
verbose_logger.debug(
|
||||
"XAI: Transforming code_interpreter tool, removing container field"
|
||||
)
|
||||
transformed_tools.append({"type": "code_interpreter"})
|
||||
|
||||
elif tool_type == "web_search":
|
||||
# Transform web_search to XAI format
|
||||
verbose_logger.debug(
|
||||
"XAI: Transforming web_search tool to XAI format"
|
||||
)
|
||||
transformed_tools.append(self._transform_web_search_tool(tool))
|
||||
|
||||
elif tool_type == "x_search":
|
||||
# Transform x_search to XAI format
|
||||
verbose_logger.debug(
|
||||
"XAI: Transforming x_search tool to XAI format"
|
||||
)
|
||||
transformed_tools.append(self._transform_x_search_tool(tool))
|
||||
|
||||
else:
|
||||
# Keep other tools as-is
|
||||
transformed_tools.append(tool)
|
||||
if isinstance(tool, dict) and tool.get("type") == "code_interpreter":
|
||||
# XAI supports code_interpreter but doesn't use the container field
|
||||
# Keep only the type field
|
||||
verbose_logger.debug(
|
||||
"XAI: Transforming code_interpreter tool, removing container field"
|
||||
)
|
||||
transformed_tools.append({"type": "code_interpreter"})
|
||||
else:
|
||||
transformed_tools.append(tool)
|
||||
|
||||
params["tools"] = transformed_tools
|
||||
|
||||
return params
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
from typing import List, Optional, Tuple
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
|
||||
|
||||
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
|
|
@ -20,19 +19,6 @@ class ZAIChatConfig(OpenAIGPTConfig):
|
|||
dynamic_api_key = api_key or get_secret_str("ZAI_API_KEY")
|
||||
return api_base, dynamic_api_key
|
||||
|
||||
def remove_cache_control_flag_from_messages_and_tools(
|
||||
self,
|
||||
model: str,
|
||||
messages: List[AllMessageValues],
|
||||
tools: Optional[List[ChatCompletionToolParam]] = None,
|
||||
) -> Tuple[List[AllMessageValues], Optional[List[ChatCompletionToolParam]]]:
|
||||
"""
|
||||
Override to preserve cache_control for GLM/ZAI.
|
||||
GLM supports cache_control - don't strip it.
|
||||
"""
|
||||
# GLM/ZAI supports cache_control, so return messages and tools unchanged
|
||||
return messages, tools
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
base_params = [
|
||||
"max_tokens",
|
||||
|
|
|
|||
|
|
@ -148,7 +148,7 @@ from litellm.utils import (
|
|||
validate_and_fix_openai_messages,
|
||||
validate_and_fix_openai_tools,
|
||||
validate_chat_completion_tool_choice,
|
||||
validate_openai_optional_params,
|
||||
validate_openai_optional_params
|
||||
)
|
||||
|
||||
from ._logging import verbose_logger
|
||||
|
|
@ -368,7 +368,7 @@ class AsyncCompletions:
|
|||
|
||||
@tracer.wrap()
|
||||
@client
|
||||
async def acompletion( # noqa: PLR0915
|
||||
async def acompletion( # noqa: PLR0915
|
||||
model: str,
|
||||
# Optional OpenAI params: see https://platform.openai.com/docs/api-reference/chat/create
|
||||
messages: List = [],
|
||||
|
|
@ -599,8 +599,16 @@ async def acompletion( # noqa: PLR0915
|
|||
# Add the context to the function
|
||||
ctx = contextvars.copy_context()
|
||||
func_with_context = partial(ctx.run, func)
|
||||
|
||||
if timeout is not None and isinstance(timeout, (int, float)):
|
||||
timeout_value = float(timeout)
|
||||
init_response = await asyncio.wait_for(
|
||||
loop.run_in_executor(None, func_with_context),
|
||||
timeout=timeout_value
|
||||
)
|
||||
else:
|
||||
init_response = await loop.run_in_executor(None, func_with_context)
|
||||
|
||||
init_response = await loop.run_in_executor(None, func_with_context)
|
||||
if isinstance(init_response, dict) or isinstance(
|
||||
init_response, ModelResponse
|
||||
): ## CACHING SCENARIO
|
||||
|
|
@ -608,7 +616,11 @@ async def acompletion( # noqa: PLR0915
|
|||
response = ModelResponse(**init_response)
|
||||
response = init_response
|
||||
elif asyncio.iscoroutine(init_response):
|
||||
response = await init_response
|
||||
if timeout is not None and isinstance(timeout, (int, float)):
|
||||
timeout_value = float(timeout)
|
||||
response = await asyncio.wait_for(init_response, timeout=timeout_value)
|
||||
else:
|
||||
response = await init_response
|
||||
else:
|
||||
response = init_response # type: ignore
|
||||
|
||||
|
|
@ -625,6 +637,14 @@ async def acompletion( # noqa: PLR0915
|
|||
loop=loop
|
||||
) # sets the logging event loop if the user does sync streaming (e.g. on proxy for sagemaker calls)
|
||||
return response
|
||||
except asyncio.TimeoutError:
|
||||
custom_llm_provider = custom_llm_provider or "openai"
|
||||
from litellm.exceptions import Timeout
|
||||
raise Timeout(
|
||||
message=f"Request timed out after {timeout} seconds",
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
)
|
||||
except Exception as e:
|
||||
custom_llm_provider = custom_llm_provider or "openai"
|
||||
raise exception_type(
|
||||
|
|
@ -1098,6 +1118,7 @@ def completion( # type: ignore # noqa: PLR0915
|
|||
# validate optional params
|
||||
stop = validate_openai_optional_params(stop=stop)
|
||||
|
||||
|
||||
######### unpacking kwargs #####################
|
||||
args = locals()
|
||||
|
||||
|
|
@ -1114,9 +1135,7 @@ def completion( # type: ignore # noqa: PLR0915
|
|||
# Check if MCP tools are present (following responses pattern)
|
||||
# Cast tools to Optional[Iterable[ToolParam]] for type checking
|
||||
tools_for_mcp = cast(Optional[Iterable[ToolParam]], tools)
|
||||
if LiteLLM_Proxy_MCP_Handler._should_use_litellm_mcp_gateway(
|
||||
tools=tools_for_mcp
|
||||
):
|
||||
if LiteLLM_Proxy_MCP_Handler._should_use_litellm_mcp_gateway(tools=tools_for_mcp):
|
||||
# Return coroutine - acompletion will await it
|
||||
# completion() can return a coroutine when MCP tools are present, which acompletion() awaits
|
||||
return acompletion_with_mcp( # type: ignore[return-value]
|
||||
|
|
@ -1517,8 +1536,6 @@ def completion( # type: ignore # noqa: PLR0915
|
|||
max_retries=max_retries,
|
||||
timeout=timeout,
|
||||
litellm_request_debug=kwargs.get("litellm_request_debug", False),
|
||||
tpm=kwargs.get("tpm"),
|
||||
rpm=kwargs.get("rpm"),
|
||||
)
|
||||
cast(LiteLLMLoggingObj, logging).update_environment_variables(
|
||||
model=model,
|
||||
|
|
@ -2344,7 +2361,11 @@ def completion( # type: ignore # noqa: PLR0915
|
|||
input=messages, api_key=api_key, original_response=response
|
||||
)
|
||||
elif custom_llm_provider == "minimax":
|
||||
api_key = api_key or get_secret_str("MINIMAX_API_KEY") or litellm.api_key
|
||||
api_key = (
|
||||
api_key
|
||||
or get_secret_str("MINIMAX_API_KEY")
|
||||
or litellm.api_key
|
||||
)
|
||||
|
||||
api_base = (
|
||||
api_base
|
||||
|
|
@ -2392,9 +2413,7 @@ def completion( # type: ignore # noqa: PLR0915
|
|||
or custom_llm_provider == "wandb"
|
||||
or custom_llm_provider == "clarifai"
|
||||
or custom_llm_provider in litellm.openai_compatible_providers
|
||||
or JSONProviderRegistry.exists(
|
||||
custom_llm_provider
|
||||
) # JSON-configured providers
|
||||
or JSONProviderRegistry.exists(custom_llm_provider) # JSON-configured providers
|
||||
or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo
|
||||
): # allow user to make an openai call with a custom base
|
||||
# note: if a user sets a custom base - we should ensure this works
|
||||
|
|
@ -4705,7 +4724,7 @@ def embedding( # noqa: PLR0915
|
|||
|
||||
if headers is not None and headers != {}:
|
||||
optional_params["extra_headers"] = headers
|
||||
|
||||
|
||||
if encoding_format is not None:
|
||||
optional_params["encoding_format"] = encoding_format
|
||||
else:
|
||||
|
|
@ -4847,36 +4866,6 @@ def embedding( # noqa: PLR0915
|
|||
|
||||
headers = openrouter_headers
|
||||
|
||||
response = base_llm_http_handler.embedding(
|
||||
model=model,
|
||||
input=input,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
logging_obj=logging,
|
||||
timeout=timeout,
|
||||
model_response=EmbeddingResponse(),
|
||||
optional_params=optional_params,
|
||||
client=client,
|
||||
aembedding=aembedding,
|
||||
litellm_params=litellm_params_dict,
|
||||
headers=headers,
|
||||
)
|
||||
elif custom_llm_provider == "vercel_ai_gateway":
|
||||
api_base = (
|
||||
api_base
|
||||
or litellm.api_base
|
||||
or get_secret_str("VERCEL_AI_GATEWAY_API_BASE")
|
||||
or "https://ai-gateway.vercel.sh/v1"
|
||||
)
|
||||
|
||||
api_key = (
|
||||
api_key
|
||||
or litellm.api_key
|
||||
or get_secret_str("VERCEL_AI_GATEWAY_API_KEY")
|
||||
or get_secret_str("VERCEL_OIDC_TOKEN")
|
||||
)
|
||||
|
||||
response = base_llm_http_handler.embedding(
|
||||
model=model,
|
||||
input=input,
|
||||
|
|
@ -6770,7 +6759,9 @@ def speech( # noqa: PLR0915
|
|||
if text_to_speech_provider_config is None:
|
||||
text_to_speech_provider_config = MinimaxTextToSpeechConfig()
|
||||
|
||||
minimax_config = cast(MinimaxTextToSpeechConfig, text_to_speech_provider_config)
|
||||
minimax_config = cast(
|
||||
MinimaxTextToSpeechConfig, text_to_speech_provider_config
|
||||
)
|
||||
|
||||
if api_base is not None:
|
||||
litellm_params_dict["api_base"] = api_base
|
||||
|
|
@ -6910,7 +6901,7 @@ async def ahealth_check(
|
|||
custom_llm_provider_from_params = model_params.get("custom_llm_provider", None)
|
||||
api_base_from_params = model_params.get("api_base", None)
|
||||
api_key_from_params = model_params.get("api_key", None)
|
||||
|
||||
|
||||
model, custom_llm_provider, _, _ = get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider_from_params,
|
||||
|
|
@ -7280,16 +7271,12 @@ def _get_encoding():
|
|||
def __getattr__(name: str) -> Any:
|
||||
"""Lazy import handler for main module"""
|
||||
if name == "encoding":
|
||||
# Use _get_default_encoding which properly sets TIKTOKEN_CACHE_DIR
|
||||
# before loading tiktoken, ensuring the local cache is used
|
||||
# instead of downloading from the internet
|
||||
from litellm._lazy_imports import _get_default_encoding
|
||||
_encoding = _get_default_encoding()
|
||||
# Lazy load encoding to avoid heavy tiktoken import at module load time
|
||||
_encoding = tiktoken.get_encoding("cl100k_base")
|
||||
# Cache it in the module's __dict__ for subsequent accesses
|
||||
import sys
|
||||
|
||||
sys.modules[__name__].__dict__["encoding"] = _encoding
|
||||
global _encoding_cache
|
||||
_encoding_cache = _encoding
|
||||
return _encoding
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
|
@ -3130,7 +3130,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_tool_choice": false,
|
||||
"supports_vision": true
|
||||
},
|
||||
"azure/gpt-5-chat-latest": {
|
||||
|
|
@ -3162,7 +3162,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_tool_choice": false,
|
||||
"supports_vision": true
|
||||
},
|
||||
"azure/gpt-5-codex": {
|
||||
|
|
@ -3653,9 +3653,10 @@
|
|||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 16384,
|
||||
"max_tokens": 16384,
|
||||
"mode": "responses",
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.4e-05,
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/responses"
|
||||
],
|
||||
"supported_modalities": [
|
||||
|
|
@ -9787,7 +9788,6 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"deepinfra/google/gemini-2.0-flash-001": {
|
||||
"deprecation_date": "2026-03-31",
|
||||
"max_tokens": 1000000,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 1000000,
|
||||
|
|
@ -10231,48 +10231,6 @@
|
|||
"mode": "completion",
|
||||
"output_cost_per_token": 5e-07
|
||||
},
|
||||
"deepseek-v3-2-251201": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "volcengine",
|
||||
"max_input_tokens": 98304,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.0,
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"glm-4-7-251222": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "volcengine",
|
||||
"max_input_tokens": 204800,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.0,
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"kimi-k2-thinking-251104": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "volcengine",
|
||||
"max_input_tokens": 229376,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.0,
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"doubao-embedding": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "volcengine",
|
||||
|
|
@ -12147,7 +12105,6 @@
|
|||
},
|
||||
"gemini-2.0-flash": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_audio_token": 7e-07,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -12187,7 +12144,7 @@
|
|||
},
|
||||
"gemini-2.0-flash-001": {
|
||||
"cache_read_input_token_cost": 3.75e-08,
|
||||
"deprecation_date": "2026-03-31",
|
||||
"deprecation_date": "2026-02-05",
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -12273,7 +12230,6 @@
|
|||
},
|
||||
"gemini-2.0-flash-lite": {
|
||||
"cache_read_input_token_cost": 1.875e-08,
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_audio_token": 7.5e-08,
|
||||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -12309,7 +12265,7 @@
|
|||
},
|
||||
"gemini-2.0-flash-lite-001": {
|
||||
"cache_read_input_token_cost": 1.875e-08,
|
||||
"deprecation_date": "2026-03-31",
|
||||
"deprecation_date": "2026-02-25",
|
||||
"input_cost_per_audio_token": 7.5e-08,
|
||||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -13521,79 +13477,6 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true
|
||||
},
|
||||
"gemini-robotics-er-1.5-preview": {
|
||||
"cache_read_input_token_cost": 0,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 65535,
|
||||
"max_tokens": 65535,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-06,
|
||||
"output_cost_per_reasoning_token": 2.5e-06,
|
||||
"source": "https://ai.google.dev/gemini-api/docs/models#gemini-robotics-er-1-5-preview",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/completions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image",
|
||||
"video",
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_output": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"gemini/gemini-robotics-er-1.5-preview": {
|
||||
"cache_read_input_token_cost": 0,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 65535,
|
||||
"max_tokens": 65535,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-06,
|
||||
"output_cost_per_reasoning_token": 2.5e-06,
|
||||
"source": "https://ai.google.dev/gemini-api/docs/models#gemini-robotics-er-1-5-preview",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/completions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image",
|
||||
"video",
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_output": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true
|
||||
},
|
||||
"gemini-2.5-computer-use-preview-10-2025": {
|
||||
"input_cost_per_token": 1.25e-06,
|
||||
"input_cost_per_token_above_200k_tokens": 2.5e-06,
|
||||
|
|
@ -14047,7 +13930,6 @@
|
|||
},
|
||||
"gemini/gemini-2.0-flash": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_audio_token": 7e-07,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "gemini",
|
||||
|
|
@ -14088,7 +13970,6 @@
|
|||
},
|
||||
"gemini/gemini-2.0-flash-001": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_audio_token": 7e-07,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "gemini",
|
||||
|
|
@ -14176,7 +14057,6 @@
|
|||
},
|
||||
"gemini/gemini-2.0-flash-lite": {
|
||||
"cache_read_input_token_cost": 1.875e-08,
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_audio_token": 7.5e-08,
|
||||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "gemini",
|
||||
|
|
@ -20663,7 +20543,6 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_system_messages": true,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 8192
|
||||
|
|
@ -20678,7 +20557,6 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_system_messages": true,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 8192
|
||||
|
|
@ -20693,7 +20571,6 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_system_messages": true,
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 8192
|
||||
|
|
@ -23281,7 +23158,6 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/google/gemini-2.0-flash-001": {
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_audio_token": 7e-07,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "openrouter",
|
||||
|
|
@ -23578,7 +23454,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 1.02e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_prompt_caching": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -23930,11 +23806,8 @@
|
|||
"max_input_tokens": 400000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "responses",
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.4e-05,
|
||||
"supported_endpoints": [
|
||||
"/v1/responses"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
|
|
@ -24273,7 +24146,6 @@
|
|||
"output_cost_per_token": 1.75e-06,
|
||||
"source": "https://openrouter.ai/z-ai/glm-4.6",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -24287,76 +24159,9 @@
|
|||
"output_cost_per_token": 1.9e-06,
|
||||
"source": "https://openrouter.ai/z-ai/glm-4.6:exacto",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/xiaomi/mimo-v2-flash": {
|
||||
"input_cost_per_token": 9e-08,
|
||||
"output_cost_per_token": 2.9e-07,
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"litellm_provider": "openrouter",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 16384,
|
||||
"max_tokens": 16384,
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": false,
|
||||
"supports_prompt_caching": false
|
||||
},
|
||||
"openrouter/z-ai/glm-4.7": {
|
||||
"input_cost_per_token": 4e-07,
|
||||
"output_cost_per_token": 1.5e-06,
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"litellm_provider": "openrouter",
|
||||
"max_input_tokens": 202752,
|
||||
"max_output_tokens": 64000,
|
||||
"max_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": false,
|
||||
"supports_assistant_prefill": true
|
||||
},
|
||||
"openrouter/z-ai/glm-4.7-flash": {
|
||||
"input_cost_per_token": 7e-08,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"litellm_provider": "openrouter",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 32000,
|
||||
"max_tokens": 32000,
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": false
|
||||
},
|
||||
"openrouter/minimax/minimax-m2.1": {
|
||||
"input_cost_per_token": 2.7e-07,
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"litellm_provider": "openrouter",
|
||||
"max_input_tokens": 204000,
|
||||
"max_output_tokens": 64000,
|
||||
"max_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": false,
|
||||
"supports_computer_use": false
|
||||
},
|
||||
"ovhcloud/DeepSeek-R1-Distill-Llama-70B": {
|
||||
"input_cost_per_token": 6.7e-07,
|
||||
"litellm_provider": "ovhcloud",
|
||||
|
|
@ -27890,7 +27695,6 @@
|
|||
"output_cost_per_token": 9e-07
|
||||
},
|
||||
"vercel_ai_gateway/google/gemini-2.0-flash": {
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "vercel_ai_gateway",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -27900,7 +27704,6 @@
|
|||
"output_cost_per_token": 6e-07
|
||||
},
|
||||
"vercel_ai_gateway/google/gemini-2.0-flash-lite": {
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "vercel_ai_gateway",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -30570,7 +30373,6 @@
|
|||
"supports_web_search": true
|
||||
},
|
||||
"xai/grok-3": {
|
||||
"cache_read_input_token_cost": 7.5e-07,
|
||||
"input_cost_per_token": 3e-06,
|
||||
"litellm_provider": "xai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -30585,7 +30387,6 @@
|
|||
"supports_web_search": true
|
||||
},
|
||||
"xai/grok-3-beta": {
|
||||
"cache_read_input_token_cost": 7.5e-07,
|
||||
"input_cost_per_token": 3e-06,
|
||||
"litellm_provider": "xai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -30600,7 +30401,6 @@
|
|||
"supports_web_search": true
|
||||
},
|
||||
"xai/grok-3-fast-beta": {
|
||||
"cache_read_input_token_cost": 1.25e-06,
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "xai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -30615,7 +30415,6 @@
|
|||
"supports_web_search": true
|
||||
},
|
||||
"xai/grok-3-fast-latest": {
|
||||
"cache_read_input_token_cost": 1.25e-06,
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "xai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -30630,7 +30429,6 @@
|
|||
"supports_web_search": true
|
||||
},
|
||||
"xai/grok-3-latest": {
|
||||
"cache_read_input_token_cost": 7.5e-07,
|
||||
"input_cost_per_token": 3e-06,
|
||||
"litellm_provider": "xai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -30645,7 +30443,6 @@
|
|||
"supports_web_search": true
|
||||
},
|
||||
"xai/grok-3-mini": {
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "xai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -30661,7 +30458,6 @@
|
|||
"supports_web_search": true
|
||||
},
|
||||
"xai/grok-3-mini-beta": {
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "xai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -30677,7 +30473,6 @@
|
|||
"supports_web_search": true
|
||||
},
|
||||
"xai/grok-3-mini-fast": {
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "xai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -30693,7 +30488,6 @@
|
|||
"supports_web_search": true
|
||||
},
|
||||
"xai/grok-3-mini-fast-beta": {
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "xai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -30709,7 +30503,6 @@
|
|||
"supports_web_search": true
|
||||
},
|
||||
"xai/grok-3-mini-fast-latest": {
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "xai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -30725,7 +30518,6 @@
|
|||
"supports_web_search": true
|
||||
},
|
||||
"xai/grok-3-mini-latest": {
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "xai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -30992,14 +30784,11 @@
|
|||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://docs.z.ai/guides/overview/pricing"
|
||||
},
|
||||
"zai/glm-4.6": {
|
||||
"cache_creation_input_token_cost": 0,
|
||||
"cache_read_input_token_cost": 1.1e-07,
|
||||
"input_cost_per_token": 6e-07,
|
||||
"output_cost_per_token": 2.2e-06,
|
||||
"litellm_provider": "zai",
|
||||
|
|
@ -31007,8 +30796,6 @@
|
|||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://docs.z.ai/guides/overview/pricing"
|
||||
},
|
||||
|
|
@ -34656,4 +34443,4 @@
|
|||
"output_cost_per_token": 0,
|
||||
"supports_reasoning": true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -387,57 +387,25 @@ async def callback(code: str, state: str):
|
|||
1. Try resource_metadata from WWW-Authenticate header (if present)
|
||||
2. Fall back to path-based well-known URI: /.well-known/oauth-protected-resource/{path}
|
||||
(
|
||||
If the resource identifier value contains a path or query component, any terminating slash (/)
|
||||
following the host component MUST be removed before inserting /.well-known/ and the well-known
|
||||
URI path suffix between the host component and the path(include root path) and/or query components.
|
||||
If the resource identifier value contains a path or query component, any terminating slash (/)
|
||||
following the host component MUST be removed before inserting /.well-known/ and the well-known
|
||||
URI path suffix between the host component and the path(include root path) and/or query components.
|
||||
https://datatracker.ietf.org/doc/html/rfc9728#section-3.1)
|
||||
3. Fall back to root-based well-known URI: /.well-known/oauth-protected-resource
|
||||
|
||||
Dual Pattern Support:
|
||||
- Standard MCP pattern: /mcp/{server_name} (recommended, used by mcp-inspector, VSCode Copilot)
|
||||
- LiteLLM legacy pattern: /{server_name}/mcp (backward compatibility)
|
||||
|
||||
The resource URL returned matches the pattern used in the discovery request.
|
||||
"""
|
||||
|
||||
|
||||
def _build_oauth_protected_resource_response(
|
||||
request: Request,
|
||||
mcp_server_name: Optional[str],
|
||||
use_standard_pattern: bool,
|
||||
) -> dict:
|
||||
"""
|
||||
Build OAuth protected resource response with the appropriate URL pattern.
|
||||
|
||||
Args:
|
||||
request: FastAPI Request object
|
||||
mcp_server_name: Name of the MCP server
|
||||
use_standard_pattern: If True, use /mcp/{server_name} pattern;
|
||||
if False, use /{server_name}/mcp pattern
|
||||
|
||||
Returns:
|
||||
OAuth protected resource metadata dict
|
||||
"""
|
||||
@router.get(f"/.well-known/oauth-protected-resource{'' if get_server_root_path() == '/' else get_server_root_path()}/{{mcp_server_name}}/mcp")
|
||||
@router.get("/.well-known/oauth-protected-resource")
|
||||
async def oauth_protected_resource_mcp(
|
||||
request: Request, mcp_server_name: Optional[str] = None
|
||||
):
|
||||
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
|
||||
global_mcp_server_manager,
|
||||
)
|
||||
|
||||
# Get the correct base URL considering X-Forwarded-* headers
|
||||
request_base_url = get_request_base_url(request)
|
||||
mcp_server: Optional[MCPServer] = None
|
||||
if mcp_server_name:
|
||||
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name)
|
||||
|
||||
# Build resource URL based on the pattern
|
||||
if mcp_server_name:
|
||||
if use_standard_pattern:
|
||||
# Standard MCP pattern: /mcp/{server_name}
|
||||
resource_url = f"{request_base_url}/mcp/{mcp_server_name}"
|
||||
else:
|
||||
# LiteLLM legacy pattern: /{server_name}/mcp
|
||||
resource_url = f"{request_base_url}/{mcp_server_name}/mcp"
|
||||
else:
|
||||
resource_url = f"{request_base_url}/mcp"
|
||||
|
||||
return {
|
||||
"authorization_servers": [
|
||||
(
|
||||
|
|
@ -446,55 +414,14 @@ def _build_oauth_protected_resource_response(
|
|||
else f"{request_base_url}"
|
||||
)
|
||||
],
|
||||
"resource": resource_url,
|
||||
"resource": (
|
||||
f"{request_base_url}/{mcp_server_name}/mcp"
|
||||
if mcp_server_name
|
||||
else f"{request_base_url}/mcp"
|
||||
), # this is what Claude will call
|
||||
"scopes_supported": mcp_server.scopes if mcp_server else [],
|
||||
}
|
||||
|
||||
|
||||
# Standard MCP pattern: /.well-known/oauth-protected-resource/mcp/{server_name}
|
||||
# This is the pattern expected by standard MCP clients (mcp-inspector, VSCode Copilot)
|
||||
@router.get(f"/.well-known/oauth-protected-resource{'' if get_server_root_path() == '/' else get_server_root_path()}/mcp/{{mcp_server_name}}")
|
||||
async def oauth_protected_resource_mcp_standard(
|
||||
request: Request, mcp_server_name: str
|
||||
):
|
||||
"""
|
||||
OAuth protected resource discovery endpoint using standard MCP URL pattern.
|
||||
|
||||
Standard pattern: /mcp/{server_name}
|
||||
Discovery path: /.well-known/oauth-protected-resource/mcp/{server_name}
|
||||
|
||||
This endpoint is compliant with MCP specification and works with standard
|
||||
MCP clients like mcp-inspector and VSCode Copilot.
|
||||
"""
|
||||
return _build_oauth_protected_resource_response(
|
||||
request=request,
|
||||
mcp_server_name=mcp_server_name,
|
||||
use_standard_pattern=True,
|
||||
)
|
||||
|
||||
|
||||
# LiteLLM legacy pattern: /.well-known/oauth-protected-resource/{server_name}/mcp
|
||||
# Kept for backward compatibility with existing deployments
|
||||
@router.get(f"/.well-known/oauth-protected-resource{'' if get_server_root_path() == '/' else get_server_root_path()}/{{mcp_server_name}}/mcp")
|
||||
@router.get("/.well-known/oauth-protected-resource")
|
||||
async def oauth_protected_resource_mcp(
|
||||
request: Request, mcp_server_name: Optional[str] = None
|
||||
):
|
||||
"""
|
||||
OAuth protected resource discovery endpoint using LiteLLM legacy URL pattern.
|
||||
|
||||
Legacy pattern: /{server_name}/mcp
|
||||
Discovery path: /.well-known/oauth-protected-resource/{server_name}/mcp
|
||||
|
||||
This endpoint is kept for backward compatibility. New integrations should
|
||||
use the standard MCP pattern (/mcp/{server_name}) instead.
|
||||
"""
|
||||
return _build_oauth_protected_resource_response(
|
||||
request=request,
|
||||
mcp_server_name=mcp_server_name,
|
||||
use_standard_pattern=False,
|
||||
)
|
||||
|
||||
"""
|
||||
https://datatracker.ietf.org/doc/html/rfc8414#section-3.1
|
||||
RFC 8414: Path-aware OAuth discovery
|
||||
|
|
@ -503,26 +430,15 @@ async def oauth_protected_resource_mcp(
|
|||
the well-known URI suffix between the host component and the path(include root path)
|
||||
component.
|
||||
"""
|
||||
|
||||
|
||||
def _build_oauth_authorization_server_response(
|
||||
request: Request,
|
||||
mcp_server_name: Optional[str],
|
||||
) -> dict:
|
||||
"""
|
||||
Build OAuth authorization server metadata response.
|
||||
|
||||
Args:
|
||||
request: FastAPI Request object
|
||||
mcp_server_name: Name of the MCP server
|
||||
|
||||
Returns:
|
||||
OAuth authorization server metadata dict
|
||||
"""
|
||||
@router.get(f"/.well-known/oauth-authorization-server{'' if get_server_root_path() == '/' else get_server_root_path()}/{{mcp_server_name}}")
|
||||
@router.get("/.well-known/oauth-authorization-server")
|
||||
async def oauth_authorization_server_mcp(
|
||||
request: Request, mcp_server_name: Optional[str] = None
|
||||
):
|
||||
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
|
||||
global_mcp_server_manager,
|
||||
)
|
||||
|
||||
# Get the correct base URL considering X-Forwarded-* headers
|
||||
request_base_url = get_request_base_url(request)
|
||||
|
||||
authorization_endpoint = (
|
||||
|
|
@ -554,58 +470,18 @@ def _build_oauth_authorization_server_response(
|
|||
}
|
||||
|
||||
|
||||
# Standard MCP pattern: /.well-known/oauth-authorization-server/mcp/{server_name}
|
||||
@router.get(f"/.well-known/oauth-authorization-server{'' if get_server_root_path() == '/' else get_server_root_path()}/mcp/{{mcp_server_name}}")
|
||||
async def oauth_authorization_server_mcp_standard(
|
||||
request: Request, mcp_server_name: str
|
||||
):
|
||||
"""
|
||||
OAuth authorization server discovery endpoint using standard MCP URL pattern.
|
||||
|
||||
Standard pattern: /mcp/{server_name}
|
||||
Discovery path: /.well-known/oauth-authorization-server/mcp/{server_name}
|
||||
"""
|
||||
return _build_oauth_authorization_server_response(
|
||||
request=request,
|
||||
mcp_server_name=mcp_server_name,
|
||||
)
|
||||
|
||||
|
||||
# LiteLLM legacy pattern and root endpoint
|
||||
@router.get(f"/.well-known/oauth-authorization-server{'' if get_server_root_path() == '/' else get_server_root_path()}/{{mcp_server_name}}")
|
||||
@router.get("/.well-known/oauth-authorization-server")
|
||||
async def oauth_authorization_server_mcp(
|
||||
request: Request, mcp_server_name: Optional[str] = None
|
||||
):
|
||||
"""
|
||||
OAuth authorization server discovery endpoint.
|
||||
|
||||
Supports both legacy pattern (/{server_name}) and root endpoint.
|
||||
"""
|
||||
return _build_oauth_authorization_server_response(
|
||||
request=request,
|
||||
mcp_server_name=mcp_server_name,
|
||||
)
|
||||
|
||||
|
||||
# Alias for standard OpenID discovery
|
||||
@router.get("/.well-known/openid-configuration")
|
||||
async def openid_configuration(request: Request):
|
||||
return await oauth_authorization_server_mcp(request)
|
||||
|
||||
|
||||
# Additional legacy pattern support
|
||||
@router.get("/.well-known/oauth-authorization-server/{mcp_server_name}/mcp")
|
||||
async def oauth_authorization_server_legacy(
|
||||
request: Request, mcp_server_name: str
|
||||
@router.get("/.well-known/oauth-authorization-server")
|
||||
async def oauth_authorization_server_root(
|
||||
request: Request, mcp_server_name: Optional[str] = None
|
||||
):
|
||||
"""
|
||||
OAuth authorization server discovery for legacy /{server_name}/mcp pattern.
|
||||
"""
|
||||
return _build_oauth_authorization_server_response(
|
||||
request=request,
|
||||
mcp_server_name=mcp_server_name,
|
||||
)
|
||||
return await oauth_authorization_server_mcp(request, mcp_server_name)
|
||||
|
||||
|
||||
@router.post("/{mcp_server_name}/register")
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import datetime
|
|||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union, cast, Callable
|
||||
from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union, cast
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
|
@ -63,20 +63,7 @@ from litellm.types.mcp_server.mcp_server_manager import (
|
|||
MCPOAuthMetadata,
|
||||
MCPServer,
|
||||
)
|
||||
|
||||
try:
|
||||
from mcp.shared.tool_name_validation import SEP_986_URL, validate_tool_name # type: ignore
|
||||
except ImportError:
|
||||
SEP_986_URL = "https://github.com/modelcontextprotocol/protocol/blob/main/proposals/0001-tool-name-validation.md"
|
||||
|
||||
def validate_tool_name(name: str):
|
||||
from pydantic import BaseModel
|
||||
|
||||
class MockResult(BaseModel):
|
||||
is_valid: bool = True
|
||||
warnings: list = []
|
||||
|
||||
return MockResult()
|
||||
from mcp.shared.tool_name_validation import SEP_986_URL, validate_tool_name
|
||||
|
||||
|
||||
# Probe includes characters on both sides of the separator to mimic real prefixed tool names.
|
||||
|
|
@ -103,9 +90,7 @@ def _warn_on_server_name_fields(
|
|||
if result.is_valid:
|
||||
return
|
||||
|
||||
warning_text = (
|
||||
"; ".join(result.warnings) if result.warnings else "Validation failed"
|
||||
)
|
||||
warning_text = "; ".join(result.warnings) if result.warnings else "Validation failed"
|
||||
verbose_logger.warning(
|
||||
"MCP server '%s' has invalid %s '%s': %s",
|
||||
server_id,
|
||||
|
|
@ -118,6 +103,7 @@ def _warn_on_server_name_fields(
|
|||
_warn("server_name", server_name)
|
||||
|
||||
|
||||
|
||||
def _deserialize_json_dict(data: Any) -> Optional[Dict[str, str]]:
|
||||
"""
|
||||
Deserialize optional JSON mappings stored in the database.
|
||||
|
|
@ -405,13 +391,10 @@ class MCPServerManager:
|
|||
# Note: `extra_headers` on MCPServer is a List[str] of header names to forward
|
||||
# from the client request (not available in this OpenAPI tool generation step).
|
||||
# `static_headers` is a dict of concrete headers to always send.
|
||||
headers = (
|
||||
merge_mcp_headers(
|
||||
extra_headers=headers,
|
||||
static_headers=server.static_headers,
|
||||
)
|
||||
or {}
|
||||
)
|
||||
headers = merge_mcp_headers(
|
||||
extra_headers=headers,
|
||||
static_headers=server.static_headers,
|
||||
) or {}
|
||||
|
||||
verbose_logger.debug(
|
||||
f"Using headers for OpenAPI tools (excluding sensitive values): "
|
||||
|
|
@ -1842,7 +1825,6 @@ class MCPServerManager:
|
|||
oauth2_headers: Optional[Dict[str, str]],
|
||||
raw_headers: Optional[Dict[str, str]],
|
||||
proxy_logging_obj: Optional[ProxyLogging],
|
||||
host_progress_callback: Optional[Callable] = None,
|
||||
) -> CallToolResult:
|
||||
"""
|
||||
Call a regular MCP tool using the MCP client.
|
||||
|
|
@ -1927,7 +1909,7 @@ class MCPServerManager:
|
|||
)
|
||||
|
||||
async def _call_tool_via_client(client, params):
|
||||
return await client.call_tool(params, host_progress_callback=host_progress_callback)
|
||||
return await client.call_tool(params)
|
||||
|
||||
tasks.append(
|
||||
asyncio.create_task(_call_tool_via_client(client, call_tool_params))
|
||||
|
|
@ -1964,8 +1946,6 @@ class MCPServerManager:
|
|||
proxy_logging_obj: Optional[ProxyLogging] = None,
|
||||
oauth2_headers: Optional[Dict[str, str]] = None,
|
||||
raw_headers: Optional[Dict[str, str]] = None,
|
||||
host_progress_callback: Optional[Callable] = None,
|
||||
|
||||
) -> CallToolResult:
|
||||
"""
|
||||
Call a tool with the given name and arguments
|
||||
|
|
@ -2041,7 +2021,6 @@ class MCPServerManager:
|
|||
oauth2_headers=oauth2_headers,
|
||||
raw_headers=raw_headers,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
host_progress_callback=host_progress_callback,
|
||||
)
|
||||
|
||||
# For OpenAPI tools, await outside the client context
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue