mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-05 08:07:05 +00:00
Merge branch 'main' into litellm_dev_02_09_2026_p1
This commit is contained in:
Krish Dholakia
GitHub
commit
545b6c832a
266 changed files with 16767 additions and 2776 deletions
|
|
@ -112,6 +112,24 @@ jobs:
|
|||
python -m mypy .
|
||||
cd ..
|
||||
no_output_timeout: 10m
|
||||
|
||||
semgrep:
|
||||
docker:
|
||||
- image: cimg/python:3.12
|
||||
auth:
|
||||
username: ${DOCKERHUB_USERNAME}
|
||||
password: ${DOCKERHUB_PASSWORD}
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- setup_google_dns
|
||||
- run:
|
||||
name: Install Semgrep
|
||||
command: pip install semgrep
|
||||
- run:
|
||||
name: Run Semgrep (custom rules only)
|
||||
command: semgrep scan --config .semgrep/rules . --error
|
||||
|
||||
local_testing_part1:
|
||||
docker:
|
||||
- image: cimg/python:3.12
|
||||
|
|
@ -2277,6 +2295,7 @@ jobs:
|
|||
- run: python ./tests/code_coverage_tests/router_code_coverage.py
|
||||
- run: python ./tests/code_coverage_tests/test_chat_completion_imports.py
|
||||
- run: python ./tests/code_coverage_tests/info_log_check.py
|
||||
- run: python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py
|
||||
- run: python ./tests/code_coverage_tests/test_ban_set_verbose.py
|
||||
- run: python ./tests/code_coverage_tests/code_qa_check_tests.py
|
||||
- run: python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
|
||||
|
|
@ -3801,7 +3820,6 @@ jobs:
|
|||
- run:
|
||||
name: Get new version
|
||||
command: |
|
||||
cd litellm-proxy-extras
|
||||
NEW_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
|
||||
echo "export NEW_VERSION=$NEW_VERSION" >> $BASH_ENV
|
||||
|
||||
|
|
@ -3826,7 +3844,6 @@ jobs:
|
|||
- run:
|
||||
name: Publish to PyPI
|
||||
command: |
|
||||
cd litellm-proxy-extras
|
||||
echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc
|
||||
python -m pip install --upgrade pip build twine setuptools wheel
|
||||
rm -rf build dist
|
||||
|
|
@ -3933,6 +3950,9 @@ jobs:
|
|||
image: ubuntu-2204:2023.10.1
|
||||
resource_class: xlarge
|
||||
working_directory: ~/project
|
||||
parameters:
|
||||
browser:
|
||||
type: string
|
||||
steps:
|
||||
- checkout
|
||||
- setup_google_dns
|
||||
|
|
@ -3962,7 +3982,7 @@ jobs:
|
|||
echo "Expires at: $EXPIRES_AT"
|
||||
neon branches create \
|
||||
--project-id $NEON_PROJECT_ID \
|
||||
--name preview/commit-${CIRCLE_SHA1:0:7} \
|
||||
--name preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \
|
||||
--expires-at $EXPIRES_AT \
|
||||
--parent br-fancy-paper-ad1olsb3 \
|
||||
--api-key $NEON_API_KEY || true
|
||||
|
|
@ -3972,7 +3992,7 @@ jobs:
|
|||
E2E_UI_TEST_DATABASE_URL=$(neon connection-string \
|
||||
--project-id $NEON_PROJECT_ID \
|
||||
--api-key $NEON_API_KEY \
|
||||
--branch preview/commit-${CIRCLE_SHA1:0:7} \
|
||||
--branch preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \
|
||||
--database-name yuneng-trial-db \
|
||||
--role neondb_owner)
|
||||
echo $E2E_UI_TEST_DATABASE_URL
|
||||
|
|
@ -3984,7 +4004,7 @@ jobs:
|
|||
-e UI_USERNAME="admin" \
|
||||
-e UI_PASSWORD="gm" \
|
||||
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
||||
--name litellm-docker-database \
|
||||
--name litellm-docker-database-<< parameters.browser >> \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
|
|
@ -4000,7 +4020,7 @@ jobs:
|
|||
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
|
||||
- run:
|
||||
name: Start outputting logs
|
||||
command: docker logs -f litellm-docker-database
|
||||
command: docker logs -f litellm-docker-database-<< parameters.browser >>
|
||||
background: true
|
||||
- run:
|
||||
name: Wait for app to be ready
|
||||
|
|
@ -4009,6 +4029,7 @@ jobs:
|
|||
name: Run Playwright Tests
|
||||
command: |
|
||||
npx playwright test \
|
||||
--project << parameters.browser >> \
|
||||
--config ui/litellm-dashboard/e2e_tests/playwright.config.ts \
|
||||
--reporter=html \
|
||||
--output=test-results
|
||||
|
|
@ -4115,6 +4136,12 @@ workflows:
|
|||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- semgrep:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- local_testing_part1:
|
||||
filters:
|
||||
branches:
|
||||
|
|
@ -4214,6 +4241,20 @@ workflows:
|
|||
- main
|
||||
- /litellm_.*/
|
||||
- e2e_ui_testing:
|
||||
name: e2e_ui_testing_chromium
|
||||
browser: chromium
|
||||
context: e2e_ui_tests
|
||||
requires:
|
||||
- ui_build
|
||||
- build_docker_database_image
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- e2e_ui_testing:
|
||||
name: e2e_ui_testing_firefox
|
||||
browser: firefox
|
||||
context: e2e_ui_tests
|
||||
requires:
|
||||
- ui_build
|
||||
|
|
@ -4493,6 +4534,7 @@ workflows:
|
|||
- publish_to_pypi:
|
||||
requires:
|
||||
- mypy_linting
|
||||
- semgrep
|
||||
- local_testing_part1
|
||||
- local_testing_part2
|
||||
- build_and_test
|
||||
|
|
@ -4525,7 +4567,8 @@ workflows:
|
|||
- litellm_assistants_api_testing
|
||||
- auth_ui_unit_tests
|
||||
- db_migration_disable_update_check
|
||||
- e2e_ui_testing
|
||||
- e2e_ui_testing_chromium
|
||||
- e2e_ui_testing_firefox
|
||||
- litellm_proxy_unit_testing_key_generation
|
||||
- litellm_proxy_unit_testing_part1
|
||||
- litellm_proxy_unit_testing_part2
|
||||
|
|
|
|||
1
.github/pull_request_template.md
vendored
1
.github/pull_request_template.md
vendored
|
|
@ -9,6 +9,7 @@
|
|||
- [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
|
||||
- [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
|
||||
- [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
|
||||
- [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review
|
||||
|
||||
## CI (LiteLLM team)
|
||||
|
||||
|
|
|
|||
52
.semgrep/rules/README.md
Normal file
52
.semgrep/rules/README.md
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
# Custom Semgrep Rules
|
||||
|
||||
All `.yml` files under `.semgrep/rules/` run in CI (CircleCI `semgrep` job).
|
||||
|
||||
## Add a Rule
|
||||
|
||||
* Add a `.yml` file under `.semgrep/rules/<language>/<domain>/`
|
||||
|
||||
|
||||
[Rule syntax →](https://semgrep.dev/docs/writing-rules/rule-syntax/)
|
||||
|
||||
## Organizing Rules
|
||||
|
||||
### Structure: language → domain
|
||||
|
||||
```
|
||||
.semgrep/rules/<language>/<domain>/<rule-name>.yml
|
||||
```
|
||||
|
||||
Examples:
|
||||
|
||||
- `python/security/unsafe-yaml-load.yml`
|
||||
- `python/reliability/missing-timeout-http.yml`
|
||||
- `python/performance/blocking-io-in-async.yml`
|
||||
|
||||
### Rule metadata
|
||||
|
||||
Match tags to the folder for consistent filtering:
|
||||
|
||||
```yaml
|
||||
metadata:
|
||||
tags: [python, security]
|
||||
```
|
||||
|
||||
### Severity expectations
|
||||
|
||||
All rules must fail CI on findings. No warn-only rules.
|
||||
|
||||
- Use `severity: ERROR` in rule metadata
|
||||
- If a rule is noisy → refine until low false positives before adding
|
||||
|
||||
## Run Locally
|
||||
|
||||
```bash
|
||||
semgrep scan --config .semgrep/rules . --error
|
||||
```
|
||||
|
||||
With Semgrep registry:
|
||||
|
||||
```bash
|
||||
semgrep scan --config auto --config .semgrep/rules .
|
||||
```
|
||||
17
.semgrep/rules/python/reliability/unbounded-memory.yml
Normal file
17
.semgrep/rules/python/reliability/unbounded-memory.yml
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
# Unbounded memory growth – data structures without a clear max limit
|
||||
# Can lead to OOM under load.
|
||||
|
||||
rules:
|
||||
- id: unbounded-asyncio-queue
|
||||
message: asyncio.Queue() with no maxsize can grow unbounded. Use asyncio.Queue(maxsize=N) for integrations (e.g. log queues).
|
||||
severity: ERROR
|
||||
languages: [python]
|
||||
pattern-either:
|
||||
- pattern: asyncio.Queue()
|
||||
- pattern: asyncio.Queue(maxsize=0)
|
||||
metadata:
|
||||
category: reliability
|
||||
cwe: "CWE-400: Uncontrolled Resource Consumption"
|
||||
tags: [python, reliability]
|
||||
confidence: HIGH
|
||||
source: https://docs.python.org/3/library/asyncio-queue.html
|
||||
|
|
@ -1,3 +1,36 @@
|
|||
ignore:
|
||||
- vulnerability: CVE-2026-22184
|
||||
reason: no fixed zlib package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists
|
||||
# Wolfi base image: Python 3.13 and Node from apk have no fixed builds in Wolfi yet / not applicable
|
||||
- vulnerability: CVE-2025-55130
|
||||
reason: Node in Wolfi apk; only used for Admin UI build/prisma
|
||||
- vulnerability: CVE-2025-59465
|
||||
reason: Node in Wolfi apk; only used for Admin UI build/prisma
|
||||
- vulnerability: CVE-2025-55131
|
||||
reason: Node in Wolfi apk; only used for Admin UI build/prisma
|
||||
- vulnerability: CVE-2025-59466
|
||||
reason: Node in Wolfi apk; only used for Admin UI build/prisma
|
||||
- vulnerability: CVE-2026-21637
|
||||
reason: Node in Wolfi apk; only used for Admin UI build/prisma
|
||||
- vulnerability: CVE-2025-55132
|
||||
reason: Node in Wolfi apk; only used for Admin UI build/prisma
|
||||
- vulnerability: GHSA-hx9q-6w63-j58v
|
||||
reason: orjson dumps recursion; allowlisted
|
||||
- vulnerability: GHSA-73rr-hh4g-fpgx
|
||||
reason: diff npm transitive dep; override in package.json, allowlisted
|
||||
- vulnerability: CVE-2026-0865
|
||||
reason: Python 3.13 in Wolfi base; no fixed apk build yet
|
||||
- vulnerability: CVE-2025-15282
|
||||
reason: Python 3.13 in Wolfi base; no fixed apk build yet
|
||||
- vulnerability: CVE-2026-0672
|
||||
reason: Python 3.13 in Wolfi base; no fixed apk build yet
|
||||
- vulnerability: CVE-2025-15366
|
||||
reason: Python 3.13 in Wolfi base; no fixed apk build yet
|
||||
- vulnerability: CVE-2025-15367
|
||||
reason: Python 3.13 in Wolfi base; no fixed apk build yet
|
||||
- vulnerability: CVE-2025-11468
|
||||
reason: Python 3.13 in Wolfi base; no fixed apk build yet
|
||||
- vulnerability: CVE-2025-12781
|
||||
reason: Python 3.13 in Wolfi base; no fixed apk build yet
|
||||
- vulnerability: CVE-2026-1299
|
||||
reason: Python 3.13 in Wolfi base; no fixed apk build yet
|
||||
|
|
|
|||
|
|
@ -140,12 +140,14 @@ run_grype_scans() {
|
|||
"GHSA-34x7-hfp2-rc4v" # node-tar hardlink path traversal - not applicable, tar CLI not exposed in application code
|
||||
"GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit
|
||||
"GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel
|
||||
"CVE-2025-59465" # We do not use Node in application runtime, only used for building Admin UI
|
||||
"CVE-2025-55131" # We do not use Node in application runtime, only used for building Admin UI
|
||||
"CVE-2025-59466" # We do not use Node in application runtime, only used for building Admin UI
|
||||
"CVE-2025-55130" # We do not use Node in application runtime, only used for building Admin UI
|
||||
"CVE-2025-59467" # We do not use Node in application runtime, only used for building Admin UI
|
||||
"CVE-2026-21637" # We do not use Node in application runtime, only used for building Admin UI
|
||||
"CVE-2025-59465" # Node only used for Admin UI build/prisma
|
||||
"CVE-2025-55131" # Node only used for Admin UI build/prisma
|
||||
"CVE-2025-59466" # Node only used for Admin UI build/prisma
|
||||
"CVE-2025-55130" # Node only used for Admin UI build/prisma
|
||||
"CVE-2025-59467" # Node only used for Admin UI build/prisma
|
||||
"CVE-2026-21637" # Node only used for Admin UI build/prisma
|
||||
"CVE-2025-55132" # Node only used for Admin UI build/prisma
|
||||
"GHSA-hx9q-6w63-j58v" # orjson dumps recursion; allowlisted
|
||||
"CVE-2025-15281" # No fix available yet
|
||||
"CVE-2026-0865" # No fix available yet
|
||||
"CVE-2025-15282" # No fix available yet
|
||||
|
|
|
|||
|
|
@ -16,10 +16,14 @@ Usage:
|
|||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import pyaudio
|
||||
import websockets
|
||||
from typing import Optional
|
||||
|
||||
# Bounded queue size for audio chunks (configurable via env to avoid unbounded memory)
|
||||
AUDIO_QUEUE_MAXSIZE = int(os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 10_000))
|
||||
|
||||
# Audio configuration (matching Nova Sonic requirements)
|
||||
INPUT_SAMPLE_RATE = 16000 # Nova Sonic expects 16kHz input
|
||||
OUTPUT_SAMPLE_RATE = 24000 # Nova Sonic outputs 24kHz
|
||||
|
|
@ -40,7 +44,7 @@ class RealtimeClient:
|
|||
self.api_key = api_key
|
||||
self.ws: Optional[websockets.WebSocketClientProtocol] = None
|
||||
self.is_active = False
|
||||
self.audio_queue = asyncio.Queue()
|
||||
self.audio_queue = asyncio.Queue(maxsize=AUDIO_QUEUE_MAXSIZE)
|
||||
self.pyaudio = pyaudio.PyAudio()
|
||||
self.input_stream = None
|
||||
self.output_stream = None
|
||||
|
|
|
|||
95
docs/my-website/blog/model_cost_map_incident/index.md
Normal file
95
docs/my-website/blog/model_cost_map_incident/index.md
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
---
|
||||
slug: model-cost-map-incident
|
||||
title: "Incident Report: Invalid model cost map on main"
|
||||
date: 2026-02-10T10:00:00
|
||||
authors:
|
||||
- name: Ishaan Jaffer
|
||||
title: "CTO, LiteLLM"
|
||||
url: https://www.linkedin.com/in/ishaanjaffer/
|
||||
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
|
||||
tags: [incident-report, stability]
|
||||
hide_table_of_contents: false
|
||||
---
|
||||
|
||||
**Date:** January 27, 2026
|
||||
**Duration:** ~20 minutes
|
||||
**Severity:** Low
|
||||
**Status:** Resolved
|
||||
|
||||
## Summary
|
||||
|
||||
A malformed JSON entry in `model_prices_and_context_window.json` was merged to `main` ([`562f0a0`](https://github.com/BerriAI/litellm/commit/562f0a028251750e3d75386bee0e630d9796d0df)). This caused LiteLLM to silently fall back to a stale local copy of the model cost map. Users on older package versions lost cost tracking for newer models only (e.g. `azure/gpt-5.2`). No LLM calls were blocked.
|
||||
|
||||
- **LLM calls and proxy routing:** No impact.
|
||||
- **Cost tracking:** Impacted for newer models not present in the local backup. Older models were unaffected. The incident lasted ~20 minutes until the commit was reverted.
|
||||
|
||||
{/* truncate */}
|
||||
|
||||
---
|
||||
|
||||
## Background
|
||||
|
||||
The model cost map is not in the request path. It is used after the LLM response comes back, inside a try/catch, to calculate spend. A missing entry never blocks a call.
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A["1. litellm.completion() receives request
|
||||
litellm/main.py"] --> B["2. Route to provider
|
||||
litellm/litellm_core_utils/get_llm_provider_logic.py"]
|
||||
B --> C["3. LLM returns response
|
||||
litellm/main.py"]
|
||||
C --> D["4. Post-call: look up model in cost map
|
||||
litellm/cost_calculator.py"]
|
||||
D -->|"found"| E["5a. Attach cost to response"]
|
||||
D -->|"not found (try/catch)"| F["5b. Log warning, set cost=0"]
|
||||
E --> G["6. Return response to caller"]
|
||||
F --> G
|
||||
|
||||
style D fill:#fff3cd,stroke:#ffc107
|
||||
style F fill:#fff3cd,stroke:#ffc107
|
||||
style E fill:#d4edda,stroke:#28a745
|
||||
style G fill:#d4edda,stroke:#28a745
|
||||
```
|
||||
|
||||
Both paths return a response to the caller. When the cost map lookup fails, the only difference is `cost=0` on that request.
|
||||
|
||||
---
|
||||
|
||||
## Root cause
|
||||
|
||||
LiteLLM fetches the model cost map from GitHub `main` at import time. If the fetch fails, it falls back to a local backup bundled with the package. Before this incident, the fallback was completely silent -- no warning was logged.
|
||||
|
||||
A contributor PR introduced an extra `{` bracket, producing invalid JSON. The remote fetch failed with `JSONDecodeError`, triggering the silent fallback. Users on older package versions had backup files missing newer models.
|
||||
|
||||
**Timeline:**
|
||||
|
||||
1. Malformed JSON merged to `main`
|
||||
2. LiteLLM installations fall back to local backup on next import
|
||||
3. Users report `"This model isn't mapped yet"` for newer models
|
||||
4. Bad commit identified and reverted (~20 minutes)
|
||||
|
||||
---
|
||||
|
||||
## Remediation
|
||||
|
||||
| # | Action | Status | Code |
|
||||
|---|---|---|---|
|
||||
| 1 | CI validation on `model_prices_and_context_window.json` | ✅ Done | [`test-model-map.yaml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test-model-map.yaml) |
|
||||
| 2 | Warning log on fallback to local backup | ✅ Done | [`get_model_cost_map.py#L57-L68`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py#L57-L68) |
|
||||
| 3 | `GetModelCostMap` class with integrity validation helpers | ✅ Done | [`get_model_cost_map.py#L24-L149`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py#L24-L149) |
|
||||
| 4 | Resilience test suite (bad hosted map, fallback, completion) | ✅ Done | [`test_model_cost_map_resilience.py#L150-L291`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py#L150-L291) |
|
||||
| 5 | Test that backup model cost map always exists and contains common models | ✅ Done | [`test_model_cost_map_resilience.py#L213-L228`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py#L213-L228) |
|
||||
|
||||
Enterprises that require zero external dependencies at import time can set `LITELLM_LOCAL_MODEL_COST_MAP=True` to skip the GitHub fetch entirely.
|
||||
|
||||
---
|
||||
|
||||
## Other dependencies on external resources
|
||||
|
||||
| Dependency | Impact if unavailable | Fallback |
|
||||
|---|---|---|
|
||||
| Model cost map (GitHub) | Cost tracking for newer models | Local backup (now with warning) |
|
||||
| JWT public keys (IDP/SSO) | Auth fails | None |
|
||||
| OIDC UserInfo (IDP/SSO) | Auth fails | None |
|
||||
| HuggingFace model API | HF provider calls fail | None |
|
||||
| Ollama tags (localhost) | Ollama model list stale | Static list |
|
||||
|
|
@ -5,6 +5,13 @@ import Image from '@theme/IdealImage';
|
|||
|
||||
Benchmarks for LiteLLM Gateway (Proxy Server) tested against a fake OpenAI endpoint.
|
||||
|
||||
## Setting Up a Fake OpenAI Endpoint
|
||||
|
||||
For load testing and benchmarking, you can use a fake OpenAI proxy server. LiteLLM provides:
|
||||
|
||||
1. **Hosted endpoint**: Use our free hosted fake endpoint at `https://exampleopenaiendpoint-production.up.railway.app/`
|
||||
2. **Self-hosted**: Set up your own fake OpenAI proxy server using [github.com/BerriAI/example_openai_endpoint](https://github.com/BerriAI/example_openai_endpoint)
|
||||
|
||||
Use this config for testing:
|
||||
|
||||
```yaml
|
||||
|
|
@ -12,7 +19,7 @@ model_list:
|
|||
- model_name: "fake-openai-endpoint"
|
||||
litellm_params:
|
||||
model: openai/any
|
||||
api_base: https://your-fake-openai-endpoint.com/chat/completions
|
||||
api_base: https://exampleopenaiendpoint-production.up.railway.app/ # or your self-hosted endpoint
|
||||
api_key: "test"
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -4,8 +4,9 @@ import Image from '@theme/IdealImage';
|
|||
|
||||
## Locust Load Test LiteLLM Proxy
|
||||
|
||||
1. Add `fake-openai-endpoint` to your proxy config.yaml and start your litellm proxy
|
||||
litellm provides a free hosted `fake-openai-endpoint` you can load test against
|
||||
1. Add `fake-openai-endpoint` to your proxy config.yaml and start your litellm proxy.
|
||||
|
||||
LiteLLM provides a free hosted `fake-openai-endpoint` you can load test against. You can also self-host your own fake OpenAI proxy server using [github.com/BerriAI/example_openai_endpoint](https://github.com/BerriAI/example_openai_endpoint).
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
|
|
|
|||
|
|
@ -29,12 +29,16 @@ Tutorial on how to get to 1K+ RPS with LiteLLM Proxy on locust
|
|||
|
||||
**Note:** we're currently migrating to aiohttp which has 10x higher throughput. We recommend using the `openai/` provider for load testing.
|
||||
|
||||
:::tip Setting Up a Fake OpenAI Endpoint
|
||||
You can use our hosted fake endpoint or self-host your own using [github.com/BerriAI/example_openai_endpoint](https://github.com/BerriAI/example_openai_endpoint).
|
||||
:::
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: "fake-openai-endpoint"
|
||||
litellm_params:
|
||||
model: openai/any
|
||||
api_base: https://your-fake-openai-endpoint.com/chat/completions
|
||||
api_base: https://exampleopenaiendpoint-production.up.railway.app/ # or your self-hosted endpoint
|
||||
api_key: "test"
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,3 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# MCP OAuth
|
||||
|
||||
LiteLLM supports two OAuth 2.0 flows for MCP servers:
|
||||
|
|
@ -98,8 +95,71 @@ LiteLLM automatically fetches, caches, and refreshes OAuth2 tokens using the `cl
|
|||
|
||||
### Setup
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="config" label="config.yaml">
|
||||
You can configure M2M OAuth via the LiteLLM UI or `config.yaml`.
|
||||
|
||||
### UI Setup
|
||||
|
||||
Navigate to the **MCP Servers** page and click **+ Add New MCP Server**.
|
||||
|
||||
|
||||
|
||||
Enter a name for your server and select **HTTP** as the transport type.
|
||||
|
||||
|
||||
|
||||
Paste the MCP server URL.
|
||||
|
||||
|
||||
|
||||
Under **Authentication**, select **OAuth**.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Choose **Machine-to-Machine (M2M)** as the OAuth flow type. This is for server-to-server authentication using the `client_credentials` grant — no browser interaction required.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Fill in the **Client ID** and **Client Secret** provided by your OAuth provider.
|
||||
|
||||
|
||||
|
||||
Enter the **Token URL** — this is the endpoint LiteLLM will call to fetch access tokens using `client_credentials`.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Scroll down and review the server URL and all fields, then click **Create MCP Server**.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Once created, open the server and navigate to the **MCP Tools** tab to verify that LiteLLM can connect and list available tools.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Select a tool (e.g. **echo**) to test it. Fill in the required parameters and click **Call Tool**.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
LiteLLM automatically fetches an OAuth token behind the scenes and calls the tool. The result confirms the M2M OAuth flow is working end-to-end.
|
||||
|
||||
|
||||
|
||||
### Config.yaml Setup
|
||||
|
||||
```yaml title="config.yaml" showLineNumbers
|
||||
mcp_servers:
|
||||
|
|
@ -112,14 +172,6 @@ mcp_servers:
|
|||
scopes: ["mcp:read", "mcp:write"] # optional
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="ui" label="LiteLLM UI">
|
||||
|
||||
Navigate to **MCP Servers → Add Server → Authentication → OAuth**, then fill in `client_id`, `client_secret`, and `token_url`.
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### How It Works
|
||||
|
||||
1. On first MCP request, LiteLLM POSTs to `token_url` with `grant_type=client_credentials`
|
||||
|
|
|
|||
|
|
@ -556,3 +556,147 @@ for event in response.get("completion"):
|
|||
|
||||
print(completion)
|
||||
```
|
||||
|
||||
## Using LangChain AWS SDK with LiteLLM
|
||||
|
||||
You can use the [LangChain AWS SDK](https://python.langchain.com/docs/integrations/chat/bedrock/) with LiteLLM Proxy to get cost tracking, load balancing, and other LiteLLM features.
|
||||
|
||||
### Quick Start
|
||||
|
||||
**1. Install LangChain AWS**:
|
||||
|
||||
```bash showLineNumbers
|
||||
pip install langchain-aws
|
||||
```
|
||||
|
||||
**2. Setup LiteLLM Proxy**:
|
||||
|
||||
Create a `config.yaml`:
|
||||
|
||||
```yaml showLineNumbers
|
||||
model_list:
|
||||
- model_name: claude-sonnet
|
||||
litellm_params:
|
||||
model: bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0
|
||||
aws_region_name: us-east-1
|
||||
custom_llm_provider: bedrock
|
||||
```
|
||||
|
||||
Start the proxy:
|
||||
|
||||
```bash showLineNumbers
|
||||
export AWS_ACCESS_KEY_ID="your-access-key"
|
||||
export AWS_SECRET_ACCESS_KEY="your-secret-key"
|
||||
|
||||
litellm --config config.yaml
|
||||
|
||||
# RUNNING on http://0.0.0.0:4000
|
||||
```
|
||||
|
||||
**3. Use LangChain with LiteLLM**:
|
||||
|
||||
```python showLineNumbers
|
||||
from langchain_aws import ChatBedrockConverse
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
# Your LiteLLM API key
|
||||
API_KEY = "Bearer sk-1234"
|
||||
|
||||
# Initialize ChatBedrockConverse pointing to LiteLLM proxy
|
||||
llm = ChatBedrockConverse(
|
||||
model_id="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
endpoint_url="http://localhost:4000/bedrock",
|
||||
region_name="us-east-1",
|
||||
aws_access_key_id=API_KEY,
|
||||
aws_secret_access_key="bedrock" # Any non-empty value works
|
||||
)
|
||||
|
||||
# Invoke the model
|
||||
messages = [HumanMessage(content="Hello, how are you?")]
|
||||
response = llm.invoke(messages)
|
||||
|
||||
print(response.content)
|
||||
```
|
||||
|
||||
### Advanced Example: PDF Document Processing with Citations
|
||||
|
||||
LangChain AWS SDK supports Bedrock's document processing features. Here's how to use it with LiteLLM:
|
||||
|
||||
```python showLineNumbers
|
||||
import os
|
||||
import json
|
||||
from langchain_aws import ChatBedrockConverse
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
# Your LiteLLM API key
|
||||
API_KEY = "Bearer sk-1234"
|
||||
|
||||
def get_llm() -> ChatBedrockConverse:
|
||||
"""Initialize LLM pointing to LiteLLM proxy"""
|
||||
llm = ChatBedrockConverse(
|
||||
model_id="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
base_model_id="anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
endpoint_url="http://localhost:4000/bedrock",
|
||||
region_name="us-east-1",
|
||||
aws_access_key_id=API_KEY,
|
||||
aws_secret_access_key="bedrock"
|
||||
)
|
||||
return llm
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Initialize the LLM
|
||||
llm = get_llm()
|
||||
|
||||
# Read PDF file as bytes (Converse API requires raw bytes)
|
||||
with open("your-document.pdf", "rb") as file:
|
||||
file_bytes = file.read()
|
||||
|
||||
# Prepare messages with document attachment
|
||||
messages = [
|
||||
HumanMessage(content=[
|
||||
{"text": "What is the policy number in this document?"},
|
||||
{
|
||||
"document": {
|
||||
"format": "pdf",
|
||||
"name": "PolicyDocument",
|
||||
"source": {"bytes": file_bytes},
|
||||
"citations": {"enabled": True}
|
||||
}
|
||||
}
|
||||
])
|
||||
]
|
||||
|
||||
# Invoke the LLM
|
||||
response = llm.invoke(messages)
|
||||
|
||||
# Print response with citations
|
||||
print(json.dumps(response.content, indent=4))
|
||||
```
|
||||
|
||||
### Supported LangChain Features
|
||||
|
||||
All LangChain AWS features work with LiteLLM:
|
||||
|
||||
| Feature | Supported | Notes |
|
||||
|---------|-----------|-------|
|
||||
| Text Generation | ✅ | Full support |
|
||||
| Streaming | ✅ | Use `stream()` method |
|
||||
| Document Processing | ✅ | PDF, images, etc. |
|
||||
| Citations | ✅ | Enable in document config |
|
||||
| Tool Use | ✅ | Function calling support |
|
||||
| Multi-modal | ✅ | Text + images + documents |
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
**Issue**: `UnknownOperationException` error
|
||||
|
||||
**Solution**: Make sure you're using the correct endpoint URL format:
|
||||
- ✅ Correct: `http://localhost:4000/bedrock`
|
||||
- ❌ Wrong: `http://localhost:4000/bedrock/v2`
|
||||
|
||||
**Issue**: Authentication errors
|
||||
|
||||
**Solution**: Ensure your API key is in the correct format:
|
||||
```python
|
||||
aws_access_key_id="Bearer sk-1234" # Include "Bearer " prefix
|
||||
```
|
||||
|
|
|
|||
|
|
@ -120,6 +120,293 @@ All models listed here https://docs.perplexity.ai/docs/model-cards are supported
|
|||
|
||||
|
||||
|
||||
## Agentic Research API (Responses API)
|
||||
|
||||
Requires v1.72.6+
|
||||
|
||||
|
||||
### Using Presets
|
||||
|
||||
Presets provide optimized defaults for specific use cases. Start with a preset for quick setup:
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import responses
|
||||
import os
|
||||
|
||||
os.environ['PERPLEXITY_API_KEY'] = ""
|
||||
|
||||
# Using the pro-search preset
|
||||
response = responses(
|
||||
model="perplexity/preset/pro-search",
|
||||
input="What are the latest developments in AI?",
|
||||
custom_llm_provider="perplexity",
|
||||
)
|
||||
|
||||
print(response.output)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="Proxy">
|
||||
|
||||
1. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: perplexity-pro-search
|
||||
litellm_params:
|
||||
model: perplexity/preset/pro-search
|
||||
api_key: os.environ/PERPLEXITY_API_KEY
|
||||
```
|
||||
|
||||
2. Start proxy
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
3. Test it!
|
||||
|
||||
```bash
|
||||
curl http://0.0.0.0:4000/v1/responses \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer anything" \
|
||||
-d '{
|
||||
"model": "perplexity-pro-search",
|
||||
"input": "What are the latest developments in AI?"
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Using Third-Party Models
|
||||
|
||||
Access models from OpenAI, Anthropic, Google, xAI, and other providers through Perplexity's unified API:
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="openai" label="OpenAI">
|
||||
|
||||
```python
|
||||
from litellm import responses
|
||||
import os
|
||||
|
||||
os.environ['PERPLEXITY_API_KEY'] = ""
|
||||
|
||||
response = responses(
|
||||
model="perplexity/openai/gpt-4o",
|
||||
input="Explain quantum computing in simple terms",
|
||||
custom_llm_provider="perplexity",
|
||||
max_output_tokens=500,
|
||||
)
|
||||
|
||||
print(response.output)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="anthropic" label="Anthropic">
|
||||
|
||||
```python
|
||||
from litellm import responses
|
||||
import os
|
||||
|
||||
os.environ['PERPLEXITY_API_KEY'] = ""
|
||||
|
||||
response = responses(
|
||||
model="perplexity/anthropic/claude-3-5-sonnet-20241022",
|
||||
input="Write a short story about a robot learning to paint",
|
||||
custom_llm_provider="perplexity",
|
||||
max_output_tokens=500,
|
||||
)
|
||||
|
||||
print(response.output)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="google" label="Google">
|
||||
|
||||
```python
|
||||
from litellm import responses
|
||||
import os
|
||||
|
||||
os.environ['PERPLEXITY_API_KEY'] = ""
|
||||
|
||||
response = responses(
|
||||
model="perplexity/google/gemini-2.0-flash-exp",
|
||||
input="Explain the concept of neural networks",
|
||||
custom_llm_provider="perplexity",
|
||||
max_output_tokens=500,
|
||||
)
|
||||
|
||||
print(response.output)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="xai" label="xAI">
|
||||
|
||||
```python
|
||||
from litellm import responses
|
||||
import os
|
||||
|
||||
os.environ['PERPLEXITY_API_KEY'] = ""
|
||||
|
||||
response = responses(
|
||||
model="perplexity/xai/grok-2-1212",
|
||||
input="What makes a good AI assistant?",
|
||||
custom_llm_provider="perplexity",
|
||||
max_output_tokens=500,
|
||||
)
|
||||
|
||||
print(response.output)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Web Search Tool
|
||||
|
||||
Enable web search capabilities to access real-time information:
|
||||
|
||||
```python
|
||||
from litellm import responses
|
||||
import os
|
||||
|
||||
os.environ['PERPLEXITY_API_KEY'] = ""
|
||||
|
||||
response = responses(
|
||||
model="perplexity/openai/gpt-4o",
|
||||
input="What's the weather in San Francisco today?",
|
||||
custom_llm_provider="perplexity",
|
||||
tools=[{"type": "web_search"}],
|
||||
instructions="You have access to a web_search tool. Use it for questions about current events.",
|
||||
)
|
||||
|
||||
print(response.output)
|
||||
```
|
||||
|
||||
|
||||
### Reasoning Effort (Responses API)
|
||||
|
||||
Control the reasoning effort level for reasoning-capable models:
|
||||
|
||||
```python
|
||||
from litellm import responses
|
||||
import os
|
||||
|
||||
os.environ['PERPLEXITY_API_KEY'] = ""
|
||||
|
||||
response = responses(
|
||||
model="perplexity/openai/gpt-5.2",
|
||||
input="Solve this complex problem step by step",
|
||||
custom_llm_provider="perplexity",
|
||||
reasoning={"effort": "high"}, # Options: low, medium, high
|
||||
max_output_tokens=1000,
|
||||
)
|
||||
|
||||
print(response.output)
|
||||
```
|
||||
|
||||
### Multi-Turn Conversations
|
||||
|
||||
Use message arrays for multi-turn conversations with context:
|
||||
|
||||
```python
|
||||
from litellm import responses
|
||||
import os
|
||||
|
||||
os.environ['PERPLEXITY_API_KEY'] = ""
|
||||
|
||||
response = responses(
|
||||
model="perplexity/anthropic/claude-3-5-sonnet-20241022",
|
||||
input=[
|
||||
{"type": "message", "role": "system", "content": "You are a helpful assistant."},
|
||||
{"type": "message", "role": "user", "content": "What are the latest AI developments?"},
|
||||
],
|
||||
custom_llm_provider="perplexity",
|
||||
instructions="Provide detailed, well-researched answers.",
|
||||
max_output_tokens=800,
|
||||
)
|
||||
|
||||
print(response.output)
|
||||
```
|
||||
|
||||
### Streaming Responses
|
||||
|
||||
Stream responses for real-time output:
|
||||
|
||||
```python
|
||||
from litellm import responses
|
||||
import os
|
||||
|
||||
os.environ['PERPLEXITY_API_KEY'] = ""
|
||||
|
||||
response = responses(
|
||||
model="perplexity/openai/gpt-4o",
|
||||
input="Tell me a story about space exploration",
|
||||
custom_llm_provider="perplexity",
|
||||
stream=True,
|
||||
max_output_tokens=500,
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
if hasattr(chunk, 'type'):
|
||||
if chunk.type == "response.output_text.delta":
|
||||
print(chunk.delta, end="", flush=True)
|
||||
```
|
||||
|
||||
### Supported Third-Party Models
|
||||
|
||||
| Provider | Model Name | Function Call |
|
||||
|----------|------------|---------------|
|
||||
| OpenAI | gpt-4o | `responses(model="perplexity/openai/gpt-4o", ...)` |
|
||||
| OpenAI | gpt-4o-mini | `responses(model="perplexity/openai/gpt-4o-mini", ...)` |
|
||||
| OpenAI | gpt-5.2 | `responses(model="perplexity/openai/gpt-5.2", ...)` |
|
||||
| Anthropic | claude-3-5-sonnet-20241022 | `responses(model="perplexity/anthropic/claude-3-5-sonnet-20241022", ...)` |
|
||||
| Anthropic | claude-3-5-haiku-20241022 | `responses(model="perplexity/anthropic/claude-3-5-haiku-20241022", ...)` |
|
||||
| Google | gemini-2.0-flash-exp | `responses(model="perplexity/google/gemini-2.0-flash-exp", ...)` |
|
||||
| Google | gemini-2.0-flash-thinking-exp | `responses(model="perplexity/google/gemini-2.0-flash-thinking-exp", ...)` |
|
||||
| xAI | grok-2-1212 | `responses(model="perplexity/xai/grok-2-1212", ...)` |
|
||||
| xAI | grok-2-vision-1212 | `responses(model="perplexity/xai/grok-2-vision-1212", ...)` |
|
||||
|
||||
### Available Presets
|
||||
|
||||
| Preset Name | Function Call |
|
||||
|----------------|--------------------------------------------------------|
|
||||
| fast-search | `responses(model="perplexity/preset/fast-search", ...)`|
|
||||
| pro-search | `responses(model="perplexity/preset/pro-search", ...)` |
|
||||
| deep-research | `responses(model="perplexity/preset/deep-research", ...)`|
|
||||
|
||||
### Complete Example
|
||||
|
||||
```python
|
||||
from litellm import responses
|
||||
import os
|
||||
|
||||
os.environ['PERPLEXITY_API_KEY'] = ""
|
||||
|
||||
# Comprehensive example with multiple features
|
||||
response = responses(
|
||||
model="perplexity/openai/gpt-4o",
|
||||
input="Research the latest developments in quantum computing and provide sources",
|
||||
custom_llm_provider="perplexity",
|
||||
tools=[
|
||||
{"type": "web_search"},
|
||||
{"type": "fetch_url"}
|
||||
],
|
||||
instructions="Use web_search to find relevant information and fetch_url to retrieve detailed content from sources. Provide citations for all claims.",
|
||||
max_output_tokens=1000,
|
||||
temperature=0.7,
|
||||
)
|
||||
|
||||
print(f"Response ID: {response.id}")
|
||||
print(f"Model: {response.model}")
|
||||
print(f"Status: {response.status}")
|
||||
print(f"Output: {response.output}")
|
||||
print(f"Usage: {response.usage}")
|
||||
```
|
||||
|
||||
:::info
|
||||
|
||||
For more information about passing provider-specific parameters, [go here](../completion/provider_specific_params.md)
|
||||
|
|
|
|||
|
|
@ -223,6 +223,7 @@ GENERIC_USER_FIRST_NAME_ATTRIBUTE = "first_name"
|
|||
GENERIC_USER_LAST_NAME_ATTRIBUTE = "last_name"
|
||||
GENERIC_USER_ROLE_ATTRIBUTE = "given_role"
|
||||
GENERIC_USER_PROVIDER_ATTRIBUTE = "provider"
|
||||
GENERIC_USER_EXTRA_ATTRIBUTES = "department,employee_id,manager" # comma-separated list of additional fields to extract from SSO response
|
||||
GENERIC_CLIENT_STATE = "some-state" # if the provider needs a state parameter
|
||||
GENERIC_INCLUDE_CLIENT_ID = "false" # some providers enforce that the client_id is not in the body
|
||||
GENERIC_SCOPE = "openid profile email" # default scope openid is sometimes not enough to retrieve basic user info like first_name and last_name located in profile scope
|
||||
|
|
@ -239,6 +240,40 @@ Use `GENERIC_USER_ROLE_ATTRIBUTE` to specify which attribute in the SSO token co
|
|||
|
||||
Nested attribute paths are supported (e.g., `claims.role` or `attributes.litellm_role`).
|
||||
|
||||
**Capturing Additional SSO Fields**
|
||||
|
||||
Use `GENERIC_USER_EXTRA_ATTRIBUTES` to extract additional fields from the SSO provider response beyond the standard user attributes (id, email, name, etc.). This is useful when you need to access custom organization-specific data (e.g., department, employee ID, groups) in your [custom SSO handler](./custom_sso.md).
|
||||
|
||||
```shell
|
||||
# Comma-separated list of field names to extract
|
||||
GENERIC_USER_EXTRA_ATTRIBUTES="department,employee_id,manager,groups"
|
||||
```
|
||||
|
||||
**Accessing Extra Fields in Custom SSO Handler:**
|
||||
|
||||
```python
|
||||
from litellm.proxy.management_endpoints.types import CustomOpenID
|
||||
|
||||
async def custom_sso_handler(userIDPInfo: CustomOpenID):
|
||||
# Access the extra fields
|
||||
extra_fields = getattr(userIDPInfo, 'extra_fields', None) or {}
|
||||
|
||||
user_department = extra_fields.get("department")
|
||||
employee_id = extra_fields.get("employee_id")
|
||||
user_groups = extra_fields.get("groups", [])
|
||||
|
||||
# Use these fields for custom logic (e.g., team assignment, access control)
|
||||
# ...
|
||||
```
|
||||
|
||||
**Nested Field Paths:**
|
||||
|
||||
Dot notation is supported for nested fields:
|
||||
|
||||
```shell
|
||||
GENERIC_USER_EXTRA_ATTRIBUTES="org_info.department,org_info.cost_center,metadata.employee_type"
|
||||
```
|
||||
|
||||
- Set Redirect URI, if your provider requires it
|
||||
- Set a redirect url = `<your proxy base url>/sso/callback`
|
||||
```shell
|
||||
|
|
|
|||
|
|
@ -395,7 +395,7 @@ router_settings:
|
|||
| ATHINA_API_KEY | API key for Athina service
|
||||
| ATHINA_BASE_URL | Base URL for Athina service (defaults to `https://log.athina.ai`)
|
||||
| AUTH_STRATEGY | Strategy used for authentication (e.g., OAuth, API key)
|
||||
| AUTO_REDIRECT_UI_LOGIN_TO_SSO | Flag to enable automatic redirect of UI login page to SSO when SSO is configured. Default is **true**
|
||||
| AUTO_REDIRECT_UI_LOGIN_TO_SSO | Flag to enable automatic redirect of UI login page to SSO when SSO is configured. Default is **false**
|
||||
| AUDIO_SPEECH_CHUNK_SIZE | Chunk size for audio speech processing. Default is 1024
|
||||
| ANTHROPIC_API_KEY | API key for Anthropic service
|
||||
| ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com
|
||||
|
|
@ -548,6 +548,10 @@ router_settings:
|
|||
| DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL | Default embedding model for MCP semantic tool filtering. Default is "text-embedding-3-small"
|
||||
| DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD | Default similarity threshold for MCP semantic tool filtering. Default is 0.3
|
||||
| DEFAULT_MCP_SEMANTIC_FILTER_TOP_K | Default number of top results to return for MCP semantic tool filtering. Default is 10
|
||||
| MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL | Default TTL in seconds for MCP OAuth2 token cache. Default is 3600
|
||||
| MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE | Maximum number of entries in MCP OAuth2 token cache. Default is 200
|
||||
| MCP_OAUTH2_TOKEN_CACHE_MIN_TTL | Minimum TTL in seconds for MCP OAuth2 token cache. Default is 10
|
||||
| MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS | Seconds to subtract from token expiry when computing cache TTL. Default is 60
|
||||
| DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20
|
||||
| DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10
|
||||
| DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602
|
||||
|
|
@ -640,6 +644,7 @@ router_settings:
|
|||
| GENERIC_TOKEN_ENDPOINT | Token endpoint for generic OAuth providers
|
||||
| GENERIC_USER_DISPLAY_NAME_ATTRIBUTE | Attribute for user's display name in generic auth
|
||||
| GENERIC_USER_EMAIL_ATTRIBUTE | Attribute for user's email in generic auth
|
||||
| GENERIC_USER_EXTRA_ATTRIBUTES | Comma-separated list of additional fields to extract from generic SSO provider response (e.g., "department,employee_id,groups"). Accessible via `CustomOpenID.extra_fields` in custom SSO handlers. Supports dot notation for nested fields
|
||||
| GENERIC_USER_FIRST_NAME_ATTRIBUTE | Attribute for user's first name in generic auth
|
||||
| GENERIC_USER_ID_ATTRIBUTE | Attribute for user ID in generic auth
|
||||
| GENERIC_USER_LAST_NAME_ATTRIBUTE | Attribute for user's last name in generic auth
|
||||
|
|
@ -779,6 +784,7 @@ router_settings:
|
|||
| LITELLM_USER_AGENT | Custom user agent string for LiteLLM API requests. Used for partner telemetry attribution
|
||||
| LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging
|
||||
| LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
|
||||
| LITELLM_ASYNCIO_QUEUE_MAXSIZE | Maximum size for asyncio queues (e.g. log queues, spend update queues, and cookbook examples such as realtime audio in `nova_sonic_realtime.py`). Bounds in-memory growth to prevent OOM. Default is 1000.
|
||||
| LOGFIRE_TOKEN | Token for Logfire logging service
|
||||
| LOGFIRE_BASE_URL | Base URL for Logfire logging service (useful for self hosted deployments)
|
||||
| LOGGING_WORKER_CONCURRENCY | Maximum number of concurrent coroutine slots for the logging worker on the asyncio event loop. Default is 100. Setting too high will flood the event loop with logging tasks which will lower the overall latency of the requests.
|
||||
|
|
@ -806,6 +812,7 @@ router_settings:
|
|||
| MAX_RETRY_DELAY | Maximum delay in seconds for retrying requests. Default is 8.0
|
||||
| MAX_LANGFUSE_INITIALIZED_CLIENTS | Maximum number of Langfuse clients to initialize on proxy. Default is 50. This is set since langfuse initializes 1 thread everytime a client is initialized. We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times.
|
||||
| MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH | Maximum header length for MCP semantic filter tools. Default is 150
|
||||
| MAX_POLICY_ESTIMATE_IMPACT_ROWS | Maximum number of rows returned when estimating the impact of a policy. Default is 1000
|
||||
| MIN_NON_ZERO_TEMPERATURE | Minimum non-zero temperature value. Default is 0.0001
|
||||
| MINIMUM_PROMPT_CACHE_TOKEN_COUNT | Minimum token count for caching a prompt. Default is 1024
|
||||
| MISTRAL_API_BASE | Base URL for Mistral API. Default is https://api.mistral.ai
|
||||
|
|
@ -822,6 +829,8 @@ router_settings:
|
|||
| MICROSOFT_USER_ID_ATTRIBUTE | Field name for user ID in Microsoft SSO response. Default is `id`
|
||||
| MICROSOFT_USER_LAST_NAME_ATTRIBUTE | Field name for user last name in Microsoft SSO response. Default is `surname`
|
||||
| MICROSOFT_USERINFO_ENDPOINT | Custom userinfo endpoint URL for Microsoft SSO (overrides default Microsoft Graph userinfo endpoint)
|
||||
| MODEL_COST_MAP_MAX_SHRINK_RATIO | Maximum allowed shrinkage ratio when validating a fetched model cost map against the local backup. Rejects the fetched map if it is smaller than this fraction of the backup. Default is 0.5
|
||||
| MODEL_COST_MAP_MIN_MODEL_COUNT | Minimum number of models a fetched cost map must contain to be considered valid. Default is 50
|
||||
| NO_DOCS | Flag to disable Swagger UI documentation
|
||||
| NO_REDOC | Flag to disable Redoc documentation
|
||||
| NO_PROXY | List of addresses to bypass proxy
|
||||
|
|
|
|||
|
|
@ -142,6 +142,18 @@ async def custom_sso_handler(userIDPInfo: OpenID) -> SSOUserDefinedValues:
|
|||
f"No ID found for user. userIDPInfo.id is None {userIDPInfo}"
|
||||
)
|
||||
|
||||
#################################################
|
||||
# Access extra fields from SSO provider (requires GENERIC_USER_EXTRA_ATTRIBUTES env var)
|
||||
# Example: Set GENERIC_USER_EXTRA_ATTRIBUTES="department,employee_id,groups"
|
||||
extra_fields = getattr(userIDPInfo, 'extra_fields', None) or {}
|
||||
user_department = extra_fields.get("department")
|
||||
employee_id = extra_fields.get("employee_id")
|
||||
user_groups = extra_fields.get("groups", [])
|
||||
|
||||
print(f"User department: {user_department}") # noqa
|
||||
print(f"Employee ID: {employee_id}") # noqa
|
||||
print(f"User groups: {user_groups}") # noqa
|
||||
#################################################
|
||||
|
||||
#################################################
|
||||
# Run your custom code / logic here
|
||||
|
|
|
|||
|
|
@ -1,3 +1,7 @@
|
|||
import Image from '@theme/IdealImage';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# [Beta] Guardrail Policies
|
||||
|
||||
Use policies to group guardrails and control which ones run for specific teams, keys, or models.
|
||||
|
|
@ -10,6 +14,9 @@ Use policies to group guardrails and control which ones run for specific teams,
|
|||
|
||||
## Quick Start
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="config" label="config.yaml">
|
||||
|
||||
```yaml showLineNumbers title="config.yaml"
|
||||
model_list:
|
||||
- model_name: gpt-4
|
||||
|
|
@ -43,6 +50,26 @@ policy_attachments:
|
|||
scope: "*" # apply to all requests
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="ui" label="UI (LiteLLM Dashboard)">
|
||||
|
||||
**Step 1: Create a Policy**
|
||||
|
||||
Go to **Policies** tab and click **+ Create New Policy**. Fill in the policy name, description, and select guardrails to add.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
Response headers show what ran:
|
||||
|
||||
```
|
||||
|
|
@ -58,6 +85,9 @@ x-litellm-applied-guardrails: pii_masking,prompt_injection
|
|||
|
||||
You have a global baseline, but want to add extra guardrails for a specific team.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="config" label="config.yaml">
|
||||
|
||||
```yaml showLineNumbers title="config.yaml"
|
||||
policies:
|
||||
global-baseline:
|
||||
|
|
@ -81,6 +111,30 @@ policy_attachments:
|
|||
- finance # team alias from /team/new
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="ui" label="UI (LiteLLM Dashboard)">
|
||||
|
||||
**Option 1: Create a team-scoped attachment**
|
||||
|
||||
Go to **Policies** > **Attachments** tab and click **+ Create New Attachment**. Select the policy and the teams to scope it to.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
**Option 2: Attach from team settings**
|
||||
|
||||
Go to **Teams** > click on a team > **Settings** tab > under **Policies**, select the policies to attach.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<Image img={require('../../../img/policy_team_attach.png')} />
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
Now the `finance` team gets `pii_masking` + `strict_compliance_check` + `audit_logger`, while everyone else just gets `pii_masking`.
|
||||
|
||||
## Remove guardrails for a specific team
|
||||
|
|
@ -201,6 +255,60 @@ policy_attachments:
|
|||
- "test-*" # key alias pattern
|
||||
```
|
||||
|
||||
**Tag-based** (matches keys/teams by metadata tags, wildcards supported):
|
||||
|
||||
```yaml showLineNumbers title="config.yaml"
|
||||
policy_attachments:
|
||||
- policy: hipaa-compliance
|
||||
tags:
|
||||
- "healthcare"
|
||||
- "health-*" # wildcard - matches health-team, health-dev, etc.
|
||||
```
|
||||
|
||||
Tags are read from key and team `metadata.tags`. For example, a key created with `metadata: {"tags": ["healthcare"]}` would match the attachment above.
|
||||
|
||||
## Test Policy Matching
|
||||
|
||||
Debug which policies and guardrails apply for a given context. Use this to verify your policy configuration before deploying.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="ui" label="UI (LiteLLM Dashboard)">
|
||||
|
||||
Go to **Policies** > **Test** tab. Enter a team alias, key alias, model, or tags and click **Test** to see which policies match and what guardrails would be applied.
|
||||
|
||||
<Image img={require('../../../img/policy_test_matching.png')} />
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="api" label="API">
|
||||
|
||||
```bash
|
||||
curl -X POST "http://localhost:4000/policies/resolve" \
|
||||
-H "Authorization: Bearer <your_api_key>" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"tags": ["healthcare"],
|
||||
"model": "gpt-4"
|
||||
}'
|
||||
```
|
||||
|
||||
Response:
|
||||
|
||||
```json
|
||||
{
|
||||
"effective_guardrails": ["pii_masking"],
|
||||
"matched_policies": [
|
||||
{
|
||||
"policy_name": "hipaa-compliance",
|
||||
"matched_via": "tag:healthcare",
|
||||
"guardrails_added": ["pii_masking"]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Config Reference
|
||||
|
||||
### `policies`
|
||||
|
|
@ -233,14 +341,18 @@ policy_attachments:
|
|||
scope: ...
|
||||
teams: [...]
|
||||
keys: [...]
|
||||
models: [...]
|
||||
tags: [...]
|
||||
```
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `policy` | `string` | **Required.** Name of the policy to attach. |
|
||||
| `scope` | `string` | Use `"*"` to apply globally. |
|
||||
| `teams` | `list[string]` | Team aliases (from `/team/new`). |
|
||||
| `teams` | `list[string]` | Team aliases (from `/team/new`). Supports `*` wildcard. |
|
||||
| `keys` | `list[string]` | Key aliases (from `/key/generate`). Supports `*` wildcard. |
|
||||
| `models` | `list[string]` | Model names. Supports `*` wildcard. |
|
||||
| `tags` | `list[string]` | Tag patterns (from key/team `metadata.tags`). Supports `*` wildcard. |
|
||||
|
||||
### Response Headers
|
||||
|
||||
|
|
@ -248,6 +360,7 @@ policy_attachments:
|
|||
|--------|-------------|
|
||||
| `x-litellm-applied-policies` | Policies that matched this request |
|
||||
| `x-litellm-applied-guardrails` | Guardrails that actually ran |
|
||||
| `x-litellm-policy-sources` | Why each policy matched (e.g., `hipaa=tag:healthcare; baseline=scope:*`) |
|
||||
|
||||
## How it works
|
||||
|
||||
|
|
|
|||
139
docs/my-website/docs/proxy/guardrails/policy_tags.md
Normal file
139
docs/my-website/docs/proxy/guardrails/policy_tags.md
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
# Tag-Based Policy Attachments
|
||||
|
||||
Apply guardrail policies automatically to any key or team that has a specific tag. Instead of attaching policies one-by-one, tag your keys and let the policy engine handle the rest.
|
||||
|
||||
**Example:** Your security team requires all healthcare-related keys to run PII masking and PHI detection. Tag those keys with `health`, create a single tag-based attachment, and every matching key gets the guardrails automatically.
|
||||
|
||||
## 1. Create a Policy with Guardrails
|
||||
|
||||
Navigate to **Policies** in the left sidebar. You'll see a list of existing policies along with their guardrails.
|
||||
|
||||
|
||||
|
||||
Click **+ Add New Policy**. In the modal, enter a name for your policy (e.g., `high-risk-policy2`). You can also type to search existing policy names if you want to reference them.
|
||||
|
||||
|
||||
|
||||
Scroll down to **Guardrails to Add**. Click the dropdown to see all available guardrails configured on your proxy — select the ones this policy should enforce.
|
||||
|
||||
|
||||
|
||||
After selecting your guardrails, they appear as chips in the input field. The **Resolved Guardrails** section below shows the final set that will be applied (including any inherited from a parent policy).
|
||||
|
||||
|
||||
|
||||
Click **Create Policy** to save.
|
||||
|
||||
|
||||
|
||||
## 2. Add a Tag Attachment for the Policy
|
||||
|
||||
After creating the policy, switch to the **Attachments** tab. This is where you define *where* the policy applies.
|
||||
|
||||
|
||||
|
||||
Click **+ Add New Attachment**. The Attachments page explains the available scopes: Global, Teams, Keys, Models, and **Tags**.
|
||||
|
||||
|
||||
|
||||
In the **Create Policy Attachment** modal, first select the policy you just created from the dropdown.
|
||||
|
||||
|
||||
|
||||
Choose **Specific (teams, keys, models, or tags)** as the scope type. This expands the form to show fields for Teams, Keys, Models, and Tags.
|
||||
|
||||
|
||||
|
||||
Scroll down to the **Tags** field and type the tag to match — here we enter `health`. You can enter any string, or use a wildcard pattern like `health-*` to match all tags starting with `health-` (e.g., `health-team`, `health-dev`).
|
||||
|
||||
|
||||
|
||||
## 3. Check the Impact of the Attachment
|
||||
|
||||
Before creating the attachment, click **Estimate Impact** to preview how many keys and teams would be affected. This is your blast-radius check — make sure the scope is what you expect before applying.
|
||||
|
||||
|
||||
|
||||
The **Impact Preview** appears inline, showing exactly how many keys and teams would be affected. In this example: "This attachment would affect **1 key** and **0 teams**", with the key alias `hi` listed.
|
||||
|
||||
|
||||
|
||||
Once you're satisfied with the impact, click **Create Attachment** to save.
|
||||
|
||||
|
||||
|
||||
The attachment now appears in the table with the policy name `high-risk-policy2` and tag `health` visible.
|
||||
|
||||
|
||||
|
||||
## 4. Create a Key with the Tag
|
||||
|
||||
Navigate to **Virtual Keys** in the left sidebar. Click **+ Create New Key**.
|
||||
|
||||
|
||||
|
||||
Enter a key name and select a model. Then expand **Optional Settings** and scroll down to the **Tags** field.
|
||||
|
||||
|
||||
|
||||
In the **Tags** field, type `health` and press Enter. This is the tag the policy engine will match against.
|
||||
|
||||
|
||||
|
||||
The tag `health` now appears as a chip in the Tags field. Confirm your settings look correct.
|
||||
|
||||
|
||||
|
||||
Click **Create Key** at the bottom of the form.
|
||||
|
||||
|
||||
|
||||
A dialog appears with your new virtual key. Click **Copy Virtual Key** — you'll need this to test in the next step.
|
||||
|
||||
|
||||
|
||||
## 5. Test the Key and Validate the Policy is Applied
|
||||
|
||||
Navigate to **Playground** in the left sidebar to test the key interactively.
|
||||
|
||||
|
||||
|
||||
Under **Virtual Key Source**, select "Virtual Key" and paste the key you just copied into the input field.
|
||||
|
||||
|
||||
|
||||
Select a model from the **Select Model** dropdown.
|
||||
|
||||
|
||||
|
||||
Type a message and press Enter. If a guardrail blocks the request, you'll see it in the response. In this example, the `testing-pl` guardrail detected an email pattern and returned a 403 error — confirming the policy is working.
|
||||
|
||||
|
||||
|
||||
**Using curl:**
|
||||
|
||||
You can also verify via the command line. The response headers confirm which policies and guardrails were applied:
|
||||
|
||||
```bash
|
||||
curl -v http://localhost:4000/chat/completions \
|
||||
-H "Authorization: Bearer <your-tagged-key>" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "gpt-4o",
|
||||
"messages": [{"role": "user", "content": "say hi"}]
|
||||
}'
|
||||
```
|
||||
|
||||
Check the response headers:
|
||||
|
||||
```
|
||||
x-litellm-applied-policies: high-risk-policy2
|
||||
x-litellm-applied-guardrails: pii-pre-guard,phi-pre-guard,testing-pl
|
||||
x-litellm-policy-sources: high-risk-policy2=tag:health
|
||||
```
|
||||
|
||||
| Header | What it tells you |
|
||||
|--------|-------------------|
|
||||
| `x-litellm-applied-policies` | Which policies matched this request |
|
||||
| `x-litellm-applied-guardrails` | Which guardrails actually ran |
|
||||
| `x-litellm-policy-sources` | **Why** each policy matched — `tag:health` confirms it was the tag |
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
import Image from '@theme/IdealImage';
|
||||
|
||||
# Claude Code - Fixing Invalid Beta Header Errors
|
||||
# Claude Code - Managing Anthropic Beta Headers
|
||||
|
||||
When using Claude Code with LiteLLM and non-Anthropic providers (Bedrock, Azure AI, Vertex AI), you may encounter "invalid beta header" errors. This guide explains how to fix these errors locally or contribute a fix to LiteLLM.
|
||||
When using Claude Code with LiteLLM and non-Anthropic providers (Bedrock, Azure AI, Vertex AI), you need to ensure that only supported beta headers are sent to each provider. This guide explains how to add support for new beta headers or fix invalid beta header errors.
|
||||
|
||||
## What Are Beta Headers?
|
||||
|
||||
|
|
@ -12,7 +12,7 @@ Anthropic uses beta headers to enable experimental features in Claude. When you
|
|||
anthropic-beta: prompt-caching-scope-2026-01-05,advanced-tool-use-2025-11-20
|
||||
```
|
||||
|
||||
However, not all providers support all Anthropic beta features. When an unsupported beta header is sent to a provider, you'll see an error.
|
||||
However, not all providers support all Anthropic beta features. LiteLLM uses `anthropic_beta_headers_config.json` to manage which beta headers are supported by each provider.
|
||||
|
||||
## Common Error Message
|
||||
|
||||
|
|
@ -22,17 +22,22 @@ Error: The model returned the following errors: invalid beta flag
|
|||
|
||||
## How LiteLLM Handles Beta Headers
|
||||
|
||||
LiteLLM automatically filters out unsupported beta headers using a configuration file:
|
||||
LiteLLM uses a strict validation approach with a configuration file:
|
||||
|
||||
```
|
||||
litellm/litellm/anthropic_beta_headers_config.json
|
||||
```
|
||||
|
||||
This JSON file lists which beta headers are **unsupported** for each provider. Headers not in the unsupported list are passed through to the provider.
|
||||
This JSON file contains a **mapping** of beta headers for each provider:
|
||||
- **Keys**: Input beta header names (from Anthropic)
|
||||
- **Values**: Provider-specific header names (or `null` if unsupported)
|
||||
- **Validation**: Only headers present in the mapping with non-null values are forwarded
|
||||
|
||||
## Quick Fix: Update Config Locally
|
||||
This enforces stricter validation than just filtering unsupported headers - headers must be explicitly defined to be allowed.
|
||||
|
||||
If you encounter an invalid beta header error, you can fix it immediately by updating the config file locally.
|
||||
## Adding Support for a New Beta Header
|
||||
|
||||
When Anthropic releases a new beta feature, you need to add it to the configuration file for each provider.
|
||||
|
||||
### Step 1: Locate the Config File
|
||||
|
||||
|
|
@ -46,43 +51,47 @@ cd $(python -c "import litellm; import os; print(os.path.dirname(litellm.__file_
|
|||
# litellm/anthropic_beta_headers_config.json
|
||||
```
|
||||
|
||||
### Step 2: Add the Unsupported Header
|
||||
### Step 2: Add the New Beta Header
|
||||
|
||||
Open `anthropic_beta_headers_config.json` and add the problematic header to the appropriate provider's list:
|
||||
Open `anthropic_beta_headers_config.json` and add the new header to each provider's mapping:
|
||||
|
||||
```json title="anthropic_beta_headers_config.json"
|
||||
{
|
||||
"description": "Unsupported Anthropic beta headers for each provider. Headers listed here will be dropped. Headers not listed are passed through as-is.",
|
||||
"anthropic": [],
|
||||
"azure_ai": [],
|
||||
"bedrock_converse": [
|
||||
"prompt-caching-scope-2026-01-05",
|
||||
"bash_20250124",
|
||||
"bash_20241022",
|
||||
"text_editor_20250124",
|
||||
"text_editor_20241022",
|
||||
"compact-2026-01-12",
|
||||
"advanced-tool-use-2025-11-20",
|
||||
"web-fetch-2025-09-10",
|
||||
"code-execution-2025-08-25",
|
||||
"skills-2025-10-02",
|
||||
"files-api-2025-04-14"
|
||||
],
|
||||
"bedrock": [
|
||||
"advanced-tool-use-2025-11-20",
|
||||
"prompt-caching-scope-2026-01-05",
|
||||
"structured-outputs-2025-11-13",
|
||||
"web-fetch-2025-09-10",
|
||||
"code-execution-2025-08-25",
|
||||
"skills-2025-10-02",
|
||||
"files-api-2025-04-14"
|
||||
],
|
||||
"vertex_ai": [
|
||||
"prompt-caching-scope-2026-01-05"
|
||||
]
|
||||
"description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded.",
|
||||
"anthropic": {
|
||||
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
|
||||
"new-feature-2026-03-01": "new-feature-2026-03-01",
|
||||
...
|
||||
},
|
||||
"azure_ai": {
|
||||
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
|
||||
"new-feature-2026-03-01": "new-feature-2026-03-01",
|
||||
...
|
||||
},
|
||||
"bedrock_converse": {
|
||||
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
|
||||
"new-feature-2026-03-01": null,
|
||||
...
|
||||
},
|
||||
"bedrock": {
|
||||
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
|
||||
"new-feature-2026-03-01": null,
|
||||
...
|
||||
},
|
||||
"vertex_ai": {
|
||||
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
|
||||
"new-feature-2026-03-01": null,
|
||||
...
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Key Points:**
|
||||
- **Supported headers**: Set the value to the provider-specific header name (often the same as the key)
|
||||
- **Unsupported headers**: Set the value to `null`
|
||||
- **Header transformations**: Some providers use different header names (e.g., Bedrock maps `advanced-tool-use-2025-11-20` to `tool-search-tool-2025-10-19`)
|
||||
- **Alphabetical order**: Keep headers sorted alphabetically for maintainability
|
||||
|
||||
### Step 3: Restart Your Application
|
||||
|
||||
After updating the config file, restart your LiteLLM proxy or application:
|
||||
|
|
@ -97,9 +106,64 @@ litellm --config config.yaml
|
|||
|
||||
The updated configuration will be loaded automatically.
|
||||
|
||||
## Fixing Invalid Beta Header Errors
|
||||
|
||||
If you encounter an "invalid beta flag" error, it means a beta header is being sent that the provider doesn't support.
|
||||
|
||||
### Step 1: Identify the Problematic Header
|
||||
|
||||
Check your logs to see which header is causing the issue:
|
||||
|
||||
```bash
|
||||
Error: The model returned the following errors: invalid beta flag: new-feature-2026-03-01
|
||||
```
|
||||
|
||||
### Step 2: Update the Config
|
||||
|
||||
Set the header value to `null` for that provider:
|
||||
|
||||
```json title="anthropic_beta_headers_config.json"
|
||||
{
|
||||
"bedrock_converse": {
|
||||
"new-feature-2026-03-01": null
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Step 3: Restart and Test
|
||||
|
||||
Restart your application and verify the header is now filtered out.
|
||||
|
||||
## Contributing a Fix to LiteLLM
|
||||
|
||||
Help the community by contributing your fix! If your local changes work, please raise a PR with the addition of the header and we will merge it.
|
||||
Help the community by contributing your fix!
|
||||
|
||||
### What to Include in Your PR
|
||||
|
||||
1. **Update the config file**: Add the new beta header to `litellm/anthropic_beta_headers_config.json`
|
||||
2. **Test your changes**: Verify the header is correctly filtered/mapped for each provider
|
||||
3. **Documentation**: Include provider documentation links showing which headers are supported
|
||||
|
||||
### Example PR Description
|
||||
|
||||
```markdown
|
||||
## Add support for new-feature-2026-03-01 beta header
|
||||
|
||||
### Changes
|
||||
- Added `new-feature-2026-03-01` to anthropic_beta_headers_config.json
|
||||
- Set to `null` for bedrock_converse (unsupported)
|
||||
- Set to header name for anthropic, azure_ai (supported)
|
||||
|
||||
### Testing
|
||||
Tested with:
|
||||
- ✅ Anthropic: Header passed through correctly
|
||||
- ✅ Azure AI: Header passed through correctly
|
||||
- ✅ Bedrock Converse: Header filtered out (returns error without fix)
|
||||
|
||||
### References
|
||||
- Anthropic docs: [link]
|
||||
- AWS Bedrock docs: [link]
|
||||
```
|
||||
|
||||
|
||||
## How Beta Header Filtering Works
|
||||
|
|
@ -116,14 +180,51 @@ sequenceDiagram
|
|||
CC->>LP: Request with beta headers
|
||||
Note over CC,LP: anthropic-beta: header1,header2,header3
|
||||
|
||||
LP->>Config: Load unsupported headers for provider
|
||||
Config-->>LP: Returns unsupported list
|
||||
LP->>Config: Load header mapping for provider
|
||||
Config-->>LP: Returns mapping (header→value or null)
|
||||
|
||||
Note over LP: Filter headers:<br/>- Remove unsupported<br/>- Keep supported
|
||||
Note over LP: Validate & Transform:<br/>1. Check if header exists in mapping<br/>2. Filter out null values<br/>3. Map to provider-specific names
|
||||
|
||||
LP->>Provider: Request with filtered headers
|
||||
Note over LP,Provider: anthropic-beta: header2<br/>(header1, header3 removed)
|
||||
LP->>Provider: Request with filtered & mapped headers
|
||||
Note over LP,Provider: anthropic-beta: mapped-header2<br/>(header1, header3 filtered out)
|
||||
|
||||
Provider-->>LP: Success response
|
||||
LP-->>CC: Response
|
||||
```
|
||||
```
|
||||
|
||||
### Filtering Rules
|
||||
|
||||
1. **Header must exist in mapping**: Unknown headers are filtered out
|
||||
2. **Header must have non-null value**: Headers with `null` values are filtered out
|
||||
3. **Header transformation**: Headers are mapped to provider-specific names (e.g., `advanced-tool-use-2025-11-20` → `tool-search-tool-2025-10-19` for Bedrock)
|
||||
|
||||
### Example
|
||||
|
||||
Request with headers:
|
||||
```
|
||||
anthropic-beta: advanced-tool-use-2025-11-20,computer-use-2025-01-24,unknown-header
|
||||
```
|
||||
|
||||
For Bedrock Converse:
|
||||
- ✅ `computer-use-2025-01-24` → `computer-use-2025-01-24` (supported, passed through)
|
||||
- ❌ `advanced-tool-use-2025-11-20` → filtered out (null value in config)
|
||||
- ❌ `unknown-header` → filtered out (not in config)
|
||||
|
||||
Result sent to Bedrock:
|
||||
```
|
||||
anthropic-beta: computer-use-2025-01-24
|
||||
```
|
||||
|
||||
## Provider-Specific Notes
|
||||
|
||||
### Bedrock
|
||||
- Beta headers appear in both HTTP headers AND request body (`additionalModelRequestFields.anthropic_beta`)
|
||||
- Some headers are transformed (e.g., `advanced-tool-use` → `tool-search-tool`)
|
||||
|
||||
### Azure AI
|
||||
- Uses same header names as Anthropic
|
||||
- Some features not yet supported (check config for null values)
|
||||
|
||||
### Vertex AI
|
||||
- Some headers are transformed to match Vertex AI's implementation
|
||||
- Limited beta feature support compared to Anthropic
|
||||
BIN
docs/my-website/img/policy_team_attach.png
Normal file
BIN
docs/my-website/img/policy_team_attach.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 225 KiB |
BIN
docs/my-website/img/policy_test_matching.png
Normal file
BIN
docs/my-website/img/policy_test_matching.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 200 KiB |
|
|
@ -42,49 +42,62 @@ const sidebars = {
|
|||
label: "Guardrails",
|
||||
items: [
|
||||
"proxy/guardrails/quick_start",
|
||||
"proxy/guardrails/guardrail_policies",
|
||||
"proxy/guardrails/guardrail_load_balancing",
|
||||
"proxy/guardrails/test_playground",
|
||||
"proxy/guardrails/litellm_content_filter",
|
||||
{
|
||||
type: "category",
|
||||
"label": "Contributing to Guardrails",
|
||||
label: "Providers",
|
||||
items: [
|
||||
...[
|
||||
"proxy/guardrails/qualifire",
|
||||
"proxy/guardrails/aim_security",
|
||||
"proxy/guardrails/onyx_security",
|
||||
"proxy/guardrails/aporia_api",
|
||||
"proxy/guardrails/azure_content_guardrail",
|
||||
"proxy/guardrails/bedrock",
|
||||
"proxy/guardrails/enkryptai",
|
||||
"proxy/guardrails/ibm_guardrails",
|
||||
"proxy/guardrails/grayswan",
|
||||
"proxy/guardrails/hiddenlayer",
|
||||
"proxy/guardrails/lasso_security",
|
||||
"proxy/guardrails/guardrails_ai",
|
||||
"proxy/guardrails/lakera_ai",
|
||||
"proxy/guardrails/model_armor",
|
||||
"proxy/guardrails/noma_security",
|
||||
"proxy/guardrails/dynamoai",
|
||||
"proxy/guardrails/openai_moderation",
|
||||
"proxy/guardrails/pangea",
|
||||
"proxy/guardrails/pillar_security",
|
||||
"proxy/guardrails/pii_masking_v2",
|
||||
"proxy/guardrails/panw_prisma_airs",
|
||||
"proxy/guardrails/secret_detection",
|
||||
"proxy/guardrails/custom_guardrail",
|
||||
"proxy/guardrails/custom_code_guardrail",
|
||||
"proxy/guardrails/prompt_injection",
|
||||
"proxy/guardrails/tool_permission",
|
||||
"proxy/guardrails/zscaler_ai_guard",
|
||||
"proxy/guardrails/javelin"
|
||||
].sort(),
|
||||
],
|
||||
},
|
||||
{
|
||||
type: "category",
|
||||
label: "Contributing to Guardrails",
|
||||
items: [
|
||||
"adding_provider/generic_guardrail_api",
|
||||
"adding_provider/simple_guardrail_tutorial",
|
||||
"adding_provider/adding_guardrail_support",
|
||||
]
|
||||
},
|
||||
"proxy/guardrails/test_playground",
|
||||
"proxy/guardrails/litellm_content_filter",
|
||||
...[
|
||||
"proxy/guardrails/qualifire",
|
||||
"proxy/guardrails/aim_security",
|
||||
"proxy/guardrails/onyx_security",
|
||||
"proxy/guardrails/aporia_api",
|
||||
"proxy/guardrails/azure_content_guardrail",
|
||||
"proxy/guardrails/bedrock",
|
||||
"proxy/guardrails/enkryptai",
|
||||
"proxy/guardrails/ibm_guardrails",
|
||||
"proxy/guardrails/grayswan",
|
||||
"proxy/guardrails/hiddenlayer",
|
||||
"proxy/guardrails/lasso_security",
|
||||
"proxy/guardrails/guardrails_ai",
|
||||
"proxy/guardrails/lakera_ai",
|
||||
"proxy/guardrails/model_armor",
|
||||
"proxy/guardrails/noma_security",
|
||||
"proxy/guardrails/dynamoai",
|
||||
"proxy/guardrails/openai_moderation",
|
||||
"proxy/guardrails/pangea",
|
||||
"proxy/guardrails/pillar_security",
|
||||
"proxy/guardrails/pii_masking_v2",
|
||||
"proxy/guardrails/panw_prisma_airs",
|
||||
"proxy/guardrails/secret_detection",
|
||||
"proxy/guardrails/custom_guardrail",
|
||||
"proxy/guardrails/custom_code_guardrail",
|
||||
"proxy/guardrails/prompt_injection",
|
||||
"proxy/guardrails/tool_permission",
|
||||
"proxy/guardrails/zscaler_ai_guard",
|
||||
"proxy/guardrails/javelin"
|
||||
].sort(),
|
||||
],
|
||||
},
|
||||
{
|
||||
type: "category",
|
||||
label: "Policies",
|
||||
items: [
|
||||
"proxy/guardrails/guardrail_policies",
|
||||
"proxy/guardrails/policy_tags",
|
||||
],
|
||||
},
|
||||
{
|
||||
|
|
@ -396,6 +409,16 @@ const sidebars = {
|
|||
],
|
||||
},
|
||||
"proxy/caching",
|
||||
{
|
||||
type: "link",
|
||||
label: "Guardrails",
|
||||
href: "https://docs.litellm.ai/docs/proxy/guardrails/quick_start",
|
||||
},
|
||||
{
|
||||
type: "link",
|
||||
label: "Policies",
|
||||
href: "https://docs.litellm.ai/docs/proxy/guardrails/guardrail_policies",
|
||||
},
|
||||
{
|
||||
type: "category",
|
||||
label: "Create Custom Plugins",
|
||||
|
|
@ -1085,6 +1108,17 @@ const sidebars = {
|
|||
"troubleshoot/max_callbacks",
|
||||
],
|
||||
},
|
||||
{
|
||||
type: "category",
|
||||
label: "Blog",
|
||||
items: [
|
||||
{
|
||||
type: "link",
|
||||
label: "Incident: Broken Model Cost Map",
|
||||
href: "/blog/model-cost-map-incident",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -899,49 +899,49 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
batch_id=response.id, model_id=model_id
|
||||
)
|
||||
|
||||
if (
|
||||
response.output_file_id and model_id
|
||||
): # return a file id with the model_id and output_file_id
|
||||
original_output_file_id = response.output_file_id
|
||||
response.output_file_id = self.get_unified_output_file_id(
|
||||
output_file_id=response.output_file_id,
|
||||
model_id=model_id,
|
||||
model_name=model_name,
|
||||
)
|
||||
|
||||
# Fetch the actual file object for the output file
|
||||
file_object = None
|
||||
try:
|
||||
# Use litellm to retrieve the file object from the provider
|
||||
from litellm import afile_retrieve
|
||||
file_object = await afile_retrieve(
|
||||
custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai",
|
||||
file_id=original_output_file_id
|
||||
# Handle both output_file_id and error_file_id
|
||||
for file_attr in ["output_file_id", "error_file_id"]:
|
||||
file_id_value = getattr(response, file_attr, None)
|
||||
if file_id_value and model_id:
|
||||
original_file_id = file_id_value
|
||||
unified_file_id = self.get_unified_output_file_id(
|
||||
output_file_id=original_file_id,
|
||||
model_id=model_id,
|
||||
model_name=model_name,
|
||||
)
|
||||
verbose_logger.debug(
|
||||
f"Successfully retrieved file object for output_file_id={original_output_file_id}"
|
||||
setattr(response, file_attr, unified_file_id)
|
||||
|
||||
# Fetch the actual file object from the provider
|
||||
file_object = None
|
||||
try:
|
||||
# Use litellm to retrieve the file object from the provider
|
||||
from litellm import afile_retrieve
|
||||
file_object = await afile_retrieve(
|
||||
custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai",
|
||||
file_id=original_file_id
|
||||
)
|
||||
verbose_logger.debug(
|
||||
f"Successfully retrieved file object for {file_attr}={original_file_id}"
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.warning(
|
||||
f"Failed to retrieve file object for {file_attr}={original_file_id}: {str(e)}. Storing with None and will fetch on-demand."
|
||||
)
|
||||
|
||||
await self.store_unified_file_id(
|
||||
file_id=unified_file_id,
|
||||
file_object=file_object,
|
||||
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
model_mappings={model_id: original_file_id},
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.warning(
|
||||
f"Failed to retrieve file object for output_file_id={original_output_file_id}: {str(e)}. Storing with None and will fetch on-demand."
|
||||
)
|
||||
|
||||
await self.store_unified_file_id(
|
||||
file_id=response.output_file_id,
|
||||
file_object=file_object,
|
||||
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
model_mappings={model_id: original_output_file_id},
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
asyncio.create_task(
|
||||
self.store_unified_object_id(
|
||||
unified_object_id=response.id,
|
||||
file_object=response,
|
||||
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
model_object_id=original_response_id,
|
||||
file_purpose="batch",
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
await self.store_unified_object_id(
|
||||
unified_object_id=response.id,
|
||||
file_object=response,
|
||||
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
model_object_id=original_response_id,
|
||||
file_purpose="batch",
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
elif isinstance(response, LiteLLMFineTuningJob):
|
||||
## Check if unified_file_id is in the response
|
||||
|
|
@ -958,15 +958,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
response.id = self.get_unified_generic_response_id(
|
||||
model_id=model_id, generic_response_id=response.id
|
||||
)
|
||||
asyncio.create_task(
|
||||
self.store_unified_object_id(
|
||||
unified_object_id=response.id,
|
||||
file_object=response,
|
||||
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
model_object_id=original_response_id,
|
||||
file_purpose="fine-tune",
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
await self.store_unified_object_id(
|
||||
unified_object_id=response.id,
|
||||
file_object=response,
|
||||
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
model_object_id=original_response_id,
|
||||
file_purpose="fine-tune",
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
elif isinstance(response, AsyncCursorPage):
|
||||
"""
|
||||
|
|
|
|||
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.34-py3-none-any.whl
vendored
Normal file
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.34-py3-none-any.whl
vendored
Normal file
Binary file not shown.
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.34.tar.gz
vendored
Normal file
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.34.tar.gz
vendored
Normal file
Binary file not shown.
|
|
@ -914,6 +914,7 @@ model LiteLLM_PolicyAttachmentTable {
|
|||
teams String[] @default([]) // Team aliases or patterns
|
||||
keys String[] @default([]) // Key aliases or patterns
|
||||
models String[] @default([]) // Model names or patterns
|
||||
tags String[] @default([]) // Tag patterns (e.g., ["healthcare", "prod-*"])
|
||||
created_at DateTime @default(now())
|
||||
created_by String?
|
||||
updated_at DateTime @default(now()) @updatedAt
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.33"
|
||||
version = "0.4.34"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
authors = ["BerriAI"]
|
||||
readme = "README.md"
|
||||
|
|
@ -22,7 +22,7 @@ requires = ["poetry-core"]
|
|||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.33"
|
||||
version = "0.4.34"
|
||||
version_files = [
|
||||
"pyproject.toml:version",
|
||||
"../requirements.txt:litellm-proxy-extras==",
|
||||
|
|
|
|||
|
|
@ -1155,6 +1155,7 @@ from .exceptions import (
|
|||
BadRequestError,
|
||||
ImageFetchError,
|
||||
NotFoundError,
|
||||
PermissionDeniedError,
|
||||
RateLimitError,
|
||||
ServiceUnavailableError,
|
||||
BadGatewayError,
|
||||
|
|
@ -1393,6 +1394,7 @@ if TYPE_CHECKING:
|
|||
from .llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig
|
||||
from .llms.volcengine.responses.transformation import VolcEngineResponsesAPIConfig as VolcEngineResponsesAPIConfig
|
||||
from .llms.manus.responses.transformation import ManusResponsesAPIConfig as ManusResponsesAPIConfig
|
||||
from .llms.perplexity.responses.transformation import PerplexityResponsesConfig as PerplexityResponsesConfig
|
||||
from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig
|
||||
from .llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config
|
||||
from .llms.anthropic.skills.transformation import AnthropicSkillsConfig as AnthropicSkillsConfig
|
||||
|
|
|
|||
|
|
@ -226,6 +226,7 @@ LLM_CONFIG_NAMES = (
|
|||
"XAIResponsesAPIConfig",
|
||||
"LiteLLMProxyResponsesAPIConfig",
|
||||
"VolcEngineResponsesAPIConfig",
|
||||
"PerplexityResponsesConfig",
|
||||
"GoogleAIStudioInteractionsConfig",
|
||||
"OpenAIOSeriesConfig",
|
||||
"AnthropicSkillsConfig",
|
||||
|
|
@ -274,6 +275,7 @@ LLM_CONFIG_NAMES = (
|
|||
"LmStudioEmbeddingConfig",
|
||||
"NscaleConfig",
|
||||
"PerplexityChatConfig",
|
||||
"PerplexityResponsesConfig",
|
||||
"AzureOpenAIO1Config",
|
||||
"IBMWatsonXAIConfig",
|
||||
"IBMWatsonXChatConfig",
|
||||
|
|
@ -901,6 +903,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
|
|||
".llms.manus.responses.transformation",
|
||||
"ManusResponsesAPIConfig",
|
||||
),
|
||||
"PerplexityResponsesConfig": (
|
||||
".llms.perplexity.responses.transformation",
|
||||
"PerplexityResponsesConfig",
|
||||
),
|
||||
"GoogleAIStudioInteractionsConfig": (
|
||||
".llms.gemini.interactions.transformation",
|
||||
"GoogleAIStudioInteractionsConfig",
|
||||
|
|
|
|||
|
|
@ -1,9 +1,13 @@
|
|||
import json
|
||||
import ast
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from logging import Formatter
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
|
||||
set_verbose = False
|
||||
|
||||
|
|
@ -19,6 +23,67 @@ handler = logging.StreamHandler()
|
|||
handler.setLevel(numeric_level)
|
||||
|
||||
|
||||
def _try_parse_json_message(message: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Try to parse a log message as JSON. Returns parsed dict if valid, else None.
|
||||
Handles messages that are entirely valid JSON (e.g. json.dumps output).
|
||||
Uses shared safe_json_loads for consistent error handling.
|
||||
"""
|
||||
if not message or not isinstance(message, str):
|
||||
return None
|
||||
msg_stripped = message.strip()
|
||||
if not (msg_stripped.startswith("{") or msg_stripped.startswith("[")):
|
||||
return None
|
||||
parsed = safe_json_loads(message, default=None)
|
||||
if parsed is None or not isinstance(parsed, dict):
|
||||
return None
|
||||
return parsed
|
||||
|
||||
|
||||
def _try_parse_embedded_python_dict(message: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Try to find and parse a Python dict repr (e.g. str(d) or repr(d)) embedded in
|
||||
the message. Handles patterns like:
|
||||
"get_available_deployment for model: X, Selected deployment: {'model_name': '...', ...} for model: X"
|
||||
Uses ast.literal_eval for safe parsing. Returns the parsed dict or None.
|
||||
"""
|
||||
if not message or not isinstance(message, str) or "{" not in message:
|
||||
return None
|
||||
i = 0
|
||||
while i < len(message):
|
||||
start = message.find("{", i)
|
||||
if start == -1:
|
||||
break
|
||||
depth = 0
|
||||
for j in range(start, len(message)):
|
||||
c = message[j]
|
||||
if c == "{":
|
||||
depth += 1
|
||||
elif c == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
substr = message[start : j + 1]
|
||||
try:
|
||||
result = ast.literal_eval(substr)
|
||||
if isinstance(result, dict) and len(result) > 0:
|
||||
return result
|
||||
except (ValueError, SyntaxError, TypeError):
|
||||
pass
|
||||
break
|
||||
i = start + 1
|
||||
return None
|
||||
|
||||
|
||||
# Standard LogRecord attribute names - used to identify 'extra' fields.
|
||||
# Derived at runtime so we automatically include version-specific attrs (e.g. taskName).
|
||||
def _get_standard_record_attrs() -> frozenset:
|
||||
"""Standard LogRecord attribute names - excludes extra keys from logger.debug(..., extra={...})."""
|
||||
return frozenset(logging.LogRecord("", 0, "", 0, "", (), None).__dict__.keys())
|
||||
|
||||
|
||||
_STANDARD_RECORD_ATTRS = _get_standard_record_attrs()
|
||||
|
||||
|
||||
class JsonFormatter(Formatter):
|
||||
def __init__(self):
|
||||
super(JsonFormatter, self).__init__()
|
||||
|
|
@ -29,16 +94,31 @@ class JsonFormatter(Formatter):
|
|||
return dt.isoformat()
|
||||
|
||||
def format(self, record):
|
||||
json_record = {
|
||||
"message": record.getMessage(),
|
||||
message_str = record.getMessage()
|
||||
json_record: Dict[str, Any] = {
|
||||
"message": message_str,
|
||||
"level": record.levelname,
|
||||
"timestamp": self.formatTime(record),
|
||||
}
|
||||
|
||||
# Parse embedded JSON or Python dict repr in message so sub-fields become first-class properties
|
||||
parsed = _try_parse_json_message(message_str)
|
||||
if parsed is None:
|
||||
parsed = _try_parse_embedded_python_dict(message_str)
|
||||
if parsed is not None:
|
||||
for key, value in parsed.items():
|
||||
if key not in json_record:
|
||||
json_record[key] = value
|
||||
|
||||
# Include extra attributes passed via logger.debug("msg", extra={...})
|
||||
for key, value in record.__dict__.items():
|
||||
if key not in _STANDARD_RECORD_ATTRS and key not in json_record:
|
||||
json_record[key] = value
|
||||
|
||||
if record.exc_info:
|
||||
json_record["stacktrace"] = self.formatException(record.exc_info)
|
||||
|
||||
return json.dumps(json_record)
|
||||
return safe_dumps(json_record)
|
||||
|
||||
|
||||
# Function to set up exception handlers for JSON logging
|
||||
|
|
@ -169,15 +249,15 @@ def _initialize_loggers_with_handler(handler: logging.Handler):
|
|||
def _get_uvicorn_json_log_config():
|
||||
"""
|
||||
Generate a uvicorn log_config dictionary that applies JSON formatting to all loggers.
|
||||
|
||||
|
||||
This ensures that uvicorn's access logs, error logs, and all application logs
|
||||
are formatted as JSON when json_logs is enabled.
|
||||
"""
|
||||
json_formatter_class = "litellm._logging.JsonFormatter"
|
||||
|
||||
|
||||
# Use the module-level log_level variable for consistency
|
||||
uvicorn_log_level = log_level.upper()
|
||||
|
||||
|
||||
log_config = {
|
||||
"version": 1,
|
||||
"disable_existing_loggers": False,
|
||||
|
|
@ -222,7 +302,7 @@ def _get_uvicorn_json_log_config():
|
|||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
return log_config
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,33 +1,151 @@
|
|||
{
|
||||
"description": "Unsupported Anthropic beta headers for each provider. Headers listed here will be dropped. Headers not listed are passed through as-is.",
|
||||
"anthropic": [],
|
||||
"azure_ai": [],
|
||||
"bedrock_converse": [
|
||||
"prompt-caching-scope-2026-01-05",
|
||||
"bash_20250124",
|
||||
"bash_20241022",
|
||||
"text_editor_20250124",
|
||||
"text_editor_20241022",
|
||||
"compact-2026-01-12",
|
||||
"advanced-tool-use-2025-11-20",
|
||||
"web-fetch-2025-09-10",
|
||||
"code-execution-2025-08-25",
|
||||
"skills-2025-10-02",
|
||||
"files-api-2025-04-14",
|
||||
"fast-mode-2026-02-01"
|
||||
],
|
||||
"bedrock": [
|
||||
"advanced-tool-use-2025-11-20",
|
||||
"prompt-caching-scope-2026-01-05",
|
||||
"structured-outputs-2025-11-13",
|
||||
"web-fetch-2025-09-10",
|
||||
"code-execution-2025-08-25",
|
||||
"skills-2025-10-02",
|
||||
"files-api-2025-04-14",
|
||||
"fast-mode-2026-02-01",
|
||||
"mcp-servers-2025-12-04"
|
||||
],
|
||||
"vertex_ai": [
|
||||
"prompt-caching-scope-2026-01-05"
|
||||
]
|
||||
}
|
||||
"description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded.",
|
||||
"anthropic": {
|
||||
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
|
||||
"bash_20241022": "bash_20241022",
|
||||
"bash_20250124": "bash_20250124",
|
||||
"code-execution-2025-08-25": "code-execution-2025-08-25",
|
||||
"compact-2026-01-12": "compact-2026-01-12",
|
||||
"computer-use-2025-01-24": "computer-use-2025-01-24",
|
||||
"computer-use-2025-11-24": "computer-use-2025-11-24",
|
||||
"context-1m-2025-08-07": "context-1m-2025-08-07",
|
||||
"context-management-2025-06-27": "context-management-2025-06-27",
|
||||
"effort-2025-11-24": "effort-2025-11-24",
|
||||
"fast-mode-2026-02-01": "fast-mode-2026-02-01",
|
||||
"files-api-2025-04-14": "files-api-2025-04-14",
|
||||
"structured-output-2024-03-01": "structured-output-2024-03-01",
|
||||
"fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
|
||||
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
|
||||
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
|
||||
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
|
||||
"mcp-servers-2025-12-04": "mcp-servers-2025-12-04",
|
||||
"output-128k-2025-02-19": "output-128k-2025-02-19",
|
||||
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
|
||||
"skills-2025-10-02": "skills-2025-10-02",
|
||||
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
|
||||
"text_editor_20241022": "text_editor_20241022",
|
||||
"text_editor_20250124": "text_editor_20250124",
|
||||
"token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
|
||||
"web-fetch-2025-09-10": "web-fetch-2025-09-10",
|
||||
"web-search-2025-03-05": "web-search-2025-03-05"
|
||||
},
|
||||
"azure_ai": {
|
||||
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
|
||||
"bash_20241022": "bash_20241022",
|
||||
"bash_20250124": "bash_20250124",
|
||||
"code-execution-2025-08-25": "code-execution-2025-08-25",
|
||||
"compact-2026-01-12": null,
|
||||
"computer-use-2025-01-24": "computer-use-2025-01-24",
|
||||
"computer-use-2025-11-24": "computer-use-2025-11-24",
|
||||
"context-1m-2025-08-07": "context-1m-2025-08-07",
|
||||
"context-management-2025-06-27": "context-management-2025-06-27",
|
||||
"effort-2025-11-24": "effort-2025-11-24",
|
||||
"fast-mode-2026-02-01": null,
|
||||
"files-api-2025-04-14": "files-api-2025-04-14",
|
||||
"fine-grained-tool-streaming-2025-05-14": null,
|
||||
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
|
||||
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
|
||||
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
|
||||
"mcp-servers-2025-12-04": "mcp-servers-2025-12-04",
|
||||
"output-128k-2025-02-19": null,
|
||||
"structured-output-2024-03-01": null,
|
||||
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
|
||||
"skills-2025-10-02": "skills-2025-10-02",
|
||||
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
|
||||
"text_editor_20241022": null,
|
||||
"text_editor_20250124": null,
|
||||
"token-efficient-tools-2025-02-19": null,
|
||||
"web-fetch-2025-09-10": "web-fetch-2025-09-10",
|
||||
"web-search-2025-03-05": "web-search-2025-03-05"
|
||||
},
|
||||
"bedrock_converse": {
|
||||
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
|
||||
"bash_20241022": null,
|
||||
"bash_20250124": null,
|
||||
"code-execution-2025-08-25": null,
|
||||
"compact-2026-01-12": null,
|
||||
"computer-use-2025-01-24": "computer-use-2025-01-24",
|
||||
"computer-use-2025-11-24": "computer-use-2025-11-24",
|
||||
"context-1m-2025-08-07": null,
|
||||
"context-management-2025-06-27": "context-management-2025-06-27",
|
||||
"effort-2025-11-24": null,
|
||||
"fast-mode-2026-02-01": null,
|
||||
"files-api-2025-04-14": null,
|
||||
"fine-grained-tool-streaming-2025-05-14": null,
|
||||
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
|
||||
"mcp-client-2025-11-20": null,
|
||||
"mcp-client-2025-04-04": null,
|
||||
"mcp-servers-2025-12-04": null,
|
||||
"output-128k-2025-02-19": null,
|
||||
"structured-output-2024-03-01": null,
|
||||
"prompt-caching-scope-2026-01-05": null,
|
||||
"skills-2025-10-02": null,
|
||||
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
|
||||
"text_editor_20241022": null,
|
||||
"text_editor_20250124": null,
|
||||
"token-efficient-tools-2025-02-19": null,
|
||||
"tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
|
||||
"web-fetch-2025-09-10": null,
|
||||
"web-search-2025-03-05": null
|
||||
},
|
||||
"bedrock": {
|
||||
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
|
||||
"bash_20241022": null,
|
||||
"bash_20250124": null,
|
||||
"code-execution-2025-08-25": null,
|
||||
"compact-2026-01-12": "compact-2026-01-12",
|
||||
"computer-use-2025-01-24": "computer-use-2025-01-24",
|
||||
"computer-use-2025-11-24": "computer-use-2025-11-24",
|
||||
"context-1m-2025-08-07": "context-1m-2025-08-07",
|
||||
"context-management-2025-06-27": "context-management-2025-06-27",
|
||||
"effort-2025-11-24": null,
|
||||
"fast-mode-2026-02-01": null,
|
||||
"files-api-2025-04-14": null,
|
||||
"fine-grained-tool-streaming-2025-05-14": null,
|
||||
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
|
||||
"mcp-client-2025-11-20": null,
|
||||
"mcp-client-2025-04-04": null,
|
||||
"mcp-servers-2025-12-04": null,
|
||||
"output-128k-2025-02-19": null,
|
||||
"structured-output-2024-03-01": null,
|
||||
"prompt-caching-scope-2026-01-05": null,
|
||||
"skills-2025-10-02": null,
|
||||
"structured-outputs-2025-11-13": null,
|
||||
"text_editor_20241022": null,
|
||||
"text_editor_20250124": null,
|
||||
"token-efficient-tools-2025-02-19": null,
|
||||
"tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
|
||||
"web-fetch-2025-09-10": null,
|
||||
"web-search-2025-03-05": null
|
||||
},
|
||||
"vertex_ai": {
|
||||
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
|
||||
"bash_20241022": null,
|
||||
"bash_20250124": null,
|
||||
"code-execution-2025-08-25": null,
|
||||
"compact-2026-01-12": null,
|
||||
"computer-use-2025-01-24": "computer-use-2025-01-24",
|
||||
"computer-use-2025-11-24": "computer-use-2025-11-24",
|
||||
"context-1m-2025-08-07": null,
|
||||
"context-management-2025-06-27": "context-management-2025-06-27",
|
||||
"effort-2025-11-24": null,
|
||||
"fast-mode-2026-02-01": null,
|
||||
"files-api-2025-04-14": null,
|
||||
"fine-grained-tool-streaming-2025-05-14": null,
|
||||
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
|
||||
"mcp-client-2025-11-20": null,
|
||||
"mcp-client-2025-04-04": null,
|
||||
"mcp-servers-2025-12-04": null,
|
||||
"output-128k-2025-02-19": null,
|
||||
"structured-output-2024-03-01": null,
|
||||
"prompt-caching-scope-2026-01-05": null,
|
||||
"skills-2025-10-02": null,
|
||||
"structured-outputs-2025-11-13": null,
|
||||
"text_editor_20241022": null,
|
||||
"text_editor_20250124": null,
|
||||
"token-efficient-tools-2025-02-19": null,
|
||||
"tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
|
||||
"web-fetch-2025-09-10": null,
|
||||
"web-search-2025-03-05": "web-search-2025-03-05"
|
||||
}
|
||||
}
|
||||
|
|
@ -2,14 +2,15 @@
|
|||
Centralized manager for Anthropic beta headers across different providers.
|
||||
|
||||
This module provides utilities to:
|
||||
1. Load beta header configuration from JSON (lists unsupported headers per provider)
|
||||
2. Filter out unsupported beta headers
|
||||
1. Load beta header configuration from JSON (mapping of supported headers per provider)
|
||||
2. Filter and map beta headers based on provider support
|
||||
3. Handle provider-specific header name mappings (e.g., advanced-tool-use -> tool-search-tool)
|
||||
|
||||
Design:
|
||||
- JSON config lists UNSUPPORTED headers for each provider
|
||||
- Headers not in the unsupported list are passed through
|
||||
- Header mappings allow renaming headers for specific providers
|
||||
- JSON config contains mapping of beta headers for each provider
|
||||
- Keys are input header names, values are provider-specific header names (or null if unsupported)
|
||||
- Only headers present in mapping keys with non-null values can be forwarded
|
||||
- This enforces stricter validation than the previous unsupported list approach
|
||||
"""
|
||||
|
||||
import json
|
||||
|
|
@ -47,13 +48,13 @@ def _load_beta_headers_config() -> Dict:
|
|||
return _BETA_HEADERS_CONFIG
|
||||
except Exception as e:
|
||||
verbose_logger.error(f"Failed to load beta headers config: {e}")
|
||||
# Return empty config as fallback
|
||||
# Return empty config as fallback (empty mappings)
|
||||
return {
|
||||
"anthropic": [],
|
||||
"azure_ai": [],
|
||||
"bedrock": [],
|
||||
"bedrock_converse": [],
|
||||
"vertex_ai": []
|
||||
"anthropic": {},
|
||||
"azure_ai": {},
|
||||
"bedrock": {},
|
||||
"bedrock_converse": {},
|
||||
"vertex_ai": {}
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -77,21 +78,19 @@ def filter_and_transform_beta_headers(
|
|||
provider: str,
|
||||
) -> List[str]:
|
||||
"""
|
||||
Filter beta headers based on provider's unsupported list.
|
||||
Filter and transform beta headers based on provider's mapping configuration.
|
||||
|
||||
This function:
|
||||
1. Removes headers that are in the provider's unsupported list
|
||||
2. Passes through all other headers as-is
|
||||
|
||||
Note: Header transformations/mappings (e.g., advanced-tool-use -> tool-search-tool)
|
||||
are handled in each provider's transformation code, not here.
|
||||
1. Only allows headers that are present in the provider's mapping keys
|
||||
2. Filters out headers with null values (unsupported)
|
||||
3. Maps headers to provider-specific names (e.g., advanced-tool-use -> tool-search-tool)
|
||||
|
||||
Args:
|
||||
beta_headers: List of Anthropic beta header values
|
||||
provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai")
|
||||
|
||||
Returns:
|
||||
List of filtered beta headers for the provider
|
||||
List of filtered and transformed beta headers for the provider
|
||||
"""
|
||||
if not beta_headers:
|
||||
return []
|
||||
|
|
@ -99,23 +98,33 @@ def filter_and_transform_beta_headers(
|
|||
config = _load_beta_headers_config()
|
||||
provider = get_provider_name(provider)
|
||||
|
||||
# Get unsupported headers for this provider
|
||||
unsupported_headers = set(config.get(provider, []))
|
||||
# Get the header mapping for this provider
|
||||
provider_mapping = config.get(provider, {})
|
||||
|
||||
filtered_headers: Set[str] = set()
|
||||
|
||||
for header in beta_headers:
|
||||
header = header.strip()
|
||||
|
||||
# Skip if header is unsupported
|
||||
if header in unsupported_headers:
|
||||
# Check if header is in the mapping
|
||||
if header not in provider_mapping:
|
||||
verbose_logger.debug(
|
||||
f"Dropping unknown beta header '{header}' for provider '{provider}' (not in mapping)"
|
||||
)
|
||||
continue
|
||||
|
||||
# Get the mapped header value
|
||||
mapped_header = provider_mapping[header]
|
||||
|
||||
# Skip if header is unsupported (null value)
|
||||
if mapped_header is None:
|
||||
verbose_logger.debug(
|
||||
f"Dropping unsupported beta header '{header}' for provider '{provider}'"
|
||||
)
|
||||
continue
|
||||
|
||||
# Pass through as-is
|
||||
filtered_headers.add(header)
|
||||
# Add the mapped header
|
||||
filtered_headers.add(mapped_header)
|
||||
|
||||
return sorted(list(filtered_headers))
|
||||
|
||||
|
|
@ -132,12 +141,14 @@ def is_beta_header_supported(
|
|||
provider: Provider name
|
||||
|
||||
Returns:
|
||||
True if the header is supported (not in unsupported list), False otherwise
|
||||
True if the header is in the mapping with a non-null value, False otherwise
|
||||
"""
|
||||
config = _load_beta_headers_config()
|
||||
provider = get_provider_name(provider)
|
||||
unsupported_headers = set(config.get(provider, []))
|
||||
return beta_header not in unsupported_headers
|
||||
provider_mapping = config.get(provider, {})
|
||||
|
||||
# Header is supported if it's in the mapping and has a non-null value
|
||||
return beta_header in provider_mapping and provider_mapping[beta_header] is not None
|
||||
|
||||
|
||||
def get_provider_beta_header(
|
||||
|
|
@ -145,27 +156,29 @@ def get_provider_beta_header(
|
|||
provider: str,
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
Check if a beta header is supported by a provider.
|
||||
Get the provider-specific beta header name for a given Anthropic beta header.
|
||||
|
||||
Note: This does NOT handle header transformations/mappings.
|
||||
Those are handled in each provider's transformation code.
|
||||
This function handles header transformations/mappings (e.g., advanced-tool-use -> tool-search-tool).
|
||||
|
||||
Args:
|
||||
anthropic_beta_header: The Anthropic beta header value
|
||||
provider: Provider name
|
||||
|
||||
Returns:
|
||||
The original header if supported, or None if unsupported
|
||||
The provider-specific header name if supported, or None if unsupported/unknown
|
||||
"""
|
||||
config = _load_beta_headers_config()
|
||||
provider = get_provider_name(provider)
|
||||
|
||||
# Check if unsupported
|
||||
unsupported_headers = set(config.get(provider, []))
|
||||
if anthropic_beta_header in unsupported_headers:
|
||||
# Get the header mapping for this provider
|
||||
provider_mapping = config.get(provider, {})
|
||||
|
||||
# Check if header is in the mapping
|
||||
if anthropic_beta_header not in provider_mapping:
|
||||
return None
|
||||
|
||||
return anthropic_beta_header
|
||||
# Return the mapped value (could be None if unsupported)
|
||||
return provider_mapping[anthropic_beta_header]
|
||||
|
||||
|
||||
def update_headers_with_filtered_beta(
|
||||
|
|
@ -208,7 +221,7 @@ def update_headers_with_filtered_beta(
|
|||
|
||||
def get_unsupported_headers(provider: str) -> List[str]:
|
||||
"""
|
||||
Get all beta headers that are unsupported by a provider.
|
||||
Get all beta headers that are unsupported by a provider (have null values in mapping).
|
||||
|
||||
Args:
|
||||
provider: Provider name
|
||||
|
|
@ -218,4 +231,7 @@ def get_unsupported_headers(provider: str) -> List[str]:
|
|||
"""
|
||||
config = _load_beta_headers_config()
|
||||
provider = get_provider_name(provider)
|
||||
return config.get(provider, [])
|
||||
provider_mapping = config.get(provider, {})
|
||||
|
||||
# Return headers with null values
|
||||
return [header for header, value in provider_mapping.items() if value is None]
|
||||
|
|
|
|||
|
|
@ -237,17 +237,37 @@ def batch_completion_models_all_responses(*args, **kwargs):
|
|||
if "model" in kwargs:
|
||||
kwargs.pop("model")
|
||||
if "models" in kwargs:
|
||||
models = kwargs["models"]
|
||||
kwargs.pop("models")
|
||||
models = kwargs.pop("models")
|
||||
else:
|
||||
raise Exception("'models' param not in kwargs")
|
||||
|
||||
if isinstance(models, str):
|
||||
models = [models]
|
||||
elif isinstance(models, (list, tuple)):
|
||||
models = list(models)
|
||||
else:
|
||||
raise TypeError("'models' must be a string or list of strings")
|
||||
|
||||
if len(models) == 0:
|
||||
return []
|
||||
|
||||
responses = []
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=len(models)) as executor:
|
||||
for idx, model in enumerate(models):
|
||||
future = executor.submit(litellm.completion, *args, model=model, **kwargs)
|
||||
if future.result() is not None:
|
||||
responses.append(future.result())
|
||||
futures = [
|
||||
executor.submit(litellm.completion, *args, model=model, **kwargs)
|
||||
for model in models
|
||||
]
|
||||
|
||||
for future in futures:
|
||||
try:
|
||||
result = future.result()
|
||||
if result is not None:
|
||||
responses.append(result)
|
||||
except Exception as e:
|
||||
print_verbose(
|
||||
f"batch_completion_models_all_responses: model request failed: {str(e)}"
|
||||
)
|
||||
continue
|
||||
|
||||
return responses
|
||||
|
|
|
|||
|
|
@ -48,6 +48,14 @@ DEFAULT_REPLICATE_POLLING_DELAY_SECONDS = int(
|
|||
os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)
|
||||
)
|
||||
DEFAULT_IMAGE_TOKEN_COUNT = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250))
|
||||
|
||||
# Model cost map validation constants
|
||||
MODEL_COST_MAP_MIN_MODEL_COUNT = int(
|
||||
os.getenv("MODEL_COST_MAP_MIN_MODEL_COUNT", 50)
|
||||
) # Minimum number of models a fetched cost map must contain to be considered valid
|
||||
MODEL_COST_MAP_MAX_SHRINK_RATIO = float(
|
||||
os.getenv("MODEL_COST_MAP_MAX_SHRINK_RATIO", 0.5)
|
||||
) # Maximum allowed shrinkage ratio vs local backup (0.5 = reject if fetched map is <50% of backup)
|
||||
DEFAULT_IMAGE_WIDTH = int(os.getenv("DEFAULT_IMAGE_WIDTH", 300))
|
||||
DEFAULT_IMAGE_HEIGHT = int(os.getenv("DEFAULT_IMAGE_HEIGHT", 300))
|
||||
# Maximum size for image URL downloads in MB (default 50MB, set to 0 to disable limit)
|
||||
|
|
@ -205,6 +213,10 @@ REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_agent_spend_update_bu
|
|||
REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_tag_spend_update_buffer"
|
||||
MAX_REDIS_BUFFER_DEQUEUE_COUNT = int(os.getenv("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100))
|
||||
MAX_SIZE_IN_MEMORY_QUEUE = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", 2000))
|
||||
# Bounds asyncio.Queue() instances (log queues, spend update queues, etc.) to prevent unbounded memory growth
|
||||
LITELLM_ASYNCIO_QUEUE_MAXSIZE = int(
|
||||
os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 1000)
|
||||
)
|
||||
MAX_IN_MEMORY_QUEUE_FLUSH_COUNT = int(
|
||||
os.getenv("MAX_IN_MEMORY_QUEUE_FLUSH_COUNT", 1000)
|
||||
)
|
||||
|
|
@ -1298,6 +1310,9 @@ DEFAULT_SLACK_ALERTING_THRESHOLD = int(
|
|||
os.getenv("DEFAULT_SLACK_ALERTING_THRESHOLD", 300)
|
||||
)
|
||||
MAX_TEAM_LIST_LIMIT = int(os.getenv("MAX_TEAM_LIST_LIMIT", 20))
|
||||
MAX_POLICY_ESTIMATE_IMPACT_ROWS = int(
|
||||
os.getenv("MAX_POLICY_ESTIMATE_IMPACT_ROWS", 1000)
|
||||
)
|
||||
DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = float(
|
||||
os.getenv("DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD", 0.7)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -28,6 +28,41 @@ else:
|
|||
|
||||
|
||||
class ArizeLogger(OpenTelemetry):
|
||||
"""
|
||||
Arize logger that sends traces to an Arize endpoint.
|
||||
|
||||
Creates its own dedicated TracerProvider so it can coexist with the
|
||||
generic ``otel`` callback (or any other OTEL-based integration) without
|
||||
fighting over the global ``opentelemetry.trace`` TracerProvider singleton.
|
||||
"""
|
||||
|
||||
def _init_tracing(self, tracer_provider):
|
||||
"""
|
||||
Override to always create a *private* TracerProvider for Arize.
|
||||
|
||||
See ArizePhoenixLogger._init_tracing for full rationale.
|
||||
"""
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
if tracer_provider is not None:
|
||||
self.tracer = tracer_provider.get_tracer("litellm")
|
||||
self.span_kind = SpanKind
|
||||
return
|
||||
|
||||
provider = TracerProvider(resource=self._get_litellm_resource(self.config))
|
||||
provider.add_span_processor(self._get_span_processor())
|
||||
self.tracer = provider.get_tracer("litellm")
|
||||
self.span_kind = SpanKind
|
||||
|
||||
def _init_otel_logger_on_litellm_proxy(self):
|
||||
"""
|
||||
Override: Arize should NOT overwrite the proxy's
|
||||
``open_telemetry_logger``. That attribute is reserved for the
|
||||
primary ``otel`` callback which handles proxy-level parent spans.
|
||||
"""
|
||||
pass
|
||||
|
||||
def set_attributes(self, span: Span, kwargs, response_obj: Optional[Any]):
|
||||
ArizeLogger.set_arize_attributes(span, kwargs, response_obj)
|
||||
return
|
||||
|
|
|
|||
|
|
@ -5,43 +5,211 @@ from litellm._logging import verbose_logger
|
|||
from litellm.integrations.arize import _utils
|
||||
from litellm.integrations.arize._utils import ArizeOTELAttributes
|
||||
from litellm.types.integrations.arize_phoenix import ArizePhoenixConfig
|
||||
from litellm.integrations.opentelemetry import OpenTelemetry
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.trace import Span as _Span
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from litellm.integrations.opentelemetry import OpenTelemetry as _OpenTelemetry
|
||||
from litellm.integrations.opentelemetry import OpenTelemetryConfig as _OpenTelemetryConfig
|
||||
from litellm.types.integrations.arize import Protocol as _Protocol
|
||||
|
||||
Protocol = _Protocol
|
||||
OpenTelemetryConfig = _OpenTelemetryConfig
|
||||
Span = Union[_Span, Any]
|
||||
OpenTelemetry = _OpenTelemetry
|
||||
else:
|
||||
Protocol = Any
|
||||
OpenTelemetryConfig = Any
|
||||
Span = Any
|
||||
TracerProvider = Any
|
||||
SpanKind = Any
|
||||
# Import OpenTelemetry at runtime
|
||||
try:
|
||||
from litellm.integrations.opentelemetry import OpenTelemetry
|
||||
except ImportError:
|
||||
OpenTelemetry = None # type: ignore
|
||||
|
||||
|
||||
ARIZE_HOSTED_PHOENIX_ENDPOINT = "https://otlp.arize.com/v1/traces"
|
||||
|
||||
|
||||
class ArizePhoenixLogger(OpenTelemetry):
|
||||
class ArizePhoenixLogger(OpenTelemetry): # type: ignore
|
||||
"""
|
||||
Arize Phoenix logger that sends traces to a Phoenix endpoint.
|
||||
|
||||
Creates its own dedicated TracerProvider so it can coexist with the
|
||||
generic ``otel`` callback (or any other OTEL-based integration) without
|
||||
fighting over the global ``opentelemetry.trace`` TracerProvider singleton.
|
||||
"""
|
||||
|
||||
def _init_tracing(self, tracer_provider):
|
||||
"""
|
||||
Override to always create a *private* TracerProvider for Arize Phoenix.
|
||||
|
||||
The base ``OpenTelemetry._init_tracing`` falls back to the global
|
||||
TracerProvider when one already exists. That causes whichever
|
||||
integration initialises second to silently reuse the first one's
|
||||
exporter, so spans only reach one destination.
|
||||
|
||||
By creating our own provider we guarantee Arize Phoenix always gets
|
||||
its own exporter pipeline, regardless of initialisation order.
|
||||
"""
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
if tracer_provider is not None:
|
||||
# Explicitly supplied (e.g. in tests) — honour it.
|
||||
self.tracer = tracer_provider.get_tracer("litellm")
|
||||
self.span_kind = SpanKind
|
||||
return
|
||||
|
||||
# Always create a dedicated provider — never touch the global one.
|
||||
provider = TracerProvider(resource=self._get_litellm_resource(self.config))
|
||||
provider.add_span_processor(self._get_span_processor())
|
||||
self.tracer = provider.get_tracer("litellm")
|
||||
self.span_kind = SpanKind
|
||||
verbose_logger.debug(
|
||||
"ArizePhoenixLogger: Created dedicated TracerProvider "
|
||||
"(endpoint=%s, exporter=%s)",
|
||||
self.config.endpoint,
|
||||
self.config.exporter,
|
||||
)
|
||||
|
||||
def _init_otel_logger_on_litellm_proxy(self):
|
||||
"""
|
||||
Override: Arize Phoenix should NOT overwrite the proxy's
|
||||
``open_telemetry_logger``. That attribute is reserved for the
|
||||
primary ``otel`` callback which handles proxy-level parent spans.
|
||||
"""
|
||||
pass
|
||||
|
||||
def set_attributes(self, span: Span, kwargs, response_obj: Optional[Any]):
|
||||
ArizePhoenixLogger.set_arize_phoenix_attributes(span, kwargs, response_obj)
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def set_arize_phoenix_attributes(span: Span, kwargs, response_obj):
|
||||
from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes import safe_set_attribute
|
||||
|
||||
_utils.set_attributes(span, kwargs, response_obj, ArizeOTELAttributes)
|
||||
|
||||
# Set project name on the span for all traces to go to custom Phoenix projects
|
||||
config = ArizePhoenixLogger.get_arize_phoenix_config()
|
||||
if config.project_name:
|
||||
from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes import safe_set_attribute
|
||||
safe_set_attribute(span, "openinference.project.name", config.project_name)
|
||||
|
||||
|
||||
# Dynamic project name: check metadata first, then fall back to env var config
|
||||
dynamic_project_name = ArizePhoenixLogger._get_dynamic_project_name(kwargs)
|
||||
if dynamic_project_name:
|
||||
safe_set_attribute(span, "openinference.project.name", dynamic_project_name)
|
||||
else:
|
||||
# Fall back to static config from env var
|
||||
config = ArizePhoenixLogger.get_arize_phoenix_config()
|
||||
if config.project_name:
|
||||
safe_set_attribute(span, "openinference.project.name", config.project_name)
|
||||
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def _get_dynamic_project_name(kwargs) -> Optional[str]:
|
||||
"""
|
||||
Retrieve dynamic Phoenix project name from request metadata.
|
||||
|
||||
Users can set `metadata.phoenix_project_name` in their request to route
|
||||
traces to different Phoenix projects dynamically.
|
||||
"""
|
||||
standard_logging_payload = kwargs.get("standard_logging_object")
|
||||
if isinstance(standard_logging_payload, dict):
|
||||
metadata = standard_logging_payload.get("metadata")
|
||||
if isinstance(metadata, dict):
|
||||
project_name = metadata.get("phoenix_project_name")
|
||||
if project_name:
|
||||
return str(project_name)
|
||||
|
||||
# Also check litellm_params.metadata for SDK usage
|
||||
litellm_params = kwargs.get("litellm_params")
|
||||
if isinstance(litellm_params, dict):
|
||||
metadata = litellm_params.get("metadata") or {}
|
||||
else:
|
||||
metadata = {}
|
||||
if isinstance(metadata, dict):
|
||||
project_name = metadata.get("phoenix_project_name")
|
||||
if project_name:
|
||||
return str(project_name)
|
||||
|
||||
return None
|
||||
|
||||
def _handle_success(self, kwargs, response_obj, start_time, end_time):
|
||||
"""
|
||||
Override to prevent creating duplicate litellm_request spans when a proxy parent span exists.
|
||||
|
||||
ArizePhoenixLogger should reuse the proxy parent span instead of creating a new litellm_request span,
|
||||
to maintain a shallow span hierarchy as expected by Arize Phoenix.
|
||||
"""
|
||||
from opentelemetry.trace import Status, StatusCode
|
||||
from litellm.secret_managers.main import get_secret_bool
|
||||
from litellm.integrations.opentelemetry import LITELLM_PROXY_REQUEST_SPAN_NAME
|
||||
|
||||
verbose_logger.debug(
|
||||
"ArizePhoenixLogger: Logging kwargs: %s, OTEL config settings=%s",
|
||||
kwargs,
|
||||
self.config,
|
||||
)
|
||||
ctx, parent_span = self._get_span_context(kwargs)
|
||||
|
||||
# ArizePhoenixLogger NEVER creates a litellm_request span when a proxy parent span exists
|
||||
# This is different from the base OpenTelemetry behavior which respects USE_OTEL_LITELLM_REQUEST_SPAN
|
||||
should_create_primary_span = parent_span is None or (
|
||||
parent_span.name != LITELLM_PROXY_REQUEST_SPAN_NAME
|
||||
and get_secret_bool("USE_OTEL_LITELLM_REQUEST_SPAN")
|
||||
)
|
||||
|
||||
if should_create_primary_span:
|
||||
# Create a new litellm_request span
|
||||
span = self._start_primary_span(
|
||||
kwargs, response_obj, start_time, end_time, ctx
|
||||
)
|
||||
# Raw-request sub-span (if enabled) - child of litellm_request span
|
||||
self._maybe_log_raw_request(
|
||||
kwargs, response_obj, start_time, end_time, span
|
||||
)
|
||||
# Ensure proxy-request parent span is annotated with the actual operation kind
|
||||
if (
|
||||
parent_span is not None
|
||||
and parent_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME
|
||||
):
|
||||
self.set_attributes(parent_span, kwargs, response_obj)
|
||||
else:
|
||||
# Do not create primary span (keep hierarchy shallow when parent exists)
|
||||
span = None
|
||||
# Only set attributes if the span is still recording (not closed)
|
||||
# Note: parent_span is guaranteed to be not None here
|
||||
if parent_span.is_recording():
|
||||
parent_span.set_status(Status(StatusCode.OK))
|
||||
self.set_attributes(parent_span, kwargs, response_obj)
|
||||
# Raw-request as direct child of parent_span
|
||||
self._maybe_log_raw_request(
|
||||
kwargs, response_obj, start_time, end_time, parent_span
|
||||
)
|
||||
|
||||
# 3. Guardrail span
|
||||
self._create_guardrail_span(kwargs=kwargs, context=ctx)
|
||||
|
||||
# 4. Metrics & cost recording
|
||||
self._record_metrics(kwargs, response_obj, start_time, end_time)
|
||||
|
||||
# 5. Semantic logs.
|
||||
if self.config.enable_events:
|
||||
log_span = span if span is not None else parent_span
|
||||
if log_span is not None:
|
||||
self._emit_semantic_logs(kwargs, response_obj, log_span)
|
||||
|
||||
# 6. Do NOT end parent span - it should be managed by its creator
|
||||
# External spans (from Langfuse, user code, HTTP headers, global context) must not be closed by LiteLLM
|
||||
# However, proxy-created spans should be closed here
|
||||
if (
|
||||
parent_span is not None
|
||||
and parent_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME
|
||||
):
|
||||
parent_span.end(end_time=self._to_ns(end_time))
|
||||
|
||||
@staticmethod
|
||||
def get_arize_phoenix_config() -> ArizePhoenixConfig:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -103,10 +103,15 @@ class CBFTransformer:
|
|||
# Use team_alias if available, otherwise team_id, otherwise fallback to 'unknown'
|
||||
entity_id = str(team_alias) if team_alias else (str(team_id) if team_id else 'unknown')
|
||||
|
||||
# Get alias fields if they exist
|
||||
api_key_alias = row.get('api_key_alias')
|
||||
organization_alias = row.get('organization_alias')
|
||||
project_alias = row.get('project_alias')
|
||||
user_alias = row.get('user_alias')
|
||||
|
||||
dimensions = {
|
||||
'entity_type': CZEntityType.TEAM.value,
|
||||
'entity_id': entity_id,
|
||||
'team_id': str(team_id) if team_id else 'unknown',
|
||||
'team_alias': str(team_alias) if team_alias else 'unknown',
|
||||
'model': model,
|
||||
'model_group': str(row.get('model_group', '')),
|
||||
|
|
@ -119,28 +124,37 @@ class CBFTransformer:
|
|||
'failed_requests': str(row.get('failed_requests', 0)),
|
||||
'cache_creation_tokens': str(row.get('cache_creation_input_tokens', 0)),
|
||||
'cache_read_tokens': str(row.get('cache_read_input_tokens', 0)),
|
||||
'organization_alias': str(organization_alias) if organization_alias else '',
|
||||
'project_alias': str(project_alias) if project_alias else '',
|
||||
'user_alias': str(user_alias) if user_alias else '',
|
||||
}
|
||||
|
||||
# Extract CZRN components to populate corresponding CBF columns
|
||||
czrn_components = self.czrn_generator.extract_components(resource_id)
|
||||
service_type, provider, region, owner_account_id, resource_type, cloud_local_id = czrn_components
|
||||
|
||||
# Build resource/account as concat of api_key_alias and api_key_prefix
|
||||
resource_account = f"{api_key_alias}|{api_key_hash}" if api_key_alias else api_key_hash
|
||||
|
||||
# CloudZero CBF format with proper column names
|
||||
cbf_record = {
|
||||
# Required CBF fields
|
||||
'time/usage_start': usage_date.isoformat() if usage_date else None, # Required: ISO-formatted UTC datetime
|
||||
'cost/cost': float(row.get('spend', 0.0)), # Required: billed cost
|
||||
'resource/id': resource_id, # Required when resource tags are present
|
||||
'resource/id': model, # Send model name
|
||||
|
||||
# Usage metrics for token consumption
|
||||
'usage/amount': total_tokens, # Numeric value of tokens consumed
|
||||
'usage/units': 'tokens', # Description of token units
|
||||
|
||||
# CBF fields that correspond to CZRN components
|
||||
'resource/service': service_type, # Maps to CZRN service-type (litellm)
|
||||
'resource/account': owner_account_id, # Maps to CZRN owner-account-id (entity_id)
|
||||
# CBF fields - updated per LIT-1907
|
||||
'resource/service': str(row.get('model_group', '')), # Send model_group
|
||||
'resource/account': resource_account, # Send api_key_alias|api_key_prefix
|
||||
'resource/region': region, # Maps to CZRN region (cross-region)
|
||||
'resource/usage_family': resource_type, # Maps to CZRN resource-type (llm-usage)
|
||||
'resource/usage_family': str(row.get('custom_llm_provider', '')), # Send provider
|
||||
|
||||
# Action field
|
||||
'action/operation': str(team_id) if team_id else '', # Send team_id
|
||||
|
||||
# Line item details
|
||||
'lineitem/type': 'Usage', # Standard usage line item
|
||||
|
|
@ -155,13 +169,11 @@ class CBFTransformer:
|
|||
if value and value != 'N/A' and value != 'unknown': # Only add meaningful tags
|
||||
cbf_record[f'resource/tag:{key}'] = str(value)
|
||||
|
||||
# Add token breakdown as resource tags for analysis
|
||||
# Add token breakdown as resource tags for analysis (excluding total_tokens per LIT-1907)
|
||||
if prompt_tokens > 0:
|
||||
cbf_record['resource/tag:prompt_tokens'] = str(prompt_tokens)
|
||||
if completion_tokens > 0:
|
||||
cbf_record['resource/tag:completion_tokens'] = str(completion_tokens)
|
||||
if total_tokens > 0:
|
||||
cbf_record['resource/tag:total_tokens'] = str(total_tokens)
|
||||
|
||||
return CBFRecord(cbf_record)
|
||||
|
||||
|
|
|
|||
|
|
@ -616,6 +616,7 @@ class CustomGuardrail(CustomLogger):
|
|||
end_time: Optional[float] = None,
|
||||
duration: Optional[float] = None,
|
||||
event_type: Optional[GuardrailEventHooks] = None,
|
||||
original_inputs: Optional[Dict] = None,
|
||||
):
|
||||
"""
|
||||
Add StandardLoggingGuardrailInformation to the request data
|
||||
|
|
@ -625,6 +626,17 @@ class CustomGuardrail(CustomLogger):
|
|||
# Convert None to empty dict to satisfy type requirements
|
||||
guardrail_response = {} if response is None else response
|
||||
|
||||
# For apply_guardrail functions in custom_code_guardrail scenario,
|
||||
# simplify the logged response to "allow", "deny", or "mask"
|
||||
if original_inputs is not None and isinstance(response, dict):
|
||||
# Check if inputs were modified by comparing them
|
||||
if self._inputs_were_modified(original_inputs, response):
|
||||
guardrail_response = "mask"
|
||||
else:
|
||||
guardrail_response = "allow"
|
||||
|
||||
verbose_logger.debug(f"Guardrail response: {response}")
|
||||
|
||||
self.add_standard_logging_guardrail_information_to_request_data(
|
||||
guardrail_json_response=guardrail_response,
|
||||
request_data=request_data,
|
||||
|
|
@ -650,8 +662,14 @@ class CustomGuardrail(CustomLogger):
|
|||
|
||||
This gets logged on downsteam Langfuse, DataDog, etc.
|
||||
"""
|
||||
# For custom_code_guardrail scenario, log as "deny" instead of full exception
|
||||
# Check if this is from custom_code_guardrail by checking the class name
|
||||
guardrail_response: Union[Exception, str] = e
|
||||
if "CustomCodeGuardrail" in self.__class__.__name__:
|
||||
guardrail_response = "deny"
|
||||
|
||||
self.add_standard_logging_guardrail_information_to_request_data(
|
||||
guardrail_json_response=e,
|
||||
guardrail_json_response=guardrail_response,
|
||||
request_data=request_data,
|
||||
guardrail_status="guardrail_failed_to_respond",
|
||||
duration=duration,
|
||||
|
|
@ -661,6 +679,25 @@ class CustomGuardrail(CustomLogger):
|
|||
)
|
||||
raise e
|
||||
|
||||
def _inputs_were_modified(self, original_inputs: Dict, response: Dict) -> bool:
|
||||
"""
|
||||
Compare original inputs with response to determine if content was modified.
|
||||
|
||||
Returns True if the inputs were modified (mask scenario), False otherwise (allow scenario).
|
||||
"""
|
||||
# Get all keys from both dictionaries
|
||||
all_keys = set(original_inputs.keys()) | set(response.keys())
|
||||
|
||||
# Compare each key's value
|
||||
for key in all_keys:
|
||||
original_value = original_inputs.get(key)
|
||||
response_value = response.get(key)
|
||||
if original_value != response_value:
|
||||
return True
|
||||
|
||||
# No modifications detected
|
||||
return False
|
||||
|
||||
def mask_content_in_string(
|
||||
self,
|
||||
content_string: str,
|
||||
|
|
@ -768,6 +805,12 @@ def log_guardrail_information(func):
|
|||
self: CustomGuardrail = args[0]
|
||||
request_data: dict = kwargs.get("data") or kwargs.get("request_data") or {}
|
||||
event_type = _infer_event_type_from_function_name(func.__name__)
|
||||
|
||||
# Store original inputs for comparison (for apply_guardrail functions)
|
||||
original_inputs = None
|
||||
if func.__name__ == "apply_guardrail" and "inputs" in kwargs:
|
||||
original_inputs = kwargs.get("inputs")
|
||||
|
||||
try:
|
||||
response = await func(*args, **kwargs)
|
||||
return self._process_response(
|
||||
|
|
@ -777,6 +820,7 @@ def log_guardrail_information(func):
|
|||
end_time=datetime.now().timestamp(),
|
||||
duration=(datetime.now() - start_time).total_seconds(),
|
||||
event_type=event_type,
|
||||
original_inputs=original_inputs,
|
||||
)
|
||||
except Exception as e:
|
||||
return self._process_error(
|
||||
|
|
@ -794,6 +838,12 @@ def log_guardrail_information(func):
|
|||
self: CustomGuardrail = args[0]
|
||||
request_data: dict = kwargs.get("data") or kwargs.get("request_data") or {}
|
||||
event_type = _infer_event_type_from_function_name(func.__name__)
|
||||
|
||||
# Store original inputs for comparison (for apply_guardrail functions)
|
||||
original_inputs = None
|
||||
if func.__name__ == "apply_guardrail" and "inputs" in kwargs:
|
||||
original_inputs = kwargs.get("inputs")
|
||||
|
||||
try:
|
||||
response = func(*args, **kwargs)
|
||||
return self._process_response(
|
||||
|
|
@ -801,6 +851,7 @@ def log_guardrail_information(func):
|
|||
request_data=request_data,
|
||||
duration=(datetime.now() - start_time).total_seconds(),
|
||||
event_type=event_type,
|
||||
original_inputs=original_inputs,
|
||||
)
|
||||
except Exception as e:
|
||||
return self._process_error(
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
|
|||
from urllib.parse import quote
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import LITELLM_ASYNCIO_QUEUE_MAXSIZE
|
||||
from litellm.integrations.additional_logging_utils import AdditionalLoggingUtils
|
||||
from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
|
||||
from litellm.proxy._types import CommonProxyErrors
|
||||
|
|
@ -41,7 +42,9 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
|
|||
batch_size=self.batch_size,
|
||||
flush_interval=self.flush_interval,
|
||||
)
|
||||
self.log_queue: asyncio.Queue[GCSLogQueueItem] = asyncio.Queue() # type: ignore[assignment]
|
||||
self.log_queue: asyncio.Queue[GCSLogQueueItem] = asyncio.Queue( # type: ignore[assignment]
|
||||
maxsize=LITELLM_ASYNCIO_QUEUE_MAXSIZE
|
||||
)
|
||||
asyncio.create_task(self.periodic_flush())
|
||||
AdditionalLoggingUtils.__init__(self)
|
||||
|
||||
|
|
@ -69,6 +72,9 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
|
|||
)
|
||||
if logging_payload is None:
|
||||
raise ValueError("standard_logging_object not found in kwargs")
|
||||
# When queue is at maxsize, flush immediately to make room (no blocking, no data dropped)
|
||||
if self.log_queue.full():
|
||||
await self.flush_queue()
|
||||
await self.log_queue.put(
|
||||
GCSLogQueueItem(
|
||||
payload=logging_payload, kwargs=kwargs, response_obj=response_obj
|
||||
|
|
@ -91,9 +97,9 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
|
|||
)
|
||||
if logging_payload is None:
|
||||
raise ValueError("standard_logging_object not found in kwargs")
|
||||
# Add to logging queue - this will be flushed periodically
|
||||
# Use asyncio.Queue.put() for thread-safe concurrent access
|
||||
# If queue is full, this will block until space is available (backpressure)
|
||||
# When queue is at maxsize, flush immediately to make room (no blocking, no data dropped)
|
||||
if self.log_queue.full():
|
||||
await self.flush_queue()
|
||||
await self.log_queue.put(
|
||||
GCSLogQueueItem(
|
||||
payload=logging_payload, kwargs=kwargs, response_obj=response_obj
|
||||
|
|
|
|||
|
|
@ -5,6 +5,10 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations._types.open_inference import (
|
||||
OpenInferenceSpanKindValues,
|
||||
SpanAttributes,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.secret_managers.main import get_secret_bool
|
||||
|
|
@ -17,10 +21,6 @@ from litellm.types.utils import (
|
|||
StandardCallbackDynamicParams,
|
||||
StandardLoggingPayload,
|
||||
)
|
||||
from litellm.integrations._types.open_inference import (
|
||||
OpenInferenceSpanKindValues,
|
||||
SpanAttributes,
|
||||
)
|
||||
|
||||
# OpenTelemetry imports moved to individual functions to avoid import errors when not installed
|
||||
|
||||
|
|
@ -40,7 +40,9 @@ if TYPE_CHECKING:
|
|||
Context = Union[_Context, Any]
|
||||
SpanExporter = Union[_SpanExporter, Any]
|
||||
UserAPIKeyAuth = Union[_UserAPIKeyAuth, Any]
|
||||
ManagementEndpointLoggingPayload = Union[_ManagementEndpointLoggingPayload, Any]
|
||||
ManagementEndpointLoggingPayload = Union[
|
||||
_ManagementEndpointLoggingPayload, Any
|
||||
]
|
||||
else:
|
||||
Span = Any
|
||||
Tracer = Any
|
||||
|
|
@ -70,6 +72,13 @@ class OpenTelemetryConfig:
|
|||
model_id: Optional[str] = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
# If endpoint is specified but exporter is still the default "console",
|
||||
# automatically infer "otlp_http" to send traces to the endpoint.
|
||||
# This fixes an issue where UI-configured OTEL settings would default
|
||||
# to console output instead of sending traces to the configured endpoint.
|
||||
if self.endpoint and isinstance(self.exporter, str) and self.exporter == "console":
|
||||
self.exporter = "otlp_http"
|
||||
|
||||
if not self.service_name:
|
||||
self.service_name = os.getenv("OTEL_SERVICE_NAME", "litellm")
|
||||
if not self.deployment_environment:
|
||||
|
|
@ -95,12 +104,16 @@ class OpenTelemetryConfig:
|
|||
exporter = os.getenv(
|
||||
"OTEL_EXPORTER_OTLP_PROTOCOL", os.getenv("OTEL_EXPORTER", "console")
|
||||
)
|
||||
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", os.getenv("OTEL_ENDPOINT"))
|
||||
endpoint = os.getenv(
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT", os.getenv("OTEL_ENDPOINT")
|
||||
)
|
||||
headers = os.getenv(
|
||||
"OTEL_EXPORTER_OTLP_HEADERS", os.getenv("OTEL_HEADERS")
|
||||
) # example: OTEL_HEADERS=x-honeycomb-team=B85YgLm96***"
|
||||
enable_metrics: bool = (
|
||||
os.getenv("LITELLM_OTEL_INTEGRATION_ENABLE_METRICS", "false").lower()
|
||||
os.getenv(
|
||||
"LITELLM_OTEL_INTEGRATION_ENABLE_METRICS", "false"
|
||||
).lower()
|
||||
== "true"
|
||||
)
|
||||
enable_events: bool = (
|
||||
|
|
@ -108,7 +121,9 @@ class OpenTelemetryConfig:
|
|||
== "true"
|
||||
)
|
||||
service_name = os.getenv("OTEL_SERVICE_NAME", "litellm")
|
||||
deployment_environment = os.getenv("OTEL_ENVIRONMENT_NAME", "production")
|
||||
deployment_environment = os.getenv(
|
||||
"OTEL_ENVIRONMENT_NAME", "production"
|
||||
)
|
||||
model_id = os.getenv("OTEL_MODEL_ID", service_name)
|
||||
|
||||
if exporter == "in_memory":
|
||||
|
|
@ -157,7 +172,9 @@ class OpenTelemetry(CustomLogger):
|
|||
logging.getLogger(__name__)
|
||||
|
||||
# Enable OpenTelemetry logging
|
||||
otel_exporter_logger = logging.getLogger("opentelemetry.sdk.trace.export")
|
||||
otel_exporter_logger = logging.getLogger(
|
||||
"opentelemetry.sdk.trace.export"
|
||||
)
|
||||
otel_exporter_logger.setLevel(logging.DEBUG)
|
||||
|
||||
# init CustomLogger params
|
||||
|
|
@ -253,7 +270,9 @@ class OpenTelemetry(CustomLogger):
|
|||
# Don't call set_provider to preserve existing context
|
||||
else:
|
||||
# Default proxy provider or unknown type, create our own
|
||||
verbose_logger.debug("OpenTelemetry: Creating new %s", provider_name)
|
||||
verbose_logger.debug(
|
||||
"OpenTelemetry: Creating new %s", provider_name
|
||||
)
|
||||
provider = create_new_provider_fn()
|
||||
set_provider_fn(provider)
|
||||
except Exception as e:
|
||||
|
|
@ -274,7 +293,9 @@ class OpenTelemetry(CustomLogger):
|
|||
from opentelemetry.trace import SpanKind
|
||||
|
||||
def create_tracer_provider():
|
||||
provider = TracerProvider(resource=self._get_litellm_resource(self.config))
|
||||
provider = TracerProvider(
|
||||
resource=self._get_litellm_resource(self.config)
|
||||
)
|
||||
provider.add_span_processor(self._get_span_processor())
|
||||
return provider
|
||||
|
||||
|
|
@ -388,10 +409,14 @@ class OpenTelemetry(CustomLogger):
|
|||
def log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
self._handle_failure(kwargs, response_obj, start_time, end_time)
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
async def async_log_success_event(
|
||||
self, kwargs, response_obj, start_time, end_time
|
||||
):
|
||||
self._handle_success(kwargs, response_obj, start_time, end_time)
|
||||
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
async def async_log_failure_event(
|
||||
self, kwargs, response_obj, start_time, end_time
|
||||
):
|
||||
self._handle_failure(kwargs, response_obj, start_time, end_time)
|
||||
|
||||
async def async_service_success_hook(
|
||||
|
|
@ -588,7 +613,9 @@ class OpenTelemetry(CustomLogger):
|
|||
|
||||
if dynamic_headers is not None:
|
||||
# Create spans using a temporary tracer with dynamic headers
|
||||
tracer_to_use = self._get_tracer_with_dynamic_headers(dynamic_headers)
|
||||
tracer_to_use = self._get_tracer_with_dynamic_headers(
|
||||
dynamic_headers
|
||||
)
|
||||
verbose_logger.debug(
|
||||
"Using dynamic headers for this request: %s", dynamic_headers
|
||||
)
|
||||
|
|
@ -624,7 +651,9 @@ class OpenTelemetry(CustomLogger):
|
|||
)
|
||||
|
||||
# Create a temporary tracer provider with dynamic headers
|
||||
temp_provider = TracerProvider(resource=self._get_litellm_resource(self.config))
|
||||
temp_provider = TracerProvider(
|
||||
resource=self._get_litellm_resource(self.config)
|
||||
)
|
||||
temp_provider.add_span_processor(
|
||||
self._get_span_processor(dynamic_headers=dynamic_headers)
|
||||
)
|
||||
|
|
@ -755,7 +784,9 @@ class OpenTelemetry(CustomLogger):
|
|||
metadata = litellm_params.get("metadata") or {}
|
||||
generation_name = metadata.get("generation_name")
|
||||
|
||||
raw_span_name = generation_name if generation_name else RAW_REQUEST_SPAN_NAME
|
||||
raw_span_name = (
|
||||
generation_name if generation_name else RAW_REQUEST_SPAN_NAME
|
||||
)
|
||||
|
||||
otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
|
||||
raw_span = otel_tracer.start_span(
|
||||
|
|
@ -780,7 +811,9 @@ class OpenTelemetry(CustomLogger):
|
|||
}
|
||||
|
||||
std_log = kwargs.get("standard_logging_object")
|
||||
md = getattr(std_log, "metadata", None) or (std_log or {}).get("metadata", {})
|
||||
md = getattr(std_log, "metadata", None) or (std_log or {}).get(
|
||||
"metadata", {}
|
||||
)
|
||||
for key in [
|
||||
"user_api_key_hash",
|
||||
"user_api_key_alias",
|
||||
|
|
@ -802,9 +835,9 @@ class OpenTelemetry(CustomLogger):
|
|||
common_attrs[f"metadata.{key}"] = str(md[key])
|
||||
|
||||
# get hidden params
|
||||
hidden_params = getattr(std_log, "hidden_params", None) or (std_log or {}).get(
|
||||
"hidden_params", {}
|
||||
)
|
||||
hidden_params = getattr(std_log, "hidden_params", None) or (
|
||||
std_log or {}
|
||||
).get("hidden_params", {})
|
||||
if hidden_params:
|
||||
common_attrs["hidden_params"] = safe_dumps(hidden_params)
|
||||
|
||||
|
|
@ -838,7 +871,9 @@ class OpenTelemetry(CustomLogger):
|
|||
self._record_response_duration_metric(kwargs, end_time, common_attrs)
|
||||
|
||||
@staticmethod
|
||||
def _to_timestamp(val: Optional[Union[datetime, float, str]]) -> Optional[float]:
|
||||
def _to_timestamp(
|
||||
val: Optional[Union[datetime, float, str]],
|
||||
) -> Optional[float]:
|
||||
"""Convert datetime/float/string to timestamp."""
|
||||
if val is None:
|
||||
return None
|
||||
|
|
@ -855,7 +890,9 @@ class OpenTelemetry(CustomLogger):
|
|||
except ValueError:
|
||||
return None
|
||||
|
||||
def _record_time_to_first_token_metric(self, kwargs: dict, common_attrs: dict):
|
||||
def _record_time_to_first_token_metric(
|
||||
self, kwargs: dict, common_attrs: dict
|
||||
):
|
||||
"""Record Time to First Token (TTFT) metric for streaming requests."""
|
||||
optional_params = kwargs.get("optional_params", {})
|
||||
is_streaming = optional_params.get("stream", False)
|
||||
|
|
@ -868,7 +905,10 @@ class OpenTelemetry(CustomLogger):
|
|||
api_call_start_time = kwargs.get("api_call_start_time", None)
|
||||
completion_start_time = kwargs.get("completion_start_time", None)
|
||||
|
||||
if api_call_start_time is not None and completion_start_time is not None:
|
||||
if (
|
||||
api_call_start_time is not None
|
||||
and completion_start_time is not None
|
||||
):
|
||||
# Convert to timestamps if needed (handles datetime, float, and string)
|
||||
api_call_start_ts = self._to_timestamp(api_call_start_time)
|
||||
completion_start_ts = self._to_timestamp(completion_start_time)
|
||||
|
|
@ -876,7 +916,9 @@ class OpenTelemetry(CustomLogger):
|
|||
if api_call_start_ts is None or completion_start_ts is None:
|
||||
return # Skip recording if conversion failed
|
||||
|
||||
time_to_first_token_seconds = completion_start_ts - api_call_start_ts
|
||||
time_to_first_token_seconds = (
|
||||
completion_start_ts - api_call_start_ts
|
||||
)
|
||||
self._time_to_first_token_histogram.record(
|
||||
time_to_first_token_seconds, attributes=common_attrs
|
||||
)
|
||||
|
|
@ -946,7 +988,9 @@ class OpenTelemetry(CustomLogger):
|
|||
generation_time_seconds = duration_s
|
||||
|
||||
if generation_time_seconds > 0:
|
||||
time_per_output_token_seconds = generation_time_seconds / completion_tokens
|
||||
time_per_output_token_seconds = (
|
||||
generation_time_seconds / completion_tokens
|
||||
)
|
||||
self._time_per_output_token_histogram.record(
|
||||
time_per_output_token_seconds, attributes=common_attrs
|
||||
)
|
||||
|
|
@ -1007,21 +1051,26 @@ class OpenTelemetry(CustomLogger):
|
|||
# See: https://github.com/open-telemetry/opentelemetry-python/pull/4676
|
||||
# TODO: Refactor to use the proper OTEL Logs API instead of directly creating SDK LogRecords
|
||||
|
||||
from opentelemetry._logs import SeverityNumber, get_logger, get_logger_provider
|
||||
from opentelemetry._logs import (
|
||||
SeverityNumber,
|
||||
get_logger,
|
||||
)
|
||||
|
||||
try:
|
||||
from opentelemetry.sdk._logs import LogRecord as SdkLogRecord # type: ignore[attr-defined] # OTEL < 1.39.0
|
||||
except ImportError:
|
||||
from opentelemetry.sdk._logs._internal import LogRecord as SdkLogRecord # type: ignore[attr-defined, no-redef] # OTEL >= 1.39.0
|
||||
# MyPy evaluates both branches of try/except imports and can fail when
|
||||
# newer OTEL stubs remove/relocate symbols. Gate the typing import so
|
||||
# only the canonical location is type-checked.
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.sdk._logs._internal import LogRecord as SdkLogRecord
|
||||
else:
|
||||
try:
|
||||
from opentelemetry.sdk._logs import (
|
||||
LogRecord as SdkLogRecord, # type: ignore[attr-defined]
|
||||
)
|
||||
except ImportError:
|
||||
from opentelemetry.sdk._logs._internal import LogRecord as SdkLogRecord
|
||||
|
||||
otel_logger = get_logger(LITELLM_LOGGER_NAME)
|
||||
|
||||
# Get the resource from the logger provider
|
||||
logger_provider = get_logger_provider()
|
||||
resource = getattr(
|
||||
logger_provider, "_resource", None
|
||||
) or self._get_litellm_resource(self.config)
|
||||
|
||||
parent_ctx = span.get_span_context()
|
||||
provider = (kwargs.get("litellm_params") or {}).get(
|
||||
"custom_llm_provider", "Unknown"
|
||||
|
|
@ -1030,7 +1079,10 @@ class OpenTelemetry(CustomLogger):
|
|||
# per-message events
|
||||
for msg in kwargs.get("messages", []):
|
||||
role = msg.get("role", "user")
|
||||
attrs = {"event_name": "gen_ai.content.prompt", "gen_ai.system": provider}
|
||||
attrs = {
|
||||
"event_name": "gen_ai.content.prompt",
|
||||
"gen_ai.system": provider,
|
||||
}
|
||||
if role == "tool" and msg.get("id"):
|
||||
attrs["id"] = msg["id"]
|
||||
if self.message_logging and msg.get("content"):
|
||||
|
|
@ -1044,7 +1096,6 @@ class OpenTelemetry(CustomLogger):
|
|||
severity_number=SeverityNumber.INFO,
|
||||
severity_text="INFO",
|
||||
body=msg.copy(),
|
||||
resource=resource,
|
||||
attributes=attrs,
|
||||
)
|
||||
otel_logger.emit(log_record)
|
||||
|
|
@ -1076,7 +1127,6 @@ class OpenTelemetry(CustomLogger):
|
|||
severity_number=SeverityNumber.INFO,
|
||||
severity_text="INFO",
|
||||
body=body,
|
||||
resource=resource,
|
||||
attributes=attrs,
|
||||
)
|
||||
otel_logger.emit(log_record)
|
||||
|
|
@ -1146,7 +1196,9 @@ class OpenTelemetry(CustomLogger):
|
|||
value=guardrail_information.get("guardrail_mode"),
|
||||
)
|
||||
|
||||
masked_entity_count = guardrail_information.get("masked_entity_count")
|
||||
masked_entity_count = guardrail_information.get(
|
||||
"masked_entity_count"
|
||||
)
|
||||
if masked_entity_count is not None:
|
||||
guardrail_span.set_attribute(
|
||||
"masked_entity_count", safe_dumps(masked_entity_count)
|
||||
|
|
@ -1173,8 +1225,9 @@ class OpenTelemetry(CustomLogger):
|
|||
# Decide whether to create a primary span
|
||||
# Always create if no parent span exists (backward compatibility)
|
||||
# OR if USE_OTEL_LITELLM_REQUEST_SPAN is explicitly enabled
|
||||
should_create_primary_span = parent_otel_span is None or get_secret_bool(
|
||||
"USE_OTEL_LITELLM_REQUEST_SPAN"
|
||||
should_create_primary_span = (
|
||||
parent_otel_span is None
|
||||
or get_secret_bool("USE_OTEL_LITELLM_REQUEST_SPAN")
|
||||
)
|
||||
|
||||
if should_create_primary_span:
|
||||
|
|
@ -1200,7 +1253,9 @@ class OpenTelemetry(CustomLogger):
|
|||
if parent_otel_span.is_recording():
|
||||
parent_otel_span.set_status(Status(StatusCode.ERROR))
|
||||
self.set_attributes(parent_otel_span, kwargs, response_obj)
|
||||
self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
|
||||
self._record_exception_on_span(
|
||||
span=parent_otel_span, kwargs=kwargs
|
||||
)
|
||||
|
||||
# Create span for guardrail information
|
||||
self._create_guardrail_span(kwargs=kwargs, context=_parent_context)
|
||||
|
|
@ -1223,7 +1278,9 @@ class OpenTelemetry(CustomLogger):
|
|||
2. Sets structured error attributes from StandardLoggingPayloadErrorInformation
|
||||
"""
|
||||
try:
|
||||
from litellm.integrations._types.open_inference import ErrorAttributes
|
||||
from litellm.integrations._types.open_inference import (
|
||||
ErrorAttributes,
|
||||
)
|
||||
|
||||
# Get the exception object if available
|
||||
exception = kwargs.get("exception")
|
||||
|
|
@ -1233,15 +1290,17 @@ class OpenTelemetry(CustomLogger):
|
|||
span.record_exception(exception)
|
||||
|
||||
# Get StandardLoggingPayload for structured error information
|
||||
standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get(
|
||||
"standard_logging_object"
|
||||
standard_logging_payload: Optional[StandardLoggingPayload] = (
|
||||
kwargs.get("standard_logging_object")
|
||||
)
|
||||
|
||||
if standard_logging_payload is None:
|
||||
return
|
||||
|
||||
# Extract error_information from StandardLoggingPayload
|
||||
error_information = standard_logging_payload.get("error_information")
|
||||
error_information = standard_logging_payload.get(
|
||||
"error_information"
|
||||
)
|
||||
|
||||
if error_information is None:
|
||||
# Fallback to error_str if error_information is not available
|
||||
|
|
@ -1331,7 +1390,9 @@ class OpenTelemetry(CustomLogger):
|
|||
)
|
||||
pass
|
||||
|
||||
def cast_as_primitive_value_type(self, value) -> Union[str, bool, int, float]:
|
||||
def cast_as_primitive_value_type(
|
||||
self, value
|
||||
) -> Union[str, bool, int, float]:
|
||||
"""
|
||||
Casts the value to a primitive OTEL type if it is not already a primitive type.
|
||||
|
||||
|
|
@ -1401,8 +1462,8 @@ class OpenTelemetry(CustomLogger):
|
|||
|
||||
optional_params = kwargs.get("optional_params", {})
|
||||
litellm_params = kwargs.get("litellm_params", {}) or {}
|
||||
standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get(
|
||||
"standard_logging_object"
|
||||
standard_logging_payload: Optional[StandardLoggingPayload] = (
|
||||
kwargs.get("standard_logging_object")
|
||||
)
|
||||
if standard_logging_payload is None:
|
||||
raise ValueError("standard_logging_object not found in kwargs")
|
||||
|
|
@ -1424,11 +1485,13 @@ class OpenTelemetry(CustomLogger):
|
|||
) or (standard_logging_payload or {}).get("hidden_params", {})
|
||||
if hidden_params:
|
||||
self.safe_set_attribute(
|
||||
span=span, key="hidden_params", value=safe_dumps(hidden_params)
|
||||
span=span,
|
||||
key="hidden_params",
|
||||
value=safe_dumps(hidden_params),
|
||||
)
|
||||
# Cost breakdown tracking
|
||||
cost_breakdown: Optional[CostBreakdown] = standard_logging_payload.get(
|
||||
"cost_breakdown"
|
||||
cost_breakdown: Optional[CostBreakdown] = (
|
||||
standard_logging_payload.get("cost_breakdown")
|
||||
)
|
||||
if cost_breakdown:
|
||||
for key, value in cost_breakdown.items():
|
||||
|
|
@ -1504,7 +1567,9 @@ class OpenTelemetry(CustomLogger):
|
|||
# The unique identifier for the completion.
|
||||
if response_obj and response_obj.get("id"):
|
||||
self.safe_set_attribute(
|
||||
span=span, key="gen_ai.response.id", value=response_obj.get("id")
|
||||
span=span,
|
||||
key="gen_ai.response.id",
|
||||
value=response_obj.get("id"),
|
||||
)
|
||||
|
||||
# The model used to generate the response.
|
||||
|
|
@ -1639,7 +1704,9 @@ class OpenTelemetry(CustomLogger):
|
|||
"OpenTelemetry logging error in set_attributes %s", str(e)
|
||||
)
|
||||
|
||||
def _cast_as_primitive_value_type(self, value) -> Union[str, bool, int, float]:
|
||||
def _cast_as_primitive_value_type(
|
||||
self, value
|
||||
) -> Union[str, bool, int, float]:
|
||||
"""
|
||||
Casts the value to a primitive OTEL type if it is not already a primitive type.
|
||||
|
||||
|
|
@ -1673,7 +1740,10 @@ class OpenTelemetry(CustomLogger):
|
|||
if isinstance(messages, str):
|
||||
# Handle system_instructions passed as a string
|
||||
return [
|
||||
{"role": "system", "parts": [{"type": "text", "content": messages}]}
|
||||
{
|
||||
"role": "system",
|
||||
"parts": [{"type": "text", "content": messages}],
|
||||
}
|
||||
]
|
||||
|
||||
transformed = []
|
||||
|
|
@ -1714,9 +1784,11 @@ class OpenTelemetry(CustomLogger):
|
|||
message = choice.get("message") or {}
|
||||
finish_reason = choice.get("finish_reason")
|
||||
|
||||
transformed_msg = self._transform_messages_to_otel_semantic_conventions(
|
||||
[message]
|
||||
)[0]
|
||||
transformed_msg = (
|
||||
self._transform_messages_to_otel_semantic_conventions(
|
||||
[message]
|
||||
)[0]
|
||||
)
|
||||
if finish_reason:
|
||||
transformed_msg["finish_reason"] = finish_reason
|
||||
|
||||
|
|
@ -1728,7 +1800,9 @@ class OpenTelemetry(CustomLogger):
|
|||
self.set_attributes(span, kwargs, response_obj)
|
||||
kwargs.get("optional_params", {})
|
||||
litellm_params = kwargs.get("litellm_params", {}) or {}
|
||||
custom_llm_provider = litellm_params.get("custom_llm_provider", "Unknown")
|
||||
custom_llm_provider = litellm_params.get(
|
||||
"custom_llm_provider", "Unknown"
|
||||
)
|
||||
|
||||
_raw_response = kwargs.get("original_response")
|
||||
_additional_args = kwargs.get("additional_args", {}) or {}
|
||||
|
|
@ -1741,7 +1815,9 @@ class OpenTelemetry(CustomLogger):
|
|||
if complete_input_dict and isinstance(complete_input_dict, dict):
|
||||
for param, val in complete_input_dict.items():
|
||||
self.safe_set_attribute(
|
||||
span=span, key=f"llm.{custom_llm_provider}.{param}", value=val
|
||||
span=span,
|
||||
key=f"llm.{custom_llm_provider}.{param}",
|
||||
value=val,
|
||||
)
|
||||
|
||||
#############################################
|
||||
|
|
@ -1773,7 +1849,8 @@ class OpenTelemetry(CustomLogger):
|
|||
)
|
||||
except Exception as e:
|
||||
verbose_logger.exception(
|
||||
"OpenTelemetry logging error in set_raw_request_attributes %s", str(e)
|
||||
"OpenTelemetry logging error in set_raw_request_attributes %s",
|
||||
str(e),
|
||||
)
|
||||
|
||||
def _to_ns(self, dt):
|
||||
|
|
@ -1813,7 +1890,9 @@ class OpenTelemetry(CustomLogger):
|
|||
)
|
||||
|
||||
litellm_params = kwargs.get("litellm_params", {}) or {}
|
||||
proxy_server_request = litellm_params.get("proxy_server_request", {}) or {}
|
||||
proxy_server_request = (
|
||||
litellm_params.get("proxy_server_request", {}) or {}
|
||||
)
|
||||
headers = proxy_server_request.get("headers", {}) or {}
|
||||
traceparent = headers.get("traceparent", None)
|
||||
_metadata = litellm_params.get("metadata", {}) or {}
|
||||
|
|
@ -1832,7 +1911,10 @@ class OpenTelemetry(CustomLogger):
|
|||
"OpenTelemetry: Using traceparent header for context propagation"
|
||||
)
|
||||
carrier = {"traceparent": traceparent}
|
||||
return TraceContextTextMapPropagator().extract(carrier=carrier), None
|
||||
return (
|
||||
TraceContextTextMapPropagator().extract(carrier=carrier),
|
||||
None,
|
||||
)
|
||||
|
||||
# Priority 3: Active span from global context (auto-detection)
|
||||
try:
|
||||
|
|
@ -1960,10 +2042,14 @@ class OpenTelemetry(CustomLogger):
|
|||
self.OTEL_HEADERS,
|
||||
)
|
||||
|
||||
_split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS)
|
||||
_split_otel_headers = OpenTelemetry._get_headers_dictionary(
|
||||
self.OTEL_HEADERS
|
||||
)
|
||||
|
||||
# Normalize endpoint for logs - ensure it points to /v1/logs instead of /v1/traces
|
||||
normalized_endpoint = self._normalize_otel_endpoint(self.OTEL_ENDPOINT, "logs")
|
||||
normalized_endpoint = self._normalize_otel_endpoint(
|
||||
self.OTEL_ENDPOINT, "logs"
|
||||
)
|
||||
|
||||
verbose_logger.debug(
|
||||
"OpenTelemetry: Log endpoint normalized from %s to %s",
|
||||
|
|
@ -2051,14 +2137,18 @@ class OpenTelemetry(CustomLogger):
|
|||
self.OTEL_HEADERS,
|
||||
)
|
||||
|
||||
_split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS)
|
||||
_split_otel_headers = OpenTelemetry._get_headers_dictionary(
|
||||
self.OTEL_HEADERS
|
||||
)
|
||||
normalized_endpoint = self._normalize_otel_endpoint(
|
||||
self.OTEL_ENDPOINT, "metrics"
|
||||
)
|
||||
|
||||
if self.OTEL_EXPORTER == "console":
|
||||
exporter = ConsoleMetricExporter()
|
||||
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
|
||||
return PeriodicExportingMetricReader(
|
||||
exporter, export_interval_millis=5000
|
||||
)
|
||||
|
||||
elif (
|
||||
self.OTEL_EXPORTER == "otlp_http"
|
||||
|
|
@ -2074,7 +2164,9 @@ class OpenTelemetry(CustomLogger):
|
|||
headers=_split_otel_headers,
|
||||
preferred_temporality={Histogram: AggregationTemporality.DELTA},
|
||||
)
|
||||
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
|
||||
return PeriodicExportingMetricReader(
|
||||
exporter, export_interval_millis=5000
|
||||
)
|
||||
|
||||
elif self.OTEL_EXPORTER == "otlp_grpc" or self.OTEL_EXPORTER == "grpc":
|
||||
try:
|
||||
|
|
@ -2092,7 +2184,9 @@ class OpenTelemetry(CustomLogger):
|
|||
headers=_split_otel_headers,
|
||||
preferred_temporality={Histogram: AggregationTemporality.DELTA},
|
||||
)
|
||||
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
|
||||
return PeriodicExportingMetricReader(
|
||||
exporter, export_interval_millis=5000
|
||||
)
|
||||
|
||||
else:
|
||||
verbose_logger.warning(
|
||||
|
|
@ -2100,7 +2194,9 @@ class OpenTelemetry(CustomLogger):
|
|||
self.OTEL_EXPORTER,
|
||||
)
|
||||
exporter = ConsoleMetricExporter()
|
||||
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
|
||||
return PeriodicExportingMetricReader(
|
||||
exporter, export_interval_millis=5000
|
||||
)
|
||||
|
||||
def _normalize_otel_endpoint(
|
||||
self, endpoint: Optional[str], signal_type: str
|
||||
|
|
@ -2171,7 +2267,9 @@ class OpenTelemetry(CustomLogger):
|
|||
return endpoint
|
||||
|
||||
@staticmethod
|
||||
def _get_headers_dictionary(headers: Optional[Union[str, dict]]) -> Dict[str, str]:
|
||||
def _get_headers_dictionary(
|
||||
headers: Optional[Union[str, dict]],
|
||||
) -> Dict[str, str]:
|
||||
"""
|
||||
Convert a string or dictionary of headers into a dictionary of headers.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -8,40 +8,187 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True
|
|||
```
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from importlib.resources import files
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm import verbose_logger
|
||||
from litellm.constants import (
|
||||
MODEL_COST_MAP_MAX_SHRINK_RATIO,
|
||||
MODEL_COST_MAP_MIN_MODEL_COUNT,
|
||||
)
|
||||
|
||||
|
||||
class GetModelCostMap:
|
||||
"""
|
||||
Handles fetching, validating, and loading the model cost map.
|
||||
|
||||
Only the backup model *count* is cached (a single int). The full
|
||||
backup dict is never held in memory — it is only parsed when it
|
||||
needs to be *returned* as a fallback.
|
||||
"""
|
||||
|
||||
_backup_model_count: int = -1 # -1 = not yet loaded
|
||||
|
||||
@staticmethod
|
||||
def load_local_model_cost_map() -> dict:
|
||||
"""Load the local backup model cost map bundled with the package."""
|
||||
content = json.loads(
|
||||
files("litellm")
|
||||
.joinpath("model_prices_and_context_window_backup.json")
|
||||
.read_text(encoding="utf-8")
|
||||
)
|
||||
return content
|
||||
|
||||
@classmethod
|
||||
def _get_backup_model_count(cls) -> int:
|
||||
"""Return the number of models in the local backup (cached int)."""
|
||||
if cls._backup_model_count < 0:
|
||||
backup = cls.load_local_model_cost_map()
|
||||
cls._backup_model_count = len(backup)
|
||||
return cls._backup_model_count
|
||||
|
||||
@staticmethod
|
||||
def _check_is_valid_dict(fetched_map: dict) -> bool:
|
||||
"""Check 1: fetched map is a non-empty dict."""
|
||||
if not isinstance(fetched_map, dict):
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Fetched model cost map is not a dict (type=%s). "
|
||||
"Falling back to local backup.",
|
||||
type(fetched_map).__name__,
|
||||
)
|
||||
return False
|
||||
|
||||
if len(fetched_map) == 0:
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Fetched model cost map is empty. "
|
||||
"Falling back to local backup.",
|
||||
)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def _check_model_count_not_reduced(
|
||||
cls,
|
||||
fetched_map: dict,
|
||||
backup_model_count: int,
|
||||
min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT,
|
||||
max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO,
|
||||
) -> bool:
|
||||
"""Check 2: model count has not reduced significantly vs backup."""
|
||||
fetched_count = len(fetched_map)
|
||||
|
||||
if fetched_count < min_model_count:
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Fetched model cost map has only %d models (minimum=%d). "
|
||||
"This may indicate a corrupted upstream file. "
|
||||
"Falling back to local backup.",
|
||||
fetched_count,
|
||||
min_model_count,
|
||||
)
|
||||
return False
|
||||
|
||||
if backup_model_count > 0 and fetched_count < backup_model_count * max_shrink_ratio:
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Fetched model cost map shrank significantly "
|
||||
"(fetched=%d, backup=%d, threshold=%.0f%%). "
|
||||
"This may indicate a corrupted upstream file. "
|
||||
"Falling back to local backup.",
|
||||
fetched_count,
|
||||
backup_model_count,
|
||||
max_shrink_ratio * 100,
|
||||
)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def validate_model_cost_map(
|
||||
cls,
|
||||
fetched_map: dict,
|
||||
backup_model_count: int,
|
||||
min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT,
|
||||
max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO,
|
||||
) -> bool:
|
||||
"""
|
||||
Validate the integrity of a fetched model cost map.
|
||||
|
||||
Runs each check in order and returns False on the first failure.
|
||||
|
||||
Checks:
|
||||
1. ``_check_is_valid_dict`` -- fetched map is a non-empty dict.
|
||||
2. ``_check_model_count_not_reduced`` -- model count meets minimum
|
||||
and has not shrunk >``max_shrink_ratio`` vs backup.
|
||||
|
||||
Returns True if all checks pass, False otherwise.
|
||||
"""
|
||||
if not cls._check_is_valid_dict(fetched_map):
|
||||
return False
|
||||
|
||||
if not cls._check_model_count_not_reduced(
|
||||
fetched_map=fetched_map,
|
||||
backup_model_count=backup_model_count,
|
||||
min_model_count=min_model_count,
|
||||
max_shrink_ratio=max_shrink_ratio,
|
||||
):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def fetch_remote_model_cost_map(url: str, timeout: int = 5) -> dict:
|
||||
"""
|
||||
Fetch the model cost map from a remote URL.
|
||||
|
||||
Returns the parsed JSON dict. Raises on network/parse errors
|
||||
(caller is expected to handle).
|
||||
"""
|
||||
response = httpx.get(url, timeout=timeout)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
|
||||
def get_model_cost_map(url: str) -> dict:
|
||||
if (
|
||||
os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", False)
|
||||
or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", False) == "True"
|
||||
):
|
||||
from importlib.resources import files
|
||||
import json
|
||||
"""
|
||||
Public entry point — returns the model cost map dict.
|
||||
|
||||
content = json.loads(
|
||||
files("litellm")
|
||||
.joinpath("model_prices_and_context_window_backup.json")
|
||||
.read_text(encoding="utf-8")
|
||||
)
|
||||
return content
|
||||
1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, uses the local backup only.
|
||||
2. Otherwise fetches from ``url``, validates integrity, and falls back
|
||||
to the local backup on any failure.
|
||||
|
||||
Only the backup model count is cached (a single int) for validation.
|
||||
The full backup dict is only parsed when it must be *returned* as a
|
||||
fallback — it is never held in memory long-term.
|
||||
"""
|
||||
# Note: can't use get_secret_bool here — this runs during litellm.__init__
|
||||
# before litellm._key_management_settings is set.
|
||||
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
|
||||
return GetModelCostMap.load_local_model_cost_map()
|
||||
|
||||
try:
|
||||
response = httpx.get(
|
||||
url, timeout=5
|
||||
) # set a 5 second timeout for the get request
|
||||
response.raise_for_status() # Raise an exception if the request is unsuccessful
|
||||
content = response.json()
|
||||
return content
|
||||
except Exception:
|
||||
from importlib.resources import files
|
||||
import json
|
||||
|
||||
content = json.loads(
|
||||
files("litellm")
|
||||
.joinpath("model_prices_and_context_window_backup.json")
|
||||
.read_text(encoding="utf-8")
|
||||
content = GetModelCostMap.fetch_remote_model_cost_map(url)
|
||||
except Exception as e:
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Failed to fetch remote model cost map from %s: %s. "
|
||||
"Falling back to local backup.",
|
||||
url,
|
||||
str(e),
|
||||
)
|
||||
return content
|
||||
return GetModelCostMap.load_local_model_cost_map()
|
||||
|
||||
# Validate using cached count (cheap int comparison, no file I/O)
|
||||
if not GetModelCostMap.validate_model_cost_map(
|
||||
fetched_map=content,
|
||||
backup_model_count=GetModelCostMap._get_backup_model_count(),
|
||||
):
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Fetched model cost map failed integrity check. "
|
||||
"Using local backup instead. url=%s",
|
||||
url,
|
||||
)
|
||||
return GetModelCostMap.load_local_model_cost_map()
|
||||
|
||||
return content
|
||||
|
|
|
|||
|
|
@ -3764,7 +3764,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
|
|||
from litellm.integrations.opentelemetry import OpenTelemetry
|
||||
|
||||
for callback in _in_memory_loggers:
|
||||
if isinstance(callback, OpenTelemetry):
|
||||
if type(callback) is OpenTelemetry:
|
||||
return callback # type: ignore
|
||||
otel_logger = OpenTelemetry(
|
||||
**_get_custom_logger_settings_from_proxy_server(
|
||||
|
|
|
|||
|
|
@ -1272,3 +1272,59 @@ def parse_tool_call_arguments(
|
|||
)
|
||||
|
||||
raise ValueError(error_message) from e
|
||||
|
||||
|
||||
def split_concatenated_json_objects(raw: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Split a string that contains one or more concatenated JSON objects into
|
||||
a list of parsed dicts.
|
||||
|
||||
LLM providers (notably Bedrock Claude Sonnet 4.5) sometimes return
|
||||
multiple tool-call argument objects concatenated in a single
|
||||
``arguments`` string, e.g.::
|
||||
|
||||
'{"command":["curl",...]}{"command":["curl",...]}{"command":["curl",...]}'
|
||||
|
||||
``json.loads()`` fails on this with ``JSONDecodeError: Extra data``.
|
||||
This helper uses ``json.JSONDecoder.raw_decode()`` to walk the string
|
||||
and extract each JSON object individually.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[dict]
|
||||
A list of parsed dicts – one per JSON object found. If *raw* is
|
||||
empty or whitespace-only, an empty list is returned.
|
||||
|
||||
Raises
|
||||
------
|
||||
json.JSONDecodeError
|
||||
If the string contains text that cannot be parsed as JSON at all.
|
||||
"""
|
||||
import json
|
||||
|
||||
raw = raw.strip()
|
||||
if not raw:
|
||||
return []
|
||||
|
||||
decoder = json.JSONDecoder()
|
||||
results: List[Dict[str, Any]] = []
|
||||
idx = 0
|
||||
length = len(raw)
|
||||
|
||||
while idx < length:
|
||||
# Skip whitespace between objects
|
||||
while idx < length and raw[idx] in " \t\n\r":
|
||||
idx += 1
|
||||
if idx >= length:
|
||||
break
|
||||
|
||||
obj, end_idx = decoder.raw_decode(raw, idx)
|
||||
if isinstance(obj, dict):
|
||||
results.append(obj)
|
||||
else:
|
||||
# Non-dict JSON value – wrap in empty dict (Bedrock requires
|
||||
# toolUse.input to be an object).
|
||||
results.append({})
|
||||
idx = end_idx
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -3287,25 +3287,68 @@ def _convert_to_bedrock_tool_call_invoke(
|
|||
- extract name
|
||||
- extract id
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
split_concatenated_json_objects,
|
||||
)
|
||||
|
||||
try:
|
||||
_parts_list: List[BedrockContentBlock] = []
|
||||
for tool in tool_calls:
|
||||
if "function" in tool:
|
||||
id = tool["id"]
|
||||
tool_id = tool["id"]
|
||||
name = tool["function"].get("name", "")
|
||||
arguments = tool["function"].get("arguments", "")
|
||||
arguments_dict = json.loads(arguments) if arguments else {}
|
||||
# Ensure arguments_dict is always a dict (Bedrock requires toolUse.input to be an object)
|
||||
# When some providers return arguments: '""' (JSON-encoded empty string), json.loads returns ""
|
||||
if not isinstance(arguments_dict, dict):
|
||||
arguments_dict = {}
|
||||
|
||||
if not arguments or not arguments.strip():
|
||||
arguments_dict = {}
|
||||
else:
|
||||
arguments_dict = json.loads(arguments)
|
||||
try:
|
||||
arguments_dict = json.loads(arguments)
|
||||
# Ensure arguments_dict is always a dict
|
||||
# (Bedrock requires toolUse.input to be an object).
|
||||
# Some providers return arguments: '""' which
|
||||
# json.loads decodes to a bare string.
|
||||
if not isinstance(arguments_dict, dict):
|
||||
arguments_dict = {}
|
||||
except json.JSONDecodeError:
|
||||
# The model may return multiple JSON objects
|
||||
# concatenated in a single arguments string, e.g.
|
||||
# '{"cmd":"a"}{"cmd":"b"}{"cmd":"c"}'
|
||||
# Split them and emit one toolUse block per object.
|
||||
# Fixes: https://github.com/BerriAI/litellm/issues/20543
|
||||
parsed_objects = split_concatenated_json_objects(
|
||||
arguments
|
||||
)
|
||||
if parsed_objects:
|
||||
# First object keeps the original tool id.
|
||||
for obj_idx, obj in enumerate(parsed_objects):
|
||||
block_id = (
|
||||
tool_id
|
||||
if obj_idx == 0
|
||||
else f"{tool_id}_{obj_idx}"
|
||||
)
|
||||
bedrock_tool = BedrockToolUseBlock(
|
||||
input=obj, name=name, toolUseId=block_id
|
||||
)
|
||||
_parts_list.append(
|
||||
BedrockContentBlock(toolUse=bedrock_tool)
|
||||
)
|
||||
# cache_control applies to the whole original
|
||||
# tool call; attach after the last split block.
|
||||
if tool.get("cache_control", None) is not None:
|
||||
_parts_list.append(
|
||||
BedrockContentBlock(
|
||||
cachePoint=CachePointBlock(
|
||||
type="default"
|
||||
)
|
||||
)
|
||||
)
|
||||
continue
|
||||
# Fallback: no objects extracted — use empty dict.
|
||||
arguments_dict = {}
|
||||
|
||||
bedrock_tool = BedrockToolUseBlock(
|
||||
input=arguments_dict, name=name, toolUseId=id
|
||||
input=arguments_dict, name=name, toolUseId=tool_id
|
||||
)
|
||||
bedrock_content_block = BedrockContentBlock(toolUse=bedrock_tool)
|
||||
_parts_list.append(bedrock_content_block)
|
||||
|
|
|
|||
|
|
@ -140,9 +140,14 @@ def should_redact_message_logging(model_call_details: dict) -> bool:
|
|||
|
||||
metadata_field = get_metadata_variable_name_from_kwargs(litellm_params)
|
||||
metadata = litellm_params.get(metadata_field, {})
|
||||
|
||||
if not isinstance(metadata, dict):
|
||||
# Fall back: litellm_metadata was None, try metadata
|
||||
metadata = litellm_params.get("metadata", {})
|
||||
if not isinstance(metadata, dict):
|
||||
metadata = {}
|
||||
|
||||
# Get headers from the metadata
|
||||
request_headers = metadata.get("headers", {}) if isinstance(metadata, dict) else {}
|
||||
request_headers = metadata.get("headers", {})
|
||||
|
||||
# Check for headers that explicitly control redaction
|
||||
if request_headers and bool(
|
||||
|
|
|
|||
|
|
@ -58,6 +58,9 @@ from litellm.types.utils import (
|
|||
|
||||
from ...base import BaseLLM
|
||||
from ..common_utils import AnthropicError, process_anthropic_headers
|
||||
from litellm.anthropic_beta_headers_manager import (
|
||||
update_headers_with_filtered_beta,
|
||||
)
|
||||
from .transformation import AnthropicConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -333,6 +336,10 @@ class AnthropicChatCompletion(BaseLLM):
|
|||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
headers = update_headers_with_filtered_beta(
|
||||
headers=headers, provider=custom_llm_provider
|
||||
)
|
||||
|
||||
config = ProviderConfigManager.get_provider_chat_config(
|
||||
model=model,
|
||||
provider=LlmProviders(custom_llm_provider),
|
||||
|
|
|
|||
|
|
@ -834,6 +834,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
"sonnet-4-5",
|
||||
"opus-4.1",
|
||||
"opus-4-1",
|
||||
"opus-4.5",
|
||||
"opus-4-5",
|
||||
"opus-4.6",
|
||||
"opus-4-6",
|
||||
}
|
||||
):
|
||||
_output_format = (
|
||||
|
|
@ -931,6 +935,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
Translate system message to anthropic format.
|
||||
|
||||
Removes system message from the original list and returns a new list of anthropic system message content.
|
||||
Filters out system messages containing x-anthropic-billing-header metadata.
|
||||
"""
|
||||
system_prompt_indices = []
|
||||
anthropic_system_message_list: List[AnthropicSystemMessageContent] = []
|
||||
|
|
@ -942,6 +947,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
# Skip empty text blocks - Anthropic API raises errors for empty text
|
||||
if not system_message_block["content"]:
|
||||
continue
|
||||
# Skip system messages containing x-anthropic-billing-header metadata
|
||||
if system_message_block["content"].startswith("x-anthropic-billing-header:"):
|
||||
continue
|
||||
anthropic_system_message_content = AnthropicSystemMessageContent(
|
||||
type="text",
|
||||
text=system_message_block["content"],
|
||||
|
|
@ -960,6 +968,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
text_value = _content.get("text")
|
||||
if _content.get("type") == "text" and not text_value:
|
||||
continue
|
||||
# Skip system messages containing x-anthropic-billing-header metadata
|
||||
if _content.get("type") == "text" and text_value and text_value.startswith("x-anthropic-billing-header:"):
|
||||
continue
|
||||
anthropic_system_message_content = (
|
||||
AnthropicSystemMessageContent(
|
||||
type=_content.get("type"),
|
||||
|
|
|
|||
|
|
@ -30,6 +30,58 @@ ANTHROPIC_ADAPTER = AnthropicAdapter()
|
|||
|
||||
|
||||
class LiteLLMMessagesToCompletionTransformationHandler:
|
||||
@staticmethod
|
||||
def _route_openai_thinking_to_responses_api_if_needed(
|
||||
completion_kwargs: Dict[str, Any],
|
||||
*,
|
||||
thinking: Optional[Dict[str, Any]],
|
||||
) -> None:
|
||||
"""
|
||||
When users call `litellm.anthropic.messages.*` with a non-Anthropic model and
|
||||
`thinking={"type": "enabled", ...}`, LiteLLM converts this into OpenAI
|
||||
`reasoning_effort`.
|
||||
|
||||
For OpenAI models, Chat Completions typically does not return reasoning text
|
||||
(only token accounting). To return a thinking-like content block in the
|
||||
Anthropic response format, we route the request through OpenAI's Responses API
|
||||
and request a reasoning summary.
|
||||
"""
|
||||
custom_llm_provider = completion_kwargs.get("custom_llm_provider")
|
||||
if custom_llm_provider is None:
|
||||
try:
|
||||
_, inferred_provider, _, _ = litellm.utils.get_llm_provider(
|
||||
model=cast(str, completion_kwargs.get("model"))
|
||||
)
|
||||
custom_llm_provider = inferred_provider
|
||||
except Exception:
|
||||
custom_llm_provider = None
|
||||
|
||||
if custom_llm_provider != "openai":
|
||||
return
|
||||
|
||||
if not isinstance(thinking, dict) or thinking.get("type") != "enabled":
|
||||
return
|
||||
|
||||
model = completion_kwargs.get("model")
|
||||
if isinstance(model, str) and model and not model.startswith("responses/"):
|
||||
# Prefix model with "responses/" to route to OpenAI Responses API
|
||||
completion_kwargs["model"] = f"responses/{model}"
|
||||
|
||||
reasoning_effort = completion_kwargs.get("reasoning_effort")
|
||||
if isinstance(reasoning_effort, str) and reasoning_effort:
|
||||
completion_kwargs["reasoning_effort"] = {
|
||||
"effort": reasoning_effort,
|
||||
"summary": "detailed",
|
||||
}
|
||||
elif isinstance(reasoning_effort, dict):
|
||||
if (
|
||||
"summary" not in reasoning_effort
|
||||
and "generate_summary" not in reasoning_effort
|
||||
):
|
||||
updated_reasoning_effort = dict(reasoning_effort)
|
||||
updated_reasoning_effort["summary"] = "detailed"
|
||||
completion_kwargs["reasoning_effort"] = updated_reasoning_effort
|
||||
|
||||
@staticmethod
|
||||
def _prepare_completion_kwargs(
|
||||
*,
|
||||
|
|
@ -123,6 +175,11 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
):
|
||||
completion_kwargs[key] = value
|
||||
|
||||
LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed(
|
||||
completion_kwargs,
|
||||
thinking=thinking,
|
||||
)
|
||||
|
||||
return completion_kwargs, tool_name_mapping
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -2,9 +2,6 @@ from typing import Any, AsyncIterator, Dict, List, Optional, Tuple
|
|||
|
||||
import httpx
|
||||
|
||||
from litellm.anthropic_beta_headers_manager import (
|
||||
update_headers_with_filtered_beta,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.litellm_logging import verbose_logger
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import (
|
||||
|
|
@ -52,6 +49,40 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
# TODO: Add Anthropic `metadata` support
|
||||
# "metadata",
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _filter_billing_headers_from_system(system_param):
|
||||
"""
|
||||
Filter out x-anthropic-billing-header metadata from system parameter.
|
||||
|
||||
Args:
|
||||
system_param: Can be a string or a list of system message content blocks
|
||||
|
||||
Returns:
|
||||
Filtered system parameter (string or list), or None if all content was filtered
|
||||
"""
|
||||
if isinstance(system_param, str):
|
||||
# If it's a string and starts with billing header, filter it out
|
||||
if system_param.startswith("x-anthropic-billing-header:"):
|
||||
return None
|
||||
return system_param
|
||||
elif isinstance(system_param, list):
|
||||
# Filter list of system content blocks
|
||||
filtered_list = []
|
||||
for content_block in system_param:
|
||||
if isinstance(content_block, dict):
|
||||
text = content_block.get("text", "")
|
||||
content_type = content_block.get("type", "")
|
||||
# Skip text blocks that start with billing header
|
||||
if content_type == "text" and text.startswith("x-anthropic-billing-header:"):
|
||||
continue
|
||||
filtered_list.append(content_block)
|
||||
else:
|
||||
# Keep non-dict items as-is
|
||||
filtered_list.append(content_block)
|
||||
return filtered_list if len(filtered_list) > 0 else None
|
||||
else:
|
||||
return system_param
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
|
|
@ -96,11 +127,6 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
optional_params=optional_params,
|
||||
)
|
||||
|
||||
headers = update_headers_with_filtered_beta(
|
||||
headers=headers,
|
||||
provider="anthropic",
|
||||
)
|
||||
|
||||
return headers, api_base
|
||||
|
||||
def transform_anthropic_messages_request(
|
||||
|
|
@ -123,6 +149,17 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
message="max_tokens is required for Anthropic /v1/messages API",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
# Filter out x-anthropic-billing-header from system messages
|
||||
system_param = anthropic_messages_optional_request_params.get("system")
|
||||
if system_param is not None:
|
||||
filtered_system = self._filter_billing_headers_from_system(system_param)
|
||||
if filtered_system is not None and len(filtered_system) > 0:
|
||||
anthropic_messages_optional_request_params["system"] = filtered_system
|
||||
else:
|
||||
# Remove system parameter if all content was filtered out
|
||||
anthropic_messages_optional_request_params.pop("system", None)
|
||||
|
||||
####### get required params for all anthropic messages requests ######
|
||||
verbose_logger.debug(f"TRANSFORMATION DEBUG - Messages: {messages}")
|
||||
anthropic_messages_request: AnthropicMessagesRequest = AnthropicMessagesRequest(
|
||||
|
|
|
|||
|
|
@ -105,6 +105,7 @@ class AzureOpenAIConfig(BaseConfig):
|
|||
"modalities",
|
||||
"audio",
|
||||
"web_search_options",
|
||||
"prompt_cache_key",
|
||||
]
|
||||
|
||||
def _is_response_format_supported_model(self, model: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -3,9 +3,6 @@ Azure Anthropic messages transformation config - extends AnthropicMessagesConfig
|
|||
"""
|
||||
from typing import TYPE_CHECKING, Any, List, Optional, Tuple
|
||||
|
||||
from litellm.anthropic_beta_headers_manager import (
|
||||
update_headers_with_filtered_beta,
|
||||
)
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
|
|
@ -65,18 +62,11 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
|
|||
if "content-type" not in headers:
|
||||
headers["content-type"] = "application/json"
|
||||
|
||||
# Update headers with anthropic beta features (context management, tool search, etc.)
|
||||
headers = self._update_headers_with_anthropic_beta(
|
||||
headers=headers,
|
||||
optional_params=optional_params,
|
||||
)
|
||||
|
||||
# Filter out unsupported beta headers for Azure AI
|
||||
headers = update_headers_with_filtered_beta(
|
||||
headers=headers,
|
||||
provider="azure_ai",
|
||||
)
|
||||
|
||||
return headers, api_base
|
||||
|
||||
def get_complete_url(
|
||||
|
|
|
|||
|
|
@ -2,10 +2,6 @@
|
|||
Azure Anthropic transformation config - extends AnthropicConfig with Azure authentication
|
||||
"""
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional, Union
|
||||
|
||||
from litellm.anthropic_beta_headers_manager import (
|
||||
update_headers_with_filtered_beta,
|
||||
)
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
|
@ -90,11 +86,6 @@ class AzureAnthropicConfig(AnthropicConfig):
|
|||
if "anthropic-version" not in headers:
|
||||
headers["anthropic-version"] = "2023-06-01"
|
||||
|
||||
# Filter out unsupported beta headers for Azure AI
|
||||
headers = update_headers_with_filtered_beta(
|
||||
headers=headers,
|
||||
provider="azure_ai",
|
||||
)
|
||||
|
||||
return headers
|
||||
|
||||
|
|
|
|||
|
|
@ -11,12 +11,14 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
_audio_or_image_in_message_content,
|
||||
convert_content_list_to_str,
|
||||
)
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM
|
||||
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
|
||||
from litellm.llms.openai.common_utils import drop_params_from_unprocessable_entity_error
|
||||
from litellm.llms.openai.openai import OpenAIConfig
|
||||
from litellm.llms.xai.chat.transformation import XAIChatConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import ModelResponse, ProviderField
|
||||
from litellm.utils import _add_path_to_api_base, supports_tool_choice
|
||||
|
||||
|
|
@ -64,12 +66,21 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
api_key: Optional[str] = None,
|
||||
api_base: Optional[str] = None,
|
||||
) -> dict:
|
||||
if api_base and self._should_use_api_key_header(api_base):
|
||||
headers["api-key"] = api_key
|
||||
if api_key:
|
||||
if api_base and self._should_use_api_key_header(api_base):
|
||||
headers["api-key"] = api_key
|
||||
else:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
else:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
# No api_key provided — fall back to Azure AD token-based auth
|
||||
litellm_params_obj = GenericLiteLLMParams(
|
||||
**(litellm_params if isinstance(litellm_params, dict) else {})
|
||||
)
|
||||
headers = BaseAzureLLM._base_validate_azure_environment(
|
||||
headers=headers, litellm_params=litellm_params_obj
|
||||
)
|
||||
|
||||
headers["Content-Type"] = "application/json" # tell Azure AI Studio to expect JSON
|
||||
headers["Content-Type"] = "application/json"
|
||||
|
||||
return headers
|
||||
|
||||
|
|
|
|||
|
|
@ -211,25 +211,13 @@ class BaseAWSLLM:
|
|||
aws_external_id=aws_external_id,
|
||||
)
|
||||
elif aws_role_name is not None:
|
||||
# Check if we're in IRSA and trying to assume the same role we already have
|
||||
current_role_arn = os.getenv("AWS_ROLE_ARN")
|
||||
web_identity_token_file = os.getenv("AWS_WEB_IDENTITY_TOKEN_FILE")
|
||||
|
||||
# In IRSA environments, we should skip role assumption if we're already running as the target role
|
||||
# This is true when:
|
||||
# 1. We have AWS_ROLE_ARN set (current role)
|
||||
# 2. We have AWS_WEB_IDENTITY_TOKEN_FILE set (IRSA environment)
|
||||
# 3. The current role matches the requested role
|
||||
if (
|
||||
current_role_arn
|
||||
and web_identity_token_file
|
||||
and current_role_arn == aws_role_name
|
||||
):
|
||||
# Check if we're already running as the target role and can skip assumption
|
||||
# This handles IRSA (EKS), ECS task roles, and EC2 instance profiles
|
||||
if self._is_already_running_as_role(aws_role_name, ssl_verify=ssl_verify):
|
||||
verbose_logger.debug(
|
||||
"Using IRSA same-role optimization: calling _auth_with_env_vars"
|
||||
"Already running as target role %s, using ambient credentials",
|
||||
aws_role_name,
|
||||
)
|
||||
# We're already running as this role via IRSA, no need to assume it again
|
||||
# Use the default boto3 credentials (which will use the IRSA credentials)
|
||||
credentials, _cache_ttl = self._auth_with_env_vars()
|
||||
else:
|
||||
verbose_logger.debug(
|
||||
|
|
@ -553,6 +541,107 @@ class BaseAWSLLM:
|
|||
aws_region_name = "us-west-2"
|
||||
return aws_region_name
|
||||
|
||||
@staticmethod
|
||||
def _parse_arn_account_and_role_name(
|
||||
arn: str,
|
||||
) -> Optional[Tuple[str, str, str]]:
|
||||
"""
|
||||
Parse an ARN and return (partition, account_id, role_name).
|
||||
|
||||
Handles:
|
||||
- arn:aws:iam::123456789012:role/MyRole
|
||||
- arn:aws:iam::123456789012:role/path/to/MyRole
|
||||
- arn:aws:sts::123456789012:assumed-role/MyRole/session-name
|
||||
|
||||
Returns None if the ARN cannot be parsed.
|
||||
"""
|
||||
# ARN format: arn:PARTITION:SERVICE:REGION:ACCOUNT:RESOURCE
|
||||
parts = arn.split(":")
|
||||
if len(parts) < 6 or parts[0] != "arn":
|
||||
return None
|
||||
|
||||
partition = parts[1] # e.g. "aws", "aws-cn", "aws-us-gov"
|
||||
account_id = parts[4]
|
||||
resource = ":".join(parts[5:]) # rejoin in case resource contains colons
|
||||
|
||||
if resource.startswith("role/"):
|
||||
# arn:aws:iam::ACCOUNT:role/[path/]ROLE_NAME
|
||||
role_name = resource.split("/")[-1]
|
||||
elif resource.startswith("assumed-role/"):
|
||||
# arn:aws:sts::ACCOUNT:assumed-role/ROLE_NAME/SESSION
|
||||
role_parts = resource.split("/")
|
||||
if len(role_parts) >= 2:
|
||||
role_name = role_parts[1]
|
||||
else:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
|
||||
return partition, account_id, role_name
|
||||
|
||||
def _is_already_running_as_role(
|
||||
self,
|
||||
aws_role_name: str,
|
||||
ssl_verify: Optional[Union[bool, str]] = None,
|
||||
) -> bool:
|
||||
"""
|
||||
Check if the current environment is already running as the target IAM role.
|
||||
|
||||
This handles multiple AWS environments:
|
||||
- IRSA (EKS): AWS_ROLE_ARN + AWS_WEB_IDENTITY_TOKEN_FILE are set
|
||||
- ECS task roles: Uses sts:GetCallerIdentity to check current role ARN
|
||||
- EC2 instance profiles: Uses sts:GetCallerIdentity to check current role ARN
|
||||
|
||||
Compares partition, account ID, and role name to avoid cross-account
|
||||
false matches.
|
||||
|
||||
Returns True if the current identity matches the target role, meaning
|
||||
we can skip sts:AssumeRole and use ambient credentials directly.
|
||||
"""
|
||||
target_parsed = self._parse_arn_account_and_role_name(aws_role_name)
|
||||
if target_parsed is None:
|
||||
return False
|
||||
|
||||
target_partition, target_account, target_role = target_parsed
|
||||
|
||||
# Fast path: IRSA environment check (no API call needed)
|
||||
current_role_arn = os.getenv("AWS_ROLE_ARN")
|
||||
web_identity_token_file = os.getenv("AWS_WEB_IDENTITY_TOKEN_FILE")
|
||||
if current_role_arn and web_identity_token_file:
|
||||
return current_role_arn == aws_role_name
|
||||
|
||||
# For ECS/EC2: call sts:GetCallerIdentity to check if already running as the role
|
||||
try:
|
||||
import boto3
|
||||
|
||||
with tracer.trace("boto3.client(sts).get_caller_identity"):
|
||||
sts_client = boto3.client(
|
||||
"sts", verify=self._get_ssl_verify(ssl_verify)
|
||||
)
|
||||
identity = sts_client.get_caller_identity()
|
||||
caller_arn = identity.get("Arn", "")
|
||||
|
||||
caller_parsed = self._parse_arn_account_and_role_name(caller_arn)
|
||||
if caller_parsed is not None:
|
||||
caller_partition, caller_account, caller_role = caller_parsed
|
||||
if (
|
||||
caller_partition == target_partition
|
||||
and caller_account == target_account
|
||||
and caller_role == target_role
|
||||
):
|
||||
verbose_logger.debug(
|
||||
"Current identity already matches target role: %s",
|
||||
aws_role_name,
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
verbose_logger.debug(
|
||||
"Could not determine current role identity: %s", str(e)
|
||||
)
|
||||
|
||||
return False
|
||||
|
||||
@tracer.wrap()
|
||||
def _auth_with_web_identity_token(
|
||||
self,
|
||||
|
|
@ -867,7 +956,35 @@ class BaseAWSLLM:
|
|||
if aws_external_id is not None:
|
||||
assume_role_params["ExternalId"] = aws_external_id
|
||||
|
||||
sts_response = sts_client.assume_role(**assume_role_params)
|
||||
try:
|
||||
sts_response = sts_client.assume_role(**assume_role_params)
|
||||
except Exception as e:
|
||||
error_str = str(e)
|
||||
if "AccessDenied" in error_str:
|
||||
# Only fall back to ambient credentials if we can positively
|
||||
# confirm the caller is already the target role (same account,
|
||||
# partition, and role name). This avoids silently using the
|
||||
# wrong identity when there is a genuine trust-policy or
|
||||
# permission misconfiguration.
|
||||
if self._is_already_running_as_role(
|
||||
aws_role_name, ssl_verify=ssl_verify
|
||||
):
|
||||
verbose_logger.warning(
|
||||
"AssumeRole failed for %s (%s). "
|
||||
"Caller is already running as this role; "
|
||||
"falling back to ambient credentials.",
|
||||
aws_role_name,
|
||||
error_str,
|
||||
)
|
||||
return self._auth_with_env_vars()
|
||||
# Genuine permission error — re-raise
|
||||
verbose_logger.error(
|
||||
"AssumeRole AccessDenied for %s and caller is NOT "
|
||||
"the same role. Re-raising. Error: %s",
|
||||
aws_role_name,
|
||||
error_str,
|
||||
)
|
||||
raise
|
||||
|
||||
# Extract the credentials from the response and convert to Session Credentials
|
||||
sts_credentials = sts_response["Credentials"]
|
||||
|
|
|
|||
|
|
@ -13,7 +13,9 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
)
|
||||
from litellm.types.utils import ModelResponse
|
||||
from litellm.utils import CustomStreamWrapper
|
||||
|
||||
from litellm.anthropic_beta_headers_manager import (
|
||||
update_headers_with_filtered_beta,
|
||||
)
|
||||
from ..base_aws_llm import BaseAWSLLM, Credentials
|
||||
from ..common_utils import BedrockError
|
||||
from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call
|
||||
|
|
@ -337,7 +339,11 @@ class BedrockConverseLLM(BaseAWSLLM):
|
|||
headers = {"Content-Type": "application/json"}
|
||||
if extra_headers is not None:
|
||||
headers = {"Content-Type": "application/json", **extra_headers}
|
||||
|
||||
|
||||
# Filter beta headers in HTTP headers before making the request
|
||||
headers = update_headers_with_filtered_beta(
|
||||
headers=headers, provider="bedrock_converse"
|
||||
)
|
||||
### ROUTING (ASYNC, STREAMING, SYNC)
|
||||
if acompletion:
|
||||
if isinstance(client, HTTPHandler):
|
||||
|
|
|
|||
|
|
@ -11,9 +11,6 @@ import httpx
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.anthropic_beta_headers_manager import (
|
||||
filter_and_transform_beta_headers,
|
||||
)
|
||||
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
filter_exceptions_from_params,
|
||||
|
|
@ -1132,24 +1129,9 @@ class AmazonConverseConfig(BaseConfig):
|
|||
|
||||
# Set anthropic_beta in additional_request_params if we have any beta features
|
||||
# ONLY apply to Anthropic/Claude models - other models (e.g., Qwen, Llama) don't support this field
|
||||
# and will error with "unknown variant anthropic_beta" if included
|
||||
base_model = BedrockModelInfo.get_base_model(model)
|
||||
if anthropic_beta_list and base_model.startswith("anthropic"):
|
||||
# Remove duplicates while preserving order
|
||||
unique_betas = []
|
||||
seen = set()
|
||||
for beta in anthropic_beta_list:
|
||||
if beta not in seen:
|
||||
unique_betas.append(beta)
|
||||
seen.add(beta)
|
||||
|
||||
filtered_betas = filter_and_transform_beta_headers(
|
||||
beta_headers=unique_betas,
|
||||
provider="bedrock_converse",
|
||||
)
|
||||
|
||||
if filtered_betas:
|
||||
additional_request_params["anthropic_beta"] = filtered_betas
|
||||
additional_request_params["anthropic_beta"] = anthropic_beta_list
|
||||
|
||||
return bedrock_tools, anthropic_beta_list
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ from typing import TYPE_CHECKING, Any, List, Optional
|
|||
|
||||
import httpx
|
||||
|
||||
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
||||
AmazonInvokeConfig,
|
||||
|
|
@ -136,13 +135,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
# Filter out beta headers that Bedrock Invoke doesn't support
|
||||
# Uses centralized configuration from anthropic_beta_headers_config.json
|
||||
beta_list = list(beta_set)
|
||||
filtered_beta_list = filter_and_transform_beta_headers(
|
||||
beta_headers=beta_list,
|
||||
provider="bedrock",
|
||||
)
|
||||
|
||||
if filtered_beta_list:
|
||||
_anthropic_request["anthropic_beta"] = filtered_beta_list
|
||||
_anthropic_request["anthropic_beta"] = beta_list
|
||||
|
||||
return _anthropic_request
|
||||
|
||||
|
|
|
|||
|
|
@ -12,9 +12,6 @@ from typing import (
|
|||
|
||||
import httpx
|
||||
|
||||
from litellm.anthropic_beta_headers_manager import (
|
||||
filter_and_transform_beta_headers,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
|
|
@ -253,71 +250,6 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
|
||||
return any(pattern in model_lower for pattern in supported_patterns)
|
||||
|
||||
def _filter_unsupported_beta_headers_for_bedrock(
|
||||
self, model: str, beta_set: set
|
||||
) -> None:
|
||||
"""
|
||||
Remove beta headers that are not supported on Bedrock for the given model.
|
||||
|
||||
Extended thinking beta headers are only supported on specific Claude 4+ models.
|
||||
Advanced tool use headers are not supported on Bedrock Invoke API, but need to be
|
||||
translated to Bedrock-specific headers for models that support tool search
|
||||
(Claude Opus 4.5, Sonnet 4.5).
|
||||
This prevents 400 "invalid beta flag" errors on Bedrock.
|
||||
|
||||
Note: Bedrock Invoke API fails with a 400 error when unsupported beta headers
|
||||
are sent, returning: {"message":"invalid beta flag"}
|
||||
|
||||
Translation for models supporting tool search (Opus 4.5, Sonnet 4.5):
|
||||
- advanced-tool-use-2025-11-20 -> tool-search-tool-2025-10-19 + tool-examples-2025-10-29
|
||||
|
||||
Args:
|
||||
model: The model name
|
||||
beta_set: The set of beta headers to filter in-place
|
||||
"""
|
||||
# 1. Handle header transformations BEFORE filtering
|
||||
# (advanced-tool-use -> tool-search-tool)
|
||||
# This must happen before filtering because advanced-tool-use is in the unsupported list
|
||||
has_advanced_tool_use = "advanced-tool-use-2025-11-20" in beta_set
|
||||
if has_advanced_tool_use and self._supports_tool_search_on_bedrock(model):
|
||||
beta_set.discard("advanced-tool-use-2025-11-20")
|
||||
beta_set.add("tool-search-tool-2025-10-19")
|
||||
beta_set.add("tool-examples-2025-10-29")
|
||||
|
||||
# 2. Apply provider-level filtering using centralized JSON config
|
||||
beta_list = list(beta_set)
|
||||
filtered_list = filter_and_transform_beta_headers(
|
||||
beta_headers=beta_list,
|
||||
provider="bedrock",
|
||||
)
|
||||
|
||||
# Update the set with filtered headers
|
||||
beta_set.clear()
|
||||
beta_set.update(filtered_list)
|
||||
|
||||
# 2.1. Handle model-specific exceptions: structured-outputs is only supported on Opus 4.6
|
||||
# Re-add structured-outputs if it was in the original set and model is Opus 4.6
|
||||
model_lower = model.lower()
|
||||
is_opus_4_6 = any(pattern in model_lower for pattern in ["opus-4.6", "opus_4.6", "opus-4-6", "opus_4_6"])
|
||||
if is_opus_4_6 and "structured-outputs-2025-11-13" in beta_list:
|
||||
beta_set.add("structured-outputs-2025-11-13")
|
||||
|
||||
# 3. Filter out extended thinking headers for models that don't support them
|
||||
extended_thinking_patterns = [
|
||||
"extended-thinking",
|
||||
"interleaved-thinking",
|
||||
]
|
||||
if not self._supports_extended_thinking_on_bedrock(model):
|
||||
beta_headers_to_remove = set()
|
||||
for beta in beta_set:
|
||||
for pattern in extended_thinking_patterns:
|
||||
if pattern in beta.lower():
|
||||
beta_headers_to_remove.add(beta)
|
||||
break
|
||||
|
||||
for beta in beta_headers_to_remove:
|
||||
beta_set.discard(beta)
|
||||
|
||||
def _get_tool_search_beta_header_for_bedrock(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -483,12 +415,11 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
beta_set=beta_set,
|
||||
)
|
||||
|
||||
# Filter out unsupported beta headers for Bedrock (e.g., advanced-tool-use, extended-thinking on non-Opus/Sonnet 4 models)
|
||||
self._filter_unsupported_beta_headers_for_bedrock(
|
||||
model=model,
|
||||
beta_set=beta_set,
|
||||
)
|
||||
|
||||
# --- Custom logic: if tool-search-tool-2025-10-19 is present, add tool-examples-2025-10-29 ---
|
||||
if "tool-search-tool-2025-10-19" in beta_set:
|
||||
beta_set.add("tool-examples-2025-10-29")
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
if beta_set:
|
||||
anthropic_messages_request["anthropic_beta"] = list(beta_set)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import asyncio
|
||||
import contextlib
|
||||
import os
|
||||
import ssl
|
||||
import typing
|
||||
import urllib.request
|
||||
from typing import Callable, Dict, Optional, Union
|
||||
|
|
@ -139,8 +140,13 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
|||
Credit to: https://github.com/karpetrosyan/httpx-aiohttp for this implementation
|
||||
"""
|
||||
|
||||
def __init__(self, client: Union[ClientSession, Callable[[], ClientSession]]):
|
||||
def __init__(
|
||||
self,
|
||||
client: Union[ClientSession, Callable[[], ClientSession]],
|
||||
ssl_verify: Optional[Union[bool, ssl.SSLContext]] = None,
|
||||
):
|
||||
self.client = client
|
||||
self._ssl_verify = ssl_verify # Store for per-request SSL override
|
||||
super().__init__(client=client)
|
||||
# Store the client factory for recreating sessions when needed
|
||||
if callable(client):
|
||||
|
|
@ -214,6 +220,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
|||
timeout: dict,
|
||||
proxy: Optional[str],
|
||||
sni_hostname: Optional[str],
|
||||
ssl_verify: Optional[Union[bool, ssl.SSLContext]] = None,
|
||||
) -> ClientResponse:
|
||||
"""
|
||||
Helper function to make an aiohttp request with the given parameters.
|
||||
|
|
@ -224,6 +231,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
|||
timeout: Timeout settings dict with 'connect', 'read', 'pool' keys
|
||||
proxy: Optional proxy URL
|
||||
sni_hostname: Optional SNI hostname for SSL
|
||||
ssl_verify: Optional SSL verification setting (False to disable, SSLContext for custom)
|
||||
|
||||
Returns:
|
||||
ClientResponse from aiohttp
|
||||
|
|
@ -237,6 +245,13 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
|||
data = request.stream # type: ignore
|
||||
request.headers.pop("transfer-encoding", None) # handled by aiohttp
|
||||
|
||||
# Only pass ssl kwarg when explicitly configured, to avoid
|
||||
# overriding the session/connector defaults with None (which is
|
||||
# not a valid value for aiohttp's ssl parameter).
|
||||
ssl_kwargs: Dict[str, Union[bool, ssl.SSLContext]] = {}
|
||||
if ssl_verify is not None:
|
||||
ssl_kwargs["ssl"] = ssl_verify
|
||||
|
||||
response = await client_session.request(
|
||||
method=request.method,
|
||||
url=YarlURL(str(request.url), encoded=True),
|
||||
|
|
@ -251,6 +266,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
|||
),
|
||||
proxy=proxy,
|
||||
server_hostname=sni_hostname,
|
||||
**ssl_kwargs,
|
||||
).__aenter__()
|
||||
|
||||
return response
|
||||
|
|
@ -268,6 +284,9 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
|||
# Resolve proxy settings from environment variables
|
||||
proxy = await self._get_proxy_settings(request)
|
||||
|
||||
# Use stored SSL configuration for per-request override
|
||||
ssl_config = self._ssl_verify
|
||||
|
||||
try:
|
||||
with map_aiohttp_exceptions():
|
||||
response = await self._make_aiohttp_request(
|
||||
|
|
@ -276,6 +295,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
|||
timeout=timeout,
|
||||
proxy=proxy,
|
||||
sni_hostname=sni_hostname,
|
||||
ssl_verify=ssl_config,
|
||||
)
|
||||
except RuntimeError as e:
|
||||
# Handle the case where session was closed between our check and actual use
|
||||
|
|
@ -296,6 +316,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
|||
timeout=timeout,
|
||||
proxy=proxy,
|
||||
sni_hostname=sni_hostname,
|
||||
ssl_verify=ssl_config,
|
||||
)
|
||||
else:
|
||||
# Re-raise if it's a different RuntimeError
|
||||
|
|
|
|||
|
|
@ -846,6 +846,16 @@ class AsyncHTTPHandler:
|
|||
if str_to_bool(os.getenv("AIOHTTP_TRUST_ENV", "False")) is True:
|
||||
trust_env = True
|
||||
|
||||
#########################################################
|
||||
# Determine SSL config to pass to transport for per-request override
|
||||
# This ensures ssl_verify works even with shared sessions
|
||||
#########################################################
|
||||
ssl_for_transport: Optional[Union[bool, ssl.SSLContext]] = None
|
||||
if ssl_context is not None:
|
||||
ssl_for_transport = ssl_context
|
||||
elif ssl_verify is False:
|
||||
ssl_for_transport = False
|
||||
|
||||
verbose_logger.debug("Creating AiohttpTransport...")
|
||||
|
||||
# Use shared session if provided and valid
|
||||
|
|
@ -853,7 +863,10 @@ class AsyncHTTPHandler:
|
|||
verbose_logger.debug(
|
||||
f"SHARED SESSION: Reusing existing ClientSession (ID: {id(shared_session)})"
|
||||
)
|
||||
return LiteLLMAiohttpTransport(client=shared_session)
|
||||
return LiteLLMAiohttpTransport(
|
||||
client=shared_session,
|
||||
ssl_verify=ssl_for_transport,
|
||||
)
|
||||
|
||||
# Create new session only if none provided or existing one is invalid
|
||||
verbose_logger.debug(
|
||||
|
|
@ -877,6 +890,7 @@ class AsyncHTTPHandler:
|
|||
connector=TCPConnector(**transport_connector_kwargs),
|
||||
trust_env=trust_env,
|
||||
),
|
||||
ssl_verify=ssl_for_transport,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -81,6 +81,9 @@ from litellm.types.llms.anthropic_skills import (
|
|||
ListSkillsResponse,
|
||||
Skill,
|
||||
)
|
||||
from litellm.anthropic_beta_headers_manager import (
|
||||
update_headers_with_filtered_beta,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
CreateBatchRequest,
|
||||
CreateFileRequest,
|
||||
|
|
@ -1858,6 +1861,10 @@ class BaseLLMHTTPHandler:
|
|||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
)
|
||||
|
||||
headers = update_headers_with_filtered_beta(
|
||||
headers=headers, provider=custom_llm_provider
|
||||
)
|
||||
|
||||
logging_obj.update_environment_variables(
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -60,6 +60,38 @@ from ...anthropic.chat.transformation import AnthropicConfig
|
|||
from ...openai_like.chat.transformation import OpenAILikeChatConfig
|
||||
from ..common_utils import DatabricksBase, DatabricksException
|
||||
|
||||
def _sanitize_empty_content(message_dict: dict[str, Any]) -> None:
|
||||
"""
|
||||
Remove or filter content so empty text blocks are not sent.
|
||||
Databricks Model Serving uses Anthropic Messages API spec and rejects empty text blocks.
|
||||
"""
|
||||
content = message_dict.get("content")
|
||||
if content is None:
|
||||
message_dict.pop("content", None)
|
||||
return
|
||||
if isinstance(content, str):
|
||||
if not content.strip():
|
||||
message_dict.pop("content")
|
||||
return
|
||||
if isinstance(content, list):
|
||||
if not content:
|
||||
message_dict.pop("content")
|
||||
return
|
||||
filtered = [
|
||||
block
|
||||
for block in content
|
||||
if not (
|
||||
isinstance(block, dict)
|
||||
and block.get("type") == "text"
|
||||
and not (block.get("text") or "").strip()
|
||||
)
|
||||
]
|
||||
if not filtered:
|
||||
message_dict.pop("content")
|
||||
else:
|
||||
message_dict["content"] = filtered
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
||||
|
|
@ -350,6 +382,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
|
|||
# Move message-level cache_control into a content block when content is a string.
|
||||
if "cache_control" in _message and isinstance(_message.get("content"), str):
|
||||
_message = self._move_cache_control_into_string_content_block(_message)
|
||||
_sanitize_empty_content(cast(dict[str, Any], _message))
|
||||
new_messages.append(_message)
|
||||
|
||||
if is_async:
|
||||
|
|
|
|||
|
|
@ -838,6 +838,15 @@ class OCIChatConfig(BaseConfig):
|
|||
if not user_messages:
|
||||
raise Exception("No user message found for Cohere model")
|
||||
|
||||
# Extract system messages into preambleOverride
|
||||
system_messages = [msg for msg in messages if msg.get("role") == "system"]
|
||||
preamble_override = None
|
||||
if system_messages:
|
||||
preamble = "\n".join(
|
||||
self._extract_text_content(msg["content"]) for msg in system_messages
|
||||
)
|
||||
if preamble:
|
||||
preamble_override = preamble
|
||||
|
||||
# Create Cohere-specific chat request
|
||||
optional_cohere_params = self._get_optional_params(OCIVendors.COHERE, optional_params)
|
||||
|
|
@ -845,6 +854,7 @@ class OCIChatConfig(BaseConfig):
|
|||
apiFormat="COHERE",
|
||||
message=self._extract_text_content(user_messages[-1]["content"]),
|
||||
chatHistory=self.adapt_messages_to_cohere_standard(messages),
|
||||
preambleOverride=preamble_override,
|
||||
**optional_cohere_params
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -502,13 +502,12 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
|
|||
reasoning_content: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
if chunk["message"].get("thinking") is not None:
|
||||
if self.started_reasoning_content is False:
|
||||
reasoning_content = chunk["message"].get("thinking")
|
||||
self.started_reasoning_content = True
|
||||
elif self.finished_reasoning_content is False:
|
||||
reasoning_content = chunk["message"].get("thinking")
|
||||
self.finished_reasoning_content = True
|
||||
reasoning_content = chunk["message"].get("thinking")
|
||||
self.started_reasoning_content = True
|
||||
elif chunk["message"].get("content") is not None:
|
||||
if self.started_reasoning_content and not self.finished_reasoning_content:
|
||||
self.finished_reasoning_content = True
|
||||
|
||||
message_content = chunk["message"].get("content")
|
||||
if "<think>" in message_content:
|
||||
message_content = message_content.replace("<think>", "")
|
||||
|
|
|
|||
|
|
@ -20,12 +20,12 @@ from typing import (
|
|||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
_extract_reasoning_content,
|
||||
_handle_invalid_parallel_tool_calls,
|
||||
_should_convert_tool_call_to_json_mode,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import get_tool_call_names
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
||||
async_convert_url_to_base64,
|
||||
|
|
@ -161,6 +161,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
"web_search_options",
|
||||
"service_tier",
|
||||
"safety_identifier",
|
||||
"prompt_cache_key",
|
||||
] # works across all models
|
||||
|
||||
model_specific_params = []
|
||||
|
|
|
|||
7
litellm/llms/perplexity/responses/__init__.py
Normal file
7
litellm/llms/perplexity/responses/__init__.py
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
"""
|
||||
Perplexity Agentic Research API (Responses API) module
|
||||
"""
|
||||
|
||||
from .transformation import PerplexityResponsesConfig
|
||||
|
||||
__all__ = ["PerplexityResponsesConfig"]
|
||||
409
litellm/llms/perplexity/responses/transformation.py
Normal file
409
litellm/llms/perplexity/responses/transformation.py
Normal file
|
|
@ -0,0 +1,409 @@
|
|||
"""
|
||||
Transformation logic for Perplexity Agentic Research API (Responses API)
|
||||
|
||||
This module handles the translation between OpenAI's Responses API format
|
||||
and Perplexity's Responses API format, which supports:
|
||||
- Third-party model access (OpenAI, Anthropic, Google, xAI, etc.)
|
||||
- Presets for optimized configurations
|
||||
- Web search and URL fetching tools
|
||||
- Reasoning effort control
|
||||
- Instructions parameter for system-level guidance
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import (
|
||||
ResponseAPIUsage,
|
||||
ResponseInputParam,
|
||||
ResponsesAPIOptionalRequestParams,
|
||||
ResponsesAPIResponse,
|
||||
ResponsesAPIStreamingResponse,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
||||
class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
|
||||
"""
|
||||
Configuration for Perplexity Agentic Research API (Responses API)
|
||||
|
||||
|
||||
Reference: https://docs.perplexity.ai/agentic-research/quickstart
|
||||
"""
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> LlmProviders:
|
||||
return LlmProviders.PERPLEXITY
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
"""
|
||||
Perplexity Responses API supports a different set of parameters
|
||||
|
||||
Ref: https://docs.perplexity.ai/api-reference/responses-post
|
||||
"""
|
||||
return [
|
||||
"max_output_tokens",
|
||||
"stream",
|
||||
"temperature",
|
||||
"top_p",
|
||||
"tools",
|
||||
"reasoning",
|
||||
"preset",
|
||||
"instructions",
|
||||
"models", # Model fallback support
|
||||
]
|
||||
|
||||
def validate_environment(
|
||||
self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams]
|
||||
) -> dict:
|
||||
"""Validate environment and set up headers"""
|
||||
# Get API key from environment
|
||||
api_key = (
|
||||
get_secret_str("PERPLEXITYAI_API_KEY")
|
||||
or get_secret_str("PERPLEXITY_API_KEY")
|
||||
)
|
||||
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
|
||||
headers["Content-Type"] = "application/json"
|
||||
|
||||
return headers
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
litellm_params: dict,
|
||||
) -> str:
|
||||
"""Get the complete URL for the Perplexity Responses API"""
|
||||
if api_base is None:
|
||||
api_base = get_secret_str("PERPLEXITY_API_BASE") or "https://api.perplexity.ai"
|
||||
|
||||
# Ensure api_base doesn't end with a slash
|
||||
api_base = api_base.rstrip("/")
|
||||
|
||||
# Add the responses endpoint
|
||||
return f"{api_base}/v1/responses"
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
response_api_optional_params: ResponsesAPIOptionalRequestParams,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> Dict:
|
||||
"""
|
||||
Map OpenAI Responses API parameters to Perplexity format
|
||||
|
||||
Key differences:
|
||||
- Supports 'preset' parameter for predefined configurations
|
||||
- Supports 'instructions' parameter for system-level guidance
|
||||
- Tools are specified differently (web_search, fetch_url)
|
||||
"""
|
||||
mapped_params: Dict[str, Any] = {}
|
||||
|
||||
# Map standard parameters
|
||||
if response_api_optional_params.get("max_output_tokens"):
|
||||
mapped_params["max_output_tokens"] = response_api_optional_params["max_output_tokens"]
|
||||
|
||||
if response_api_optional_params.get("temperature"):
|
||||
mapped_params["temperature"] = response_api_optional_params["temperature"]
|
||||
|
||||
if response_api_optional_params.get("top_p"):
|
||||
mapped_params["top_p"] = response_api_optional_params["top_p"]
|
||||
|
||||
if response_api_optional_params.get("stream"):
|
||||
mapped_params["stream"] = response_api_optional_params["stream"]
|
||||
|
||||
if response_api_optional_params.get("stream_options"):
|
||||
mapped_params["stream_options"] = response_api_optional_params["stream_options"]
|
||||
|
||||
# Map Perplexity-specific parameters (using .get() with Any dict access)
|
||||
preset = response_api_optional_params.get("preset") # type: ignore
|
||||
if preset:
|
||||
mapped_params["preset"] = preset
|
||||
|
||||
instructions = response_api_optional_params.get("instructions") # type: ignore
|
||||
if instructions:
|
||||
mapped_params["instructions"] = instructions
|
||||
|
||||
if response_api_optional_params.get("reasoning"):
|
||||
mapped_params["reasoning"] = response_api_optional_params["reasoning"]
|
||||
|
||||
tools = response_api_optional_params.get("tools")
|
||||
if tools:
|
||||
# Convert tools to list of dicts for transformation
|
||||
tools_list = [dict(tool) if hasattr(tool, '__dict__') else tool for tool in tools] # type: ignore
|
||||
mapped_params["tools"] = self._transform_tools(tools_list) # type: ignore
|
||||
|
||||
return mapped_params
|
||||
|
||||
def _transform_tools(self, tools: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Transform tools to Perplexity format
|
||||
|
||||
Perplexity supports:
|
||||
- web_search: Performs web searches
|
||||
- fetch_url: Fetches content from URLs
|
||||
"""
|
||||
perplexity_tools = []
|
||||
|
||||
for tool in tools:
|
||||
if isinstance(tool, dict):
|
||||
tool_type = tool.get("type")
|
||||
|
||||
# Direct Perplexity tool format
|
||||
if tool_type in ["web_search", "fetch_url"]:
|
||||
perplexity_tools.append(tool)
|
||||
|
||||
# OpenAI function format - try to map to Perplexity tools
|
||||
elif tool_type == "function":
|
||||
function = tool.get("function", {})
|
||||
function_name = function.get("name", "")
|
||||
|
||||
if function_name == "web_search" or "search" in function_name.lower():
|
||||
perplexity_tools.append({"type": "web_search"})
|
||||
elif function_name == "fetch_url" or "fetch" in function_name.lower():
|
||||
perplexity_tools.append({"type": "fetch_url"})
|
||||
|
||||
return perplexity_tools
|
||||
|
||||
def transform_responses_api_request(
|
||||
self,
|
||||
model: str,
|
||||
input: Union[str, ResponseInputParam],
|
||||
response_api_optional_request_params: Dict,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Dict:
|
||||
"""
|
||||
Transform request to Perplexity Responses API format
|
||||
"""
|
||||
# Check if the model is a preset (format: preset/preset-name)
|
||||
if model.startswith("preset/"):
|
||||
preset_name = model.replace("preset/", "")
|
||||
data = {
|
||||
"preset": preset_name,
|
||||
"input": self._format_input(input),
|
||||
}
|
||||
# Check if preset is explicitly provided in params
|
||||
elif response_api_optional_request_params.get("preset"):
|
||||
data = {
|
||||
"preset": response_api_optional_request_params.pop("preset"),
|
||||
"input": self._format_input(input),
|
||||
}
|
||||
else:
|
||||
# Full request format for third-party models
|
||||
data = {
|
||||
"model": model,
|
||||
"input": self._format_input(input),
|
||||
}
|
||||
|
||||
# Add all optional parameters
|
||||
for key, value in response_api_optional_request_params.items():
|
||||
data[key] = value
|
||||
|
||||
return data
|
||||
|
||||
def _format_input(self, input: Union[str, ResponseInputParam]) -> Union[str, List[Dict[str, Any]]]:
|
||||
"""
|
||||
Format input for Perplexity Responses API
|
||||
|
||||
The API accepts either:
|
||||
- A simple string for single-turn queries
|
||||
- An array of message objects for multi-turn conversations
|
||||
"""
|
||||
if isinstance(input, str):
|
||||
return input
|
||||
|
||||
# Handle ResponseInputParam format
|
||||
if isinstance(input, list):
|
||||
formatted_messages = []
|
||||
for item in input:
|
||||
if isinstance(item, dict):
|
||||
formatted_message = {
|
||||
"type": "message",
|
||||
"role": item.get("role"),
|
||||
"content": item.get("content", ""),
|
||||
}
|
||||
formatted_messages.append(formatted_message)
|
||||
return formatted_messages
|
||||
|
||||
return str(input)
|
||||
|
||||
def transform_response_api_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> ResponsesAPIResponse:
|
||||
"""
|
||||
Transform Perplexity Responses API response to OpenAI Responses API format
|
||||
"""
|
||||
try:
|
||||
raw_response_json = raw_response.json()
|
||||
except Exception as e:
|
||||
raise BaseLLMException(
|
||||
status_code=raw_response.status_code,
|
||||
message=f"Failed to parse response: {str(e)}",
|
||||
)
|
||||
|
||||
# Check for error status
|
||||
status = raw_response_json.get("status")
|
||||
if status == "failed":
|
||||
error = raw_response_json.get("error", {})
|
||||
error_message = error.get("message", "Unknown error")
|
||||
raise BaseLLMException(
|
||||
status_code=raw_response.status_code,
|
||||
message=error_message,
|
||||
)
|
||||
|
||||
# Transform usage to handle Perplexity's cost structure
|
||||
usage_data = raw_response_json.get("usage", {})
|
||||
transformed_usage_dict = self._transform_usage(usage_data)
|
||||
|
||||
# Convert usage dict to ResponseAPIUsage object
|
||||
usage_obj = ResponseAPIUsage(**transformed_usage_dict) if transformed_usage_dict else None
|
||||
|
||||
# Map Perplexity response to OpenAI Responses API format
|
||||
response = ResponsesAPIResponse(
|
||||
id=raw_response_json.get("id", ""),
|
||||
object="response",
|
||||
created_at=raw_response_json.get("created_at", 0),
|
||||
status=raw_response_json.get("status", "completed"),
|
||||
model=raw_response_json.get("model", model),
|
||||
output=raw_response_json.get("output", []),
|
||||
usage=usage_obj,
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
def _transform_usage(self, usage_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Transform Perplexity usage data to OpenAI format
|
||||
|
||||
Perplexity returns:
|
||||
{
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 200,
|
||||
"total_tokens": 300,
|
||||
"cost": {
|
||||
"currency": "USD",
|
||||
"input_cost": 0.0001,
|
||||
"output_cost": 0.0002,
|
||||
"total_cost": 0.0003
|
||||
}
|
||||
}
|
||||
|
||||
OpenAI expects:
|
||||
{
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 200,
|
||||
"total_tokens": 300,
|
||||
"cost": 0.0003
|
||||
}
|
||||
"""
|
||||
transformed = {
|
||||
"input_tokens": usage_data.get("input_tokens", 0),
|
||||
"output_tokens": usage_data.get("output_tokens", 0),
|
||||
"total_tokens": usage_data.get("total_tokens", 0),
|
||||
}
|
||||
|
||||
# Transform cost from Perplexity format (dict) to OpenAI format (float)
|
||||
cost_obj = usage_data.get("cost")
|
||||
if isinstance(cost_obj, dict) and "total_cost" in cost_obj:
|
||||
transformed["cost"] = cost_obj["total_cost"]
|
||||
verbose_logger.debug(
|
||||
"Transformed Perplexity cost object to float: %s -> %s",
|
||||
cost_obj,
|
||||
cost_obj["total_cost"]
|
||||
)
|
||||
elif cost_obj is not None:
|
||||
# If cost is already a float/number, use it as-is
|
||||
transformed["cost"] = cost_obj
|
||||
|
||||
# Add input_tokens_details if present
|
||||
if "input_tokens_details" in usage_data:
|
||||
transformed["input_tokens_details"] = usage_data["input_tokens_details"]
|
||||
|
||||
# Add output_tokens_details if present
|
||||
if "output_tokens_details" in usage_data:
|
||||
transformed["output_tokens_details"] = usage_data["output_tokens_details"]
|
||||
|
||||
return transformed
|
||||
|
||||
def transform_streaming_response(
|
||||
self,
|
||||
model: str,
|
||||
parsed_chunk: dict,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> ResponsesAPIStreamingResponse:
|
||||
"""
|
||||
Transform a parsed streaming response chunk into a ResponsesAPIStreamingResponse
|
||||
"""
|
||||
# Get the event type from the chunk
|
||||
verbose_logger.debug("Raw Perplexity Chunk=%s", parsed_chunk)
|
||||
event_type = str(parsed_chunk.get("type"))
|
||||
event_pydantic_model = PerplexityResponsesConfig.get_event_model_class(
|
||||
event_type=event_type
|
||||
)
|
||||
|
||||
# Transform Perplexity-specific fields to OpenAI format
|
||||
parsed_chunk = self._transform_perplexity_chunk(parsed_chunk)
|
||||
|
||||
# Defensive: Handle error.code being null (similar to OpenAI implementation)
|
||||
try:
|
||||
error_obj = parsed_chunk.get("error")
|
||||
if isinstance(error_obj, dict) and error_obj.get("code") is None:
|
||||
# Preserve other fields, but ensure `code` is a non-null string
|
||||
parsed_chunk = dict(parsed_chunk)
|
||||
parsed_chunk["error"] = dict(error_obj)
|
||||
parsed_chunk["error"]["code"] = "unknown_error"
|
||||
except Exception:
|
||||
# If anything unexpected happens here, fall back to attempting
|
||||
# instantiation and let higher-level handlers manage errors.
|
||||
verbose_logger.debug("Failed to coalesce error.code in parsed_chunk")
|
||||
|
||||
return event_pydantic_model(**parsed_chunk)
|
||||
|
||||
def _transform_perplexity_chunk(self, chunk: dict) -> dict:
|
||||
"""
|
||||
Transform Perplexity-specific fields in a streaming chunk to OpenAI format.
|
||||
|
||||
This handles:
|
||||
- Converting Perplexity's cost object to a simple float
|
||||
"""
|
||||
# Make a copy to avoid modifying the original
|
||||
chunk = dict(chunk)
|
||||
|
||||
# Transform usage.cost from Perplexity format to OpenAI format
|
||||
# Perplexity: {"currency": "USD", "input_cost": 0.0001, "output_cost": 0.0002, "total_cost": 0.0003}
|
||||
# OpenAI: 0.0003 (just the total_cost as a float)
|
||||
try:
|
||||
response_obj = chunk.get("response")
|
||||
if isinstance(response_obj, dict):
|
||||
usage_obj = response_obj.get("usage")
|
||||
if isinstance(usage_obj, dict):
|
||||
cost_obj = usage_obj.get("cost")
|
||||
if isinstance(cost_obj, dict) and "total_cost" in cost_obj:
|
||||
# Replace the cost object with just the total_cost value
|
||||
chunk = dict(chunk)
|
||||
chunk["response"] = dict(response_obj)
|
||||
chunk["response"]["usage"] = dict(usage_obj)
|
||||
chunk["response"]["usage"]["cost"] = cost_obj["total_cost"]
|
||||
verbose_logger.debug(
|
||||
"Transformed Perplexity cost object to float: %s -> %s",
|
||||
cost_obj,
|
||||
cost_obj["total_cost"]
|
||||
)
|
||||
except Exception as e:
|
||||
# If transformation fails, log and continue with original chunk
|
||||
verbose_logger.debug("Failed to transform Perplexity cost object: %s", e)
|
||||
|
||||
return chunk
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
import re
|
||||
from copy import deepcopy
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union, get_type_hints
|
||||
|
||||
|
|
@ -684,7 +685,7 @@ def convert_anyof_null_to_nullable(schema, depth=0):
|
|||
if anyof is not None:
|
||||
contains_null = False
|
||||
for atype in anyof:
|
||||
if atype == {"type": "null"}:
|
||||
if isinstance(atype, dict) and atype.get("type") == "null":
|
||||
# remove null type
|
||||
anyof.remove(atype)
|
||||
contains_null = True
|
||||
|
|
@ -801,8 +802,38 @@ def _convert_schema_types(schema, depth=0):
|
|||
if "type" in schema:
|
||||
type_val = schema["type"]
|
||||
if isinstance(type_val, list) and len(type_val) > 1:
|
||||
# Convert ["string", "number"] -> {"anyOf": [{"type": "STRING"}, {"type": "NUMBER"}]}
|
||||
schema["anyOf"] = [{"type": t} for t in type_val if isinstance(t, str)]
|
||||
# Convert type arrays to anyOf format
|
||||
# Fields that are specific to object/array types and should move into anyOf
|
||||
type_specific_fields = {"properties", "required", "additionalProperties", "items", "minItems", "maxItems", "minProperties", "maxProperties"}
|
||||
|
||||
any_of: List[Dict[str, Any]] = []
|
||||
for t in type_val:
|
||||
if not isinstance(t, str):
|
||||
continue
|
||||
if t == "null":
|
||||
# Keep null entry minimal so we can strip it later.
|
||||
any_of.append({"type": "null"})
|
||||
continue
|
||||
|
||||
# For object/array types, include type-specific fields
|
||||
if t in ("object", "array"):
|
||||
item_schema = {"type": t}
|
||||
# Move type-specific fields into this anyOf item
|
||||
for field in type_specific_fields:
|
||||
if field in schema:
|
||||
item_schema[field] = deepcopy(schema[field])
|
||||
any_of.append(item_schema)
|
||||
else:
|
||||
# For primitive types, only include the type
|
||||
any_of.append({"type": t})
|
||||
|
||||
# Remove type-specific fields from parent if we moved them into anyOf
|
||||
has_object_or_array = any(t in ("object", "array") for t in type_val if isinstance(t, str))
|
||||
if has_object_or_array:
|
||||
for field in type_specific_fields:
|
||||
schema.pop(field, None)
|
||||
|
||||
schema["anyOf"] = any_of
|
||||
schema.pop("type")
|
||||
elif isinstance(type_val, list) and len(type_val) == 1:
|
||||
schema["type"] = type_val[0]
|
||||
|
|
|
|||
|
|
@ -437,6 +437,27 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
|
|||
else:
|
||||
assistant_content.append(PartType(text=assistant_text)) # type: ignore
|
||||
|
||||
## HANDLE ASSISTANT IMAGES FIELD
|
||||
# Process images field if present (for generated images from assistant)
|
||||
assistant_images = assistant_msg.get("images")
|
||||
if assistant_images is not None and isinstance(assistant_images, list):
|
||||
for image_item in assistant_images:
|
||||
if isinstance(image_item, dict):
|
||||
image_url_obj = image_item.get("image_url")
|
||||
if isinstance(image_url_obj, dict):
|
||||
assistant_image_url = image_url_obj.get("url")
|
||||
format = image_url_obj.get("format")
|
||||
detail = image_url_obj.get("detail")
|
||||
media_resolution_enum = _convert_detail_to_media_resolution_enum(detail)
|
||||
if assistant_image_url:
|
||||
_part = _process_gemini_media(
|
||||
image_url=assistant_image_url,
|
||||
format=format,
|
||||
media_resolution_enum=media_resolution_enum,
|
||||
model=model,
|
||||
)
|
||||
assistant_content.append(_part)
|
||||
|
||||
## HANDLE ASSISTANT FUNCTION CALL
|
||||
if (
|
||||
assistant_msg.get("tool_calls", []) is not None
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from litellm.anthropic_beta_headers_manager import (
|
||||
update_headers_with_filtered_beta,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
|
|
@ -105,12 +102,6 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
|
|||
if beta_values:
|
||||
headers["anthropic-beta"] = ",".join(beta_values)
|
||||
|
||||
# Filter out unsupported beta headers for Vertex AI
|
||||
headers = update_headers_with_filtered_beta(
|
||||
headers=headers,
|
||||
provider="vertex_ai",
|
||||
)
|
||||
|
||||
return headers, api_base
|
||||
|
||||
def get_complete_url(
|
||||
|
|
|
|||
|
|
@ -107,6 +107,11 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
|
|||
vertex_project = self.get_vertex_ai_project(litellm_params)
|
||||
vertex_location = self.get_vertex_ai_location(litellm_params)
|
||||
|
||||
# Map empty location/cluade models to a supported region for count-tokens endpoint
|
||||
# https://docs.cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/count-tokens
|
||||
if not vertex_location or "claude" in model.lower():
|
||||
vertex_location = "us-central1"
|
||||
|
||||
# Get access token and resolved project ID
|
||||
access_token, project_id = await self._ensure_access_token_async(
|
||||
credentials=vertex_credentials,
|
||||
|
|
@ -118,7 +123,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
|
|||
endpoint_url = self._build_count_tokens_endpoint(
|
||||
model=model,
|
||||
project_id=project_id,
|
||||
vertex_location=vertex_location or "us-central1",
|
||||
vertex_location=vertex_location,
|
||||
api_base=litellm_params.get("api_base"),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -5848,6 +5848,19 @@
|
|||
"output_cost_per_token": 7e-07,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/kimi-k2.5": {
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"max_tokens": 262144,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-06,
|
||||
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/kimi-k2-5-now-in-microsoft-foundry/4492321",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"azure_ai/ministral-3b": {
|
||||
"input_cost_per_token": 4e-08,
|
||||
"litellm_provider": "azure_ai",
|
||||
|
|
@ -6091,6 +6104,39 @@
|
|||
"output_cost_per_token": 2.4e-05,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": {
|
||||
"input_cost_per_token": 7.3e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"max_tokens": 262144,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.03e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"bedrock/moonshotai.kimi-k2-thinking": {
|
||||
"input_cost_per_token": 7.3e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"max_tokens": 262144,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.03e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"bedrock/moonshotai.kimi-k2.5": {
|
||||
"input_cost_per_token": 7.3e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"max_tokens": 262144,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.03e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": {
|
||||
"input_cost_per_token": 3.18e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
|
|
@ -6109,6 +6155,17 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 7.2e-07
|
||||
},
|
||||
"bedrock/ap-south-1/moonshotai.kimi-k2-thinking": {
|
||||
"input_cost_per_token": 7.1e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"max_tokens": 262144,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.94e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": {
|
||||
"input_cost_per_token": 3.05e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
|
|
@ -6314,6 +6371,17 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 1.01e-06
|
||||
},
|
||||
"bedrock/sa-east-1/moonshotai.kimi-k2-thinking": {
|
||||
"input_cost_per_token": 7.3e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"max_tokens": 262144,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.03e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": {
|
||||
"input_cost_per_second": 0.011,
|
||||
"litellm_provider": "bedrock",
|
||||
|
|
@ -6450,6 +6518,28 @@
|
|||
"output_cost_per_token": 7e-07,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock/us-east-1/moonshotai.kimi-k2-thinking": {
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"max_tokens": 262144,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"bedrock/us-east-2/moonshotai.kimi-k2-thinking": {
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"max_tokens": 262144,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"bedrock/us-gov-east-1/amazon.nova-pro-v1:0": {
|
||||
"input_cost_per_token": 9.6e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
|
|
@ -6856,6 +6946,17 @@
|
|||
"output_cost_per_token": 7e-07,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock/us-west-2/moonshotai.kimi-k2-thinking": {
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"max_tokens": 262144,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": {
|
||||
"cache_creation_input_token_cost": 1e-06,
|
||||
"cache_read_input_token_cost": 8e-08,
|
||||
|
|
@ -8945,6 +9046,43 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"dashscope/qwen3-max": {
|
||||
"litellm_provider": "dashscope",
|
||||
"max_input_tokens": 258048,
|
||||
"max_output_tokens": 65536,
|
||||
"max_tokens": 65536,
|
||||
"mode": "chat",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"tiered_pricing": [
|
||||
{
|
||||
"input_cost_per_token": 1.2e-06,
|
||||
"output_cost_per_token": 6e-06,
|
||||
"range": [
|
||||
0,
|
||||
32000.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"input_cost_per_token": 2.4e-06,
|
||||
"output_cost_per_token": 1.2e-05,
|
||||
"range": [
|
||||
32000.0,
|
||||
128000.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"input_cost_per_token": 3e-06,
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"range": [
|
||||
128000.0,
|
||||
252000.0
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"dashscope/qwq-plus": {
|
||||
"input_cost_per_token": 8e-07,
|
||||
"litellm_provider": "dashscope",
|
||||
|
|
@ -10618,14 +10756,22 @@
|
|||
"input_cost_per_token": 2.8e-07,
|
||||
"input_cost_per_token_cache_hit": 2.8e-08,
|
||||
"litellm_provider": "deepseek",
|
||||
"max_input_tokens": 128000,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.2e-07,
|
||||
"source": "https://api-docs.deepseek.com/quick_start/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions"
|
||||
],
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_native_streaming": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"deepseek/deepseek-coder": {
|
||||
|
|
@ -10662,16 +10808,24 @@
|
|||
"input_cost_per_token": 2.8e-07,
|
||||
"input_cost_per_token_cache_hit": 2.8e-08,
|
||||
"litellm_provider": "deepseek",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 65536,
|
||||
"max_tokens": 65536,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.2e-07,
|
||||
"source": "https://api-docs.deepseek.com/quick_start/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions"
|
||||
],
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_function_calling": false,
|
||||
"supports_native_streaming": true,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": false
|
||||
},
|
||||
"deepseek/deepseek-v3": {
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
|
|
@ -25614,6 +25768,66 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"perplexity/preset/pro-search": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_web_search": true,
|
||||
"supports_preset": true
|
||||
},
|
||||
"perplexity/openai/gpt-4o": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_web_search": true,
|
||||
"supports_reasoning": false
|
||||
},
|
||||
"perplexity/openai/gpt-4o-mini": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_web_search": true,
|
||||
"supports_reasoning": false
|
||||
},
|
||||
"perplexity/openai/gpt-5.2": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_web_search": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"perplexity/anthropic/claude-3-5-sonnet-20241022": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_web_search": true,
|
||||
"supports_reasoning": false
|
||||
},
|
||||
"perplexity/anthropic/claude-3-5-haiku-20241022": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_web_search": true,
|
||||
"supports_reasoning": false
|
||||
},
|
||||
"perplexity/google/gemini-2.0-flash-exp": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_web_search": true,
|
||||
"supports_reasoning": false
|
||||
},
|
||||
"perplexity/google/gemini-2.0-flash-thinking-exp": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_web_search": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"perplexity/xai/grok-2-1212": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_web_search": true,
|
||||
"supports_reasoning": false
|
||||
},
|
||||
"perplexity/xai/grok-2-vision-1212": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_web_search": true,
|
||||
"supports_reasoning": false
|
||||
},
|
||||
"publicai/aisingapore/Qwen-SEA-LION-v4-32B-IT": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "publicai",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import json
|
||||
from typing import Optional
|
||||
from urllib.parse import urlencode, urlparse, urlunparse
|
||||
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
|
||||
|
||||
from fastapi import APIRouter, Form, HTTPException, Request
|
||||
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
|
||||
|
|
@ -16,6 +16,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
|
|||
)
|
||||
from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
|
||||
from litellm.proxy.utils import get_server_root_path
|
||||
from litellm.types.mcp import MCPAuth
|
||||
from litellm.types.mcp_server.mcp_server_manager import MCPServer
|
||||
|
||||
router = APIRouter(
|
||||
|
|
@ -125,6 +126,29 @@ def decode_state_hash(encrypted_state: str) -> dict:
|
|||
return state_data
|
||||
|
||||
|
||||
def _resolve_oauth2_server_for_root_endpoints(
|
||||
client_ip: Optional[str] = None,
|
||||
) -> Optional[MCPServer]:
|
||||
"""
|
||||
Resolve the MCP server for root-level OAuth endpoints (no server name in path).
|
||||
|
||||
When the MCP SDK hits root-level endpoints like /register, /authorize, /token
|
||||
without a server name prefix, we try to find the right server automatically.
|
||||
Returns the server if exactly one OAuth2 server is configured, else None.
|
||||
"""
|
||||
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
|
||||
global_mcp_server_manager,
|
||||
)
|
||||
|
||||
registry = global_mcp_server_manager.get_filtered_registry(client_ip=client_ip)
|
||||
oauth2_servers = [
|
||||
s for s in registry.values() if s.auth_type == MCPAuth.oauth2
|
||||
]
|
||||
if len(oauth2_servers) == 1:
|
||||
return oauth2_servers[0]
|
||||
return None
|
||||
|
||||
|
||||
async def authorize_with_server(
|
||||
request: Request,
|
||||
mcp_server: MCPServer,
|
||||
|
|
@ -170,7 +194,13 @@ async def authorize_with_server(
|
|||
if code_challenge_method:
|
||||
params["code_challenge_method"] = code_challenge_method
|
||||
|
||||
return RedirectResponse(f"{mcp_server.authorization_url}?{urlencode(params)}")
|
||||
parsed_auth_url = urlparse(mcp_server.authorization_url)
|
||||
existing_params = dict(parse_qsl(parsed_auth_url.query))
|
||||
existing_params.update(params)
|
||||
final_url = urlunparse(
|
||||
parsed_auth_url._replace(query=urlencode(existing_params))
|
||||
)
|
||||
return RedirectResponse(final_url)
|
||||
|
||||
|
||||
async def exchange_token_with_server(
|
||||
|
|
@ -305,6 +335,8 @@ async def authorize(
|
|||
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(
|
||||
lookup_name, client_ip=client_ip
|
||||
)
|
||||
if mcp_server is None and mcp_server_name is None:
|
||||
mcp_server = _resolve_oauth2_server_for_root_endpoints()
|
||||
if mcp_server is None:
|
||||
raise HTTPException(status_code=404, detail="MCP server not found")
|
||||
return await authorize_with_server(
|
||||
|
|
@ -350,6 +382,8 @@ async def token_endpoint(
|
|||
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(
|
||||
lookup_name, client_ip=client_ip
|
||||
)
|
||||
if mcp_server is None and mcp_server_name is None:
|
||||
mcp_server = _resolve_oauth2_server_for_root_endpoints()
|
||||
if mcp_server is None:
|
||||
raise HTTPException(status_code=404, detail="MCP server not found")
|
||||
return await exchange_token_with_server(
|
||||
|
|
@ -430,6 +464,13 @@ def _build_oauth_protected_resource_response(
|
|||
)
|
||||
|
||||
request_base_url = get_request_base_url(request)
|
||||
|
||||
# When no server name provided, try to resolve the single OAuth2 server
|
||||
if mcp_server_name is None:
|
||||
resolved = _resolve_oauth2_server_for_root_endpoints()
|
||||
if resolved:
|
||||
mcp_server_name = resolved.server_name or resolved.name
|
||||
|
||||
mcp_server: Optional[MCPServer] = None
|
||||
if mcp_server_name:
|
||||
client_ip = IPAddressUtils.get_mcp_client_ip(request)
|
||||
|
|
@ -535,6 +576,12 @@ def _build_oauth_authorization_server_response(
|
|||
|
||||
request_base_url = get_request_base_url(request)
|
||||
|
||||
# When no server name provided, try to resolve the single OAuth2 server
|
||||
if mcp_server_name is None:
|
||||
resolved = _resolve_oauth2_server_for_root_endpoints()
|
||||
if resolved:
|
||||
mcp_server_name = resolved.server_name or resolved.name
|
||||
|
||||
authorization_endpoint = (
|
||||
f"{request_base_url}/{mcp_server_name}/authorize"
|
||||
if mcp_server_name
|
||||
|
|
@ -640,6 +687,19 @@ async def register_client(request: Request, mcp_server_name: Optional[str] = Non
|
|||
"redirect_uris": [f"{request_base_url}/callback"],
|
||||
}
|
||||
if not mcp_server_name:
|
||||
resolved = _resolve_oauth2_server_for_root_endpoints()
|
||||
if resolved:
|
||||
return await register_client_with_server(
|
||||
request=request,
|
||||
mcp_server=resolved,
|
||||
client_name=data.get("client_name", ""),
|
||||
grant_types=data.get("grant_types", []),
|
||||
response_types=data.get("response_types", []),
|
||||
token_endpoint_auth_method=data.get(
|
||||
"token_endpoint_auth_method", ""
|
||||
),
|
||||
fallback_client_id=resolved.server_name or resolved.name,
|
||||
)
|
||||
return dummy_return
|
||||
|
||||
client_ip = IPAddressUtils.get_mcp_client_ip(request)
|
||||
|
|
|
|||
|
|
@ -339,7 +339,7 @@ class MCPServerManager:
|
|||
verbose_logger.info(
|
||||
f"Loading OpenAPI spec from {spec_path} for server {server_name}"
|
||||
)
|
||||
self._register_openapi_tools(
|
||||
await self._register_openapi_tools(
|
||||
spec_path=spec_path,
|
||||
server=new_server,
|
||||
base_url=server_config.get("url", ""),
|
||||
|
|
@ -351,7 +351,9 @@ class MCPServerManager:
|
|||
|
||||
self.initialize_tool_name_to_mcp_server_name_mapping()
|
||||
|
||||
def _register_openapi_tools(self, spec_path: str, server: MCPServer, base_url: str):
|
||||
async def _register_openapi_tools(
|
||||
self, spec_path: str, server: MCPServer, base_url: str
|
||||
):
|
||||
"""
|
||||
Register tools from an OpenAPI specification for a given server.
|
||||
|
||||
|
|
@ -373,15 +375,15 @@ class MCPServerManager:
|
|||
get_base_url as get_openapi_base_url,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
|
||||
load_openapi_spec,
|
||||
load_openapi_spec_async,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.tool_registry import (
|
||||
global_mcp_tool_registry,
|
||||
)
|
||||
|
||||
try:
|
||||
# Load OpenAPI spec
|
||||
spec = load_openapi_spec(spec_path)
|
||||
# Load OpenAPI spec (async to avoid "called from within a running event loop")
|
||||
spec = await load_openapi_spec_async(spec_path)
|
||||
|
||||
# Use base_url from config if provided, otherwise extract from spec
|
||||
if not base_url:
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ This module is used to generate MCP tools from OpenAPI specs.
|
|||
"""
|
||||
|
||||
import json
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import PurePosixPath
|
||||
from typing import Any, Dict, Optional
|
||||
from urllib.parse import quote
|
||||
|
|
@ -45,8 +47,36 @@ def _sanitize_path_parameter_value(param_value: Any, param_name: str) -> str:
|
|||
|
||||
|
||||
def load_openapi_spec(filepath: str) -> Dict[str, Any]:
|
||||
"""Load OpenAPI specification from JSON file."""
|
||||
with open(filepath, "r") as f:
|
||||
"""
|
||||
Sync wrapper. For URL specs, use the shared/custom MCP httpx client.
|
||||
"""
|
||||
try:
|
||||
# If we're already inside an event loop, prefer the async function.
|
||||
asyncio.get_running_loop()
|
||||
raise RuntimeError(
|
||||
"load_openapi_spec() was called from within a running event loop. "
|
||||
"Use 'await load_openapi_spec_async(...)' instead."
|
||||
)
|
||||
except RuntimeError as e:
|
||||
# "no running event loop" is fine; other RuntimeErrors we re-raise
|
||||
if "no running event loop" not in str(e).lower():
|
||||
raise
|
||||
return asyncio.run(load_openapi_spec_async(filepath))
|
||||
|
||||
async def load_openapi_spec_async(filepath: str) -> Dict[str, Any]:
|
||||
if filepath.startswith("http://") or filepath.startswith("https://"):
|
||||
client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)
|
||||
# NOTE: do not close shared client if get_async_httpx_client returns a shared singleton.
|
||||
# If it returns a new client each time, consider wrapping it in an async context manager.
|
||||
r = await client.get(filepath)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
# fallback: local file
|
||||
# Local filesystem path
|
||||
if not os.path.exists(filepath):
|
||||
raise FileNotFoundError(f"OpenAPI spec not found at {filepath}")
|
||||
with open(filepath, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import importlib
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Union
|
||||
from typing import Any, Awaitable, Callable, Dict, List, Optional, Union
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
|
||||
|
|
@ -528,24 +528,50 @@ if MCP_AVAILABLE:
|
|||
NewMCPServerRequest,
|
||||
)
|
||||
|
||||
def _extract_credentials(
|
||||
request: NewMCPServerRequest,
|
||||
) -> tuple:
|
||||
"""
|
||||
Extract OAuth credentials from the nested ``request.credentials`` dict.
|
||||
|
||||
Returns:
|
||||
(client_id, client_secret, scopes) — any value may be ``None``.
|
||||
"""
|
||||
creds = request.credentials if isinstance(request.credentials, dict) else {}
|
||||
client_id: Optional[str] = creds.get("client_id")
|
||||
client_secret: Optional[str] = creds.get("client_secret")
|
||||
scopes_raw = creds.get("scopes")
|
||||
scopes: Optional[List[str]] = scopes_raw if isinstance(scopes_raw, list) else None
|
||||
return client_id, client_secret, scopes
|
||||
|
||||
async def _execute_with_mcp_client(
|
||||
request: NewMCPServerRequest,
|
||||
operation,
|
||||
operation: Callable[..., Awaitable[Any]],
|
||||
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
|
||||
oauth2_headers: Optional[Dict[str, str]] = None,
|
||||
raw_headers: Optional[Dict[str, str]] = None,
|
||||
):
|
||||
) -> dict:
|
||||
"""
|
||||
Common helper to create MCP client, execute operation, and ensure proper cleanup.
|
||||
Create a temporary MCP client from *request*, run *operation*, and return the result.
|
||||
|
||||
For M2M OAuth servers (those with ``client_id``, ``client_secret``, and
|
||||
``token_url``), the incoming ``oauth2_headers`` are dropped so that
|
||||
``resolve_mcp_auth`` can auto-fetch a token via ``client_credentials``.
|
||||
|
||||
Args:
|
||||
request: MCP server configuration
|
||||
operation: Async function that takes a client and returns the operation result
|
||||
request: MCP server configuration submitted by the UI.
|
||||
operation: Async callable that receives the created client and returns a result dict.
|
||||
mcp_auth_header: Pre-resolved credential header (API-key / bearer token).
|
||||
oauth2_headers: Headers extracted from the incoming request (may contain the
|
||||
litellm API key — must NOT be forwarded for M2M servers).
|
||||
raw_headers: Raw request headers forwarded for stdio env construction.
|
||||
|
||||
Returns:
|
||||
Operation result or error response
|
||||
The dict returned by *operation*, or an error dict on failure.
|
||||
"""
|
||||
try:
|
||||
client_id, client_secret, scopes = _extract_credentials(request)
|
||||
|
||||
server_model = MCPServer(
|
||||
server_id=request.server_id or "",
|
||||
name=request.alias or request.server_name or "",
|
||||
|
|
@ -557,14 +583,26 @@ if MCP_AVAILABLE:
|
|||
args=request.args,
|
||||
env=request.env,
|
||||
static_headers=request.static_headers,
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
token_url=request.token_url,
|
||||
scopes=scopes,
|
||||
authorization_url=request.authorization_url,
|
||||
registration_url=request.registration_url,
|
||||
)
|
||||
|
||||
stdio_env = global_mcp_server_manager._build_stdio_env(
|
||||
server_model, raw_headers
|
||||
)
|
||||
|
||||
# For M2M OAuth servers, drop the incoming Authorization header so that
|
||||
# resolve_mcp_auth can auto-fetch a token via client_credentials.
|
||||
effective_oauth2_headers = (
|
||||
None if server_model.has_client_credentials else oauth2_headers
|
||||
)
|
||||
|
||||
merged_headers = merge_mcp_headers(
|
||||
extra_headers=oauth2_headers,
|
||||
extra_headers=effective_oauth2_headers,
|
||||
static_headers=request.static_headers,
|
||||
)
|
||||
|
||||
|
|
@ -577,11 +615,14 @@ if MCP_AVAILABLE:
|
|||
|
||||
return await operation(client)
|
||||
|
||||
except Exception as e:
|
||||
verbose_logger.error(f"Error in MCP operation: {e}", exc_info=True)
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
raise
|
||||
except BaseException as e:
|
||||
verbose_logger.error("Error in MCP operation: %s", e, exc_info=True)
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "An internal error has occurred while testing the MCP server.",
|
||||
"error": True,
|
||||
"message": "Failed to connect to MCP server. Check proxy logs for details.",
|
||||
}
|
||||
|
||||
@router.post("/test/connection", dependencies=[Depends(user_api_key_auth)])
|
||||
|
|
|
|||
|
|
@ -31,6 +31,9 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
|||
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
|
||||
MCPRequestHandler,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
|
||||
get_request_base_url,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.utils import (
|
||||
LITELLM_MCP_SERVER_DESCRIPTION,
|
||||
LITELLM_MCP_SERVER_NAME,
|
||||
|
|
@ -1972,7 +1975,7 @@ if MCP_AVAILABLE:
|
|||
)
|
||||
if server and server.auth_type == MCPAuth.oauth2 and not oauth2_headers:
|
||||
request = StarletteRequest(scope)
|
||||
base_url = str(request.base_url).rstrip("/")
|
||||
base_url = get_request_base_url(request)
|
||||
|
||||
authorization_uri = (
|
||||
f"Bearer authorization_uri="
|
||||
|
|
|
|||
|
|
@ -633,6 +633,9 @@ class LiteLLMRoutes(enum.Enum):
|
|||
"/model/{model_id}/update",
|
||||
"/prompt/list",
|
||||
"/prompt/info",
|
||||
# Invitation routes - org/team admins checked in endpoint via _user_has_admin_privileges
|
||||
"/invitation/new",
|
||||
"/invitation/delete",
|
||||
] # routes that manage their own allowed/disallowed logic
|
||||
|
||||
## Org Admin Routes ##
|
||||
|
|
@ -1911,6 +1914,10 @@ class PassThroughGenericEndpoint(LiteLLMPydanticObjectBase):
|
|||
default=None,
|
||||
description="Guardrails configuration for this passthrough endpoint. Dict keys are guardrail names, values are optional settings for field targeting. When set, all org/team/key level guardrails will also execute. Defaults to None (no guardrails execute).",
|
||||
)
|
||||
is_from_config: bool = Field(
|
||||
default=False,
|
||||
description="True if this endpoint is defined in the config file, False if from DB. Config-defined endpoints cannot be edited via the UI.",
|
||||
)
|
||||
|
||||
|
||||
class PassThroughEndpointResponse(LiteLLMPydanticObjectBase):
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ Returns a UserAPIKeyAuth object if the API key is valid
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
from typing import List, Optional, Tuple, cast
|
||||
|
|
@ -115,6 +116,18 @@ def _get_bearer_token_or_received_api_key(api_key: str) -> str:
|
|||
api_key = api_key.replace("Basic ", "") # handle langfuse input
|
||||
elif api_key.startswith("bearer "):
|
||||
api_key = api_key.replace("bearer ", "")
|
||||
elif api_key.startswith("AWS4-HMAC-SHA256"):
|
||||
# Handle AWS Signature V4 format from LangChain
|
||||
# Format: AWS4-HMAC-SHA256 Credential=Bearer sk-12345/date/region/service/aws4_request, SignedHeaders=..., Signature=...
|
||||
# Extract the Bearer token from the Credential field
|
||||
match = re.search(r'Credential=Bearer\s+([^/\s,]+)', api_key)
|
||||
if match:
|
||||
api_key = match.group(1)
|
||||
else:
|
||||
# If no Bearer token found in Credential, try to extract just the credential value
|
||||
match = re.search(r'Credential=([^/\s,]+)', api_key)
|
||||
if match:
|
||||
api_key = match.group(1)
|
||||
|
||||
return api_key
|
||||
|
||||
|
|
@ -128,6 +141,20 @@ def _get_bearer_token(
|
|||
api_key = api_key.replace("Basic ", "") # handle langfuse input
|
||||
elif api_key.startswith("bearer "):
|
||||
api_key = api_key.replace("bearer ", "")
|
||||
elif api_key.startswith("AWS4-HMAC-SHA256"):
|
||||
# Handle AWS Signature V4 format from LangChain
|
||||
# Format: AWS4-HMAC-SHA256 Credential=Bearer sk-12345/date/region/service/aws4_request, SignedHeaders=..., Signature=...
|
||||
# Extract the Bearer token from the Credential field
|
||||
match = re.search(r'Credential=Bearer\s+([^/\s,]+)', api_key)
|
||||
if match:
|
||||
api_key = match.group(1)
|
||||
else:
|
||||
# If no Bearer token found in Credential, try to extract just the credential value
|
||||
match = re.search(r'Credential=([^/\s,]+)', api_key)
|
||||
if match:
|
||||
api_key = match.group(1)
|
||||
else:
|
||||
api_key = ""
|
||||
else:
|
||||
api_key = ""
|
||||
return api_key
|
||||
|
|
|
|||
|
|
@ -394,6 +394,14 @@ def get_logging_caching_headers(request_data: Dict) -> Optional[Dict]:
|
|||
_metadata["applied_policies"]
|
||||
)
|
||||
|
||||
if "policy_sources" in _metadata:
|
||||
sources = _metadata["policy_sources"]
|
||||
if isinstance(sources, dict) and sources:
|
||||
# Use ';' as delimiter — matched_via reasons may contain commas
|
||||
headers["x-litellm-policy-sources"] = "; ".join(
|
||||
f"{name}={reason}" for name, reason in sources.items()
|
||||
)
|
||||
|
||||
if "semantic-similarity" in _metadata:
|
||||
headers["x-litellm-semantic-similarity"] = str(_metadata["semantic-similarity"])
|
||||
|
||||
|
|
@ -441,6 +449,27 @@ def add_policy_to_applied_policies_header(
|
|||
request_data["metadata"] = _metadata
|
||||
|
||||
|
||||
def add_policy_sources_to_metadata(
|
||||
request_data: Dict, policy_sources: Dict[str, str]
|
||||
):
|
||||
"""
|
||||
Store policy match reasons in metadata for x-litellm-policy-sources header.
|
||||
|
||||
Args:
|
||||
request_data: The request data dict
|
||||
policy_sources: Map of policy_name -> matched_via reason
|
||||
"""
|
||||
if not policy_sources:
|
||||
return
|
||||
_metadata = request_data.get("metadata", None) or {}
|
||||
existing = _metadata.get("policy_sources", {})
|
||||
if not isinstance(existing, dict):
|
||||
existing = {}
|
||||
existing.update(policy_sources)
|
||||
_metadata["policy_sources"] = existing
|
||||
request_data["metadata"] = _metadata
|
||||
|
||||
|
||||
def add_guardrail_response_to_standard_logging_object(
|
||||
litellm_logging_obj: Optional["LiteLLMLogging"],
|
||||
guardrail_response: StandardLoggingGuardrailInformation,
|
||||
|
|
|
|||
|
|
@ -24,9 +24,12 @@ async def custom_sso_handler(userIDPInfo: OpenID) -> SSOUserDefinedValues:
|
|||
print(f"userIDPInfo: {userIDPInfo}") # noqa
|
||||
|
||||
if userIDPInfo.id is None:
|
||||
raise ValueError(
|
||||
f"No ID found for user. userIDPInfo.id is None {userIDPInfo}"
|
||||
)
|
||||
raise ValueError(f"No ID found for user. userIDPInfo.id is None {userIDPInfo}")
|
||||
|
||||
# Access extra fields from the IDP response (requires GENERIC_USER_EXTRA_ATTRIBUTES env var)
|
||||
# Example: Set GENERIC_USER_EXTRA_ATTRIBUTES="group,NTID,domain" to capture these fields
|
||||
# extra_fields = getattr(userIDPInfo, 'extra_fields', None) or {}
|
||||
# user_groups = extra_fields.get("group", [])
|
||||
|
||||
# check if user exists in litellm proxy DB
|
||||
_user_info = await user_info(user_id=userIDPInfo.id)
|
||||
|
|
|
|||
|
|
@ -10,14 +10,18 @@ from litellm._service_logger import ServiceLogging
|
|||
service_logger_obj = (
|
||||
ServiceLogging()
|
||||
) # used for tracking metrics for In memory buffer, redis buffer, pod lock manager
|
||||
from litellm.constants import MAX_IN_MEMORY_QUEUE_FLUSH_COUNT, MAX_SIZE_IN_MEMORY_QUEUE
|
||||
from litellm.constants import (
|
||||
LITELLM_ASYNCIO_QUEUE_MAXSIZE,
|
||||
MAX_IN_MEMORY_QUEUE_FLUSH_COUNT,
|
||||
MAX_SIZE_IN_MEMORY_QUEUE,
|
||||
)
|
||||
|
||||
|
||||
class BaseUpdateQueue:
|
||||
"""Base class for in memory buffer for database transactions"""
|
||||
|
||||
def __init__(self):
|
||||
self.update_queue = asyncio.Queue()
|
||||
self.update_queue = asyncio.Queue(maxsize=LITELLM_ASYNCIO_QUEUE_MAXSIZE)
|
||||
self.MAX_SIZE_IN_MEMORY_QUEUE = MAX_SIZE_IN_MEMORY_QUEUE
|
||||
|
||||
async def add_update(self, update):
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from copy import deepcopy
|
|||
from typing import Dict, List, Optional
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import LITELLM_ASYNCIO_QUEUE_MAXSIZE
|
||||
from litellm.proxy._types import BaseDailySpendTransaction
|
||||
from litellm.proxy.db.db_transaction_queue.base_update_queue import (
|
||||
BaseUpdateQueue,
|
||||
|
|
@ -54,7 +55,7 @@ class DailySpendUpdateQueue(BaseUpdateQueue):
|
|||
def __init__(self):
|
||||
super().__init__()
|
||||
self.update_queue: asyncio.Queue[Dict[str, BaseDailySpendTransaction]] = (
|
||||
asyncio.Queue()
|
||||
asyncio.Queue(maxsize=LITELLM_ASYNCIO_QUEUE_MAXSIZE)
|
||||
)
|
||||
|
||||
async def add_update(self, update: Dict[str, BaseDailySpendTransaction]):
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import asyncio
|
|||
from typing import Dict, List, Optional
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import LITELLM_ASYNCIO_QUEUE_MAXSIZE
|
||||
from litellm.proxy._types import (
|
||||
DBSpendUpdateTransactions,
|
||||
Litellm_EntityType,
|
||||
|
|
@ -21,7 +22,9 @@ class SpendUpdateQueue(BaseUpdateQueue):
|
|||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.update_queue: asyncio.Queue[SpendUpdateQueueItem] = asyncio.Queue()
|
||||
self.update_queue: asyncio.Queue[SpendUpdateQueueItem] = asyncio.Queue(
|
||||
maxsize=LITELLM_ASYNCIO_QUEUE_MAXSIZE
|
||||
)
|
||||
|
||||
async def flush_and_get_aggregated_db_spend_update_transactions(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -19,13 +19,15 @@ async def get_ui_config():
|
|||
from litellm.proxy.utils import get_proxy_base_url, get_server_root_path
|
||||
|
||||
auto_redirect_ui_login_to_sso = (
|
||||
os.getenv("AUTO_REDIRECT_UI_LOGIN_TO_SSO", "true").lower() == "true"
|
||||
os.getenv("AUTO_REDIRECT_UI_LOGIN_TO_SSO", "false").lower() == "true"
|
||||
)
|
||||
admin_ui_disabled = os.getenv("DISABLE_ADMIN_UI", "false").lower() == "true"
|
||||
|
||||
sso_configured = _has_user_setup_sso()
|
||||
return UiDiscoveryEndpoints(
|
||||
server_root_path=get_server_root_path(),
|
||||
proxy_base_url=get_proxy_base_url(),
|
||||
auto_redirect_to_sso=_has_user_setup_sso() and auto_redirect_ui_login_to_sso,
|
||||
auto_redirect_to_sso=sso_configured and auto_redirect_ui_login_to_sso,
|
||||
admin_ui_disabled=admin_ui_disabled,
|
||||
sso_configured=sso_configured,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from litellm.proxy._types import UserAPIKeyAuth
|
|||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.guardrails.guardrail_registry import GuardrailRegistry
|
||||
from litellm.types.guardrails import (
|
||||
BaseLitellmParams,
|
||||
PII_ENTITY_CATEGORIES_MAP,
|
||||
ApplyGuardrailRequest,
|
||||
ApplyGuardrailResponse,
|
||||
|
|
@ -150,6 +151,7 @@ async def list_guardrails_v2():
|
|||
}
|
||||
```
|
||||
"""
|
||||
from litellm.litellm_core_utils.litellm_logging import _get_masked_values
|
||||
from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
|
|
@ -164,11 +166,29 @@ async def list_guardrails_v2():
|
|||
guardrail_configs: List[GuardrailInfoResponse] = []
|
||||
seen_guardrail_ids = set()
|
||||
for guardrail in guardrails:
|
||||
litellm_params: Optional[Union[LitellmParams, dict]] = guardrail.get(
|
||||
"litellm_params"
|
||||
)
|
||||
litellm_params_dict = (
|
||||
litellm_params.model_dump(exclude_none=True)
|
||||
if isinstance(litellm_params, LitellmParams)
|
||||
else litellm_params
|
||||
) or {}
|
||||
masked_litellm_params_dict = _get_masked_values(
|
||||
litellm_params_dict,
|
||||
unmasked_length=4,
|
||||
number_of_asterisks=4,
|
||||
)
|
||||
masked_litellm_params = (
|
||||
BaseLitellmParams(**masked_litellm_params_dict)
|
||||
if masked_litellm_params_dict
|
||||
else None
|
||||
)
|
||||
guardrail_configs.append(
|
||||
GuardrailInfoResponse(
|
||||
guardrail_id=guardrail.get("guardrail_id"),
|
||||
guardrail_name=guardrail.get("guardrail_name"),
|
||||
litellm_params=guardrail.get("litellm_params"),
|
||||
litellm_params=masked_litellm_params,
|
||||
guardrail_info=guardrail.get("guardrail_info"),
|
||||
created_at=guardrail.get("created_at"),
|
||||
updated_at=guardrail.get("updated_at"),
|
||||
|
|
@ -182,11 +202,27 @@ async def list_guardrails_v2():
|
|||
for guardrail in in_memory_guardrails:
|
||||
# only add guardrails that are not in DB guardrail list already
|
||||
if guardrail.get("guardrail_id") not in seen_guardrail_ids:
|
||||
in_memory_litellm_params_raw = guardrail.get("litellm_params")
|
||||
in_memory_litellm_params_dict = (
|
||||
in_memory_litellm_params_raw.model_dump(exclude_none=True)
|
||||
if isinstance(in_memory_litellm_params_raw, LitellmParams)
|
||||
else in_memory_litellm_params_raw
|
||||
) or {}
|
||||
masked_in_memory_litellm_params = _get_masked_values(
|
||||
in_memory_litellm_params_dict,
|
||||
unmasked_length=4,
|
||||
number_of_asterisks=4,
|
||||
)
|
||||
masked_in_memory_litellm_params_typed = (
|
||||
BaseLitellmParams(**masked_in_memory_litellm_params)
|
||||
if masked_in_memory_litellm_params
|
||||
else None
|
||||
)
|
||||
guardrail_configs.append(
|
||||
GuardrailInfoResponse(
|
||||
guardrail_id=guardrail.get("guardrail_id"),
|
||||
guardrail_name=guardrail.get("guardrail_name"),
|
||||
litellm_params=dict(guardrail.get("litellm_params") or {}),
|
||||
litellm_params=masked_in_memory_litellm_params_typed,
|
||||
guardrail_info=dict(guardrail.get("guardrail_info") or {}),
|
||||
guardrail_definition_location="config",
|
||||
)
|
||||
|
|
@ -666,11 +702,16 @@ async def get_guardrail_info(guardrail_id: str):
|
|||
unmasked_length=4,
|
||||
number_of_asterisks=4,
|
||||
)
|
||||
masked_litellm_params = (
|
||||
BaseLitellmParams(**masked_litellm_params_dict)
|
||||
if masked_litellm_params_dict
|
||||
else None
|
||||
)
|
||||
|
||||
return GuardrailInfoResponse(
|
||||
guardrail_id=result.get("guardrail_id"),
|
||||
guardrail_name=result.get("guardrail_name"),
|
||||
litellm_params=masked_litellm_params_dict,
|
||||
litellm_params=masked_litellm_params,
|
||||
guardrail_info=dict(result.get("guardrail_info") or {}),
|
||||
created_at=result.get("created_at"),
|
||||
updated_at=result.get("updated_at"),
|
||||
|
|
|
|||
|
|
@ -461,12 +461,37 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
|
|||
data=prepared_request.body, # type: ignore
|
||||
headers=prepared_request.headers, # type: ignore
|
||||
)
|
||||
except HTTPException:
|
||||
# Propagate HTTPException (e.g. from non-200 path) as-is
|
||||
raise
|
||||
except Exception as e:
|
||||
# If this is an HTTP error with a response body (e.g. httpx.HTTPStatusError),
|
||||
# extract the AWS error message and propagate it
|
||||
response = getattr(e, "response", None)
|
||||
if isinstance(response, httpx.Response):
|
||||
try:
|
||||
status_code, detail_message = (
|
||||
self._parse_bedrock_guardrail_error_response(response)
|
||||
)
|
||||
self.add_standard_logging_guardrail_information_to_request_data(
|
||||
guardrail_provider=self.guardrail_provider,
|
||||
guardrail_json_response={"error": detail_message},
|
||||
request_data=request_data or {},
|
||||
guardrail_status="guardrail_failed_to_respond",
|
||||
start_time=start_time.timestamp(),
|
||||
end_time=datetime.now().timestamp(),
|
||||
duration=(datetime.now() - start_time).total_seconds(),
|
||||
event_type=event_type,
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status_code, detail=detail_message
|
||||
) from e
|
||||
except HTTPException:
|
||||
raise
|
||||
# Endpoint down, timeout, or other HTTP/network errors
|
||||
verbose_proxy_logger.error(
|
||||
"Bedrock AI: failed to make guardrail request: %s", str(e)
|
||||
)
|
||||
# Add guardrail information with failure status
|
||||
self.add_standard_logging_guardrail_information_to_request_data(
|
||||
guardrail_provider=self.guardrail_provider,
|
||||
guardrail_json_response={"error": str(e)},
|
||||
|
|
@ -477,7 +502,6 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
|
|||
duration=(datetime.now() - start_time).total_seconds(),
|
||||
event_type=event_type,
|
||||
)
|
||||
# Re-raise the exception to maintain existing behavior
|
||||
raise
|
||||
|
||||
#########################################################
|
||||
|
|
@ -509,11 +533,15 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
|
|||
bedrock_guardrail_response
|
||||
)
|
||||
else:
|
||||
status_code, detail_message = self._parse_bedrock_guardrail_error_response(
|
||||
httpx_response
|
||||
)
|
||||
verbose_proxy_logger.error(
|
||||
"Bedrock AI: error in response. Status code: %s, response: %s",
|
||||
httpx_response.status_code,
|
||||
httpx_response.text,
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail_message)
|
||||
|
||||
return bedrock_guardrail_response
|
||||
|
||||
|
|
@ -579,6 +607,34 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
|
|||
return "success"
|
||||
return "guardrail_failed_to_respond"
|
||||
|
||||
def _parse_bedrock_guardrail_error_response(
|
||||
self, response: httpx.Response
|
||||
) -> Tuple[int, str]:
|
||||
"""
|
||||
Parse AWS Bedrock guardrail error response body to extract status code and message.
|
||||
|
||||
AWS may return shapes like {"message": "..."} or {"error": {"message": "..."}}.
|
||||
Returns (status_code, message) for use in HTTPException.
|
||||
"""
|
||||
status_code = response.status_code
|
||||
message = "Bedrock guardrail request failed"
|
||||
try:
|
||||
body = response.json()
|
||||
except Exception:
|
||||
text = getattr(response, "text", None) or ""
|
||||
if isinstance(text, str) and text.strip():
|
||||
return (status_code, text.strip())
|
||||
return (status_code, message)
|
||||
if isinstance(body, dict):
|
||||
if isinstance(body.get("message"), str):
|
||||
return (status_code, body["message"])
|
||||
err = body.get("error")
|
||||
if isinstance(err, dict) and isinstance(err.get("message"), str):
|
||||
return (status_code, err["message"])
|
||||
if isinstance(err, str):
|
||||
return (status_code, err)
|
||||
return (status_code, message)
|
||||
|
||||
def _get_http_exception_for_blocked_guardrail(
|
||||
self, response: BedrockGuardrailResponse
|
||||
) -> Union[HTTPException, GuardrailInterventionNormalStringError]:
|
||||
|
|
@ -739,9 +795,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
|
|||
#########################################################
|
||||
########## 1. Make the Bedrock API request ##########
|
||||
#########################################################
|
||||
bedrock_guardrail_response: Optional[
|
||||
Union[BedrockGuardrailResponse, str]
|
||||
] = None
|
||||
bedrock_guardrail_response: Optional[Union[BedrockGuardrailResponse, str]] = (
|
||||
None
|
||||
)
|
||||
try:
|
||||
bedrock_guardrail_response = await self.make_bedrock_api_request(
|
||||
source="INPUT", messages=filtered_messages, request_data=data
|
||||
|
|
@ -811,9 +867,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
|
|||
#########################################################
|
||||
########## 1. Make the Bedrock API request ##########
|
||||
#########################################################
|
||||
bedrock_guardrail_response: Optional[
|
||||
Union[BedrockGuardrailResponse, str]
|
||||
] = None
|
||||
bedrock_guardrail_response: Optional[Union[BedrockGuardrailResponse, str]] = (
|
||||
None
|
||||
)
|
||||
try:
|
||||
bedrock_guardrail_response = await self.make_bedrock_api_request(
|
||||
source="INPUT", messages=filtered_messages, request_data=data
|
||||
|
|
|
|||
|
|
@ -35,7 +35,10 @@ from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Type, cast
|
|||
from fastapi import HTTPException
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
)
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
|
||||
from litellm.types.utils import GenericGuardrailAPIInputs
|
||||
|
|
@ -179,6 +182,7 @@ class CustomCodeGuardrail(CustomGuardrail):
|
|||
self._compile_error = f"Failed to compile custom code: {e}"
|
||||
raise CustomCodeCompilationError(self._compile_error) from e
|
||||
|
||||
@log_guardrail_information
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
|
|
|
|||
|
|
@ -23,7 +23,10 @@ import httpx
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
|
|
@ -483,6 +486,7 @@ class EnkryptAIGuardrails(CustomGuardrail):
|
|||
request_data=data, guardrail_name=self.guardrail_name
|
||||
)
|
||||
|
||||
@log_guardrail_information
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: "GenericGuardrailAPIInputs",
|
||||
|
|
|
|||
|
|
@ -10,7 +10,10 @@ from typing import TYPE_CHECKING, Any, Dict, Literal, Optional
|
|||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.exceptions import GuardrailRaisedException
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
|
|
@ -150,6 +153,7 @@ class GenericGuardrailAPI(CustomGuardrail):
|
|||
|
||||
return result_metadata
|
||||
|
||||
@log_guardrail_information
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@ from fastapi import HTTPException
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
ModifyResponseException
|
||||
ModifyResponseException,
|
||||
log_guardrail_information,
|
||||
)
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
|
|
@ -108,7 +109,9 @@ class GraySwanGuardrail(CustomGuardrail):
|
|||
self.categories = categories
|
||||
self.policy_id = policy_id
|
||||
self.fail_open = True if fail_open is None else bool(fail_open)
|
||||
self.guardrail_timeout = 30.0 if guardrail_timeout is None else float(guardrail_timeout)
|
||||
self.guardrail_timeout = (
|
||||
30.0 if guardrail_timeout is None else float(guardrail_timeout)
|
||||
)
|
||||
|
||||
# Streaming configuration
|
||||
self.streaming_end_of_stream_only = streaming_end_of_stream_only
|
||||
|
|
@ -155,6 +158,7 @@ class GraySwanGuardrail(CustomGuardrail):
|
|||
# Unified Guardrail Interface (works with ALL endpoints automatically)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@log_guardrail_information
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
|
|
@ -208,7 +212,9 @@ class GraySwanGuardrail(CustomGuardrail):
|
|||
messages = [{"role": role, "content": text} for text in texts]
|
||||
|
||||
# Get dynamic params from request metadata
|
||||
dynamic_body = self.get_guardrail_dynamic_request_body_params(request_data) or {}
|
||||
dynamic_body = (
|
||||
self.get_guardrail_dynamic_request_body_params(request_data) or {}
|
||||
)
|
||||
if dynamic_body:
|
||||
verbose_proxy_logger.debug(
|
||||
"Gray Swan Guardrail: dynamic extra_body=%s", safe_dumps(dynamic_body)
|
||||
|
|
@ -271,12 +277,12 @@ class GraySwanGuardrail(CustomGuardrail):
|
|||
async def run_grayswan_guardrail(self, payload: dict) -> Dict[str, Any]:
|
||||
"""
|
||||
Run the GraySwan guardrail on a payload.
|
||||
|
||||
|
||||
This is a legacy method for testing purposes.
|
||||
|
||||
|
||||
Args:
|
||||
payload: The payload to scan
|
||||
|
||||
|
||||
Returns:
|
||||
Dict containing the GraySwan API response
|
||||
"""
|
||||
|
|
@ -293,11 +299,11 @@ class GraySwanGuardrail(CustomGuardrail):
|
|||
) -> None:
|
||||
"""
|
||||
Legacy method for processing GraySwan API responses.
|
||||
|
||||
|
||||
This method is maintained for backward compatibility with existing tests.
|
||||
It handles the test scenarios where responses need to be processed with
|
||||
knowledge of the request context (pre/during/post call hooks).
|
||||
|
||||
|
||||
Args:
|
||||
response_json: Response from GraySwan API
|
||||
data: Optional request data (for passthrough exceptions)
|
||||
|
|
@ -365,7 +371,10 @@ class GraySwanGuardrail(CustomGuardrail):
|
|||
)
|
||||
|
||||
# If hook_type is provided and in pre/during call, raise exception
|
||||
if hook_type in [GuardrailEventHooks.pre_call, GuardrailEventHooks.during_call]:
|
||||
if hook_type in [
|
||||
GuardrailEventHooks.pre_call,
|
||||
GuardrailEventHooks.during_call,
|
||||
]:
|
||||
# Raise ModifyResponseException to short-circuit LLM call
|
||||
if data is None:
|
||||
data = {}
|
||||
|
|
@ -540,7 +549,9 @@ class GraySwanGuardrail(CustomGuardrail):
|
|||
if isinstance(litellm_metadata, dict) and litellm_metadata:
|
||||
cleaned_litellm_metadata = dict(litellm_metadata)
|
||||
# cleaned_litellm_metadata.pop("user_api_key_auth", None)
|
||||
sanitized = safe_json_loads(safe_dumps(cleaned_litellm_metadata), default={})
|
||||
sanitized = safe_json_loads(
|
||||
safe_dumps(cleaned_litellm_metadata), default={}
|
||||
)
|
||||
if isinstance(sanitized, dict) and sanitized:
|
||||
payload["litellm_metadata"] = sanitized
|
||||
|
||||
|
|
@ -566,7 +577,9 @@ class GraySwanGuardrail(CustomGuardrail):
|
|||
detection_info = detection_info[0]
|
||||
|
||||
# Extract fields from detection_info dict
|
||||
detection_dict: dict = detection_info if isinstance(detection_info, dict) else {}
|
||||
detection_dict: dict = (
|
||||
detection_info if isinstance(detection_info, dict) else {}
|
||||
)
|
||||
violation_score = detection_dict.get("violation_score", 0.0)
|
||||
violated_rules = detection_dict.get("violated_rules", [])
|
||||
mutation = detection_dict.get("mutation", False)
|
||||
|
|
@ -582,7 +595,9 @@ class GraySwanGuardrail(CustomGuardrail):
|
|||
if violated_rules:
|
||||
formatted_rules = self._format_violated_rules(violated_rules)
|
||||
if formatted_rules:
|
||||
message_parts.append(f"It was violating the rule(s): {formatted_rules}.")
|
||||
message_parts.append(
|
||||
f"It was violating the rule(s): {formatted_rules}."
|
||||
)
|
||||
|
||||
if mutation:
|
||||
message_parts.append(
|
||||
|
|
@ -590,9 +605,7 @@ class GraySwanGuardrail(CustomGuardrail):
|
|||
)
|
||||
|
||||
if ipi:
|
||||
message_parts.append(
|
||||
"Indirect Prompt Injection was DETECTED."
|
||||
)
|
||||
message_parts.append("Indirect Prompt Injection was DETECTED.")
|
||||
|
||||
return "\n".join(message_parts)
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,10 @@ from httpx import HTTPStatusError
|
|||
from requests.auth import HTTPBasicAuth
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client,
|
||||
|
|
@ -110,6 +113,7 @@ class HiddenlayerGuardrail(CustomGuardrail):
|
|||
)
|
||||
super().__init__(**kwargs)
|
||||
|
||||
@log_guardrail_information
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
|
|
|
|||
|
|
@ -28,7 +28,10 @@ from fastapi import HTTPException
|
|||
|
||||
from litellm import Router
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
)
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.types.utils import ModelResponseStream
|
||||
|
||||
|
|
@ -50,6 +53,7 @@ from litellm.types.proxy.guardrails.guardrail_hooks.litellm_content_filter impor
|
|||
ContentFilterDetection,
|
||||
PatternDetection,
|
||||
)
|
||||
|
||||
from .patterns import PATTERN_EXTRA_CONFIG, get_compiled_pattern
|
||||
|
||||
MAX_KEYWORD_VALUE_GAP_WORDS = 1
|
||||
|
|
@ -168,9 +172,9 @@ class ContentFilterGuardrail(CustomGuardrail):
|
|||
self.image_model = image_model
|
||||
# Store loaded categories
|
||||
self.loaded_categories: Dict[str, CategoryConfig] = {}
|
||||
self.category_keywords: Dict[
|
||||
str, Tuple[str, str, ContentFilterAction]
|
||||
] = {} # keyword -> (category, severity, action)
|
||||
self.category_keywords: Dict[str, Tuple[str, str, ContentFilterAction]] = (
|
||||
{}
|
||||
) # keyword -> (category, severity, action)
|
||||
|
||||
# Load categories if provided
|
||||
if categories:
|
||||
|
|
@ -994,6 +998,7 @@ class ContentFilterGuardrail(CustomGuardrail):
|
|||
masked_entity_count=masked_entity_count,
|
||||
)
|
||||
|
||||
@log_guardrail_information
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: "GenericGuardrailAPIInputs",
|
||||
|
|
|
|||
|
|
@ -12,7 +12,10 @@ import httpx
|
|||
from fastapi import HTTPException
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client,
|
||||
|
|
@ -26,7 +29,11 @@ if TYPE_CHECKING:
|
|||
|
||||
class OnyxGuardrail(CustomGuardrail):
|
||||
def __init__(
|
||||
self, api_base: Optional[str] = None, api_key: Optional[str] = None, timeout: Optional[float] = 10.0, **kwargs
|
||||
self,
|
||||
api_base: Optional[str] = None,
|
||||
api_key: Optional[str] = None,
|
||||
timeout: Optional[float] = 10.0,
|
||||
**kwargs,
|
||||
):
|
||||
timeout = timeout or int(os.getenv("ONYX_TIMEOUT", 10.0))
|
||||
self.async_handler = get_async_httpx_client(
|
||||
|
|
@ -79,6 +86,7 @@ class OnyxGuardrail(CustomGuardrail):
|
|||
)
|
||||
return result
|
||||
|
||||
@log_guardrail_information
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
|
|
|
|||
|
|
@ -58,7 +58,9 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
|
|||
guardrail_name: str,
|
||||
api_key: Optional[str] = None,
|
||||
api_base: Optional[str] = None,
|
||||
model: Optional[Literal["omni-moderation-latest", "text-moderation-latest"]] = None,
|
||||
model: Optional[
|
||||
Literal["omni-moderation-latest", "text-moderation-latest"]
|
||||
] = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Initialize OpenAI Moderation guardrail handler."""
|
||||
|
|
@ -75,7 +77,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
|
|||
supported_event_hooks=supported_event_hooks,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
self.async_handler = get_async_httpx_client(
|
||||
llm_provider=httpxSpecialProvider.GuardrailCallback
|
||||
)
|
||||
|
|
@ -83,10 +85,14 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
|
|||
# Store configuration
|
||||
self.api_key = api_key or self._get_api_key()
|
||||
self.api_base = api_base or "https://api.openai.com/v1"
|
||||
self.model: Literal["omni-moderation-latest", "text-moderation-latest"] = model or "omni-moderation-latest"
|
||||
self.model: Literal["omni-moderation-latest", "text-moderation-latest"] = (
|
||||
model or "omni-moderation-latest"
|
||||
)
|
||||
|
||||
if not self.api_key:
|
||||
raise ValueError("OpenAI Moderation: api_key is required. Set OPENAI_API_KEY environment variable or pass it in configuration.")
|
||||
raise ValueError(
|
||||
"OpenAI Moderation: api_key is required. Set OPENAI_API_KEY environment variable or pass it in configuration."
|
||||
)
|
||||
|
||||
verbose_proxy_logger.debug(
|
||||
f"Initialized OpenAI Moderation Guardrail: {guardrail_name} with model: {self.model}"
|
||||
|
|
@ -98,7 +104,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
|
|||
|
||||
import litellm
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
|
||||
return (
|
||||
os.environ.get("OPENAI_API_KEY")
|
||||
or litellm.api_key
|
||||
|
|
@ -106,21 +112,14 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
|
|||
or get_secret_str("OPENAI_API_KEY")
|
||||
)
|
||||
|
||||
async def async_make_request(
|
||||
self, input_text: str
|
||||
) -> "OpenAIModerationResponse":
|
||||
async def async_make_request(self, input_text: str) -> "OpenAIModerationResponse":
|
||||
"""
|
||||
Make a request to the OpenAI Moderation API.
|
||||
"""
|
||||
request_body = {
|
||||
"model": self.model,
|
||||
"input": input_text
|
||||
}
|
||||
|
||||
verbose_proxy_logger.debug(
|
||||
"OpenAI Moderation guard request: %s", request_body
|
||||
)
|
||||
|
||||
request_body = {"model": self.model, "input": input_text}
|
||||
|
||||
verbose_proxy_logger.debug("OpenAI Moderation guard request: %s", request_body)
|
||||
|
||||
response = await self.async_handler.post(
|
||||
url=f"{self.api_base}/moderations",
|
||||
headers={
|
||||
|
|
@ -133,7 +132,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
|
|||
verbose_proxy_logger.debug(
|
||||
"OpenAI Moderation guard response: %s", response.json()
|
||||
)
|
||||
|
||||
|
||||
if response.status_code != 200:
|
||||
raise HTTPException(
|
||||
status_code=response.status_code,
|
||||
|
|
@ -144,9 +143,12 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
|
|||
)
|
||||
|
||||
from litellm.types.llms.openai import OpenAIModerationResponse
|
||||
|
||||
return OpenAIModerationResponse(**response.json())
|
||||
|
||||
def _check_moderation_result(self, moderation_response: "OpenAIModerationResponse") -> None:
|
||||
def _check_moderation_result(
|
||||
self, moderation_response: "OpenAIModerationResponse"
|
||||
) -> None:
|
||||
"""
|
||||
Check if the moderation response indicates harmful content and raise exception if needed.
|
||||
"""
|
||||
|
|
@ -168,10 +170,10 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
|
|||
}
|
||||
|
||||
verbose_proxy_logger.warning(
|
||||
"OpenAI Moderation: Content flagged for violations: %s",
|
||||
violation_details
|
||||
"OpenAI Moderation: Content flagged for violations: %s",
|
||||
violation_details,
|
||||
)
|
||||
|
||||
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
|
|
@ -180,6 +182,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
|
|||
},
|
||||
)
|
||||
|
||||
@log_guardrail_information
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
|
|
@ -189,51 +192,50 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
|
|||
) -> GenericGuardrailAPIInputs:
|
||||
"""
|
||||
Apply OpenAI moderation guardrail using the unified guardrail interface.
|
||||
|
||||
|
||||
This method is called by the UnifiedLLMGuardrails system for all endpoint types
|
||||
(chat completions, embeddings, responses API, etc.).
|
||||
|
||||
|
||||
Args:
|
||||
inputs: GenericGuardrailAPIInputs containing texts and/or structured_messages
|
||||
request_data: The original request data
|
||||
input_type: Whether this is a "request" (pre-call) or "response" (post-call)
|
||||
logging_obj: Optional logging object
|
||||
|
||||
|
||||
Returns:
|
||||
The inputs unchanged (moderation doesn't modify content, only blocks)
|
||||
|
||||
|
||||
Raises:
|
||||
HTTPException: If content violates moderation policy
|
||||
"""
|
||||
# Extract text to moderate from inputs
|
||||
text_to_moderate: Optional[str] = None
|
||||
|
||||
|
||||
# Prefer structured_messages if available (has role context)
|
||||
if structured_messages := inputs.get("structured_messages"):
|
||||
text_to_moderate = self.get_user_prompt(structured_messages)
|
||||
|
||||
|
||||
# Fall back to texts
|
||||
if not text_to_moderate:
|
||||
if texts := inputs.get("texts"):
|
||||
# Join all texts for moderation
|
||||
text_to_moderate = "\n".join(texts)
|
||||
|
||||
|
||||
if not text_to_moderate:
|
||||
verbose_proxy_logger.debug(
|
||||
"OpenAI Moderation: No text content to moderate in inputs"
|
||||
)
|
||||
return inputs
|
||||
|
||||
|
||||
# Make moderation request
|
||||
moderation_response = await self.async_make_request(input_text=text_to_moderate)
|
||||
|
||||
|
||||
# Check if content is flagged and raise exception if needed
|
||||
self._check_moderation_result(moderation_response)
|
||||
|
||||
|
||||
# Moderation doesn't modify content, just blocks - return inputs unchanged
|
||||
return inputs
|
||||
|
||||
|
||||
@log_guardrail_information
|
||||
async def async_post_call_streaming_iterator_hook(
|
||||
self,
|
||||
|
|
@ -252,9 +254,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
|
|||
from litellm.main import stream_chunk_builder
|
||||
from litellm.types.utils import TextCompletionResponse
|
||||
|
||||
verbose_proxy_logger.debug(
|
||||
"OpenAI Moderation: Running streaming response scan"
|
||||
)
|
||||
verbose_proxy_logger.debug("OpenAI Moderation: Running streaming response scan")
|
||||
|
||||
# Collect all chunks to process them together
|
||||
all_chunks: List["ModelResponseStream"] = []
|
||||
|
|
@ -269,7 +269,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
|
|||
)
|
||||
|
||||
if isinstance(assembled_model_response, (type(None), TextCompletionResponse)):
|
||||
# If we can't assemble a ModelResponse or it's a text completion,
|
||||
# If we can't assemble a ModelResponse or it's a text completion,
|
||||
# just yield the original chunks without moderation
|
||||
verbose_proxy_logger.warning(
|
||||
"OpenAI Moderation: Could not assemble ModelResponse from chunks, skipping moderation"
|
||||
|
|
@ -284,19 +284,17 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
|
|||
verbose_proxy_logger.debug(
|
||||
f"OpenAI Moderation: Streaming response text: {response_text[:100]}..." # Log first 100 chars
|
||||
)
|
||||
|
||||
|
||||
# Make moderation request - this will raise HTTPException if content is flagged
|
||||
moderation_response = await self.async_make_request(
|
||||
input_text=response_text,
|
||||
)
|
||||
|
||||
|
||||
# Check if content is flagged and raise exception if needed
|
||||
self._check_moderation_result(moderation_response)
|
||||
|
||||
# If we reach here, content passed moderation - yield the original chunks
|
||||
mock_response = MockResponseIterator(
|
||||
model_response=assembled_model_response
|
||||
)
|
||||
mock_response = MockResponseIterator(model_response=assembled_model_response)
|
||||
|
||||
# Return the reconstructed stream
|
||||
async for chunk in mock_response:
|
||||
|
|
@ -306,34 +304,34 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
|
|||
"""
|
||||
Extract text content from the model response for moderation.
|
||||
"""
|
||||
if not hasattr(response, 'choices') or not response.choices:
|
||||
if not hasattr(response, "choices") or not response.choices:
|
||||
return None
|
||||
|
||||
response_texts = []
|
||||
for choice in response.choices:
|
||||
try:
|
||||
# Try to get content from message (chat completion)
|
||||
message = getattr(choice, 'message', None)
|
||||
message = getattr(choice, "message", None)
|
||||
if message:
|
||||
content = getattr(message, 'content', None)
|
||||
content = getattr(message, "content", None)
|
||||
if content and isinstance(content, str):
|
||||
response_texts.append(content)
|
||||
continue
|
||||
|
||||
|
||||
# Try to get text (text completion)
|
||||
text = getattr(choice, 'text', None)
|
||||
text = getattr(choice, "text", None)
|
||||
if text and isinstance(text, str):
|
||||
response_texts.append(text)
|
||||
continue
|
||||
|
||||
|
||||
# Try to get content from delta (streaming)
|
||||
delta = getattr(choice, 'delta', None)
|
||||
delta = getattr(choice, "delta", None)
|
||||
if delta:
|
||||
content = getattr(delta, 'content', None)
|
||||
content = getattr(delta, "content", None)
|
||||
if content and isinstance(content, str):
|
||||
response_texts.append(content)
|
||||
continue
|
||||
|
||||
|
||||
except (AttributeError, TypeError):
|
||||
# Skip choices that don't have expected attributes
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -9,10 +9,10 @@
|
|||
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
import json
|
||||
from datetime import datetime
|
||||
import threading
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
|
|
@ -38,8 +38,11 @@ if TYPE_CHECKING:
|
|||
|
||||
from litellm._uuid import uuid
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.exceptions import BlockedPiiEntityError
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
)
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.types.guardrails import (
|
||||
GuardrailEventHooks,
|
||||
|
|
@ -229,6 +232,14 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
|
|||
"""Cleanup: we try to close, but doing async cleanup in __del__ is risky."""
|
||||
pass
|
||||
|
||||
def _has_block_action(self) -> bool:
|
||||
"""Return True if pii_entities_config has any BLOCK action (fail-closed on analyzer errors)."""
|
||||
if not self.pii_entities_config:
|
||||
return False
|
||||
return any(
|
||||
action == PiiAction.BLOCK for action in self.pii_entities_config.values()
|
||||
)
|
||||
|
||||
def _get_presidio_analyze_request_payload(
|
||||
self,
|
||||
text: str,
|
||||
|
|
@ -313,13 +324,30 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
|
|||
|
||||
# Handle error responses from Presidio (e.g., {'error': 'No text provided'})
|
||||
# Presidio may return a dict instead of a list when errors occur
|
||||
def _fail_on_invalid_response(
|
||||
reason: str,
|
||||
) -> List[PresidioAnalyzeResponseItem]:
|
||||
should_fail_closed = (
|
||||
bool(self.pii_entities_config)
|
||||
or self.output_parse_pii
|
||||
or self.apply_to_output
|
||||
)
|
||||
if should_fail_closed:
|
||||
raise GuardrailRaisedException(
|
||||
guardrail_name=self.guardrail_name,
|
||||
message=f"Presidio analyzer returned invalid response; cannot verify PII when PII protection is configured: {reason}",
|
||||
should_wrap_with_default_message=False,
|
||||
)
|
||||
verbose_proxy_logger.warning(
|
||||
"Presidio analyzer %s, returning empty list", reason
|
||||
)
|
||||
return []
|
||||
|
||||
if isinstance(analyze_results, dict):
|
||||
if "error" in analyze_results:
|
||||
verbose_proxy_logger.warning(
|
||||
"Presidio analyzer returned error: %s, returning empty list",
|
||||
analyze_results.get("error"),
|
||||
return _fail_on_invalid_response(
|
||||
f"error: {analyze_results.get('error')}"
|
||||
)
|
||||
return []
|
||||
# If it's a dict but not an error, try to process it as a single item
|
||||
verbose_proxy_logger.debug(
|
||||
"Presidio returned dict (not list), attempting to process as single item"
|
||||
|
|
@ -327,23 +355,33 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
|
|||
try:
|
||||
return [PresidioAnalyzeResponseItem(**analyze_results)]
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.warning(
|
||||
"Failed to parse Presidio dict response: %s, returning empty list",
|
||||
e,
|
||||
return _fail_on_invalid_response(
|
||||
f"failed to parse dict response: {e}"
|
||||
)
|
||||
return []
|
||||
|
||||
# Handle unexpected types (str, None, etc.) - e.g. from malformed/error
|
||||
if not isinstance(analyze_results, list):
|
||||
return _fail_on_invalid_response(
|
||||
f"unexpected type {type(analyze_results).__name__} (expected list or dict), response: {str(analyze_results)[:200]}"
|
||||
)
|
||||
|
||||
# Normal case: list of results
|
||||
final_results = []
|
||||
for item in analyze_results:
|
||||
if not isinstance(item, dict):
|
||||
verbose_proxy_logger.warning(
|
||||
"Skipping invalid Presidio result item (expected dict, got %s): %s",
|
||||
type(item).__name__,
|
||||
str(item)[:100],
|
||||
)
|
||||
continue
|
||||
try:
|
||||
final_results.append(PresidioAnalyzeResponseItem(**item))
|
||||
except TypeError as te:
|
||||
# Handle case where item is not a dict (shouldn't happen, but be defensive)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.warning(
|
||||
"Skipping invalid Presidio result item: %s (error: %s)",
|
||||
"Failed to parse Presidio result item: %s (error: %s)",
|
||||
item,
|
||||
te,
|
||||
e,
|
||||
)
|
||||
continue
|
||||
return final_results
|
||||
|
|
@ -568,9 +606,9 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
|
|||
if messages is None:
|
||||
return data
|
||||
tasks = []
|
||||
task_mappings: List[
|
||||
Tuple[int, Optional[int]]
|
||||
] = [] # Track (message_index, content_index) for each task
|
||||
task_mappings: List[Tuple[int, Optional[int]]] = (
|
||||
[]
|
||||
) # Track (message_index, content_index) for each task
|
||||
|
||||
for msg_idx, m in enumerate(messages):
|
||||
content = m.get("content", None)
|
||||
|
|
@ -671,9 +709,9 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
|
|||
): # /chat/completions requests
|
||||
messages: Optional[List] = kwargs.get("messages", None)
|
||||
tasks = []
|
||||
task_mappings: List[
|
||||
Tuple[int, Optional[int]]
|
||||
] = [] # Track (message_index, content_index) for each task
|
||||
task_mappings: List[Tuple[int, Optional[int]]] = (
|
||||
[]
|
||||
) # Track (message_index, content_index) for each task
|
||||
|
||||
if messages is None:
|
||||
return kwargs, result
|
||||
|
|
@ -792,11 +830,11 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
|
|||
# Type narrowing: StreamingChoices doesn't have .message attribute
|
||||
if not hasattr(choice, "message"):
|
||||
continue
|
||||
content = getattr(choice.message, "content", None)
|
||||
content = getattr(choice.message, "content", None) # type: ignore
|
||||
if content is None:
|
||||
continue
|
||||
if isinstance(content, str):
|
||||
choice.message.content = await self.check_pii(
|
||||
choice.message.content = await self.check_pii( # type: ignore
|
||||
text=content,
|
||||
output_parse_pii=False,
|
||||
presidio_config=presidio_config,
|
||||
|
|
@ -989,6 +1027,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
@log_guardrail_information
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: "GenericGuardrailAPIInputs",
|
||||
|
|
|
|||
|
|
@ -6,7 +6,10 @@ from typing import TYPE_CHECKING, Any, List, Literal, Optional, Type
|
|||
from fastapi import HTTPException
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
|
|
@ -67,6 +70,7 @@ class PromptSecurityGuardrail(CustomGuardrail):
|
|||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
@log_guardrail_information
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
|
|
|
|||
|
|
@ -12,10 +12,11 @@ from typing import Any, Dict, List, Literal, Optional, Type
|
|||
from fastapi import HTTPException
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
Logging as LiteLLMLoggingObj,
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
|
|
@ -343,9 +344,7 @@ class QualifireGuardrail(CustomGuardrail):
|
|||
)
|
||||
url = f"{self.qualifire_api_base}/api/evaluation/evaluate"
|
||||
|
||||
verbose_proxy_logger.debug(
|
||||
f"Qualifire Guardrail: Making request to {url}"
|
||||
)
|
||||
verbose_proxy_logger.debug(f"Qualifire Guardrail: Making request to {url}")
|
||||
|
||||
# Make the API request
|
||||
response = await self.async_handler.post(
|
||||
|
|
@ -393,6 +392,7 @@ class QualifireGuardrail(CustomGuardrail):
|
|||
verbose_proxy_logger.exception(f"Qualifire Guardrail error: {e}")
|
||||
raise
|
||||
|
||||
@log_guardrail_information
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
|
|
|
|||
|
|
@ -9,7 +9,10 @@ from typing import TYPE_CHECKING, Literal, Optional
|
|||
from fastapi import HTTPException
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
|
|
@ -70,6 +73,7 @@ class ZscalerAIGuard(CustomGuardrail):
|
|||
return str(value).strip()
|
||||
return "N/A"
|
||||
|
||||
@log_guardrail_information
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: "GenericGuardrailAPIInputs",
|
||||
|
|
@ -92,7 +96,7 @@ class ZscalerAIGuard(CustomGuardrail):
|
|||
Raises:
|
||||
Exception: If content is blocked by Zscaler AI Guard
|
||||
"""
|
||||
|
||||
|
||||
texts = inputs.get("texts", [])
|
||||
try:
|
||||
verbose_proxy_logger.debug(f"ZscalerAIGuard: Checking {len(texts)} text(s)")
|
||||
|
|
@ -102,8 +106,8 @@ class ZscalerAIGuard(CustomGuardrail):
|
|||
team_metadata = metadata.get("team_metadata", {}) or {}
|
||||
|
||||
# Precedence for policy_id:
|
||||
# 1. metadata.zguard_policy_id # request level
|
||||
# 2. user_api_key_metadata.zguard_policy_id # Key level
|
||||
# 1. metadata.zguard_policy_id # request level
|
||||
# 2. user_api_key_metadata.zguard_policy_id # Key level
|
||||
# 3. team_metadata.zguard_policy_id # Team level
|
||||
# 4. self.policy_id (from environment) # Global
|
||||
policy_id = (
|
||||
|
|
@ -154,9 +158,7 @@ class ZscalerAIGuard(CustomGuardrail):
|
|||
zscaler_ai_guard_result
|
||||
and zscaler_ai_guard_result.get("action") == "BLOCK"
|
||||
):
|
||||
blocking_info = zscaler_ai_guard_result.get(
|
||||
"zscaler_ai_guard_response"
|
||||
)
|
||||
blocking_info = zscaler_ai_guard_result.get("zscaler_ai_guard_response")
|
||||
error_message = f"Content blocked by Zscaler AI Guard: {self.extract_blocking_info(blocking_info)}"
|
||||
raise Exception(error_message)
|
||||
except Exception as e:
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue