scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 86

Merge branch 'litellm_internal_staging' into feature/improve-gigachat-provider

Browse source
This commit is contained in:
KnyazSh 2026-06-16 22:20:51 +03:00
commit 6eb8eb901f
356 changed files with 23618 additions and 7776 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
.github/workflows/osv-scan.yml vendored Normal file
View file

@ -0,0 +1,49 @@
name: OSV Scan
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
paths:
- uv.lock
- ui/litellm-dashboard/package-lock.json
- osv-scanner.toml
- .github/workflows/osv-scan.yml
schedule:
- cron: "23 6 * * *"
workflow_dispatch:
permissions: {}
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
osv-scan:
name: osv-scan
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Download osv-scanner v2.3.8
run: |
curl -fsSL --retry 3 -o "$RUNNER_TEMP/osv-scanner" \
https://github.com/google/osv-scanner/releases/download/v2.3.8/osv-scanner_linux_amd64
echo "bc98e15319ed0d515e3f9235287ba53cdc5535d576d24fd573978ecfe9ab92dc $RUNNER_TEMP/osv-scanner" | sha256sum -c -
chmod +x "$RUNNER_TEMP/osv-scanner"
- name: Scan lockfiles
run: |
"$RUNNER_TEMP/osv-scanner" scan source \
--config osv-scanner.toml \
-L uv.lock \
-L ui/litellm-dashboard/package-lock.json

69
.github/workflows/test-linting.yml vendored
View file

@ -14,11 +14,15 @@ permissions:
jobs:
lint:
runs-on: ubuntu-latest
timeout-minutes: 5
timeout-minutes: 10
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
# Check out the PR head, not the default refs/pull/N/merge: the merge ref
# folds in newer base commits, which the diff-based gates (ruff delta,
# Any-discipline) would otherwise blame on this branch.
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
clean: true
persist-credentials: false
@ -67,6 +71,12 @@ jobs:
uv run --no-sync ruff check .
cd ..
- name: Check strict-rule budget (delta vs base)
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
uv run --no-sync python scripts/ruff_strict_gate.py --base "$BASE_SHA"
- name: Print OpenAI version
run: |
uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
@ -74,8 +84,11 @@ jobs:
- name: Run MyPy type checking
run: |
cd litellm
uv run --no-sync mypy .
cd ..
(uv run --no-sync mypy . || true) | uv run --no-sync python ../scripts/type_check_gate.py --tool mypy
- name: Run basedpyright type checking
run: |
(uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --tool basedpyright
- name: Check for circular imports
run: |
@ -87,6 +100,56 @@ jobs:
run: |
uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
any-discipline:
# Separate job: the first run cold-builds litellm's type cache (~2 min, ~3 GB),
# so keep it off the main lint job's time budget. Subsequent runs reuse the
# cached .mypy_cache_any and only re-type-check the changed files.
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
# Check out the PR head, not the default refs/pull/N/merge: the merge ref
# folds in newer base commits, which the diff-based gates (ruff delta,
# Any-discipline) would otherwise blame on this branch.
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
clean: true
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
- name: Install dependencies
run: |
uv sync --frozen
# Keyed on deps + mypy config (which fix the type cache's validity), not on
# source content, so changed files always differ from the restored cache.
# The gate also defensively invalidates each target's cache entry, so
# correctness never depends on cache freshness -- this is purely for speed.
- name: Restore Any-gate type cache
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: .mypy_cache_any
key: any-mypy-cache-${{ runner.os }}-py3.12-${{ hashFiles('uv.lock', 'litellm/mypy.ini') }}
restore-keys: |
any-mypy-cache-${{ runner.os }}-py3.12-
- name: Check Any discipline on changed lines
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
uv run --no-sync python scripts/check_any_discipline.py --changed --base "$BASE_SHA"
secret-scan:
runs-on: ubuntu-latest
timeout-minutes: 5

1
.gitignore vendored
View file

@ -75,6 +75,7 @@ tests/local_testing/log.txt
litellm/proxy/_new_new_secret_config.yaml
litellm/proxy/custom_guardrail.py
**/.mypy_cache/
**/.mypy_cache_any/
litellm/proxy/application.log
tests/llm_translation/vertex_test_account.json
tests/llm_translation/test_vertex_key.json

12
CLAUDE.md
View file

@ -36,6 +36,12 @@ Don't hesitate to use values in .env to get needed API keys and other secrets, a
Run tests, format your code, and lint your code before each commit
When you fix violations gated by `ruff-strict-budget.json`, `mypy-code-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered baselines so the ceilings ratchet down instead of leaving stale headroom
If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and bringing it closer to the max, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
The Any-discipline gate (`make lint-any`, also a CI job) fails when a line you changed under `litellm/` holds a value typed `Any`, including the `X | Any`. Ideally `# any-ok: <reason>` is never used; treat it as a last resort for a genuine typed/untyped boundary that Pydantic truly can't model
Ask to commit and push your work when you're done (or if you're confident that your code is good and works, just do it)
When you must use real LLM models to, for example, write e2e tests, write a QA runbook, etc., make sure to use the latest models (doesn't have to be smartest, can also be a modern small, fast one. No strong preference for smart vs fast here, just use something modern) as of the year and month of the current date. Do a web search as necessary to figure that out
@ -57,11 +63,13 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
- Composition over inheritance
- Never-nester: early returns over deep nesting
- Don't throw; model failures as values (One function (e.g., raise_public) maps error union to existing public exception contracts via exhaustive match + assert_never)
- No mutation; instead of mutable lists and dicts, prefer tuples, NamedTuples, frozen dataclasses, etc.
- No mutation; don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), etc.
- Use dependency injection
- Fully typed; no `Any` or coarse types like dict[str, Any]. Every function parameter must be strongly typed
- Fully typed; no `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed
- Use tagged unions + match
- No monster files or god objects
- No file sprawl: deliberate file and folder structure
- Standard over hand-rolled: use the official SDK or a library where one exists; where none does, follow industry standards instead of inventing local conventions
Follow conventional commits for commit names and PR titles

1
CONTRIBUTING.md
View file

@ -155,6 +155,7 @@ Individual linting commands:
make format-check # Check Black formatting
make lint-ruff # Run Ruff linting
make lint-mypy # Run MyPy type checking
make lint-any # Fail on Any-typed values on changed lines
make check-circular-imports # Check for circular imports
make check-import-safety # Check import safety
```

38
Makefile
View file

@ -5,6 +5,8 @@
test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
info lint lint-dev format \
lint-mypy lint-mypy-budget-update lint-basedpyright lint-basedpyright-budget-update \
lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-any \
install-dev install-proxy-dev install-test-deps install-hooks \
install-helm-unittest check-circular-imports check-import-safety
@ -22,8 +24,15 @@ help:
@echo " make format-check - Check Black code formatting (matches CI)"
@echo " make lint - Run all linting (Ruff, MyPy, Black check, circular imports, import safety)"
@echo " make lint-ruff - Run Ruff linting only"
@echo " make lint-mypy - Run MyPy type checking only"
@echo " make lint-mypy - Run MyPy (disallow_untyped_defs), gated by per-rule error counts"
@echo " make lint-mypy-budget-update - Re-capture the MyPy per-rule budget (ratchet)"
@echo " make lint-basedpyright - Run basedpyright strict, gated by per-rule error counts"
@echo " make lint-basedpyright-budget-update - Re-capture the basedpyright per-rule budget (ratchet)"
@echo " make lint-black - Check Black formatting (matches CI)"
@echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its ceiling"
@echo " make lint-ruff-budget-update - Re-capture per-rule baselines in ruff-strict-budget.json (ratchet)"
@echo " make lint-budget-update - Re-capture all three ratchet budgets (ruff + mypy + basedpyright)"
@echo " make lint-any - Fail if changed lines under litellm/ hold an Any-typed value"
@echo " make check-circular-imports - Check for circular imports"
@echo " make check-import-safety - Check import safety"
@echo " make test - Run all tests"
@ -118,10 +127,31 @@ lint-ruff-FULL-dev: install-dev
else echo "No changed .py files to check."; fi
lint-mypy: install-dev
cd litellm && $(UV_RUN) mypy . --ignore-missing-imports && cd ..
cd litellm && ($(UV_RUN) mypy . || true) | $(UV_RUN) python ../scripts/type_check_gate.py --tool mypy
lint-mypy-budget-update: install-dev
cd litellm && ($(UV_RUN) mypy . || true) | $(UV_RUN) python ../scripts/type_check_gate.py --tool mypy --update
lint-basedpyright: install-dev
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --tool basedpyright
lint-basedpyright-budget-update: install-dev
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --tool basedpyright --update
lint-black: format-check
lint-ruff-budget: install-dev
$(UV_RUN) python scripts/ruff_strict_gate.py
lint-ruff-budget-update: install-dev
$(UV_RUN) python scripts/ruff_strict_gate.py --update
# Ratchet all three budgets in one shot (ruff strict + mypy + basedpyright)
lint-budget-update: lint-ruff-budget-update lint-mypy-budget-update lint-basedpyright-budget-update
lint-any: install-dev
$(UV_RUN) python scripts/check_any_discipline.py --changed
check-circular-imports: install-dev
cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd ..
@ -129,10 +159,10 @@ check-import-safety: install-dev
@$(UV_RUN) python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
# Combined linting (matches test-linting.yml workflow)
lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety
lint: format-check lint-ruff lint-mypy lint-basedpyright check-circular-imports check-import-safety lint-ruff-budget lint-any
# Faster linting for local development (only checks changed code)
lint-dev: lint-format-changed lint-mypy check-circular-imports check-import-safety
lint-dev: lint-format-changed lint-mypy lint-any check-circular-imports check-import-safety
# Testing targets
test: install-test-deps

1
README.md
View file

@ -327,6 +327,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
| [Maritalk (`maritalk`)](https://docs.litellm.ai/docs/providers/maritalk) | ✅ | ✅ | ✅ | | | | | | | |
| [Meta - Llama API (`meta_llama`)](https://docs.litellm.ai/docs/providers/meta_llama) | ✅ | ✅ | ✅ | | | | | | | |
| [Mistral AI API (`mistral`)](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [ModelScope (`modelscope`)](https://docs.litellm.ai/docs/providers/modelscope) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Moonshot (`moonshot`)](https://docs.litellm.ai/docs/providers/moonshot) | ✅ | ✅ | ✅ | | | | | | | |
| [Morph (`morph`)](https://docs.litellm.ai/docs/providers/morph) | ✅ | ✅ | ✅ | | | | | | | |
| [Nebius AI Studio (`nebius`)](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | | | | | | |

11
backend/main.py
View file

@ -20,7 +20,11 @@ DatabaseURLSettings.from_env().apply_to_env()
from litellm.proxy.proxy_server import app
from backend.routes.allowlist import BACKEND_EXACT_PATHS, BACKEND_PATH_PREFIXES
from backend.routes.allowlist import (
BACKEND_EXACT_PATHS,
BACKEND_MOUNT_PATHS,
BACKEND_PATH_PREFIXES,
)
def _is_backend_route(route) -> bool:
@ -29,8 +33,9 @@ def _is_backend_route(route) -> bool:
if path is None:
return False
if isinstance(route, Mount):
# Static UI mounts are served by the dedicated UI container, not here.
return False
# The dashboard UI static mounts are served by the dedicated UI container.
# Only Mounts in the backend allowlist (e.g. swagger docs) remain on backend.
return path in BACKEND_MOUNT_PATHS
if path in BACKEND_EXACT_PATHS:
return True
return any(path.startswith(prefix) for prefix in BACKEND_PATH_PREFIXES)

6
backend/routes/allowlist.py
View file

@ -133,3 +133,9 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
"/fallback/login",
}
)
BACKEND_MOUNT_PATHS: frozenset[str] = frozenset(
{
"/swagger", # API documentation static assets belong to the backend
}
)

194
basedpyright-code-budget.json Normal file
View file

@ -0,0 +1,194 @@
{
"reportAny": {
"baseline": 24954,
"slack": 10
},
"reportArgumentType": {
"baseline": 1863,
"slack": 3
},
"reportAssignmentType": {
"baseline": 220,
"slack": 3
},
"reportAttributeAccessIssue": {
"baseline": 335,
"slack": 3
},
"reportCallIssue": {
"baseline": 77,
"slack": 10
},
"reportConstantRedefinition": {
"baseline": 39,
"slack": 3
},
"reportDeprecated": {
"baseline": 217,
"slack": 10
},
"reportDuplicateImport": {
"baseline": 28,
"slack": 3
},
"reportExplicitAny": {
"baseline": 6931,
"slack": 10
},
"reportFunctionMemberAccess": {
"baseline": 7,
"slack": 3
},
"reportGeneralTypeIssues": {
"baseline": 151,
"slack": 3
},
"reportIncompatibleMethodOverride": {
"baseline": 52,
"slack": 10
},
"reportIncompatibleVariableOverride": {
"baseline": 8,
"slack": 3
},
"reportInconsistentOverload": {
"baseline": 12,
"slack": 3
},
"reportIndexIssue": {
"baseline": 26,
"slack": 3
},
"reportInvalidTypeForm": {
"baseline": 23,
"slack": 3
},
"reportInvalidTypeVarUse": {
"baseline": 2,
"slack": 3
},
"reportMatchNotExhaustive": {
"baseline": 1,
"slack": 3
},
"reportMissingParameterType": {
"baseline": 3933,
"slack": 10
},
"reportMissingTypeArgument": {
"baseline": 10612,
"slack": 10
},
"reportMissingTypeStubs": {
"baseline": 27,
"slack": 10
},
"reportOperatorIssue": {
"baseline": 6,
"slack": 3
},
"reportOptionalCall": {
"baseline": 4,
"slack": 3
},
"reportOptionalIterable": {
"baseline": 3,
"slack": 3
},
"reportOptionalMemberAccess": {
"baseline": 724,
"slack": 10
},
"reportOptionalOperand": {
"baseline": 3,
"slack": 3
},
"reportOptionalSubscript": {
"baseline": 11,
"slack": 3
},
"reportPossiblyUnboundVariable": {
"baseline": 52,
"slack": 10
},
"reportPrivateUsage": {
"baseline": 1625,
"slack": 10
},
"reportRedeclaration": {
"baseline": 8,
"slack": 3
},
"reportReturnType": {
"baseline": 118,
"slack": 10
},
"reportTypedDictNotRequiredAccess": {
"baseline": 20,
"slack": 3
},
"reportUndefinedVariable": {
"baseline": 2,
"slack": 3
},
"reportUnknownArgumentType": {
"baseline": 30603,
"slack": 10
},
"reportUnknownLambdaType": {
"baseline": 76,
"slack": 10
},
"reportUnknownMemberType": {
"baseline": 27322,
"slack": 10
},
"reportUnknownParameterType": {
"baseline": 13636,
"slack": 10
},
"reportUnknownVariableType": {
"baseline": 21776,
"slack": 10
},
"reportUnnecessaryCast": {
"baseline": 118,
"slack": 10
},
"reportUnnecessaryComparison": {
"baseline": 680,
"slack": 10
},
"reportUnnecessaryContains": {
"baseline": 4,
"slack": 3
},
"reportUnnecessaryIsInstance": {
"baseline": 807,
"slack": 10
},
"reportUntypedBaseClass": {
"baseline": 110,
"slack": 3
},
"reportUntypedFunctionDecorator": {
"baseline": 22,
"slack": 3
},
"reportUnusedClass": {
"baseline": 22,
"slack": 3
},
"reportUnusedFunction": {
"baseline": 137,
"slack": 10
},
"reportUnusedImport": {
"baseline": 670,
"slack": 10
},
"reportUnusedVariable": {
"baseline": 865,
"slack": 10
}
}

16
codecov.yaml
View file

@ -35,6 +35,22 @@ component_management:
- component_id: "Enterprise"
paths:
- "enterprise/**"
- component_id: "Batches"
paths:
- "*/proxy/batches_endpoints/**"
- "litellm/batches/**"
- "*/llms/*/batches/**"
- component_id: "Videos"
paths:
- "litellm/videos/**"
- "*/proxy/video_endpoints/**"
- "*/llms/*/videos/**"
- component_id: "Realtime"
paths:
- "litellm/realtime_api/**"
- "*/proxy/realtime_endpoints/**"
- "*/llms/*/realtime/**"
- "litellm/litellm_core_utils/realtime_streaming.py"
comment:
layout: "header, diff, flags, components" # show component info in the PR comment

11
litellm/__init__.py
View file

@ -73,6 +73,7 @@ from litellm.constants import (
replicate_models,
clarifai_models,
huggingface_models,
modelscope_models,
empower_models,
together_ai_models,
baseten_models,
@ -900,6 +901,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
heroku_models.add(key)
elif value.get("litellm_provider") == "dashscope":
dashscope_models.add(key)
elif value.get("litellm_provider") == "modelscope":
modelscope_models.add(key)
elif value.get("litellm_provider") == "moonshot":
moonshot_models.add(key)
elif value.get("litellm_provider") == "publicai":
@ -1019,6 +1022,7 @@ model_list = list(
| zai_models
| fal_ai_models
| deepseek_models
| modelscope_models
| azure_ai_models
| voyage_models
| infinity_models
@ -1152,6 +1156,7 @@ models_by_provider: dict = {
"elevenlabs": elevenlabs_models,
"heroku": heroku_models,
"dashscope": dashscope_models,
"modelscope": modelscope_models,
"moonshot": moonshot_models,
"publicai": publicai_models,
"v0": v0_models,
@ -1731,6 +1736,9 @@ if TYPE_CHECKING:
from .llms.voyage.embedding.transformation_contextual import (
VoyageContextualEmbeddingConfig as VoyageContextualEmbeddingConfig,
)
from .llms.voyage.embedding.transformation_multimodal import (
VoyageMultimodalEmbeddingConfig as VoyageMultimodalEmbeddingConfig,
)
from .llms.infinity.embedding.transformation import (
InfinityEmbeddingConfig as InfinityEmbeddingConfig,
)
@ -1972,6 +1980,9 @@ if TYPE_CHECKING:
from .llms.dashscope.rerank.transformation import (
DashScopeRerankConfig as DashScopeRerankConfig,
)
from .llms.modelscope.chat.transformation import (
ModelScopeChatConfig as ModelScopeChatConfig,
)
from .llms.moonshot.chat.transformation import (
MoonshotChatConfig as MoonshotChatConfig,
)

10
litellm/_lazy_imports_registry.py
View file

@ -223,6 +223,7 @@ LLM_CONFIG_NAMES = (
"GenAIHubOrchestrationConfig",
"VoyageEmbeddingConfig",
"VoyageContextualEmbeddingConfig",
"VoyageMultimodalEmbeddingConfig",
"InfinityEmbeddingConfig",
"PerplexityEmbeddingConfig",
"AzureAIStudioConfig",
@ -305,6 +306,7 @@ LLM_CONFIG_NAMES = (
"GigaChatConfig",
"GigaChatEmbeddingConfig",
"DashScopeChatConfig",
"ModelScopeChatConfig",
"MoonshotChatConfig",
"DockerModelRunnerChatConfig",
"V0ChatConfig",
@ -903,6 +905,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
".llms.voyage.embedding.transformation_contextual",
"VoyageContextualEmbeddingConfig",
),
"VoyageMultimodalEmbeddingConfig": (
".llms.voyage.embedding.transformation_multimodal",
"VoyageMultimodalEmbeddingConfig",
),
"InfinityEmbeddingConfig": (
".llms.infinity.embedding.transformation",
"InfinityEmbeddingConfig",
@ -1156,6 +1162,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
".llms.dashscope.chat.transformation",
"DashScopeChatConfig",
),
"ModelScopeChatConfig": (
".llms.modelscope.chat.transformation",
"ModelScopeChatConfig",
),
"MoonshotChatConfig": (".llms.moonshot.chat.transformation", "MoonshotChatConfig"),
"DockerModelRunnerChatConfig": (
".llms.docker_model_runner.chat.transformation",

2
litellm/_logging.py
View file

@ -419,7 +419,7 @@ def _enable_debugging():
def print_verbose(print_statement):
try:
if set_verbose:
print(redact_secrets(str(print_statement))) # noqa
print(redact_secrets(str(print_statement))) # noqa: T201
except Exception:
pass

2
litellm/_redis.py
View file

@ -567,7 +567,7 @@ def get_redis_client(**env_overrides):
return redis.Redis(**redis_kwargs)
def get_redis_async_client( # noqa: PLR0915
def get_redis_async_client(
connection_pool: Optional[async_redis.BlockingConnectionPool] = None,
**env_overrides,
) -> Union[async_redis.Redis, async_redis.RedisCluster]:

4
litellm/anthropic_beta_headers_config.json
View file

@ -75,7 +75,7 @@
"effort-2025-11-24": "effort-2025-11-24",
"fast-mode-2026-02-01": null,
"files-api-2025-04-14": null,
"fine-grained-tool-streaming-2025-05-14": null,
"fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
"interleaved-thinking-2025-05-14": null,
"mcp-client-2025-11-20": null,
"mcp-client-2025-04-04": null,
@ -106,7 +106,7 @@
"effort-2025-11-24": "effort-2025-11-24",
"fast-mode-2026-02-01": null,
"files-api-2025-04-14": null,
"fine-grained-tool-streaming-2025-05-14": null,
"fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
"interleaved-thinking-2025-05-14": null,
"mcp-client-2025-11-20": null,
"mcp-client-2025-04-04": null,

4
litellm/caching/caching.py
View file

@ -27,7 +27,7 @@ from litellm.types.utils import EmbeddingResponse, all_litellm_params
from .azure_blob_cache import AzureBlobCache
from .base_cache import BaseCache
from .disk_cache import DiskCache
from .dual_cache import DualCache # noqa
from .dual_cache import DualCache # noqa: F401
from .gcs_cache import GCSCache
from .in_memory_cache import InMemoryCache
from .qdrant_semantic_cache import QdrantSemanticCache
@ -41,7 +41,7 @@ def print_verbose(print_statement):
try:
verbose_logger.debug(print_statement)
if litellm.set_verbose:
print(print_statement) # noqa
print(print_statement) # noqa: T201
except Exception:
pass

10
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -693,7 +693,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
original_response = model_call_details.get("original_response")
return cls._recover_output_items_from_raw_sse(original_response)
def transform_response( # noqa: PLR0915
def transform_response(
self,
model: str,
raw_response: "BaseModel",
@ -1293,9 +1293,15 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
provider_specific_fields
)
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
tool_call_index = parsed_chunk.get("output_index", 0)
tool_call_chunk = ChatCompletionToolCallChunk(
id=output_item.get("call_id"),
id=LiteLLMCompletionResponsesConfig._tool_call_id_from_responses_item(
output_item.get("id"), output_item.get("call_id")
),
index=tool_call_index,
type="function",
function=function_chunk,

52
litellm/constants.py
View file

@ -190,6 +190,10 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE = int(
# Override with LITELLM_MAX_CALLBACKS env var for large deployments (e.g., many teams with guardrails)
MAX_CALLBACKS = get_env_int("LITELLM_MAX_CALLBACKS", 100)
# Metadata key recording which pre_call guardrails the proxy loop already ran,
# so the deployment-level hook does not re-run them for the same request
PRE_CALL_EXECUTED_GUARDRAILS_KEY = "_pre_call_executed_guardrails"
# Generic fallback for unknown models
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET = int(
os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET", 128)
@ -618,6 +622,7 @@ LITELLM_CHAT_PROVIDERS = [
"nscale",
"nebius",
"dashscope",
"modelscope",
"moonshot",
"publicai",
"v0",
@ -776,6 +781,7 @@ openai_compatible_endpoints: List = [
"inference.api.nscale.com/v1",
"api.studio.nebius.ai/v1",
"https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
"https://api-inference.modelscope.cn/v1",
"https://api.moonshot.ai/v1",
"https://api.publicai.co/v1",
"https://api.synthetic.new/openai/v1",
@ -793,6 +799,7 @@ openai_compatible_endpoints: List = [
"https://ai-gateway.vercel.sh/v1",
"https://api.inference.wandb.ai/v1",
"https://api.clarifai.com/v2/ext/openai/v1",
"https://api.libertai.io/v1",
]
@ -836,10 +843,12 @@ openai_compatible_providers: List = [
"poe", # Poe - JSON-configured provider
"chutes", # Chutes - JSON-configured provider
"parasail", # Parasail - JSON-configured provider
"libertai", # LibertAI - JSON-configured provider
"featherless_ai",
"nscale",
"nebius",
"dashscope",
"modelscope",
"moonshot",
"v0",
"helicone",
@ -865,6 +874,7 @@ openai_text_completion_compatible_providers: List = (
"featherless_ai",
"nebius",
"dashscope",
"modelscope",
"moonshot",
"publicai",
"synthetic",
@ -1125,6 +1135,48 @@ WANDB_MODELS: set = set(
]
)
modelscope_models: set = set(
[
# Qwen series models
"Qwen/Qwen3-0.6B",
"Qwen/Qwen3-1.7B",
"Qwen/Qwen3-4B",
"Qwen/Qwen3-8B",
"Qwen/Qwen3-14B",
"Qwen/Qwen3-30B-A3B",
"Qwen/Qwen3-32B",
"Qwen/Qwen3-235B-A22B",
"Qwen/Qwen3-235B-A22B-Instruct-2507",
"Qwen/Qwen3-235B-A22B-Thinking-2507",
"Qwen/Qwen3-30B-A3B-Thinking-2507",
"Qwen/Qwen3-Coder-30B-A3B-Instruct",
"Qwen/Qwen3-Coder-480B-A35B-Instruct",
"Qwen/Qwen3-Next-80B-A3B-Instruct",
"Qwen/Qwen3-Next-80B-A3B-Thinking",
"Qwen/Qwen3-VL-235B-A22B-Instruct",
"Qwen/Qwen3-VL-8B-Instruct",
"Qwen/Qwen3-VL-8B-Thinking",
"Qwen/Qwen3.5-122B-A10B",
"Qwen/Qwen3.5-27B",
"Qwen/Qwen3.5-35B-A3B",
"Qwen/Qwen3.5-397B-A17B",
"Qwen/QwQ-32B",
"Qwen/QwQ-32B-Preview",
"Qwen/QVQ-72B-Preview",
"Qwen/Qwen-Image-Edit",
# DeepSeek series models
"deepseek-ai/DeepSeek-R1-0528",
"deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
"deepseek-ai/DeepSeek-R1-Distill-Llama-8B",
"deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B",
"deepseek-ai/DeepSeek-R1-Distill-Qwen-14B",
"deepseek-ai/DeepSeek-R1-Distill-Qwen-32B",
"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B",
"deepseek-ai/DeepSeek-V3.2",
"deepseek-ai/DeepSeek-V4-Flash",
]
)
BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
"cohere",
"anthropic",

30
litellm/integrations/SlackAlerting/hanging_request_check.py
View file

@ -8,6 +8,7 @@ Notes:
"""
import asyncio
import time
from typing import TYPE_CHECKING, Any, Optional
import litellm
@ -36,11 +37,15 @@ class AlertingHangingRequestCheck:
slack_alerting_object: SlackAlerting,
):
self.slack_alerting_object = slack_alerting_object
# checks run every alerting_threshold / 2 seconds, so entries must
# stay cached for at least 1.5x the threshold to guarantee a check
# happens after they cross it
self.hanging_request_cache_ttl = int(
self.slack_alerting_object.alerting_threshold * 1.5
+ HANGING_ALERT_BUFFER_TIME_SECONDS
)
self.hanging_request_cache = InMemoryCache(
default_ttl=int(
self.slack_alerting_object.alerting_threshold
+ HANGING_ALERT_BUFFER_TIME_SECONDS
),
default_ttl=self.hanging_request_cache_ttl,
)
async def add_request_to_hanging_request_check(
@ -76,10 +81,7 @@ class AlertingHangingRequestCheck:
await self.hanging_request_cache.async_set_cache(
key=hanging_request_data.request_id,
value=hanging_request_data,
ttl=int(
self.slack_alerting_object.alerting_threshold
+ HANGING_ALERT_BUFFER_TIME_SECONDS
),
ttl=self.hanging_request_cache_ttl,
)
return
@ -111,6 +113,9 @@ class AlertingHangingRequestCheck:
if hanging_request_data is None:
continue
if hanging_request_data.alerted:
continue
request_status = (
await proxy_logging_obj.internal_usage_cache.async_get_cache(
key="request_status:{}".format(hanging_request_data.request_id),
@ -127,12 +132,21 @@ class AlertingHangingRequestCheck:
)
continue
request_age_seconds = time.time() - hanging_request_data.created_at
if request_age_seconds < self.slack_alerting_object.alerting_threshold:
# in-flight but below the alerting threshold; keep it cached
# so a later check can alert if it never completes
continue
################
# Send the Alert on Slack
################
await self.send_hanging_request_alert(
hanging_request_data=hanging_request_data
)
# flag so the entry is skipped on later ticks; one alert per hang,
# with the existing TTL still handling cleanup
hanging_request_data.alerted = True
return

2
litellm/integrations/SlackAlerting/slack_alerting.py
View file

@ -1179,7 +1179,7 @@ Model Info:
if response.status_code == 200:
return True
else:
print("Error sending webhook alert. Error=", response.text) # noqa
print("Error sending webhook alert. Error=", response.text) # noqa: T201
return False

155
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -27,6 +27,11 @@ else:
LiteLLMLoggingObj = Any
# Anthropic (and Bedrock Claude) reject requests with more than 4 cache_control
# breakpoints: "A maximum of 4 blocks with cache_control may be provided."
MAX_CACHE_CONTROL_BLOCKS = 4
class AnthropicCacheControlHook(CustomPromptManagement):
def get_chat_completion_prompt(
self,
@ -61,16 +66,30 @@ class AnthropicCacheControlHook(CustomPromptManagement):
processed_messages = copy.deepcopy(messages)
# Separate message-level and non-message-level injection points
remaining_points = []
message_points: List[CacheControlMessageInjectionPoint] = []
remaining_points: List[CacheControlInjectionPoint] = []
for point in injection_points:
if point.get("location") == "message":
point = cast(CacheControlMessageInjectionPoint, point)
processed_messages = self._process_message_injection(
point=point, messages=processed_messages
)
message_points.append(cast(CacheControlMessageInjectionPoint, point))
else:
remaining_points.append(point)
# Non-message points (currently Bedrock tool_config) are handled in the
# provider transform, where each tool_config point appends at most one
# cachePoint to the tools. That block also counts toward Anthropic's
# limit, so reserve a slot for it here to leave room.
reserved_blocks = (
1
if any(p.get("location") == "tool_config" for p in remaining_points)
else 0
)
processed_messages = self._apply_message_injections(
points=message_points,
messages=processed_messages,
max_blocks=MAX_CACHE_CONTROL_BLOCKS - reserved_blocks,
)
# Pass through non-message injection points for provider-specific handling
if remaining_points:
non_default_params["cache_control_injection_points"] = remaining_points
@ -78,14 +97,71 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return model, processed_messages, non_default_params
@staticmethod
def _process_message_injection(
point: CacheControlMessageInjectionPoint, messages: List[AllMessageValues]
def _apply_message_injections(
points: List[CacheControlMessageInjectionPoint],
messages: List[AllMessageValues],
max_blocks: int,
) -> List[AllMessageValues]:
"""Process message-level cache control injection."""
control: ChatCompletionCachedContent = point.get(
"control", None
) or ChatCompletionCachedContent(type="ephemeral")
"""Apply message-level cache control injection points in order.
Anthropic allows at most ``MAX_CACHE_CONTROL_BLOCKS`` cache_control
breakpoints per request. Client-supplied breakpoints count toward that
limit, so we never inject onto a message that already carries
cache_control (preserving the client's TTL) and we stop injecting once
``max_blocks`` is reached. Injection points are honored in config order,
so earlier points win when slots are scarce.
"""
used_blocks = sum(
AnthropicCacheControlHook._count_cache_control_blocks(msg)
for msg in messages
)
limit_reached = False
for point in points:
if used_blocks >= max_blocks:
limit_reached = True
break
control: ChatCompletionCachedContent = point.get(
"control", None
) or ChatCompletionCachedContent(type="ephemeral")
for target_index in AnthropicCacheControlHook._resolve_target_indices(
point=point, messages=messages
):
if used_blocks >= max_blocks:
limit_reached = True
break
if AnthropicCacheControlHook._message_has_cache_control(
messages[target_index]
):
# Client already marked this message; don't overwrite it.
continue
messages[target_index] = (
AnthropicCacheControlHook._safe_insert_cache_control_in_message(
messages[target_index], control
)
)
used_blocks += 1
if limit_reached:
break
if limit_reached:
verbose_logger.warning(
f"AnthropicCacheControlHook: Reached the Anthropic limit of "
f"{MAX_CACHE_CONTROL_BLOCKS} cache_control blocks. Skipping further injection."
)
return messages
@staticmethod
def _resolve_target_indices(
point: CacheControlMessageInjectionPoint, messages: List[AllMessageValues]
) -> List[int]:
"""Resolve which message indices an injection point targets."""
_targetted_index: Optional[Union[int, str]] = point.get("index", None)
targetted_index: Optional[int] = None
if isinstance(_targetted_index, str):
@ -96,36 +172,49 @@ class AnthropicCacheControlHook(CustomPromptManagement):
else:
targetted_index = _targetted_index
targetted_role = point.get("role", None)
# Case 1: Target by specific index
if targetted_index is not None:
original_index = targetted_index
# Handle negative indices (convert to positive)
if targetted_index < 0:
targetted_index += len(messages)
if 0 <= targetted_index < len(messages):
messages[targetted_index] = (
AnthropicCacheControlHook._safe_insert_cache_control_in_message(
messages[targetted_index], control
)
)
else:
verbose_logger.warning(
f"AnthropicCacheControlHook: Provided index {original_index} is out of bounds for message list of length {len(messages)}. "
f"Targeted index was {targetted_index}. Skipping cache control injection for this point."
)
return [targetted_index]
verbose_logger.warning(
f"AnthropicCacheControlHook: Provided index {original_index} is out of bounds for message list of length {len(messages)}. "
f"Targeted index was {targetted_index}. Skipping cache control injection for this point."
)
return []
# Case 2: Target by role
elif targetted_role is not None:
for msg in messages:
if msg.get("role") == targetted_role:
msg = (
AnthropicCacheControlHook._safe_insert_cache_control_in_message(
message=msg, control=control
)
)
return messages
targetted_role = point.get("role", None)
if targetted_role is not None:
return [
idx
for idx, msg in enumerate(messages)
if msg.get("role") == targetted_role
]
return []
@staticmethod
def _count_cache_control_blocks(message: AllMessageValues) -> int:
"""Count cache_control breakpoints on a message (message + content level)."""
count = 0
if message.get("cache_control") is not None:
count += 1
content = message.get("content")
if isinstance(content, list):
for block in content:
if isinstance(block, dict) and block.get("cache_control") is not None:
count += 1
return count
@staticmethod
def _message_has_cache_control(message: AllMessageValues) -> bool:
"""Return True if the message already carries any cache_control."""
return AnthropicCacheControlHook._count_cache_control_blocks(message) > 0
@staticmethod
def _safe_insert_cache_control_in_message(

57
litellm/integrations/custom_guardrail.py
View file

@ -1,3 +1,4 @@
import secrets
from datetime import datetime
from typing import (
TYPE_CHECKING,
@ -43,6 +44,7 @@ if TYPE_CHECKING:
dc = DualCache()
from litellm.constants import PRE_CALL_EXECUTED_GUARDRAILS_KEY
from litellm.exceptions import (
BlockedPiiEntityError,
GuardrailRaisedException,
@ -50,6 +52,12 @@ from litellm.exceptions import (
SensitiveDataRouteException,
)
# Per-process secret tagging each recorded marker. The deployment hook only
# honors markers carrying this token, so a caller cannot forge the metadata
# field to suppress a guardrail on the direct-SDK path that never reaches the
# proxy's metadata sanitizer.
_PRE_CALL_EXECUTED_TOKEN = secrets.token_hex(16)
def get_session_id_from_request_data(request_data: Dict[str, Any]) -> Optional[str]:
"""Extract session_id from request data (litellm_session_id or metadata)."""
@ -458,6 +466,49 @@ class CustomGuardrail(CustomLogger):
return False
def _pre_call_marker(self) -> Optional[str]:
name = self.guardrail_name
if not name:
return None
return f"{_PRE_CALL_EXECUTED_TOKEN}:{name}"
def mark_pre_call_hook_ran(self, data: Dict[str, Any]) -> None:
"""
Record that this guardrail's ``async_pre_call_hook`` already ran for this
request, so the deployment-level hook does not run it a second time.
The proxy runs pre-call guardrails in ``ProxyLogging.pre_call_hook``. The
router later spreads a deployment's model-level ``guardrails`` into the
top-level request kwargs, which would otherwise re-trigger the same hook
from ``async_pre_call_deployment_hook``.
"""
marker = self._pre_call_marker()
if marker is None:
return
for meta_key in ("metadata", "litellm_metadata"):
meta = data.get(meta_key)
if isinstance(meta, dict):
executed = meta.get(PRE_CALL_EXECUTED_GUARDRAILS_KEY)
if isinstance(executed, list):
if marker not in executed:
executed.append(marker)
else:
meta[PRE_CALL_EXECUTED_GUARDRAILS_KEY] = [marker]
return
data["metadata"] = {PRE_CALL_EXECUTED_GUARDRAILS_KEY: [marker]}
def _pre_call_hook_already_ran(self, data: Dict[str, Any]) -> bool:
marker = self._pre_call_marker()
if marker is None:
return False
for meta_key in ("metadata", "litellm_metadata"):
meta = data.get(meta_key)
if isinstance(meta, dict):
executed = meta.get(PRE_CALL_EXECUTED_GUARDRAILS_KEY)
if isinstance(executed, list) and marker in executed:
return True
return False
async def async_pre_call_deployment_hook(
self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
) -> Optional[dict]:
@ -468,6 +519,9 @@ class CustomGuardrail(CustomLogger):
if litellm_guardrails is None or not isinstance(litellm_guardrails, list):
return kwargs
if self._pre_call_hook_already_ran(kwargs):
return kwargs
if (
self.should_run_guardrail(
data=kwargs, event_type=GuardrailEventHooks.pre_call
@ -567,6 +621,9 @@ class CustomGuardrail(CustomLogger):
):
return False
if self.default_on is True and disable_global_guardrail is True:
return False
if self.default_on is True and disable_global_guardrail is not True:
if self._event_hook_is_event_type(event_type):
if isinstance(self.event_hook, Mode):

81
litellm/integrations/datadog/datadog.py
View file

@ -92,12 +92,26 @@ class DataDogLogger(
# Class variables or attributes
def __init__(
self,
dd_api_key: Optional[str] = None,
dd_site: Optional[str] = None,
dd_agent_host: Optional[str] = None,
dd_agent_port: Optional[str] = None,
allow_env_credentials: bool = True,
**kwargs,
):
"""
Initializes the datadog logger, checks if the correct env variables are set
Required environment variables (Direct API):
Args:
dd_api_key: Datadog API key. Falls back to DD_API_KEY env var when allow_env_credentials is True.
dd_site: Datadog site (e.g. "us5.datadoghq.com"). Falls back to DD_SITE env var.
dd_agent_host: Hostname or IP of DataDog agent. Falls back to LITELLM_DD_AGENT_HOST env var.
dd_agent_port: Port of DataDog agent (default: 10518). Falls back to LITELLM_DD_AGENT_PORT env var.
allow_env_credentials: When False, the API key is never read from DD_API_KEY env var. Set to
False for team/key-scoped loggers whose destination (dd_agent_host/dd_site) is caller-supplied,
so the proxy's global DD_API_KEY is never sent to an untrusted host.
Required environment variables (Direct API) when kwargs not provided:
`DD_API_KEY` - your datadog api key
`DD_SITE` - your datadog site, example = `"us5.datadoghq.com"`
@ -130,12 +144,21 @@ class DataDogLogger(
)
# Configure DataDog endpoint (Agent or Direct API)
# Use LITELLM_DD_AGENT_HOST to avoid conflicts with ddtrace's DD_AGENT_HOST
dd_agent_host = os.getenv("LITELLM_DD_AGENT_HOST")
if dd_agent_host:
self._configure_dd_agent(dd_agent_host=dd_agent_host)
# Prefer explicit kwargs, then fall back to env vars
resolved_agent_host = dd_agent_host or os.getenv("LITELLM_DD_AGENT_HOST")
if resolved_agent_host:
self._configure_dd_agent(
dd_agent_host=resolved_agent_host,
dd_agent_port=dd_agent_port,
dd_api_key=dd_api_key,
allow_env_credentials=allow_env_credentials,
)
else:
self._configure_dd_direct_api()
self._configure_dd_direct_api(
dd_api_key=dd_api_key,
dd_site=dd_site,
allow_env_credentials=allow_env_credentials,
)
# Optional override for testing
dd_base_url = get_datadog_base_url_from_env()
@ -172,34 +195,60 @@ class DataDogLogger(
).model_dump()
return dict_datadog_params
def _configure_dd_agent(self, dd_agent_host: str) -> None:
def _configure_dd_agent(
self,
dd_agent_host: str,
dd_agent_port: Optional[str] = None,
dd_api_key: Optional[str] = None,
allow_env_credentials: bool = True,
) -> None:
"""
Configure DataDog Agent for log forwarding
Args:
dd_agent_host: Hostname or IP of DataDog agent
dd_agent_port: Port of DataDog agent. Falls back to LITELLM_DD_AGENT_PORT env var (default: 10518).
dd_api_key: Datadog API key. Falls back to DD_API_KEY env var when allow_env_credentials is True. Optional when using agent.
allow_env_credentials: When False, never read the API key from DD_API_KEY env var.
"""
dd_agent_port = os.getenv(
resolved_port = dd_agent_port or os.getenv(
"LITELLM_DD_AGENT_PORT", "10518"
) # default port for logs
self.intake_url = f"http://{dd_agent_host}:{dd_agent_port}/api/v2/logs"
self.DD_API_KEY = os.getenv("DD_API_KEY") # Optional when using agent
self.intake_url = f"http://{dd_agent_host}:{resolved_port}/api/v2/logs"
self.DD_API_KEY = dd_api_key or (
os.getenv("DD_API_KEY") if allow_env_credentials else None
) # Optional when using agent
verbose_logger.debug(f"Datadog: Using DD Agent at {self.intake_url}")
def _configure_dd_direct_api(self) -> None:
def _configure_dd_direct_api(
self,
dd_api_key: Optional[str] = None,
dd_site: Optional[str] = None,
allow_env_credentials: bool = True,
) -> None:
"""
Configure direct DataDog API connection
Args:
dd_api_key: Datadog API key. Falls back to DD_API_KEY env var when allow_env_credentials is True.
dd_site: Datadog site. Falls back to DD_SITE env var.
allow_env_credentials: When False, never read the API key from DD_API_KEY env var.
Raises:
Exception: If required environment variables are not set
Exception: If required credentials are not provided via args or env vars
"""
if os.getenv("DD_API_KEY", None) is None:
resolved_api_key = dd_api_key or (
os.getenv("DD_API_KEY") if allow_env_credentials else None
)
resolved_site = dd_site or os.getenv("DD_SITE")
if resolved_api_key is None:
raise Exception("DD_API_KEY is not set, set 'DD_API_KEY=<>")
if os.getenv("DD_SITE", None) is None:
if resolved_site is None:
raise Exception("DD_SITE is not set in .env, set 'DD_SITE=<>")
self.DD_API_KEY = os.getenv("DD_API_KEY")
self.intake_url = f"https://http-intake.logs.{os.getenv('DD_SITE')}/api/v2/logs"
self.DD_API_KEY = resolved_api_key
self.intake_url = f"https://http-intake.logs.{resolved_site}/api/v2/logs"
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
"""

124
litellm/integrations/datadog/datadog_team_handler.py Normal file
View file

@ -0,0 +1,124 @@
"""
DataDog Team Handler
Used to get the DataDogLogger for a given request.
Handles Key/Team Based Datadog Logging, following the same pattern as LangFuseHandler.
"""
from typing import TYPE_CHECKING, Any, Dict, Optional, TypedDict
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import StandardCallbackDynamicParams
from .datadog import DataDogLogger
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import DynamicLoggingCache
else:
DynamicLoggingCache = Any
class DatadogLoggingConfig(TypedDict):
dd_api_key: Optional[str]
dd_site: Optional[str]
dd_agent_host: Optional[str]
dd_agent_port: Optional[str]
class DataDogHandler:
@staticmethod
def get_datadog_logger_for_request(
standard_callback_dynamic_params: StandardCallbackDynamicParams,
in_memory_dynamic_logger_cache: DynamicLoggingCache,
) -> DataDogLogger:
"""
Get a team-scoped DataDogLogger for a given request.
Resolves and caches per-team DataDogLogger instances using DynamicLoggingCache,
keyed by the team's DD credentials. Each unique set of credentials gets its own
logger instance with its own batch/flush loop.
Note: This handler is only called when team-scoped DD credentials are present.
The global (env-var based) DataDogLogger is managed separately by
_init_custom_logger_compatible_class via _in_memory_loggers.
"""
_credentials = DataDogHandler.get_dynamic_datadog_logging_config(
standard_callback_dynamic_params=standard_callback_dynamic_params,
)
credentials_dict = dict(_credentials)
# check if datadog logger is already cached
temp_datadog_logger = in_memory_dynamic_logger_cache.get_cache(
credentials=credentials_dict, service_name="datadog"
)
# if not cached, create a new datadog logger and cache it
if temp_datadog_logger is None:
temp_datadog_logger = (
DataDogHandler._create_datadog_logger_from_credentials(
credentials=credentials_dict,
in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
)
)
return temp_datadog_logger
@staticmethod
def _create_datadog_logger_from_credentials(
credentials: Dict,
in_memory_dynamic_logger_cache: DynamicLoggingCache,
) -> DataDogLogger:
"""
Create a DataDogLogger from the credentials and cache it.
"""
# When the destination is caller-supplied (dd_agent_host/dd_site), never fall back to the
# proxy's DD_API_KEY env var, otherwise it would be sent to a team-controlled host.
allow_env_credentials = (
credentials.get("dd_agent_host") is None
and credentials.get("dd_site") is None
)
datadog_logger = DataDogLogger(
dd_api_key=credentials.get("dd_api_key"),
dd_site=credentials.get("dd_site"),
dd_agent_host=credentials.get("dd_agent_host"),
dd_agent_port=credentials.get("dd_agent_port"),
allow_env_credentials=allow_env_credentials,
)
in_memory_dynamic_logger_cache.set_cache(
credentials=credentials,
service_name="datadog",
logging_obj=datadog_logger,
)
verbose_logger.debug(
"Datadog: Created and cached new DataDogLogger for team-scoped credentials"
)
return datadog_logger
@staticmethod
def get_dynamic_datadog_logging_config(
standard_callback_dynamic_params: StandardCallbackDynamicParams,
) -> DatadogLoggingConfig:
"""
Get the Datadog logging config for a given request from dynamic params.
"""
return DatadogLoggingConfig(
dd_api_key=standard_callback_dynamic_params.get("dd_api_key"),
dd_site=standard_callback_dynamic_params.get("dd_site"),
dd_agent_host=standard_callback_dynamic_params.get("dd_agent_host"),
dd_agent_port=standard_callback_dynamic_params.get("dd_agent_port"),
)
@staticmethod
def _dynamic_datadog_credentials_are_passed(
standard_callback_dynamic_params: StandardCallbackDynamicParams,
) -> bool:
"""
Check if dynamic Datadog credentials are passed in standard_callback_dynamic_params.
"""
if (
standard_callback_dynamic_params.get("dd_api_key") is not None
or standard_callback_dynamic_params.get("dd_site") is not None
or standard_callback_dynamic_params.get("dd_agent_host") is not None
):
return True
return False

1
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -43,6 +43,7 @@ class LangfuseOtelLogger(OpenTelemetry):
"""
_utils.set_attributes(span, kwargs, response_obj, LangfuseLLMObsOTELAttributes)
span.set_attribute("langfuse.observation.type", "generation")
#########################################################
# Set Langfuse specific attributes

6
litellm/integrations/lunary.py
View file

@ -75,16 +75,16 @@ class LunaryLogger:
version = importlib.metadata.version("lunary") # type: ignore
# if version < 0.1.43 then raise ImportError
if packaging.version.Version(version) < packaging.version.Version("0.1.43"): # type: ignore
print( # noqa
print( # noqa: T201
"Lunary version outdated. Required: >= 0.1.43. Upgrade via 'pip install lunary --upgrade'"
)
raise ImportError
self.lunary_client = lunary
except ImportError:
print( # noqa
print( # noqa: T201
"Lunary not installed. Please install it using 'pip install lunary'"
) # noqa
)
raise ImportError
def log_event(

168
litellm/integrations/opentelemetry.py
View file

@ -1,7 +1,18 @@
import os
from dataclasses import dataclass, field
from datetime import datetime
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union, cast
from typing import (
TYPE_CHECKING,
Any,
Dict,
FrozenSet,
List,
Optional,
Set,
Tuple,
Union,
cast,
)
import litellm
from litellm._logging import verbose_logger
@ -82,6 +93,88 @@ _VALID_CAPTURE_MODES = {
CAPTURE_MODE_SPAN_AND_EVENT,
}
METRIC_METADATA_KEYS: Tuple[str, ...] = (
"user_api_key_hash",
"user_api_key_alias",
"user_api_key_team_id",
"user_api_key_org_id",
"user_api_key_user_id",
"user_api_key_team_alias",
"user_api_key_user_email",
"spend_logs_metadata",
"requester_ip_address",
"requester_metadata",
"user_api_key_end_user_id",
"prompt_management_metadata",
"applied_guardrails",
"mcp_tool_call_metadata",
"vector_store_request_metadata",
)
TOKEN_TYPE_ATTRIBUTE: str = "gen_ai.token.type"
VALID_METRIC_ATTRIBUTE_NAMES: FrozenSet[str] = frozenset(
(
"gen_ai.operation.name",
"gen_ai.system",
"gen_ai.request.model",
"gen_ai.framework",
"hidden_params",
)
+ tuple(f"metadata.{key}" for key in METRIC_METADATA_KEYS)
)
@dataclass(frozen=True)
class OTELMetricAttributeFilter:
include_list: Optional[List[str]] = None
exclude_list: Optional[List[str]] = None
def _build_metric_attribute_filter(value: Any) -> OTELMetricAttributeFilter:
if isinstance(value, OTELMetricAttributeFilter):
return value
if not isinstance(value, dict):
raise ValueError(
"otel.attributes must be a mapping with optional 'include_list' / "
f"'exclude_list', got {type(value).__name__}"
)
return OTELMetricAttributeFilter(
include_list=value.get("include_list"),
exclude_list=value.get("exclude_list"),
)
def _resolve_metric_attribute_filter(
attributes: Optional[OTELMetricAttributeFilter],
) -> Tuple[Optional[FrozenSet[str]], Optional[FrozenSet[str]]]:
if attributes is None:
return None, None
include = attributes.include_list or None
exclude = attributes.exclude_list or None
if include and exclude:
raise ValueError(
"otel.attributes: include_list and exclude_list are mutually exclusive"
)
requested = include or exclude or []
if TOKEN_TYPE_ATTRIBUTE in requested:
raise ValueError(
f"otel.attributes: {TOKEN_TYPE_ATTRIBUTE} is a structural token-usage "
"discriminator and cannot be filtered"
)
unknown = sorted(
name for name in requested if name not in VALID_METRIC_ATTRIBUTE_NAMES
)
if unknown:
raise ValueError(
f"otel.attributes: unknown attribute name(s) {unknown}. "
f"Valid names: {sorted(VALID_METRIC_ATTRIBUTE_NAMES)}"
)
return (
frozenset(include) if include else None,
frozenset(exclude) if exclude else None,
)
def _normalize_team_metadata_keys(value: Any) -> List[str]:
"""Coerce a team-metadata allowlist from a list or comma-separated string.
@ -117,6 +210,9 @@ class OpenTelemetryConfig:
# under ``litellm.team.metadata``. Empty by default so none of a team's
# metadata leaves the process until explicitly allowlisted.
baggage_team_metadata_keys: List[str] = field(default_factory=list)
# Prometheus-style include/exclude control over which attributes are stamped
# on emitted metrics, to cap metric cardinality.
attributes: Optional[OTELMetricAttributeFilter] = None
def __post_init__(self) -> None:
# If endpoint is specified but exporter is still the default "console",
@ -211,15 +307,29 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
**kwargs,
):
team_metadata_keys_override = kwargs.pop("baggage_team_metadata_keys", None)
metric_attributes_override = kwargs.pop("attributes", None)
if config is None:
config = OpenTelemetryConfig.from_env()
if team_metadata_keys_override is not None:
config.baggage_team_metadata_keys = _normalize_team_metadata_keys(
team_metadata_keys_override
)
if metric_attributes_override is not None:
config.attributes = _build_metric_attribute_filter(
metric_attributes_override
)
self.config = config
self.callback_name = callback_name
# Resolved on first metric record, not here: the proxy populates
# callback_settings.otel.attributes after this logger is constructed, so
# reading it now would miss it. An explicit config is validated eagerly so
# a bad config still fails at startup.
self._metric_attr_include: Optional[FrozenSet[str]] = None
self._metric_attr_exclude: Optional[FrozenSet[str]] = None
self._metric_attr_filter_resolved = False
if config.attributes is not None:
self._ensure_metric_attribute_filter()
self.OTEL_EXPORTER = self.config.exporter
self.OTEL_ENDPOINT = self.config.endpoint
self.OTEL_HEADERS = self.config.headers
@ -1318,6 +1428,38 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
return None
return safe_dumps(filtered)
def _ensure_metric_attribute_filter(self) -> None:
"""Resolve the include/exclude filter once, falling back to the proxy's
callback_settings.otel.attributes when no explicit config was passed."""
if self._metric_attr_filter_resolved:
return
attributes = self.config.attributes
if attributes is None and self.callback_name in (None, "otel"):
otel_settings = (litellm.callback_settings or {}).get("otel") or {}
raw = (
otel_settings.get("attributes")
if isinstance(otel_settings, dict)
else None
)
if raw is not None:
attributes = _build_metric_attribute_filter(raw)
(
self._metric_attr_include,
self._metric_attr_exclude,
) = _resolve_metric_attribute_filter(attributes)
self._metric_attr_filter_resolved = True
def _filter_metric_attributes(self, attrs: Dict[str, Any]) -> Dict[str, Any]:
if not self._metric_attr_filter_resolved:
self._ensure_metric_attribute_filter()
if self._metric_attr_include is not None:
return {k: v for k, v in attrs.items() if k in self._metric_attr_include}
if self._metric_attr_exclude is not None:
return {
k: v for k, v in attrs.items() if k not in self._metric_attr_exclude
}
return attrs
def _record_metrics(self, kwargs, response_obj, start_time, end_time):
duration_s = (end_time - start_time).total_seconds()
params = kwargs.get("litellm_params") or {}
@ -1336,23 +1478,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
std_log = kwargs.get("standard_logging_object")
md = getattr(std_log, "metadata", None) or (std_log or {}).get("metadata", {})
for key in [
"user_api_key_hash",
"user_api_key_alias",
"user_api_key_team_id",
"user_api_key_org_id",
"user_api_key_user_id",
"user_api_key_team_alias",
"user_api_key_user_email",
"spend_logs_metadata",
"requester_ip_address",
"requester_metadata",
"user_api_key_end_user_id",
"prompt_management_metadata",
"applied_guardrails",
"mcp_tool_call_metadata",
"vector_store_request_metadata",
]:
for key in METRIC_METADATA_KEYS:
value = md.get(key)
if value is None:
continue
@ -1368,6 +1494,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
if hidden_params:
common_attrs["hidden_params"] = safe_dumps(hidden_params)
common_attrs = self._filter_metric_attributes(common_attrs)
if self._operation_duration_histogram:
self._operation_duration_histogram.record(
duration_s, attributes=common_attrs
@ -1377,8 +1505,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
and (usage := response_obj.get("usage"))
and self._token_usage_histogram
):
in_attrs = {**common_attrs, "gen_ai.token.type": "input"}
out_attrs = {**common_attrs, "gen_ai.token.type": "output"}
in_attrs = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "input"}
out_attrs = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "output"}
self._token_usage_histogram.record(
usage.get("prompt_tokens", 0), attributes=in_attrs
)

8
litellm/integrations/otel/README.md
View file

@ -216,7 +216,13 @@ lives in [`plumbing/`](./plumbing):
`TracerProvider` so one logger serves many tenants. The cache is a bounded LRU
that flushes + shuts down evicted providers, since the key derives from
request-supplied credentials and must not grow (or leak threads) without limit.
- [`metrics.py`](./plumbing/metrics.py) — GenAI client metric instruments.
- [`metrics.py`](./plumbing/metrics.py) — GenAI client metric instruments. The
six `gen_ai.client.*` histograms are recorded through the meter resolved by
`providers.resolve_meter_provider`: an injected provider wins (tests/DI),
otherwise the operator's globally configured `MeterProvider` is reused so its
readers/exporters receive them alongside the server metrics, and one is built
and registered as the global only when none is set (mirroring how V2 owns trace
export).
### Adapter

16
litellm/integrations/otel/emitter.py
View file

@ -17,7 +17,7 @@ from litellm.integrations.otel.model.payloads import (
ServiceSpanData,
)
from litellm.integrations.otel.plumbing.providers import to_otel_span_kind
from litellm.integrations.otel.model.semconv import Error
from litellm.integrations.otel.model.semconv import Error, ExceptionEvent
from litellm.integrations.otel.model.spans import (
SPAN_REGISTRY,
SpanRole,
@ -179,9 +179,17 @@ class SpanEmitter:
else None
)
if error and (error.error_type or error.message):
span.set_attribute(Error.TYPE, error.error_type or "error")
span.set_status(
Status(StatusCode.ERROR, error.message or error.error_type or "error")
error_type = error.error_type or "error"
message = error.message or error.error_type or "error"
span.set_attribute(Error.TYPE, error_type)
span.set_status(Status(StatusCode.ERROR, message))
# Carry the full message on the standard ``exception`` event so backends
# map it as full text under ``exception.message``. Setting it as a bare
# string attribute instead lets backends like Elasticsearch dynamic-map
# it to a ``keyword`` capped at 1024 chars, truncating the message.
span.add_event(
ExceptionEvent.NAME,
{ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message},
)
# On success leave the status UNSET (the semconv default) rather than
# forcing OK — that matches the FastAPI server span and avoids implying a

44
litellm/integrations/otel/logger.py
View file

@ -10,6 +10,7 @@ from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.trace import Span, Tracer, get_current_span, use_span
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.otel.model.baggage import promoted_baggage
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
@ -36,9 +37,15 @@ from litellm.integrations.otel.model.payloads import (
SpanError,
is_mcp_tool_call,
)
from litellm.integrations.otel.plumbing.metrics import (
GenAIMetricRecorder,
create_genai_metrics,
)
from litellm.integrations.otel.plumbing.providers import (
build_tracer_provider,
get_meter,
get_tracer,
resolve_meter_provider,
)
from litellm.integrations.otel.plumbing.routing import TenantTracerCache
from litellm.integrations.otel.model.spans import SpanRole, span_role_for_service
@ -95,7 +102,7 @@ class OpenTelemetryV2(CustomLogger):
callback_name: str | None = None,
tracer_provider: TracerProvider | None = None,
logger_provider: Any | None = None, # reserved for OTel logs
meter_provider: Any | None = None, # reserved for metrics
meter_provider: Any | None = None,
**kwargs: Any,
) -> None:
super().__init__(**kwargs)
@ -107,6 +114,8 @@ class OpenTelemetryV2(CustomLogger):
else build_tracer_provider(self.config)
)
self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME)
self._metrics_recorder = self._init_metrics(meter_provider)
self._metric_filter_error_logged = False
self._emitter = SpanEmitter(
self.tracer, self.config, mappers=resolve_mappers(self.config.mapper_names)
)
@ -116,6 +125,20 @@ class OpenTelemetryV2(CustomLogger):
self._open_llm_calls: "OrderedDict[str, _LLMCallSpan]" = OrderedDict()
self._init_otel_logger_on_litellm_proxy()
def _init_metrics(self, meter_provider: Any | None) -> "GenAIMetricRecorder | None":
"""Create the six GenAI histograms when metrics are enabled, else ``None``.
``meter_provider`` is an explicit override (tests inject one); otherwise the
provider is resolved from the OTel global so the operator's configured
readers/exporters receive the metrics, building and registering one only
when no global provider is set.
"""
if not self.config.enable_metrics:
return None
provider = resolve_meter_provider(self.config, meter_provider)
meter = get_meter(provider, LITELLM_TRACER_NAME)
return GenAIMetricRecorder(create_genai_metrics(meter), self.callback_name)
# ====================================================================== #
# Proxy global registration
# ====================================================================== #
@ -208,6 +231,25 @@ class OpenTelemetryV2(CustomLogger):
if self._emit_mcp_tool_call(kwargs, start_time, end_time):
return
self._close_llm_call(kwargs, start_time, end_time)
self._record_metrics(kwargs, response_obj, start_time, end_time)
def _record_metrics(self, kwargs, response_obj, start_time, end_time) -> None:
"""Record the GenAI metrics for a successful LLM call. Best-effort: a
recording failure (e.g. a malformed payload) must never break the span
close or the request itself."""
if self._metrics_recorder is None:
return
try:
self._metrics_recorder.record(kwargs, response_obj, start_time, end_time)
except ValueError as exc:
if not self._metric_filter_error_logged:
verbose_logger.error(
"OpenTelemetryV2: invalid otel.attributes metric filter, metrics disabled: %s",
exc,
)
self._metric_filter_error_logged = True
except Exception as exc:
verbose_logger.debug("OpenTelemetryV2: metric recording failed: %s", exc)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
if self._emit_mcp_tool_call(kwargs, start_time, end_time):

23
litellm/integrations/otel/mappers/genai.py
View file

@ -10,7 +10,12 @@ table: one lambda per mapping operation, applied against the typed span data.
from typing import Callable
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
from litellm.integrations.otel.mappers.utils import collect, drop_none
from litellm.integrations.otel.mappers.utils import (
collect,
drop_none,
output_messages,
serialize_messages,
)
from litellm.integrations.otel.model.payloads import (
GuardrailSpanData,
LLMCallSpanData,
@ -47,6 +52,8 @@ class GenAIMapper:
else None
),
GenAI.REQUEST_SEED: lambda d: d.request_params.seed,
GenAI.INPUT_MESSAGES: lambda d: serialize_messages(d.messages_in),
GenAI.OUTPUT_MESSAGES: lambda d: serialize_messages(output_messages(d)),
GenAI.RESPONSE_MODEL: lambda d: d.response_model,
GenAI.RESPONSE_ID: lambda d: d.response_id,
GenAI.RESPONSE_FINISH_REASONS: lambda d: (
@ -63,6 +70,20 @@ class GenAIMapper:
# routing) onto the boundary-born LLM span — stamp it directly here.
LiteLLM.PROVIDER_MODEL: lambda d: d.identity.provider_model or None,
f"{LiteLLM.COST_PREFIX}total": lambda d: d.response_cost,
# Per-component cost breakdown (from the StandardLoggingPayload
# ``cost_breakdown``). Each component is omitted when the source didn't
# report it, so spans stay sparse rather than carrying zeros.
f"{LiteLLM.COST_PREFIX}input": lambda d: d.cost.input,
f"{LiteLLM.COST_PREFIX}output": lambda d: d.cost.output,
f"{LiteLLM.COST_PREFIX}cache_read": lambda d: d.cost.cache_read,
f"{LiteLLM.COST_PREFIX}cache_creation": lambda d: d.cost.cache_creation,
f"{LiteLLM.COST_PREFIX}tool_usage": lambda d: d.cost.tool_usage,
f"{LiteLLM.COST_PREFIX}original": lambda d: d.cost.original,
f"{LiteLLM.COST_PREFIX}discount_amount": lambda d: d.cost.discount_amount,
f"{LiteLLM.COST_PREFIX}discount_percent": lambda d: d.cost.discount_percent,
f"{LiteLLM.COST_PREFIX}margin_fixed_amount": lambda d: d.cost.margin_fixed_amount,
f"{LiteLLM.COST_PREFIX}margin_percent": lambda d: d.cost.margin_percent,
f"{LiteLLM.COST_PREFIX}margin_total_amount": lambda d: d.cost.margin_total_amount,
LiteLLM.REQUEST_STREAMING: lambda d: d.is_streaming,
}

48
litellm/integrations/otel/model/payloads.py
View file

@ -34,6 +34,7 @@ __all__ = [
"RequestIdentity",
"GuardrailSpanData",
"LLMCallSpanData",
"LLMCost",
"LLMRequestParams",
"LLMUsage",
"MCPToolCallSpanData",
@ -91,6 +92,49 @@ class LLMUsage:
total_tokens: int | None = None
@dataclass(frozen=True)
class LLMCost:
"""Per-component cost breakdown, from the StandardLoggingPayload
``cost_breakdown`` (``litellm.types.utils.CostBreakdown``).
Each field is the USD cost of one component, or ``None`` when the source did
not report it — so the mapper omits absent components instead of emitting 0.
The final (post-discount/post-margin) total is carried separately on
``LLMCallSpanData.response_cost``. Free-form ``additional_costs`` are not
surfaced here: span attributes are scalar and there is no agreed key shape
for them yet.
"""
input: float | None = None
output: float | None = None
cache_read: float | None = None
cache_creation: float | None = None
tool_usage: float | None = None
original: float | None = None
discount_amount: float | None = None
discount_percent: float | None = None
margin_fixed_amount: float | None = None
margin_percent: float | None = None
margin_total_amount: float | None = None
@classmethod
def from_breakdown(cls, breakdown: Mapping[str, object] | None) -> "LLMCost":
b = breakdown or {}
return cls(
input=as_float(b.get("input_cost")),
output=as_float(b.get("output_cost")),
cache_read=as_float(b.get("cache_read_cost")),
cache_creation=as_float(b.get("cache_creation_cost")),
tool_usage=as_float(b.get("tool_usage_cost")),
original=as_float(b.get("original_cost")),
discount_amount=as_float(b.get("discount_amount")),
discount_percent=as_float(b.get("discount_percent")),
margin_fixed_amount=as_float(b.get("margin_fixed_amount")),
margin_percent=as_float(b.get("margin_percent")),
margin_total_amount=as_float(b.get("margin_total_amount")),
)
@dataclass(frozen=True)
class SpanError:
error_type: str | None = None
@ -255,6 +299,7 @@ class LLMCallSpanData:
server: ServerInfo | None
identity: RequestIdentity
is_streaming: bool | None = None
cost: LLMCost = field(default_factory=LLMCost)
tools: tuple[ToolDefinition, ...] = ()
# Raw messages and response, needed by vendor mappers (OpenInference,
# Langfuse, Weave) that stamp message-level attributes. ``messages_in`` is
@ -302,6 +347,9 @@ class LLMCallSpanData:
finish_reasons=finish_reasons,
error=_parse_error(payload),
response_cost=as_float(payload.get("response_cost")),
cost=LLMCost.from_breakdown(
cast("Mapping[str, object] | None", payload.get("cost_breakdown"))
),
server=ServerInfo.from_api_base(context.api_base),
identity=context.identity,
is_streaming=as_bool(payload.get("stream")),

19
litellm/integrations/otel/model/semconv.py
View file

@ -146,6 +146,21 @@ class Error:
TYPE: Final = "error.type"
class ExceptionEvent:
"""OTel exception-event name and attribute keys (semconv ``exception.*``).
The full error message rides ``exception.message`` on a span event rather than
a custom string attribute. Backends recognise these semantic-convention names
and map them as full text; an unrecognised key (e.g. ``error_message``) falls
into the default dynamic template, which truncates strings to a 1024-char
``keyword``.
"""
NAME: Final = "exception"
TYPE: Final = "exception.type"
MESSAGE: Final = "exception.message"
class Server:
ADDRESS: Final = "server.address"
PORT: Final = "server.port"
@ -215,6 +230,10 @@ class Metric:
TOKEN_USAGE: Final = "gen_ai.client.token.usage"
OPERATION_DURATION: Final = "gen_ai.client.operation.duration"
TOKEN_COST: Final = "gen_ai.client.token.cost"
TIME_TO_FIRST_TOKEN: Final = "gen_ai.client.response.time_to_first_token"
TIME_PER_OUTPUT_TOKEN: Final = "gen_ai.client.response.time_per_output_token"
RESPONSE_DURATION: Final = "gen_ai.client.response.duration"
# litellm ``custom_llm_provider`` -> ``gen_ai.provider.name`` value.

255
litellm/integrations/otel/plumbing/metrics.py
View file

@ -1,28 +1,265 @@
"""GenAI client metrics (token usage + operation duration histograms)."""
"""GenAI client metrics: the six ``gen_ai.client.*`` histograms plus the
recorder that builds attributes, applies the shared cardinality filter, and
records a request's metrics in the success path.
The instrument names/units/descriptions and the recording + timing math mirror
the v1 :mod:`litellm.integrations.opentelemetry` integration so both engines emit
identical metrics. The attribute cardinality filter is reused from v1 by import
(no duplication of the valid-name set or its validation).
"""
from dataclasses import dataclass
from datetime import datetime
from typing import Any, FrozenSet, Mapping, Optional
from opentelemetry.metrics import Histogram, Meter
from litellm.integrations.otel.model.semconv import Metric
import litellm
from litellm.integrations.opentelemetry import (
METRIC_METADATA_KEYS,
TOKEN_TYPE_ATTRIBUTE,
_build_metric_attribute_filter,
_resolve_metric_attribute_filter,
)
from litellm.integrations.otel.model.semconv import Metric, resolve_operation
from litellm.integrations.otel.model.utils import to_seconds
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@dataclass(frozen=True)
class GenAIMetrics:
token_usage: Histogram
operation_duration: Histogram
token_usage: Histogram
token_cost: Histogram
time_to_first_token: Histogram
time_per_output_token: Histogram
response_duration: Histogram
def create_genai_metrics(meter: Meter) -> GenAIMetrics:
return GenAIMetrics(
token_usage=meter.create_histogram(
name=Metric.TOKEN_USAGE,
unit="{token}",
description="Number of tokens used per GenAI request.",
),
operation_duration=meter.create_histogram(
name=Metric.OPERATION_DURATION,
unit="s",
description="GenAI operation duration.",
description="GenAI operation duration",
),
token_usage=meter.create_histogram(
name=Metric.TOKEN_USAGE,
unit="{token}",
description="GenAI token usage",
),
token_cost=meter.create_histogram(
name=Metric.TOKEN_COST,
unit="USD",
description="GenAI request cost",
),
time_to_first_token=meter.create_histogram(
name=Metric.TIME_TO_FIRST_TOKEN,
unit="s",
description="Time to first token for streaming requests",
),
time_per_output_token=meter.create_histogram(
name=Metric.TIME_PER_OUTPUT_TOKEN,
unit="s",
description="Average time per output token (generation time / completion tokens)",
),
response_duration=meter.create_histogram(
name=Metric.RESPONSE_DURATION,
unit="s",
description="Total LLM API generation time (excludes LiteLLM overhead)",
),
)
class GenAIMetricRecorder:
"""Records the six GenAI histograms for one successful LLM call.
The cardinality filter is resolved lazily on the first record: the proxy
populates ``callback_settings.otel.attributes`` after the logger is built, so
reading it at construction time would miss it. ``gen_ai.token.type`` is added
to the token-usage attributes after filtering so the input/output split always
survives.
"""
def __init__(
self, metrics: GenAIMetrics, callback_name: Optional[str] = None
) -> None:
self._metrics = metrics
self._callback_name = callback_name
self._include: Optional[FrozenSet[str]] = None
self._exclude: Optional[FrozenSet[str]] = None
self._filter_resolved = False
def record(
self,
kwargs: Mapping[str, Any],
response_obj: Any,
start_time: datetime,
end_time: datetime,
) -> None:
common_attrs = self._filter_attributes(self._common_attributes(kwargs))
duration_s = (end_time - start_time).total_seconds()
self._metrics.operation_duration.record(duration_s, attributes=common_attrs)
self._record_token_usage(response_obj, common_attrs)
cost = kwargs.get("response_cost")
if cost:
self._metrics.token_cost.record(cost, attributes=common_attrs)
self._record_time_to_first_token(kwargs, common_attrs)
self._record_time_per_output_token(
kwargs, response_obj, end_time, duration_s, common_attrs
)
self._record_response_duration(kwargs, end_time, common_attrs)
# ------------------------------------------------------------------ #
# Attribute building + cardinality filter
# ------------------------------------------------------------------ #
def _common_attributes(self, kwargs: Mapping[str, Any]) -> dict:
params = kwargs.get("litellm_params") or {}
provider = params.get("custom_llm_provider", "Unknown")
common_attrs: dict = {
"gen_ai.operation.name": resolve_operation(kwargs.get("call_type")).value,
"gen_ai.system": provider,
"gen_ai.request.model": kwargs.get("model"),
"gen_ai.framework": "litellm",
}
std_log = kwargs.get("standard_logging_object")
md = getattr(std_log, "metadata", None) or (std_log or {}).get("metadata", {})
for key in METRIC_METADATA_KEYS:
value = md.get(key)
if value is None:
continue
if isinstance(value, (dict, list)):
common_attrs[f"metadata.{key}"] = safe_dumps(value)
else:
common_attrs[f"metadata.{key}"] = str(value)
hidden_params = getattr(std_log, "hidden_params", None) or (std_log or {}).get(
"hidden_params", {}
)
if hidden_params:
common_attrs["hidden_params"] = safe_dumps(hidden_params)
return common_attrs
def _ensure_filter(self) -> None:
if self._filter_resolved:
return
attributes = None
if self._callback_name in (None, "otel"):
otel_settings = (litellm.callback_settings or {}).get("otel") or {}
raw = (
otel_settings.get("attributes")
if isinstance(otel_settings, dict)
else None
)
if raw is not None:
attributes = _build_metric_attribute_filter(raw)
# A bad filter (include_list + exclude_list both set, an unfilterable name)
# raises here; the caller (logger._record_metrics) surfaces it once at ERROR
# so the operator-fixable config error is visible. Not cached on the raise
# path -- _filter_resolved stays False -- so a corrected config takes effect
# without reconstructing the recorder.
self._include, self._exclude = _resolve_metric_attribute_filter(attributes)
self._filter_resolved = True
def _filter_attributes(self, attrs: dict) -> dict:
self._ensure_filter()
if self._include is not None:
return {k: v for k, v in attrs.items() if k in self._include}
if self._exclude is not None:
return {k: v for k, v in attrs.items() if k not in self._exclude}
return attrs
# ------------------------------------------------------------------ #
# Per-metric recording
# ------------------------------------------------------------------ #
def _record_token_usage(self, response_obj: Any, common_attrs: dict) -> None:
if not response_obj:
return
usage = response_obj.get("usage")
if not usage:
return
in_attrs = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "input"}
out_attrs = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "output"}
self._metrics.token_usage.record(
usage.get("prompt_tokens", 0), attributes=in_attrs
)
self._metrics.token_usage.record(
usage.get("completion_tokens", 0), attributes=out_attrs
)
def _record_time_to_first_token(
self, kwargs: Mapping[str, Any], common_attrs: dict
) -> None:
if not kwargs.get("optional_params", {}).get("stream", False):
return
api_call_start = to_seconds(kwargs.get("api_call_start_time"))
completion_start = to_seconds(kwargs.get("completion_start_time"))
if api_call_start is None or completion_start is None:
return
self._metrics.time_to_first_token.record(
completion_start - api_call_start, attributes=common_attrs
)
def _record_time_per_output_token(
self,
kwargs: Mapping[str, Any],
response_obj: Any,
end_time: datetime,
duration_s: float,
common_attrs: dict,
) -> None:
completion_tokens = None
if response_obj and (usage := response_obj.get("usage")):
completion_tokens = usage.get("completion_tokens")
if completion_tokens is None or completion_tokens <= 0:
return
end_ts = to_seconds(end_time)
if end_ts is None:
generation_time = duration_s
else:
completion_start_time = kwargs.get("completion_start_time")
api_call_start_time = kwargs.get("api_call_start_time")
if completion_start_time is not None:
completion_start = to_seconds(completion_start_time)
generation_time = (
duration_s
if completion_start is None
else end_ts - completion_start
)
elif api_call_start_time is not None:
api_call_start = to_seconds(api_call_start_time)
generation_time = (
duration_s if api_call_start is None else end_ts - api_call_start
)
else:
generation_time = duration_s
if generation_time > 0:
self._metrics.time_per_output_token.record(
generation_time / completion_tokens, attributes=common_attrs
)
def _record_response_duration(
self, kwargs: Mapping[str, Any], end_time: datetime, common_attrs: dict
) -> None:
api_call_start_time = kwargs.get("api_call_start_time")
if api_call_start_time is None:
return
_end_time = kwargs.get("end_time") or end_time
if _end_time is None:
_end_time = datetime.now()
api_call_start = to_seconds(api_call_start_time)
end_ts = to_seconds(_end_time)
if api_call_start is None or end_ts is None:
return
duration = end_ts - api_call_start
if duration > 0:
self._metrics.response_duration.record(duration, attributes=common_attrs)

130
litellm/integrations/otel/plumbing/providers.py
View file

@ -1,9 +1,11 @@
"""Provider / exporter factory + the Baggage span processor."""
from typing import Callable, Iterable
from typing import TYPE_CHECKING, Any, Callable, Iterable
from opentelemetry import baggage
from opentelemetry import baggage, metrics
from opentelemetry.context import Context
from opentelemetry.metrics import MeterProvider, NoOpMeterProvider
from opentelemetry.sdk.metrics import MeterProvider as SDKMeterProvider
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider
from opentelemetry.sdk.trace.export import (
@ -17,6 +19,7 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
)
from opentelemetry.trace import Span, SpanKind, Tracer
from litellm._version import version as litellm_version
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.model.semconv import LiteLLM
from litellm.integrations.otel.model.spans import LiteLLMSpanKind
@ -24,6 +27,10 @@ from litellm.integrations.otel.model.spans import LiteLLMSpanKind
# Re-exported so ``providers.parse_headers`` remains a stable entry point.
from litellm.integrations.otel.model.utils import parse_headers as parse_headers
if TYPE_CHECKING:
from opentelemetry.metrics import Meter
from opentelemetry.sdk.metrics.export import MetricReader
_SPAN_KIND_BY_ROLE_KIND: dict[LiteLLMSpanKind, SpanKind] = {
LiteLLMSpanKind.SERVER: SpanKind.SERVER,
LiteLLMSpanKind.CLIENT: SpanKind.CLIENT,
@ -156,6 +163,120 @@ def build_span_exporter(config: OpenTelemetryV2Config) -> SpanExporter:
)
def _otlp_metrics_endpoint(endpoint: str | None) -> str | None:
"""Point an OTLP/HTTP base endpoint at the ``/v1/metrics`` signal path.
The OTLP/HTTP exporter only appends ``/v1/metrics`` when it reads
``OTEL_EXPORTER_OTLP_ENDPOINT`` itself; an explicitly passed endpoint is used
verbatim, so a base URL would POST to the root. Mirror ``_otlp_traces_endpoint``
for the metrics signal (rewriting a sibling signal path when present).
"""
if not endpoint:
return endpoint
endpoint = endpoint.rstrip("/")
if endpoint.endswith("/v1/metrics"):
return endpoint
for other_signal in ("/v1/traces", "/v1/logs"):
if endpoint.endswith(other_signal):
return endpoint[: -len(other_signal)] + "/v1/metrics"
return endpoint + "/v1/metrics"
def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader":
"""Build a metric reader mirroring v1's exporter selection.
``console`` (and any unrecognized kind) exports to the console; ``otlp_http``
and ``otlp_grpc`` export over OTLP with the configured endpoint/headers. The
reader exports on a 5s period, matching v1.
"""
from opentelemetry.sdk.metrics.export import (
ConsoleMetricExporter,
PeriodicExportingMetricReader,
)
kind = (config.exporter or "console").lower()
if kind in ("otlp_http", "http", "http/protobuf", "http/json"):
from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
OTLPMetricExporter as HTTPMetricExporter,
)
from opentelemetry.sdk.metrics import Histogram
from opentelemetry.sdk.metrics.export import AggregationTemporality
exporter: Any = HTTPMetricExporter(
endpoint=_otlp_metrics_endpoint(config.endpoint),
headers=parse_headers(config.headers),
preferred_temporality={Histogram: AggregationTemporality.DELTA},
)
elif kind in ("otlp_grpc", "grpc"):
from opentelemetry.sdk.metrics import Histogram
from opentelemetry.sdk.metrics.export import AggregationTemporality
try:
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
OTLPMetricExporter as GRPCMetricExporter,
)
except ImportError as exc:
raise ImportError(
"OpenTelemetry OTLP gRPC metric exporter is not available. Install "
"`opentelemetry-exporter-otlp` and `grpcio` (or `litellm[grpc]`)."
) from exc
exporter = GRPCMetricExporter(
endpoint=config.endpoint,
headers=parse_headers(config.headers),
preferred_temporality={Histogram: AggregationTemporality.DELTA},
)
else:
exporter = ConsoleMetricExporter()
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
def build_meter_provider(
config: OpenTelemetryV2Config,
metric_reader: "MetricReader | None" = None,
) -> SDKMeterProvider:
"""Build the :class:`MeterProvider` for GenAI metrics.
``metric_reader`` is an explicit override (tests inject an
``InMemoryMetricReader``); otherwise the reader is selected from the config's
exporter kind via :func:`build_metric_reader`.
"""
reader = metric_reader if metric_reader is not None else build_metric_reader(config)
return SDKMeterProvider(metric_readers=[reader], resource=build_resource(config))
def resolve_meter_provider(
config: OpenTelemetryV2Config,
meter_provider: MeterProvider | None = None,
) -> MeterProvider:
"""Resolve the :class:`MeterProvider` GenAI metrics record through.
An injected provider wins (DI/tests). Otherwise reuse whatever the operator has
configured as the global, whether a real SDK provider or an explicit
``NoOpMeterProvider``, so the GenAI histograms ride the operator's
readers/exporters and an explicit opt-out is honored. Only when the global is
still the default proxy placeholder does V2 build one from the config and
publish it as the global, mirroring how V2 owns trace export. The built
provider is the one returned, so its reader thread is always live, never
orphaned.
"""
if meter_provider is not None:
return meter_provider
existing = metrics.get_meter_provider()
if isinstance(existing, (SDKMeterProvider, NoOpMeterProvider)):
return existing
provider = build_meter_provider(config)
metrics.set_meter_provider(provider)
return provider
def get_meter(provider: MeterProvider, name: str = "litellm") -> "Meter":
return provider.get_meter(name, litellm_version)
def build_resource(config: OpenTelemetryV2Config) -> Resource:
attributes: dict[str, str] = {"service.name": config.service_name}
if config.deployment_environment:
@ -207,7 +328,10 @@ def build_tracer_provider(
def get_tracer(provider: TracerProvider, name: str = "litellm") -> Tracer:
return provider.get_tracer(name)
# Stamp the instrumentation scope with the LiteLLM package version so every
# emitted span carries a deterministic ``scope.version`` (the standard OTel
# location for the emitting library's version) for downstream consumers.
return provider.get_tracer(name, litellm_version)
def in_memory_provider(

4
litellm/integrations/websearch_interception/handler.py
View file

@ -1128,7 +1128,7 @@ class WebSearchInterceptionLogger(CustomLogger):
)
raise
async def _execute_chat_completion_agentic_loop( # noqa: PLR0915
async def _execute_chat_completion_agentic_loop(
self,
model: str,
messages: List[Dict],
@ -1159,7 +1159,7 @@ class WebSearchInterceptionLogger(CustomLogger):
**request_patch.kwargs,
)
async def _build_chat_completion_request_patch( # noqa: PLR0915
async def _build_chat_completion_request_patch(
self,
model: str,
messages: List[Dict],

7
litellm/integrations/weights_biases.py
View file

@ -21,10 +21,11 @@ try:
# contains a (known) object attribute
object: Literal["chat.completion", "edit", "text_completion"]
def __getitem__(self, key: K) -> V: ... # noqa
def __getitem__(self, key: K) -> V: ...
def get(self, key: K, default: Optional[V] = None) -> Optional[V]: # noqa
... # pragma: no cover
def get(
self, key: K, default: Optional[V] = None
) -> Optional[V]: ... # pragma: no cover
class OpenAIRequestResponseResolver:
def __call__(

27
litellm/litellm_core_utils/core_helpers.py
View file

@ -242,9 +242,28 @@ def _get_parent_otel_span_from_kwargs(
return None
def process_response_headers(response_headers: Union[httpx.Headers, dict]) -> dict:
def process_response_headers(
response_headers: Union[httpx.Headers, dict],
preserve_litellm_internal_headers: bool = False,
) -> dict:
"""
`preserve_litellm_internal_headers` must only be True when the input is a
LiteLLM-owned dict (e.g. `_hidden_params["additional_headers"]` that has
already been through one round of processing). For raw upstream provider
headers — whether passed as `httpx.Headers` or a plain dict — it must
remain False, otherwise a malicious provider returning `x-litellm-*` could
spoof LiteLLM-internal markers (e.g. `x-litellm-attempted-fallbacks`).
When the input is an `httpx.Headers` object the flag is always treated as
False regardless of what the caller requested, because `httpx.Headers` is
always a raw provider response and can never be LiteLLM-owned.
"""
from litellm.types.utils import OPENAI_RESPONSE_HEADERS
# Raw httpx.Headers objects come directly from provider HTTP responses and
# must never be treated as LiteLLM-owned, regardless of caller intent.
_preserve = preserve_litellm_internal_headers and isinstance(response_headers, dict)
openai_headers = {}
processed_headers = {}
additional_headers = {}
@ -256,6 +275,12 @@ def process_response_headers(response_headers: Union[httpx.Headers, dict]) -> di
"llm_provider-"
): # return raw provider headers (incl. openai-compatible ones)
processed_headers[k] = v
elif _preserve and k.startswith("x-litellm-"):
# LiteLLM's own internal headers (e.g. x-litellm-attempted-fallbacks,
# x-litellm-model-group) are not LLM provider headers and must not be
# prefixed. Downstream consumers (proxy override, callers checking
# whether a fallback happened) look up the bare key.
processed_headers[k] = v
else:
additional_headers["{}-{}".format("llm_provider", k)] = v

16
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -250,14 +250,14 @@ def exception_type( # type: ignore # noqa: PLR0915
exception_mapping_worked = False
exception_provider = custom_llm_provider
if litellm.suppress_debug_info is False:
print() # noqa
print( # noqa
"\033[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new\033[0m" # noqa
) # noqa
print( # noqa
"LiteLLM.Info: If you need to debug this error, use `litellm._turn_on_debug()'." # noqa
) # noqa
print() # noqa
print() # noqa: T201
print( # noqa: T201
"\033[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new\033[0m"
)
print( # noqa: T201
"LiteLLM.Info: If you need to debug this error, use `litellm._turn_on_debug()'."
)
print() # noqa: T201
litellm_response_headers = _get_response_headers(
original_exception=original_exception

10
litellm/litellm_core_utils/fallback_utils.py
View file

@ -7,6 +7,9 @@ from litellm.litellm_core_utils.core_helpers import (
safe_deep_copy,
filter_internal_params,
)
from litellm.router_utils.add_retry_fallback_headers import (
add_fallback_headers_to_response,
)
from .asyncify import run_async_function
@ -42,7 +45,7 @@ async def async_completion_with_fallbacks(**kwargs):
# Try each fallback model
most_recent_exception_str: Optional[str] = None
for fallback in fallbacks:
for attempted_fallbacks, fallback in enumerate(fallbacks):
try:
completion_kwargs = safe_deep_copy(base_kwargs)
# Handle dictionary fallback configurations
@ -63,7 +66,10 @@ async def async_completion_with_fallbacks(**kwargs):
)
if response is not None:
return response
return add_fallback_headers_to_response(
response=response,
attempted_fallbacks=attempted_fallbacks,
)
except Exception as e:
verbose_logger.exception(

20
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -334,6 +334,9 @@ def get_llm_provider( # noqa: PLR0915
elif endpoint == "dashscope-intl.aliyuncs.com/compatible-mode/v1":
custom_llm_provider = "dashscope"
dynamic_api_key = get_secret_str("DASHSCOPE_API_KEY")
elif endpoint == "https://api-inference.modelscope.cn/v1":
custom_llm_provider = "modelscope"
dynamic_api_key = get_secret_str("MODELSCOPE_API_KEY")
elif endpoint == "api.moonshot.ai/v1":
custom_llm_provider = "moonshot"
dynamic_api_key = get_secret_str("MOONSHOT_API_KEY")
@ -531,11 +534,11 @@ def get_llm_provider( # noqa: PLR0915
custom_llm_provider = "gigachat"
if not custom_llm_provider:
if litellm.suppress_debug_info is False:
print() # noqa
print( # noqa
"\033[1;31mProvider List: https://docs.litellm.ai/docs/providers\033[0m" # noqa
) # noqa
print() # noqa
print() # noqa: T201
print( # noqa: T201
"\033[1;31mProvider List: https://docs.litellm.ai/docs/providers\033[0m"
)
print() # noqa: T201
error_str = f"LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model={model}\n Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers"
# maps to openai.NotFoundError, this is raised when openai does not recognize the llm
raise litellm.exceptions.BadRequestError( # type: ignore
@ -932,6 +935,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
) = litellm.DashScopeChatConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
elif custom_llm_provider == "modelscope":
(
api_base,
dynamic_api_key,
) = litellm.ModelScopeChatConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
elif custom_llm_provider == "moonshot":
(
api_base,

9
litellm/litellm_core_utils/get_supported_openai_params.py
View file

@ -295,6 +295,15 @@ def get_supported_openai_params( # noqa: PLR0915
elif custom_llm_provider == "predibase":
return litellm.PredibaseConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "voyage":
if (
request_type == "embeddings"
and litellm.VoyageMultimodalEmbeddingConfig.is_multimodal_embeddings(model)
):
return (
litellm.VoyageMultimodalEmbeddingConfig().get_supported_openai_params(
model=model
)
)
return litellm.VoyageEmbeddingConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "infinity":
return litellm.InfinityEmbeddingConfig().get_supported_openai_params(

8
litellm/litellm_core_utils/initialize_dynamic_callback_params.py
View file

@ -53,11 +53,19 @@ _supported_callback_params = [
"braintrust_host",
"slack_webhook_url",
"lunary_public_key",
"dd_api_key",
"dd_site",
"dd_agent_host",
"dd_agent_port",
]
_request_blocked_callback_params = {
"gcs_bucket_name",
"gcs_path_service_account",
"dd_api_key",
"dd_site",
"dd_agent_host",
"dd_agent_port",
}

42
litellm/litellm_core_utils/litellm_logging.py
View file

@ -381,13 +381,14 @@ class Logging(LiteLLMLoggingBaseClass):
List[Union[str, Callable, CustomLogger]]
] = dynamic_async_failure_callbacks
# Process dynamic callbacks
self.process_dynamic_callbacks()
## DYNAMIC LANGFUSE / GCS / logging callback KEYS ##
self.standard_callback_dynamic_params: StandardCallbackDynamicParams = (
self.initialize_standard_callback_dynamic_params(kwargs)
)
# Process dynamic callbacks (after standard_callback_dynamic_params is initialized,
# so team-scoped credentials are available for callback initialization)
self.process_dynamic_callbacks()
self.standard_built_in_tools_params: StandardBuiltInToolsParams = (
self.initialize_standard_built_in_tools_params(kwargs)
)
@ -482,8 +483,21 @@ class Logging(LiteLLMLoggingBaseClass):
isinstance(callback, str)
and callback in litellm._known_custom_logger_compatible_callbacks
):
# For callbacks that support team-scoped credentials (e.g. datadog),
# pass only the relevant dynamic params as custom_logger_init_args.
_custom_logger_init_args: Optional[dict] = None
if callback == "datadog":
_custom_logger_init_args = {
k: v
for k, v in self.standard_callback_dynamic_params.items()
if k.startswith("dd_")
}
callback_class = _init_custom_logger_compatible_class(
callback, internal_usage_cache=None, llm_router=None # type: ignore
callback, # type: ignore[arg-type]
internal_usage_cache=None,
llm_router=None, # type: ignore
custom_logger_init_args=_custom_logger_init_args,
)
if callback_class is not None:
processed_list.append(callback_class)
@ -3946,6 +3960,24 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
_in_memory_loggers.append(_prometheus_logger)
return _prometheus_logger # type: ignore
elif logging_integration == "datadog":
# Check if team-scoped credentials are provided
_dd_api_key = custom_logger_init_args.get("dd_api_key")
_dd_site = custom_logger_init_args.get("dd_site")
_dd_agent_host = custom_logger_init_args.get("dd_agent_host")
_dd_agent_port = custom_logger_init_args.get("dd_agent_port")
if _dd_api_key or _dd_site or _dd_agent_host:
# Team-scoped credentials: use DynamicLoggingCache for per-credential isolation
from litellm.integrations.datadog.datadog_team_handler import (
DataDogHandler,
)
return DataDogHandler.get_datadog_logger_for_request(
standard_callback_dynamic_params=custom_logger_init_args, # type: ignore
in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
)
# Global (env-var based): reuse cached instance
for callback in _in_memory_loggers:
if isinstance(callback, DataDogLogger):
return callback # type: ignore
@ -5867,7 +5899,7 @@ def get_standard_logging_object_payload(
def emit_standard_logging_payload(payload: StandardLoggingPayload):
if os.getenv("LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD"):
try:
print(json.dumps(payload, indent=4, default=str)) # noqa
print(json.dumps(payload, indent=4, default=str)) # noqa: T201
except Exception as e:
verbose_logger.exception(
"Error serializing standard logging payload for debug output: {}".format(

3
litellm/litellm_core_utils/llm_response_utils/response_metadata.py
View file

@ -49,7 +49,8 @@ class ResponseMetadata:
result=self.result, litellm_model_name=model, router_model_id=model_id
),
"additional_headers": process_response_headers(
self._get_value_from_hidden_params("additional_headers") or {}
self._get_value_from_hidden_params("additional_headers") or {},
preserve_litellm_internal_headers=True,
),
"litellm_model_name": model,
}

16
litellm/litellm_core_utils/logging_callback_manager.py
View file

@ -394,6 +394,22 @@ class LoggingCallbackManager:
+ litellm._async_failure_callback
)
def remove_callback_from_all_lists(self, obj, require_self=False) -> None:
"""
Remove a callback object from every callback list it may have been
promoted into, so a re-initialized callback leaves no stale instance behind.
"""
for callback_list in (
litellm.callbacks,
litellm.success_callback,
litellm.failure_callback,
litellm._async_success_callback,
litellm._async_failure_callback,
):
self.remove_callback_from_list_by_object(
callback_list, obj, require_self=require_self
)
def get_active_additional_logging_utils_from_custom_logger(
self,
) -> Set[AdditionalLoggingUtils]:

2
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -604,6 +604,8 @@ class ChunkProcessor:
usage_chunk = chunk._hidden_params.get("usage", None)
if usage_chunk is not None:
if isinstance(usage_chunk, dict):
usage_chunk = Usage(**usage_chunk)
usage_chunk_dict = self._usage_chunk_calculation_helper(usage_chunk)
if (
usage_chunk_dict["prompt_tokens"] is not None

10
litellm/litellm_core_utils/streaming_handler.py
View file

@ -92,7 +92,7 @@ def is_async_iterable(obj: Any) -> bool:
def print_verbose(print_statement):
try:
if litellm.set_verbose:
print(print_statement) # noqa
print(print_statement) # noqa: T201
except Exception:
pass
@ -295,6 +295,12 @@ class CustomStreamWrapper:
if len(self.chunks) < 2:
return
# Providers like Vertex Gemini (Flash / Flash Lite with web search) emit
# metadata-only / usage-only chunks with no choices. These get stored in
# self.chunks but carry no comparable content, so skip repetition detection.
if not self.chunks[-1].choices or not self.chunks[-2].choices:
return
last_content = self.chunks[-1].choices[0].delta.content
if (
@ -961,7 +967,7 @@ class CustomStreamWrapper:
delta, model_response.choices[0].delta, attribute
)
def return_processed_chunk_logic( # noqa
def return_processed_chunk_logic( # noqa: PLR0915, C901
self,
completion_obj: Dict[str, Any],
model_response: ModelResponseStream,

44
litellm/llms/anthropic/chat/transformation.py
View file

@ -2214,18 +2214,33 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if "inference_geo" in _usage and _usage["inference_geo"] is not None:
inference_geo = _usage["inference_geo"]
if (
"cache_creation_input_tokens" in _usage
and _usage["cache_creation_input_tokens"] is not None
):
cache_creation_input_tokens = _usage["cache_creation_input_tokens"]
prompt_tokens += cache_creation_input_tokens
if (
"cache_read_input_tokens" in _usage
and _usage["cache_read_input_tokens"] is not None
):
cache_read_input_tokens = _usage["cache_read_input_tokens"]
prompt_tokens += cache_read_input_tokens
iterations: Optional[List[Any]] = _usage.get("iterations")
if iterations:
prompt_tokens = sum(it.get("input_tokens", 0) or 0 for it in iterations)
completion_tokens = sum(
it.get("output_tokens", 0) or 0 for it in iterations
)
cache_creation_input_tokens = sum(
it.get("cache_creation_input_tokens", 0) or 0 for it in iterations
)
cache_read_input_tokens = sum(
it.get("cache_read_input_tokens", 0) or 0 for it in iterations
)
prompt_tokens += cache_creation_input_tokens + cache_read_input_tokens
if not iterations:
if (
"cache_creation_input_tokens" in _usage
and _usage["cache_creation_input_tokens"] is not None
):
cache_creation_input_tokens = _usage["cache_creation_input_tokens"]
prompt_tokens += cache_creation_input_tokens
if (
"cache_read_input_tokens" in _usage
and _usage["cache_read_input_tokens"] is not None
):
cache_read_input_tokens = _usage["cache_read_input_tokens"]
prompt_tokens += cache_read_input_tokens
if "server_tool_use" in _usage and _usage["server_tool_use"] is not None:
if (
"web_search_requests" in _usage["server_tool_use"]
@ -2264,7 +2279,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
),
)
raw_input_tokens = usage_object.get("input_tokens", 0) or 0
raw_input_tokens = (
prompt_tokens - cache_read_input_tokens - cache_creation_input_tokens
)
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=cache_read_input_tokens,
cache_creation_tokens=cache_creation_input_tokens,
@ -2296,6 +2313,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
cache_creation_input_tokens=cache_creation_input_tokens,
cache_read_input_tokens=cache_read_input_tokens,
completion_tokens_details=completion_token_details,
iterations=iterations,
server_tool_use=(
ServerToolUse(
web_search_requests=web_search_requests,

2
litellm/llms/azure/azure.py
View file

@ -189,7 +189,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
except Exception as e:
raise e
def completion( # noqa: PLR0915
def completion(
self,
model: str,
messages: list,

2
litellm/llms/azure/completion/handler.py
View file

@ -25,7 +25,7 @@ class AzureTextCompletion(BaseAzureLLM):
headers["Authorization"] = f"Bearer {azure_ad_token}"
return headers
def completion( # noqa: PLR0915
def completion(
self,
model: str,
messages: list,

17
litellm/llms/base_llm/base_model_iterator.py
View file

@ -1,8 +1,11 @@
import json
from abc import abstractmethod
from typing import List, Optional, Union, cast
from typing import TYPE_CHECKING, List, Optional, Union, cast
import litellm
if TYPE_CHECKING:
import httpx
from litellm.types.utils import (
Choices,
Delta,
@ -69,6 +72,18 @@ class BaseModelResponseIterator:
self.streaming_response = streaming_response
self.response_iterator = self.streaming_response
self.json_mode = json_mode
self.http_response: Optional["httpx.Response"] = None
async def aclose(self) -> None:
"""Close the upstream HTTP response so the provider connection is
released (and a backend like vLLM aborts generation) when the stream
is abandoned before its natural end.
``streaming_response`` is usually a bare ``aiter_lines()`` generator
that holds no reference to the response, so the handler that owns the
response attaches it here after construction."""
if self.http_response is not None:
await self.http_response.aclose()
def chunk_parser(
self, chunk: dict

46
litellm/llms/bedrock/base_aws_llm.py
View file

@ -861,14 +861,58 @@ class BaseAWSLLM:
with tracer.trace("boto3.client(sts)"):
sts_client = boto3.client("sts", **sts_client_kwargs)
# The session policy is an IAM PERMISSION CEILING — effective
# permissions are the intersection of the role's identity policies
# and this policy. Any action not listed here is silently denied
# even when the IAM role grants it. So every Bedrock route we
# support needs a matching action statement, or it 403s on OIDC
# auth only (static creds + IRSA take other code paths).
# https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html
bedrock_session_policy = {
"Version": "2012-10-17",
"Statement": [
{
"Sid": "BedrockLiteLLM",
"Effect": "Allow",
"Action": [
"bedrock:InvokeModel",
"bedrock:InvokeModelWithResponseStream",
"bedrock:ApplyGuardrail",
"bedrock:GetGuardrail",
"bedrock:ListGuardrails",
],
"Resource": "*",
"Condition": {"Bool": {"aws:SecureTransport": "true"}},
},
# Claude Platform on AWS (added by #27678 for the
# ``bedrock/claude_platform/<model>`` route) lives under
# a separate IAM action namespace; without these entries
# the OIDC path 403s on every claude_platform request
# even with a fully permissive identity policy (#30200).
{
"Sid": "ClaudePlatformLiteLLM",
"Effect": "Allow",
"Action": [
"aws-external-anthropic:CreateInference",
"aws-external-anthropic:CreateBatchInference",
"aws-external-anthropic:CancelBatchInference",
"aws-external-anthropic:DeleteBatchInference",
"aws-external-anthropic:CountTokens",
"aws-external-anthropic:Get*",
"aws-external-anthropic:List*",
],
"Resource": "*",
"Condition": {"Bool": {"aws:SecureTransport": "true"}},
},
],
}
assume_role_params = {
"RoleArn": aws_role_name,
"RoleSessionName": aws_session_name,
"WebIdentityToken": oidc_token,
"DurationSeconds": 3600,
"Policy": '{"Version":"2012-10-17","Statement":[{"Sid":"BedrockLiteLLM","Effect":"Allow","Action":["bedrock:InvokeModel","bedrock:InvokeModelWithResponseStream","bedrock:ApplyGuardrail","bedrock:GetGuardrail","bedrock:ListGuardrails"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}}]}',
"Policy": json.dumps(bedrock_session_policy, separators=(",", ":")),
}
# Add ExternalId parameter if provided

6
litellm/llms/bedrock/chat/converse_handler.py
View file

@ -32,7 +32,7 @@ def make_sync_call(
logging_obj: LiteLLMLoggingObject,
json_mode: Optional[bool] = False,
fake_stream: bool = False,
stream_chunk_size: int = 1024,
stream_chunk_size: Optional[int] = None,
):
if client is None:
client = _get_httpx_client() # Create a new client if none provided
@ -108,7 +108,7 @@ class BedrockConverseLLM(BaseAWSLLM):
fake_stream: bool = False,
json_mode: Optional[bool] = False,
api_key: Optional[str] = None,
stream_chunk_size: int = 1024,
stream_chunk_size: Optional[int] = None,
) -> CustomStreamWrapper:
request_data = await litellm.AmazonConverseConfig()._async_transform_request(
model=model,
@ -268,7 +268,7 @@ class BedrockConverseLLM(BaseAWSLLM):
):
## SETUP ##
stream = optional_params.pop("stream", None)
stream_chunk_size = optional_params.pop("stream_chunk_size", 1024)
stream_chunk_size = optional_params.pop("stream_chunk_size", None)
unencoded_model_id = optional_params.pop("model_id", None)
fake_stream = optional_params.pop("fake_stream", False)
json_mode = optional_params.get("json_mode", False)

10
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -197,7 +197,7 @@ async def make_call(
fake_stream: bool = False,
json_mode: Optional[bool] = False,
bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None,
stream_chunk_size: int = 1024,
stream_chunk_size: Optional[int] = None,
):
try:
if client is None:
@ -294,7 +294,7 @@ def make_sync_call(
fake_stream: bool = False,
json_mode: Optional[bool] = False,
bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None,
stream_chunk_size: int = 1024,
stream_chunk_size: Optional[int] = None,
):
try:
if client is None:
@ -790,7 +790,7 @@ class BedrockLLM(BaseAWSLLM):
## SETUP ##
stream = optional_params.pop("stream", None)
stream_chunk_size = optional_params.pop("stream_chunk_size", 1024)
stream_chunk_size = optional_params.pop("stream_chunk_size", None)
provider = self.get_bedrock_invoke_provider(model)
modelId = self.get_bedrock_model_id(
@ -1203,7 +1203,7 @@ class BedrockLLM(BaseAWSLLM):
extra_headers: Optional[dict] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[AsyncHTTPHandler] = None,
stream_chunk_size: int = 1024,
stream_chunk_size: Optional[int] = None,
) -> Union[ModelResponse, CustomStreamWrapper]:
transformed_request = (
await litellm.AmazonAnthropicClaudeConfig().async_transform_request(
@ -1350,7 +1350,7 @@ class BedrockLLM(BaseAWSLLM):
logger_fn=None,
headers={},
client: Optional[AsyncHTTPHandler] = None,
stream_chunk_size: int = 1024,
stream_chunk_size: Optional[int] = None,
) -> CustomStreamWrapper:
# The call is not made here; instead, we prepare the necessary objects for the stream.

1
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -215,6 +215,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
anthropic_request.pop("model", None)
anthropic_request.pop("stream", None)
anthropic_request.pop("stream_chunk_size", None)
output_format = anthropic_request.pop("output_format", None)
output_config_format = pop_bedrock_invoke_output_config_format(
anthropic_request

3
litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py
View file

@ -150,6 +150,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
) -> dict:
## SETUP ##
stream = optional_params.pop("stream", None)
optional_params.pop("stream_chunk_size", None)
custom_prompt_dict: dict = litellm_params.pop("custom_prompt_dict", None) or {}
hf_model_name = litellm_params.get("hf_model_name", None)
@ -256,7 +257,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
return request_data
def transform_response( # noqa: PLR0915
def transform_response(
self,
model: str,
raw_response: httpx.Response,

2
litellm/llms/bytez/chat/transformation.py
View file

@ -191,7 +191,7 @@ class BytezChatConfig(BaseConfig):
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ModelResponse:
json = raw_response.json() # noqa: F811
json = raw_response.json()
error = json.get("error")

7
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -33,7 +33,10 @@ from litellm.llms.base_llm.anthropic_messages.transformation import (
from litellm.llms.base_llm.audio_transcription.transformation import (
BaseAudioTranscriptionConfig,
)
from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
from litellm.llms.base_llm.base_model_iterator import (
BaseModelResponseIterator,
MockResponseIterator,
)
from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
from litellm.llms.base_llm.chat.transformation import BaseConfig
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
@ -814,6 +817,8 @@ class BaseLLMHTTPHandler:
completion_stream = provider_config.get_model_response_iterator(
streaming_response=response.aiter_lines(), sync_stream=False
)
if isinstance(completion_stream, BaseModelResponseIterator):
completion_stream.http_response = response
# LOGGING
logging_obj.post_call(
input=messages,

7
litellm/llms/fastcrw/__init__.py Normal file
View file

@ -0,0 +1,7 @@
"""
fastCRW API integration module.
"""
from litellm.llms.fastcrw.search.transformation import FastCRWSearchConfig
__all__ = ["FastCRWSearchConfig"]

7
litellm/llms/fastcrw/search/__init__.py Normal file
View file

@ -0,0 +1,7 @@
"""
fastCRW Search API module.
"""
from litellm.llms.fastcrw.search.transformation import FastCRWSearchConfig
__all__ = ["FastCRWSearchConfig"]

182
litellm/llms/fastcrw/search/transformation.py Normal file
View file

@ -0,0 +1,182 @@
"""
Calls fastCRW's /v1/search endpoint to search the web.
fastCRW is a Firecrawl-compatible web data engine (single Rust binary; self-host
or cloud). The search response uses the Firecrawl-compatible envelope
{ "success": true, "data": [ { "title", "url", "description", "markdown"? } ] }.
fastCRW API Reference: https://fastcrw.com/docs/rest-api
"""
from typing import Optional, TypedDict, Union
import httpx
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.search.transformation import (
BaseSearchConfig,
SearchResponse,
SearchResult,
)
from litellm.secret_managers.main import get_secret_str
class _FastCRWSearchRequestRequired(TypedDict):
"""Required fields for fastCRW Search API request."""
query: str # Required - search query
class FastCRWSearchRequest(_FastCRWSearchRequestRequired, total=False):
"""
fastCRW Search API request format.
Based on: https://fastcrw.com/docs/rest-api
"""
limit: int # Optional - maximum number of results to return
sources: list[
str
] # Optional - sources to search ('web', 'images'), default ['web']
scrapeOptions: dict # Optional - options for scraping search results
class FastCRWSearchConfig(BaseSearchConfig):
FASTCRW_API_BASE = "https://fastcrw.com/api/v1"
@staticmethod
def ui_friendly_name() -> str:
return "fastCRW"
def validate_environment(
self,
headers: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
**kwargs,
) -> dict:
"""
Validate environment and return headers.
"""
api_key = api_key or get_secret_str("CRW_API_KEY")
if not api_key:
raise ValueError(
"CRW_API_KEY is not set. Set `CRW_API_KEY` environment variable."
)
headers["Authorization"] = f"Bearer {api_key}"
headers["Content-Type"] = "application/json"
return headers
def get_complete_url(
self,
api_base: Optional[str],
optional_params: dict,
data: Optional[Union[dict, list[dict]]] = None,
**kwargs,
) -> str:
"""
Get complete URL for Search endpoint.
"""
api_base = api_base or get_secret_str("CRW_API_BASE") or self.FASTCRW_API_BASE
# Append "/search" to the api base if it's not already there
if not api_base.endswith("/search"):
api_base = f"{api_base}/search"
return api_base
def transform_search_request(
self,
query: Union[str, list[str]],
optional_params: dict,
**kwargs,
) -> dict:
"""
Transform Search request to fastCRW API format.
Transforms Perplexity unified spec parameters:
- query -> query (same)
- max_results -> limit
All other fastCRW-specific parameters are passed through as-is.
Args:
query: Search query (string or list of strings). fastCRW only supports single string queries.
optional_params: Optional parameters for the request
Returns:
Dict with typed request data following FastCRWSearchRequest spec
"""
if isinstance(query, list):
# fastCRW only supports single string queries, join with spaces
query = " ".join(query)
request_data: FastCRWSearchRequest = {
"query": query,
}
# Transform Perplexity unified spec parameters to fastCRW format
if "max_results" in optional_params:
request_data["limit"] = optional_params["max_results"]
# Convert to dict before dynamic key assignments
result_data = dict(request_data)
# pass through all other parameters as-is
for param, value in optional_params.items():
if (
param not in self.get_supported_perplexity_optional_params()
and param not in result_data
):
result_data[param] = value
# By default, request markdown content if not explicitly specified
# fastCRW doesn't return content unless explicitly requested via scrapeOptions
if "scrapeOptions" not in result_data:
result_data["scrapeOptions"] = {
"formats": ["markdown"],
"onlyMainContent": True,
}
return result_data
def transform_search_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
**kwargs,
) -> SearchResponse:
"""
Transform fastCRW API response to LiteLLM unified SearchResponse format.
fastCRW (Firecrawl-compatible) returns:
{"success": true, "data": [{"url": "...", "title": "...", "description": "...", "markdown"?: "..."}, ...]}
Args:
raw_response: Raw httpx response from fastCRW API
logging_obj: Logging object for tracking
Returns:
SearchResponse with standardized format
"""
response_json = raw_response.json()
results = []
data = response_json.get("data", [])
if isinstance(data, list):
for result in data:
snippet = result.get("markdown") or result.get("description", "")
search_result = SearchResult(
title=result.get("title", ""),
url=result.get("url", ""),
snippet=snippet,
date=None,
last_updated=None,
)
results.append(search_result)
return SearchResponse(
results=results,
object="search",
)

93
litellm/llms/modelscope/chat/transformation.py Normal file
View file

@ -0,0 +1,93 @@
"""
Translates from OpenAI's `/v1/chat/completions` to ModelScope's `/v1/chat/completions`
"""
from typing import Any, Coroutine, Literal, Optional, Tuple, Union, cast, overload
from typing_extensions import override
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
def _has_non_text_content(message: AllMessageValues) -> bool:
"""Check if a message has non-text content items (e.g. image_url)."""
content = message.get("content")
if not isinstance(content, list):
return False
return any(item.get("type") != "text" for item in content)
class ModelScopeChatConfig(OpenAIGPTConfig):
DEFAULT_BASE_URL: str = "https://api-inference.modelscope.cn/v1"
@overload
def _transform_messages(
self, messages: list[AllMessageValues], model: str, is_async: Literal[True]
) -> Coroutine[Any, Any, list[AllMessageValues]]: ...
@overload
def _transform_messages(
self,
messages: list[AllMessageValues],
model: str,
is_async: Literal[False] = False,
) -> list[AllMessageValues]: ...
def _transform_messages(
self, messages: list[AllMessageValues], model: str, is_async: bool = False
) -> Union[list[AllMessageValues], Coroutine[Any, Any, list[AllMessageValues]]]:
"""
Flatten text-only content lists to strings for ModelScope.
Messages with non-text content (e.g. image_url for vision models)
are kept as lists so the parent class can normalize them properly.
"""
messages = [cast(AllMessageValues, {**m}) for m in messages]
for message in messages:
if _has_non_text_content(message):
continue
content = message.get("content")
if isinstance(content, list):
message["content"] = "".join(item.get("text") or "" for item in content)
if is_async:
return super()._transform_messages(
messages=messages, model=model, is_async=True
)
else:
return super()._transform_messages(
messages=messages, model=model, is_async=False
)
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:
api_base = (
api_base or get_secret_str("MODELSCOPE_API_BASE") or self.DEFAULT_BASE_URL
) # type: ignore
dynamic_api_key = api_key or get_secret_str("MODELSCOPE_API_KEY")
return api_base, dynamic_api_key
@override
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
If api_base is not provided, use the default ModelScope /chat/completions endpoint.
"""
if not api_base:
api_base = self.DEFAULT_BASE_URL
if not api_base.endswith("/chat/completions"):
api_base = f"{api_base}/chat/completions"
return api_base

31
litellm/llms/modelscope/image_generation/__init__.py Normal file
View file

@ -0,0 +1,31 @@
"""
ModelScope Image Generation Module
Factory function for getting the appropriate config class.
"""
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from .transformation import ModelScopeImageGenerationConfig
__all__ = [
"ModelScopeImageGenerationConfig",
"get_modelscope_image_generation_config",
]
def get_modelscope_image_generation_config(
model: str,
) -> BaseImageGenerationConfig:
"""
Get the ModelScope config for image generation.
Args:
model: The model name (e.g., "modelscope/Qwen/Qwen-Image-Edit")
Returns:
BaseImageGenerationConfig instance for ModelScope
"""
return ModelScopeImageGenerationConfig()

248
litellm/llms/modelscope/image_generation/transformation.py Normal file
View file

@ -0,0 +1,248 @@
"""
ModelScope Image Generation Config
Handles transformation between OpenAI-compatible format and ModelScope API format.
API Reference: https://modelscope.cn/docs/model-service/API-Inference/intro
"""
from typing import TYPE_CHECKING, Optional, Union
import httpx
from typing_extensions import override
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
AllMessageValues,
OpenAIImageGenerationOptionalParams,
)
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = object
class ModelScopeImageGenerationConfig(BaseImageGenerationConfig):
"""
Configuration for ModelScope image generation.
Supports text-to-image models like:
- Qwen/Qwen-Image-Edit
- And other ModelScope-hosted image generation models
"""
DEFAULT_BASE_URL: str = "https://api-inference.modelscope.cn/v1"
def get_supported_openai_params(
self, model: str
) -> list[OpenAIImageGenerationOptionalParams]:
"""
Return list of OpenAI params supported by ModelScope.
ModelScope supports standard OpenAI image generation parameters.
"""
return [
"n", # Number of images to generate
"size", # Size of the generated images
"response_format", # url or b64_json
"user", # User identifier
]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
"""
Map OpenAI parameters to ModelScope parameters.
ModelScope uses the same parameter names as OpenAI.
"""
supported_params = self.get_supported_openai_params(model)
if drop_params:
non_default_params = {
k: v for k, v in non_default_params.items() if k in supported_params
}
optional_params.update(non_default_params)
return optional_params
@override
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
Get the complete URL for the ModelScope image generation API request.
"""
base_url: str = (
api_base or get_secret_str("MODELSCOPE_API_BASE") or self.DEFAULT_BASE_URL
)
base_url = base_url.rstrip("/")
# Return the images endpoint
return f"{base_url}/images/generations"
@override
def validate_environment(
self,
headers: dict,
model: str,
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
"""
Validate environment and set up headers for ModelScope.
"""
final_api_key: Optional[str] = api_key or get_secret_str("MODELSCOPE_API_KEY")
if not final_api_key:
raise ValueError(
"MODELSCOPE_API_KEY is not set. "
"Please set it via environment variable or pass api_key parameter."
)
default_headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {final_api_key}",
}
headers = {**headers, **default_headers}
return headers
def transform_image_generation_request(
self,
model: str,
prompt: str,
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
"""
Transform OpenAI-style request to ModelScope request format.
ModelScope uses the same format as OpenAI for image generation.
"""
# Build the request body (same as OpenAI)
request_data: dict = {
"model": model,
"prompt": prompt,
}
# Add optional params
for key, value in optional_params.items():
if key.startswith("_"):
continue
request_data[key] = value
return request_data
@override
def transform_image_generation_response(
self,
model: str,
raw_response: httpx.Response,
model_response: ImageResponse,
logging_obj: LiteLLMLoggingObj,
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: object,
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ImageResponse:
"""
Transform ModelScope response to OpenAI-compatible ImageResponse.
ModelScope returns the same format as OpenAI:
{"created": timestamp, "data": [{"url": "..."}]}
"""
try:
response_data = raw_response.json()
except Exception as e:
raise self.get_error_class(
error_message=f"Error parsing ModelScope response: {e}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
# Check for errors in response
if "error" in response_data:
error_msg = response_data["error"].get(
"message", str(response_data["error"])
)
raise self.get_error_class(
error_message=f"ModelScope error: {error_msg}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
# Extract images from response
data_list = response_data.get("data", [])
if not model_response.data:
model_response.data = []
for item in data_list:
image_obj = ImageObject(
url=item.get("url"),
b64_json=item.get("b64_json"),
revised_prompt=item.get("revised_prompt"),
)
model_response.data.append(image_obj)
return model_response
def get_error_class(
self,
error_message: str,
status_code: int,
headers: Union[dict, httpx.Headers],
) -> BaseLLMException:
"""Return the appropriate error class for ModelScope."""
from litellm.exceptions import (
AuthenticationError,
BadRequestError,
InternalServerError,
)
if status_code == 400:
return BadRequestError( # type: ignore[return-value]
message=error_message,
model="",
llm_provider="modelscope",
)
elif status_code == 401:
return AuthenticationError( # type: ignore[return-value]
message=error_message,
model="",
llm_provider="modelscope",
)
elif status_code >= 500:
return InternalServerError( # type: ignore[return-value]
message=error_message,
model="",
llm_provider="modelscope",
)
else:
return BadRequestError( # type: ignore[return-value]
message=error_message,
model="",
llm_provider="modelscope",
)

20
litellm/llms/openai_like/providers.json
View file

@ -131,7 +131,8 @@
"base_class": "openai_gpt",
"param_mappings": {
"max_completion_tokens": "max_tokens"
}
},
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
},
"parasail": {
"base_url": "https://api.parasail.io/v1",
@ -141,5 +142,22 @@
"special_handling": {
"force_store_false": true
}
},
"libertai": {
"base_url": "https://api.libertai.io/v1",
"api_key_env": "LIBERTAI_API_KEY",
"api_base_env": "LIBERTAI_API_BASE",
"param_mappings": {
"max_completion_tokens": "max_tokens"
}
},
"empiriolabs": {
"base_url": "https://api.empiriolabs.ai/v1",
"api_key_env": "EMPIRIOLABS_API_KEY",
"api_base_env": "EMPIRIOLABS_API_BASE",
"param_mappings": {
"max_completion_tokens": "max_tokens"
},
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
}
}

2
litellm/llms/sagemaker/completion/handler.py
View file

@ -138,7 +138,7 @@ class SagemakerLLM(BaseAWSLLM):
return prepped_request
def completion( # noqa: PLR0915
def completion(
self,
model: str,
messages: list,

827
litellm/llms/snowflake/chat/transformation.py
View file

@ -1,17 +1,32 @@
"""
Support for Snowflake REST API
Snowflake Cortex REST API — Chat Transformation
Routes to native Cortex REST API endpoints based on model:
- Claude models → POST /api/v2/cortex/v1/messages (Anthropic format)
- All other models → POST /api/v2/cortex/v1/chat/completions (OpenAI format)
Ref: https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api
"""
import json
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from typing import TYPE_CHECKING, Any, Dict, List, Optional
import httpx
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ChatCompletionMessageToolCall, Function, ModelResponse
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk
from litellm.types.utils import (
ChatCompletionMessageToolCall,
ChatCompletionUsageBlock,
Choices,
Function,
GenericStreamingChunk,
Message,
ModelResponse,
Usage,
)
from ...base_llm.base_model_iterator import BaseModelResponseIterator
from ...openai_like.chat.transformation import OpenAIGPTConfig
from ..utils import SnowflakeBaseConfig
if TYPE_CHECKING:
@ -21,69 +36,343 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
ANTHROPIC_VERSION = "2023-06-01"
_CLAUDE_MODEL_PREFIXES = (
"claude-",
"claude_",
)
def _is_claude_model(model: str) -> bool:
"""Return True if model name (after stripping snowflake/ prefix) is a Claude model."""
name = model.lower().removeprefix("snowflake/")
return any(name.startswith(p) for p in _CLAUDE_MODEL_PREFIXES)
class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
"""
Reference: https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-llm-rest-api
Snowflake Cortex REST API — unified provider.
Snowflake Cortex LLM REST API supports function calling with specific models (e.g., Claude 3.5 Sonnet).
This config handles transformation between OpenAI format and Snowflake's tool_spec format.
Auto-routes based on model name:
- Claude models → /api/v2/cortex/v1/messages (Anthropic Messages format)
- All others → /api/v2/cortex/v1/chat/completions (OpenAI format)
Auth:
PAT: api_key="pat/<token>" → X-Snowflake-Authorization-Token-Type: PROGRAMMATIC_ACCESS_TOKEN
JWT: api_key="<jwt>" → X-Snowflake-Authorization-Token-Type: KEYPAIR_JWT
"""
@classmethod
def get_config(cls):
return super().get_config()
def _transform_tool_calls_from_snowflake_to_openai(
self, content_list: List[Dict[str, Any]]
) -> Tuple[str, Optional[List[ChatCompletionMessageToolCall]]]:
def get_supported_openai_params(self, model: str) -> List[str]:
params = [
"temperature",
"max_tokens",
"max_completion_tokens",
"top_p",
"stream",
"tools",
"tool_choice",
]
if _is_claude_model(model):
params.append("thinking")
return params
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
api_base = self._get_api_base(api_base, optional_params)
if _is_claude_model(model):
return f"{api_base}/cortex/v1/messages"
return f"{api_base}/cortex/v1/chat/completions"
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
headers = super().validate_environment(
headers=headers,
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
api_key=api_key,
api_base=api_base,
)
if _is_claude_model(model):
headers["anthropic-version"] = ANTHROPIC_VERSION
return headers
def _transform_tools_to_anthropic(self, tools: List[Dict]) -> List[Dict]:
"""
Transform Snowflake tool calls to OpenAI format.
Convert tools from OpenAI format to Anthropic format.
Args:
content_list: Snowflake's content_list array containing text and tool_use items
OpenAI: {"type": "function", "function": {"name": ..., "parameters": {...}}}
Anthropic: {"name": ..., "description": ..., "input_schema": {...}}
"""
anthropic_tools = []
for tool in tools:
if tool.get("type") == "function" and "function" in tool:
func = tool["function"]
anthropic_tool: Dict[str, Any] = {
"name": func.get("name", ""),
}
if "description" in func:
anthropic_tool["description"] = func["description"]
if "parameters" in func:
anthropic_tool["input_schema"] = func["parameters"]
else:
anthropic_tool["input_schema"] = {
"type": "object",
"properties": {},
}
anthropic_tools.append(anthropic_tool)
else:
anthropic_tools.append(tool)
return anthropic_tools
Returns:
Tuple of (text_content, tool_calls)
def _extract_system_and_messages(
self, messages: List[AllMessageValues]
) -> tuple[Optional[str], List[Dict]]:
"""
Split messages into system prompt and conversation turns for Anthropic format.
Snowflake format in content_list:
{
"type": "tool_use",
"tool_use": {
"tool_use_id": "tooluse_...",
"name": "get_weather",
"input": {"location": "Paris"}
}
- system messages → collected and joined (preserves guardrail prompts)
- assistant messages with tool_calls → tool_use content blocks
- tool role messages → user role with tool_result content blocks
"""
system_parts: List[str] = []
conversation: List[Dict] = []
for msg in messages:
if isinstance(msg, dict):
role = msg.get("role", "")
content: Any = msg.get("content", "")
else:
role = getattr(msg, "role", "")
content = getattr(msg, "content", "")
if role == "system":
if isinstance(content, str) and content:
system_parts.append(content)
elif isinstance(content, list):
system_parts.append(
"\n".join(
b.get("text", "")
for b in content
if b.get("type") == "text"
)
)
elif role == "assistant":
tool_calls = (
msg.get("tool_calls")
if isinstance(msg, dict)
else getattr(msg, "tool_calls", None)
)
if tool_calls: # type: ignore[truthy-bool]
content_blocks: List[Dict[str, Any]] = []
if content:
content_blocks.append({"type": "text", "text": content})
for tc in tool_calls: # type: ignore[attr-defined]
func = (
tc.get("function", {})
if isinstance(tc, dict)
else getattr(tc, "function", {})
)
tc_id = (
tc.get("id", "")
if isinstance(tc, dict)
else getattr(tc, "id", "")
)
func_name = (
func.get("name", "")
if isinstance(func, dict)
else getattr(func, "name", "")
)
func_args = (
func.get("arguments", "{}")
if isinstance(func, dict)
else getattr(func, "arguments", "{}")
)
try:
input_data = (
json.loads(func_args)
if isinstance(func_args, str)
else func_args
)
except (json.JSONDecodeError, TypeError):
input_data = {}
content_blocks.append(
{
"type": "tool_use",
"id": tc_id,
"name": func_name,
"input": input_data,
}
)
conversation.append(
{"role": "assistant", "content": content_blocks}
)
else:
conversation.append({"role": "assistant", "content": content})
elif role == "tool":
tool_call_id = (
msg.get("tool_call_id", "")
if isinstance(msg, dict)
else getattr(msg, "tool_call_id", "")
)
tool_content = (
content if isinstance(content, str) else json.dumps(content)
)
tool_result_block = {
"type": "tool_result",
"tool_use_id": tool_call_id,
"content": tool_content,
}
if (
conversation
and conversation[-1]["role"] == "user"
and isinstance(conversation[-1]["content"], list)
and conversation[-1]["content"]
and conversation[-1]["content"][0].get("type") == "tool_result"
):
conversation[-1]["content"].append(tool_result_block)
else:
conversation.append(
{"role": "user", "content": [tool_result_block]}
)
else:
conversation.append({"role": role, "content": content})
system: Optional[str] = "\n\n".join(system_parts) if system_parts else None
return system, conversation
def transform_request(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
stream: bool = optional_params.pop("stream", False) or False
extra_body = optional_params.pop("extra_body", {})
if _is_claude_model(model):
return self._transform_request_anthropic(
model, messages, optional_params, stream, extra_body
)
return self._transform_request_openai(
model, messages, optional_params, stream, extra_body
)
def _transform_request_openai(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
stream: bool,
extra_body: dict,
) -> dict:
"""OpenAI format for /chat/completions endpoint."""
max_tokens = optional_params.pop("max_tokens", None)
max_completion_tokens = optional_params.pop("max_completion_tokens", None)
resolved_max = max_completion_tokens or max_tokens
body: dict = {
"model": model.removeprefix("snowflake/"),
"messages": messages,
"stream": stream,
**optional_params,
**extra_body,
}
OpenAI format (returned tool_calls):
ChatCompletionMessageToolCall(
id="tooluse_...",
type="function",
function=Function(name="get_weather", arguments='{"location": "Paris"}')
)
if resolved_max is not None:
body["max_completion_tokens"] = resolved_max
return body
def _transform_tool_choice_to_anthropic(self, tool_choice: Any) -> Dict[str, Any]:
"""
text_content = ""
tool_calls: List[ChatCompletionMessageToolCall] = []
Convert tool_choice from OpenAI format to Anthropic format.
for idx, content_item in enumerate(content_list):
if content_item.get("type") == "text":
text_content += content_item.get("text", "")
OpenAI string values: "auto", "required", "none"
OpenAI dict: {"type": "function", "function": {"name": "..."}}
Anthropic: {"type": "auto"}, {"type": "any"}, {"type": "tool", "name": "..."}
"""
if isinstance(tool_choice, str):
mapping = {
"auto": {"type": "auto"},
"required": {"type": "any"},
"none": {"type": "none"},
}
return mapping.get(tool_choice, {"type": "auto"})
elif isinstance(tool_choice, dict):
if tool_choice.get("type") == "function":
func = tool_choice.get("function", {})
return {"type": "tool", "name": func.get("name", "")}
return tool_choice
return {"type": "auto"}
## TOOL CALLING
elif content_item.get("type") == "tool_use":
tool_use_data = content_item.get("tool_use", {})
tool_call = ChatCompletionMessageToolCall(
id=tool_use_data.get("tool_use_id", ""),
type="function",
function=Function(
name=tool_use_data.get("name", ""),
arguments=json.dumps(tool_use_data.get("input", {})),
),
)
tool_calls.append(tool_call)
def _transform_request_anthropic(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
stream: bool,
extra_body: dict,
) -> dict:
"""Anthropic Messages format for /messages endpoint."""
system, conversation = self._extract_system_and_messages(messages)
return text_content, tool_calls if tool_calls else None
if "tools" in optional_params:
optional_params["tools"] = self._transform_tools_to_anthropic(
optional_params["tools"]
)
if "tool_choice" in optional_params:
optional_params["tool_choice"] = self._transform_tool_choice_to_anthropic(
optional_params["tool_choice"]
)
max_completion_tokens = optional_params.pop("max_completion_tokens", None)
if max_completion_tokens and "max_tokens" not in optional_params:
optional_params["max_tokens"] = max_completion_tokens
model_name = model.removeprefix("snowflake/")
body: Dict[str, Any] = {
"model": model_name,
"messages": conversation,
"stream": stream,
**optional_params,
**extra_body,
}
if system is not None:
body["system"] = system
if "max_tokens" not in body:
body["max_tokens"] = (
4096 # reasonable default; Anthropic API max varies by model
)
return body
def transform_response(
self,
@ -99,6 +388,24 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ModelResponse:
if _is_claude_model(model):
return self._transform_response_anthropic(
model, raw_response, model_response, logging_obj, request_data, messages
)
return self._transform_response_openai(
model, raw_response, model_response, logging_obj, request_data, messages
)
def _transform_response_openai(
self,
model: str,
raw_response: httpx.Response,
model_response: ModelResponse,
logging_obj: LiteLLMLoggingObj,
request_data: dict,
messages: List[AllMessageValues],
) -> ModelResponse:
"""Parse standard OpenAI chat completions response."""
response_json = raw_response.json()
logging_obj.post_call(
@ -108,180 +415,278 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
additional_args={"complete_input_dict": request_data},
)
## RESPONSE TRANSFORMATION
# Snowflake returns content_list (not content) with tool_use objects
# We need to transform this to OpenAI's format with content + tool_calls
if "choices" in response_json and len(response_json["choices"]) > 0:
choice = response_json["choices"][0]
if "message" in choice and "content_list" in choice["message"]:
content_list = choice["message"]["content_list"]
(
text_content,
tool_calls,
) = self._transform_tool_calls_from_snowflake_to_openai(content_list)
# Update the choice message with OpenAI format
choice["message"]["content"] = text_content
if tool_calls:
choice["message"]["tool_calls"] = tool_calls
# Remove Snowflake-specific content_list
del choice["message"]["content_list"]
returned_response = ModelResponse(**response_json)
returned_response.model = "snowflake/" + (returned_response.model or "")
if model is not None:
returned_response._hidden_params["model"] = model
return returned_response
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
If api_base is not provided, use the default DeepSeek /chat/completions endpoint.
"""
api_base = self._get_api_base(api_base, optional_params)
return f"{api_base}/cortex/inference:complete"
def _transform_tools(self, tools: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""
Transform OpenAI tool format to Snowflake tool format.
Args:
tools: List of tools in OpenAI format
Returns:
List of tools in Snowflake format
OpenAI format:
{
"type": "function",
"function": {
"name": "get_weather",
"description": "...",
"parameters": {...}
}
}
Snowflake format:
{
"tool_spec": {
"type": "generic",
"name": "get_weather",
"description": "...",
"input_schema": {...}
}
}
"""
snowflake_tools: List[Dict[str, Any]] = []
for tool in tools:
if tool.get("type") == "function":
function = tool.get("function", {})
snowflake_tool: Dict[str, Any] = {
"tool_spec": {
"type": "generic",
"name": function.get("name"),
"input_schema": function.get(
"parameters",
{"type": "object", "properties": {}},
),
}
}
# Add description if present
if "description" in function:
snowflake_tool["tool_spec"]["description"] = function["description"]
snowflake_tools.append(snowflake_tool)
return snowflake_tools
def _transform_tool_choice(
self, tool_choice: Union[str, Dict[str, Any]]
) -> Dict[str, Any]:
"""
Transform OpenAI tool_choice format to Snowflake format.
Snowflake requires tool_choice to be an object, not a string.
Ref: https://docs.snowflake.com/en/developer-guide/snowflake-rest-api/reference/cortex-inference#post--api-v2-cortex-inference-complete-req-body-schema
Args:
tool_choice: Tool choice in OpenAI format (str or dict)
Returns:
Tool choice in Snowflake format (always an object, never a string)
OpenAI format (string):
"auto", "required", "none"
OpenAI format (dict):
{"type": "function", "function": {"name": "get_weather"}}
Snowflake format:
{"type": "auto"} / {"type": "any"} / {"type": "none"}
{"type": "tool", "name": ["get_weather"]}
Snowflake's API (like Anthropic) requires tool_choice as an object
with a "type" field, not as a bare string.
"""
if isinstance(tool_choice, str):
# Snowflake requires object format, not string.
# Map OpenAI string values to Snowflake object format.
# "required" maps to "any" (Snowflake/Anthropic convention).
_type_map = {
"auto": "auto",
"required": "any",
"none": "none",
}
mapped_type = _type_map.get(tool_choice, tool_choice)
return {"type": mapped_type}
if isinstance(tool_choice, dict):
if tool_choice.get("type") == "function":
function_name = tool_choice.get("function", {}).get("name")
if function_name:
return {
"type": "tool",
"name": [function_name], # Snowflake expects array
}
return tool_choice
def transform_request(
def _transform_response_anthropic(
self,
model: str,
raw_response: httpx.Response,
model_response: ModelResponse,
logging_obj: LiteLLMLoggingObj,
request_data: dict,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
stream: bool = optional_params.pop("stream", None) or False
extra_body = optional_params.pop("extra_body", {})
) -> ModelResponse:
"""Parse Anthropic Messages response into OpenAI format."""
response_json = raw_response.json()
## TOOL CALLING
# Transform tools from OpenAI format to Snowflake's tool_spec format
tools = optional_params.pop("tools", None)
if tools:
optional_params["tools"] = self._transform_tools(tools)
logging_obj.post_call(
input=messages,
api_key="",
original_response=response_json,
additional_args={"complete_input_dict": request_data},
)
# Transform tool_choice from OpenAI format to Snowflake's tool name array format
tool_choice = optional_params.pop("tool_choice", None)
if tool_choice:
optional_params["tool_choice"] = self._transform_tool_choice(tool_choice)
text_content = ""
tool_calls = []
return {
"model": model,
"messages": messages,
"stream": stream,
**optional_params,
**extra_body,
for block in response_json.get("content", []):
if block.get("type") == "text":
text_content += block.get("text", "")
elif block.get("type") == "tool_use":
tool_calls.append(
ChatCompletionMessageToolCall(
id=block.get("id", ""),
type="function",
function=Function(
name=block.get("name", ""),
arguments=json.dumps(block.get("input", {})),
),
)
)
_stop_reason_map = {
"end_turn": "stop",
"max_tokens": "length",
"tool_use": "tool_calls",
"stop_sequence": "stop",
}
finish_reason = _stop_reason_map.get(
response_json.get("stop_reason", "end_turn"), "stop"
)
message = Message(content=text_content or None, role="assistant")
if tool_calls:
message.tool_calls = tool_calls
choice = Choices(
finish_reason=finish_reason,
index=0,
message=message,
)
usage_data = response_json.get("usage", {})
usage = Usage(
prompt_tokens=usage_data.get("input_tokens", 0),
completion_tokens=usage_data.get("output_tokens", 0),
total_tokens=usage_data.get("input_tokens", 0)
+ usage_data.get("output_tokens", 0),
)
model_response.choices = [choice]
model_response.usage = usage # type: ignore[attr-defined]
model_response.model = "snowflake/" + response_json.get("model", model)
model_response.id = response_json.get("id", "")
if model is not None:
model_response._hidden_params["model"] = model
return model_response
def get_model_response_iterator(
self,
streaming_response: Any,
sync_stream: bool,
json_mode: Optional[bool] = False,
) -> Any:
return SnowflakeStreamingHandler(
streaming_response=streaming_response,
sync_stream=sync_stream,
json_mode=json_mode,
)
class SnowflakeStreamingHandler(BaseModelResponseIterator):
"""
Parse streaming events from both Snowflake endpoints.
- /chat/completions: OpenAI SSE format (has "choices" key)
- /messages: Anthropic SSE format (has "type" key like content_block_delta)
"""
def __init__(
self,
streaming_response: Any,
sync_stream: bool,
json_mode: Optional[bool] = False,
):
super().__init__(streaming_response=streaming_response, sync_stream=sync_stream)
self._tool_index = 0
self._tool_id = ""
self._tool_name = ""
self._input_tokens = 0
def chunk_parser(self, chunk: dict) -> GenericStreamingChunk:
if "choices" in chunk:
return self._parse_openai_chunk(chunk)
return self._parse_anthropic_chunk(chunk)
def _parse_openai_chunk(self, chunk: dict) -> GenericStreamingChunk:
choices = chunk.get("choices", [])
if not choices:
return GenericStreamingChunk(
text="",
is_finished=False,
finish_reason="",
usage=None,
index=0,
tool_use=None,
)
choice = choices[0]
delta = choice.get("delta", {})
finish_reason = choice.get("finish_reason") or ""
text = delta.get("content") or ""
tool_use = None
tool_calls = delta.get("tool_calls")
if tool_calls:
tc = tool_calls[0]
func = tc.get("function", {})
tool_use = ChatCompletionToolCallChunk(
id=tc.get("id", ""),
type="function",
function={
"name": func.get("name", ""),
"arguments": func.get("arguments", ""),
},
index=tc.get("index", 0),
)
return GenericStreamingChunk(
text=text,
is_finished=finish_reason != "",
finish_reason=finish_reason,
usage=None,
index=choice.get("index", 0),
tool_use=tool_use,
)
def _parse_anthropic_chunk(self, chunk: dict) -> GenericStreamingChunk:
event_type = chunk.get("type", "")
if event_type == "message_start":
message = chunk.get("message", {})
usage_data = message.get("usage", {})
self._input_tokens = usage_data.get("input_tokens", 0)
return GenericStreamingChunk(
text="",
is_finished=False,
finish_reason="",
usage=None,
index=0,
tool_use=None,
)
elif event_type == "content_block_delta":
delta = chunk.get("delta", {})
delta_type = delta.get("type", "")
if delta_type == "text_delta":
return GenericStreamingChunk(
text=delta.get("text", ""),
is_finished=False,
finish_reason="",
usage=None,
index=chunk.get("index", 0),
tool_use=None,
)
elif delta_type == "input_json_delta":
return GenericStreamingChunk(
text="",
is_finished=False,
finish_reason="",
usage=None,
index=chunk.get("index", 0),
tool_use=ChatCompletionToolCallChunk(
id=self._tool_id,
type="function",
function={
"name": self._tool_name,
"arguments": delta.get("partial_json", ""),
},
index=self._tool_index,
),
)
elif event_type == "content_block_start":
content_block = chunk.get("content_block", {})
if content_block.get("type") == "tool_use":
self._tool_id = content_block.get("id", "")
self._tool_name = content_block.get("name", "")
self._tool_index = chunk.get("index", 0)
return GenericStreamingChunk(
text="",
is_finished=False,
finish_reason="",
usage=None,
index=chunk.get("index", 0),
tool_use=ChatCompletionToolCallChunk(
id=self._tool_id,
type="function",
function={"name": self._tool_name, "arguments": ""},
index=self._tool_index,
),
)
elif event_type == "message_delta":
delta = chunk.get("delta", {})
stop_reason = delta.get("stop_reason", "")
usage_data = chunk.get("usage", {})
_stop_map = {
"end_turn": "stop",
"max_tokens": "length",
"tool_use": "tool_calls",
"stop_sequence": "stop",
}
usage = None
if usage_data or self._input_tokens:
output_t = usage_data.get("output_tokens", 0)
input_t = self._input_tokens or usage_data.get("input_tokens", 0)
usage = ChatCompletionUsageBlock(
prompt_tokens=input_t,
completion_tokens=output_t,
total_tokens=input_t + output_t,
)
return GenericStreamingChunk(
text="",
is_finished=True,
finish_reason=_stop_map.get(stop_reason, "stop"),
usage=usage,
index=0,
tool_use=None,
)
elif event_type == "message_stop":
return GenericStreamingChunk(
text="",
is_finished=True,
finish_reason="stop",
usage=None,
index=0,
tool_use=None,
)
return GenericStreamingChunk(
text="",
is_finished=False,
finish_reason="",
usage=None,
index=0,
tool_use=None,
)

2
litellm/llms/vertex_ai/vertex_ai_non_gemini.py
View file

@ -650,7 +650,7 @@ async def async_completion( # noqa: PLR0915
raise VertexAIError(status_code=500, message=str(e))
async def async_streaming( # noqa: PLR0915
async def async_streaming(
llm_model,
mode: str,
prompt: str,

183
litellm/llms/voyage/embedding/transformation_multimodal.py Normal file
View file

@ -0,0 +1,183 @@
"""
Transform request/response for Voyage multimodal embeddings.
Voyage multimodal models use /v1/multimodalembeddings and accept `inputs`
containing content blocks, unlike standard Voyage embeddings which use
/v1/embeddings and a string/list `input` field.
"""
from typing import Any, Dict, List, Optional, Union
import httpx
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
from litellm.types.utils import EmbeddingResponse, Usage
class VoyageMultimodalEmbeddingError(BaseLLMException):
def __init__(
self,
status_code: int,
message: str,
headers: Union[dict, httpx.Headers] = {},
):
self.status_code = status_code
self.message = message
self.request = httpx.Request(
method="POST", url="https://api.voyageai.com/v1/multimodalembeddings"
)
self.response = httpx.Response(status_code=status_code, request=self.request)
super().__init__(
status_code=status_code,
message=message,
headers=headers,
)
class VoyageMultimodalEmbeddingConfig(BaseEmbeddingConfig):
"""
Reference: https://docs.voyageai.com/reference/multimodal-embeddings-api
"""
@staticmethod
def is_multimodal_embeddings(model: str) -> bool:
return "multimodal" in model.lower()
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
if api_base:
if not api_base.endswith("/multimodalembeddings"):
api_base = f"{api_base}/multimodalembeddings"
return api_base
return "https://api.voyageai.com/v1/multimodalembeddings"
def get_supported_openai_params(self, model: str) -> list:
return ["dimensions"]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
if "dimensions" in non_default_params:
optional_params["output_dimension"] = non_default_params["dimensions"]
return optional_params
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
if api_key is None:
api_key = (
get_secret_str("VOYAGE_API_KEY")
or get_secret_str("VOYAGE_AI_API_KEY")
or get_secret_str("VOYAGE_AI_TOKEN")
)
if not api_key:
raise ValueError(
"Voyage API key is required for multimodal embeddings. "
"Set VOYAGE_API_KEY / VOYAGE_AI_API_KEY / VOYAGE_AI_TOKEN "
"or pass `api_key` explicitly."
)
return {"Authorization": f"Bearer {api_key}"}
def _normalize_content_item(self, item: Dict[str, Any]) -> Dict[str, Any]:
item_type = item.get("type")
if item_type == "image_url":
image_url = item.get("image_url")
if isinstance(image_url, dict):
image_url = image_url.get("url")
if image_url is None:
raise ValueError(
"Voyage multimodal embeddings require a non-empty `image_url`. "
"Got an image content block without a `url`."
)
if isinstance(image_url, str) and image_url.startswith("data:image/"):
_, _, encoded = image_url.partition(",")
return {"type": "image_base64", "image_base64": encoded}
return {"type": "image_url", "image_url": image_url}
return item
def _normalize_input_item(self, item: Any) -> Dict[str, Any]:
if isinstance(item, str):
return {"content": [{"type": "text", "text": item}]}
if isinstance(item, dict) and "content" in item:
content = item.get("content") or []
return {
**item,
"content": [
self._normalize_content_item(content_item)
for content_item in content
],
}
return item
def transform_embedding_request(
self,
model: str,
input: AllEmbeddingInputValues,
optional_params: dict,
headers: dict,
) -> dict:
inputs = input if isinstance(input, list) else [input]
return {
"inputs": [self._normalize_input_item(item) for item in inputs],
"model": model,
**optional_params,
}
def transform_embedding_response(
self,
model: str,
raw_response: httpx.Response,
model_response: EmbeddingResponse,
logging_obj: LiteLLMLoggingObj,
api_key: Optional[str] = None,
request_data: dict = {},
optional_params: dict = {},
litellm_params: dict = {},
) -> EmbeddingResponse:
try:
raw_response_json = raw_response.json()
except Exception:
raise VoyageMultimodalEmbeddingError(
message=raw_response.text, status_code=raw_response.status_code
)
model_response.model = raw_response_json.get("model")
model_response.data = raw_response_json.get("data")
model_response.object = raw_response_json.get("object")
usage_payload = raw_response_json.get("usage", {})
total_tokens = usage_payload.get("total_tokens", 0)
model_response.usage = Usage(
prompt_tokens=total_tokens,
total_tokens=total_tokens,
)
return model_response
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
) -> BaseLLMException:
return VoyageMultimodalEmbeddingError(
message=error_message, status_code=status_code, headers=headers
)

4
litellm/main.py
View file

@ -392,7 +392,7 @@ class AsyncCompletions:
@tracer.wrap()
@client
async def acompletion( # noqa: PLR0915
async def acompletion(
model: str,
# Optional OpenAI params: see https://platform.openai.com/docs/api-reference/chat/create
messages: List = [],
@ -7575,7 +7575,7 @@ def print_verbose(print_statement):
try:
verbose_logger.debug(print_statement)
if litellm.set_verbose:
print(print_statement) # noqa
print(print_statement) # noqa: T201
except Exception:
pass

254
litellm/model_prices_and_context_window_backup.json
View file

@ -18822,6 +18822,38 @@
"supports_response_schema": true,
"supports_vision": true
},
"github_copilot/mai-code-1-flash": {
"cache_read_input_token_cost": 7.5e-08,
"input_cost_per_token": 7.5e-07,
"litellm_provider": "github_copilot",
"max_input_tokens": 128000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 4.5e-06,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true
},
"github_copilot/mai-code-1-flash-internal": {
"cache_read_input_token_cost": 7.5e-08,
"input_cost_per_token": 7.5e-07,
"litellm_provider": "github_copilot",
"max_input_tokens": 128000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 4.5e-06,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true
},
"github_copilot/text-embedding-3-small": {
"litellm_provider": "github_copilot",
"max_input_tokens": 8191,
@ -35852,7 +35884,17 @@
"max_input_tokens": 32000,
"max_tokens": 32000,
"mode": "embedding",
"output_cost_per_token": 0.0
"output_cost_per_token": 0.0,
"supports_vision": true
},
"voyage/voyage-multimodal-3.5": {
"input_cost_per_token": 1.2e-07,
"litellm_provider": "voyage",
"max_input_tokens": 32000,
"max_tokens": 32000,
"mode": "embedding",
"output_cost_per_token": 0.0,
"supports_vision": true
},
"wandb/openai/gpt-oss-120b": {
"max_tokens": 131072,
@ -40774,6 +40816,174 @@
"litellm_provider": "llamagate",
"mode": "embedding"
},
"libertai/hermes-3-8b-tee": {
"max_tokens": 16000,
"max_input_tokens": 16000,
"max_output_tokens": 16000,
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 6e-07,
"litellm_provider": "libertai",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
"supports_vision": false,
"source": "https://docs.libertai.io/apis/text/"
},
"libertai/gemma-4-31b-it": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 4e-07,
"litellm_provider": "libertai",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
"supports_vision": true,
"source": "https://docs.libertai.io/apis/text/"
},
"libertai/gemma-4-31b-it-thinking": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 4e-07,
"litellm_provider": "libertai",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
"supports_vision": true,
"supports_reasoning": true,
"source": "https://docs.libertai.io/apis/text/"
},
"libertai/qwen3.6-27b": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 5e-07,
"litellm_provider": "libertai",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
"supports_vision": true,
"source": "https://docs.libertai.io/apis/text/"
},
"libertai/qwen3.6-27b-thinking": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 5e-07,
"litellm_provider": "libertai",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
"supports_vision": true,
"supports_reasoning": true,
"source": "https://docs.libertai.io/apis/text/"
},
"libertai/qwen3.6-35b-a3b": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 5e-07,
"litellm_provider": "libertai",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
"supports_vision": true,
"source": "https://docs.libertai.io/apis/text/"
},
"libertai/qwen3.6-35b-a3b-thinking": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 5e-07,
"litellm_provider": "libertai",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
"supports_vision": true,
"supports_reasoning": true,
"source": "https://docs.libertai.io/apis/text/"
},
"libertai/qwen3.5-122b-a10b": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"input_cost_per_token": 2.5e-07,
"output_cost_per_token": 1.75e-06,
"litellm_provider": "libertai",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
"supports_vision": true,
"source": "https://docs.libertai.io/apis/text/"
},
"libertai/qwen3.5-122b-a10b-thinking": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"input_cost_per_token": 2.5e-07,
"output_cost_per_token": 1.75e-06,
"litellm_provider": "libertai",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
"supports_vision": true,
"supports_reasoning": true,
"source": "https://docs.libertai.io/apis/text/"
},
"libertai/deepseek-v4-flash": {
"max_tokens": 200000,
"max_input_tokens": 200000,
"max_output_tokens": 200000,
"input_cost_per_token": 2.5e-07,
"output_cost_per_token": 1.75e-06,
"litellm_provider": "libertai",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
"supports_vision": false,
"source": "https://docs.libertai.io/apis/text/"
},
"libertai/deepseek-v4-flash-thinking": {
"max_tokens": 200000,
"max_input_tokens": 200000,
"max_output_tokens": 200000,
"input_cost_per_token": 2.5e-07,
"output_cost_per_token": 1.75e-06,
"litellm_provider": "libertai",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_system_messages": true,
"supports_vision": false,
"supports_reasoning": true,
"source": "https://docs.libertai.io/apis/text/"
},
"libertai/bge-m3": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"input_cost_per_token": 1e-08,
"output_cost_per_token": 0.0,
"litellm_provider": "libertai",
"mode": "embedding",
"source": "https://docs.libertai.io/apis/text/"
},
"sarvam/sarvam-m": {
"cache_creation_input_token_cost": 0,
"cache_creation_input_token_cost_above_1hr": 0,
@ -41753,6 +41963,48 @@
"supports_tool_choice": true,
"supports_vision": true
},
"bedrock_mantle/google.gemma-4-31b": {
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 4e-07,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 256000,
"max_output_tokens": 256000,
"max_tokens": 256000,
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": false,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"bedrock_mantle/google.gemma-4-26b-a4b": {
"input_cost_per_token": 1.3e-07,
"output_cost_per_token": 4e-07,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 256000,
"max_output_tokens": 256000,
"max_tokens": 256000,
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": false,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"bedrock_mantle/google.gemma-4-e2b": {
"input_cost_per_token": 4e-08,
"output_cost_per_token": 8e-08,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": false,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"volcengine/doubao-seed-2-0-pro-260215": {
"litellm_provider": "volcengine",
"max_input_tokens": 256000,

7
litellm/mypy.ini
View file

@ -1,13 +1,16 @@
[mypy]
warn_return_any = False
warn_return_any = True
ignore_missing_imports = True
disallow_untyped_defs = True
mypy_path = litellm/stubs
namespace_packages = True
disable_error_code =
valid-type,
annotation-unchecked,
import-untyped
[mypy-litellm.*]
ignore_missing_imports = False
[mypy-google.*]
ignore_missing_imports = True

17
litellm/provider_endpoints_support_backup.json
View file

@ -1288,6 +1288,23 @@
"interactions": true
}
},
"libertai": {
"display_name": "LibertAI (`libertai`)",
"url": "https://docs.litellm.ai/docs/providers/libertai",
"endpoints": {
"chat_completions": true,
"messages": true,
"responses": false,
"embeddings": false,
"image_generations": false,
"audio_transcriptions": false,
"audio_speech": false,
"moderations": false,
"batches": false,
"rerank": false,
"a2a": false
}
},
"litellm_proxy": {
"display_name": "LiteLLM Proxy (`litellm_proxy`)",
"url": "https://docs.litellm.ai/docs/providers/litellm_proxy",

171
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -67,9 +67,10 @@ def _is_mcp_passthrough_cold_start(
spec-compliant WWW-Authenticate challenge instead of surfacing a generic
admission error.
Uses "all" semantics (mirrors :meth:`MCPRequestHandler._target_servers_use_oauth2`):
one non-passthrough target in a co-targeted set must not flip the bypass
open for the others. Fails closed when any target cannot be resolved."""
Uses "all" semantics (mirrors
:meth:`MCPRequestHandler._target_servers_delegate_auth_to_upstream`): one
non-passthrough target in a co-targeted set must not flip the bypass open
for the others. Fails closed when any target cannot be resolved."""
if not mcp_servers:
return False
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
@ -214,101 +215,64 @@ class MCPRequestHandler:
# Only OAuth metadata routes registered under /.well-known/ are public.
if request_route.startswith("/.well-known/"):
validated_user_api_key_auth = UserAPIKeyAuth()
elif (
not litellm_api_key
and MCPRequestHandler._target_servers_delegate_auth_to_upstream( # noqa: E501
path=request_route,
mcp_servers=mcp_servers,
client_ip=IPAddressUtils.get_mcp_client_ip(request),
)
):
# Operator opted this oauth2 server into upstream-delegated auth
# (PKCE passthrough): skip LiteLLM API-key/SSO entirely so the
# client authenticates directly with the upstream MCP server.
# Fires ONLY when neither x-litellm-api-key nor Authorization is
# present. If any LiteLLM key is supplied (primary or secondary
# header), we fall through so user_id is resolved, spend/rate
# limiting apply, and any stored OAuth token can be retrieved
# and forwarded upstream. Gated by
# _target_servers_delegate_auth_to_upstream, which only returns
# True when EVERY target is auth_type=oauth2 AND has the
# delegate_auth_to_upstream flag set — fails closed otherwise.
validated_user_api_key_auth = UserAPIKeyAuth()
elif has_explicit_litellm_key:
# Explicit x-litellm-api-key provided - always validate normally
# An explicit x-litellm-api-key is always a LiteLLM credential, even
# for a delegated server, so validate it: identity / spend / rate
# limits resolve and any stored upstream token can be forwarded.
validated_user_api_key_auth = await user_api_key_auth(
api_key=litellm_api_key, request=request
)
elif MCPRequestHandler._target_servers_delegate_auth_to_upstream(
path=request_route,
mcp_servers=mcp_servers,
client_ip=IPAddressUtils.get_mcp_client_ip(request),
):
# Operator opted this oauth2 server into upstream-delegated auth: the
# client authenticates directly with the upstream MCP server, so any
# Authorization bearer is an upstream token, never a LiteLLM key. Skip
# LiteLLM validation entirely — covering both the no-credential
# discovery request and the authenticated call carrying the upstream
# bearer — so a tool call that succeeds never carries a phantom 401
# auth span; the bearer is forwarded upstream unchanged. Gated by
# _target_servers_delegate_auth_to_upstream, which returns True only
# when EVERY target is auth_type=oauth2 with delegate_auth_to_upstream
# set; fails closed otherwise.
validated_user_api_key_auth = UserAPIKeyAuth()
elif oauth2_headers:
# No x-litellm-api-key, but Authorization header present.
# Could be a LiteLLM key (backward compat) OR an opaque OAuth2 token
# the operator wants forwarded to an upstream OAuth2-mode MCP server.
# Try LiteLLM auth first; on auth failure, only fall back to anonymous
# passthrough when the request actually targets a server whose operator
# configured ``auth_type=oauth2``. For any other server (api_key,
# bearer_token, basic, etc.), a failed LiteLLM auth is a real failure
# and must propagate — otherwise an attacker can exchange any garbage
# bearer for an anonymous session.
# Authorization on a non-delegated server: the bearer must be a real
# LiteLLM credential, so a failed validation is a genuine 401/403 and
# propagates. The sole anonymous fallback is the auth_type=none
# pass-through cold-start (RFC 9728 discovery return), gated on a 401
# so a recognized-but-forbidden key still fails closed.
client_ip = IPAddressUtils.get_mcp_client_ip(request)
try:
validated_user_api_key_auth = await user_api_key_auth(
api_key=litellm_api_key, request=request
)
except (HTTPException, ProxyException) as e:
# HTTPException.status_code is int; ProxyException.code is
# normalized to str in its __init__ but can be ``"None"`` or any
# non-numeric string when the caller didn't supply a numeric
# code, so we compare against both int and str forms rather
# than coercing (``int("None")`` would raise ValueError and
# rewrite the auth error as a 500).
# ProxyException.code is normalized to str (possibly "None"), so
# compare both int and str forms rather than coercing.
status = e.status_code if isinstance(e, HTTPException) else e.code
is_auth_error = status in (401, 403, "401", "403")
is_unauthenticated = status in (401, "401")
client_ip = IPAddressUtils.get_mcp_client_ip(request)
if is_auth_error and MCPRequestHandler._target_servers_use_oauth2(
path=request_route,
mcp_servers=mcp_servers,
client_ip=client_ip,
mcp_servers_from_path = _parse_mcp_server_names_from_path(
request_route, mcp_servers
)
if (
is_unauthenticated
and mcp_servers_from_path is not None
and not _has_client_supplied_mcp_auth(
mcp_auth_header,
mcp_server_auth_headers,
)
and _is_mcp_passthrough_cold_start(
mcp_servers_from_path, client_ip=client_ip
)
):
verbose_logger.debug(
"MCP OAuth2: target server is OAuth2-mode, treating "
"Authorization as upstream OAuth2 token passthrough"
"MCP pass-through return: forwarding Authorization as "
"upstream OAuth token for delegated auth"
)
validated_user_api_key_auth = UserAPIKeyAuth()
elif is_unauthenticated:
# Pass-through cold-start return: per RFC 9728 / MCP
# Authorization spec the client completes upstream OAuth
# discovery and returns with ``Authorization: Bearer
# <upstream-token>``. For ``auth_type=none`` passthrough
# servers that bearer is not a LiteLLM key (auth above
# failed) but is meant to be forwarded upstream
# unchanged. Fall back to anonymous admission so the
# caller is not rejected for following the discovery
# flow without also setting ``x-litellm-api-key``.
# Only trigger on 401 (token unrecognized); a 403 means
# the key WAS recognized but is forbidden (e.g. over
# budget / rate limited) and must propagate so those
# controls are not bypassed via anonymous admission.
mcp_servers_from_path = _parse_mcp_server_names_from_path(
request_route, mcp_servers
)
if (
mcp_servers_from_path is not None
and not _has_client_supplied_mcp_auth(
mcp_auth_header,
mcp_server_auth_headers,
)
and _is_mcp_passthrough_cold_start(
mcp_servers_from_path, client_ip=client_ip
)
):
verbose_logger.debug(
"MCP pass-through return: target server is "
"passthrough, treating Authorization as "
"upstream OAuth token for delegated auth"
)
validated_user_api_key_auth = UserAPIKeyAuth()
else:
raise
else:
raise
else:
@ -412,45 +376,6 @@ class MCPRequestHandler:
return [single_server_match.group(1)]
return [servers_and_path]
@staticmethod
def _target_servers_use_oauth2(
path: str, mcp_servers: Optional[List[str]], client_ip: Optional[str]
) -> bool:
"""
True only when EVERY MCP server the request targets is configured for
``auth_type == oauth2``. If any target is non-OAuth2 — or if the target
cannot be resolved at all — return False so the caller fails closed.
Used to gate the "treat Authorization as opaque OAuth2 token" fallback
in :meth:`process_mcp_request` so a failed LiteLLM-auth cannot be
exchanged for an anonymous session against a non-OAuth2 server.
"""
# Inline imports avoid a circular dependency: mcp_server_manager imports
# from this module.
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
from litellm.types.mcp import MCPAuth
# Resolve the same target list downstream routing will use. For
# ``/mcp/...`` routes, ``extract_mcp_auth_context`` overrides the
# ``x-mcp-servers`` header with path-derived names, so we must mirror
# that here — otherwise a caller could set the header to a permissive
# server while the path targets a stricter one (header/path TOCTOU).
target_names = MCPRequestHandler._resolve_target_server_names(
path=path, mcp_servers_header=mcp_servers
)
if not target_names:
return False
for name in target_names:
server = global_mcp_server_manager.get_mcp_server_by_name(
name, client_ip=client_ip
)
if server is None or server.auth_type != MCPAuth.oauth2:
return False
return True
@staticmethod
def _target_servers_delegate_auth_to_upstream(
path: str, mcp_servers: Optional[List[str]], client_ip: Optional[str]
@ -472,8 +397,8 @@ class MCPRequestHandler:
)
from litellm.types.mcp import MCPAuth
# See _target_servers_use_oauth2: must mirror the downstream
# header-vs-path override or an attacker could set
# Must mirror the downstream header-vs-path override
# (``extract_mcp_auth_context``) or an attacker could set
# ``x-mcp-servers`` to a delegate-enabled server while the URL path
# targets a non-delegate server, skipping LiteLLM auth for it.
target_names = MCPRequestHandler._resolve_target_server_names(

BIN
litellm/proxy/_experimental/out/assets/logos/cisco.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.9 KiB

6
litellm/proxy/_types.py
View file

@ -373,6 +373,8 @@ class LiteLLMRoutes(enum.Enum):
# vector stores
"/vector_stores",
"/v1/vector_stores",
"/vector_stores/{vector_store_id}",
"/v1/vector_stores/{vector_store_id}",
"/vector_stores/{vector_store_id}/search",
"/v1/vector_stores/{vector_store_id}/search",
"/vector_stores/{vector_store_id}/files",
@ -2227,6 +2229,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
None,
description="max response size in MB, if a response is larger than this size it will be rejected",
)
cancel_on_disconnect: Optional[bool] = Field(
None,
description="cancel the in-flight upstream LLM request (non-streaming) when the client disconnects, freeing backend capacity (e.g. a vLLM GPU slot); the request is logged as a 499 failure",
)
infer_model_from_keys: Optional[bool] = Field(
None,
description="for `/models` endpoint, infers available model based on environment keys (e.g. OPENAI_API_KEY)",

2
litellm/proxy/anthropic_endpoints/endpoints.py
View file

@ -28,7 +28,7 @@ router = APIRouter()
tags=["[beta] Anthropic `/v1/messages`"],
dependencies=[Depends(user_api_key_auth)],
)
async def anthropic_response( # noqa: PLR0915
async def anthropic_response(
fastapi_response: Response,
request: Request,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),

2
litellm/proxy/auth/login_utils.py
View file

@ -103,7 +103,7 @@ class LoginResult:
self.login_method = login_method
async def authenticate_user( # noqa: PLR0915
async def authenticate_user(
username: str,
password: str,
master_key: Optional[str],

159
litellm/proxy/common_request_processing.py
View file

@ -1,6 +1,7 @@
import asyncio
import json
import logging
import math
import time
import traceback
from datetime import datetime
@ -16,10 +17,12 @@ from typing import (
Union,
)
import anyio
import httpx
import orjson
from fastapi import HTTPException, Request, status
from fastapi.responses import JSONResponse, Response, StreamingResponse
from starlette.types import Receive, Scope, Send
import litellm
from litellm._logging import _redact_string, verbose_proxy_logger
@ -49,6 +52,7 @@ from litellm.proxy.route_llm_request import route_request
from litellm.proxy.utils import ProxyLogging
from litellm.router import Router
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.router import RouterRateLimitError
from litellm.types.utils import ServerToolUse
# Type alias for streaming chunk serializer (chunk after hooks + cost injection -> wire format)
@ -238,6 +242,64 @@ def _extract_error_from_sse_chunk(event_line: Union[str, bytes]) -> dict:
return default_error
async def _aclose_upstream_response(response: Any) -> None:
"""Release the upstream HTTP connection when a stream ends for any
reason, including client disconnect. Mirrors the finally block of
async_data_generator in proxy_server.py."""
with anyio.CancelScope(shield=True):
if hasattr(response, "aclose"):
try:
await response.aclose()
except BaseException as e:
verbose_proxy_logger.debug(
"error closing upstream response stream: %s", e
)
class _UpstreamClosingStreamingResponse(StreamingResponse):
"""StreamingResponse that always closes its body iterator and the wrapped
upstream generator.
When the client disconnects mid-stream, Starlette abandons the body
iterator without calling aclose(), leaving the upstream LLM connection
open until garbage collection; the backend (e.g. vLLM) keeps generating
into a dead pipe. The upstream generator is closed directly (not via the
body iterator) because aclose() on a never-started generator skips its
body, so a cascade through it would be a no-op if the client disconnects
before the first chunk is sent.
"""
def __init__(
self,
content: AsyncGenerator[str, None],
*,
media_type: Optional[str] = None,
headers: Optional[dict] = None,
status_code: int = status.HTTP_200_OK,
upstream_generator: Optional[AsyncGenerator[str, None]] = None,
) -> None:
super().__init__(
content, status_code=status_code, headers=headers, media_type=media_type
)
self._upstream_generator = upstream_generator
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
try:
await super().__call__(scope, receive, send)
finally:
with anyio.CancelScope(shield=True):
for target in (self.body_iterator, self._upstream_generator):
aclose = getattr(target, "aclose", None)
if aclose is None:
continue
try:
await aclose()
except BaseException as e:
verbose_proxy_logger.debug(
"error closing streaming generator: %s", e
)
async def create_response( # noqa: PLR0915
generator: AsyncGenerator[str, None],
media_type: str,
@ -364,11 +426,12 @@ async def create_response( # noqa: PLR0915
with tracer.trace(DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE):
yield chunk
return StreamingResponse(
return _UpstreamClosingStreamingResponse(
combined_generator(),
media_type=media_type,
headers=streaming_headers,
status_code=final_status_code,
upstream_generator=generator,
)
@ -556,6 +619,64 @@ def _has_attribute_error_in_chain(exc: Exception) -> bool:
return False
_CLIENT_DISCONNECT_DETAIL = "Client disconnected the request"
def _log_llm_api_exception(e: Exception) -> None:
if (
getattr(e, "status_code", None) == 499
and getattr(e, "detail", None) == _CLIENT_DISCONNECT_DETAIL
):
verbose_proxy_logger.info(
"litellm.proxy.proxy_server._handle_llm_api_exception(): client disconnected, upstream LLM request cancelled"
)
return
verbose_proxy_logger.exception(
f"litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - {str(e)}"
)
async def _cancel_llm_call_on_client_disconnect(
request: Request,
llm_api_call: "asyncio.Future[Any]",
disconnect_event: asyncio.Event,
) -> None:
try:
while True:
message = await request.receive()
if message["type"] == "http.disconnect":
disconnect_event.set()
llm_api_call.cancel()
return
except Exception as exc:
verbose_proxy_logger.warning(
"cancel_on_disconnect: request.receive() raised %s; "
"upstream LLM call will not be cancelled on disconnect",
exc,
)
async def _await_llm_call_cancelling_on_disconnect(
request: Request,
llm_api_call: "asyncio.Future[Any]",
) -> Any:
disconnect_event = asyncio.Event()
monitor = asyncio.create_task(
_cancel_llm_call_on_client_disconnect(request, llm_api_call, disconnect_event)
)
try:
return await llm_api_call
except asyncio.CancelledError:
if disconnect_event.is_set():
raise HTTPException(
status_code=499,
detail=_CLIENT_DISCONNECT_DETAIL,
)
raise
finally:
monitor.cancel()
class ProxyBaseLLMRequestProcessing:
def __init__(self, data: dict):
self.data = data
@ -1244,7 +1365,12 @@ class ProxyBaseLLMRequestProcessing:
*tasks
) # run the moderation check in parallel to the actual llm api call
responses = await llm_responses
if general_settings.get("cancel_on_disconnect", False):
responses = await _await_llm_call_cancelling_on_disconnect(
request, llm_responses
)
else:
responses = await llm_responses
response = responses[1]
@ -1666,6 +1792,23 @@ class ProxyBaseLLMRequestProcessing:
response=completed_obj,
user_api_key_dict=user_api_key_dict,
)
else:
# Silent skip caused #30210: the proxy's Router wrapper
# of the responses streaming iterator wasn't propagating
# ``completed_response``, so this hook recorded nothing
# and follow-up /v1/containers/<id>/files calls 403'd
# for non-admin keys with no proxy-side hint. Log a
# warning so future regressions of the same shape
# surface in operator logs.
verbose_proxy_logger.warning(
"Container ownership recording skipped on streaming "
"/v1/responses: no completed_response on stream "
"iterator %s. If this stream created any tool "
"container (e.g. code_interpreter), follow-up "
"/v1/containers/<id>/files calls will 403 for "
"non-admin keys.",
type(original_stream_response).__name__,
)
except Exception as e:
verbose_proxy_logger.exception(
"Container ownership recording failed after streaming responses call: %s",
@ -2096,6 +2239,10 @@ class ProxyBaseLLMRequestProcessing:
e,
)
def _apply_router_cooldown_retry_after(self, headers: dict, e: Exception) -> None:
if isinstance(e, RouterRateLimitError) and e.cooldown_time > 0:
headers["retry-after"] = str(math.ceil(e.cooldown_time))
async def _handle_llm_api_exception(
self,
e: Exception,
@ -2104,9 +2251,7 @@ class ProxyBaseLLMRequestProcessing:
version: Optional[str] = None,
):
"""Raises ProxyException (OpenAI API compatible) if an exception is raised"""
verbose_proxy_logger.exception(
f"litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - {str(e)}"
)
_log_llm_api_exception(e)
# Allow callbacks to transform the error response
transformed_exception = await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
@ -2177,6 +2322,8 @@ class ProxyBaseLLMRequestProcessing:
except Exception:
pass
self._apply_router_cooldown_retry_after(headers, e)
if isinstance(e, HTTPException):
raw_detail = getattr(e, "detail", str(e))
message, structured_fields = _serialize_http_exception_detail(raw_detail)
@ -2384,6 +2531,8 @@ class ProxyBaseLLMRequestProcessing:
code=getattr(e, "status_code", 500),
)
yield serialize_error(proxy_exception)
finally:
await _aclose_upstream_response(response)
@staticmethod
def async_sse_data_generator(

2
litellm/proxy/common_utils/callback_utils.py
View file

@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Literal,
import litellm
from litellm import get_secret
from litellm._logging import verbose_proxy_logger
from litellm.constants import PRE_CALL_EXECUTED_GUARDRAILS_KEY
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
from litellm.proxy._types import CommonProxyErrors, LiteLLMPromptInjectionParams
@ -497,6 +498,7 @@ LITELLM_PROXY_INTERNAL_METADATA_KEYS = frozenset(
"guardrail_config",
"_guardrail_pipelines",
"_pipeline_managed_guardrails",
PRE_CALL_EXECUTED_GUARDRAILS_KEY,
"disable_global_guardrails",
"disable_global_guardrail",
"opted_out_global_guardrails",

8
litellm/proxy/common_utils/debug_utils.py
View file

@ -92,12 +92,12 @@ if os.environ.get("LITELLM_PROFILE", "false").lower() == "true":
try:
import objgraph # type: ignore
print("growth of objects") # noqa
print("growth of objects") # noqa: T201
objgraph.show_growth()
print("\n\nMost common types") # noqa
print("\n\nMost common types") # noqa: T201
objgraph.show_most_common_types()
roots = objgraph.get_leaking_objects()
print("\n\nLeaking objects") # noqa
print("\n\nLeaking objects") # noqa: T201
objgraph.show_most_common_types(objects=roots)
except ImportError:
raise ImportError(
@ -739,7 +739,7 @@ async def get_otel_spans():
else:
recorded_spans = []
print("Spans: ", recorded_spans) # noqa
print("Spans: ", recorded_spans) # noqa: T201
most_recent_parent = None
most_recent_start_time = 1000000

2
litellm/proxy/db/check_migration.py
View file

@ -54,7 +54,7 @@ def check_prisma_schema_diff_helper(db_url: str) -> Tuple[bool, List[str]]:
subprocess.CalledProcessError: If the Prisma command fails.
Exception: For any other errors during execution.
"""
verbose_logger.debug("Checking for Prisma schema diff...") # noqa: T201
verbose_logger.debug("Checking for Prisma schema diff...")
try:
result = subprocess.run(
[

108
litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/__init__.py Normal file
View file

@ -0,0 +1,108 @@
"""Cisco AI Defense Guardrail Integration for LiteLLM."""
from typing import TYPE_CHECKING
from litellm.types.guardrails import SupportedGuardrailIntegrations
from .cisco_ai_defense import (
CiscoAIDefenseGuardrail,
CiscoAIDefenseGuardrailAPIError,
CiscoAIDefenseGuardrailMissingSecrets,
)
if TYPE_CHECKING:
from litellm.types.guardrails import Guardrail, LitellmParams
def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"):
import litellm
guardrail_name = guardrail.get("guardrail_name")
if not guardrail_name:
raise ValueError("Cisco AI Defense: guardrail_name is required")
optional_params = getattr(litellm_params, "optional_params", None)
_callback = CiscoAIDefenseGuardrail(
guardrail_name=guardrail_name,
api_key=litellm_params.api_key,
api_base=litellm_params.api_base,
inspection_type=_get_optional_value(
litellm_params, optional_params, "inspection_type"
),
inspect_path=_get_optional_value(
litellm_params, optional_params, "inspect_path"
),
enabled_rules=_get_optional_value(
litellm_params, optional_params, "enabled_rules"
),
integration_profile_id=_get_optional_value(
litellm_params, optional_params, "integration_profile_id"
),
integration_profile_version=_get_optional_value(
litellm_params, optional_params, "integration_profile_version"
),
integration_tenant_id=_get_optional_value(
litellm_params, optional_params, "integration_tenant_id"
),
integration_type=_get_optional_value(
litellm_params, optional_params, "integration_type"
),
on_flagged_action=_get_optional_value(
litellm_params, optional_params, "on_flagged_action"
),
fallback_on_error=_get_optional_value(
litellm_params, optional_params, "fallback_on_error"
),
timeout=_get_optional_value(litellm_params, optional_params, "timeout"),
event_hook=litellm_params.mode,
default_on=litellm_params.default_on or False,
)
litellm.logging_callback_manager.add_litellm_callback(_callback)
# MCP post-tool-call hooks are dispatched through success callbacks.
litellm.logging_callback_manager.add_litellm_success_callback(_callback)
return _callback
def _get_optional_value(litellm_params, optional_params, attribute_name):
"""Resolve Cisco optional params without inheriting sibling defaults."""
if optional_params is not None:
if isinstance(optional_params, dict):
if attribute_name in optional_params:
return optional_params[attribute_name]
else:
nested_fields_set = getattr(optional_params, "model_fields_set", None)
if nested_fields_set is None or attribute_name in nested_fields_set:
value = getattr(optional_params, attribute_name, None)
if value is not None:
return value
if litellm_params is None:
return None
# Only accept flattened values the caller explicitly set.
fields_set = getattr(litellm_params, "model_fields_set", None)
if fields_set is None or attribute_name not in fields_set:
return None
return getattr(litellm_params, attribute_name, None)
guardrail_initializer_registry = {
SupportedGuardrailIntegrations.CISCO_AI_DEFENSE.value: initialize_guardrail,
}
guardrail_class_registry = {
SupportedGuardrailIntegrations.CISCO_AI_DEFENSE.value: CiscoAIDefenseGuardrail,
}
__all__ = [
"CiscoAIDefenseGuardrail",
"CiscoAIDefenseGuardrailAPIError",
"CiscoAIDefenseGuardrailMissingSecrets",
"initialize_guardrail",
"guardrail_initializer_registry",
"guardrail_class_registry",
]

2358
litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py Normal file
View file

File diff suppressed because it is too large Load diff

704
litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense_mcp.py Normal file
View file

@ -0,0 +1,704 @@
"""MCP-specific inspection logic for the Cisco AI Defense guardrail.
The public guardrail class imports this private mixin from
``cisco_ai_defense.py``. Keeping MCP logic here avoids circular imports
while preserving the existing public import path.
"""
from datetime import datetime
from typing import TYPE_CHECKING, Any, Dict, List, Optional
from fastapi import HTTPException
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.callback_utils import (
add_guardrail_to_applied_guardrails_header,
)
from litellm.types.guardrails import GuardrailEventHooks
if TYPE_CHECKING:
from litellm.types.mcp import MCPPostCallResponseObject
from .cisco_ai_defense import _ScanContext
def _serialize_mcp_content_item(item: object) -> Dict[str, Any]:
"""Serialize an MCP content item to a JSON-friendly dict.
Handles raw dicts, MCP SDK Pydantic models, and simple ``.text`` objects.
"""
if isinstance(item, dict):
return dict(item)
model_dump = getattr(item, "model_dump", None)
if callable(model_dump):
try:
return dict(model_dump(exclude_none=True))
except TypeError:
return dict(model_dump())
text = getattr(item, "text", None)
if isinstance(text, str):
return {"type": getattr(item, "type", "text"), "text": text}
return {"type": "text", "text": str(item)}
class _CiscoAIDefenseMcpMixin:
"""MCP-specific instance methods for ``CiscoAIDefenseGuardrail``.
Holds the MCP hooks, JSON-RPC payload builders, and redaction helpers.
"""
if TYPE_CHECKING:
api_base: str
inspect_path: str
inspection_type: str
_PROVIDER_NAME: str
guardrail_name: Optional[str]
def should_run_guardrail(
self, data: dict, event_type: GuardrailEventHooks
) -> bool: ...
async def _post_inspection(
self, url: str, payload: Dict[str, Any], surface: str
) -> Dict[str, Any]: ...
def _handle_api_error(
self,
error: Exception,
*,
request_data: Optional[dict] = ...,
start_time: Optional[datetime] = ...,
surface: str = ...,
direction: str = ...,
) -> Dict[str, Any]: ...
def _finalize_inspection(
self,
inspect_response: Dict[str, Any],
request_data: dict,
context: "_ScanContext",
start_time: datetime,
response_obj: object = ...,
) -> Dict[str, Any]: ...
# ------------------------------------------------------------------
# MCP post-tool hook (dispatcher contract)
# ------------------------------------------------------------------
async def async_post_mcp_tool_call_hook(
self,
kwargs: dict,
response_obj: "MCPPostCallResponseObject",
start_time: datetime,
end_time: datetime,
) -> Optional["MCPPostCallResponseObject"]:
"""Scan MCP tool output and return a replacement object on block."""
del start_time, end_time
if self.inspection_type != "mcp":
return None
request_data: Dict[str, Any] = {}
for key in (
"name",
"litellm_call_id",
"id",
"user",
"mcp_tool_name",
"tool_name",
"mcp_arguments",
"arguments",
"mcp_server_name",
"server_name",
"metadata",
"litellm_metadata",
"mcp_tool_call_metadata",
"guardrails",
):
if key in kwargs and kwargs[key] is not None:
request_data[key] = kwargs[key]
self._hydrate_mcp_tool_context(request_data)
if not (
self.should_run_guardrail(
data=request_data,
event_type=GuardrailEventHooks.during_mcp_call,
)
or self.should_run_guardrail(
data=request_data,
event_type=GuardrailEventHooks.pre_mcp_call,
)
):
verbose_proxy_logger.debug(
"Cisco AI Defense guardrail (%s): no MCP mode configured "
"— skipping MCP response scan.",
self.guardrail_name,
)
return None
mcp_tool_response = self._extract_mcp_tool_call_response(response_obj)
if mcp_tool_response is None:
verbose_proxy_logger.debug(
"Cisco AI Defense guardrail: no MCP tool response payload "
"to scan, skipping"
)
return None
original_response = kwargs.get("original_response")
try:
await self._inspect_mcp_response(
request_data=request_data,
response=mcp_tool_response,
redact_response_obj=(
original_response
if original_response is not None
else mcp_tool_response
),
)
except HTTPException as exc:
blocking_response = self._build_blocking_mcp_response(
detail=exc.detail, original_response_obj=response_obj
)
self._replace_mcp_tool_response(response_obj, blocking_response)
if original_response is not None:
self._replace_mcp_tool_response(original_response, blocking_response)
add_guardrail_to_applied_guardrails_header(
request_data=request_data, guardrail_name=self.guardrail_name
)
verbose_proxy_logger.warning(
"Cisco AI Defense guardrail (%s): MCP response blocked — "
"tool output replaced with synthesized violation message.",
self.guardrail_name,
)
return blocking_response
add_guardrail_to_applied_guardrails_header(
request_data=request_data, guardrail_name=self.guardrail_name
)
return None
def _build_blocking_mcp_response(
self,
detail: object,
original_response_obj: object,
) -> "MCPPostCallResponseObject":
"""Build a synthetic MCPPostCallResponseObject for blocked output."""
import json as _json
from litellm.types.llms.base import HiddenParams
from litellm.types.mcp import MCPPostCallResponseObject
from mcp.types import TextContent
if isinstance(detail, dict):
payload = detail
else:
payload = {
"error": "Blocked by Cisco AI Defense Guardrail",
"message": (
str(detail) if detail else "Blocked by Cisco AI Defense Guardrail"
),
"provider": self._PROVIDER_NAME,
"guardrail": self.guardrail_name,
"surface": "mcp",
"direction": "output",
"action": "block",
}
original_hidden = getattr(original_response_obj, "hidden_params", None)
if isinstance(original_hidden, HiddenParams):
hidden_params: Any = original_hidden
else:
response_cost = getattr(original_hidden, "response_cost", None)
hidden_params = (
HiddenParams(response_cost=response_cost)
if response_cost is not None
else HiddenParams()
)
return MCPPostCallResponseObject(
mcp_tool_call_response=[
TextContent(type="text", text=_json.dumps(payload))
],
hidden_params=hidden_params,
)
@staticmethod
def _replace_mcp_tool_response(
response_obj: object, replacement_obj: object
) -> bool:
replacement = getattr(replacement_obj, "mcp_tool_call_response", None)
if replacement is None:
return False
inner = getattr(response_obj, "mcp_tool_call_response", None)
if inner is not None:
if _CiscoAIDefenseMcpMixin._replace_mcp_tool_response(
inner, replacement_obj
):
return True
try:
setattr(response_obj, "mcp_tool_call_response", replacement)
return True
except (AttributeError, TypeError, ValueError):
return False
content = getattr(response_obj, "content", None)
if isinstance(content, list):
content[:] = replacement
structured_replacement = (
_CiscoAIDefenseMcpMixin._replacement_structured_content(replacement)
)
if hasattr(response_obj, "structuredContent"):
try:
setattr(response_obj, "structuredContent", structured_replacement)
except (AttributeError, TypeError, ValueError):
pass
if hasattr(response_obj, "isError"):
try:
setattr(response_obj, "isError", True)
except (AttributeError, TypeError, ValueError):
pass
return True
if isinstance(response_obj, list):
response_obj[:] = replacement
return True
if isinstance(response_obj, dict):
result = response_obj.get("result")
if isinstance(result, dict):
result["content"] = replacement
result["structuredContent"] = (
_CiscoAIDefenseMcpMixin._replacement_structured_content(replacement)
)
result["isError"] = True
return True
response_obj["result"] = {
"content": replacement,
"structuredContent": _CiscoAIDefenseMcpMixin._replacement_structured_content(
replacement
),
"isError": True,
}
return True
return False
@staticmethod
def _replacement_structured_content(
replacement: object,
) -> Optional[Dict[str, str]]:
if not isinstance(replacement, list) or not replacement:
return None
first = replacement[0]
text = (
first.get("text")
if isinstance(first, dict)
else getattr(first, "text", None)
)
return {"result": text} if isinstance(text, str) else None
@staticmethod
def _extract_mcp_tool_call_response(response_obj: object) -> object:
"""Pull the raw tool-call response off a MCPPostCallResponseObject."""
inner = getattr(response_obj, "mcp_tool_call_response", None)
if inner is None and isinstance(response_obj, dict):
inner = response_obj.get("mcp_tool_call_response")
return inner if inner is not None else response_obj
# ------------------------------------------------------------------
# MCP request / response inspection
# ------------------------------------------------------------------
async def _inspect_mcp_request(
self,
data: dict,
user_api_key_dict: UserAPIKeyAuth,
) -> Dict[str, Any]:
del user_api_key_dict # carried via logging metadata, not the wire payload
url = f"{self.api_base}{self.inspect_path}"
payload = self._build_mcp_request_payload(data=data)
if payload is None:
verbose_proxy_logger.debug(
"Cisco AI Defense guardrail: could not build MCP request "
"payload, skipping"
)
return {}
start_time = datetime.now()
try:
inspect_response = await self._post_inspection(
url=url, payload=payload, surface="mcp"
)
except HTTPException:
raise
except Exception as exc:
return self._handle_api_error(
exc,
request_data=data,
start_time=start_time,
surface="mcp",
direction="input",
)
from .cisco_ai_defense import _ScanContext
return self._finalize_inspection(
inspect_response=inspect_response,
request_data=data,
context=_ScanContext(surface="mcp", direction="input"),
start_time=start_time,
)
async def _inspect_mcp_response(
self,
request_data: dict,
response: object,
user_api_key_dict: Optional[UserAPIKeyAuth] = None,
redact_response_obj: object = None,
) -> Dict[str, Any]:
del user_api_key_dict # carried via logging metadata, not the wire payload
url = f"{self.api_base}{self.inspect_path}"
payload = self._build_mcp_response_payload(
request_data=request_data,
response=response,
)
if payload is None:
verbose_proxy_logger.debug(
"Cisco AI Defense guardrail: could not build MCP response "
"payload, skipping"
)
return {}
start_time = datetime.now()
try:
inspect_response = await self._post_inspection(
url=url, payload=payload, surface="mcp"
)
except HTTPException:
raise
except Exception as exc:
return self._handle_api_error(
exc,
request_data=request_data,
start_time=start_time,
surface="mcp",
direction="output",
)
from .cisco_ai_defense import _ScanContext
return self._finalize_inspection(
inspect_response=inspect_response,
request_data=request_data,
context=_ScanContext(surface="mcp", direction="output"),
start_time=start_time,
response_obj=(
response if redact_response_obj is None else redact_response_obj
),
)
def _build_mcp_request_payload(
self,
data: dict,
) -> Optional[Dict[str, Any]]:
"""Build the JSON-RPC ``tools/call`` envelope sent to ``/inspect/mcp``.
The Cisco AI Defense MCP inspect endpoint expects the JSON-RPC
envelope itself as the request body — *not* wrapped under a
``request`` key with sibling ``metadata`` / ``config`` keys. Policies
are applied based on the API key linked to the request. Operator
metadata (user, call id, src/dst app, etc.) is carried out-of-band
via the standard logging payload so the wire contract stays
identical to a hand-rolled ``curl`` against ``/inspect/mcp``.
"""
if data.get("jsonrpc") == "2.0":
return {
"jsonrpc": "2.0",
"id": (data.get("id") or data.get("litellm_call_id") or "litellm-mcp"),
"method": data.get("method") or "tools/call",
"params": data.get("params") or {},
}
tool_name = (
data.get("mcp_tool_name") or data.get("tool_name") or data.get("name")
)
if not tool_name:
return None
arguments = data.get("mcp_arguments")
if arguments is None:
arguments = data.get("arguments")
return {
"jsonrpc": "2.0",
"id": data.get("litellm_call_id") or "litellm-mcp",
"method": "tools/call",
"params": {
"name": tool_name,
"arguments": (arguments if isinstance(arguments, dict) else {}),
},
}
def _build_mcp_response_payload(
self,
request_data: dict,
response: object,
) -> Optional[Dict[str, Any]]:
"""Build the MCP response-inspection body sent to ``/inspect/mcp``."""
request_payload = self._build_mcp_request_payload(data=request_data)
if request_payload is None:
return None
normalized = self._normalize_mcp_response(response)
if normalized is None:
return None
payload = dict(request_payload)
response_id = normalized.get("id")
if response_id not in (None, "litellm-mcp"):
payload["id"] = response_id
elif payload.get("id") in (None, "litellm-mcp"):
request_id = request_data.get("litellm_call_id") or request_data.get("id")
if request_id:
payload["id"] = request_id
if "result" in normalized:
payload["result"] = normalized["result"]
if "error" in normalized:
payload["error"] = normalized["error"]
return payload
@staticmethod
def _hydrate_mcp_tool_context(request_data: Dict[str, Any]) -> None:
metadata = request_data.get("mcp_tool_call_metadata")
if metadata is None:
nested = request_data.get("metadata") or request_data.get(
"litellm_metadata"
)
if isinstance(nested, dict):
metadata = nested.get("mcp_tool_call_metadata")
if not isinstance(metadata, dict):
return
name = metadata.get("name")
arguments = metadata.get("arguments")
server_name = metadata.get("mcp_server_name")
if name:
request_data.setdefault("mcp_tool_name", name)
request_data.setdefault("tool_name", name)
request_data.setdefault("name", name)
if arguments is not None:
request_data.setdefault("mcp_arguments", arguments)
request_data.setdefault("arguments", arguments)
if server_name:
request_data.setdefault("mcp_server_name", server_name)
request_data.setdefault("server_name", server_name)
@staticmethod
def _normalize_mcp_response(response: object) -> Optional[Dict[str, Any]]:
"""Normalize an MCP tool response into a JSON-RPC envelope.
Handles JSON-RPC dicts, raw content lists, MCP SDK models, and
Pydantic-coerced ``[(field_name, value)]`` lists.
"""
if isinstance(response, dict):
if response.get("jsonrpc") == "2.0":
return dict(response)
if isinstance(response.get("result"), dict):
return {
"jsonrpc": "2.0",
"id": response.get("id") or "litellm-mcp",
"result": response["result"],
}
content = response.get("content")
if isinstance(content, list):
return {
"jsonrpc": "2.0",
"id": response.get("id") or "litellm-mcp",
"result": _CiscoAIDefenseMcpMixin._build_mcp_result(
content=content, source=response
),
}
if isinstance(response, list):
if response and all(
isinstance(item, tuple) and len(item) == 2 and isinstance(item[0], str)
for item in response
):
response_fields = dict(response)
inner_content = response_fields.get("content")
if isinstance(inner_content, list):
return {
"jsonrpc": "2.0",
"id": "litellm-mcp",
"result": _CiscoAIDefenseMcpMixin._build_mcp_result(
content=inner_content, source=response_fields
),
}
else:
return None
return {
"jsonrpc": "2.0",
"id": "litellm-mcp",
"result": _CiscoAIDefenseMcpMixin._build_mcp_result(content=response),
}
model_dump = getattr(response, "model_dump", None)
if callable(model_dump):
try:
dumped = model_dump(exclude_none=True)
except TypeError:
dumped = model_dump()
if isinstance(dumped, dict):
return _CiscoAIDefenseMcpMixin._normalize_mcp_response(dumped)
content = getattr(response, "content", None)
if isinstance(content, list):
return {
"jsonrpc": "2.0",
"id": "litellm-mcp",
"result": _CiscoAIDefenseMcpMixin._build_mcp_result(
content=content, source=response
),
}
return None
@staticmethod
def _build_mcp_result(
content: List[Any],
source: object = None,
) -> Dict[str, Any]:
result: Dict[str, Any] = {
"content": [_serialize_mcp_content_item(item) for item in content]
}
for key in ("structuredContent", "isError"):
value = (
source.get(key)
if isinstance(source, dict)
else getattr(source, key, None)
)
if value is not None and (key != "isError" or isinstance(value, bool)):
result[key] = value
return result
# ------------------------------------------------------------------
# MCP redact (in-place rewrite of tool output)
# ------------------------------------------------------------------
@staticmethod
def _set_mcp_tool_response_text(response_obj: object, text: str) -> bool:
"""Replace text content in any supported MCP response shape."""
if response_obj is None:
return False
inner = getattr(response_obj, "mcp_tool_call_response", None)
if inner is not None:
return _CiscoAIDefenseMcpMixin._set_mcp_tool_response_text(inner, text)
content_list = _CiscoAIDefenseMcpMixin._coerce_to_content_list(response_obj)
replaced = False
if isinstance(content_list, list):
for item in content_list:
if isinstance(item, dict) and item.get("type") == "text":
item["text"] = text
replaced = True
elif hasattr(item, "type") and getattr(item, "type", None) == "text":
try:
setattr(item, "text", text)
replaced = True
except (AttributeError, TypeError, ValueError):
continue
replacement = {"result": text}
if (
isinstance(response_obj, list)
and response_obj
and all(
isinstance(item, tuple) and len(item) == 2 and isinstance(item[0], str)
for item in response_obj
)
):
for index, item in enumerate(response_obj):
if item[0] == "structuredContent":
response_obj[index] = (item[0], replacement)
replaced = True
elif hasattr(response_obj, "structuredContent"):
try:
setattr(response_obj, "structuredContent", replacement)
replaced = True
except (AttributeError, TypeError, ValueError):
pass
elif isinstance(response_obj, dict):
result = response_obj.get("result")
target: Dict[Any, Any] = (
result if isinstance(result, dict) else response_obj
)
if "structuredContent" in target:
target["structuredContent"] = replacement
replaced = True
return replaced
@staticmethod
def _coerce_to_content_list(response_obj: object) -> Optional[List[Any]]:
"""Find the MCP content list inside supported response shapes."""
if response_obj is None:
return None
inner = getattr(response_obj, "mcp_tool_call_response", None)
if inner is not None:
return _CiscoAIDefenseMcpMixin._coerce_to_content_list(inner)
content = getattr(response_obj, "content", None)
if isinstance(content, list):
return content
if isinstance(response_obj, list):
if response_obj and all(
isinstance(item, tuple) and len(item) == 2 and isinstance(item[0], str)
for item in response_obj
):
inner_content = dict(response_obj).get("content")
if isinstance(inner_content, list):
return inner_content
return None
return response_obj
return None
# ------------------------------------------------------------------
# MCP-specific verdict extraction
# ------------------------------------------------------------------
@staticmethod
def _extract_sanitized_mcp_arguments(
inspect_response: Dict[str, Any],
) -> Optional[Dict[str, Any]]:
"""Pull sanitized MCP tool-call arguments off the verdict.
Cisco can return them at the top level (``params.arguments``) or
under ``sanitized_payload`` / ``modified_payload``.
"""
containers = [inspect_response]
for container_key in ("result", "data"):
container = inspect_response.get(container_key)
if isinstance(container, dict):
containers.append(container)
for container in containers:
params = container.get("params")
if isinstance(params, dict):
args = params.get("arguments")
if isinstance(args, dict) and args:
return dict(args)
for key in (
"sanitized_payload",
"sanitizedPayload",
"modified_payload",
"modifiedPayload",
):
payload = container.get(key)
if isinstance(payload, dict):
inner_params = payload.get("params")
if isinstance(inner_params, dict):
args = inner_params.get("arguments")
if isinstance(args, dict) and args:
return dict(args)
direct = payload.get("arguments")
if isinstance(direct, dict) and direct:
return dict(direct)
return None

4
litellm/proxy/guardrails/guardrail_hooks/presidio.py
View file

@ -28,7 +28,7 @@ from typing import (
import aiohttp
import litellm # noqa: E401
import litellm
from litellm import get_secret
from litellm._logging import verbose_proxy_logger
from litellm.types.utils import GenericGuardrailAPIInputs
@ -1432,7 +1432,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
try:
verbose_proxy_logger.debug(print_statement)
if litellm.set_verbose:
print(print_statement) # noqa
print(print_statement) # noqa: T201
except Exception:
pass

64
litellm/proxy/guardrails/guardrail_registry.py
View file

@ -5,6 +5,8 @@ import os
from datetime import datetime, timezone
from typing import Any, Dict, List, Literal, Optional, Set, Type, cast
from pydantic import ValidationError
import litellm
from litellm import Router
from litellm._logging import verbose_proxy_logger
@ -601,21 +603,25 @@ class InMemoryGuardrailHandler:
def delete_in_memory_guardrail(self, guardrail_id: str) -> None:
"""
Delete a guardrail in memory and remove from litellm callbacks.
The callback is purged from every callback list, not just
litellm.callbacks: request handling promotes guardrail callbacks into the
success/failure/async lists, so removing it from only litellm.callbacks
leaves the old instance stranded in those lists on every re-initialization.
"""
# Remove from in-memory storage
self.IN_MEMORY_GUARDRAILS.pop(guardrail_id, None)
self._sources.pop(guardrail_id, None)
# Remove the callback from litellm.callbacks
custom_guardrail_callback = self.guardrail_id_to_custom_guardrail.pop(
guardrail_id, None
)
if custom_guardrail_callback:
litellm.logging_callback_manager.remove_callback_from_list_by_object(
callback_list=litellm.callbacks,
obj=custom_guardrail_callback,
require_self=False,
)
if custom_guardrail_callback is None:
return
litellm.logging_callback_manager.remove_callback_from_all_lists(
custom_guardrail_callback
)
def list_in_memory_guardrails(self) -> List[Guardrail]:
"""
@ -657,6 +663,34 @@ class InMemoryGuardrailHandler:
self.delete_in_memory_guardrail(guardrail_id)
return stale_ids
@staticmethod
def _normalize_litellm_params_for_comparison(
params: Optional[Any],
) -> Optional[Dict[str, Any]]:
"""
Render litellm_params to a canonical dict so an in-memory LitellmParams and
the raw dict loaded from the DB compare equal when they describe the same
config. The in-memory side is a LitellmParams whose model_dump() carries
every field default and coerces enums, while the DB side is the raw stored
dict holding only the keys originally provided. Comparing those two shapes
directly never matches, so each DB poll would re-initialize the guardrail
forever; normalizing both through LitellmParams keeps the diff meaningful.
"""
if params is None:
return None
if isinstance(params, LitellmParams):
return params.model_dump()
if isinstance(params, dict):
try:
return LitellmParams(**params).model_dump()
except ValidationError as e:
verbose_proxy_logger.warning(
f"Could not normalize guardrail litellm_params for comparison; "
f"treating the guardrail as changed. Error: {e}"
)
return params
return params
def _has_guardrail_params_changed(
self, guardrail_id: str, new_guardrail: Guardrail
) -> bool:
@ -673,19 +707,11 @@ class InMemoryGuardrailHandler:
return True
# Compare litellm_params
existing_params = existing.get("litellm_params")
new_params = new_guardrail.get("litellm_params")
# Convert to dicts for comparison
existing_dict = (
existing_params.model_dump()
if isinstance(existing_params, LitellmParams)
else existing_params
existing_dict = self._normalize_litellm_params_for_comparison(
existing.get("litellm_params")
)
new_dict = (
new_params.model_dump()
if isinstance(new_params, LitellmParams)
else new_params
new_dict = self._normalize_litellm_params_for_comparison(
new_guardrail.get("litellm_params")
)
# Compare and identify specific differences

25
litellm/proxy/health_endpoints/_health_endpoints.py
View file

@ -24,6 +24,7 @@ from litellm.proxy._types import (
LitellmUserRoles,
ProxyErrorTypes,
ProxyException,
SpecialModelNames,
UserAPIKeyAuth,
WebhookEvent,
)
@ -1074,8 +1075,26 @@ async def health_endpoint(
# response but NOT in the background-cache /health response. This is
# surfaced via the "warnings" field below so operators can fix the
# missing model_info.id rather than guess at the discrepancy.
if len(user_api_key_dict.models) > 0:
allowed_models = set(user_api_key_dict.models)
# Keys granted SpecialModelNames.all_proxy_models carry the literal
# "all-proxy-models" entry, which matches no real model_name; treat
# them as unrestricted instead of filtering the list down to nothing.
# Keys granted SpecialModelNames.all_team_models inherit the parent
# team's allowlist (same semantics as get_key_models in
# model_checks.py). Without a team_id the sentinel cannot resolve and
# stays in the list, matching nothing; denied rather than
# unrestricted, mirroring _resolve_key_models_for_auth_check.
accessible_models = list(user_api_key_dict.models)
if (
SpecialModelNames.all_team_models.value in accessible_models
and user_api_key_dict.team_id is not None
):
accessible_models = list(user_api_key_dict.team_models)
restrict_to_allowed_models = (
len(accessible_models) > 0
and SpecialModelNames.all_proxy_models.value not in accessible_models
)
if restrict_to_allowed_models:
allowed_models = set(accessible_models)
_llm_model_list = [
m for m in _llm_model_list if m.get("model_name") in allowed_models
]
@ -1087,7 +1106,7 @@ async def health_endpoint(
# other healthy model would still report healthy_count > 0 and
# the targeted-503 path would never fire.
targeted_ids = _resolve_targeted_model_ids(_llm_model_list, model, model_id)
if len(user_api_key_dict.models) > 0:
if restrict_to_allowed_models:
allowed_model_ids = {
(m.get("model_info") or {}).get("id")
for m in _llm_model_list

2
litellm/proxy/hooks/batch_redis_get.py
View file

@ -33,7 +33,7 @@ class _PROXY_BatchRedisRequests(CustomLogger):
elif debug_level == "INFO":
verbose_proxy_logger.debug(print_statement)
if litellm.set_verbose is True:
print(print_statement) # noqa
print(print_statement) # noqa: T201
async def async_pre_call_hook(
self,

4
litellm/proxy/hooks/parallel_request_limiter.py
View file

@ -50,7 +50,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
try:
verbose_proxy_logger.debug(print_statement)
if litellm.set_verbose:
print(print_statement) # noqa
print(print_statement) # noqa: T201
except Exception:
pass
@ -769,7 +769,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
litellm_parent_otel_span=litellm_parent_otel_span,
)
except Exception as e:
self.print_verbose(e) # noqa
self.print_verbose(e)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
try:

2
litellm/proxy/hooks/prompt_injection_detection.py
View file

@ -72,7 +72,7 @@ class _OPTIONAL_PromptInjectionDetection(CustomLogger):
verbose_proxy_logger.debug(print_statement)
if litellm.set_verbose is True:
print(print_statement) # noqa
print(print_statement) # noqa: T201
def update_environment(self, router: Optional[Router] = None):
self.llm_router = router

31
litellm/proxy/litellm_pre_call_utils.py
View file

@ -13,6 +13,7 @@ from starlette.datastructures import Headers
import litellm
from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm._service_logger import ServiceLogging
from litellm.constants import PRE_CALL_EXECUTED_GUARDRAILS_KEY
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.litellm_core_utils.url_utils import is_url_destination_allowed_by_host
@ -161,6 +162,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS = (
"secret_fields",
"_guardrail_pipelines",
"_pipeline_managed_guardrails",
PRE_CALL_EXECUTED_GUARDRAILS_KEY,
)
_UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS = frozenset(
@ -397,6 +399,32 @@ def get_chain_id_from_headers(headers: Optional[Dict[str, str]]) -> Optional[str
)
def is_claude_code_user_agent(user_agent: str) -> bool:
"""Claude Code identifies itself as ``claude-cli/<version> ...``; the IDE
extensions and the Agent SDK run through the same CLI and share that prefix."""
return user_agent.startswith("claude-cli/")
def should_auto_drop_params_for_claude_code(
user_agent: str, data: dict, proxy_config: ProxyConfig
) -> bool:
"""drop_params defaults to on for Claude Code so its Anthropic-specific
params (e.g. thinking) don't fail requests routed to non-Anthropic
providers. An explicit drop_params from the caller or in the operator's
``litellm_settings`` always wins over this default."""
if not is_claude_code_user_agent(user_agent):
return False
if "drop_params" in data:
return False
config = getattr(proxy_config, "config", None)
litellm_settings = (
config.get("litellm_settings") if isinstance(config, dict) else None
)
return not (
isinstance(litellm_settings, dict) and "drop_params" in litellm_settings
)
def safe_add_api_version_from_query_params(data: dict, request: Request):
try:
if hasattr(request, "query_params"):
@ -1742,6 +1770,9 @@ async def add_litellm_data_to_request( # noqa: PLR0915
user_agent = request.headers["user-agent"]
data[_metadata_variable_name]["user_agent"] = user_agent
if should_auto_drop_params_for_claude_code(user_agent, data, proxy_config):
data["drop_params"] = True
# Merge caller-supplied tags (x-litellm-tags header, data["tags"] root-level)
# into request metadata for tag-based routing and spend attribution.
tags = LiteLLMProxyRequestSetup.add_request_tag_to_metadata(

13
litellm/proxy/management_endpoints/customer_endpoints.py
View file

@ -885,6 +885,19 @@ async def get_customer_daily_activity(
"""
Get daily activity for specific organizations or all accessible organizations.
"""
if (
user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN
and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY
):
raise HTTPException(
status_code=401,
detail={
"error": "Admin-only endpoint. Your user role={}".format(
user_api_key_dict.user_role
)
},
)
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:

4
litellm/proxy/management_endpoints/internal_user_endpoints.py
View file

@ -755,7 +755,7 @@ def _build_user_info_response(
response_model=UserInfoResponse,
)
@management_endpoint_wrapper
async def user_info( # noqa: PLR0915
async def user_info(
request: Request,
user_id: Optional[str] = fastapi.Query(
default=None, description="User ID in the request parameters"
@ -1082,7 +1082,7 @@ def _process_keys_for_user_info(
continue
try:
_key: dict = key.model_dump() # noqa
_key: dict = key.model_dump()
except Exception:
# if using pydantic v1
_key = key.dict()

31
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -2428,7 +2428,7 @@ async def _validate_update_key_data(
"/key/update", tags=["key management"], dependencies=[Depends(user_api_key_auth)]
)
@management_endpoint_wrapper
async def update_key_fn( # noqa: PLR0915
async def update_key_fn(
request: Request,
data: UpdateKeyRequest,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
@ -3377,7 +3377,7 @@ async def info_key_fn(
)
## REMOVE HASHED TOKEN INFO BEFORE RETURNING ##
try:
key_info = key_info.model_dump() # noqa
key_info = key_info.model_dump()
except Exception:
# if using pydantic v1
key_info = key_info.dict()
@ -4412,7 +4412,7 @@ async def _execute_virtual_key_regeneration(
dependencies=[Depends(user_api_key_auth)],
)
@management_endpoint_wrapper
async def regenerate_key_fn( # noqa: PLR0915
async def regenerate_key_fn(
key: Optional[str] = None,
data: Optional[RegenerateKeyRequest] = None,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
@ -4794,12 +4794,27 @@ async def reset_key_spend_fn(
proxy_logging_obj=proxy_logging_obj,
)
try:
from litellm.proxy.proxy_server import _invalidate_spend_counter
# Set Redis spend counter to the new value so get_current_spend()
# returns the correct amount immediately instead of the stale pre-reset value.
# We use reset_to (not 0.0) so partial resets are reflected correctly.
from litellm.proxy.proxy_server import spend_counter_cache
await _invalidate_spend_counter(counter_key=f"spend:key:{hashed_api_key}")
except Exception:
pass
_counter_key = f"spend:key:{hashed_api_key}"
spend_counter_cache.in_memory_cache.set_cache(
key=_counter_key, value=reset_to, ttl=60
)
if spend_counter_cache.redis_cache is not None:
try:
await spend_counter_cache.redis_cache.async_set_cache(
key=_counter_key, value=reset_to, ttl=60
)
except Exception as redis_err:
verbose_proxy_logger.warning(
"Failed to update spend counter %s in Redis: %s. "
"Budget checks may use stale value until counter expires.",
_counter_key,
redis_err,
)
max_budget = updated_key.max_budget
budget_reset_at = updated_key.budget_reset_at

2
litellm/proxy/management_endpoints/policy_endpoints/__init__.py
View file

@ -7,7 +7,7 @@ continue to work. Patch targets also resolve correctly since names
are imported directly into this namespace.
"""
from litellm.proxy.management_endpoints.policy_endpoints.endpoints import * # noqa: F401, F403
from litellm.proxy.management_endpoints.policy_endpoints.endpoints import * # noqa: F403
from litellm.proxy.management_endpoints.policy_endpoints.endpoints import ( # noqa: F401
_build_all_names_per_competitor,
_build_comparison_blocked_words,

43
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -3647,7 +3647,7 @@ async def team_info(
## REMOVE HASHED TOKEN INFO before returning ##
for key in keys:
try:
key = key.model_dump() # noqa
key = key.model_dump()
except Exception:
# if using pydantic v1
key = key.dict()
@ -4244,6 +4244,14 @@ async def _enforce_list_team_v2_access(
status_code=403,
detail={"error": "You can only view teams within your organizations."},
)
# When the caller is an org admin querying their own teams (or no
# specific user), null out user_id so that
# _build_team_list_where_conditions scopes only by organization_id
# — org admins should see all teams in their orgs, not just teams
# they are a direct member of. Keep user_id when the org admin
# explicitly queries a *different* user's teams.
if user_id is None or user_id == user_api_key_dict.user_id:
user_id = None
verbose_proxy_logger.debug(
"list_team_v2: org admin access for user=%s, org_ids=%s, user_id_filter=%s",
user_api_key_dict.user_id,
@ -4850,15 +4858,34 @@ async def team_model_add(
detail={"error": "Only proxy admin or team admin can modify team models"},
)
updated_models = add_new_models_to_team(team_obj=team_obj, new_models=data.models)
# Update team. `include` mirrors the relations the auth path consumes
# off the cached team object so that `_refresh_cached_team` doesn't
# null them out — see object_permission_utils.validate_key_search_tools_against_team
# and the MCP/agent authz paths, which treat a missing object_permission
# as "no team-level restriction".
# Atomic array append with dedup at the database level so concurrent
# BYOK model creates don't overwrite each other's team.models entries.
# When the team currently has models=[] (unrestricted access), the
# CASE expression inserts the 'all-proxy-models' sentinel first.
models_to_add = list(data.models)
await prisma_client.db.execute_raw(
'UPDATE "LiteLLM_TeamTable" '
"SET models = ("
" SELECT ARRAY(SELECT DISTINCT unnest("
" CASE WHEN cardinality(COALESCE(models, ARRAY[]::text[])) = 0 "
" THEN ARRAY['all-proxy-models']::text[] "
" ELSE models "
" END || $1::text[]"
" ))"
") "
"WHERE team_id = $2",
models_to_add,
data.team_id,
)
# Re-fetch via update (write-routed) instead of find_unique (read-routed)
# to avoid returning stale data from a read replica. The models column
# was already set by execute_raw above; this just retrieves the row from
# the writer and lets Prisma bump updated_at.
# `include` mirrors the relations the auth path consumes off the cached
# team object so that `_refresh_cached_team` doesn't null them out.
updated_team = await TeamRepository(prisma_client).table.update(
where={"team_id": data.team_id},
data={"models": updated_models},
data={"updated_at": datetime.now(timezone.utc)},
include={"object_permission": True}, # type: ignore
)

6
litellm/proxy/management_endpoints/ui_sso.py
View file

@ -114,7 +114,7 @@ from litellm.repositories.table_repositories import SSOConfigRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.secret_managers.main import get_secret_bool, str_to_bool
from litellm.types.proxy.management_endpoints.ui_sso import * # noqa: F403, F401
from litellm.types.proxy.management_endpoints.ui_sso import * # noqa: F403
from litellm.types.proxy.management_endpoints.ui_sso import (
DefaultTeamSSOParams,
MicrosoftGraphAPIUserGroupDirectoryObject,
@ -829,7 +829,7 @@ async def google_login(
key: Optional[str] = None,
existing_key: Optional[str] = None,
return_to: Optional[str] = None,
): # noqa: PLR0915
):
"""
Create Proxy API Keys using Google Workspace SSO. Requires setting PROXY_BASE_URL in .env
PROXY_BASE_URL should be the your deployed proxy endpoint, e.g. PROXY_BASE_URL="https://litellm-production-7002.up.railway.app/"
@ -1833,7 +1833,7 @@ async def check_and_update_if_proxy_admin_id(
@router.get("/sso/callback", tags=["experimental"], include_in_schema=False)
async def auth_callback(request: Request, state: Optional[str] = None): # noqa: PLR0915
async def auth_callback(request: Request, state: Optional[str] = None):
"""Verify login"""
verbose_proxy_logger.info(f"Starting SSO callback with state: {state}")

Some files were not shown because too many files have changed in this diff Show more