scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-13 08:52:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 175

Merge branch 'main' into fix/skip-unsatisfiable-tag-fallback-legs-43736

Browse source
This commit is contained in:
Quinn Xu 2026-10-25 09:53:18 -07:00 • committed by GitHub
commit 6ad0c0acdf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
881 changed files with 46414 additions and 9565 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/CODEOWNERS vendored
View file

@ -1,4 +1,4 @@
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/helm/ @yassin-berriai
/terraform/ @yassin-berriai

91
.github/workflows/interactions-spec-canary.yml vendored Normal file
View file

@ -0,0 +1,91 @@
name: Interactions spec canary
on:
schedule:
- cron: "17 7 * * *"
pull_request:
paths:
- .github/workflows/interactions-spec-canary.yml
- tests/unit/interactions/**
- litellm/types/interactions/**
workflow_dispatch:
permissions: {}
concurrency: interactions-spec-canary
jobs:
interactions-spec-canary:
name: interactions-spec-canary
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
outputs:
compliance: ${{ steps.compliance.outcome }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Install dependencies
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy --no-install-project
- name: Run the compliance tests against Google's live spec
id: compliance
run: |
uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \
--interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \
--tb=short -vv -p no:rerunfailures -p no:pytest-retry \
--allow-hosts=127.0.0.1,::1,localhost,ai.google.dev
tracking-issue:
name: tracking-issue
needs: interactions-spec-canary
if: always() && github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
issues: write
steps:
- name: File the tracking issue unless the compliance step passed, close it when it did
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
env:
COMPLIANCE_OUTCOME: ${{ needs.interactions-spec-canary.outputs.compliance }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
with:
script: |
const title = "Interactions spec canary is red";
const outcome = process.env.COMPLIANCE_OUTCOME;
const runUrl = process.env.RUN_URL;
const query = `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open in:title "${title}"`;
const { data: found } = await github.rest.search.issuesAndPullRequests({ q: query, per_page: 20 });
const open = found.items.find((issue) => issue.title === title);
if (outcome === "success") {
if (open === undefined) return;
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Green again: ${runUrl}` });
await github.rest.issues.update({ ...context.repo, issue_number: open.number, state: "closed" });
return;
}
const state = outcome === "failure" ? "red" : `not run (compliance step outcome: ${outcome || "none"})`;
if (open !== undefined) {
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Still ${state}: ${runUrl}` });
return;
}
await github.rest.issues.create({
...context.repo,
title,
assignees: ["mateo-berri"],
body: `The scheduled run of tests/unit/interactions/test_openapi_compliance.py against Google's live Interactions spec is ${state}: ${runUrl}\n\nlitellm/types/interactions/README.md says how to refresh the snapshot and regenerate the types.`,
});

1
.github/workflows/test-unit.yml vendored
View file

@ -425,6 +425,7 @@ jobs:
tests/unit/realtime_api
tests/unit/rerank_api
tests/unit/search
tests/unit/text_completions
tests/unit/vector_stores
tests/unit/videos
python-version: "3.12"

9
Dockerfile
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@ -61,6 +61,7 @@ RUN apk add --no-cache \
python-3.13-dev \
rust \
openssl \
libssl3 \
openssl-dev \
nodejs \
npm \
@ -120,8 +121,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder
COPY --from=uvbin /uv /usr/local/bin/uv
RUN apk add --no-cache python-3.13
ADD --checksum=sha256:2f7ae5cdd9d91731c0990e74a58239dc3e3fd2bf28dab23b55eafcdc47aaf87e \
https://github.com/BerriAI/litellm-admin-agent/archive/ef501e94bc9fbacb9233b922abf71427f030408c.tar.gz /tmp/liteadmin.tar.gz
ADD --checksum=sha256:4fb1847e28e2be30ba76dac94f02c4903291326516a0f4ba772f6ffb8b25d05c \
https://github.com/BerriAI/litellm-admin-agent/archive/5661c805ce479a0cf11b842ea133cf8d22cd7e34.tar.gz /tmp/liteadmin.tar.gz
RUN mkdir /tmp/liteadmin && tar xzf /tmp/liteadmin.tar.gz --strip-components=1 -C /tmp/liteadmin && \
uv venv /opt/liteadmin --python python3.13 && \
uv pip install --python /opt/liteadmin/bin/python --require-hashes -r /tmp/liteadmin/requirements.txt && \

6
README.md
View file

@ -616,15 +616,15 @@ The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws
### Verify Docker Image Signatures
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Releases are signed with the key introduced in [commit `e82a3c6`](https://github.com/BerriAI/litellm/commit/e82a3c646908ba0a6ee0beddc4310d7a6cac4065).
**Verify using the pinned commit hash (recommended):**
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the published signing key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
--key https://raw.githubusercontent.com/BerriAI/litellm/e82a3c646908ba0a6ee0beddc4310d7a6cac4065/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```

13
backend/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
@ -43,7 +43,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
@ -124,7 +124,12 @@ ENV HOME=/home/nonroot \
PYTHONUNBUFFERED=1 \
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
COPY --from=builder --chown=nonroot:nonroot /app /app
COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
COPY --from=builder --chown=nonroot:nonroot /app/backend /app/backend
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
COPY --from=builder /opt/prisma /opt/prisma
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \

4
cosign.pub
View file

@ -1,4 +1,4 @@
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb
POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g==
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEG06dpKM+vEDnmp2zcjyXgVCXxzRg
JA5H2LnmwBKQafZGka6OOrWujCEWATQewiAgyD4WQdJ0pvU9RC4mvNAjvQ==
-----END PUBLIC KEY-----

5
docker/Dockerfile.database
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@ -59,6 +59,7 @@ RUN apk add --no-cache \
python-3.13 \
python-3.13-dev \
openssl \
libssl3 \
openssl-dev \
nodejs \
npm \

5
docker/Dockerfile.non_root
View file

@ -1,8 +1,8 @@
# syntax=docker/dockerfile:1.7
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@ -60,6 +60,7 @@ RUN for i in 1 2 3; do \
coreutils \
curl \
openssl \
libssl3 \
libsndfile \
nodejs \
npm && break || sleep 5; \

2
docker/build_from_pip/Dockerfile.build_from_pip
View file

@ -30,7 +30,7 @@ RUN uv venv --python python && \
"google-cloud-aiplatform==1.133.0" \
"google-genai==1.37.0" \
"anthropic[vertex]==0.84.0" \
"grpcio==1.78.0" \
"grpcio==1.84.1" \
"prometheus-client==0.20.0" \
"langfuse==2.59.7" \
"opentelemetry-api==1.28.0" \

13
enterprise/enterprise_hooks/aporia_ai.py
View file

@ -15,9 +15,10 @@ sys.path.insert(
) # Adds the parent directory to the system path
import json
import sys
from typing import Any, List, Literal, Optional
from typing import Final, List, Literal, Optional
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -29,11 +30,13 @@ from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.llms.openai.common_utils import decoded_json_object
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata
from litellm.types.guardrails import GuardrailEventHooks
GUARDRAIL_NAME = "aporia"
_DECODED_JSON: Final = TypeAdapter(object)
class AporiaGuardrail(CustomGuardrail):
@ -68,7 +71,7 @@ class AporiaGuardrail(CustomGuardrail):
async def prepare_aporia_request(
self, new_messages: List[dict], response_string: Optional[str] = None
) -> dict:
data: dict[str, Any] = {}
data: dict[str, object] = {}
if new_messages is not None:
data["messages"] = new_messages
if response_string is not None:
@ -122,8 +125,10 @@ class AporiaGuardrail(CustomGuardrail):
verbose_proxy_logger.debug("Aporia AI response: %s", response.text)
if response.status_code == 200:
# check if the response was flagged
_json_response = response.json()
action: str = _json_response.get(
_json_response: Final = decoded_json_object(
_DECODED_JSON.validate_python(response.json())
)
action: Final = _json_response.get(
"action"
) # possible values are modify, passthrough, block, rephrase
if action == "block":

17
enterprise/enterprise_hooks/blocked_user_list.py
View file

@ -7,9 +7,10 @@
## This accepts a list of user id's for whom calls will be rejected
from typing import Literal, Optional
from typing import Final, Literal, Optional
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._internal_context import with_service_target
@ -19,6 +20,9 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import LiteLLM_EndUserTable, UserAPIKeyAuth
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET
from litellm.proxy.utils import PrismaClient
from litellm.repositories.table_repositories import EndUserRepository
_REQUEST_BODY_VALUE: Final = TypeAdapter(object)
class ENTERPRISE_BlockedUserList(CustomLogger):
@ -94,15 +98,14 @@ class ENTERPRISE_BlockedUserList(CustomLogger):
)
if end_user_cache_obj is None and self.prisma_client is not None:
# check db
end_user_obj = (
await self.prisma_client.db.litellm_endusertable.find_unique(
where={"user_id": user}
)
)
end_user_id: Final = _REQUEST_BODY_VALUE.validate_python(user)
end_user_obj = await EndUserRepository(
self.prisma_client
).table.find_unique(where={"user_id": end_user_id})
if end_user_obj is None: # user not in db - assume not blocked
end_user_obj = LiteLLM_EndUserTable(user_id=user, blocked=False)
cache.set_cache(key=cache_key, value=end_user_obj, ttl=60)
if end_user_obj is not None and end_user_obj.blocked is True:
if end_user_obj.blocked is True:
raise HTTPException(
status_code=400,
detail={

13
enterprise/enterprise_hooks/google_text_moderation.py
View file

@ -6,7 +6,10 @@
# +-----------------------------------------------+
# Thank you users! We ❤️ you! - Krrish & Ishaan
from typing import Final
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -15,6 +18,8 @@ from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import iter_message_text
from litellm.types.utils import CallTypesLiteral
_THRESHOLD_VALUE: Final = TypeAdapter(object)
class ENTERPRISE_GoogleTextModeration(CustomLogger):
user_api_key_cache = None
@ -69,9 +74,11 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
f"{category}_confidence_threshold",
default_confidence_threshold,
)
set_confidence_value = getattr(
self,
f"{category}_confidence_threshold",
set_confidence_value = _THRESHOLD_VALUE.validate_python(
getattr(
self,
f"{category}_confidence_threshold",
)
)
verbose_proxy_logger.info(
f"Google Text Moderation: {category}_confidence_threshold: {set_confidence_value}"

7
enterprise/litellm_enterprise/enterprise_callbacks/example_logging_api.py
View file

@ -1,15 +1,20 @@
# this is an example endpoint to receive data from litellm
from typing import Final
from fastapi import FastAPI, HTTPException, Request
from pydantic import TypeAdapter
app = FastAPI()
_LOGGED_EVENT: Final = TypeAdapter(object)
@app.post("/log-event")
async def log_event(request: Request):
try:
print("Received /log-event request") # noqa
# Assuming the incoming request has JSON data
data = await request.json()
data: Final = _LOGGED_EVENT.validate_python(await request.json())
print("Received request data:") # noqa
print(data) # noqa

18
enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py
View file

@ -8,25 +8,30 @@
## This provides an LLM Guard Integration for content moderation on the proxy
import asyncio
from typing import Final, Optional
from collections.abc import Mapping
from typing import Final
import aiohttp
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.openai.common_utils import decoded_json_object
from litellm.proxy._types import UserAPIKeyAuth
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import CallTypesLiteral
_DECODED_JSON: Final = TypeAdapter(object)
class _ENTERPRISE_LLMGuard(CustomLogger):
# Class variables or attributes
def __init__(
self,
mock_testing: bool = False,
mock_redacted_text: Optional[dict] = None,
mock_redacted_text: Mapping[str, object] | None = None,
):
self.mock_redacted_text = mock_redacted_text
self.llm_guard_mode = litellm.llm_guard_mode
@ -66,7 +71,9 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
async with session.post(
analyze_url, json={"prompt": text}
) as response:
redacted_text = await response.json()
redacted_text = _DECODED_JSON.validate_python(
await response.json()
)
verbose_proxy_logger.debug(
f"LLM Guard: Received response - {redacted_text}"
)
@ -77,12 +84,13 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
"error": f"Invalid content moderation response: {redacted_text}"
},
)
if redacted_text.get("is_valid", None) is False:
redacted_response: Final = decoded_json_object(redacted_text)
if redacted_response.get("is_valid", None) is False:
raise HTTPException(
status_code=400,
detail={"error": "Violated content safety policy"},
)
sanitized_prompt = redacted_text.get("sanitized_prompt")
sanitized_prompt: Final = redacted_response.get("sanitized_prompt")
return sanitized_prompt if isinstance(sanitized_prompt, str) else text
except Exception as e:
verbose_proxy_logger.exception(

4
enterprise/litellm_enterprise/proxy/auth/route_checks.py
View file

@ -2,6 +2,8 @@ import os
from fastapi import HTTPException, status
from litellm.proxy._types import MODEL_LIST_ROUTES
class EnterpriseRouteChecks:
@staticmethod
@ -43,7 +45,7 @@ class EnterpriseRouteChecks:
# Routes that should remain accessible even when LLM API endpoints are disabled.
# These are read-only model listing routes needed by the Admin UI.
LLM_API_EXEMPT_ROUTES = ["/models", "/v1/models"]
LLM_API_EXEMPT_ROUTES = MODEL_LIST_ROUTES
@staticmethod
def should_call_route(route: str):

4
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -1040,8 +1040,8 @@ class CheckBatchCost:
from litellm.proxy.proxy_server import proxy_config
from litellm.secret_managers.main import str_to_bool
api_key: Final = getattr(job, "api_key", None)
team_id: Final = getattr(job, "team_id", None)
api_key: Final = job.api_key if hasattr(job, "api_key") else None
team_id: Final = job.team_id if hasattr(job, "team_id") else None
has_key_row: Final = api_key is not None and is_valid_sha256_hash(api_key)
try:
key_row: Final = (

27
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -44,6 +44,7 @@ from litellm.constants import (
)
from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.litellm_core_utils.internal_call_metadata import requests_background
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -133,7 +134,7 @@ if TYPE_CHECKING:
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
from litellm.proxy.utils import PrismaClient as _PrismaClient
Span = Union[_Span, Any]
Span = _Span
InternalUsageCache = _InternalUsageCache
PrismaClient = _PrismaClient
else:
@ -490,6 +491,14 @@ _PROVIDER_FILE_ID_KEYS: Final = tuple(dict.fromkeys(chain.from_iterable(_PROVIDE
_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS: Final = (0.5, 1.0, 2.0)
class _SupportsKeyedGet(Protocol):
def get(self, key: str, default: object, /) -> object: ...
def _keyed_get(source: object, key: str, default: object = None) -> object:
return require_attributes(source, _SupportsKeyedGet, "get").get(key, default)
class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Class variables or attributes
def __init__(
@ -1611,7 +1620,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
return holders if holders or "input_file_id" in request else healthy_deployments
async def async_pre_call_deployment_hook(
self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
self, kwargs: dict[str, object], call_type: Optional[CallTypes]
) -> Optional[dict]:
"""
Allow modifying the request just before it's sent to the deployment.
@ -1623,11 +1632,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping"))
# model_info may be at top-level or nested under litellm_metadata
# (batch/file operations use litellm_metadata)
model_id = cast(Optional[str], kwargs.get("model_info", {}).get("id", None))
model_id = cast(Optional[str], _keyed_get(kwargs.get("model_info", {}), "id"))
if model_id is None:
model_id = cast(
Optional[str],
kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None),
_keyed_get(_keyed_get(kwargs.get("litellm_metadata", {}), "model_info", {}), "id"),
)
for accessor_key in accessor_keys:
input_file_id = cast(Optional[str], kwargs.get(accessor_key))
@ -1824,7 +1833,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
),
)
async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span) -> dict:
async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span | None) -> dict:
"""
Get model-specific file IDs for a list of proxy file IDs.
Returns a dictionary mapping litellm_proxy/ file_id -> model_id -> model_file_id
@ -1866,7 +1875,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
llm_router: Optional[Router],
_create_file_request: CreateFileRequest,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
) -> List[OpenAIFileObject]:
if llm_router is None:
raise Exception("LLM Router not initialized. Ensure models added to proxy.")
@ -1882,7 +1891,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
create_file_request: CreateFileRequest,
llm_router: Router,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
user_api_key_dict: UserAPIKeyAuth,
) -> OpenAIFileObject:
responses = await self.create_file_for_each_model(
@ -1949,7 +1958,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
file_objects: List[OpenAIFileObject],
create_file_request: CreateFileRequest,
internal_usage_cache: InternalUsageCache,
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
target_model_names_list: List[str],
) -> OpenAIFileObject:
## GET THE FILE TYPE FROM THE CREATE FILE REQUEST
@ -2652,7 +2661,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Continue with other files even if one fails
continue
def _get_content_type_from_file_object(self, file_object: Optional[Any]) -> str:
def _get_content_type_from_file_object(self, file_object: object) -> str:
"""
Determine content type from file object.

6
enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py
View file

@ -29,7 +29,7 @@ if TYPE_CHECKING:
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
from litellm.proxy.utils import PrismaClient as _PrismaClient
Span = Union[_Span, Any]
Span = _Span
InternalUsageCache = _InternalUsageCache
PrismaClient = _PrismaClient
else:
@ -107,7 +107,7 @@ class _PROXY_LiteLLMManagedVectorStores(
llm_router: Router,
model: str,
request_data: dict[str, object] | VectorStoreCreateOptionalRequestParams,
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
) -> VectorStoreCreateResponse:
"""
Create a vector store for a specific model.
@ -134,7 +134,7 @@ class _PROXY_LiteLLMManagedVectorStores(
create_request: VectorStoreCreateOptionalRequestParams,
llm_router: Router,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
user_api_key_dict: UserAPIKeyAuth,
) -> VectorStoreCreateResponse:
"""

13
enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py
View file

@ -36,6 +36,7 @@ from litellm.repositories.budget_repository import BudgetRepository
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.project_repository import ProjectRepository
from litellm.repositories.table_repositories import PrismaTableRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
@ -73,6 +74,16 @@ def _verification_token_table(
return VerificationTokenRepository(prisma_client).table
class _WriterVerificationTokenRepository(PrismaTableRepository["prisma_models.LiteLLM_VerificationToken"]):
table_name = "litellm_verificationtoken"
def _writer_verification_token_table(
prisma_client: PrismaClient,
) -> TableActions["prisma_models.LiteLLM_VerificationToken"]:
return _WriterVerificationTokenRepository(prisma_client, use_writer=True).table
def _budget_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_BudgetTable"]:
return BudgetRepository(prisma_client).table
@ -849,7 +860,7 @@ async def update_project(
else None
)
if current_project is not None and data.team_id != current_project.team_id:
mismatched_key_count: Final = await prisma_client.writer_db.litellm_verificationtoken.count(
mismatched_key_count: Final = await _writer_verification_token_table(prisma_client).count(
where={
"project_id": data.project_id,
"OR": [{"team_id": {"not": data.team_id}}, {"team_id": None}],

8
enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py
View file

@ -14,6 +14,7 @@ from collections.abc import Mapping
from typing import TYPE_CHECKING, Final, List, Optional, Protocol
from fastapi import APIRouter, Depends, HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -37,6 +38,7 @@ if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
router = APIRouter()
_DECODED_JSON: Final = TypeAdapter(object)
class ManagedVectorStoreRow(Protocol):
@ -348,11 +350,11 @@ async def update_vector_store(
raise HTTPException(status_code=500, detail="Database not connected")
try:
update_data = data.model_dump(exclude_unset=True)
vector_store_id: Final[str] = update_data.pop("vector_store_id")
update_data = data.model_dump(exclude_unset=True, exclude={"vector_store_id"})
vector_store_id: Final = data.vector_store_id
if update_data.get("vector_store_metadata") is not None:
update_data["vector_store_metadata"] = safe_dumps(
update_data["vector_store_metadata"]
_DECODED_JSON.validate_python(update_data["vector_store_metadata"])
)
updated = await managed_vector_store_table(prisma_client).update(

13
gateway/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
ARG PGBOUNCER_VERSION=1.26.0
@ -59,7 +59,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
@ -140,7 +140,12 @@ ENV HOME=/home/nonroot \
PYTHONUNBUFFERED=1 \
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
COPY --from=builder --chown=nonroot:nonroot /app /app
COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
COPY --from=builder /opt/prisma /opt/prisma
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer

6
litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs
View file

@ -235,7 +235,10 @@ impl AnthropicBeta {
BetaProvider::Databricks,
],
Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
Self::ComputerUse20250124
| Self::ComputerUse20251124
| Self::Context1m20250807
| Self::ThinkingBindingControls20260801 => &[],
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
Self::Effort20251124 => &[BetaProvider::VertexAi],
Self::FastMode20260201 | Self::Oauth20250420 => &[
@ -272,7 +275,6 @@ impl AnthropicBeta {
BetaProvider::Databricks,
],
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
Self::ThinkingBindingControls20260801 => &[BetaProvider::AzureAi],
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
Self::ToolSearchTool20251019 => &[
BetaProvider::Anthropic,

21
litellm/__init__.py
View file

@ -175,7 +175,7 @@ argilla_batch_size: Optional[int] = None
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload.
gcs_pub_sub_use_v1: Optional[bool] = False # if you want to use v1 gcs pubsub logged payload
generic_api_use_v1: Optional[bool] = False # if you want to use v1 generic api logged payload
argilla_transformation_object: Optional[Dict[str, Any]] = None
argilla_transformation_object: Mapping[str, str] | None = None
_async_input_callback: List[Union[str, Callable[..., object], "CustomLogger"]] = ( # CustomLogger is lazy-loaded
[]
) # internal variable - async custom callbacks are routed here.
@ -333,7 +333,7 @@ in_memory_llm_clients_cache: "LLMClientCache"
safe_memory_mode: bool = False
enable_azure_ad_token_refresh: Optional[bool] = False
# Proxy Authentication - auto-obtain/refresh OAuth2/JWT tokens for LiteLLM Proxy
proxy_auth: Optional[Any] = None
proxy_auth: _ProxyAuthHandlerType | None = None
### DEFAULT AZURE API VERSION ###
AZURE_DEFAULT_API_VERSION = "2025-02-01-preview" # this is updated to the latest
### DEFAULT WATSONX API VERSION ###
@ -503,7 +503,7 @@ skill_search_embedding_model: Optional[str] = None
# Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]])
# New format: { "displayName": { "url": "...", "index": 0 } }
# Old format: { "displayName": "url" } (for backward compatibility)
public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {}
public_model_groups_links: dict[str, str | dict[str, object]] = {}
#### REQUEST PRIORITIZATION #######
priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]]] = None
# priority_reservation_settings is lazy-loaded via __getattr__
@ -682,6 +682,7 @@ darkbloom_models: Set = set()
openinfer_models: Final[Set[str]] = set()
v0_models: Set = set()
morph_models: Set = set()
opencode_go_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads
lambda_ai_models: Set = set()
inception_models: Set = set()
hyperbolic_models: Set = set()
@ -970,6 +971,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
v0_models.add(key)
elif value.get("litellm_provider") == "morph":
morph_models.add(key)
elif value.get("litellm_provider") == "opencode_go":
opencode_go_models.add(key)
elif value.get("litellm_provider") == "lambda_ai":
lambda_ai_models.add(key)
elif value.get("litellm_provider") == "inception":
@ -1137,6 +1140,7 @@ model_list = list(
| openinfer_models
| v0_models
| morph_models
| opencode_go_models
| lambda_ai_models
| inception_models
| black_forest_labs_models
@ -1254,6 +1258,7 @@ def _build_models_by_provider() -> dict:
"openinfer": openinfer_models,
"v0": v0_models,
"morph": morph_models,
"opencode_go": opencode_go_models,
"lambda_ai": lambda_ai_models,
"inception": inception_models,
"hyperbolic": hyperbolic_models,
@ -2289,6 +2294,10 @@ if TYPE_CHECKING:
LiteLLM_UpperboundKeyGenerateParams,
)
from litellm.utils import ModelResponseListIterator as _ModelResponseListIterator
from collections.abc import Awaitable
from litellm.types.utils import ModelResponse as _ModelResponseType
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper as _CustomStreamWrapperType
from litellm.proxy_auth import ProxyAuthHandler as _ProxyAuthHandlerType
# Cost calculator functions
cost_per_token: Callable[..., Tuple[float, float]]
@ -2312,7 +2321,7 @@ if TYPE_CHECKING:
supports_audio_output: Callable[..., bool]
supports_system_messages: Callable[..., bool]
supports_reasoning: Callable[..., bool]
acreate: Callable[..., Any]
acreate: Callable[..., Awaitable[_ModelResponseType | _CustomStreamWrapperType]]
get_max_tokens: Callable[..., int]
get_model_info: Callable[..., _ModelInfoType]
register_prompt_template: Callable[..., None]
@ -2610,8 +2619,10 @@ def __getattr__(name: str) -> Any:
cancel_run,
delete_run,
)
from types import MappingProxyType
return locals()[name]
evals_exports: Final[Mapping[str, object]] = MappingProxyType(locals())
return evals_exports[name]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

4
litellm/_lazy_imports.py
View file

@ -19,7 +19,7 @@ import importlib
import sys
from collections.abc import Callable, Mapping
from types import ModuleType
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final
from typing_extensions import ReadOnly, TypedDict
@ -456,7 +456,7 @@ def _lazy_import_http_handlers(name: str) -> object:
params: Final = {"timeout": async_timeout, "client_alias": "module level aclient"}
# Create the client instance
provider_id: Final = cast(Any, "litellm_module_level_client")
provider_id: Final = "litellm_module_level_client"
async_client: Final = get_async_httpx_client(
llm_provider=provider_id,
params=params,

20
litellm/a2a_protocol/cost_calculator.py
View file

@ -5,7 +5,10 @@ Supports dynamic cost parameters that allow platform owners
to define custom costs per agent query or per token.
"""
from typing import TYPE_CHECKING, Any, Final
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, SupportsFloat, SupportsIndex
from typing_extensions import Buffer
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import (
@ -15,6 +18,15 @@ else:
LitellmLoggingObject = Any
def _custom_response_cost(model_call_details: Mapping[str, object]) -> float | None:
response_cost: Final = model_call_details.get("response_cost", None)
if response_cost is None:
return None
if isinstance(response_cost, (str, Buffer, SupportsFloat, SupportsIndex)):
return float(response_cost)
raise TypeError(f"float() argument must be a string or a real number, not {type(response_cost).__name__!r}")
class A2ACostCalculator:
@staticmethod
def calculate_a2a_cost(
@ -45,9 +57,9 @@ class A2ACostCalculator:
model_call_details: Final = litellm_logging_obj.model_call_details
# Check if user set a custom response cost (backward compatibility)
response_cost: Final = model_call_details.get("response_cost", None)
if response_cost is not None:
return float(response_cost)
custom_response_cost: Final = _custom_response_cost(model_call_details)
if custom_response_cost is not None:
return custom_response_cost
# Get litellm_params for cost parameters
litellm_params: Final = model_call_details.get("litellm_params", {}) or {}

27
litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py
View file

@ -84,14 +84,23 @@ def _filter_reserved_headers(
return filtered or None
def _calling_key_hash(litellm_params: Mapping[str, object]) -> str | None:
principal: Final = litellm_params.get(A2A_USER_API_KEY_HASH_PARAM)
if principal is None or isinstance(principal, str):
return principal
if not principal:
return None
raise AttributeError(f"{type(principal).__name__!r} object has no attribute 'encode'", name="encode", obj=principal)
def _request_scoped_runtime_session_id(
params: Mapping[str, object],
litellm_params: Mapping[str, Any],
litellm_params: Mapping[str, object],
) -> str | None:
context_id: Final = get_session_id_from_a2a_params(params)
if not isinstance(context_id, str) or not context_id:
return None
return scope_session_to_principal(context_id, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM))
return scope_session_to_principal(context_id, _calling_key_hash(litellm_params))
def _validate_runtime_session_id(session_id: str, model: str) -> str:
@ -109,6 +118,13 @@ def _validate_runtime_session_id(session_id: str, model: str) -> str:
)
def _agentcore_model_name(litellm_params: Mapping[str, object]) -> str:
model: Final = litellm_params.get("model", "")
if isinstance(model, str):
return model
raise AttributeError(f"{type(model).__name__!r} object has no attribute 'startswith'", name="startswith", obj=model)
class BedrockAgentCoreA2ATransformation:
"""
Request/response transformation for Bedrock AgentCore A2A agents.
@ -152,11 +168,8 @@ class BedrockAgentCoreA2ATransformation:
"""
# Extract model and strip the "bedrock/" prefix
# "bedrock/agentcore/arn:aws:..." → "agentcore/arn:aws:..."
model: Final = litellm_params.get("model", "")
if model.startswith("bedrock/"):
agentcore_model = model[len("bedrock/") :]
else:
agentcore_model = model
model: Final = _agentcore_model_name(litellm_params)
agentcore_model: Final = model.removeprefix("bedrock/")
# Build optional_params from litellm_params (everything except model and custom_llm_provider)
optional_params: Final = {k: v for k, v in litellm_params.items() if k not in ("model", "custom_llm_provider")}

40
litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py
View file

@ -8,11 +8,31 @@ WXO uses a REST API (not A2A/JSON-RPC) with an async-poll execution model:
"""
import asyncio
from collections.abc import AsyncIterator, Mapping
from typing import Any, Final
from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
from itertools import chain
from typing import Final
from uuid import uuid4
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
from litellm.llms.openai.common_utils import decoded_json_object
def _a2a_text_part(part: object) -> Sequence[object]:
part_fields: Final = as_str_mapping(part)
if not isinstance(part_fields, dict):
return ()
if part_fields.get("kind") in (None, "", "text") and part_fields.get("text"):
return (part_fields["text"],)
return ()
def _join_with_spaces(texts: Sequence[object]) -> str:
strings: Final = [text for text in texts if isinstance(text, str)]
if len(strings) == len(texts):
return " ".join(strings)
first_bad: Final = next(position for position, text in enumerate(texts) if not isinstance(text, str))
raise TypeError(f"sequence item {first_bad}: expected str instance, {type(texts[first_bad]).__name__} found")
class WatsonxOrchestrateTransformation:
@ -29,22 +49,16 @@ class WatsonxOrchestrateTransformation:
return f"{cp4d_host.rstrip('/')}/orchestrate/cpd/instances/{instance_id}"
@staticmethod
def extract_text_from_a2a_params(params: dict[str, Any]) -> str:
def extract_text_from_a2a_params(params: Mapping[str, object]) -> str:
"""
Extract user message text from A2A MessageSendParams.
A2A format: params.message.parts[*] where part.kind == "text"
"""
message: Final = params.get("message", {})
parts: Final = message.get("parts", [])
texts: Final = []
for part in parts:
if not isinstance(part, dict):
continue
kind = part.get("kind")
if kind in (None, "", "text") and part.get("text"):
texts.append(part["text"])
return " ".join(texts) or ""
message: Final = decoded_json_object(params.get("message", {}))
parts: Final[Iterable[object]] = iter_items(message.get("parts", []))
texts: Final = list(chain.from_iterable(map(_a2a_text_part, parts)))
return _join_with_spaces(texts) or ""
@staticmethod
def build_wxo_run_body(

21
litellm/a2a_protocol/utils.py
View file

@ -4,15 +4,20 @@ Utility functions for A2A protocol.
import hashlib
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final, Protocol
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import has_attributes
if TYPE_CHECKING:
from a2a.types import SendMessageRequest, SendStreamingMessageRequest
class _SupportsModelDump(Protocol):
def model_dump(self, *, mode: str) -> object: ...
class A2ARequestUtils:
"""Utility class for A2A request/response processing."""
@ -74,7 +79,7 @@ class A2ARequestUtils:
@staticmethod
def get_input_message_from_request(
request: "SendMessageRequest | SendStreamingMessageRequest",
) -> Any:
) -> object:
"""
Extract the input message from an A2A request.
@ -84,7 +89,7 @@ class A2ARequestUtils:
Returns:
The message object/dict or None
"""
params: Final = getattr(request, "params", None)
params: Final[object] = getattr(request, "params", None)
if params is None:
return None
return getattr(params, "message", None)
@ -127,10 +132,12 @@ class A2ARequestUtils:
# the dict branch — request-side parts are a2a-sdk Part RootModels whose
# kind/text live on part.root, which the object branch cannot read. This
# mirrors how the response side already works (it operates on model_dump).
input_message = A2ARequestUtils.get_input_message_from_request(request)
if input_message is not None and hasattr(input_message, "model_dump"):
input_message = input_message.model_dump(mode="json")
input_text: Final = A2ARequestUtils.extract_text_from_message(input_message)
input_message: Final = A2ARequestUtils.get_input_message_from_request(request)
input_text: Final = A2ARequestUtils.extract_text_from_message(
input_message.model_dump(mode="json")
if has_attributes(input_message, _SupportsModelDump, "model_dump")
else input_message
)
prompt_tokens: Final = A2ARequestUtils.count_tokens(input_text)
# Count output tokens

2
litellm/anthropic_beta_headers_config.json
View file

@ -67,7 +67,7 @@
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
"text_editor_20241022": null,
"text_editor_20250124": null,
"thinking-binding-controls-2026-08-01": null,
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
"token-efficient-tools-2025-02-19": null,
"web-fetch-2025-09-10": "web-fetch-2025-09-10",

16
litellm/anthropic_beta_headers_manager.py
View file

@ -36,6 +36,7 @@ import httpx
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
# Cache for the loaded configuration
_BETA_HEADERS_CONFIG: dict | None = None
@ -44,6 +45,7 @@ _BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_na
BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
_DECODED_JSON: Final = TypeAdapter(object)
class GetAnthropicBetaHeadersConfig:
@ -75,7 +77,7 @@ class GetAnthropicBetaHeadersConfig:
}
@staticmethod
def _check_is_valid_dict(fetched_config: dict) -> bool:
def _check_is_valid_dict(fetched_config: object) -> bool:
"""Check if fetched config is a non-empty dict with expected structure."""
if not isinstance(fetched_config, dict):
verbose_logger.warning(
@ -109,7 +111,7 @@ class GetAnthropicBetaHeadersConfig:
return True
@classmethod
def validate_beta_headers_config(cls, fetched_config: dict) -> bool:
def validate_beta_headers_config(cls, fetched_config: object) -> bool:
"""
Validate the integrity of a fetched beta headers config.
@ -118,7 +120,7 @@ class GetAnthropicBetaHeadersConfig:
return cls._check_is_valid_dict(fetched_config)
@staticmethod
def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> dict:
def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> object:
"""
Fetch the beta headers config from a remote URL.
@ -127,7 +129,7 @@ class GetAnthropicBetaHeadersConfig:
"""
response: Final = httpx.get(url, timeout=timeout)
response.raise_for_status()
return response.json()
return _DECODED_JSON.validate_python(response.json())
def get_beta_headers_config(url: str) -> dict:
@ -145,7 +147,7 @@ def get_beta_headers_config(url: str) -> dict:
Dict containing the beta headers configuration
"""
# Check if local-only mode is enabled
if os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
# verbose_logger.debug("Using local Anthropic beta headers config (LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True)")
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
@ -160,7 +162,9 @@ def get_beta_headers_config(url: str) -> dict:
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
# Validate the fetched config
if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content):
if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content) or not isinstance(
content, dict
):
verbose_logger.warning(
"LiteLLM: Fetched beta headers config failed integrity check. Using local backup instead. url=%s",
url,

11
litellm/anthropic_interface/exceptions/exception_mapping_utils.py
View file

@ -7,10 +7,14 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop
import json
from typing import Final
from pydantic import TypeAdapter
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicErrorType
_PARSED_ERROR_BODY: Final = TypeAdapter(object)
# HTTP status code -> Anthropic error type
# Source: https://docs.anthropic.com/en/api/errors
ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = {
@ -79,7 +83,7 @@ class AnthropicExceptionMapping:
- Generic: {"message": "..."}
- Plain strings
"""
parsed: Final = safe_json_loads(raw_message)
parsed: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message))
if isinstance(parsed, dict):
# Bedrock format
if "detail" in parsed and isinstance(parsed["detail"], dict):
@ -145,9 +149,8 @@ class AnthropicExceptionMapping:
AnthropicErrorResponse dict
"""
# Try to parse as JSON once
parsed: dict | None = safe_json_loads(raw_message)
if not isinstance(parsed, dict):
parsed = None
decoded: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message))
parsed: Final[dict[str, object] | None] = decoded if isinstance(decoded, dict) else None
# If parsed and already in Anthropic format - passthrough
if parsed is not None and AnthropicExceptionMapping._is_anthropic_error_dict(parsed):

24
litellm/assistants/main.py
View file

@ -77,7 +77,7 @@ def get_assistants(
api_base: str | None = None,
api_version: str | None = None,
**kwargs,
) -> SyncCursorPage[Assistant]:
) -> SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]]:
aget_assistants: Final[bool | None] = kwargs.pop("aget_assistants", None)
if aget_assistants is not None and not isinstance(aget_assistants, bool):
raise Exception("Invalid value passed in for aget_assistants. Only bool or None allowed")
@ -100,7 +100,7 @@ def get_assistants(
elif timeout is None:
timeout = 600.0
response: SyncCursorPage[Assistant] | None = None
response: SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -564,7 +564,7 @@ def create_thread(
tool_resources: OpenAICreateThreadParamsToolResources | None = None,
client: OpenAI | None = None,
**kwargs,
) -> Thread:
) -> Thread | Coroutine[None, None, Thread]:
"""
- get the llm provider
- if openai - route it there
@ -610,7 +610,7 @@ def create_thread(
api_base: str | None = None
api_key: str | None = None
response: Thread | None = None
response: Thread | Coroutine[None, None, Thread] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -735,7 +735,7 @@ def get_thread(
thread_id: str,
client=None,
**kwargs,
) -> Thread:
) -> Thread | Coroutine[None, None, Thread]:
"""Get the thread object, given a thread_id"""
aget_thread: Final = kwargs.pop("aget_thread", None)
optional_params: Final = GenericLiteLLMParams(**kwargs)
@ -757,7 +757,7 @@ def get_thread(
timeout = 600.0
api_base: str | None = None
api_key: str | None = None
response: Thread | None = None
response: Thread | Coroutine[None, None, Thread] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -903,7 +903,7 @@ def add_message(
metadata: dict | None = None,
client=None,
**kwargs,
) -> OpenAIMessage:
) -> OpenAIMessage | Coroutine[None, None, OpenAIMessage]:
### COMMON OBJECTS ###
a_add_message: Final = kwargs.pop("a_add_message", None)
_message_data: Final = MessageData(role=role, content=content, attachments=attachments, metadata=metadata)
@ -935,7 +935,7 @@ def add_message(
timeout = 600.0
api_key: str | None = None
api_base: str | None = None
response: OpenAIMessage | None = None
response: OpenAIMessage | Coroutine[None, None, OpenAIMessage] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -1065,7 +1065,7 @@ def get_messages(
thread_id: str,
client: Any | None = None,
**kwargs,
) -> SyncCursorPage[OpenAIMessage]:
) -> SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]:
aget_messages: Final = kwargs.pop("aget_messages", None)
optional_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params_dict: Final = get_litellm_params(**kwargs)
@ -1086,7 +1086,7 @@ def get_messages(
elif timeout is None:
timeout = 600.0
response: SyncCursorPage[OpenAIMessage] | None = None
response: SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]] | None = None
api_key: str | None = None
api_base: str | None = None
if custom_llm_provider == "openai":
@ -1256,7 +1256,7 @@ def run_thread(
client: object | None = None,
event_handler: AssistantEventHandler | None = None, # for stream=True calls
**kwargs,
) -> Run:
) -> Run | Coroutine[None, None, Run]:
"""Run a given thread + assistant."""
arun_thread: Final = kwargs.pop("arun_thread", None)
optional_params: Final = GenericLiteLLMParams(**kwargs)
@ -1278,7 +1278,7 @@ def run_thread(
elif timeout is None:
timeout = 600.0
response: Run | None = None
response: Run | Coroutine[None, None, Run] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there

6
litellm/caching/_embedding_router.py
View file

@ -13,7 +13,7 @@ This module is dependency-injected: callers pass the proxy ``llm_router`` and
from __future__ import annotations
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final
import litellm
from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
@ -25,12 +25,12 @@ if TYPE_CHECKING:
def resolve_embedding_router(
embedding_model: str,
llm_router: Router | None,
llm_model_list: list[dict[str, Any]] | None,
llm_model_list: Sequence[Mapping[str, object]] | None,
) -> Router | None:
"""Return ``llm_router`` iff it serves ``embedding_model`` as a deployment."""
if llm_router is None:
return None
router_model_names: Final[list[str]] = (
router_model_names: Final[Sequence[object]] = (
[m["model_name"] for m in llm_model_list if "model_name" in m] if llm_model_list is not None else []
)
if embedding_model in router_model_names:

29
litellm/caching/affinity_cache.py
View file

@ -24,6 +24,16 @@ if current == false then
end
if ARGV[3] then
local decoded, stored = pcall(cjson.decode, current)
if ARGV[4] then
local legacy = current
if decoded and type(stored) == 'string' then legacy = stored end
for _, eligible in ipairs(cjson.decode(ARGV[3])) do
if eligible['model_id'] == legacy then
redis.call('EXPIRE', KEYS[1], ARGV[2])
return current
end
end
end
if decoded and type(stored) == 'table' then
for _, eligible in ipairs(cjson.decode(ARGV[3])) do
local matches = true
@ -68,6 +78,7 @@ def claim_affinity_pin_in_memory(
ttl_seconds: int,
*,
eligible_values: tuple[Mapping[str, str], ...] | None = None,
preserve_legacy_model_ids: bool = False,
) -> object:
"""No await between read and write, so same-loop claims agree during a Redis outage."""
existing: Final[object] = cache.in_memory_cache.get_cache(cache_key)
@ -75,7 +86,12 @@ def claim_affinity_pin_in_memory(
if _legacy_pin_matches(existing, pin_value):
set_local_affinity_pin(cache, cache_key, pin_value, ttl_seconds)
return existing
winner: Final = existing if existing is not None and existing in (eligible_values or ()) else pin_value
eligible_existing: Final = existing in (eligible_values or ()) or (
preserve_legacy_model_ids
and isinstance(existing, str)
and any("model_id" in value and _legacy_pin_matches(existing, value) for value in (eligible_values or ()))
)
winner: Final = existing if existing is not None and eligible_existing else pin_value
set_local_affinity_pin(cache, cache_key, winner, ttl_seconds)
return winner
@ -94,6 +110,7 @@ async def claim_affinity_pin(
ttl_seconds: int,
*,
eligible_values: tuple[Mapping[str, str], ...] | None = None,
preserve_legacy_model_ids: bool = False,
) -> object:
"""Return the authoritative first writer, replacing it only when it becomes ineligible.
@ -112,6 +129,7 @@ async def claim_affinity_pin(
if eligible_values is not None
else ()
),
*(("1",) if preserve_legacy_model_ids and eligible_values is not None else ()),
)
raw: Final = cast( # cast-ok: Redis scripts return heterogeneous values; only object is asserted here
object, await claim_script(keys=(cache_key,), args=args)
@ -124,4 +142,11 @@ async def claim_affinity_pin(
return winner
except Exception as error: # noqa: BLE001 # Redis/Lua faults retain same-pod affinity through local claims
verbose_router_logger.debug("Affinity cache: Redis claim failed, using pod-local claim. error=%s", error)
return claim_affinity_pin_in_memory(cache, cache_key, pin_value, ttl_seconds, eligible_values=eligible_values)
return claim_affinity_pin_in_memory(
cache,
cache_key,
pin_value,
ttl_seconds,
eligible_values=eligible_values,
preserve_legacy_model_ids=preserve_legacy_model_ids,
)

8
litellm/caching/azure_blob_cache.py
View file

@ -13,10 +13,14 @@ import json
from contextlib import suppress
from typing import Final
from pydantic import TypeAdapter
from litellm._logging import print_verbose, verbose_logger
from .base_cache import BaseCache
_DECODED_BLOB: Final = TypeAdapter(object)
class AzureBlobCache(BaseCache):
def __init__(self, account_url, container) -> None:
@ -65,7 +69,7 @@ class AzureBlobCache(BaseCache):
print_verbose(f"Get Azure Blob Cache: key: {key}")
as_bytes: Final = self.container_client.download_blob(key).readall()
as_str: Final = as_bytes.decode("utf-8")
cached_response: Final[object] = json.loads(as_str)
cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str))
verbose_logger.debug(
"Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s",
@ -86,7 +90,7 @@ class AzureBlobCache(BaseCache):
blob: Final = await self.async_container_client.download_blob(key)
as_bytes: Final = await blob.readall()
as_str: Final = as_bytes.decode("utf-8")
cached_response: Final[object] = json.loads(as_str)
cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str))
verbose_logger.debug(
"Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s",
key,

9
litellm/caching/base_cache.py
View file

@ -9,14 +9,7 @@ Has 4 methods:
"""
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, Final
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span | Any
else:
Span = Any
from typing import Final
class BaseCache(ABC):

8
litellm/caching/disk_cache.py
View file

@ -1,15 +1,19 @@
import json
from typing import TYPE_CHECKING, Any, Final
from pydantic import TypeAdapter
from .base_cache import BaseCache
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span | Any
Span = _Span
else:
Span = Any
_DECODED_DISK_VALUE: Final = TypeAdapter(object)
class DiskCache(BaseCache):
def __init__(self, disk_cache_dir: str | None = None):
@ -44,7 +48,7 @@ class DiskCache(BaseCache):
original_cached_response: Final = self.disk_cache.get(key)
if original_cached_response:
try:
cached_response = json.loads(original_cached_response)
cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response))
except Exception:
cached_response = original_cached_response
return cached_response

6
litellm/caching/redis_cache.py
View file

@ -1738,7 +1738,9 @@ class RedisCache(BaseCache):
return key_value_dict
@_redis_circuit_breaker_guard
async def async_get_cache(self, key, parent_otel_span: Span | None = None, **kwargs):
async def async_get_cache(
self, key, parent_otel_span: Span | None = None, *, raise_on_error: bool = False, **kwargs
):
from redis.asyncio import Redis
_redis_client: Final[Redis] = self.init_async_client()
@ -1781,6 +1783,8 @@ class RedisCache(BaseCache):
)
)
print_verbose(f"litellm.caching.caching: async get() - Got exception from REDIS: {e}")
if raise_on_error:
raise
_record_swallowed_redis_failure(self._circuit_breaker, e)
@_redis_circuit_breaker_guard

4
litellm/constants.py
View file

@ -480,6 +480,7 @@ REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_end_user_sp
REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_agent_spend_update_buffer"
REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_tag_spend_update_buffer"
REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_window_spend_update_buffer"
REDIS_TOOL_DISCOVERY_BUFFER_KEY: Final = "litellm_tool_discovery_buffer"
MAX_REDIS_BUFFER_DEQUEUE_COUNT: Final = get_env_int("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100)
REDIS_SPEND_LOGS_BUFFER_KEY: Final = "litellm_spend_logs_buffer"
REDIS_SPEND_LOGS_BUFFER_MAX_ROWS: Final = 100000
@ -861,6 +862,7 @@ LITELLM_CHAT_PROVIDERS: Final = [
"heroku",
"oci",
"morph",
"opencode_go",
"lambda_ai",
"inception",
"vercel_ai_gateway",
@ -1116,6 +1118,7 @@ openai_compatible_providers: Final[list[str]] = [
"v0",
"helicone",
"morph",
"opencode_go",
"lambda_ai",
"inception",
"hyperbolic",
@ -1842,6 +1845,7 @@ LITELLM_UI_SESSION_DURATION: Final = os.getenv("LITELLM_UI_SESSION_DURATION", "2
########################### DB CRON JOB NAMES ###########################
DB_SPEND_UPDATE_JOB_NAME: Final = "db_spend_update_job"
TOOL_DISCOVERY_COMMIT_JOB_NAME: Final = "tool_discovery_commit_job"
DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME: Final = "db_daily_tag_spend_update_job"
PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME: Final = "prometheus_emit_budget_metrics"
CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME: Final = "cloudzero_export_usage_data"

79
litellm/cost_calculator.py
View file

@ -37,6 +37,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
_SERVICE_TIER_TO_COST_KEY_SUFFIX,
BilledTokenRates,
CostCalculatorUtils,
calculate_cache_writing_cost,
calculate_cost_component,
generic_cost_per_character,
generic_cost_per_token,
@ -1200,12 +1201,16 @@ def get_usage_object(
elif isinstance(usage_obj, dict):
return Usage(**usage_obj)
elif isinstance(usage_obj, BaseModel):
return Usage(**usage_obj.model_dump())
return _usage_from_model(usage_obj)
else:
verbose_logger.debug("Unknown usage object type: %s, usage_obj: %s", type(usage_obj), usage_obj)
return None
def _usage_from_model(usage_model: BaseModel) -> Usage:
return Usage(**usage_model.model_dump())
def _decisions_prompt_tokens_details(usage: DecisionsUsage | OpenAIDecisionUsage) -> PromptTokensDetailsWrapper:
match usage:
case DecisionsUsage():
@ -1546,6 +1551,8 @@ def completion_cost(
"""
try:
call_type = infer_call_type(call_type, completion_response) or "completion"
if isinstance(completion_response, DecisionsResponse) and completion_response.usage is None:
raise ValueError("Cannot calculate Decisions cost: provider did not report usage")
if pricing_provider is not None:
custom_llm_provider = pricing_provider # rebind-ok: model_info.pricing_provider overrides the routing provider for cost-map lookups
@ -1686,7 +1693,7 @@ def completion_cost(
setattr(
completion_response,
"usage",
litellm.Usage(**_usage_for_dump.model_dump()),
_usage_from_model(_usage_for_dump),
)
if isinstance(usage_obj, (DecisionsUsage, OpenAIDecisionUsage)):
_usage = _decisions_usage(usage_obj).model_dump()
@ -2264,6 +2271,8 @@ def response_cost_calculator(
### VERTEX LOCATION ###
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
region_name: str | None = None,
custom_cost_per_token: CostPerToken | None = None,
custom_cost_per_second: float | None = None,
) -> float:
"""
Returns
@ -2292,6 +2301,8 @@ def response_cost_calculator(
optional_params=optional_params,
custom_pricing=custom_pricing,
base_model=base_model,
custom_cost_per_token=custom_cost_per_token,
custom_cost_per_second=custom_cost_per_second,
prompt=prompt,
standard_built_in_tools_params=standard_built_in_tools_params,
litellm_model_name=litellm_model_name,
@ -2875,6 +2886,7 @@ def _batch_rate(
model_info: ModelInfo,
key: Literal[
"input_cost_per_audio_token_batches",
"cache_read_input_audio_token_cost_batches",
"input_cost_per_image_token_batches",
"input_cost_per_video_token_batches",
],
@ -2960,7 +2972,12 @@ def batch_cost_calculator(
total_completion_cost = 0.0
if batch_rates.input is not None:
total_prompt_cost = _batch_prompt_cost(
usage, model_info, batch_rates.input, batch_rates.cache_read, batch_rates.cache_creation
usage,
model_info,
batch_rates.input,
batch_rates.cache_read,
batch_rates.cache_creation,
batch_rates.cache_creation_rates_by_ttl,
)
elif input_cost_per_token:
details: Final = parse_prompt_tokens_details(usage)
@ -3005,10 +3022,13 @@ def _batch_prompt_cost(
input_rate: float,
cache_read_rate: float | None,
cache_creation_rate: float | None,
cache_creation_rates_by_ttl: Mapping[str, float],
) -> float:
details: Final = parse_prompt_tokens_details(usage)
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl)
written_tokens: Final = details["cache_creation_tokens"] if bills_cache_writes else 0
audio_tokens, image_tokens, video_tokens = (
details["audio_tokens"],
details["image_tokens"],
@ -3022,13 +3042,27 @@ def _batch_prompt_cost(
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
+ cached_tokens * (cache_read_rate or 0.0)
+ written_tokens * (cache_creation_rate or 0.0)
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
+ cached_audio_tokens
* _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0)
+ (
calculate_cache_writing_cost(
written_tokens,
details["cache_creation_token_details"],
input_rate if cache_creation_rate is None else cache_creation_rate,
cache_creation_rates_by_ttl,
)
if bills_cache_writes
else 0.0
)
)
_ATTRIBUTE_VALUE: Final = TypeAdapter(object)
def _attribute_value(obj: object, name: str) -> object:
return getattr(obj, name)
return _ATTRIBUTE_VALUE.validate_python(getattr(obj, name))
def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]:
@ -3663,7 +3697,26 @@ def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo |
return None
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...]) -> float | None:
_TranscriptionRateKey: TypeAlias = Literal[
"input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
]
def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None:
match key:
case "input_cost_per_second":
return info.get("input_cost_per_second")
case "input_cost_per_audio_token":
return info.get("input_cost_per_audio_token")
case "input_cost_per_token":
return info.get("input_cost_per_token")
case "output_cost_per_token":
return info.get("output_cost_per_token")
case _:
assert_never(key)
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
"""First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry
because ``get_model_info`` synthesizes zero token rates for entries that omit them."""
if info is None:
@ -3672,12 +3725,18 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...])
if declared is None:
return None
return next(
(float(value) for key in keys if declared.get(key) is not None and (value := info.get(key)) is not None),
(
float(value)
for key in keys
if declared.get(key) is not None and (value := _model_info_rate(info, key)) is not None
),
None,
)
def _transcription_rate(keys: tuple[str, ...], override: ModelInfo | None, base: ModelInfo | None) -> float:
def _transcription_rate(
keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None
) -> float:
rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base))
return next((rate for rate in rates if rate is not None), 0.0)

2
litellm/exceptions.py
View file

@ -1273,7 +1273,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
self.message = _saved_message
self.args = (_saved_message,)
original_headers: Final = getattr(original_exception, "litellm_response_headers", None)
original_headers: Final[object] = getattr(original_exception, "litellm_response_headers", None)
if original_headers is not None:
self.litellm_response_headers = original_headers

20
litellm/experimental_mcp_client/tools.py
View file

@ -10,6 +10,7 @@ from mcp.types import Tool as MCPTool
from openai.types.chat import ChatCompletionToolParam
from openai.types.responses.function_tool_param import FunctionToolParam
from openai.types.shared_params.function_definition import FunctionDefinition
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import (
@ -20,6 +21,8 @@ from litellm.constants import (
from litellm.types.llms.anthropic import AnthropicMessagesTool
from litellm.types.utils import ChatCompletionMessageToolCall
_DECODED_JSON: Final = TypeAdapter(object)
########################################################
# List MCP Tool functions
@ -200,15 +203,18 @@ async def call_mcp_tool(
return tool_result
def _decode_json_arguments(arguments: str) -> object:
try:
return _DECODED_JSON.validate_python(json.loads(arguments))
except json.JSONDecodeError:
return {}
def _get_function_arguments(function: FunctionDefinition) -> dict:
"""Helper to safely get and parse function arguments."""
arguments = function.get("arguments", {})
if isinstance(arguments, str):
try:
arguments = json.loads(arguments)
except json.JSONDecodeError:
arguments = {}
return arguments if isinstance(arguments, dict) else {}
arguments: Final = function.get("arguments", {})
decoded: Final = _decode_json_arguments(arguments) if isinstance(arguments, str) else arguments
return decoded if isinstance(decoded, dict) else {}
def transform_openai_tool_call_request_to_mcp_tool_call_request(

14
litellm/files/main.py
View file

@ -9,7 +9,7 @@ import asyncio
import contextvars
import time
import uuid as uuid_module
from collections.abc import Coroutine
from collections.abc import Awaitable, Coroutine
from functools import partial
from typing import Final, Literal, cast
@ -293,6 +293,14 @@ def create_file(
raise e
async def _await_retrieved_file(pending: Awaitable[FileObject | OpenAIFileObject]) -> FileObject | OpenAIFileObject:
return await pending
async def _await_deleted_file(pending: Awaitable[object]) -> object:
return await pending
@client
async def afile_retrieve(
file_id: str,
@ -325,7 +333,7 @@ async def afile_retrieve(
func_with_context: Final = partial(ctx.run, func)
init_response: Final = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
response = await _await_retrieved_file(init_response)
else:
response = init_response
@ -505,7 +513,7 @@ async def afile_delete(
func_with_context: Final = partial(ctx.run, func)
init_response: Final = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
response = await _await_deleted_file(init_response)
else:
response = init_response

4
litellm/files/streaming.py
View file

@ -1,7 +1,7 @@
import datetime
import traceback
from collections.abc import AsyncIterator, Iterator
from typing import TYPE_CHECKING, Any, Final, Optional, cast
from typing import TYPE_CHECKING, Final, Optional, cast
import anyio
@ -163,7 +163,7 @@ class FileContentStreamingResponse:
merged_hidden_params: Final = cast(
"StandardLoggingHiddenParams",
{
**cast(dict[str, Any], payload.get("hidden_params") or {}),
**(payload.get("hidden_params") or {}),
**self._hidden_params,
},
)

3
litellm/harness/handlers/base.py
View file

@ -4,7 +4,6 @@ from __future__ import annotations
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
from typing import Any
from litellm.harness.context import SessionContext
from litellm.harness.errors import CapabilityUnsupported
@ -38,5 +37,5 @@ class BaseHarnessHandler(ABC):
async def resume(self, ctx: SessionContext, native_session_id: str) -> None:
"""Continue the runtime's own session on the next turn."""
async def history(self, ctx: SessionContext) -> list[dict[str, Any]]: # mutable-ok: public history() API shape
async def history(self, ctx: SessionContext) -> list[dict[str, object]]: # mutable-ok: public history() API shape
raise CapabilityUnsupported(f"Harness.{self.config.harness.name} does not expose history")

4
litellm/harness/handlers/cli_handler.py
View file

@ -70,10 +70,10 @@ async def send_stdin(proc: Process, data: str) -> None:
async def private_dir_for(sandbox: Sandbox) -> str:
tempdir = getattr(sandbox, "tempdir", None)
tempdir: Final = sandbox.tempdir if hasattr(sandbox, "tempdir") else None
if tempdir is None:
raise SandboxError(f"{type(sandbox).__name__} has no tempdir(); CLI harnesses need a private config dir")
path: str = await tempdir()
path: Final = await tempdir()
return path

2
litellm/harness/handlers/deepagents_handler.py
View file

@ -161,7 +161,7 @@ class DeepAgentsHandler(BaseHarnessHandler):
async def history(
self, ctx: SessionContext
) -> list[dict[str, Any]]: # mutable-ok: BaseHarnessHandler.history API returns OpenAI message dicts
) -> list[dict[str, object]]: # mutable-ok: BaseHarnessHandler.history API returns OpenAI message dicts
agent, deps = self._require_agent()
snapshot = await agent.aget_state(self._run_config(ctx, None))
messages = (snapshot.values or MappingProxyType({})).get("messages") or ()

11
litellm/harness/runtime.py
View file

@ -12,11 +12,10 @@ import inspect
import logging
import os
import uuid
from collections.abc import AsyncIterator, Callable, Coroutine, Generator, Mapping, Sequence
from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Generator, Mapping, Sequence
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import (
Any,
Final,
get_args,
)
@ -331,11 +330,15 @@ async def _put_end(queue: asyncio.Queue[Event | _End], end: _End) -> None:
queue.put_nowait(end)
async def _await_decision(pending: Awaitable[object]) -> object:
return await pending
async def call_approval_handler(handler: ApprovalHandler, approval: Approval) -> None:
"""Run on_approval (sync in a worker thread, or async) and resolve approval."""
try:
if inspect.iscoroutinefunction(handler):
decision: object = await handler(approval)
decision: object = await _await_decision(handler(approval))
else:
decision = await asyncio.to_thread(handler, approval)
if inspect.isawaitable(decision):
@ -749,7 +752,7 @@ class AsyncSession:
async def history(
self,
) -> list[dict[str, Any]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler
) -> list[dict[str, object]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler
if not self.harness_config.capabilities.history:
raise CapabilityUnsupported(f"Harness.{self.config.harness.name} does not expose history")
await self._ensure_ready()

11
litellm/images/main.py
View file

@ -3,7 +3,7 @@ import contextvars
import importlib
from collections.abc import Coroutine
from functools import partial
from typing import TYPE_CHECKING, Final, Literal, Optional, cast, overload
from typing import TYPE_CHECKING, Final, Literal, cast, overload
from pydantic import TypeAdapter, ValidationError
@ -70,16 +70,17 @@ from litellm.utils import (
_PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
# Cache for ImageEditRequestUtils to avoid repeated __getattr__ calls
_ImageEditRequestUtils_cache: Optional["ImageEditRequestUtils"] = None
_ImageEditRequestUtils_cache: type["ImageEditRequestUtils"] | None = None
def _get_ImageEditRequestUtils() -> "ImageEditRequestUtils":
def _get_ImageEditRequestUtils() -> type["ImageEditRequestUtils"]:
"""Get ImageEditRequestUtils, loading it lazily if needed."""
global _ImageEditRequestUtils_cache
if _ImageEditRequestUtils_cache is None:
# Access via module to trigger __getattr__ if not cached
module: Final = importlib.import_module(__name__)
_ImageEditRequestUtils_cache = module.ImageEditRequestUtils
from litellm.images import main as images_main
_ImageEditRequestUtils_cache = images_main.ImageEditRequestUtils
assert _ImageEditRequestUtils_cache is not None # Type narrowing for type checker
return _ImageEditRequestUtils_cache

14
litellm/integrations/SlackAlerting/hanging_request_check.py
View file

@ -115,14 +115,16 @@ class AlertingHangingRequestCheck:
if hanging_request_data.alerted:
continue
request_status = await proxy_logging_obj.internal_usage_cache.async_get_cache(
key=f"request_status:{hanging_request_data.request_id}",
litellm_parent_otel_span=None,
local_only=True,
)
# this means the request status was either success or fail
# and is not hanging
if request_status is not None:
if (
await proxy_logging_obj.internal_usage_cache.async_get_cache(
key=f"request_status:{hanging_request_data.request_id}",
litellm_parent_otel_span=None,
local_only=True,
)
is not None
):
# clear this request from hanging request cache since the request was either success or failed
self.hanging_request_cache._remove_key(
key=request_id,

5
litellm/integrations/SlackAlerting/user_spend_alerts.py
View file

@ -56,8 +56,9 @@ async def fetch_user_spend_rows(
today_str: Final = today.strftime("%Y-%m-%d")
month_start_str: Final = today.replace(day=1).strftime("%Y-%m-%d")
baseline_start_str: Final = (today - datetime.timedelta(days=max(baseline_days, 1))).strftime("%Y-%m-%d")
raw: Final = await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
return USER_SPEND_ROWS_ADAPTER.validate_python(raw)
return USER_SPEND_ROWS_ADAPTER.validate_python(
await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
)
def _daily_threshold_event(row: UserSpendRow, args: SlackAlertingArgs, today_str: str) -> UserSpendAlertEvent | None:

23
litellm/integrations/agentops/agentops.py
View file

@ -4,11 +4,16 @@ AgentOps integration for LiteLLM - Provides OpenTelemetry tracing for LLM calls
import os
from dataclasses import dataclass
from typing import Any, Final
from typing import Final
from pydantic import TypeAdapter
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.custom_httpx.http_handler import get_httpx_client
_DECODED_JSON: Final = TypeAdapter(object)
@dataclass
class AgentOpsConfig:
@ -58,9 +63,10 @@ class AgentOps(OpenTelemetry):
project_id = None
if config.api_key:
try:
response: Final = self._fetch_auth_token(config.api_key, config.auth_endpoint)
jwt_token = response.get("token")
project_id = response.get("project_id")
response: Final = as_str_mapping(self._fetch_auth_token(config.api_key, config.auth_endpoint))
if response is not None:
jwt_token = response.get("token")
project_id = response.get("project_id")
except Exception:
pass
@ -78,12 +84,9 @@ class AgentOps(OpenTelemetry):
"telemetry.sdk.name": "agentops",
}
if project_id:
resource_attrs["project.id"] = project_id
self.resource_attributes = {**resource_attrs, "project.id": project_id} if project_id else resource_attrs
self.resource_attributes = resource_attrs
def _fetch_auth_token(self, api_key: str, auth_endpoint: str) -> dict[str, Any]:
def _fetch_auth_token(self, api_key: str, auth_endpoint: str) -> object:
"""
Fetch JWT authentication token from AgentOps API
@ -111,6 +114,6 @@ class AgentOps(OpenTelemetry):
if response.status_code != 200:
raise Exception(f"Failed to fetch auth token: {response.text}")
return response.json()
return _DECODED_JSON.validate_python(response.json())
finally:
client.close()

2
litellm/integrations/arize/arize.py
View file

@ -166,7 +166,7 @@ class ArizeLogger(OpenTelemetry):
primary ``otel`` callback which handles proxy-level parent spans.
"""
def set_attributes(self, span: Span, kwargs, response_obj: Any | None):
def set_attributes(self, span: Span, kwargs, response_obj: object):
ArizeLogger.set_arize_attributes(span, kwargs, response_obj)
@staticmethod

29
litellm/integrations/braintrust_logging.py
View file

@ -3,9 +3,10 @@
import os
from datetime import datetime
from typing import Final
from typing import Final, Protocol
import httpx
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm import verbose_logger
@ -24,6 +25,18 @@ from litellm.utils import print_verbose
API_BASE: Final = "https://api.braintrustdata.com/v1"
class _BraintrustProjectBody(TypedDict):
id: ReadOnly[str]
class _BraintrustProjectResponse(Protocol):
def json(self) -> _BraintrustProjectBody: ...
def _project_id(response: _BraintrustProjectResponse) -> str:
return response.json()["id"]
def get_utc_datetime():
import datetime as dt
@ -80,8 +93,7 @@ class BraintrustLogger(CustomLogger):
headers=self.headers,
json={"name": project_name},
)
project_dict: Final = response.json()
project_id: Final = project_dict["id"]
project_id: Final = _project_id(response)
self._project_id_cache[project_name] = project_id
return project_id
except httpx.HTTPStatusError as e:
@ -100,8 +112,7 @@ class BraintrustLogger(CustomLogger):
headers=self.headers,
json={"name": project_name},
)
project_dict: Final = response.json()
project_id: Final = project_dict["id"]
project_id: Final = _project_id(response)
self._project_id_cache[project_name] = project_id
return project_id
except httpx.HTTPStatusError as e:
@ -112,18 +123,14 @@ class BraintrustLogger(CustomLogger):
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
)
project_dict: Final = project.json()
self.default_project_id = project_dict["id"]
self.default_project_id = _project_id(project)
def create_sync_default_project_and_experiment(self):
project: Final = self.global_braintrust_sync_http_handler.post(
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
)
project_dict: Final = project.json()
self.default_project_id = project_dict["id"]
self.default_project_id = _project_id(project)
def log_success_event(self, kwargs, response_obj, start_time, end_time):
verbose_logger.debug("REACHES BRAINTRUST SUCCESS")

7
litellm/integrations/cloudzero/cz_stream_api.py
View file

@ -19,8 +19,9 @@
"""Output modules for writing CBF data to various destinations."""
import zoneinfo
from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Any, Final
from typing import Final
import httpx
import polars as pl
@ -188,7 +189,7 @@ class CloudZeroStreamer:
)
raise
def _prepare_batch_payload(self, batch_date: str, batch_data: pl.DataFrame, operation: str) -> dict[str, Any]:
def _prepare_batch_payload(self, batch_date: str, batch_data: pl.DataFrame, operation: str) -> Mapping[str, object]:
"""Prepare batch payload according to CloudZero AnyCost API format."""
# Convert batch_date to month for the API (YYYY-MM format)
try:
@ -205,7 +206,7 @@ class CloudZeroStreamer:
if record:
data_records.append(record)
payload: Final = {"month": month_str, "operation": operation, "data": data_records}
payload: Final[Mapping[str, object]] = {"month": month_str, "operation": operation, "data": data_records}
return payload

7
litellm/integrations/custom_guardrail.py
View file

@ -352,7 +352,7 @@ class CustomGuardrail(CustomLogger):
self,
violation_message: str,
request_data: dict[str, Any],
detection_info: dict[str, Any] | None = None,
detection_info: Mapping[str, object] | None = None,
original_response: object = None,
) -> None:
"""
@ -1368,7 +1368,10 @@ class CustomGuardrail(CustomLogger):
if event_type is not None:
guardrail_mode = event_type
elif isinstance(self.event_hook, Mode):
guardrail_mode = GuardrailMode(**dict(self.event_hook.model_dump()))
guardrail_mode = GuardrailMode(
tags=copy.deepcopy(self.event_hook.tags),
default=copy.deepcopy(self.event_hook.default),
)
else:
guardrail_mode = self.event_hook

3
litellm/integrations/custom_secret_manager.py
View file

@ -37,7 +37,6 @@ Usage:
"""
from abc import abstractmethod
from typing import Any
import httpx
@ -154,7 +153,7 @@ class CustomSecretManager(BaseSecretManager):
optional_params: dict | None = None,
timeout: float | httpx.Timeout | None = None,
tags: dict | list | None = None,
) -> dict[str, Any]:
) -> dict[str, object]:
"""
Asynchronously write a secret to your custom secret manager.

4
litellm/integrations/dotprompt/__init__.py
View file

@ -61,7 +61,9 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom
prompt_file: Final = getattr(litellm_params, "prompt_file", None)
# Handle dotprompt_content from database
dotprompt_content: Final = getattr(litellm_params, "dotprompt_content", None)
dotprompt_content: Final = (
litellm_params.dotprompt_content if hasattr(litellm_params, "dotprompt_content") else None
)
if dotprompt_content and not prompt_data and not prompt_file:
prompt_data = get_prompt_data_from_dotprompt_content(dotprompt_content)

15
litellm/integrations/dotprompt/dotprompt_manager.py
View file

@ -7,6 +7,8 @@ import json
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing_extensions import ReadOnly, TypedDict
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import PromptManagementClient
from litellm.types.llms.openai import AllMessageValues
@ -18,7 +20,12 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
from .prompt_manager import PromptManager, PromptTemplate
from .prompt_manager import PromptJsonEntry, PromptManager, PromptTemplate
class _PromptJsonData(TypedDict, total=False):
content: ReadOnly[str]
metadata: ReadOnly[dict[str, object]]
class DotpromptManager(CustomPromptManagement):
@ -342,17 +349,17 @@ class DotpromptManager(CustomPromptManagement):
if self._prompt_manager:
self._prompt_manager.reload_prompts()
def add_prompt_from_json(self, prompt_id: str, json_data: dict[str, Any]) -> None:
def add_prompt_from_json(self, prompt_id: str, json_data: _PromptJsonData) -> None:
"""Add a prompt from JSON data."""
content: Final = json_data.get("content", "")
metadata: Final = json_data.get("metadata", {})
self.prompt_manager.add_prompt(prompt_id, content, metadata)
def load_prompts_from_json(self, prompts_data: dict[str, dict[str, object]]) -> None:
def load_prompts_from_json(self, prompts_data: Mapping[str, object]) -> None:
"""Load multiple prompts from JSON data."""
self.prompt_manager.load_prompts_from_json_data(prompts_data)
def get_prompts_as_json(self) -> dict[str, dict[str, object]]:
def get_prompts_as_json(self) -> dict[str, PromptJsonEntry]:
"""Get all prompts in JSON format."""
return self.prompt_manager.get_all_prompts_as_json()

29
litellm/integrations/dotprompt/prompt_manager.py
View file

@ -12,6 +12,8 @@ from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.litellm_core_utils.core_helpers import as_str_mapping
class _PromptFileJson(TypedDict):
"""JSON form of a .prompt file: rendered template text plus its frontmatter."""
@ -20,6 +22,11 @@ class _PromptFileJson(TypedDict):
metadata: ReadOnly[NotRequired[dict[str, object]]]
class PromptJsonEntry(TypedDict):
content: ReadOnly[str]
metadata: ReadOnly[dict[str, object]]
def strip_version_suffix(prompt_id: str) -> str | None:
base, separator, version = prompt_id.rpartition(".v")
if separator and base and version.isdigit():
@ -358,20 +365,26 @@ class PromptManager:
return f"---\n{frontmatter_yaml}---\n{content}"
def get_all_prompts_as_json(self) -> dict[str, dict[str, Any]]:
def get_all_prompts_as_json(self) -> dict[str, PromptJsonEntry]:
"""Get all loaded prompts in JSON format.
Returns:
Dictionary mapping prompt_id to prompt data
"""
result: Final = {}
for prompt_id, template in self.prompts.items():
result[prompt_id] = {
return {
prompt_id: {
"content": template.content,
"metadata": template.metadata,
}
return result
for prompt_id, template in self.prompts.items()
}
def load_prompts_from_json_data(self, prompt_data: dict[str, dict[str, object]]) -> None:
"""Load additional prompts from JSON data (merges with existing prompts)."""
self._load_prompts_from_json(prompt_data)
def load_prompts_from_json_data(self, prompt_data: Mapping[str, object]) -> None:
"""Load additional prompts from JSON data (merges with existing prompts). Entries that are not
mappings are skipped, the same as a malformed entry inside the loader."""
entries: Final = {
prompt_id: dict(entry)
for prompt_id, raw in prompt_data.items()
if (entry := as_str_mapping(raw)) is not None
}
self._load_prompts_from_json(entries)

5
litellm/integrations/focus/destinations/factory.py
View file

@ -3,6 +3,7 @@
from __future__ import annotations
import os
from collections.abc import Mapping
from typing import Any, Final
from .base import FocusDestination
@ -20,7 +21,7 @@ class FocusDestinationFactory:
*,
provider: str,
prefix: str,
config: dict[str, Any] | None = None,
config: Mapping[str, object] | None = None,
) -> FocusDestination:
"""Return a destination implementation for the requested provider."""
provider_lower: Final = provider.lower()
@ -39,7 +40,7 @@ class FocusDestinationFactory:
def _resolve_config(
*,
provider: str,
overrides: dict[str, object],
overrides: Mapping[str, object],
) -> dict[str, Any]:
if provider == "s3":
resolved = {

8
litellm/integrations/focus/export_engine.py
View file

@ -2,8 +2,8 @@
from __future__ import annotations
from collections.abc import Callable
from typing import Any, Final, SupportsFloat, SupportsIndex, SupportsInt, cast
from collections.abc import Callable, Mapping
from typing import Final, SupportsFloat, SupportsIndex, SupportsInt, cast
import polars as pl
@ -24,7 +24,7 @@ class FocusExportEngine:
provider: str,
export_format: str,
prefix: str,
destination_config: dict[str, Any] | None = None,
destination_config: Mapping[str, object] | None = None,
) -> None:
self.provider = provider
self.export_format = export_format
@ -77,7 +77,7 @@ class FocusExportEngine:
return FocusParquetSerializer()
raise NotImplementedError(f"Export format '{self.export_format}' not supported. Use 'parquet' or 'csv'.")
async def dry_run_export_usage_data(self, limit: int | None) -> dict[str, Any]:
async def dry_run_export_usage_data(self, limit: int | None) -> dict[str, object]:
data: Final = await self.database.get_usage_data(limit=limit)
normalized: Final = self.transformer.transform(data)

8
litellm/integrations/focus/focus_logger.py
View file

@ -6,6 +6,8 @@ import os
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Any, ClassVar, Final, cast
from typing_extensions import TypedDict, Unpack
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
@ -25,6 +27,10 @@ FOCUS_USAGE_DATA_JOB_NAME: Final = "focus_export_usage_data"
DEFAULT_DRY_RUN_LIMIT: Final = 500
class _CustomLoggerOptions(TypedDict, total=False, extra_items=object):
pass
class FocusLogger(CustomLogger):
"""Coordinates Focus export jobs across transformer/serializer/destination layers."""
@ -40,7 +46,7 @@ class FocusLogger(CustomLogger):
interval_seconds: int | None = None,
prefix: str | None = None,
destination_config: dict[str, Any] | None = None,
**kwargs: Any,
**kwargs: Unpack[_CustomLoggerOptions],
) -> None:
super().__init__(**kwargs)
self.provider = (provider or os.getenv("FOCUS_PROVIDER") or "s3").lower()

8
litellm/integrations/gcs_bucket/gcs_bucket.py
View file

@ -3,6 +3,7 @@ import hashlib
import json
import os
import time
from collections.abc import Mapping
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Any, Final
@ -15,6 +16,7 @@ from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
from litellm.litellm_core_utils.cloud_storage_security import (
sanitize_cloud_object_component,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.gcs_bucket import *
from litellm.types.utils import StandardBatchLineItemsPayload, StandardLoggingPayload
@ -134,7 +136,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
"""
return f"{date_str}/batch-{batch_id}.ndjson"
def _get_config_key(self, kwargs: dict[str, Any]) -> str:
def _get_config_key(self, kwargs: Mapping[str, object]) -> str:
"""
Extract a synchronous grouping key from kwargs to group items by GCS config.
This allows us to batch items with the same bucket/credentials together.
@ -143,7 +145,9 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
This key may contain sensitive information (bucket names, paths) - use _sanitize_config_key()
for logging purposes.
"""
standard_callback_dynamic_params: Final = kwargs.get("standard_callback_dynamic_params", None) or {}
standard_callback_dynamic_params: Final = (
as_str_mapping(kwargs.get("standard_callback_dynamic_params", None)) or {}
)
bucket_name = standard_callback_dynamic_params.get("gcs_bucket_name", None) or self.BUCKET_NAME or "default"
path_service_account: Final = (

3
litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py
View file

@ -9,6 +9,7 @@ Usage:
"""
import asyncio
import os
from collections.abc import AsyncIterable, Iterable
from typing import Final
@ -40,7 +41,7 @@ _mocks_initialized = False
# Default mock latency in seconds (simulates network round-trip)
# Typical GCS API calls take 100-300ms for uploads, 50-150ms for GET/DELETE
_MOCK_LATENCY_SECONDS: Final = float(__import__("os").getenv("GCS_MOCK_LATENCY_MS", "150")) / 1000.0
_MOCK_LATENCY_SECONDS: Final = float(os.getenv("GCS_MOCK_LATENCY_MS", "150")) / 1000.0
async def _mock_async_handler_get(self, url, params=None, headers=None, follow_redirects=None):

8
litellm/integrations/gcs_pubsub/pub_sub.py
View file

@ -12,6 +12,8 @@ import os
import traceback
from typing import TYPE_CHECKING, Any, Final
from pydantic import TypeAdapter
from litellm.types.utils import StandardLoggingPayload
if TYPE_CHECKING:
@ -27,6 +29,8 @@ from litellm.llms.custom_httpx.http_handler import (
httpxSpecialProvider,
)
_DECODED_JSON: Final = TypeAdapter(object)
class GcsPubSubLogger(CustomBatchLogger):
def __init__(
@ -152,7 +156,7 @@ class GcsPubSubLogger(CustomBatchLogger):
finally:
self.log_queue.clear()
async def publish_message(self, message: SpendLogsPayload | StandardLoggingPayload) -> dict[str, Any] | None:
async def publish_message(self, message: SpendLogsPayload | StandardLoggingPayload) -> object:
"""
Publish message to Google Cloud Pub/Sub using REST API
@ -188,7 +192,7 @@ class GcsPubSubLogger(CustomBatchLogger):
raise Exception(f"Failed to publish message: {response.text}")
verbose_logger.debug("Pub/Sub response: %s", response.text)
return response.json()
return _DECODED_JSON.validate_python(response.json())
except Exception as e:
verbose_logger.error("Pub/Sub publish error: %s", str(e))

5
litellm/integrations/langfuse/langfuse.py
View file

@ -20,6 +20,7 @@ from litellm.integrations.langfuse.langfuse_mock_client import (
should_use_langfuse_mock,
)
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
filter_exceptions_from_params,
reconstruct_model_name,
safe_deep_copy,
@ -1268,11 +1269,11 @@ def log_provider_specific_information_as_span(
*,
tracing: "LangfuseTracing",
parent: "LangfuseObservation",
enrichments: Mapping[str, Any],
enrichments: Mapping[str, object],
):
"""Logs provider-specific information as spans under the generation."""
_hidden_params: Final[Mapping[str, object] | None] = enrichments.get("hidden_params", None)
_hidden_params: Final = as_str_mapping(enrichments.get("hidden_params", None))
if _hidden_params is None:
return

5
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -12,6 +12,7 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import (
serialize_langfuse_datetime,
)
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.types.integrations.langfuse_otel import (
LangfuseSpanAttributes,
@ -222,7 +223,7 @@ class LangfuseOtelLogger(OpenTelemetry):
)
@staticmethod
def _set_langfuse_specific_attributes(span: Span, kwargs: Mapping[str, Any], response_obj):
def _set_langfuse_specific_attributes(span: Span, kwargs: Mapping[str, object], response_obj):
"""
Sets Langfuse specific metadata attributes onto the OTEL span.
@ -234,7 +235,7 @@ class LangfuseOtelLogger(OpenTelemetry):
from litellm.integrations.arize._utils import safe_set_attribute
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
dynamic_params: Final = kwargs.get("standard_callback_dynamic_params")
dynamic_params: Final = as_str_mapping(kwargs.get("standard_callback_dynamic_params"))
langfuse_environment: Final = (
dynamic_params.get("langfuse_environment") if dynamic_params else None
) or os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")

12
litellm/integrations/langfuse/langfuse_prompt_management.py
View file

@ -2,6 +2,7 @@
Call Hook for LiteLLM Proxy which allows Langfuse prompt management.
"""
from collections.abc import Mapping
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
@ -110,6 +111,10 @@ def _remember_trace_id(litellm_call_id: object, logged: LangfuseLoggedEvent) ->
in_memory_trace_id_cache.set_cache(litellm_call_id=litellm_call_id, service_name="langfuse", trace_id=trace_id)
def _params_without_model(config: Mapping[str, object]) -> Mapping[str, object]:
return {k: v for k, v in config.items() if k != "model"}
class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogger):
def __init__(
self,
@ -182,12 +187,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
return compiled_prompt
def _get_optional_params_from_langfuse(self, langfuse_prompt_client: PROMPT_CLIENT) -> dict:
config: Final = langfuse_prompt_client.config
optional_params: Final = {}
for k, v in config.items():
if k != "model":
optional_params[k] = v
return optional_params
return dict(_params_without_model(langfuse_prompt_client.config))
async def async_get_chat_completion_prompt(
self,

11
litellm/integrations/langsmith.py
View file

@ -7,10 +7,11 @@ import random
import traceback
import types
from datetime import datetime, timezone
from typing import Any, Final
from typing import Final
import httpx
from pydantic import BaseModel, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -31,6 +32,10 @@ from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPa
_RUN_RESPONSE: Final = TypeAdapter(object)
class _DynamicParamsKwargs(TypedDict, total=False):
standard_callback_dynamic_params: ReadOnly[StandardCallbackDynamicParams | None]
def is_serializable(value):
non_serializable_types: Final = (
types.CoroutineType,
@ -474,7 +479,7 @@ class LangsmithLogger(CustomBatchLogger):
return log_queue_by_credentials
def _get_sampling_rate_to_use_for_request(self, kwargs: dict[str, Any]) -> float:
def _get_sampling_rate_to_use_for_request(self, kwargs: _DynamicParamsKwargs) -> float:
standard_callback_dynamic_params: Final[StandardCallbackDynamicParams | None] = kwargs.get(
"standard_callback_dynamic_params", None
)
@ -485,7 +490,7 @@ class LangsmithLogger(CustomBatchLogger):
sampling_rate = float(_sampling_rate)
return sampling_rate
def _get_credentials_to_use_for_request(self, kwargs: dict[str, Any]) -> LangsmithCredentialsObject:
def _get_credentials_to_use_for_request(self, kwargs: _DynamicParamsKwargs) -> LangsmithCredentialsObject:
"""
Handles key/team based logging

2
litellm/integrations/langtrace.py
View file

@ -6,7 +6,7 @@ from litellm.types.integrations.otel_span_attributes import SpanAttributes
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span | Any
Span = _Span
else:
Span = Any

4
litellm/integrations/levo/levo.py
View file

@ -4,8 +4,6 @@ from typing import TYPE_CHECKING, Any, Final
from litellm.integrations.opentelemetry import OpenTelemetry
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.integrations.opentelemetry import (
OpenTelemetryConfig as _OpenTelemetryConfig,
)
@ -13,11 +11,9 @@ if TYPE_CHECKING:
Protocol = _Protocol
OpenTelemetryConfig = _OpenTelemetryConfig
Span = _Span | Any
else:
Protocol = Any
OpenTelemetryConfig = Any
Span = Any
class LevoConfig:

5
litellm/integrations/logfire_logger.py
View file

@ -3,8 +3,9 @@
import os
import traceback
from collections.abc import Mapping
from enum import Enum
from typing import Any, Final, NamedTuple
from typing import Final, NamedTuple
from typing_extensions import LiteralString
@ -15,7 +16,7 @@ from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
class SpanConfig(NamedTuple):
message_template: LiteralString
span_data: dict[str, Any]
span_data: Mapping[str, object]
class LogfireLevel(str, Enum):

19
litellm/integrations/newrelic/newrelic.py
View file

@ -49,11 +49,12 @@ import time
import uuid
from collections.abc import Mapping, Sequence
from datetime import datetime
from typing import Any, Final
from typing import Final, Protocol
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.newrelic import NewRelicInitParams
@ -65,6 +66,10 @@ except ImportError:
_newrelic_agent = None
class _SupportsTimestamp(Protocol):
def timestamp(self) -> float: ...
class NewRelicLogger(CustomLogger):
"""
New Relic logger for LiteLLM to send AI monitoring events.
@ -126,13 +131,13 @@ class NewRelicLogger(CustomLogger):
verbose_logger.error("Failed to initialize New Relic agent: %s. Integration will be disabled.", e)
self.enabled = False
def _get_newrelic_params(self) -> dict:
def _get_newrelic_params(self) -> Mapping[str, object]:
"""
Get the newrelic_params from litellm.newrelic_params
These are params specific to initializing the NewRelicLogger e.g. turn_off_message_logging
"""
dict_newrelic_params: dict = {}
dict_newrelic_params: Mapping[str, object] = {}
if litellm.newrelic_params is not None:
if isinstance(litellm.newrelic_params, NewRelicInitParams):
dict_newrelic_params = litellm.newrelic_params.model_dump()
@ -401,9 +406,9 @@ class NewRelicLogger(CustomLogger):
return choices[0].get("finish_reason") or "unknown"
return "unknown"
def _to_epoch_ms(self, t: Any) -> float:
def _to_epoch_ms(self, t: datetime | float) -> float:
"""Convert a datetime or float timestamp to epoch milliseconds."""
if hasattr(t, "timestamp"):
if has_attributes(t, _SupportsTimestamp, "timestamp"):
return t.timestamp() * 1000.0
return float(t) * 1000.0
@ -518,13 +523,13 @@ class NewRelicLogger(CustomLogger):
sequence = 0
# Extract timestamps, preferring StandardLoggingPayload
start_time = None
start_time: datetime | float | None = None
if standard_logging_object:
start_time = standard_logging_object.get("startTime")
if not start_time:
start_time = kwargs.get("start_time")
end_time = None
end_time: datetime | float | None = None
if standard_logging_object:
end_time = standard_logging_object.get("endTime")
if not end_time:

2
litellm/integrations/newrelic/newrelic_metrics.py
View file

@ -254,7 +254,7 @@ class NewRelicMetricsLogger(CustomBatchLogger):
if not self.log_queue:
return
if _pass < NEWRELIC_METRICS_MAX_DRAIN_PASSES - 1:
await asyncio.sleep(2**_pass)
await asyncio.sleep(1 << _pass)
async with self.flush_lock:
tried_ids: Final = frozenset(id(record) for record in attempted)
survivors: Final = tuple(record for record in self.log_queue if id(record) not in tried_ids)

4
litellm/integrations/opik/opik_payload_builder/payload_builders.py
View file

@ -57,9 +57,7 @@ def build_span_payload(
span_id: Final = utils.create_uuid7()
model: Final[str] = response_obj.get("model", "unknown-model")
obj_type: Final[str] = response_obj.get("object", "unknown-object")
created: Final[int] = response_obj.get("created", 0)
span_name: Final = f"{model}_{obj_type}_{created}"
span_name: Final = f"{model}_{response_obj.get('object', 'unknown-object')}_{response_obj.get('created', 0)}"
_logging.verbose_logger.debug("OpikLogger creating span with id %s for trace %s", span_id, trace_id)

3
litellm/integrations/otel/model/config.py
View file

@ -130,7 +130,8 @@ class _EnvWithoutBareExcludedServices(PydanticBaseSettingsSource):
return self._env_settings.get_field_value(field, field_name)
def __call__(self) -> dict[str, object]:
return {key: value for key, value in self._env_settings().items() if key != "excluded_services"}
settings: Final = self._env_settings()
return {key: settings[key] for key in settings if key != "excluded_services"}
class OpenTelemetryV2Config(BaseSettings):

6
litellm/integrations/otel/model/metadata.py
View file

@ -332,7 +332,7 @@ def caller_session_id(kwargs: Mapping[str, object], trace: TraceControls) -> str
return (None if echoes_trace_id else explicit) or trace.session_id or None
def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None:
def time_to_first_chunk_seconds(kwargs: Mapping[str, object]) -> float | None:
"""Seconds from the upstream request being issued (``api_call_start_time``)
to the first streamed chunk (``completion_start_time``); ``None`` for
non-streaming calls, where ``completion_start_time`` is backfilled with the
@ -340,8 +340,8 @@ def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None:
optional_params: Final = cast(Mapping[str, object], kwargs.get("optional_params") or {})
if not optional_params.get("stream"):
return None
api_call_start: Final = to_seconds(kwargs.get("api_call_start_time"))
completion_start: Final = to_seconds(kwargs.get("completion_start_time"))
api_call_start: Final = _epoch_seconds(kwargs.get("api_call_start_time"))
completion_start: Final = _epoch_seconds(kwargs.get("completion_start_time"))
if api_call_start is None or completion_start is None:
return None
return completion_start - api_call_start

15
litellm/integrations/otel/model/payloads.py
View file

@ -7,7 +7,7 @@ from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from enum import Enum
from types import MappingProxyType
from typing import TYPE_CHECKING, ClassVar, Final, Literal, cast
from typing import TYPE_CHECKING, ClassVar, Final, Literal, Protocol, cast
from urllib.parse import urlsplit
from typing_extensions import ReadOnly, TypedDict
@ -716,6 +716,15 @@ def _json_or_none(value: object) -> str | None:
return None
class _EnumValue(Protocol):
@property
def value(self) -> object: ...
def _enum_value(item: _EnumValue) -> object:
return item.value
def _guardrail_mode_str(value: object) -> str | None:
"""Normalize ``guardrail_mode`` to a stable string.
@ -732,12 +741,12 @@ def _guardrail_mode_str(value: object) -> str | None:
for item in value:
if item is None:
continue
part = as_str(item.value) if isinstance(item, Enum) else as_str(item)
part = as_str(_enum_value(item)) if isinstance(item, Enum) else as_str(item)
if part:
parts.append(part)
return ",".join(parts) or None
if isinstance(value, Enum):
return as_str(value.value)
return as_str(_enum_value(value))
return as_str(value)

7
litellm/integrations/otel/mount.py
View file

@ -107,8 +107,8 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
Safe no-op when the V2 gate is off or ``opentelemetry-instrumentation-fastapi``
is unavailable. This MUST be called at app-creation time — once the lifespan
runs, the middleware stack is frozen and ``instrument_app`` raises "Cannot add
middleware after an application has started".
runs, the middleware stack is frozen and middleware added later never runs, so
a late call is skipped without registering anything.
No ``TracerProvider`` is passed, so the instrumentation binds to the OTel global
``ProxyTracerProvider``; the proxy publishes the real provider as the global
@ -126,6 +126,9 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
try:
if not is_otel_v2_enabled():
return
if app.middleware_stack is not None:
verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: the app has already started")
return
# Lazy: only the V2-enabled path needs the optional
# ``opentelemetry-instrumentation-fastapi`` package, which is not part of the

8
litellm/integrations/otel/plumbing/metrics.py
View file

@ -173,6 +173,7 @@ class _MetricKwargs(TypedDict, total=False):
response_cost: ReadOnly[float | None]
completion_start_time: ReadOnly[datetime | float | str | None]
api_call_start_time: ReadOnly[datetime | float | str | None]
end_time: ReadOnly[datetime | float | str | None]
def resolve_error_type(kwargs: Mapping[str, Any]) -> str:
@ -414,15 +415,12 @@ class GenAIMetricRecorder:
if generation_time > 0:
self._metrics.time_per_output_token.record(generation_time / completion_tokens, attributes=common_attrs)
def _record_response_duration(self, kwargs: Mapping[str, Any], end_time: datetime, common_attrs: dict) -> None:
def _record_response_duration(self, kwargs: _MetricKwargs, end_time: datetime, common_attrs: dict) -> None:
api_call_start_time: Final = kwargs.get("api_call_start_time")
if api_call_start_time is None:
return
_end_time = kwargs.get("end_time") or end_time
if _end_time is None:
_end_time = datetime.now()
api_call_start: Final = to_seconds(api_call_start_time)
end_ts: Final = to_seconds(_end_time)
end_ts: Final = to_seconds(kwargs.get("end_time") or end_time)
if api_call_start is None or end_ts is None:
return
duration: Final = end_ts - api_call_start

52
litellm/integrations/otel/plumbing/otlp_json.py
View file

@ -6,6 +6,7 @@ retry loop and swaps the payload for OTLP/JSON (enums as integers, ids as hex).
import base64
import json
import logging
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final, TypeAlias
@ -13,9 +14,13 @@ from typing import Final, TypeAlias
import requests
from google.protobuf.json_format import MessageToDict
from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
from opentelemetry.exporter.otlp.proto.http._common import RequestPayloadTooLargeError
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace import ReadableSpan
from opentelemetry.sdk.trace.export import SpanExportResult
from opentelemetry.semconv.attributes.http_attributes import HTTP_RESPONSE_STATUS_CODE
_logger: Final = logging.getLogger(__name__)
JSON_CONTENT_TYPE: Final = "application/json"
_HEX_ID_KEYS: Final = frozenset({"traceId", "spanId", "parentSpanId"})
@ -66,12 +71,49 @@ class OTLPJsonSpanExporter(OTLPSpanExporter):
def __init__(
self,
endpoint: str | None,
headers: dict[str, str], # mutable-ok: SDK __init__ takes Dict
headers: Mapping[str, str],
certificate_file: str | None = None,
session: "requests.Session | None" = None,
*,
max_request_size: int | None = None,
) -> None:
super().__init__(endpoint=endpoint, headers=headers, certificate_file=certificate_file, session=session)
self._session.headers["Content-Type"] = JSON_CONTENT_TYPE
super().__init__(
endpoint=endpoint,
headers={**{key.lower(): value for key, value in headers.items()}, "content-type": JSON_CONTENT_TYPE},
certificate_file=certificate_file,
session=session,
max_request_size=max_request_size,
)
def _serialize_spans(self, spans: Sequence[ReadableSpan]) -> bytes:
return encode_spans_json(spans)
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
if self._shutdown:
_logger.warning("Exporter already shutdown, ignoring batch")
return SpanExportResult.FAILURE
with self._metrics.export_operation(len(spans)) as result:
try:
serialized_data: Final = encode_spans_json(spans)
except Exception as error: # noqa: BLE001 # SDK export contract reports encoding failures as results
_logger.error("Failed to encode span batch: %s", error)
result.error = error
return SpanExportResult.FAILURE
if 0 < self._max_request_size < len(serialized_data):
result.error = RequestPayloadTooLargeError(
f"Serialized span request size {len(serialized_data)} bytes "
f"exceeds max_request_size {self._max_request_size} bytes."
)
_logger.warning("Dropping span batch: %s", result.error)
return SpanExportResult.FAILURE
export_result: Final = self._client.export(serialized_data)
if not export_result.success:
result.error = export_result.error
result.error_attrs = (
{HTTP_RESPONSE_STATUS_CODE: export_result.status_code}
if export_result.status_code is not None
else None
)
return SpanExportResult.FAILURE
return SpanExportResult.SUCCESS

9
litellm/integrations/otel/presets/agentops.py
View file

@ -76,6 +76,7 @@ def agentops_preset(
def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
"""Factory for the ``agentops`` exporter kind: a lazy-auth OTLP/HTTP exporter."""
import requests
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
OTLPSpanExporter,
)
@ -89,7 +90,8 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
"""
def __init__(self, *, endpoint: str | None, api_key: str | None) -> None:
super().__init__(endpoint=endpoint)
self._agentops_session = requests.Session()
super().__init__(endpoint=endpoint, session=self._agentops_session)
self._agentops_api_key = api_key
self._auth_resolved = False
@ -102,10 +104,7 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
try:
token: Final = _fetch_agentops_jwt(self._agentops_api_key).get("token")
if token:
# ``_session`` is the requests.Session the base exporter
# POSTs through; updating its Authorization header is how the
# minted JWT reaches every subsequent export.
self._session.headers["Authorization"] = f"Bearer {token}"
self._agentops_session.headers["Authorization"] = f"Bearer {token}"
except Exception as e:
verbose_logger.debug("AgentOps JWT fetch failed: %s", e)

5
litellm/integrations/sqs.py
View file

@ -13,6 +13,8 @@ import re
import traceback
from typing import Final
from pydantic import TypeAdapter
import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.constants import (
@ -32,6 +34,7 @@ from litellm.types.utils import StandardLoggingPayload
from .custom_batch_logger import CustomBatchLogger
_DECODED_JSON: Final = TypeAdapter(object)
_BASE64_INLINE_PATTERN: Final = re.compile(
r"data:(?:application|image|audio|video)/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=\s]+",
re.MULTILINE,
@ -270,7 +273,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM):
if self.sqs_queue_url is None:
raise ValueError("sqs_queue_url not set")
json_data: Final = json.loads(safe_dumps(payload))
json_data: Final = _DECODED_JSON.validate_python(json.loads(safe_dumps(payload)))
if self.app_crypto:
aad_bytes: Final = self.sqs_app_encryption_aad.encode("utf-8") if self.sqs_app_encryption_aad else None
encrypted: Final = self.app_crypto.encrypt_json(json_data, aad=aad_bytes)

5
litellm/interactions/litellm_responses_transformation/handler.py
View file

@ -116,10 +116,7 @@ class LiteLLMResponsesInteractionsHandler:
"""Async handler for interactions API requests."""
# Call litellm.aresponses()
# Note: litellm.aresponses() returns Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]
aresponses_fn: Final[
Callable[..., Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]]
] = vars(litellm)["aresponses"]
responses_response: Final = await aresponses_fn(
responses_response: Final = await litellm.aresponses(
**responses_request,
)

9
litellm/litellm_core_utils/app_crypto.py
View file

@ -4,6 +4,9 @@ import os
from typing import Final
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
from pydantic import TypeAdapter
_DECODED_JSON: Final = TypeAdapter(object)
class AppCrypto:
@ -12,7 +15,7 @@ class AppCrypto:
raise ValueError("Master key must be 32 bytes for AES-256-GCM")
self.key = master_key
def encrypt_json(self, data: dict, aad: bytes | None = None) -> dict:
def encrypt_json(self, data: object, aad: bytes | None = None) -> dict:
aes: Final = AESGCM(self.key)
nonce: Final = os.urandom(12)
plaintext: Final = json.dumps(data).encode("utf-8")
@ -24,10 +27,10 @@ class AppCrypto:
"tag": base64.b64encode(tag).decode(),
}
def decrypt_json(self, enc: dict, aad: bytes | None = None) -> dict:
def decrypt_json(self, enc: dict, aad: bytes | None = None) -> object:
aes: Final = AESGCM(self.key)
nonce: Final = base64.b64decode(enc["nonce"])
ct: Final = base64.b64decode(enc["ciphertext"])
tag: Final = base64.b64decode(enc["tag"])
data: Final = aes.decrypt(nonce, ct + tag, aad)
return json.loads(data.decode())
return _DECODED_JSON.validate_python(json.loads(data.decode()))

6
litellm/litellm_core_utils/cloud_storage_security.py
View file

@ -3,10 +3,11 @@ import re
from collections.abc import Mapping, Sequence
from itertools import accumulate, repeat
from types import MappingProxyType
from typing import Any, Final, cast
from typing import Final
from urllib.parse import quote, unquote
from litellm._uuid import uuid
from litellm.litellm_core_utils.core_helpers import as_str_mapping
VERTEX_AI_MANAGED_GCS_PREFIX: Final = "litellm-vertex-files/"
BEDROCK_MANAGED_S3_BATCH_PREFIX: Final = "litellm-bedrock-files-"
@ -141,7 +142,8 @@ def should_allow_legacy_cloud_file_ids(
if isinstance(litellm_params, Mapping):
trusted_model_credentials: Final = litellm_params.get("_litellm_internal_model_credentials")
if isinstance(trusted_model_credentials, _MAPPING_PROXY_TYPE):
value = cast(Mapping[str, Any], trusted_model_credentials).get("allow_legacy_cloud_file_ids")
trusted_mapping: Final = as_str_mapping(trusted_model_credentials)
value = trusted_mapping.get("allow_legacy_cloud_file_ids") if trusted_mapping is not None else None
if isinstance(value, bool):
return value

112
litellm/litellm_core_utils/core_helpers.py
View file

@ -5,10 +5,11 @@ import logging
import re
from collections.abc import Collection, Iterable, Iterator, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_checkable
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
import httpx
from pydantic import TypeAdapter, ValidationError
from typing_extensions import TypeIs # noqa: TID251 # narrows a duck-typed value to the Protocol its caller reads
import litellm
from litellm._logging import verbose_logger
@ -58,6 +59,13 @@ def as_str_mapping(value: object) -> Mapping[str, object] | None:
return value if isinstance(value, Mapping) else None
def decoded_json_object(value: object) -> Mapping[str, object]:
mapping: Final = as_str_mapping(value)
if mapping is not None:
return mapping
raise AttributeError(f"{type(value).__name__!r} object has no attribute 'get'", name="get", obj=value)
@runtime_checkable
class _SupportsIter(Protocol):
def __iter__(self) -> Iterator[object]: ...
@ -72,6 +80,24 @@ def attribute_or_none(source: object, name: str) -> object:
return getattr(source, name, None)
_Shape: Final = TypeVar("_Shape")
def has_attributes(
value: object,
shape: type[_Shape],
*names: str,
) -> TypeIs[_Shape]: # guard-ok: hasattr sees what __getattr__ serves, which Protocol isinstance skips on 3.12+
return all(hasattr(value, name) for name in names)
def require_attributes(value: object, shape: type[_Shape], *names: str) -> _Shape:
if has_attributes(value, shape, *names):
return value
missing: Final = next(name for name in names if not hasattr(value, name))
raise AttributeError(f"'{type(value).__name__}' object has no attribute '{missing}'", name=missing, obj=value)
def iter_items(value: object) -> Iterable[object]:
if isinstance(value, _SupportsIter):
return value
@ -634,11 +660,14 @@ def preserve_upstream_non_openai_attributes(
for key in candidate_keys
)
if all(value is None or isinstance(value, (str, int, float, bool)) for _, value in candidate_values):
for key, value in candidate_values:
setattr(model_response, key, value)
_assign_attributes(model_response, candidate_values)
return
for key, value in original_chunk.model_dump(include=frozenset(candidate_keys)).items():
setattr(model_response, key, value)
_assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items())
def _assign_attributes(target: object, values: Iterable[tuple[str, object]]) -> None:
for key, value in values:
setattr(target, key, value)
def safe_deep_copy(data):
@ -764,6 +793,10 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any:
Filtered data structure with Exception and callable objects removed, or None if the
entire input was an Exception or callable
"""
return _filter_exceptions_from_params(data, max_depth)
def _filter_exceptions_from_params(data: object, max_depth: int) -> object:
if max_depth <= 0:
return data
@ -785,7 +818,7 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any:
if isinstance(v, Exception) or (callable(v) and not isinstance(v, type)):
continue
try:
filtered = filter_exceptions_from_params(v, max_depth - 1)
filtered = _filter_exceptions_from_params(v, max_depth - 1)
if filtered is not None:
result[k] = filtered
except Exception:
@ -799,7 +832,7 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any:
if isinstance(item, Exception) or (callable(item) and not isinstance(item, type)):
continue
try:
filtered = filter_exceptions_from_params(item, max_depth - 1)
filtered = _filter_exceptions_from_params(item, max_depth - 1)
if filtered is not None:
result_list.append(filtered)
except Exception:
@ -895,6 +928,71 @@ class _CarriesHiddenParams(Protocol):
def hidden_params(self, hidden_params: dict[str, object]) -> None: ... # mutable-ok: API requires mutation
_USAGE_BLOCK_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def _usage_field(usage: object, name: str) -> object:
if isinstance(usage, Mapping):
return _USAGE_BLOCK_ADAPTER.validate_python(usage).get(name)
return getattr(usage, name, None)
def _cost_number(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, int | float):
return None
return float(value)
def _reported_cost_value(value: object) -> float | None:
"""A reported cost is a number, or a breakdown whose total lives under ``total_cost``."""
raw: Final[object] = value
if isinstance(value, Mapping):
return _cost_number(_usage_field(raw, "total_cost"))
return _cost_number(value)
def response_cost_in_hidden_params(response: _CarriesHiddenParams) -> float | None:
additional_headers: Final[object] = response.hidden_params.get("additional_headers")
if not isinstance(additional_headers, Mapping):
return None
return _cost_number(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER))
def override_stamped_response_cost(response: _CarriesHiddenParams, cost: float) -> None:
"""Replace the cost an earlier builder stamped so every reader of the hidden params agrees with the header."""
set_response_cost_in_hidden_params(response, cost)
hidden_params: Final = response.hidden_params
hidden_params["response_cost"] = cost
def chunk_usage(chunk: object) -> object:
"""Stream chunks reach the builders both as mappings and as ModelResponseStream objects."""
if isinstance(chunk, Mapping):
return _USAGE_BLOCK_ADAPTER.validate_python(chunk).get("usage")
return getattr(chunk, "usage", None)
_MESSAGES_REPORTED_COST_PROVIDERS: Final = frozenset({"opencode_go"})
def native_messages_reported_cost(usage: object, custom_llm_provider: object) -> float | None:
"""Only a Messages upstream known to price its own responses is trusted, every other one keeps the price map."""
if not isinstance(custom_llm_provider, str) or custom_llm_provider not in _MESSAGES_REPORTED_COST_PROVIDERS:
return None
return provider_reported_usage_cost(usage, custom_llm_provider)
def provider_reported_usage_cost(usage: object, custom_llm_provider: str | None) -> float | None:
"""The request cost a provider states on its usage block, as a number or a breakdown with ``total_cost``.
OpenCode Go reports the model's list price (peak hours and long-context tiers included) under
``usage.opencode.list_cost``, while its ``usage.cost`` is what hit the prepaid balance, 0 inside the plan.
"""
if custom_llm_provider == "opencode_go":
return _reported_cost_value(_usage_field(_usage_field(usage, "opencode"), "list_cost"))
return _reported_cost_value(_usage_field(usage, "cost"))
def set_response_cost_in_hidden_params(response: _CarriesHiddenParams, cost: float | None) -> None:
"""Record a provider-reported cost where the cost calculator looks before the price map."""
if cost is None:

2
litellm/litellm_core_utils/cost_discount.py
View file

@ -23,7 +23,7 @@ def parse_cost_discount_key(key: str) -> CostDiscountKey:
def _literal_length(pattern: str) -> int:
return sum(1 for token in _GLOB_TOKEN.findall(pattern) if len(token) == 1 and token not in "*?")
return sum(1 for match in _GLOB_TOKEN.finditer(pattern) if len(match.group()) == 1 and match.group() not in "*?")
def _model_names(model: str, provider_prefix: str) -> tuple[str, ...]:

2
litellm/litellm_core_utils/default_encoding.py
View file

@ -48,5 +48,5 @@ for attempt in range(_max_retries):
# Last attempt, re-raise the exception
raise
# Exponential backoff with jitter to reduce collision probability
delay = _retry_delay * (2**attempt) + random.uniform(0, 0.1)
delay = _retry_delay * (2.0**attempt) + random.uniform(0, 0.1)
time.sleep(delay)

4
litellm/litellm_core_utils/dot_notation_indexing.py
View file

@ -181,7 +181,7 @@ def _delete_nested_value_custom(
def delete_nested_value(
data: dict[str, Any],
data: Mapping[str, object],
path: str,
depth: int = 0,
max_depth: int = 20,
@ -214,7 +214,7 @@ def delete_nested_value(
"""
import copy
result: Final = copy.deepcopy(data)
result: Final = copy.deepcopy(data if isinstance(data, dict) else dict(data))
try:
# Parse path into segments

5
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -16,7 +16,7 @@ from litellm.litellm_core_utils.bug_report import (
build_bug_report,
should_report_bug,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
from litellm.types.utils import LlmProviders
@ -102,6 +102,7 @@ class ExceptionCheckers:
"`inputs` tokens + `max_new_tokens` must be",
"exceeds the available context size", # llama.cpp/Lemonade
"exceeds the maximum number of tokens allowed", # Gemini
'"code": "context_length_exceeded"',
]
for substring in known_exception_substrings:
if substring in _error_str_lowercase:
@ -168,7 +169,7 @@ def get_error_message(error_obj) -> str | None:
return None
if hasattr(error_obj, "body"):
_error_obj_body: Final = getattr(error_obj, "body")
_error_obj_body: Final = attribute_or_none(error_obj, "body")
if isinstance(_error_obj_body, dict):
# OpenAI-style: {"message": "...", "type": "...", ...}
if _error_obj_body.get("message"):

3
litellm/litellm_core_utils/get_blog_posts.py
View file

@ -19,6 +19,7 @@ import httpx
from pydantic import ConfigDict
from litellm import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.types.llms.base import LiteLLMBaseModel
BLOG_POSTS_TTL_SECONDS: Final[int] = 3600 # 1 hour
@ -129,7 +130,7 @@ class GetBlogPosts:
Uses the in-process cache if within BLOG_POSTS_TTL_SECONDS.
Fetches from ``url`` otherwise, falling back to local backup on failure.
"""
if os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
return cls.load_local_blog_posts()
now: Final = time.time()

10
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -902,6 +902,10 @@ def _get_openai_compatible_provider_info(
api_base,
dynamic_api_key,
) = litellm.MorphChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "opencode_go":
from litellm.llms.opencode.common_utils import go_api_base, go_api_key
return model, custom_llm_provider, go_api_key(api_key), go_api_base(api_base, model)
elif custom_llm_provider == "lambda_ai":
(
api_base,
@ -972,3 +976,9 @@ def _get_openai_compatible_provider_info(
if dynamic_api_key is None and api_key is not None:
dynamic_api_key = api_key
return model, custom_llm_provider, dynamic_api_key, api_base
def get_provider_affinity_header(api_base: str | None) -> str | None:
from litellm.llms.opencode.transformation import OpenCodeProviderAdapter
return OpenCodeProviderAdapter().get_provider_affinity_header(api_base)

11
litellm/litellm_core_utils/get_model_cost_map.py
View file

@ -39,9 +39,11 @@ from litellm.constants import (
from litellm.litellm_core_utils.fallback_generalizations import (
set_fallback_generalizations,
)
from litellm.litellm_core_utils.offline_mode import is_offline_mode
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
_DECODED_JSON: Final = TypeAdapter(object)
_BUNDLED_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
_CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"})
@ -313,7 +315,7 @@ def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -
def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome:
try:
parsed: Final[object] = json.loads(content)
parsed: Final = _DECODED_JSON.validate_python(json.loads(content))
except ValueError as e:
return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
if not isinstance(parsed, dict):
@ -382,7 +384,8 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp
)
if response.is_error:
return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}")
return _classify_fetch_content(response.content, url, response.headers.get("etag"))
etag_values: Final = response.headers.get_list("etag")
return _classify_fetch_content(response.content, url, ", ".join(etag_values) if etag_values else None)
def _next_retry_wait(
@ -456,7 +459,7 @@ async def refetch_model_cost_map(
on failure it returns ``ModelCostMapReloadUnavailable`` so callers keep the
map they already have.
"""
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
_cost_map_source_info.source = "local"
_cost_map_source_info.url = None
@ -740,7 +743,7 @@ def get_model_cost_map(
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
# Note: can't use get_secret_bool here — this runs during litellm.__init__
# before litellm._key_management_settings is set.
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process():
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process():
_cost_map_source_info.source = "local"
_cost_map_source_info.url = None
_cost_map_source_info.is_env_forced = True

3
litellm/litellm_core_utils/hidden_params.py
View file

@ -1,4 +1,5 @@
from collections.abc import Iterator, Mapping, MutableMapping
from types import MappingProxyType
from typing import Final, Protocol, cast
from pydantic import BaseModel
@ -52,7 +53,7 @@ class HiddenParamsModelView(MutableMapping[str, object]):
def stored_attribute(obj: object, name: str) -> object:
if not isinstance(obj, BaseModel):
return getattr(obj, name, None)
values: Final[Mapping[str, object]] = vars(obj)
values: Final[Mapping[str, object]] = MappingProxyType[str, object](vars(obj))
if name in values:
return values[name]
extras: Final[Mapping[str, object]] = obj.__pydantic_extra__ or {}

2
litellm/litellm_core_utils/initialize_dynamic_callback_params.py
View file

@ -28,7 +28,7 @@ def inherit_message_logging_privacy(disabled: bool) -> Generator[None]:
def iter_client_callback_metadata_dicts(
kwargs: dict[str, Any],
kwargs: Mapping[str, object],
) -> Iterator[tuple[str, dict[str, Any]]]:
litellm_params: Final = kwargs.get("litellm_params")
if isinstance(litellm_params, dict):

95
litellm/litellm_core_utils/litellm_logging.py
View file

@ -77,6 +77,8 @@ from litellm.litellm_core_utils.classifier_logging import (
from litellm.litellm_core_utils.core_helpers import (
get_provider_response_headers_from_hidden_params,
is_expected_client_error,
native_messages_reported_cost,
provider_reported_usage_cost,
proxy_stamped_kubernetes_pod_routing,
proxy_stamped_used_client_oauth_token,
reconstruct_model_name,
@ -161,6 +163,7 @@ from litellm.types.utils import (
CachingDetails,
CallTypes,
CostBreakdown,
CostPerToken,
CustomPricingLiteLLMParams,
DynamicPromptManagementParamLiteral,
EmbeddingResponse,
@ -555,6 +558,7 @@ def _resolve_mantle_region_for_cost(
_PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
_REDACTED_RESULT: Final = TypeAdapter(object)
def _provider_response_id(source: object) -> str | None:
@ -621,6 +625,8 @@ class Logging(LiteLLMLoggingBaseClass):
prometheusLogger, \
slack_app
custom_pricing: bool = False
custom_cost_per_token: CostPerToken | None = None
custom_cost_per_second: float | None = None
replaced_global_callback_keys: frozenset[str] = frozenset()
stream_options = None
litellm_request_debug: bool = False
@ -1168,7 +1174,7 @@ class Logging(LiteLLMLoggingBaseClass):
automatically extracts metadata/litellm_metadata from kwargs,
so callers don't need to manually plumb them into litellm_params.
"""
base_litellm_params: Final[dict[str, Any]] = {}
base_litellm_params: Final[dict[str, object]] = {}
if isinstance(kwargs.get("metadata"), dict):
base_litellm_params["metadata"] = kwargs["metadata"].copy()
@ -1185,10 +1191,11 @@ class Logging(LiteLLMLoggingBaseClass):
lp_metadata: Final = litellm_params.pop("metadata", None)
base_litellm_params.update(litellm_params)
if lp_metadata and isinstance(lp_metadata, dict):
base_litellm_params.setdefault("metadata", {})
for k, v in lp_metadata.items():
if k not in base_litellm_params["metadata"]:
base_litellm_params["metadata"][k] = v
merged_metadata: Final = base_litellm_params.setdefault("metadata", {})
if isinstance(merged_metadata, dict):
for k, v in lp_metadata.items():
if k not in merged_metadata:
merged_metadata[k] = v
self.update_environment_variables(
litellm_params=base_litellm_params,
@ -1790,7 +1797,7 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.exception(
"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e
)
original_response = redact_message_input_output_from_logging(
redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
result=original_response,
)
@ -2020,10 +2027,9 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details.pop("response_cost_failure_debug_information", None)
if cache_hit is None:
cache_hit = self.model_call_details.get("cache_hit", False)
resolved_cache_hit: Final = self.model_call_details.get("cache_hit", False) if cache_hit is None else cache_hit
if cache_hit is True:
if resolved_cache_hit is True:
return 0.0
if is_unbilled_non_inference_call(
@ -2075,7 +2081,7 @@ class Logging(LiteLLMLoggingBaseClass):
model_value: Final = litellm_model_name or self.model
cost_cache_hit_value: Final = (
self.model_call_details.get("cache_hit", False) if cache_hit is None else cache_hit
self.model_call_details.get("cache_hit", False) if resolved_cache_hit is None else resolved_cache_hit
)
cost_cache_hit: Final[bool | None] = cast( # cast-ok: callback metadata is caller-provided
bool | None, cost_cache_hit_value
@ -2096,6 +2102,8 @@ class Logging(LiteLLMLoggingBaseClass):
"call_type": self.call_type,
"optional_params": self.optional_params,
"custom_pricing": custom_pricing,
"custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None,
"custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None,
"prompt": prompt,
"standard_built_in_tools_params": self.standard_built_in_tools_params,
"router_model_id": router_model_id,
@ -2507,8 +2515,10 @@ class Logging(LiteLLMLoggingBaseClass):
def carry_failed_attempt_usage(self) -> None:
"""Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its
own usage while the request's failure log still bills every attempt."""
usage: Final = self.model_call_details.pop("combined_usage_object", None)
response_cost: Final = self.model_call_details.pop("response_cost", None)
usage: Final = self._model_call_details_view.get("combined_usage_object")
response_cost: Final = self._model_call_details_view.get("response_cost")
self.model_call_details.pop("combined_usage_object", None)
self.model_call_details.pop("response_cost", None)
if isinstance(usage, Usage):
self.model_call_details["current_attempt_api_call_start_time"] = None
self.model_call_details["failed_attempts_usage"] = (
@ -3297,7 +3307,7 @@ class Logging(LiteLLMLoggingBaseClass):
if callback_logging_payload is not None or shared_logging_payload is None
else shared_logging_payload.copy()
)
call_details: dict[str, Any] # mutable-ok: legacy loggers take a plain dict
call_details: dict[str, object] # mutable-ok: legacy loggers take a plain dict
call_details = ( # rebind-ok: logging hooks may return a new dict
self.model_call_details
if not for_async_request
@ -3318,9 +3328,11 @@ class Logging(LiteLLMLoggingBaseClass):
)
## REDACT MESSAGES ##
result = redact_message_input_output_from_logging(
model_call_details=call_details,
result=result,
result = _REDACTED_RESULT.validate_python(
redact_message_input_output_from_logging(
model_call_details=call_details,
result=result,
)
)
## LOGGING HOOK ##
hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks)))
@ -3586,7 +3598,7 @@ class Logging(LiteLLMLoggingBaseClass):
print_verbose("S3Logger Logger: Got Stream Event - Completed Stream Response")
s3Logger.log_event(
kwargs=call_details,
response_obj=call_details["complete_streaming_response"],
response_obj=call_details_view["complete_streaming_response"],
start_time=start_time,
end_time=end_time,
print_verbose=print_verbose,
@ -3909,9 +3921,11 @@ class Logging(LiteLLMLoggingBaseClass):
self._get_completed_success_callbacks(asynchronous=True) if not self.stream else callbacks
)
result = redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
result=result,
result = _REDACTED_RESULT.validate_python(
redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
result=result,
)
)
## LOGGING HOOK ##
@ -4256,9 +4270,11 @@ class Logging(LiteLLMLoggingBaseClass):
result: object = None # result sent to all loggers, init this to None incase it's not created
result = redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
result=result,
result = _REDACTED_RESULT.validate_python(
redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
result=result,
)
)
self.mark_logging_complete(event_type="sync_failure")
for callback in callbacks:
@ -4742,7 +4758,12 @@ class Logging(LiteLLMLoggingBaseClass):
):
## return unified Usage object
if isinstance(result.response.usage, ResponseAPIUsage):
set_response_cost_in_hidden_params(result.response, result.response.usage.cost)
set_response_cost_in_hidden_params(
result.response,
provider_reported_usage_cost(
result.response.usage, self.model_call_details.get("custom_llm_provider")
),
)
transformed_usage: Final = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(
result.response.usage
)
@ -4808,8 +4829,11 @@ class Logging(LiteLLMLoggingBaseClass):
bridge is replaced by the ModelResponse its spend row is built from, anything else is
priced as is."""
served: Final[object] = result
if self.bridged_model_response is None or not isinstance(result, dict):
return result
call_type: Final[object] = getattr(self, "call_type", None)
if not isinstance(result, dict):
return served
if self.bridged_model_response is None and call_type != CallTypes.anthropic_messages.value:
return served
return self._anthropic_messages_logged_response(result=served)
def _anthropic_messages_logged_response(self, result: object) -> ModelResponse:
@ -4852,7 +4876,13 @@ class Logging(LiteLLMLoggingBaseClass):
if isinstance(result, ResponsesAPIResponse):
return self._translate_responses_api_response_to_model_response(result)
provider_response_id: Final = _provider_response_id(result)
raw_body: Final[object] = result
served_body: Final = _LITELLM_PARAMS.validate_python(raw_body) if isinstance(result, dict) else None
reported_cost: Final = native_messages_reported_cost(
served_body.get("usage") if served_body is not None else None,
self.model_call_details.get("custom_llm_provider"),
)
provider_response_id: Final = _provider_response_id(raw_body)
bridged: Final = self.bridged_model_response
if bridged is not None and isinstance(result, dict):
return bridged if provider_response_id is None else bridged.model_copy(update={"id": provider_response_id})
@ -4877,7 +4907,7 @@ class Logging(LiteLLMLoggingBaseClass):
else:
from litellm.types.llms.anthropic import AnthropicResponse
pydantic_result: Final = AnthropicResponse.model_validate(result)
pydantic_result: Final = AnthropicResponse.model_validate({"stop_sequence": None, **(served_body or {})})
import httpx
result = litellm.AnthropicConfig().transform_parsed_response(
@ -4890,6 +4920,7 @@ class Logging(LiteLLMLoggingBaseClass):
json_mode=None,
speed=self.optional_params.get("speed") if self.optional_params else None,
)
set_response_cost_in_hidden_params(result, reported_cost)
return result
def _translate_responses_api_response_to_model_response(self, result: ResponsesAPIResponse) -> ModelResponse:
@ -4947,7 +4978,7 @@ class Logging(LiteLLMLoggingBaseClass):
"""
import httpx
httpx_response: Final = self.model_call_details.get("httpx_response", None)
httpx_response: Final[Response | None] = self.model_call_details.get("httpx_response", None)
if httpx_response is None:
raise ValueError("Google GenAI Generate Content: httpx_response is None")
dict_result: Final = httpx_response.json()
@ -4977,7 +5008,7 @@ class Logging(LiteLLMLoggingBaseClass):
The response object with usage added if available
"""
# Get usage from model_call_details (set by asend_message)
usage: Final = self.model_call_details.get("usage")
usage: Final[Usage | None] = self.model_call_details.get("usage")
if usage is None:
return result
@ -5810,7 +5841,9 @@ def build_custom_logger_compatible_class(
return callback
# Get global BitBucket config
bitbucket_config: Final = getattr(litellm, "global_bitbucket_config", None)
bitbucket_config: Final = (
litellm.global_bitbucket_config if hasattr(litellm, "global_bitbucket_config") else None
)
if bitbucket_config is None:
raise ValueError("BitBucket configuration not found. Please set litellm.global_bitbucket_config first.")

42
litellm/litellm_core_utils/llm_cost_calc/tiered_pricing.py
View file

@ -5,30 +5,42 @@ Shared by provider cost calculators (e.g. Dashscope) and the proxy budget
reservation logic so neither has to depend on the other.
"""
from typing import Final
from collections.abc import Mapping, Sequence
from typing import Final, SupportsFloat
from pydantic import TypeAdapter, ValidationError
_TIER_RANGE: Final = TypeAdapter(tuple[float, float])
def _coerce_cost_per_token(value: float | str | None) -> float:
def _coerce_cost_per_token(value: object) -> float:
"""
Coerce a per-token cost into a float.
Model cost values loaded from YAML config may arrive as strings (e.g.
scientific notation like "4e-07"), which would break arithmetic.
"""
if value is None:
return 0.0
if isinstance(value, str):
try:
return float(value)
except ValueError:
return 0.0
return float(value)
if isinstance(value, SupportsFloat):
return float(value)
return 0.0
def _tier_range(tier: Mapping[str, object]) -> tuple[float, float] | None:
try:
return _TIER_RANGE.validate_python(tier.get("range"))
except ValidationError:
return None
def select_tier_for_input(
tiered_pricing: list[dict],
tiered_pricing: Sequence[Mapping[str, object]],
input_tokens: int,
) -> dict | None:
) -> Mapping[str, object] | None:
"""
Select the pricing tier for a request based on its total input token count.
@ -44,19 +56,21 @@ def select_tier_for_input(
if not tiered_pricing or input_tokens <= 0:
return None
sorted_tiers: Final = sorted(tiered_pricing, key=lambda t: t.get("range", [0, 0])[0])
valid_tiers: Final = [tier for tier in sorted_tiers if len(tier.get("range", [])) == 2]
if not valid_tiers:
ranged_tiers: Final = sorted(
((bounds, tier) for tier in tiered_pricing if (bounds := _tier_range(tier)) is not None),
key=lambda ranged: ranged[0][0],
)
if not ranged_tiers:
return None
matching: Final = [tier for tier in valid_tiers if tier["range"][0] < input_tokens <= tier["range"][1]]
matching: Final = tuple(tier for (start, end), tier in ranged_tiers if start < input_tokens <= end)
if matching:
return matching[0]
return valid_tiers[-1]
return ranged_tiers[-1][1]
def tier_rate(
tier: dict,
tier: Mapping[str, object],
cost_key: str,
fallback_cost_key: str | None = None,
) -> float:
@ -68,4 +82,6 @@ def tier_rate(
primary: Final = tier.get(cost_key)
if primary is not None:
return _coerce_cost_per_token(primary)
if fallback_cost_key is None:
return 0.0
return _coerce_cost_per_token(tier.get(fallback_cost_key, 0))

Some files were not shown because too many files have changed in this diff Show more