scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-12 08:42:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 207

fix: resolve main conflicts in model group spend tracking

Browse source
This commit is contained in:
Fernando Jordan Silva 2026-10-26 08:44:17 +01:00
commit cd93c02838
1164 changed files with 53798 additions and 10955 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/CODEOWNERS vendored
View file

@ -1,4 +1,4 @@
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/helm/ @yassin-berriai
/terraform/ @yassin-berriai

91
.github/workflows/interactions-spec-canary.yml vendored Normal file
View file

@ -0,0 +1,91 @@
name: Interactions spec canary
on:
schedule:
- cron: "17 7 * * *"
pull_request:
paths:
- .github/workflows/interactions-spec-canary.yml
- tests/unit/interactions/**
- litellm/types/interactions/**
workflow_dispatch:
permissions: {}
concurrency: interactions-spec-canary
jobs:
interactions-spec-canary:
name: interactions-spec-canary
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
outputs:
compliance: ${{ steps.compliance.outcome }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Install dependencies
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy --no-install-project
- name: Run the compliance tests against Google's live spec
id: compliance
run: |
uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \
--interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \
--tb=short -vv -p no:rerunfailures -p no:pytest-retry \
--allow-hosts=127.0.0.1,::1,localhost,ai.google.dev
tracking-issue:
name: tracking-issue
needs: interactions-spec-canary
if: always() && github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
issues: write
steps:
- name: File the tracking issue unless the compliance step passed, close it when it did
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
env:
COMPLIANCE_OUTCOME: ${{ needs.interactions-spec-canary.outputs.compliance }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
with:
script: |
const title = "Interactions spec canary is red";
const outcome = process.env.COMPLIANCE_OUTCOME;
const runUrl = process.env.RUN_URL;
const query = `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open in:title "${title}"`;
const { data: found } = await github.rest.search.issuesAndPullRequests({ q: query, per_page: 20 });
const open = found.items.find((issue) => issue.title === title);
if (outcome === "success") {
if (open === undefined) return;
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Green again: ${runUrl}` });
await github.rest.issues.update({ ...context.repo, issue_number: open.number, state: "closed" });
return;
}
const state = outcome === "failure" ? "red" : `not run (compliance step outcome: ${outcome || "none"})`;
if (open !== undefined) {
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Still ${state}: ${runUrl}` });
return;
}
await github.rest.issues.create({
...context.repo,
title,
assignees: ["mateo-berri"],
body: `The scheduled run of tests/unit/interactions/test_openapi_compliance.py against Google's live Interactions spec is ${state}: ${runUrl}\n\nlitellm/types/interactions/README.md says how to refresh the snapshot and regenerate the types.`,
});

3
.github/workflows/test-unit.yml vendored
View file

@ -216,10 +216,12 @@ jobs:
- shard: llms-providers
test-path: |-
tests/unit/llms/test_api_base_trailing_slash_routes.py
tests/unit/llms/test_cache_control_and_reasoning.py
tests/unit/llms/test_custom_llm.py
tests/unit/llms/test_file_content_block.py
tests/unit/llms/test_file_search_responses.py
tests/unit/llms/test_get_complete_url_trailing_slash.py
tests/unit/llms/test_lifecycle_fix.py
tests/unit/llms/test_oss_decision.py
tests/unit/llms/test_polling_url_origin_match.py
@ -423,6 +425,7 @@ jobs:
tests/unit/realtime_api
tests/unit/rerank_api
tests/unit/search
tests/unit/text_completions
tests/unit/vector_stores
tests/unit/videos
python-version: "3.12"

11
Dockerfile
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@ -61,6 +61,7 @@ RUN apk add --no-cache \
python-3.13-dev \
rust \
openssl \
libssl3 \
openssl-dev \
nodejs \
npm \
@ -120,8 +121,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder
COPY --from=uvbin /uv /usr/local/bin/uv
RUN apk add --no-cache python-3.13
ADD --checksum=sha256:2f7ae5cdd9d91731c0990e74a58239dc3e3fd2bf28dab23b55eafcdc47aaf87e \
https://github.com/BerriAI/litellm-admin-agent/archive/ef501e94bc9fbacb9233b922abf71427f030408c.tar.gz /tmp/liteadmin.tar.gz
ADD --checksum=sha256:4fb1847e28e2be30ba76dac94f02c4903291326516a0f4ba772f6ffb8b25d05c \
https://github.com/BerriAI/litellm-admin-agent/archive/5661c805ce479a0cf11b842ea133cf8d22cd7e34.tar.gz /tmp/liteadmin.tar.gz
RUN mkdir /tmp/liteadmin && tar xzf /tmp/liteadmin.tar.gz --strip-components=1 -C /tmp/liteadmin && \
uv venv /opt/liteadmin --python python3.13 && \
uv pip install --python /opt/liteadmin/bin/python --require-hashes -r /tmp/liteadmin/requirements.txt && \
@ -141,7 +142,7 @@ USER root
RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories
# node (without npm) is required by the prisma CLI at runtime
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
WORKDIR /app

6
README.md
View file

@ -616,15 +616,15 @@ The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws
### Verify Docker Image Signatures
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Releases are signed with the key introduced in [commit `e82a3c6`](https://github.com/BerriAI/litellm/commit/e82a3c646908ba0a6ee0beddc4310d7a6cac4065).
**Verify using the pinned commit hash (recommended):**
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the published signing key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
--key https://raw.githubusercontent.com/BerriAI/litellm/e82a3c646908ba0a6ee0beddc4310d7a6cac4065/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```

2
security.md → SECURITY.md
View file

@ -79,7 +79,7 @@ To qualify for a bounty, reports must include clear reproduction steps, a reprod
- Audit Logs with retention policy
- Control Allowed IP Addresses that can access your Cloud LiteLLM Instance
For security inquiries, please contact us at support@berri.ai
For security inquiries, please contact us at security@berri.ai
#### Supported data regions for LiteLLM Cloud

15
backend/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
@ -43,7 +43,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
@ -109,7 +109,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
USER root
RUN for i in 1 2 3; do \
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
@ -124,7 +124,12 @@ ENV HOME=/home/nonroot \
PYTHONUNBUFFERED=1 \
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
COPY --from=builder --chown=nonroot:nonroot /app /app
COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
COPY --from=builder --chown=nonroot:nonroot /app/backend /app/backend
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
COPY --from=builder /opt/prisma /opt/prisma
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \

4
cosign.pub
View file

@ -1,4 +1,4 @@
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb
POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g==
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEG06dpKM+vEDnmp2zcjyXgVCXxzRg
JA5H2LnmwBKQafZGka6OOrWujCEWATQewiAgyD4WQdJ0pvU9RC4mvNAjvQ==
-----END PUBLIC KEY-----

7
docker/Dockerfile.database
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@ -59,6 +59,7 @@ RUN apk add --no-cache \
python-3.13 \
python-3.13-dev \
openssl \
libssl3 \
openssl-dev \
nodejs \
npm \
@ -122,7 +123,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
USER root
# node (without npm) is required by the prisma CLI at runtime
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
WORKDIR /app

7
docker/Dockerfile.non_root
View file

@ -1,8 +1,8 @@
# syntax=docker/dockerfile:1.7
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@ -60,6 +60,7 @@ RUN for i in 1 2 3; do \
coreutils \
curl \
openssl \
libssl3 \
libsndfile \
nodejs \
npm && break || sleep 5; \
@ -134,7 +135,7 @@ RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
done && \
for i in 1 2 3; do \
apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs libevent && break || sleep 5; \
apk add --no-cache python-3.13 bash openssl libssl3 tzdata libsndfile nodejs libevent && break || sleep 5; \
done
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer

2
docker/build_from_pip/Dockerfile.build_from_pip
View file

@ -30,7 +30,7 @@ RUN uv venv --python python && \
"google-cloud-aiplatform==1.133.0" \
"google-genai==1.37.0" \
"anthropic[vertex]==0.84.0" \
"grpcio==1.78.0" \
"grpcio==1.84.1" \
"prometheus-client==0.20.0" \
"langfuse==2.59.7" \
"opentelemetry-api==1.28.0" \

13
enterprise/enterprise_hooks/aporia_ai.py
View file

@ -15,9 +15,10 @@ sys.path.insert(
) # Adds the parent directory to the system path
import json
import sys
from typing import Any, List, Literal, Optional
from typing import Final, List, Literal, Optional
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -29,11 +30,13 @@ from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.llms.openai.common_utils import decoded_json_object
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata
from litellm.types.guardrails import GuardrailEventHooks
GUARDRAIL_NAME = "aporia"
_DECODED_JSON: Final = TypeAdapter(object)
class AporiaGuardrail(CustomGuardrail):
@ -68,7 +71,7 @@ class AporiaGuardrail(CustomGuardrail):
async def prepare_aporia_request(
self, new_messages: List[dict], response_string: Optional[str] = None
) -> dict:
data: dict[str, Any] = {}
data: dict[str, object] = {}
if new_messages is not None:
data["messages"] = new_messages
if response_string is not None:
@ -122,8 +125,10 @@ class AporiaGuardrail(CustomGuardrail):
verbose_proxy_logger.debug("Aporia AI response: %s", response.text)
if response.status_code == 200:
# check if the response was flagged
_json_response = response.json()
action: str = _json_response.get(
_json_response: Final = decoded_json_object(
_DECODED_JSON.validate_python(response.json())
)
action: Final = _json_response.get(
"action"
) # possible values are modify, passthrough, block, rephrase
if action == "block":

17
enterprise/enterprise_hooks/blocked_user_list.py
View file

@ -7,9 +7,10 @@
## This accepts a list of user id's for whom calls will be rejected
from typing import Literal, Optional
from typing import Final, Literal, Optional
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._internal_context import with_service_target
@ -19,6 +20,9 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import LiteLLM_EndUserTable, UserAPIKeyAuth
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET
from litellm.proxy.utils import PrismaClient
from litellm.repositories.table_repositories import EndUserRepository
_REQUEST_BODY_VALUE: Final = TypeAdapter(object)
class ENTERPRISE_BlockedUserList(CustomLogger):
@ -94,15 +98,14 @@ class ENTERPRISE_BlockedUserList(CustomLogger):
)
if end_user_cache_obj is None and self.prisma_client is not None:
# check db
end_user_obj = (
await self.prisma_client.db.litellm_endusertable.find_unique(
where={"user_id": user}
)
)
end_user_id: Final = _REQUEST_BODY_VALUE.validate_python(user)
end_user_obj = await EndUserRepository(
self.prisma_client
).table.find_unique(where={"user_id": end_user_id})
if end_user_obj is None: # user not in db - assume not blocked
end_user_obj = LiteLLM_EndUserTable(user_id=user, blocked=False)
cache.set_cache(key=cache_key, value=end_user_obj, ttl=60)
if end_user_obj is not None and end_user_obj.blocked is True:
if end_user_obj.blocked is True:
raise HTTPException(
status_code=400,
detail={

13
enterprise/enterprise_hooks/google_text_moderation.py
View file

@ -6,7 +6,10 @@
# +-----------------------------------------------+
# Thank you users! We ❤️ you! - Krrish & Ishaan
from typing import Final
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -15,6 +18,8 @@ from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import iter_message_text
from litellm.types.utils import CallTypesLiteral
_THRESHOLD_VALUE: Final = TypeAdapter(object)
class ENTERPRISE_GoogleTextModeration(CustomLogger):
user_api_key_cache = None
@ -69,9 +74,11 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
f"{category}_confidence_threshold",
default_confidence_threshold,
)
set_confidence_value = getattr(
self,
f"{category}_confidence_threshold",
set_confidence_value = _THRESHOLD_VALUE.validate_python(
getattr(
self,
f"{category}_confidence_threshold",
)
)
verbose_proxy_logger.info(
f"Google Text Moderation: {category}_confidence_threshold: {set_confidence_value}"

7
enterprise/litellm_enterprise/enterprise_callbacks/example_logging_api.py
View file

@ -1,15 +1,20 @@
# this is an example endpoint to receive data from litellm
from typing import Final
from fastapi import FastAPI, HTTPException, Request
from pydantic import TypeAdapter
app = FastAPI()
_LOGGED_EVENT: Final = TypeAdapter(object)
@app.post("/log-event")
async def log_event(request: Request):
try:
print("Received /log-event request") # noqa
# Assuming the incoming request has JSON data
data = await request.json()
data: Final = _LOGGED_EVENT.validate_python(await request.json())
print("Received request data:") # noqa
print(data) # noqa

18
enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py
View file

@ -8,25 +8,30 @@
## This provides an LLM Guard Integration for content moderation on the proxy
import asyncio
from typing import Final, Optional
from collections.abc import Mapping
from typing import Final
import aiohttp
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.openai.common_utils import decoded_json_object
from litellm.proxy._types import UserAPIKeyAuth
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import CallTypesLiteral
_DECODED_JSON: Final = TypeAdapter(object)
class _ENTERPRISE_LLMGuard(CustomLogger):
# Class variables or attributes
def __init__(
self,
mock_testing: bool = False,
mock_redacted_text: Optional[dict] = None,
mock_redacted_text: Mapping[str, object] | None = None,
):
self.mock_redacted_text = mock_redacted_text
self.llm_guard_mode = litellm.llm_guard_mode
@ -66,7 +71,9 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
async with session.post(
analyze_url, json={"prompt": text}
) as response:
redacted_text = await response.json()
redacted_text = _DECODED_JSON.validate_python(
await response.json()
)
verbose_proxy_logger.debug(
f"LLM Guard: Received response - {redacted_text}"
)
@ -77,12 +84,13 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
"error": f"Invalid content moderation response: {redacted_text}"
},
)
if redacted_text.get("is_valid", None) is False:
redacted_response: Final = decoded_json_object(redacted_text)
if redacted_response.get("is_valid", None) is False:
raise HTTPException(
status_code=400,
detail={"error": "Violated content safety policy"},
)
sanitized_prompt = redacted_text.get("sanitized_prompt")
sanitized_prompt: Final = redacted_response.get("sanitized_prompt")
return sanitized_prompt if isinstance(sanitized_prompt, str) else text
except Exception as e:
verbose_proxy_logger.exception(

4
enterprise/litellm_enterprise/proxy/auth/route_checks.py
View file

@ -2,6 +2,8 @@ import os
from fastapi import HTTPException, status
from litellm.proxy._types import MODEL_LIST_ROUTES
class EnterpriseRouteChecks:
@staticmethod
@ -43,7 +45,7 @@ class EnterpriseRouteChecks:
# Routes that should remain accessible even when LLM API endpoints are disabled.
# These are read-only model listing routes needed by the Admin UI.
LLM_API_EXEMPT_ROUTES = ["/models", "/v1/models"]
LLM_API_EXEMPT_ROUTES = MODEL_LIST_ROUTES
@staticmethod
def should_call_route(route: str):

4
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -1040,8 +1040,8 @@ class CheckBatchCost:
from litellm.proxy.proxy_server import proxy_config
from litellm.secret_managers.main import str_to_bool
api_key: Final = getattr(job, "api_key", None)
team_id: Final = getattr(job, "team_id", None)
api_key: Final = job.api_key if hasattr(job, "api_key") else None
team_id: Final = job.team_id if hasattr(job, "team_id") else None
has_key_row: Final = api_key is not None and is_valid_sha256_hash(api_key)
try:
key_row: Final = (

27
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -44,6 +44,7 @@ from litellm.constants import (
)
from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.litellm_core_utils.internal_call_metadata import requests_background
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -133,7 +134,7 @@ if TYPE_CHECKING:
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
from litellm.proxy.utils import PrismaClient as _PrismaClient
Span = Union[_Span, Any]
Span = _Span
InternalUsageCache = _InternalUsageCache
PrismaClient = _PrismaClient
else:
@ -490,6 +491,14 @@ _PROVIDER_FILE_ID_KEYS: Final = tuple(dict.fromkeys(chain.from_iterable(_PROVIDE
_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS: Final = (0.5, 1.0, 2.0)
class _SupportsKeyedGet(Protocol):
def get(self, key: str, default: object, /) -> object: ...
def _keyed_get(source: object, key: str, default: object = None) -> object:
return require_attributes(source, _SupportsKeyedGet, "get").get(key, default)
class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Class variables or attributes
def __init__(
@ -1611,7 +1620,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
return holders if holders or "input_file_id" in request else healthy_deployments
async def async_pre_call_deployment_hook(
self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
self, kwargs: dict[str, object], call_type: Optional[CallTypes]
) -> Optional[dict]:
"""
Allow modifying the request just before it's sent to the deployment.
@ -1623,11 +1632,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping"))
# model_info may be at top-level or nested under litellm_metadata
# (batch/file operations use litellm_metadata)
model_id = cast(Optional[str], kwargs.get("model_info", {}).get("id", None))
model_id = cast(Optional[str], _keyed_get(kwargs.get("model_info", {}), "id"))
if model_id is None:
model_id = cast(
Optional[str],
kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None),
_keyed_get(_keyed_get(kwargs.get("litellm_metadata", {}), "model_info", {}), "id"),
)
for accessor_key in accessor_keys:
input_file_id = cast(Optional[str], kwargs.get(accessor_key))
@ -1824,7 +1833,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
),
)
async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span) -> dict:
async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span | None) -> dict:
"""
Get model-specific file IDs for a list of proxy file IDs.
Returns a dictionary mapping litellm_proxy/ file_id -> model_id -> model_file_id
@ -1866,7 +1875,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
llm_router: Optional[Router],
_create_file_request: CreateFileRequest,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
) -> List[OpenAIFileObject]:
if llm_router is None:
raise Exception("LLM Router not initialized. Ensure models added to proxy.")
@ -1882,7 +1891,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
create_file_request: CreateFileRequest,
llm_router: Router,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
user_api_key_dict: UserAPIKeyAuth,
) -> OpenAIFileObject:
responses = await self.create_file_for_each_model(
@ -1949,7 +1958,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
file_objects: List[OpenAIFileObject],
create_file_request: CreateFileRequest,
internal_usage_cache: InternalUsageCache,
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
target_model_names_list: List[str],
) -> OpenAIFileObject:
## GET THE FILE TYPE FROM THE CREATE FILE REQUEST
@ -2652,7 +2661,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Continue with other files even if one fails
continue
def _get_content_type_from_file_object(self, file_object: Optional[Any]) -> str:
def _get_content_type_from_file_object(self, file_object: object) -> str:
"""
Determine content type from file object.

6
enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py
View file

@ -29,7 +29,7 @@ if TYPE_CHECKING:
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
from litellm.proxy.utils import PrismaClient as _PrismaClient
Span = Union[_Span, Any]
Span = _Span
InternalUsageCache = _InternalUsageCache
PrismaClient = _PrismaClient
else:
@ -107,7 +107,7 @@ class _PROXY_LiteLLMManagedVectorStores(
llm_router: Router,
model: str,
request_data: dict[str, object] | VectorStoreCreateOptionalRequestParams,
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
) -> VectorStoreCreateResponse:
"""
Create a vector store for a specific model.
@ -134,7 +134,7 @@ class _PROXY_LiteLLMManagedVectorStores(
create_request: VectorStoreCreateOptionalRequestParams,
llm_router: Router,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
user_api_key_dict: UserAPIKeyAuth,
) -> VectorStoreCreateResponse:
"""

13
enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py
View file

@ -36,6 +36,7 @@ from litellm.repositories.budget_repository import BudgetRepository
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.project_repository import ProjectRepository
from litellm.repositories.table_repositories import PrismaTableRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
@ -73,6 +74,16 @@ def _verification_token_table(
return VerificationTokenRepository(prisma_client).table
class _WriterVerificationTokenRepository(PrismaTableRepository["prisma_models.LiteLLM_VerificationToken"]):
table_name = "litellm_verificationtoken"
def _writer_verification_token_table(
prisma_client: PrismaClient,
) -> TableActions["prisma_models.LiteLLM_VerificationToken"]:
return _WriterVerificationTokenRepository(prisma_client, use_writer=True).table
def _budget_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_BudgetTable"]:
return BudgetRepository(prisma_client).table
@ -849,7 +860,7 @@ async def update_project(
else None
)
if current_project is not None and data.team_id != current_project.team_id:
mismatched_key_count: Final = await prisma_client.writer_db.litellm_verificationtoken.count(
mismatched_key_count: Final = await _writer_verification_token_table(prisma_client).count(
where={
"project_id": data.project_id,
"OR": [{"team_id": {"not": data.team_id}}, {"team_id": None}],

8
enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py
View file

@ -14,6 +14,7 @@ from collections.abc import Mapping
from typing import TYPE_CHECKING, Final, List, Optional, Protocol
from fastapi import APIRouter, Depends, HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -37,6 +38,7 @@ if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
router = APIRouter()
_DECODED_JSON: Final = TypeAdapter(object)
class ManagedVectorStoreRow(Protocol):
@ -348,11 +350,11 @@ async def update_vector_store(
raise HTTPException(status_code=500, detail="Database not connected")
try:
update_data = data.model_dump(exclude_unset=True)
vector_store_id: Final[str] = update_data.pop("vector_store_id")
update_data = data.model_dump(exclude_unset=True, exclude={"vector_store_id"})
vector_store_id: Final = data.vector_store_id
if update_data.get("vector_store_metadata") is not None:
update_data["vector_store_metadata"] = safe_dumps(
update_data["vector_store_metadata"]
_DECODED_JSON.validate_python(update_data["vector_store_metadata"])
)
updated = await managed_vector_store_table(prisma_client).update(

15
gateway/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
ARG PGBOUNCER_VERSION=1.26.0
@ -59,7 +59,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
@ -125,7 +125,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN for i in 1 2 3; do \
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic libevent && break; \
apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic libevent && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
@ -140,7 +140,12 @@ ENV HOME=/home/nonroot \
PYTHONUNBUFFERED=1 \
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
COPY --from=builder --chown=nonroot:nonroot /app /app
COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
COPY --from=builder /opt/prisma /opt/prisma
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer

8
litellm-rust/Cargo.lock generated
View file

@ -8202,18 +8202,18 @@ dependencies = [
[[package]]
name = "webpki-root-certs"
version = "1.0.9"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "webpki-roots"
version = "1.0.9"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a"
checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9"
dependencies = [
"rustls-pki-types",
]

2
litellm-rust/Cargo.toml
View file

@ -124,7 +124,7 @@ moka = { version = "0.12.16", features = ["future"] }
strum = { version = "0.28.0", features = ["derive"] }
url = "2.5.8"
percent-encoding = "2.3"
webpki-roots = "1"
webpki-roots = "1.0.10"
time = { version = "0.3.53", features = ["parsing"] }
criterion = "0.8.2"
fancy-regex = "0.19.2"

8
litellm-rust/crates/gateway-inference/src/caching.rs
View file

@ -72,26 +72,26 @@ fn duration(seconds: f64) -> Result<Duration, Error> {
#[derive(Clone, Default)]
pub(crate) struct CacheHeaders(std::sync::Arc<std::sync::OnceLock<String>>);
impl litellm_host::interceptors::Interceptors<litellm_inference::RouteError> for CacheHeaders {
impl litellm_host::interceptors::Interceptors<litellm_inference::Error> for CacheHeaders {
async fn before_provider_request(
&self,
wire: litellm_host::interceptors::WireRequest,
_: litellm_host::interceptors::RequestContext,
) -> Result<litellm_host::interceptors::WireRequest, litellm_inference::RouteError> {
) -> Result<litellm_host::interceptors::WireRequest, litellm_inference::Error> {
Ok(wire)
}
async fn after_provider_response(
&self,
_: litellm_host::interceptors::RawResponse,
) -> Result<(), litellm_inference::RouteError> {
) -> Result<(), litellm_inference::Error> {
Ok(())
}
async fn result_ready(
&self,
facts: litellm_host::interceptors::ExecutionFacts,
) -> Result<(), litellm_inference::RouteError> {
) -> Result<(), litellm_inference::Error> {
if let litellm_host::interceptors::ResultSource::Cache { key } = facts.source {
let _ = self.0.set(key);
}

26
litellm-rust/crates/gateway-inference/src/error.rs
View file

@ -4,7 +4,7 @@ use axum::{
response::{IntoResponse, Response},
};
use litellm_http::transport::Error as TransportError;
use litellm_inference::RouteError;
use litellm_inference::Error as InferenceError;
use litellm_llms::base_llm::ocr::error::Error as OcrError;
use serde_json::{Map, Value, json};
@ -19,7 +19,7 @@ pub enum Error {
)]
UnknownModel(String),
#[error(transparent)]
Route(#[from] RouteError),
Route(#[from] InferenceError),
#[error(transparent)]
Ocr(#[from] OcrError),
#[error("{0} is not implemented by the Rust gateway")]
@ -36,8 +36,8 @@ impl IntoResponse for Error {
}
}
impl From<litellm_host_http::Error<RouteError>> for Error {
fn from(error: litellm_host_http::Error<RouteError>) -> Self {
impl From<litellm_host_http::Error<InferenceError>> for Error {
fn from(error: litellm_host_http::Error<InferenceError>) -> Self {
match error {
litellm_host_http::Error::Call(error) => Self::Route(error),
litellm_host_http::Error::Protocol => Self::Internal(error.to_string()),
@ -50,7 +50,7 @@ impl Error {
match self {
Self::Auth(error) => error.status(),
Self::Unsupported(_)
| Self::Route(RouteError::Unsupported(_))
| Self::Route(InferenceError::Unsupported(_))
| Self::Ocr(OcrError::Unsupported(_)) => StatusCode::NOT_IMPLEMENTED,
Self::BodyTooLarge => StatusCode::PAYLOAD_TOO_LARGE,
Self::Ocr(
@ -64,10 +64,10 @@ impl Error {
.and_then(|status| StatusCode::from_u16(status).ok())
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
Self::InvalidBody(_) | Self::UnknownModel(_) => StatusCode::BAD_REQUEST,
Self::Route(RouteError::Transport(TransportError::Http { status, .. })) => {
Self::Route(InferenceError::Transport(TransportError::Http { status, .. })) => {
StatusCode::from_u16(*status).unwrap_or(StatusCode::BAD_GATEWAY)
}
Self::Route(RouteError::Auth(litellm_auth::Error::MissingApiKey { .. })) => {
Self::Route(InferenceError::Auth(litellm_auth::Error::MissingApiKey { .. })) => {
StatusCode::UNAUTHORIZED
}
Self::Route(error) if error.is_request() => StatusCode::BAD_REQUEST,
@ -97,7 +97,9 @@ impl Error {
/// body already in that shape passes through, any other has its message extracted.
pub fn body(&self, request_id: Option<&str>) -> Value {
let raw = match self {
Self::Route(RouteError::Transport(TransportError::Http { body, .. })) => body.clone(),
Self::Route(InferenceError::Transport(TransportError::Http { body, .. })) => {
body.clone()
}
other => other.to_string(),
};
let parsed = serde_json::from_str::<Value>(&raw).ok();
@ -175,7 +177,7 @@ mod tests {
use super::*;
fn upstream(status: u16, body: &str) -> Error {
Error::Route(RouteError::Transport(TransportError::Http {
Error::Route(InferenceError::Transport(TransportError::Http {
status,
body: body.into(),
}))
@ -225,16 +227,16 @@ mod tests {
#[rstest]
#[case::upstream_status(upstream(429, ""), StatusCode::TOO_MANY_REQUESTS)]
#[case::rejected_request(Error::Route(RouteError::InvalidRequest("top_k".into())), StatusCode::BAD_REQUEST)]
#[case::rejected_request(Error::Route(InferenceError::InvalidRequest("top_k".into())), StatusCode::BAD_REQUEST)]
#[case::missing_key(
Error::Route(RouteError::Auth(litellm_auth::Error::MissingApiKey {
Error::Route(InferenceError::Auth(litellm_auth::Error::MissingApiKey {
provider: "Anthropic",
environment_variable: "ANTHROPIC_API_KEY",
})),
StatusCode::UNAUTHORIZED,
)]
#[case::lost_connection(
Error::Route(RouteError::Transport(TransportError::Network("reset".into()))),
Error::Route(InferenceError::Transport(TransportError::Network("reset".into()))),
StatusCode::INTERNAL_SERVER_ERROR,
)]
fn status_follows_who_is_at_fault(#[case] error: Error, #[case] status: StatusCode) {

2
litellm-rust/crates/inference-chat/src/lib.rs
View file

@ -1,7 +1,7 @@
use litellm_host::observation::ObservationSender;
pub mod route;
pub mod types;
pub use litellm_inference::RouteError as Error;
pub use litellm_inference::Error;
mod common_utils;
pub mod constants;
pub(crate) mod handler;

10
litellm-rust/crates/inference-chat/tests/caching.rs
View file

@ -21,7 +21,7 @@ use litellm_host::{
lifecycle::{CallEvent, ExecutionEvent},
observation::observation_channel,
};
use litellm_inference::RouteError;
use litellm_inference::Error;
use rstest::{fixture, rstest};
use serde_json::{Value, json};
@ -166,12 +166,12 @@ struct ChangingHooks {
facts: std::sync::Mutex<Vec<ExecutionFacts>>,
}
impl Interceptors<RouteError> for ChangingHooks {
impl Interceptors<Error> for ChangingHooks {
async fn before_provider_request(
&self,
mut wire: WireRequest,
_: RequestContext,
) -> Result<WireRequest, RouteError> {
) -> Result<WireRequest, Error> {
let call = self.calls.fetch_add(1, Ordering::SeqCst);
if self.rewrite {
wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 });
@ -179,11 +179,11 @@ impl Interceptors<RouteError> for ChangingHooks {
Ok(wire)
}
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
Ok(())
}
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
self.facts.lock().unwrap().push(facts);
Ok(())
}

2
litellm-rust/crates/inference-messages/src/lib.rs
View file

@ -14,7 +14,7 @@ use std::sync::Arc;
use crate::common_utils::messages_provider;
pub use litellm_inference::RouteError as Error;
pub use litellm_inference::Error;
pub use types::{
MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params,
messages_body,

10
litellm-rust/crates/inference-messages/tests/caching.rs
View file

@ -17,7 +17,7 @@ use litellm_host::interceptors::{
ResultSource, WireRequest,
};
use litellm_inference::{
RouteError,
Error,
caching::{CacheRequest, execute_unary},
};
use rstest::{fixture, rstest};
@ -133,12 +133,12 @@ struct ChangingHooks {
facts: std::sync::Mutex<Vec<ExecutionFacts>>,
}
impl Interceptors<RouteError> for ChangingHooks {
impl Interceptors<Error> for ChangingHooks {
async fn before_provider_request(
&self,
mut wire: WireRequest,
_: RequestContext,
) -> Result<WireRequest, RouteError> {
) -> Result<WireRequest, Error> {
let call = self.calls.fetch_add(1, Ordering::SeqCst);
if self.rewrite {
wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 });
@ -146,11 +146,11 @@ impl Interceptors<RouteError> for ChangingHooks {
Ok(wire)
}
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
Ok(())
}
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
self.facts.lock().unwrap().push(facts);
Ok(())
}

2
litellm-rust/crates/inference-ocr/AGENTS.md
View file

@ -3,4 +3,4 @@
- The route returns its completed response directly
- Route closures supply OCR-specific host capabilities, such as the caller's Azure AD token provider (`src/route.rs`)
- Provider code reaches the caller's hooks mid-call only through `litellm_llms::base_llm::ocr::handler::CallHooks`, which this crate implements over its host until it folds into `litellm_host::interceptors::Interceptors`
- OCR failures use `base_llm/ocr/error.rs`, the recorded exception to `litellm_inference::RouteError` until OCR folds into `litellm_llms::Error`
- OCR failures use `base_llm/ocr/error.rs`, the recorded exception to `litellm_inference::Error` until OCR folds into `litellm_llms::Error`

2
litellm-rust/crates/inference-ocr/src/provider_config.rs
View file

@ -249,7 +249,7 @@ pub(crate) fn resolve_provider_config<'a>(
provider: LlmProviders::Mistral,
}
}
Err(litellm_inference::RouteError::InvalidProvider(provider)) => {
Err(litellm_inference::Error::InvalidProvider(provider)) => {
return Err(Error::InvalidProvider(provider));
}
Err(error) => return Err(Error::InvalidProvider(error.to_string())),

2
litellm-rust/crates/inference-responses/src/lib.rs
View file

@ -1,5 +1,5 @@
use litellm_host::observation::ObservationSender;
pub use litellm_inference::RouteError as Error;
pub use litellm_inference::Error;
pub mod ids;
pub mod route;

10
litellm-rust/crates/inference-responses/tests/caching.rs
View file

@ -18,7 +18,7 @@ use litellm_host::interceptors::{
WireRequest,
};
use litellm_inference::{
RouteError,
Error,
caching::{CacheRequest, execute_unary},
};
use rstest::{fixture, rstest};
@ -124,12 +124,12 @@ struct ChangingHooks {
facts: std::sync::Mutex<Vec<ExecutionFacts>>,
}
impl Interceptors<RouteError> for ChangingHooks {
impl Interceptors<Error> for ChangingHooks {
async fn before_provider_request(
&self,
mut wire: WireRequest,
_: RequestContext,
) -> Result<WireRequest, RouteError> {
) -> Result<WireRequest, Error> {
let call = self.calls.fetch_add(1, Ordering::SeqCst);
if self.rewrite {
wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 });
@ -137,11 +137,11 @@ impl Interceptors<RouteError> for ChangingHooks {
Ok(wire)
}
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
Ok(())
}
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
self.facts.lock().unwrap().push(facts);
Ok(())
}

2
litellm-rust/crates/inference-transcription/src/lib.rs
View file

@ -1,5 +1,5 @@
pub mod types;
pub use litellm_inference::RouteError as Error;
pub use litellm_inference::Error;
mod constants;
mod handler;

6
litellm-rust/crates/inference/AGENTS.md
View file

@ -2,7 +2,7 @@
- `gateway-inference` (serving) -> `inference-<fmt>` (one API's call) -> `inference` (shared base)
- This crate is the shared base every format crate builds on
- `RouteError` (`src/error.rs`), `CallOptions` (`src/lib.rs`), `CallContext` (`src/context.rs`)
- `Error` (`src/error.rs`), `CallOptions` (`src/lib.rs`), `CallContext` (`src/context.rs`)
- diagnostic spans (`src/diagnostic.rs`), outbound send and signing (`src/outbound.rs`), provider resolution (`src/provider.rs`), `CoreResources` (`src/resources.rs`)
- response caching (`src/caching.rs`): `Cachable`, `StreamCachable`, `CacheRequest`, `CallCache`, `execute_unary`, `execute_streaming`, stream capture
- Shared test helpers belong in litellm-inference-testing, used only as a dev dependency
@ -49,14 +49,14 @@
- `litellm-inference-<fmt>` mirrors the route packages (`litellm/ocr/`, `litellm/messages/`, ...)
- Provider code never imports from `inference` or `inference-*`
- Handlers belong in `inference-*` or `llms`, never in a host crate
- Import every item from its canonical path. Never re-export another crate's items or give an item a second public path. The only allowed re-exports are a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`) and each format crate's `pub use litellm_inference::RouteError as Error;`, which keeps the pre-split `<fmt>::Error` name
- Import every item from its canonical path. Never re-export another crate's items or give an item a second public path. The only allowed re-exports are a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`) and each format crate's `pub use litellm_inference::Error;`, which keeps the pre-split `<fmt>::Error` name
## Error placement
- The workspace `Error definitions` rules shape each crate's error. This section decides which crate and module a failure belongs to
- A failure is declared once, by the lowest crate that raises it
- every crate above nests it unchanged (`#[error(transparent)] Auth(#[from] litellm_auth::Error)`) or maps it once at its boundary, as `src/error.rs` does for `litellm_llms::Error`
- `RouteError` collects route failures for every format crate and never re-declares a variant a lower crate raises
- `Error` collects route failures for every format crate and never re-declares a variant a lower crate raises
- Scope follows the concept, not the first caller
- an error type under `litellm-llms`'s `<provider>/` is private to that provider: no other provider and nothing in `base_llm` may import it
- a failure two providers or two routes can hit (wire framing, stream event decoding, a malformed provider response) belongs to the crate that owns the concept: `litellm-framer` for framing, `litellm_llms::Error` for the transformation layer

26
litellm-rust/crates/inference/src/caching.rs
View file

@ -24,9 +24,9 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned};
use serde_json::Value;
use tokio_util::codec::Decoder;
use crate::RouteError;
use crate::Error;
pub trait Cachable: Protocol<Error = RouteError> {
pub trait Cachable: Protocol<Error = Error> {
const SURFACE: &'static str;
fn reusable(_response: &Self::Response) -> bool {
@ -145,15 +145,15 @@ pub async fn execute_unary<P, F, Fut>(
request: CacheRequest,
cache: Option<Arc<dyn ResponseCacheService>>,
options: Option<CacheOptions>,
interceptors: &impl Interceptors<RouteError>,
interceptors: &impl Interceptors<Error>,
observers: Option<&ObservationSender>,
provider: F,
) -> Result<P::Response, RouteError>
) -> Result<P::Response, Error>
where
P: Cachable,
P::Response: Serialize + DeserializeOwned,
F: FnOnce() -> Fut,
Fut: Future<Output = Result<P::Response, RouteError>>,
Fut: Future<Output = Result<P::Response, Error>>,
{
let identity = request.identity.clone();
crate::diagnostic::provider(&identity.model, &identity.provider);
@ -191,15 +191,15 @@ pub async fn execute_streaming<P, F, Fut>(
request: CacheRequest,
cache: Option<Arc<dyn ResponseCacheService>>,
options: Option<CacheOptions>,
interceptors: &impl Interceptors<RouteError>,
interceptors: &impl Interceptors<Error>,
observers: Option<&ObservationSender>,
provider: F,
) -> Result<OutputOf<P>, RouteError>
) -> Result<OutputOf<P>, Error>
where
P: StreamCachable,
P::Response: Serialize + DeserializeOwned,
F: FnOnce() -> Fut,
Fut: Future<Output = Result<OutputOf<P>, RouteError>>,
Fut: Future<Output = Result<OutputOf<P>, Error>>,
{
let identity = request.identity.clone();
crate::diagnostic::provider(&identity.model, &identity.provider);
@ -299,9 +299,9 @@ impl<P: StreamCachable> CallCache<P> {
}
fn capture_stream<P: StreamCachable>(
chunks: futures_util::stream::BoxStream<'static, Result<P::Chunk, RouteError>>,
chunks: futures_util::stream::BoxStream<'static, Result<P::Chunk, Error>>,
session: CacheSession,
) -> futures_util::stream::BoxStream<'static, Result<P::Chunk, RouteError>> {
) -> futures_util::stream::BoxStream<'static, Result<P::Chunk, Error>> {
stream::try_unfold(
(chunks, Some(Vec::<u8>::new()), session),
|(mut chunks, captured, session)| async move {
@ -330,7 +330,7 @@ fn capture_stream<P: StreamCachable>(
{
session.store(entry).await;
}
Ok::<_, RouteError>(None)
Ok::<_, Error>(None)
}
}
},
@ -369,9 +369,9 @@ fn successful_stream(text: &str, terminal: &str) -> bool {
async fn publish(
facts: ExecutionFacts,
interceptors: &impl Interceptors<RouteError>,
interceptors: &impl Interceptors<Error>,
observers: Option<&ObservationSender>,
) -> Result<(), RouteError> {
) -> Result<(), Error> {
if let Some(observers) = observers {
observers.emit(CallEvent::Execution(ExecutionEvent::ResultReady {
facts: facts.clone(),

10
litellm-rust/crates/inference/src/context.rs
View file

@ -5,7 +5,7 @@ use litellm_host::{
observation::ObservationSender,
};
use crate::{CallOptions, RouteError};
use crate::{CallOptions, Error};
pub struct CallContext<'a, I> {
pub interceptors: &'a I,
@ -13,7 +13,7 @@ pub struct CallContext<'a, I> {
pub cache: CachePolicy,
}
impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
impl<'a, I: Interceptors<Error>> CallContext<'a, I> {
pub fn new(interceptors: &'a I, options: CallOptions) -> Self {
Self {
interceptors,
@ -22,7 +22,7 @@ impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
}
}
pub async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
pub async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
if let Some(observers) = &self.observers {
observers.emit(CallEvent::Execution(ExecutionEvent::ResultReady {
facts: facts.clone(),
@ -31,7 +31,7 @@ impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
self.interceptors.result_ready(facts).await
}
pub async fn response_received(&self, body: &str) -> Result<(), RouteError> {
pub async fn response_received(&self, body: &str) -> Result<(), Error> {
let raw = RawResponse {
body: body.to_owned(),
};
@ -43,6 +43,6 @@ impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
self.interceptors
.after_provider_response(raw)
.await
.map_err(RouteError::post_call)
.map_err(Error::post_call)
}
}

24
litellm-rust/crates/inference/src/error.rs
View file

@ -11,7 +11,7 @@ use litellm_http::transport::Error as TransportError;
use litellm_llms::Error as LlmError;
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum RouteError {
pub enum Error {
#[error("expected {expected}, got {actual}")]
InvalidType {
expected: &'static str,
@ -38,10 +38,10 @@ pub enum RouteError {
#[error(transparent)]
Secret(#[from] SecretError),
#[error("post-call hook failed: {0}")]
PostCallHook(#[source] Arc<RouteError>),
PostCallHook(#[source] Arc<Error>),
}
impl From<litellm_host::machine::MachineFault> for RouteError {
impl From<litellm_host::machine::MachineFault> for Error {
fn from(fault: litellm_host::machine::MachineFault) -> Self {
use litellm_host::machine::MachineFault;
Self::InvalidRequest(match fault {
@ -51,7 +51,7 @@ impl From<litellm_host::machine::MachineFault> for RouteError {
}
}
impl RouteError {
impl Error {
pub fn post_call(error: Self) -> Self {
Self::PostCallHook(Arc::new(error))
}
@ -76,7 +76,7 @@ impl RouteError {
}
}
impl From<LlmError> for RouteError {
impl From<LlmError> for Error {
fn from(error: LlmError) -> Self {
match error {
LlmError::InvalidType { expected, actual } => Self::InvalidType { expected, actual },
@ -99,7 +99,7 @@ impl SecretError {
}
}
impl From<litellm_secrets::Error> for RouteError {
impl From<litellm_secrets::Error> for Error {
fn from(error: litellm_secrets::Error) -> Self {
Self::Secret(SecretError(Arc::new(error)))
}
@ -115,22 +115,22 @@ impl Eq for SecretError {}
#[cfg(test)]
mod tests {
use super::RouteError;
use super::Error;
use litellm_llms::{Error as LlmError, ErrorDetail};
use rstest::rstest;
#[test]
fn a_missing_api_key_is_the_environment_not_the_request() {
assert!(
!RouteError::Auth(litellm_auth::Error::MissingApiKey {
!Error::Auth(litellm_auth::Error::MissingApiKey {
provider: "Anthropic",
environment_variable: "ANTHROPIC_API_KEY",
})
.is_request()
);
assert!(RouteError::Auth(litellm_auth::Error::InvalidHeader).is_request());
assert!(RouteError::InvalidRequest("top_k".into()).is_request());
assert!(!RouteError::InvalidResponse("bad json".into()).is_request());
assert!(Error::Auth(litellm_auth::Error::InvalidHeader).is_request());
assert!(Error::InvalidRequest("top_k".into()).is_request());
assert!(!Error::InvalidResponse("bad json".into()).is_request());
}
#[rstest]
#[case::request(true)]
@ -139,7 +139,7 @@ mod tests {
let source = serde_json::from_str::<serde_json::Value>("{").unwrap_err();
let source_message = source.to_string();
let detail = ErrorDetail::invalid("test payload", source);
let error = RouteError::from(if request {
let error = Error::from(if request {
LlmError::InvalidRequest(detail)
} else {
LlmError::InvalidResponse(detail)

2
litellm-rust/crates/inference/src/lib.rs
View file

@ -7,7 +7,7 @@ pub mod outbound;
pub mod provider;
pub mod resources;
pub use error::RouteError;
pub use error::Error;
#[derive(Clone, Default)]
pub struct CallOptions {

2
litellm-rust/crates/inference/src/provider.rs
View file

@ -2,7 +2,7 @@ use litellm_core_utils::get_llm_provider_logic::{
CustomLlmProvider, LlmProviders, get_custom_llm_provider,
};
use crate::error::RouteError as Error;
use crate::error::Error;
#[derive(Debug)]
pub struct ResolvedProvider<'a> {

24
litellm-rust/crates/inference/tests/caching.rs
View file

@ -25,7 +25,7 @@ use litellm_host::{
protocol::Protocol,
};
use litellm_inference::{
RouteError,
Error,
caching::{Cachable, CacheRequest, StreamCachable, execute_streaming, execute_unary},
};
use rstest::{fixture, rstest};
@ -36,7 +36,7 @@ struct TestRoute;
impl Protocol for TestRoute {
type Request = Value;
type Response = Value;
type Error = RouteError;
type Error = Error;
type HostCall = Infallible;
type Chunk = Bytes;
type StreamHead = ();
@ -204,7 +204,7 @@ async fn streamed(
.chunks(3)
.map(|bytes| Ok(Bytes::copy_from_slice(bytes)))
.collect::<Vec<_>>();
let ending = fail.then_some(Err(RouteError::Unsupported("test transport failure")));
let ending = fail.then_some(Err(Error::Unsupported("test transport failure")));
Ok(CallOutput::Stream {
head: (),
chunks: stream::iter(chunks.into_iter().chain(ending)).boxed(),
@ -215,7 +215,7 @@ async fn streamed(
.unwrap()
}
async fn consume(output: OutputOf<TestRoute>) -> Result<Vec<u8>, RouteError> {
async fn consume(output: OutputOf<TestRoute>) -> Result<Vec<u8>, Error> {
let CallOutput::Stream { chunks, .. } = output else {
panic!("expected a stream");
};
@ -315,7 +315,7 @@ async fn a_provider_failure_never_populates_the_cache(cache: Arc<dyn ResponseCac
None,
|| async {
calls.fetch_add(1, Ordering::SeqCst);
Err(RouteError::Unsupported("test provider failure"))
Err(Error::Unsupported("test provider failure"))
},
)
.await;
@ -467,8 +467,8 @@ struct CacheHitAccounting {
reject: bool,
}
impl Interceptors<RouteError> for CacheHitAccounting {
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
impl Interceptors<Error> for CacheHitAccounting {
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
self.calls.fetch_add(1, Ordering::SeqCst);
let ResultSource::Cache { key } = facts.source else {
panic!("expected cache source")
@ -482,7 +482,7 @@ impl Interceptors<RouteError> for CacheHitAccounting {
);
self.key.set(key).unwrap();
if self.reject {
return Err(RouteError::Unsupported("cache accounting rejected"));
return Err(Error::Unsupported("cache accounting rejected"));
}
Ok(())
}
@ -491,11 +491,11 @@ impl Interceptors<RouteError> for CacheHitAccounting {
&self,
wire: WireRequest,
_: RequestContext,
) -> Result<WireRequest, RouteError> {
) -> Result<WireRequest, Error> {
Ok(wire)
}
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
Ok(())
}
}
@ -571,7 +571,7 @@ async fn cache_hits_notify_accounting_once_and_propagate_its_failure(
if reject {
assert!(matches!(
result,
Err(RouteError::Unsupported("cache accounting rejected"))
Err(Error::Unsupported("cache accounting rejected"))
));
} else {
assert_eq!(result.unwrap(), expected);
@ -590,7 +590,7 @@ async fn cache_hits_notify_accounting_once_and_propagate_its_failure(
impl Protocol for UnaryTestRoute {
type Request = Value;
type Response = Value;
type Error = RouteError;
type Error = Error;
type HostCall = Infallible;
type Chunk = Infallible;
type StreamHead = Infallible;

6
litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs
View file

@ -235,7 +235,10 @@ impl AnthropicBeta {
BetaProvider::Databricks,
],
Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
Self::ComputerUse20250124
| Self::ComputerUse20251124
| Self::Context1m20250807
| Self::ThinkingBindingControls20260801 => &[],
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
Self::Effort20251124 => &[BetaProvider::VertexAi],
Self::FastMode20260201 | Self::Oauth20250420 => &[
@ -272,7 +275,6 @@ impl AnthropicBeta {
BetaProvider::Databricks,
],
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
Self::ThinkingBindingControls20260801 => &[BetaProvider::AzureAi],
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
Self::ToolSearchTool20251019 => &[
BetaProvider::Anthropic,

28
litellm-rust/crates/python-bridge/src/errors.rs
View file

@ -1,5 +1,5 @@
use litellm_http::transport::Error as TransportError;
use litellm_inference::RouteError;
use litellm_inference::Error;
use pyo3::{
exceptions::{PyRuntimeError, PyValueError},
prelude::*,
@ -19,9 +19,9 @@ pyo3::create_exception!(
"The provider call was already issued and failed. Args are (status, message); status is 0 when there was no HTTP response."
);
pub(crate) fn route_error_to_pyerr(error: RouteError) -> PyErr {
pub(crate) fn route_error_to_pyerr(error: Error) -> PyErr {
match error {
RouteError::Transport(TransportError::Http { status, body }) => {
Error::Transport(TransportError::Http { status, body }) => {
RustUpstreamError::new_err((status, body))
}
other => by_fault(other.is_request(), other.to_string()),
@ -42,13 +42,13 @@ mod tests {
use super::*;
#[rstest::rstest]
#[case::unsupported(RouteError::Unsupported("test capability"), true)]
#[case::invalid_provider(RouteError::InvalidProvider("unknown".into()), true)]
#[case::invalid_request(RouteError::InvalidRequest("empty messages".into()), true)]
#[case::unsupported(Error::Unsupported("test capability"), true)]
#[case::invalid_provider(Error::InvalidProvider("unknown".into()), true)]
#[case::invalid_request(Error::InvalidRequest("empty messages".into()), true)]
#[case::connection(TransportError::Connect("unreachable".into()).into(), false)]
#[case::network(TransportError::Network("timed out".into()).into(), false)]
#[case::invalid_response(RouteError::InvalidResponse("missing usage".into()), false)]
fn route_failures_are_terminal(#[case] error: RouteError, #[case] is_request: bool) {
#[case::invalid_response(Error::InvalidResponse("missing usage".into()), false)]
fn route_failures_are_terminal(#[case] error: Error, #[case] is_request: bool) {
Python::initialize();
Python::attach(|py| {
let failure = route_error_to_pyerr(error);
@ -86,14 +86,12 @@ mod tests {
fn missing_api_key_stays_a_runtime_error_while_other_auth_failures_are_value_errors() {
Python::initialize();
Python::attach(|py| {
let missing =
route_error_to_pyerr(RouteError::Auth(litellm_auth::Error::MissingApiKey {
provider: "Anthropic",
environment_variable: "ANTHROPIC_API_KEY",
}));
let missing = route_error_to_pyerr(Error::Auth(litellm_auth::Error::MissingApiKey {
provider: "Anthropic",
environment_variable: "ANTHROPIC_API_KEY",
}));
assert!(missing.is_instance_of::<PyRuntimeError>(py));
let invalid =
route_error_to_pyerr(RouteError::Auth(litellm_auth::Error::InvalidHeader));
let invalid = route_error_to_pyerr(Error::Auth(litellm_auth::Error::InvalidHeader));
assert!(invalid.is_instance_of::<PyValueError>(py));
});
}

10
litellm-rust/crates/python-bridge/src/routes/inference.rs
View file

@ -7,7 +7,7 @@ use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
use litellm_host::{call::HostedMachine, protocol::Protocol};
use litellm_host_python::{PythonBinding, PythonHostCalls, from_py};
use litellm_http::transport::Error as TransportError;
use litellm_inference::RouteError;
use litellm_inference::Error;
use litellm_secrets::source::SecretSource;
use pyo3::{
exceptions::PyValueError,
@ -119,14 +119,14 @@ impl InferenceHost {
public_response(py, self.module, response)
}
pub fn error(&self, py: Python<'_>, error: RouteError) -> PyResult<PyErr> {
if let RouteError::Secret(source) = &error
pub fn error(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
if let Error::Secret(source) = &error
&& let Some(original) = crate::secrets::python_error(py, source.source_error())
{
return Ok(original);
}
let native = match error {
RouteError::Transport(TransportError::Http { status, body }) => {
Error::Transport(TransportError::Http { status, body }) => {
let error = RustUpstreamError::new_err((status, body));
error
.value(py)
@ -144,7 +144,7 @@ impl InferenceHost {
}
pub(super) trait InferenceRoute: Sized + 'static {
type Protocol: Protocol<Error = RouteError>;
type Protocol: Protocol<Error = Error>;
const OPERATION: LoggingOperation;
const SYNC_CALL_TYPE: &'static str;
const ASYNC_CALL_TYPE: &'static str;

21
litellm/__init__.py
View file

@ -175,7 +175,7 @@ argilla_batch_size: Optional[int] = None
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload.
gcs_pub_sub_use_v1: Optional[bool] = False # if you want to use v1 gcs pubsub logged payload
generic_api_use_v1: Optional[bool] = False # if you want to use v1 generic api logged payload
argilla_transformation_object: Optional[Dict[str, Any]] = None
argilla_transformation_object: Mapping[str, str] | None = None
_async_input_callback: List[Union[str, Callable[..., object], "CustomLogger"]] = ( # CustomLogger is lazy-loaded
[]
) # internal variable - async custom callbacks are routed here.
@ -333,7 +333,7 @@ in_memory_llm_clients_cache: "LLMClientCache"
safe_memory_mode: bool = False
enable_azure_ad_token_refresh: Optional[bool] = False
# Proxy Authentication - auto-obtain/refresh OAuth2/JWT tokens for LiteLLM Proxy
proxy_auth: Optional[Any] = None
proxy_auth: _ProxyAuthHandlerType | None = None
### DEFAULT AZURE API VERSION ###
AZURE_DEFAULT_API_VERSION = "2025-02-01-preview" # this is updated to the latest
### DEFAULT WATSONX API VERSION ###
@ -503,7 +503,7 @@ skill_search_embedding_model: Optional[str] = None
# Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]])
# New format: { "displayName": { "url": "...", "index": 0 } }
# Old format: { "displayName": "url" } (for backward compatibility)
public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {}
public_model_groups_links: dict[str, str | dict[str, object]] = {}
#### REQUEST PRIORITIZATION #######
priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]]] = None
# priority_reservation_settings is lazy-loaded via __getattr__
@ -682,6 +682,7 @@ darkbloom_models: Set = set()
openinfer_models: Final[Set[str]] = set()
v0_models: Set = set()
morph_models: Set = set()
opencode_go_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads
lambda_ai_models: Set = set()
inception_models: Set = set()
hyperbolic_models: Set = set()
@ -970,6 +971,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
v0_models.add(key)
elif value.get("litellm_provider") == "morph":
morph_models.add(key)
elif value.get("litellm_provider") == "opencode_go":
opencode_go_models.add(key)
elif value.get("litellm_provider") == "lambda_ai":
lambda_ai_models.add(key)
elif value.get("litellm_provider") == "inception":
@ -1137,6 +1140,7 @@ model_list = list(
| openinfer_models
| v0_models
| morph_models
| opencode_go_models
| lambda_ai_models
| inception_models
| black_forest_labs_models
@ -1254,6 +1258,7 @@ def _build_models_by_provider() -> dict:
"openinfer": openinfer_models,
"v0": v0_models,
"morph": morph_models,
"opencode_go": opencode_go_models,
"lambda_ai": lambda_ai_models,
"inception": inception_models,
"hyperbolic": hyperbolic_models,
@ -2289,6 +2294,10 @@ if TYPE_CHECKING:
LiteLLM_UpperboundKeyGenerateParams,
)
from litellm.utils import ModelResponseListIterator as _ModelResponseListIterator
from collections.abc import Awaitable
from litellm.types.utils import ModelResponse as _ModelResponseType
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper as _CustomStreamWrapperType
from litellm.proxy_auth import ProxyAuthHandler as _ProxyAuthHandlerType
# Cost calculator functions
cost_per_token: Callable[..., Tuple[float, float]]
@ -2312,7 +2321,7 @@ if TYPE_CHECKING:
supports_audio_output: Callable[..., bool]
supports_system_messages: Callable[..., bool]
supports_reasoning: Callable[..., bool]
acreate: Callable[..., Any]
acreate: Callable[..., Awaitable[_ModelResponseType | _CustomStreamWrapperType]]
get_max_tokens: Callable[..., int]
get_model_info: Callable[..., _ModelInfoType]
register_prompt_template: Callable[..., None]
@ -2610,8 +2619,10 @@ def __getattr__(name: str) -> Any:
cancel_run,
delete_run,
)
from types import MappingProxyType
return locals()[name]
evals_exports: Final[Mapping[str, object]] = MappingProxyType(locals())
return evals_exports[name]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

4
litellm/_lazy_imports.py
View file

@ -19,7 +19,7 @@ import importlib
import sys
from collections.abc import Callable, Mapping
from types import ModuleType
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final
from typing_extensions import ReadOnly, TypedDict
@ -456,7 +456,7 @@ def _lazy_import_http_handlers(name: str) -> object:
params: Final = {"timeout": async_timeout, "client_alias": "module level aclient"}
# Create the client instance
provider_id: Final = cast(Any, "litellm_module_level_client")
provider_id: Final = "litellm_module_level_client"
async_client: Final = get_async_httpx_client(
llm_provider=provider_id,
params=params,

20
litellm/a2a_protocol/cost_calculator.py
View file

@ -5,7 +5,10 @@ Supports dynamic cost parameters that allow platform owners
to define custom costs per agent query or per token.
"""
from typing import TYPE_CHECKING, Any, Final
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, SupportsFloat, SupportsIndex
from typing_extensions import Buffer
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import (
@ -15,6 +18,15 @@ else:
LitellmLoggingObject = Any
def _custom_response_cost(model_call_details: Mapping[str, object]) -> float | None:
response_cost: Final = model_call_details.get("response_cost", None)
if response_cost is None:
return None
if isinstance(response_cost, (str, Buffer, SupportsFloat, SupportsIndex)):
return float(response_cost)
raise TypeError(f"float() argument must be a string or a real number, not {type(response_cost).__name__!r}")
class A2ACostCalculator:
@staticmethod
def calculate_a2a_cost(
@ -45,9 +57,9 @@ class A2ACostCalculator:
model_call_details: Final = litellm_logging_obj.model_call_details
# Check if user set a custom response cost (backward compatibility)
response_cost: Final = model_call_details.get("response_cost", None)
if response_cost is not None:
return float(response_cost)
custom_response_cost: Final = _custom_response_cost(model_call_details)
if custom_response_cost is not None:
return custom_response_cost
# Get litellm_params for cost parameters
litellm_params: Final = model_call_details.get("litellm_params", {}) or {}

27
litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py
View file

@ -84,14 +84,23 @@ def _filter_reserved_headers(
return filtered or None
def _calling_key_hash(litellm_params: Mapping[str, object]) -> str | None:
principal: Final = litellm_params.get(A2A_USER_API_KEY_HASH_PARAM)
if principal is None or isinstance(principal, str):
return principal
if not principal:
return None
raise AttributeError(f"{type(principal).__name__!r} object has no attribute 'encode'", name="encode", obj=principal)
def _request_scoped_runtime_session_id(
params: Mapping[str, object],
litellm_params: Mapping[str, Any],
litellm_params: Mapping[str, object],
) -> str | None:
context_id: Final = get_session_id_from_a2a_params(params)
if not isinstance(context_id, str) or not context_id:
return None
return scope_session_to_principal(context_id, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM))
return scope_session_to_principal(context_id, _calling_key_hash(litellm_params))
def _validate_runtime_session_id(session_id: str, model: str) -> str:
@ -109,6 +118,13 @@ def _validate_runtime_session_id(session_id: str, model: str) -> str:
)
def _agentcore_model_name(litellm_params: Mapping[str, object]) -> str:
model: Final = litellm_params.get("model", "")
if isinstance(model, str):
return model
raise AttributeError(f"{type(model).__name__!r} object has no attribute 'startswith'", name="startswith", obj=model)
class BedrockAgentCoreA2ATransformation:
"""
Request/response transformation for Bedrock AgentCore A2A agents.
@ -152,11 +168,8 @@ class BedrockAgentCoreA2ATransformation:
"""
# Extract model and strip the "bedrock/" prefix
# "bedrock/agentcore/arn:aws:..." → "agentcore/arn:aws:..."
model: Final = litellm_params.get("model", "")
if model.startswith("bedrock/"):
agentcore_model = model[len("bedrock/") :]
else:
agentcore_model = model
model: Final = _agentcore_model_name(litellm_params)
agentcore_model: Final = model.removeprefix("bedrock/")
# Build optional_params from litellm_params (everything except model and custom_llm_provider)
optional_params: Final = {k: v for k, v in litellm_params.items() if k not in ("model", "custom_llm_provider")}

40
litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py
View file

@ -8,11 +8,31 @@ WXO uses a REST API (not A2A/JSON-RPC) with an async-poll execution model:
"""
import asyncio
from collections.abc import AsyncIterator, Mapping
from typing import Any, Final
from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
from itertools import chain
from typing import Final
from uuid import uuid4
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
from litellm.llms.openai.common_utils import decoded_json_object
def _a2a_text_part(part: object) -> Sequence[object]:
part_fields: Final = as_str_mapping(part)
if not isinstance(part_fields, dict):
return ()
if part_fields.get("kind") in (None, "", "text") and part_fields.get("text"):
return (part_fields["text"],)
return ()
def _join_with_spaces(texts: Sequence[object]) -> str:
strings: Final = [text for text in texts if isinstance(text, str)]
if len(strings) == len(texts):
return " ".join(strings)
first_bad: Final = next(position for position, text in enumerate(texts) if not isinstance(text, str))
raise TypeError(f"sequence item {first_bad}: expected str instance, {type(texts[first_bad]).__name__} found")
class WatsonxOrchestrateTransformation:
@ -29,22 +49,16 @@ class WatsonxOrchestrateTransformation:
return f"{cp4d_host.rstrip('/')}/orchestrate/cpd/instances/{instance_id}"
@staticmethod
def extract_text_from_a2a_params(params: dict[str, Any]) -> str:
def extract_text_from_a2a_params(params: Mapping[str, object]) -> str:
"""
Extract user message text from A2A MessageSendParams.
A2A format: params.message.parts[*] where part.kind == "text"
"""
message: Final = params.get("message", {})
parts: Final = message.get("parts", [])
texts: Final = []
for part in parts:
if not isinstance(part, dict):
continue
kind = part.get("kind")
if kind in (None, "", "text") and part.get("text"):
texts.append(part["text"])
return " ".join(texts) or ""
message: Final = decoded_json_object(params.get("message", {}))
parts: Final[Iterable[object]] = iter_items(message.get("parts", []))
texts: Final = list(chain.from_iterable(map(_a2a_text_part, parts)))
return _join_with_spaces(texts) or ""
@staticmethod
def build_wxo_run_body(

21
litellm/a2a_protocol/utils.py
View file

@ -4,15 +4,20 @@ Utility functions for A2A protocol.
import hashlib
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final, Protocol
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import has_attributes
if TYPE_CHECKING:
from a2a.types import SendMessageRequest, SendStreamingMessageRequest
class _SupportsModelDump(Protocol):
def model_dump(self, *, mode: str) -> object: ...
class A2ARequestUtils:
"""Utility class for A2A request/response processing."""
@ -74,7 +79,7 @@ class A2ARequestUtils:
@staticmethod
def get_input_message_from_request(
request: "SendMessageRequest | SendStreamingMessageRequest",
) -> Any:
) -> object:
"""
Extract the input message from an A2A request.
@ -84,7 +89,7 @@ class A2ARequestUtils:
Returns:
The message object/dict or None
"""
params: Final = getattr(request, "params", None)
params: Final[object] = getattr(request, "params", None)
if params is None:
return None
return getattr(params, "message", None)
@ -127,10 +132,12 @@ class A2ARequestUtils:
# the dict branch — request-side parts are a2a-sdk Part RootModels whose
# kind/text live on part.root, which the object branch cannot read. This
# mirrors how the response side already works (it operates on model_dump).
input_message = A2ARequestUtils.get_input_message_from_request(request)
if input_message is not None and hasattr(input_message, "model_dump"):
input_message = input_message.model_dump(mode="json")
input_text: Final = A2ARequestUtils.extract_text_from_message(input_message)
input_message: Final = A2ARequestUtils.get_input_message_from_request(request)
input_text: Final = A2ARequestUtils.extract_text_from_message(
input_message.model_dump(mode="json")
if has_attributes(input_message, _SupportsModelDump, "model_dump")
else input_message
)
prompt_tokens: Final = A2ARequestUtils.count_tokens(input_text)
# Count output tokens

2
litellm/anthropic_beta_headers_config.json
View file

@ -67,7 +67,7 @@
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
"text_editor_20241022": null,
"text_editor_20250124": null,
"thinking-binding-controls-2026-08-01": null,
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
"token-efficient-tools-2025-02-19": null,
"web-fetch-2025-09-10": "web-fetch-2025-09-10",

16
litellm/anthropic_beta_headers_manager.py
View file

@ -36,6 +36,7 @@ import httpx
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
# Cache for the loaded configuration
_BETA_HEADERS_CONFIG: dict | None = None
@ -44,6 +45,7 @@ _BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_na
BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
_DECODED_JSON: Final = TypeAdapter(object)
class GetAnthropicBetaHeadersConfig:
@ -75,7 +77,7 @@ class GetAnthropicBetaHeadersConfig:
}
@staticmethod
def _check_is_valid_dict(fetched_config: dict) -> bool:
def _check_is_valid_dict(fetched_config: object) -> bool:
"""Check if fetched config is a non-empty dict with expected structure."""
if not isinstance(fetched_config, dict):
verbose_logger.warning(
@ -109,7 +111,7 @@ class GetAnthropicBetaHeadersConfig:
return True
@classmethod
def validate_beta_headers_config(cls, fetched_config: dict) -> bool:
def validate_beta_headers_config(cls, fetched_config: object) -> bool:
"""
Validate the integrity of a fetched beta headers config.
@ -118,7 +120,7 @@ class GetAnthropicBetaHeadersConfig:
return cls._check_is_valid_dict(fetched_config)
@staticmethod
def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> dict:
def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> object:
"""
Fetch the beta headers config from a remote URL.
@ -127,7 +129,7 @@ class GetAnthropicBetaHeadersConfig:
"""
response: Final = httpx.get(url, timeout=timeout)
response.raise_for_status()
return response.json()
return _DECODED_JSON.validate_python(response.json())
def get_beta_headers_config(url: str) -> dict:
@ -145,7 +147,7 @@ def get_beta_headers_config(url: str) -> dict:
Dict containing the beta headers configuration
"""
# Check if local-only mode is enabled
if os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
# verbose_logger.debug("Using local Anthropic beta headers config (LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True)")
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
@ -160,7 +162,9 @@ def get_beta_headers_config(url: str) -> dict:
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
# Validate the fetched config
if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content):
if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content) or not isinstance(
content, dict
):
verbose_logger.warning(
"LiteLLM: Fetched beta headers config failed integrity check. Using local backup instead. url=%s",
url,

11
litellm/anthropic_interface/exceptions/exception_mapping_utils.py
View file

@ -7,10 +7,14 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop
import json
from typing import Final
from pydantic import TypeAdapter
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicErrorType
_PARSED_ERROR_BODY: Final = TypeAdapter(object)
# HTTP status code -> Anthropic error type
# Source: https://docs.anthropic.com/en/api/errors
ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = {
@ -79,7 +83,7 @@ class AnthropicExceptionMapping:
- Generic: {"message": "..."}
- Plain strings
"""
parsed: Final = safe_json_loads(raw_message)
parsed: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message))
if isinstance(parsed, dict):
# Bedrock format
if "detail" in parsed and isinstance(parsed["detail"], dict):
@ -145,9 +149,8 @@ class AnthropicExceptionMapping:
AnthropicErrorResponse dict
"""
# Try to parse as JSON once
parsed: dict | None = safe_json_loads(raw_message)
if not isinstance(parsed, dict):
parsed = None
decoded: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message))
parsed: Final[dict[str, object] | None] = decoded if isinstance(decoded, dict) else None
# If parsed and already in Anthropic format - passthrough
if parsed is not None and AnthropicExceptionMapping._is_anthropic_error_dict(parsed):

24
litellm/assistants/main.py
View file

@ -77,7 +77,7 @@ def get_assistants(
api_base: str | None = None,
api_version: str | None = None,
**kwargs,
) -> SyncCursorPage[Assistant]:
) -> SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]]:
aget_assistants: Final[bool | None] = kwargs.pop("aget_assistants", None)
if aget_assistants is not None and not isinstance(aget_assistants, bool):
raise Exception("Invalid value passed in for aget_assistants. Only bool or None allowed")
@ -100,7 +100,7 @@ def get_assistants(
elif timeout is None:
timeout = 600.0
response: SyncCursorPage[Assistant] | None = None
response: SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -564,7 +564,7 @@ def create_thread(
tool_resources: OpenAICreateThreadParamsToolResources | None = None,
client: OpenAI | None = None,
**kwargs,
) -> Thread:
) -> Thread | Coroutine[None, None, Thread]:
"""
- get the llm provider
- if openai - route it there
@ -610,7 +610,7 @@ def create_thread(
api_base: str | None = None
api_key: str | None = None
response: Thread | None = None
response: Thread | Coroutine[None, None, Thread] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -735,7 +735,7 @@ def get_thread(
thread_id: str,
client=None,
**kwargs,
) -> Thread:
) -> Thread | Coroutine[None, None, Thread]:
"""Get the thread object, given a thread_id"""
aget_thread: Final = kwargs.pop("aget_thread", None)
optional_params: Final = GenericLiteLLMParams(**kwargs)
@ -757,7 +757,7 @@ def get_thread(
timeout = 600.0
api_base: str | None = None
api_key: str | None = None
response: Thread | None = None
response: Thread | Coroutine[None, None, Thread] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -903,7 +903,7 @@ def add_message(
metadata: dict | None = None,
client=None,
**kwargs,
) -> OpenAIMessage:
) -> OpenAIMessage | Coroutine[None, None, OpenAIMessage]:
### COMMON OBJECTS ###
a_add_message: Final = kwargs.pop("a_add_message", None)
_message_data: Final = MessageData(role=role, content=content, attachments=attachments, metadata=metadata)
@ -935,7 +935,7 @@ def add_message(
timeout = 600.0
api_key: str | None = None
api_base: str | None = None
response: OpenAIMessage | None = None
response: OpenAIMessage | Coroutine[None, None, OpenAIMessage] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -1065,7 +1065,7 @@ def get_messages(
thread_id: str,
client: Any | None = None,
**kwargs,
) -> SyncCursorPage[OpenAIMessage]:
) -> SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]:
aget_messages: Final = kwargs.pop("aget_messages", None)
optional_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params_dict: Final = get_litellm_params(**kwargs)
@ -1086,7 +1086,7 @@ def get_messages(
elif timeout is None:
timeout = 600.0
response: SyncCursorPage[OpenAIMessage] | None = None
response: SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]] | None = None
api_key: str | None = None
api_base: str | None = None
if custom_llm_provider == "openai":
@ -1256,7 +1256,7 @@ def run_thread(
client: object | None = None,
event_handler: AssistantEventHandler | None = None, # for stream=True calls
**kwargs,
) -> Run:
) -> Run | Coroutine[None, None, Run]:
"""Run a given thread + assistant."""
arun_thread: Final = kwargs.pop("arun_thread", None)
optional_params: Final = GenericLiteLLMParams(**kwargs)
@ -1278,7 +1278,7 @@ def run_thread(
elif timeout is None:
timeout = 600.0
response: Run | None = None
response: Run | Coroutine[None, None, Run] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there

6
litellm/caching/_embedding_router.py
View file

@ -13,7 +13,7 @@ This module is dependency-injected: callers pass the proxy ``llm_router`` and
from __future__ import annotations
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final
import litellm
from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
@ -25,12 +25,12 @@ if TYPE_CHECKING:
def resolve_embedding_router(
embedding_model: str,
llm_router: Router | None,
llm_model_list: list[dict[str, Any]] | None,
llm_model_list: Sequence[Mapping[str, object]] | None,
) -> Router | None:
"""Return ``llm_router`` iff it serves ``embedding_model`` as a deployment."""
if llm_router is None:
return None
router_model_names: Final[list[str]] = (
router_model_names: Final[Sequence[object]] = (
[m["model_name"] for m in llm_model_list if "model_name" in m] if llm_model_list is not None else []
)
if embedding_model in router_model_names:

29
litellm/caching/affinity_cache.py
View file

@ -24,6 +24,16 @@ if current == false then
end
if ARGV[3] then
local decoded, stored = pcall(cjson.decode, current)
if ARGV[4] then
local legacy = current
if decoded and type(stored) == 'string' then legacy = stored end
for _, eligible in ipairs(cjson.decode(ARGV[3])) do
if eligible['model_id'] == legacy then
redis.call('EXPIRE', KEYS[1], ARGV[2])
return current
end
end
end
if decoded and type(stored) == 'table' then
for _, eligible in ipairs(cjson.decode(ARGV[3])) do
local matches = true
@ -68,6 +78,7 @@ def claim_affinity_pin_in_memory(
ttl_seconds: int,
*,
eligible_values: tuple[Mapping[str, str], ...] | None = None,
preserve_legacy_model_ids: bool = False,
) -> object:
"""No await between read and write, so same-loop claims agree during a Redis outage."""
existing: Final[object] = cache.in_memory_cache.get_cache(cache_key)
@ -75,7 +86,12 @@ def claim_affinity_pin_in_memory(
if _legacy_pin_matches(existing, pin_value):
set_local_affinity_pin(cache, cache_key, pin_value, ttl_seconds)
return existing
winner: Final = existing if existing is not None and existing in (eligible_values or ()) else pin_value
eligible_existing: Final = existing in (eligible_values or ()) or (
preserve_legacy_model_ids
and isinstance(existing, str)
and any("model_id" in value and _legacy_pin_matches(existing, value) for value in (eligible_values or ()))
)
winner: Final = existing if existing is not None and eligible_existing else pin_value
set_local_affinity_pin(cache, cache_key, winner, ttl_seconds)
return winner
@ -94,6 +110,7 @@ async def claim_affinity_pin(
ttl_seconds: int,
*,
eligible_values: tuple[Mapping[str, str], ...] | None = None,
preserve_legacy_model_ids: bool = False,
) -> object:
"""Return the authoritative first writer, replacing it only when it becomes ineligible.
@ -112,6 +129,7 @@ async def claim_affinity_pin(
if eligible_values is not None
else ()
),
*(("1",) if preserve_legacy_model_ids and eligible_values is not None else ()),
)
raw: Final = cast( # cast-ok: Redis scripts return heterogeneous values; only object is asserted here
object, await claim_script(keys=(cache_key,), args=args)
@ -124,4 +142,11 @@ async def claim_affinity_pin(
return winner
except Exception as error: # noqa: BLE001 # Redis/Lua faults retain same-pod affinity through local claims
verbose_router_logger.debug("Affinity cache: Redis claim failed, using pod-local claim. error=%s", error)
return claim_affinity_pin_in_memory(cache, cache_key, pin_value, ttl_seconds, eligible_values=eligible_values)
return claim_affinity_pin_in_memory(
cache,
cache_key,
pin_value,
ttl_seconds,
eligible_values=eligible_values,
preserve_legacy_model_ids=preserve_legacy_model_ids,
)

8
litellm/caching/azure_blob_cache.py
View file

@ -13,10 +13,14 @@ import json
from contextlib import suppress
from typing import Final
from pydantic import TypeAdapter
from litellm._logging import print_verbose, verbose_logger
from .base_cache import BaseCache
_DECODED_BLOB: Final = TypeAdapter(object)
class AzureBlobCache(BaseCache):
def __init__(self, account_url, container) -> None:
@ -65,7 +69,7 @@ class AzureBlobCache(BaseCache):
print_verbose(f"Get Azure Blob Cache: key: {key}")
as_bytes: Final = self.container_client.download_blob(key).readall()
as_str: Final = as_bytes.decode("utf-8")
cached_response: Final[object] = json.loads(as_str)
cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str))
verbose_logger.debug(
"Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s",
@ -86,7 +90,7 @@ class AzureBlobCache(BaseCache):
blob: Final = await self.async_container_client.download_blob(key)
as_bytes: Final = await blob.readall()
as_str: Final = as_bytes.decode("utf-8")
cached_response: Final[object] = json.loads(as_str)
cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str))
verbose_logger.debug(
"Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s",
key,

9
litellm/caching/base_cache.py
View file

@ -9,14 +9,7 @@ Has 4 methods:
"""
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, Final
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span | Any
else:
Span = Any
from typing import Final
class BaseCache(ABC):

8
litellm/caching/disk_cache.py
View file

@ -1,15 +1,19 @@
import json
from typing import TYPE_CHECKING, Any, Final
from pydantic import TypeAdapter
from .base_cache import BaseCache
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span | Any
Span = _Span
else:
Span = Any
_DECODED_DISK_VALUE: Final = TypeAdapter(object)
class DiskCache(BaseCache):
def __init__(self, disk_cache_dir: str | None = None):
@ -44,7 +48,7 @@ class DiskCache(BaseCache):
original_cached_response: Final = self.disk_cache.get(key)
if original_cached_response:
try:
cached_response = json.loads(original_cached_response)
cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response))
except Exception:
cached_response = original_cached_response
return cached_response

6
litellm/caching/redis_cache.py
View file

@ -1738,7 +1738,9 @@ class RedisCache(BaseCache):
return key_value_dict
@_redis_circuit_breaker_guard
async def async_get_cache(self, key, parent_otel_span: Span | None = None, **kwargs):
async def async_get_cache(
self, key, parent_otel_span: Span | None = None, *, raise_on_error: bool = False, **kwargs
):
from redis.asyncio import Redis
_redis_client: Final[Redis] = self.init_async_client()
@ -1781,6 +1783,8 @@ class RedisCache(BaseCache):
)
)
print_verbose(f"litellm.caching.caching: async get() - Got exception from REDIS: {e}")
if raise_on_error:
raise
_record_swallowed_redis_failure(self._circuit_breaker, e)
@_redis_circuit_breaker_guard

5
litellm/constants.py
View file

@ -480,6 +480,7 @@ REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_end_user_sp
REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_agent_spend_update_buffer"
REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_tag_spend_update_buffer"
REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_window_spend_update_buffer"
REDIS_TOOL_DISCOVERY_BUFFER_KEY: Final = "litellm_tool_discovery_buffer"
MAX_REDIS_BUFFER_DEQUEUE_COUNT: Final = get_env_int("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100)
REDIS_SPEND_LOGS_BUFFER_KEY: Final = "litellm_spend_logs_buffer"
REDIS_SPEND_LOGS_BUFFER_MAX_ROWS: Final = 100000
@ -611,6 +612,7 @@ NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int(
) # timeout for adjacent services (e.g. jwt auth)
MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000)
MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048)
MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024
BEDROCK_MAX_POLICY_SIZE: Final = get_env_int("BEDROCK_MAX_POLICY_SIZE", 75)
# One entry per distinct AWS credential-argument set. Per-user cost attribution passes the attributed
# identity as aws_session_name, so this bounds how many attributed identities keep a cached STS session.
@ -860,6 +862,7 @@ LITELLM_CHAT_PROVIDERS: Final = [
"heroku",
"oci",
"morph",
"opencode_go",
"lambda_ai",
"inception",
"vercel_ai_gateway",
@ -1115,6 +1118,7 @@ openai_compatible_providers: Final[list[str]] = [
"v0",
"helicone",
"morph",
"opencode_go",
"lambda_ai",
"inception",
"hyperbolic",
@ -1841,6 +1845,7 @@ LITELLM_UI_SESSION_DURATION: Final = os.getenv("LITELLM_UI_SESSION_DURATION", "2
########################### DB CRON JOB NAMES ###########################
DB_SPEND_UPDATE_JOB_NAME: Final = "db_spend_update_job"
TOOL_DISCOVERY_COMMIT_JOB_NAME: Final = "tool_discovery_commit_job"
DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME: Final = "db_daily_tag_spend_update_job"
PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME: Final = "prometheus_emit_budget_metrics"
CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME: Final = "cloudzero_export_usage_data"

79
litellm/cost_calculator.py
View file

@ -37,6 +37,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
_SERVICE_TIER_TO_COST_KEY_SUFFIX,
BilledTokenRates,
CostCalculatorUtils,
calculate_cache_writing_cost,
calculate_cost_component,
generic_cost_per_character,
generic_cost_per_token,
@ -1200,12 +1201,16 @@ def get_usage_object(
elif isinstance(usage_obj, dict):
return Usage(**usage_obj)
elif isinstance(usage_obj, BaseModel):
return Usage(**usage_obj.model_dump())
return _usage_from_model(usage_obj)
else:
verbose_logger.debug("Unknown usage object type: %s, usage_obj: %s", type(usage_obj), usage_obj)
return None
def _usage_from_model(usage_model: BaseModel) -> Usage:
return Usage(**usage_model.model_dump())
def _decisions_prompt_tokens_details(usage: DecisionsUsage | OpenAIDecisionUsage) -> PromptTokensDetailsWrapper:
match usage:
case DecisionsUsage():
@ -1546,6 +1551,8 @@ def completion_cost(
"""
try:
call_type = infer_call_type(call_type, completion_response) or "completion"
if isinstance(completion_response, DecisionsResponse) and completion_response.usage is None:
raise ValueError("Cannot calculate Decisions cost: provider did not report usage")
if pricing_provider is not None:
custom_llm_provider = pricing_provider # rebind-ok: model_info.pricing_provider overrides the routing provider for cost-map lookups
@ -1686,7 +1693,7 @@ def completion_cost(
setattr(
completion_response,
"usage",
litellm.Usage(**_usage_for_dump.model_dump()),
_usage_from_model(_usage_for_dump),
)
if isinstance(usage_obj, (DecisionsUsage, OpenAIDecisionUsage)):
_usage = _decisions_usage(usage_obj).model_dump()
@ -2264,6 +2271,8 @@ def response_cost_calculator(
### VERTEX LOCATION ###
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
region_name: str | None = None,
custom_cost_per_token: CostPerToken | None = None,
custom_cost_per_second: float | None = None,
) -> float:
"""
Returns
@ -2292,6 +2301,8 @@ def response_cost_calculator(
optional_params=optional_params,
custom_pricing=custom_pricing,
base_model=base_model,
custom_cost_per_token=custom_cost_per_token,
custom_cost_per_second=custom_cost_per_second,
prompt=prompt,
standard_built_in_tools_params=standard_built_in_tools_params,
litellm_model_name=litellm_model_name,
@ -2875,6 +2886,7 @@ def _batch_rate(
model_info: ModelInfo,
key: Literal[
"input_cost_per_audio_token_batches",
"cache_read_input_audio_token_cost_batches",
"input_cost_per_image_token_batches",
"input_cost_per_video_token_batches",
],
@ -2960,7 +2972,12 @@ def batch_cost_calculator(
total_completion_cost = 0.0
if batch_rates.input is not None:
total_prompt_cost = _batch_prompt_cost(
usage, model_info, batch_rates.input, batch_rates.cache_read, batch_rates.cache_creation
usage,
model_info,
batch_rates.input,
batch_rates.cache_read,
batch_rates.cache_creation,
batch_rates.cache_creation_rates_by_ttl,
)
elif input_cost_per_token:
details: Final = parse_prompt_tokens_details(usage)
@ -3005,10 +3022,13 @@ def _batch_prompt_cost(
input_rate: float,
cache_read_rate: float | None,
cache_creation_rate: float | None,
cache_creation_rates_by_ttl: Mapping[str, float],
) -> float:
details: Final = parse_prompt_tokens_details(usage)
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl)
written_tokens: Final = details["cache_creation_tokens"] if bills_cache_writes else 0
audio_tokens, image_tokens, video_tokens = (
details["audio_tokens"],
details["image_tokens"],
@ -3022,13 +3042,27 @@ def _batch_prompt_cost(
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
+ cached_tokens * (cache_read_rate or 0.0)
+ written_tokens * (cache_creation_rate or 0.0)
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
+ cached_audio_tokens
* _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0)
+ (
calculate_cache_writing_cost(
written_tokens,
details["cache_creation_token_details"],
input_rate if cache_creation_rate is None else cache_creation_rate,
cache_creation_rates_by_ttl,
)
if bills_cache_writes
else 0.0
)
)
_ATTRIBUTE_VALUE: Final = TypeAdapter(object)
def _attribute_value(obj: object, name: str) -> object:
return getattr(obj, name)
return _ATTRIBUTE_VALUE.validate_python(getattr(obj, name))
def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]:
@ -3663,7 +3697,26 @@ def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo |
return None
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...]) -> float | None:
_TranscriptionRateKey: TypeAlias = Literal[
"input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
]
def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None:
match key:
case "input_cost_per_second":
return info.get("input_cost_per_second")
case "input_cost_per_audio_token":
return info.get("input_cost_per_audio_token")
case "input_cost_per_token":
return info.get("input_cost_per_token")
case "output_cost_per_token":
return info.get("output_cost_per_token")
case _:
assert_never(key)
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
"""First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry
because ``get_model_info`` synthesizes zero token rates for entries that omit them."""
if info is None:
@ -3672,12 +3725,18 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...])
if declared is None:
return None
return next(
(float(value) for key in keys if declared.get(key) is not None and (value := info.get(key)) is not None),
(
float(value)
for key in keys
if declared.get(key) is not None and (value := _model_info_rate(info, key)) is not None
),
None,
)
def _transcription_rate(keys: tuple[str, ...], override: ModelInfo | None, base: ModelInfo | None) -> float:
def _transcription_rate(
keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None
) -> float:
rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base))
return next((rate for rate in rates if rate is not None), 0.0)

2
litellm/exceptions.py
View file

@ -1273,7 +1273,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
self.message = _saved_message
self.args = (_saved_message,)
original_headers: Final = getattr(original_exception, "litellm_response_headers", None)
original_headers: Final[object] = getattr(original_exception, "litellm_response_headers", None)
if original_headers is not None:
self.litellm_response_headers = original_headers

20
litellm/experimental_mcp_client/tools.py
View file

@ -10,6 +10,7 @@ from mcp.types import Tool as MCPTool
from openai.types.chat import ChatCompletionToolParam
from openai.types.responses.function_tool_param import FunctionToolParam
from openai.types.shared_params.function_definition import FunctionDefinition
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import (
@ -20,6 +21,8 @@ from litellm.constants import (
from litellm.types.llms.anthropic import AnthropicMessagesTool
from litellm.types.utils import ChatCompletionMessageToolCall
_DECODED_JSON: Final = TypeAdapter(object)
########################################################
# List MCP Tool functions
@ -200,15 +203,18 @@ async def call_mcp_tool(
return tool_result
def _decode_json_arguments(arguments: str) -> object:
try:
return _DECODED_JSON.validate_python(json.loads(arguments))
except json.JSONDecodeError:
return {}
def _get_function_arguments(function: FunctionDefinition) -> dict:
"""Helper to safely get and parse function arguments."""
arguments = function.get("arguments", {})
if isinstance(arguments, str):
try:
arguments = json.loads(arguments)
except json.JSONDecodeError:
arguments = {}
return arguments if isinstance(arguments, dict) else {}
arguments: Final = function.get("arguments", {})
decoded: Final = _decode_json_arguments(arguments) if isinstance(arguments, str) else arguments
return decoded if isinstance(decoded, dict) else {}
def transform_openai_tool_call_request_to_mcp_tool_call_request(

14
litellm/files/main.py
View file

@ -9,7 +9,7 @@ import asyncio
import contextvars
import time
import uuid as uuid_module
from collections.abc import Coroutine
from collections.abc import Awaitable, Coroutine
from functools import partial
from typing import Final, Literal, cast
@ -293,6 +293,14 @@ def create_file(
raise e
async def _await_retrieved_file(pending: Awaitable[FileObject | OpenAIFileObject]) -> FileObject | OpenAIFileObject:
return await pending
async def _await_deleted_file(pending: Awaitable[object]) -> object:
return await pending
@client
async def afile_retrieve(
file_id: str,
@ -325,7 +333,7 @@ async def afile_retrieve(
func_with_context: Final = partial(ctx.run, func)
init_response: Final = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
response = await _await_retrieved_file(init_response)
else:
response = init_response
@ -505,7 +513,7 @@ async def afile_delete(
func_with_context: Final = partial(ctx.run, func)
init_response: Final = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
response = await _await_deleted_file(init_response)
else:
response = init_response

4
litellm/files/streaming.py
View file

@ -1,7 +1,7 @@
import datetime
import traceback
from collections.abc import AsyncIterator, Iterator
from typing import TYPE_CHECKING, Any, Final, Optional, cast
from typing import TYPE_CHECKING, Final, Optional, cast
import anyio
@ -163,7 +163,7 @@ class FileContentStreamingResponse:
merged_hidden_params: Final = cast(
"StandardLoggingHiddenParams",
{
**cast(dict[str, Any], payload.get("hidden_params") or {}),
**(payload.get("hidden_params") or {}),
**self._hidden_params,
},
)

3
litellm/harness/handlers/base.py
View file

@ -4,7 +4,6 @@ from __future__ import annotations
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
from typing import Any
from litellm.harness.context import SessionContext
from litellm.harness.errors import CapabilityUnsupported
@ -38,5 +37,5 @@ class BaseHarnessHandler(ABC):
async def resume(self, ctx: SessionContext, native_session_id: str) -> None:
"""Continue the runtime's own session on the next turn."""
async def history(self, ctx: SessionContext) -> list[dict[str, Any]]: # mutable-ok: public history() API shape
async def history(self, ctx: SessionContext) -> list[dict[str, object]]: # mutable-ok: public history() API shape
raise CapabilityUnsupported(f"Harness.{self.config.harness.name} does not expose history")

4
litellm/harness/handlers/cli_handler.py
View file

@ -70,10 +70,10 @@ async def send_stdin(proc: Process, data: str) -> None:
async def private_dir_for(sandbox: Sandbox) -> str:
tempdir = getattr(sandbox, "tempdir", None)
tempdir: Final = sandbox.tempdir if hasattr(sandbox, "tempdir") else None
if tempdir is None:
raise SandboxError(f"{type(sandbox).__name__} has no tempdir(); CLI harnesses need a private config dir")
path: str = await tempdir()
path: Final = await tempdir()
return path

2
litellm/harness/handlers/deepagents_handler.py
View file

@ -161,7 +161,7 @@ class DeepAgentsHandler(BaseHarnessHandler):
async def history(
self, ctx: SessionContext
) -> list[dict[str, Any]]: # mutable-ok: BaseHarnessHandler.history API returns OpenAI message dicts
) -> list[dict[str, object]]: # mutable-ok: BaseHarnessHandler.history API returns OpenAI message dicts
agent, deps = self._require_agent()
snapshot = await agent.aget_state(self._run_config(ctx, None))
messages = (snapshot.values or MappingProxyType({})).get("messages") or ()

11
litellm/harness/runtime.py
View file

@ -12,11 +12,10 @@ import inspect
import logging
import os
import uuid
from collections.abc import AsyncIterator, Callable, Coroutine, Generator, Mapping, Sequence
from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Generator, Mapping, Sequence
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import (
Any,
Final,
get_args,
)
@ -331,11 +330,15 @@ async def _put_end(queue: asyncio.Queue[Event | _End], end: _End) -> None:
queue.put_nowait(end)
async def _await_decision(pending: Awaitable[object]) -> object:
return await pending
async def call_approval_handler(handler: ApprovalHandler, approval: Approval) -> None:
"""Run on_approval (sync in a worker thread, or async) and resolve approval."""
try:
if inspect.iscoroutinefunction(handler):
decision: object = await handler(approval)
decision: object = await _await_decision(handler(approval))
else:
decision = await asyncio.to_thread(handler, approval)
if inspect.isawaitable(decision):
@ -749,7 +752,7 @@ class AsyncSession:
async def history(
self,
) -> list[dict[str, Any]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler
) -> list[dict[str, object]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler
if not self.harness_config.capabilities.history:
raise CapabilityUnsupported(f"Harness.{self.config.harness.name} does not expose history")
await self._ensure_ready()

11
litellm/images/main.py
View file

@ -3,7 +3,7 @@ import contextvars
import importlib
from collections.abc import Coroutine
from functools import partial
from typing import TYPE_CHECKING, Final, Literal, Optional, cast, overload
from typing import TYPE_CHECKING, Final, Literal, cast, overload
from pydantic import TypeAdapter, ValidationError
@ -70,16 +70,17 @@ from litellm.utils import (
_PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
# Cache for ImageEditRequestUtils to avoid repeated __getattr__ calls
_ImageEditRequestUtils_cache: Optional["ImageEditRequestUtils"] = None
_ImageEditRequestUtils_cache: type["ImageEditRequestUtils"] | None = None
def _get_ImageEditRequestUtils() -> "ImageEditRequestUtils":
def _get_ImageEditRequestUtils() -> type["ImageEditRequestUtils"]:
"""Get ImageEditRequestUtils, loading it lazily if needed."""
global _ImageEditRequestUtils_cache
if _ImageEditRequestUtils_cache is None:
# Access via module to trigger __getattr__ if not cached
module: Final = importlib.import_module(__name__)
_ImageEditRequestUtils_cache = module.ImageEditRequestUtils
from litellm.images import main as images_main
_ImageEditRequestUtils_cache = images_main.ImageEditRequestUtils
assert _ImageEditRequestUtils_cache is not None # Type narrowing for type checker
return _ImageEditRequestUtils_cache

14
litellm/integrations/SlackAlerting/hanging_request_check.py
View file

@ -115,14 +115,16 @@ class AlertingHangingRequestCheck:
if hanging_request_data.alerted:
continue
request_status = await proxy_logging_obj.internal_usage_cache.async_get_cache(
key=f"request_status:{hanging_request_data.request_id}",
litellm_parent_otel_span=None,
local_only=True,
)
# this means the request status was either success or fail
# and is not hanging
if request_status is not None:
if (
await proxy_logging_obj.internal_usage_cache.async_get_cache(
key=f"request_status:{hanging_request_data.request_id}",
litellm_parent_otel_span=None,
local_only=True,
)
is not None
):
# clear this request from hanging request cache since the request was either success or failed
self.hanging_request_cache._remove_key(
key=request_id,

5
litellm/integrations/SlackAlerting/user_spend_alerts.py
View file

@ -56,8 +56,9 @@ async def fetch_user_spend_rows(
today_str: Final = today.strftime("%Y-%m-%d")
month_start_str: Final = today.replace(day=1).strftime("%Y-%m-%d")
baseline_start_str: Final = (today - datetime.timedelta(days=max(baseline_days, 1))).strftime("%Y-%m-%d")
raw: Final = await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
return USER_SPEND_ROWS_ADAPTER.validate_python(raw)
return USER_SPEND_ROWS_ADAPTER.validate_python(
await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
)
def _daily_threshold_event(row: UserSpendRow, args: SlackAlertingArgs, today_str: str) -> UserSpendAlertEvent | None:

23
litellm/integrations/agentops/agentops.py
View file

@ -4,11 +4,16 @@ AgentOps integration for LiteLLM - Provides OpenTelemetry tracing for LLM calls
import os
from dataclasses import dataclass
from typing import Any, Final
from typing import Final
from pydantic import TypeAdapter
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.custom_httpx.http_handler import get_httpx_client
_DECODED_JSON: Final = TypeAdapter(object)
@dataclass
class AgentOpsConfig:
@ -58,9 +63,10 @@ class AgentOps(OpenTelemetry):
project_id = None
if config.api_key:
try:
response: Final = self._fetch_auth_token(config.api_key, config.auth_endpoint)
jwt_token = response.get("token")
project_id = response.get("project_id")
response: Final = as_str_mapping(self._fetch_auth_token(config.api_key, config.auth_endpoint))
if response is not None:
jwt_token = response.get("token")
project_id = response.get("project_id")
except Exception:
pass
@ -78,12 +84,9 @@ class AgentOps(OpenTelemetry):
"telemetry.sdk.name": "agentops",
}
if project_id:
resource_attrs["project.id"] = project_id
self.resource_attributes = {**resource_attrs, "project.id": project_id} if project_id else resource_attrs
self.resource_attributes = resource_attrs
def _fetch_auth_token(self, api_key: str, auth_endpoint: str) -> dict[str, Any]:
def _fetch_auth_token(self, api_key: str, auth_endpoint: str) -> object:
"""
Fetch JWT authentication token from AgentOps API
@ -111,6 +114,6 @@ class AgentOps(OpenTelemetry):
if response.status_code != 200:
raise Exception(f"Failed to fetch auth token: {response.text}")
return response.json()
return _DECODED_JSON.validate_python(response.json())
finally:
client.close()

2
litellm/integrations/arize/arize.py
View file

@ -166,7 +166,7 @@ class ArizeLogger(OpenTelemetry):
primary ``otel`` callback which handles proxy-level parent spans.
"""
def set_attributes(self, span: Span, kwargs, response_obj: Any | None):
def set_attributes(self, span: Span, kwargs, response_obj: object):
ArizeLogger.set_arize_attributes(span, kwargs, response_obj)
@staticmethod

29
litellm/integrations/braintrust_logging.py
View file

@ -3,9 +3,10 @@
import os
from datetime import datetime
from typing import Final
from typing import Final, Protocol
import httpx
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm import verbose_logger
@ -24,6 +25,18 @@ from litellm.utils import print_verbose
API_BASE: Final = "https://api.braintrustdata.com/v1"
class _BraintrustProjectBody(TypedDict):
id: ReadOnly[str]
class _BraintrustProjectResponse(Protocol):
def json(self) -> _BraintrustProjectBody: ...
def _project_id(response: _BraintrustProjectResponse) -> str:
return response.json()["id"]
def get_utc_datetime():
import datetime as dt
@ -80,8 +93,7 @@ class BraintrustLogger(CustomLogger):
headers=self.headers,
json={"name": project_name},
)
project_dict: Final = response.json()
project_id: Final = project_dict["id"]
project_id: Final = _project_id(response)
self._project_id_cache[project_name] = project_id
return project_id
except httpx.HTTPStatusError as e:
@ -100,8 +112,7 @@ class BraintrustLogger(CustomLogger):
headers=self.headers,
json={"name": project_name},
)
project_dict: Final = response.json()
project_id: Final = project_dict["id"]
project_id: Final = _project_id(response)
self._project_id_cache[project_name] = project_id
return project_id
except httpx.HTTPStatusError as e:
@ -112,18 +123,14 @@ class BraintrustLogger(CustomLogger):
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
)
project_dict: Final = project.json()
self.default_project_id = project_dict["id"]
self.default_project_id = _project_id(project)
def create_sync_default_project_and_experiment(self):
project: Final = self.global_braintrust_sync_http_handler.post(
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
)
project_dict: Final = project.json()
self.default_project_id = project_dict["id"]
self.default_project_id = _project_id(project)
def log_success_event(self, kwargs, response_obj, start_time, end_time):
verbose_logger.debug("REACHES BRAINTRUST SUCCESS")

7
litellm/integrations/cloudzero/cz_stream_api.py
View file

@ -19,8 +19,9 @@
"""Output modules for writing CBF data to various destinations."""
import zoneinfo
from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Any, Final
from typing import Final
import httpx
import polars as pl
@ -188,7 +189,7 @@ class CloudZeroStreamer:
)
raise
def _prepare_batch_payload(self, batch_date: str, batch_data: pl.DataFrame, operation: str) -> dict[str, Any]:
def _prepare_batch_payload(self, batch_date: str, batch_data: pl.DataFrame, operation: str) -> Mapping[str, object]:
"""Prepare batch payload according to CloudZero AnyCost API format."""
# Convert batch_date to month for the API (YYYY-MM format)
try:
@ -205,7 +206,7 @@ class CloudZeroStreamer:
if record:
data_records.append(record)
payload: Final = {"month": month_str, "operation": operation, "data": data_records}
payload: Final[Mapping[str, object]] = {"month": month_str, "operation": operation, "data": data_records}
return payload

7
litellm/integrations/custom_guardrail.py
View file

@ -352,7 +352,7 @@ class CustomGuardrail(CustomLogger):
self,
violation_message: str,
request_data: dict[str, Any],
detection_info: dict[str, Any] | None = None,
detection_info: Mapping[str, object] | None = None,
original_response: object = None,
) -> None:
"""
@ -1368,7 +1368,10 @@ class CustomGuardrail(CustomLogger):
if event_type is not None:
guardrail_mode = event_type
elif isinstance(self.event_hook, Mode):
guardrail_mode = GuardrailMode(**dict(self.event_hook.model_dump()))
guardrail_mode = GuardrailMode(
tags=copy.deepcopy(self.event_hook.tags),
default=copy.deepcopy(self.event_hook.default),
)
else:
guardrail_mode = self.event_hook

3
litellm/integrations/custom_secret_manager.py
View file

@ -37,7 +37,6 @@ Usage:
"""
from abc import abstractmethod
from typing import Any
import httpx
@ -154,7 +153,7 @@ class CustomSecretManager(BaseSecretManager):
optional_params: dict | None = None,
timeout: float | httpx.Timeout | None = None,
tags: dict | list | None = None,
) -> dict[str, Any]:
) -> dict[str, object]:
"""
Asynchronously write a secret to your custom secret manager.

4
litellm/integrations/deepeval/types.py
View file

@ -1,6 +1,6 @@
# Duplicate -> https://github.com/confident-ai/deepeval/blob/main/deepeval/tracing/api.py
from enum import Enum
from typing import Any, ClassVar, Literal
from typing import ClassVar, Literal
from pydantic import ConfigDict, Field
@ -55,7 +55,7 @@ class TraceApi(LiteLLMBaseModel):
tool_spans: list[BaseApiSpan] = Field(alias="toolSpans")
start_time: str = Field(alias="startTime")
end_time: str = Field(alias="endTime")
metadata: dict[str, Any] | None = Field(None)
metadata: dict[str, object] | None = Field(None)
tags: list[str] | None = Field(None)
environment: str | None = Field(None)

4
litellm/integrations/dotprompt/__init__.py
View file

@ -61,7 +61,9 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom
prompt_file: Final = getattr(litellm_params, "prompt_file", None)
# Handle dotprompt_content from database
dotprompt_content: Final = getattr(litellm_params, "dotprompt_content", None)
dotprompt_content: Final = (
litellm_params.dotprompt_content if hasattr(litellm_params, "dotprompt_content") else None
)
if dotprompt_content and not prompt_data and not prompt_file:
prompt_data = get_prompt_data_from_dotprompt_content(dotprompt_content)

15
litellm/integrations/dotprompt/dotprompt_manager.py
View file

@ -7,6 +7,8 @@ import json
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing_extensions import ReadOnly, TypedDict
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import PromptManagementClient
from litellm.types.llms.openai import AllMessageValues
@ -18,7 +20,12 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
from .prompt_manager import PromptManager, PromptTemplate
from .prompt_manager import PromptJsonEntry, PromptManager, PromptTemplate
class _PromptJsonData(TypedDict, total=False):
content: ReadOnly[str]
metadata: ReadOnly[dict[str, object]]
class DotpromptManager(CustomPromptManagement):
@ -342,17 +349,17 @@ class DotpromptManager(CustomPromptManagement):
if self._prompt_manager:
self._prompt_manager.reload_prompts()
def add_prompt_from_json(self, prompt_id: str, json_data: dict[str, Any]) -> None:
def add_prompt_from_json(self, prompt_id: str, json_data: _PromptJsonData) -> None:
"""Add a prompt from JSON data."""
content: Final = json_data.get("content", "")
metadata: Final = json_data.get("metadata", {})
self.prompt_manager.add_prompt(prompt_id, content, metadata)
def load_prompts_from_json(self, prompts_data: dict[str, dict[str, object]]) -> None:
def load_prompts_from_json(self, prompts_data: Mapping[str, object]) -> None:
"""Load multiple prompts from JSON data."""
self.prompt_manager.load_prompts_from_json_data(prompts_data)
def get_prompts_as_json(self) -> dict[str, dict[str, object]]:
def get_prompts_as_json(self) -> dict[str, PromptJsonEntry]:
"""Get all prompts in JSON format."""
return self.prompt_manager.get_all_prompts_as_json()

29
litellm/integrations/dotprompt/prompt_manager.py
View file

@ -12,6 +12,8 @@ from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.litellm_core_utils.core_helpers import as_str_mapping
class _PromptFileJson(TypedDict):
"""JSON form of a .prompt file: rendered template text plus its frontmatter."""
@ -20,6 +22,11 @@ class _PromptFileJson(TypedDict):
metadata: ReadOnly[NotRequired[dict[str, object]]]
class PromptJsonEntry(TypedDict):
content: ReadOnly[str]
metadata: ReadOnly[dict[str, object]]
def strip_version_suffix(prompt_id: str) -> str | None:
base, separator, version = prompt_id.rpartition(".v")
if separator and base and version.isdigit():
@ -358,20 +365,26 @@ class PromptManager:
return f"---\n{frontmatter_yaml}---\n{content}"
def get_all_prompts_as_json(self) -> dict[str, dict[str, Any]]:
def get_all_prompts_as_json(self) -> dict[str, PromptJsonEntry]:
"""Get all loaded prompts in JSON format.
Returns:
Dictionary mapping prompt_id to prompt data
"""
result: Final = {}
for prompt_id, template in self.prompts.items():
result[prompt_id] = {
return {
prompt_id: {
"content": template.content,
"metadata": template.metadata,
}
return result
for prompt_id, template in self.prompts.items()
}
def load_prompts_from_json_data(self, prompt_data: dict[str, dict[str, object]]) -> None:
"""Load additional prompts from JSON data (merges with existing prompts)."""
self._load_prompts_from_json(prompt_data)
def load_prompts_from_json_data(self, prompt_data: Mapping[str, object]) -> None:
"""Load additional prompts from JSON data (merges with existing prompts). Entries that are not
mappings are skipped, the same as a malformed entry inside the loader."""
entries: Final = {
prompt_id: dict(entry)
for prompt_id, raw in prompt_data.items()
if (entry := as_str_mapping(raw)) is not None
}
self._load_prompts_from_json(entries)

5
litellm/integrations/focus/destinations/factory.py
View file

@ -3,6 +3,7 @@
from __future__ import annotations
import os
from collections.abc import Mapping
from typing import Any, Final
from .base import FocusDestination
@ -20,7 +21,7 @@ class FocusDestinationFactory:
*,
provider: str,
prefix: str,
config: dict[str, Any] | None = None,
config: Mapping[str, object] | None = None,
) -> FocusDestination:
"""Return a destination implementation for the requested provider."""
provider_lower: Final = provider.lower()
@ -39,7 +40,7 @@ class FocusDestinationFactory:
def _resolve_config(
*,
provider: str,
overrides: dict[str, object],
overrides: Mapping[str, object],
) -> dict[str, Any]:
if provider == "s3":
resolved = {

8
litellm/integrations/focus/export_engine.py
View file

@ -2,8 +2,8 @@
from __future__ import annotations
from collections.abc import Callable
from typing import Any, Final, SupportsFloat, SupportsIndex, SupportsInt, cast
from collections.abc import Callable, Mapping
from typing import Final, SupportsFloat, SupportsIndex, SupportsInt, cast
import polars as pl
@ -24,7 +24,7 @@ class FocusExportEngine:
provider: str,
export_format: str,
prefix: str,
destination_config: dict[str, Any] | None = None,
destination_config: Mapping[str, object] | None = None,
) -> None:
self.provider = provider
self.export_format = export_format
@ -77,7 +77,7 @@ class FocusExportEngine:
return FocusParquetSerializer()
raise NotImplementedError(f"Export format '{self.export_format}' not supported. Use 'parquet' or 'csv'.")
async def dry_run_export_usage_data(self, limit: int | None) -> dict[str, Any]:
async def dry_run_export_usage_data(self, limit: int | None) -> dict[str, object]:
data: Final = await self.database.get_usage_data(limit=limit)
normalized: Final = self.transformer.transform(data)

8
litellm/integrations/focus/focus_logger.py
View file

@ -6,6 +6,8 @@ import os
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Any, ClassVar, Final, cast
from typing_extensions import TypedDict, Unpack
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
@ -25,6 +27,10 @@ FOCUS_USAGE_DATA_JOB_NAME: Final = "focus_export_usage_data"
DEFAULT_DRY_RUN_LIMIT: Final = 500
class _CustomLoggerOptions(TypedDict, total=False, extra_items=object):
pass
class FocusLogger(CustomLogger):
"""Coordinates Focus export jobs across transformer/serializer/destination layers."""
@ -40,7 +46,7 @@ class FocusLogger(CustomLogger):
interval_seconds: int | None = None,
prefix: str | None = None,
destination_config: dict[str, Any] | None = None,
**kwargs: Any,
**kwargs: Unpack[_CustomLoggerOptions],
) -> None:
super().__init__(**kwargs)
self.provider = (provider or os.getenv("FOCUS_PROVIDER") or "s3").lower()

8
litellm/integrations/gcs_bucket/gcs_bucket.py
View file

@ -3,6 +3,7 @@ import hashlib
import json
import os
import time
from collections.abc import Mapping
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Any, Final
@ -15,6 +16,7 @@ from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
from litellm.litellm_core_utils.cloud_storage_security import (
sanitize_cloud_object_component,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.gcs_bucket import *
from litellm.types.utils import StandardBatchLineItemsPayload, StandardLoggingPayload
@ -134,7 +136,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
"""
return f"{date_str}/batch-{batch_id}.ndjson"
def _get_config_key(self, kwargs: dict[str, Any]) -> str:
def _get_config_key(self, kwargs: Mapping[str, object]) -> str:
"""
Extract a synchronous grouping key from kwargs to group items by GCS config.
This allows us to batch items with the same bucket/credentials together.
@ -143,7 +145,9 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
This key may contain sensitive information (bucket names, paths) - use _sanitize_config_key()
for logging purposes.
"""
standard_callback_dynamic_params: Final = kwargs.get("standard_callback_dynamic_params", None) or {}
standard_callback_dynamic_params: Final = (
as_str_mapping(kwargs.get("standard_callback_dynamic_params", None)) or {}
)
bucket_name = standard_callback_dynamic_params.get("gcs_bucket_name", None) or self.BUCKET_NAME or "default"
path_service_account: Final = (

3
litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py
View file

@ -9,6 +9,7 @@ Usage:
"""
import asyncio
import os
from collections.abc import AsyncIterable, Iterable
from typing import Final
@ -40,7 +41,7 @@ _mocks_initialized = False
# Default mock latency in seconds (simulates network round-trip)
# Typical GCS API calls take 100-300ms for uploads, 50-150ms for GET/DELETE
_MOCK_LATENCY_SECONDS: Final = float(__import__("os").getenv("GCS_MOCK_LATENCY_MS", "150")) / 1000.0
_MOCK_LATENCY_SECONDS: Final = float(os.getenv("GCS_MOCK_LATENCY_MS", "150")) / 1000.0
async def _mock_async_handler_get(self, url, params=None, headers=None, follow_redirects=None):

8
litellm/integrations/gcs_pubsub/pub_sub.py
View file

@ -12,6 +12,8 @@ import os
import traceback
from typing import TYPE_CHECKING, Any, Final
from pydantic import TypeAdapter
from litellm.types.utils import StandardLoggingPayload
if TYPE_CHECKING:
@ -27,6 +29,8 @@ from litellm.llms.custom_httpx.http_handler import (
httpxSpecialProvider,
)
_DECODED_JSON: Final = TypeAdapter(object)
class GcsPubSubLogger(CustomBatchLogger):
def __init__(
@ -152,7 +156,7 @@ class GcsPubSubLogger(CustomBatchLogger):
finally:
self.log_queue.clear()
async def publish_message(self, message: SpendLogsPayload | StandardLoggingPayload) -> dict[str, Any] | None:
async def publish_message(self, message: SpendLogsPayload | StandardLoggingPayload) -> object:
"""
Publish message to Google Cloud Pub/Sub using REST API
@ -188,7 +192,7 @@ class GcsPubSubLogger(CustomBatchLogger):
raise Exception(f"Failed to publish message: {response.text}")
verbose_logger.debug("Pub/Sub response: %s", response.text)
return response.json()
return _DECODED_JSON.validate_python(response.json())
except Exception as e:
verbose_logger.error("Pub/Sub publish error: %s", str(e))

5
litellm/integrations/langfuse/langfuse.py
View file

@ -20,6 +20,7 @@ from litellm.integrations.langfuse.langfuse_mock_client import (
should_use_langfuse_mock,
)
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
filter_exceptions_from_params,
reconstruct_model_name,
safe_deep_copy,
@ -1268,11 +1269,11 @@ def log_provider_specific_information_as_span(
*,
tracing: "LangfuseTracing",
parent: "LangfuseObservation",
enrichments: Mapping[str, Any],
enrichments: Mapping[str, object],
):
"""Logs provider-specific information as spans under the generation."""
_hidden_params: Final[Mapping[str, object] | None] = enrichments.get("hidden_params", None)
_hidden_params: Final = as_str_mapping(enrichments.get("hidden_params", None))
if _hidden_params is None:
return

5
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -12,6 +12,7 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import (
serialize_langfuse_datetime,
)
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.types.integrations.langfuse_otel import (
LangfuseSpanAttributes,
@ -222,7 +223,7 @@ class LangfuseOtelLogger(OpenTelemetry):
)
@staticmethod
def _set_langfuse_specific_attributes(span: Span, kwargs: Mapping[str, Any], response_obj):
def _set_langfuse_specific_attributes(span: Span, kwargs: Mapping[str, object], response_obj):
"""
Sets Langfuse specific metadata attributes onto the OTEL span.
@ -234,7 +235,7 @@ class LangfuseOtelLogger(OpenTelemetry):
from litellm.integrations.arize._utils import safe_set_attribute
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
dynamic_params: Final = kwargs.get("standard_callback_dynamic_params")
dynamic_params: Final = as_str_mapping(kwargs.get("standard_callback_dynamic_params"))
langfuse_environment: Final = (
dynamic_params.get("langfuse_environment") if dynamic_params else None
) or os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")

12
litellm/integrations/langfuse/langfuse_prompt_management.py
View file

@ -2,6 +2,7 @@
Call Hook for LiteLLM Proxy which allows Langfuse prompt management.
"""
from collections.abc import Mapping
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
@ -110,6 +111,10 @@ def _remember_trace_id(litellm_call_id: object, logged: LangfuseLoggedEvent) ->
in_memory_trace_id_cache.set_cache(litellm_call_id=litellm_call_id, service_name="langfuse", trace_id=trace_id)
def _params_without_model(config: Mapping[str, object]) -> Mapping[str, object]:
return {k: v for k, v in config.items() if k != "model"}
class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogger):
def __init__(
self,
@ -182,12 +187,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
return compiled_prompt
def _get_optional_params_from_langfuse(self, langfuse_prompt_client: PROMPT_CLIENT) -> dict:
config: Final = langfuse_prompt_client.config
optional_params: Final = {}
for k, v in config.items():
if k != "model":
optional_params[k] = v
return optional_params
return dict(_params_without_model(langfuse_prompt_client.config))
async def async_get_chat_completion_prompt(
self,

11
litellm/integrations/langsmith.py
View file

@ -7,10 +7,11 @@ import random
import traceback
import types
from datetime import datetime, timezone
from typing import Any, Final
from typing import Final
import httpx
from pydantic import BaseModel, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -31,6 +32,10 @@ from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPa
_RUN_RESPONSE: Final = TypeAdapter(object)
class _DynamicParamsKwargs(TypedDict, total=False):
standard_callback_dynamic_params: ReadOnly[StandardCallbackDynamicParams | None]
def is_serializable(value):
non_serializable_types: Final = (
types.CoroutineType,
@ -474,7 +479,7 @@ class LangsmithLogger(CustomBatchLogger):
return log_queue_by_credentials
def _get_sampling_rate_to_use_for_request(self, kwargs: dict[str, Any]) -> float:
def _get_sampling_rate_to_use_for_request(self, kwargs: _DynamicParamsKwargs) -> float:
standard_callback_dynamic_params: Final[StandardCallbackDynamicParams | None] = kwargs.get(
"standard_callback_dynamic_params", None
)
@ -485,7 +490,7 @@ class LangsmithLogger(CustomBatchLogger):
sampling_rate = float(_sampling_rate)
return sampling_rate
def _get_credentials_to_use_for_request(self, kwargs: dict[str, Any]) -> LangsmithCredentialsObject:
def _get_credentials_to_use_for_request(self, kwargs: _DynamicParamsKwargs) -> LangsmithCredentialsObject:
"""
Handles key/team based logging

2
litellm/integrations/langtrace.py
View file

@ -6,7 +6,7 @@ from litellm.types.integrations.otel_span_attributes import SpanAttributes
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span | Any
Span = _Span
else:
Span = Any

4
litellm/integrations/levo/levo.py
View file

@ -4,8 +4,6 @@ from typing import TYPE_CHECKING, Any, Final
from litellm.integrations.opentelemetry import OpenTelemetry
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.integrations.opentelemetry import (
OpenTelemetryConfig as _OpenTelemetryConfig,
)
@ -13,11 +11,9 @@ if TYPE_CHECKING:
Protocol = _Protocol
OpenTelemetryConfig = _OpenTelemetryConfig
Span = _Span | Any
else:
Protocol = Any
OpenTelemetryConfig = Any
Span = Any
class LevoConfig:

5
litellm/integrations/logfire_logger.py
View file

@ -3,8 +3,9 @@
import os
import traceback
from collections.abc import Mapping
from enum import Enum
from typing import Any, Final, NamedTuple
from typing import Final, NamedTuple
from typing_extensions import LiteralString
@ -15,7 +16,7 @@ from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
class SpanConfig(NamedTuple):
message_template: LiteralString
span_data: dict[str, Any]
span_data: Mapping[str, object]
class LogfireLevel(str, Enum):

19
litellm/integrations/newrelic/newrelic.py
View file

@ -49,11 +49,12 @@ import time
import uuid
from collections.abc import Mapping, Sequence
from datetime import datetime
from typing import Any, Final
from typing import Final, Protocol
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.newrelic import NewRelicInitParams
@ -65,6 +66,10 @@ except ImportError:
_newrelic_agent = None
class _SupportsTimestamp(Protocol):
def timestamp(self) -> float: ...
class NewRelicLogger(CustomLogger):
"""
New Relic logger for LiteLLM to send AI monitoring events.
@ -126,13 +131,13 @@ class NewRelicLogger(CustomLogger):
verbose_logger.error("Failed to initialize New Relic agent: %s. Integration will be disabled.", e)
self.enabled = False
def _get_newrelic_params(self) -> dict:
def _get_newrelic_params(self) -> Mapping[str, object]:
"""
Get the newrelic_params from litellm.newrelic_params
These are params specific to initializing the NewRelicLogger e.g. turn_off_message_logging
"""
dict_newrelic_params: dict = {}
dict_newrelic_params: Mapping[str, object] = {}
if litellm.newrelic_params is not None:
if isinstance(litellm.newrelic_params, NewRelicInitParams):
dict_newrelic_params = litellm.newrelic_params.model_dump()
@ -401,9 +406,9 @@ class NewRelicLogger(CustomLogger):
return choices[0].get("finish_reason") or "unknown"
return "unknown"
def _to_epoch_ms(self, t: Any) -> float:
def _to_epoch_ms(self, t: datetime | float) -> float:
"""Convert a datetime or float timestamp to epoch milliseconds."""
if hasattr(t, "timestamp"):
if has_attributes(t, _SupportsTimestamp, "timestamp"):
return t.timestamp() * 1000.0
return float(t) * 1000.0
@ -518,13 +523,13 @@ class NewRelicLogger(CustomLogger):
sequence = 0
# Extract timestamps, preferring StandardLoggingPayload
start_time = None
start_time: datetime | float | None = None
if standard_logging_object:
start_time = standard_logging_object.get("startTime")
if not start_time:
start_time = kwargs.get("start_time")
end_time = None
end_time: datetime | float | None = None
if standard_logging_object:
end_time = standard_logging_object.get("endTime")
if not end_time:

2
litellm/integrations/newrelic/newrelic_metrics.py
View file

@ -254,7 +254,7 @@ class NewRelicMetricsLogger(CustomBatchLogger):
if not self.log_queue:
return
if _pass < NEWRELIC_METRICS_MAX_DRAIN_PASSES - 1:
await asyncio.sleep(2**_pass)
await asyncio.sleep(1 << _pass)
async with self.flush_lock:
tried_ids: Final = frozenset(id(record) for record in attempted)
survivors: Final = tuple(record for record in self.log_queue if id(record) not in tried_ids)

Some files were not shown because too many files have changed in this diff Show more