Merge branch 'litellm_guardrail_choice_isolation' into litellm_guardrail_stream_inspection

# Conflicts:
#	litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py
This commit is contained in:
Alessandro Pignati 2026-10-27 16:52:43 +01:00
commit b783f41a6a
2292 changed files with 111812 additions and 26076 deletions

View file

@ -1291,7 +1291,7 @@ jobs:
name: Install MCP SDK1 peer
command: |
uv venv --python 3.12 .venv-mcp-peer
uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
- when:
condition:

View file

@ -33,7 +33,7 @@ while IFS= read -r file || [ -n "$file" ]; do
*) has_backend=true ;;
esac
case "$file" in
model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json | model_prices_and_context_window.schema.json)
model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json)
has_cost_map=true ;;
tests/unit/proxy/*) : ;;
*) outside_cost_map_set=true ;;

4
.github/CODEOWNERS vendored
View file

@ -1,4 +1,4 @@
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/helm/ @yassin-berriai
/terraform/ @yassin-berriai

View file

@ -4,6 +4,13 @@
Drop every section you have nothing to put in, heading included: a bare "## Relevant issues" or
"## Affected release" with nothing under it must not appear in the final description -->
<!-- Two or three plain sentences a teammate can read in 10 seconds, written the way you'd explain it in Slack
Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
No headings, bullets, tables, or code names here. Everything else goes inside the Details block below -->
<details>
<summary>Details</summary>
## TLDR
<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
@ -164,3 +171,5 @@ Example checklists:
## Final Attestation
- [ ] The tests check the right things, including the edge cases, and regressions in the respective real-world customer use-cases are not possible after this PR
</details>

View file

@ -0,0 +1,91 @@
name: Interactions spec canary
on:
schedule:
- cron: "17 7 * * *"
pull_request:
paths:
- .github/workflows/interactions-spec-canary.yml
- tests/unit/interactions/**
- litellm/types/interactions/**
workflow_dispatch:
permissions: {}
concurrency: interactions-spec-canary
jobs:
interactions-spec-canary:
name: interactions-spec-canary
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
outputs:
compliance: ${{ steps.compliance.outcome }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Install dependencies
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy --no-install-project
- name: Run the compliance tests against Google's live spec
id: compliance
run: |
uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \
--interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \
--tb=short -vv -p no:rerunfailures -p no:pytest-retry \
--allow-hosts=127.0.0.1,::1,localhost,ai.google.dev
tracking-issue:
name: tracking-issue
needs: interactions-spec-canary
if: always() && github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
issues: write
steps:
- name: File the tracking issue unless the compliance step passed, close it when it did
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
env:
COMPLIANCE_OUTCOME: ${{ needs.interactions-spec-canary.outputs.compliance }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
with:
script: |
const title = "Interactions spec canary is red";
const outcome = process.env.COMPLIANCE_OUTCOME;
const runUrl = process.env.RUN_URL;
const query = `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open in:title "${title}"`;
const { data: found } = await github.rest.search.issuesAndPullRequests({ q: query, per_page: 20 });
const open = found.items.find((issue) => issue.title === title);
if (outcome === "success") {
if (open === undefined) return;
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Green again: ${runUrl}` });
await github.rest.issues.update({ ...context.repo, issue_number: open.number, state: "closed" });
return;
}
const state = outcome === "failure" ? "red" : `not run (compliance step outcome: ${outcome || "none"})`;
if (open !== undefined) {
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Still ${state}: ${runUrl}` });
return;
}
await github.rest.issues.create({
...context.repo,
title,
assignees: ["mateo-berri"],
body: `The scheduled run of tests/unit/interactions/test_openapi_compliance.py against Google's live Interactions spec is ${state}: ${runUrl}\n\nlitellm/types/interactions/README.md says how to refresh the snapshot and regenerate the types.`,
});

View file

@ -139,6 +139,7 @@ jobs:
test-path: |-
tests/unit/batch_completion
tests/unit/decisions
tests/unit/extraction
tests/unit/litellm_core_utils
tests/unit/systemone
python-version: "3.12"
@ -215,10 +216,12 @@ jobs:
- shard: llms-providers
test-path: |-
tests/unit/llms/test_api_base_trailing_slash_routes.py
tests/unit/llms/test_cache_control_and_reasoning.py
tests/unit/llms/test_custom_llm.py
tests/unit/llms/test_file_content_block.py
tests/unit/llms/test_file_search_responses.py
tests/unit/llms/test_get_complete_url_trailing_slash.py
tests/unit/llms/test_lifecycle_fix.py
tests/unit/llms/test_oss_decision.py
tests/unit/llms/test_polling_url_origin_match.py
@ -422,6 +425,7 @@ jobs:
tests/unit/realtime_api
tests/unit/rerank_api
tests/unit/search
tests/unit/text_completions
tests/unit/vector_stores
tests/unit/videos
python-version: "3.12"
@ -433,6 +437,13 @@ jobs:
- shard: rust-bridge-harness
test-path: >-
tests/test_litellm_rust/test_dispatch.py
tests/test_litellm_rust/test_public_entries.py
tests/test_litellm_rust/chat_completions/test_dispatch.py
tests/test_litellm_rust/embeddings/test_dispatch.py
tests/test_litellm_rust/messages/test_dispatch.py
tests/test_litellm_rust/ocr/test_dispatch.py
tests/test_litellm_rust/responses/test_dispatch.py
tests/unit/compression
tests/unit/harness
tests/unit/rust_bridge

View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@ -61,6 +61,7 @@ RUN apk add --no-cache \
python-3.13-dev \
rust \
openssl \
libssl3 \
openssl-dev \
nodejs \
npm \
@ -120,8 +121,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder
COPY --from=uvbin /uv /usr/local/bin/uv
RUN apk add --no-cache python-3.13
ADD --checksum=sha256:2f7ae5cdd9d91731c0990e74a58239dc3e3fd2bf28dab23b55eafcdc47aaf87e \
https://github.com/BerriAI/litellm-admin-agent/archive/ef501e94bc9fbacb9233b922abf71427f030408c.tar.gz /tmp/liteadmin.tar.gz
ADD --checksum=sha256:4fb1847e28e2be30ba76dac94f02c4903291326516a0f4ba772f6ffb8b25d05c \
https://github.com/BerriAI/litellm-admin-agent/archive/5661c805ce479a0cf11b842ea133cf8d22cd7e34.tar.gz /tmp/liteadmin.tar.gz
RUN mkdir /tmp/liteadmin && tar xzf /tmp/liteadmin.tar.gz --strip-components=1 -C /tmp/liteadmin && \
uv venv /opt/liteadmin --python python3.13 && \
uv pip install --python /opt/liteadmin/bin/python --require-hashes -r /tmp/liteadmin/requirements.txt && \
@ -141,7 +142,7 @@ USER root
RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories
# node (without npm) is required by the prisma CLI at runtime
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
WORKDIR /app

View file

@ -616,15 +616,15 @@ The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws
### Verify Docker Image Signatures
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Releases are signed with the key introduced in [commit `e82a3c6`](https://github.com/BerriAI/litellm/commit/e82a3c646908ba0a6ee0beddc4310d7a6cac4065).
**Verify using the pinned commit hash (recommended):**
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the published signing key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
--key https://raw.githubusercontent.com/BerriAI/litellm/e82a3c646908ba0a6ee0beddc4310d7a6cac4065/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```

View file

@ -79,7 +79,7 @@ To qualify for a bounty, reports must include clear reproduction steps, a reprod
- Audit Logs with retention policy
- Control Allowed IP Addresses that can access your Cloud LiteLLM Instance
For security inquiries, please contact us at support@berri.ai
For security inquiries, please contact us at security@berri.ai
#### Supported data regions for LiteLLM Cloud

View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
@ -43,7 +43,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
@ -109,7 +109,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
USER root
RUN for i in 1 2 3; do \
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
@ -124,7 +124,12 @@ ENV HOME=/home/nonroot \
PYTHONUNBUFFERED=1 \
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
COPY --from=builder --chown=nonroot:nonroot /app /app
COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
COPY --from=builder --chown=nonroot:nonroot /app/backend /app/backend
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
COPY --from=builder /opt/prisma /opt/prisma
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \

View file

@ -4,8 +4,7 @@ Every pull request whose diff against its merge base touches one of the two cost
checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
get the file checks and additionally may only touch those two files or the legacy schema
path and may only add or update models.
get the file checks and additionally may only touch those two files and may only add or update models.
"""
from __future__ import annotations
@ -21,7 +20,6 @@ from typing import Final
COST_MAP_PATH: Final = "model_prices_and_context_window.json"
BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
LEGACY_SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
@ -110,7 +108,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
*(
f"bot PRs may only change the cost map files, not {path}"
for path in changed_files
if path not in GUARDED_PATHS and path != LEGACY_SCHEMA_PATH
if path not in GUARDED_PATHS
),
*(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map),
*(f"bot PRs may not remove fields: {ref}" for ref in removed_fields),

View file

@ -0,0 +1,41 @@
# Bedrock batch endpoints
Set `aws_bedrock_control_plane_endpoint` on a deployment when batch create, retrieve, and cancel requests must use a custom Bedrock control-plane endpoint. The proxy signs these requests for the configured URL. Without this setting, batch requests use the standard regional Bedrock endpoint
```yaml
model_list:
- model_name: bedrock-batch
litellm_params:
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_region_name: us-east-1
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_batch_role_arn: os.environ/AWS_BATCH_ROLE_ARN
s3_output_bucket_name: os.environ/AWS_S3_OUTPUT_BUCKET_NAME
aws_bedrock_control_plane_endpoint: os.environ/BEDROCK_CONTROL_PLANE_ENDPOINT
```
Set `BEDROCK_CONTROL_PLANE_ENDPOINT` to your Bedrock control-plane VPC endpoint or compatible gateway base URL, without `/model-invocation-job`. The proxy adds the operation path. The endpoint must be reachable from the proxy, and the configured AWS region must match the endpoint's signing region
`api_base` and `aws_bedrock_runtime_endpoint` configure runtime inference separately. They do not override the batch control-plane endpoint. `s3_endpoint_url` configures S3 file access separately
The control-plane setting belongs in deployment configuration. Sending `aws_bedrock_control_plane_endpoint` in an API request body is rejected. There is no automatic `AWS_BEDROCK_CONTROL_PLANE_ENDPOINT` environment fallback, so reference the desired environment variable explicitly as shown above
Start the proxy with this configuration and set `LITELLM_API_BASE` to its URL. Set `BEDROCK_BATCH_INPUT_URI` to a valid input JSONL file in S3 that the batch service role can read
```bash
curl -sS "$LITELLM_API_BASE/v1/batches" \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H 'Content-Type: application/json' \
-d "{\"model\":\"bedrock-batch\",\"input_file_id\":\"$BEDROCK_BATCH_INPUT_URI\",\"endpoint\":\"/v1/chat/completions\",\"completion_window\":\"24h\"}"
```
Use the returned `id` as `BATCH_ID` to retrieve or cancel the same job through the deployment's configured endpoint
```bash
curl -sS "$LITELLM_API_BASE/v1/batches/$BATCH_ID?model=bedrock-batch" \
-H "Authorization: Bearer $LITELLM_MASTER_KEY"
curl -sS -X POST "$LITELLM_API_BASE/v1/batches/$BATCH_ID/cancel?model=bedrock-batch" \
-H "Authorization: Bearer $LITELLM_MASTER_KEY"
```

View file

@ -1,4 +1,4 @@
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb
POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g==
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEG06dpKM+vEDnmp2zcjyXgVCXxzRg
JA5H2LnmwBKQafZGka6OOrWujCEWATQewiAgyD4WQdJ0pvU9RC4mvNAjvQ==
-----END PUBLIC KEY-----

View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@ -59,6 +59,7 @@ RUN apk add --no-cache \
python-3.13 \
python-3.13-dev \
openssl \
libssl3 \
openssl-dev \
nodejs \
npm \
@ -122,7 +123,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
USER root
# node (without npm) is required by the prisma CLI at runtime
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
WORKDIR /app

View file

@ -1,8 +1,8 @@
# syntax=docker/dockerfile:1.7
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@ -60,6 +60,7 @@ RUN for i in 1 2 3; do \
coreutils \
curl \
openssl \
libssl3 \
libsndfile \
nodejs \
npm && break || sleep 5; \
@ -134,7 +135,7 @@ RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
done && \
for i in 1 2 3; do \
apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs libevent && break || sleep 5; \
apk add --no-cache python-3.13 bash openssl libssl3 tzdata libsndfile nodejs libevent && break || sleep 5; \
done
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer

View file

@ -30,7 +30,7 @@ RUN uv venv --python python && \
"google-cloud-aiplatform==1.133.0" \
"google-genai==1.37.0" \
"anthropic[vertex]==0.84.0" \
"grpcio==1.78.0" \
"grpcio==1.84.1" \
"prometheus-client==0.20.0" \
"langfuse==2.59.7" \
"opentelemetry-api==1.28.0" \

View file

@ -15,9 +15,10 @@ sys.path.insert(
) # Adds the parent directory to the system path
import json
import sys
from typing import Any, List, Literal, Optional
from typing import Final, List, Literal, Optional
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -29,11 +30,13 @@ from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.llms.openai.common_utils import decoded_json_object
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata
from litellm.types.guardrails import GuardrailEventHooks
GUARDRAIL_NAME = "aporia"
_DECODED_JSON: Final = TypeAdapter(object)
class AporiaGuardrail(CustomGuardrail):
@ -68,7 +71,7 @@ class AporiaGuardrail(CustomGuardrail):
async def prepare_aporia_request(
self, new_messages: List[dict], response_string: Optional[str] = None
) -> dict:
data: dict[str, Any] = {}
data: dict[str, object] = {}
if new_messages is not None:
data["messages"] = new_messages
if response_string is not None:
@ -122,8 +125,10 @@ class AporiaGuardrail(CustomGuardrail):
verbose_proxy_logger.debug("Aporia AI response: %s", response.text)
if response.status_code == 200:
# check if the response was flagged
_json_response = response.json()
action: str = _json_response.get(
_json_response: Final = decoded_json_object(
_DECODED_JSON.validate_python(response.json())
)
action: Final = _json_response.get(
"action"
) # possible values are modify, passthrough, block, rephrase
if action == "block":

View file

@ -7,9 +7,10 @@
## This accepts a list of user id's for whom calls will be rejected
from typing import Literal, Optional
from typing import Final, Literal, Optional
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._internal_context import with_service_target
@ -19,6 +20,9 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import LiteLLM_EndUserTable, UserAPIKeyAuth
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET
from litellm.proxy.utils import PrismaClient
from litellm.repositories.table_repositories import EndUserRepository
_REQUEST_BODY_VALUE: Final = TypeAdapter(object)
class ENTERPRISE_BlockedUserList(CustomLogger):
@ -94,15 +98,14 @@ class ENTERPRISE_BlockedUserList(CustomLogger):
)
if end_user_cache_obj is None and self.prisma_client is not None:
# check db
end_user_obj = (
await self.prisma_client.db.litellm_endusertable.find_unique(
where={"user_id": user}
)
)
end_user_id: Final = _REQUEST_BODY_VALUE.validate_python(user)
end_user_obj = await EndUserRepository(
self.prisma_client
).table.find_unique(where={"user_id": end_user_id})
if end_user_obj is None: # user not in db - assume not blocked
end_user_obj = LiteLLM_EndUserTable(user_id=user, blocked=False)
cache.set_cache(key=cache_key, value=end_user_obj, ttl=60)
if end_user_obj is not None and end_user_obj.blocked is True:
if end_user_obj.blocked is True:
raise HTTPException(
status_code=400,
detail={

View file

@ -6,7 +6,10 @@
# +-----------------------------------------------+
# Thank you users! We ❤️ you! - Krrish & Ishaan
from typing import Final
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -15,6 +18,8 @@ from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import iter_message_text
from litellm.types.utils import CallTypesLiteral
_THRESHOLD_VALUE: Final = TypeAdapter(object)
class ENTERPRISE_GoogleTextModeration(CustomLogger):
user_api_key_cache = None
@ -69,9 +74,11 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
f"{category}_confidence_threshold",
default_confidence_threshold,
)
set_confidence_value = getattr(
self,
f"{category}_confidence_threshold",
set_confidence_value = _THRESHOLD_VALUE.validate_python(
getattr(
self,
f"{category}_confidence_threshold",
)
)
verbose_proxy_logger.info(
f"Google Text Moderation: {category}_confidence_threshold: {set_confidence_value}"

View file

@ -1,15 +1,20 @@
# this is an example endpoint to receive data from litellm
from typing import Final
from fastapi import FastAPI, HTTPException, Request
from pydantic import TypeAdapter
app = FastAPI()
_LOGGED_EVENT: Final = TypeAdapter(object)
@app.post("/log-event")
async def log_event(request: Request):
try:
print("Received /log-event request") # noqa
# Assuming the incoming request has JSON data
data = await request.json()
data: Final = _LOGGED_EVENT.validate_python(await request.json())
print("Received request data:") # noqa
print(data) # noqa

View file

@ -8,25 +8,30 @@
## This provides an LLM Guard Integration for content moderation on the proxy
import asyncio
from typing import Final, Optional
from collections.abc import Mapping
from typing import Final
import aiohttp
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.openai.common_utils import decoded_json_object
from litellm.proxy._types import UserAPIKeyAuth
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import CallTypesLiteral
_DECODED_JSON: Final = TypeAdapter(object)
class _ENTERPRISE_LLMGuard(CustomLogger):
# Class variables or attributes
def __init__(
self,
mock_testing: bool = False,
mock_redacted_text: Optional[dict] = None,
mock_redacted_text: Mapping[str, object] | None = None,
):
self.mock_redacted_text = mock_redacted_text
self.llm_guard_mode = litellm.llm_guard_mode
@ -66,7 +71,9 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
async with session.post(
analyze_url, json={"prompt": text}
) as response:
redacted_text = await response.json()
redacted_text = _DECODED_JSON.validate_python(
await response.json()
)
verbose_proxy_logger.debug(
f"LLM Guard: Received response - {redacted_text}"
)
@ -77,12 +84,13 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
"error": f"Invalid content moderation response: {redacted_text}"
},
)
if redacted_text.get("is_valid", None) is False:
redacted_response: Final = decoded_json_object(redacted_text)
if redacted_response.get("is_valid", None) is False:
raise HTTPException(
status_code=400,
detail={"error": "Violated content safety policy"},
)
sanitized_prompt = redacted_text.get("sanitized_prompt")
sanitized_prompt: Final = redacted_response.get("sanitized_prompt")
return sanitized_prompt if isinstance(sanitized_prompt, str) else text
except Exception as e:
verbose_proxy_logger.exception(

View file

@ -2,6 +2,8 @@ import os
from fastapi import HTTPException, status
from litellm.proxy._types import MODEL_LIST_ROUTES
class EnterpriseRouteChecks:
@staticmethod
@ -43,7 +45,7 @@ class EnterpriseRouteChecks:
# Routes that should remain accessible even when LLM API endpoints are disabled.
# These are read-only model listing routes needed by the Admin UI.
LLM_API_EXEMPT_ROUTES = ["/models", "/v1/models"]
LLM_API_EXEMPT_ROUTES = MODEL_LIST_ROUTES
@staticmethod
def should_call_route(route: str):

View file

@ -4,6 +4,7 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t
import asyncio
import json
import re
from collections.abc import Mapping, Sequence
from dataclasses import replace as dataclasses_replace
from datetime import datetime, timedelta, timezone
@ -13,6 +14,7 @@ from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tupl
from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
ManagedObjectCreatorAttribution,
ManagedObjectRow,
row_metadata,
)
import litellm
@ -39,7 +41,12 @@ if TYPE_CHECKING:
from litellm.types.utils import LiteLLMBatch
def _error_names_resource(error: Exception, resource_id: str) -> bool:
return resource_id in re.findall(r"""[^\s"'`]+""", str(error))
CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost"
RECENT_BATCH_POLL_WINDOW: Final = timedelta(hours=48)
PROVIDER_TERMINAL_BATCH_STATUSES: Final[Tuple[str, ...]] = (
"completed",
@ -84,11 +91,6 @@ def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.Li
return TeamRepository(prisma_client).table
def _row_metadata(row: object) -> dict[str, object]:
metadata: Final = getattr(row, "metadata", None)
return cast(dict[str, object], metadata) if isinstance(metadata, dict) else {} # cast-ok: prisma Json object column
class CheckBatchCost:
def __init__(
self,
@ -109,6 +111,11 @@ class CheckBatchCost:
# the guaranteed-failing primary query on every subsequent cycle.
self._has_batch_processed_column: bool = True
self.batch_processed_support_confirmed: bool = False
self._poll_cursor: tuple[datetime, str] | None = None
self._poll_before: datetime = datetime.now(timezone.utc)
self._arrival_cursor: tuple[datetime, str] | None = None
self._arrival_before: datetime = self._poll_before
self._poll_arrivals: bool = False
self._line_item_stores: Final[set[asyncio.Task[None]]] = set()
self._line_item_store_turn: Final = asyncio.Lock()
self._poll_idle: Final = asyncio.Event()
@ -161,29 +168,9 @@ class CheckBatchCost:
f"(older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days) as stale_expired"
)
if not self._has_batch_processed_column:
return
# A row already in a terminal status is never rewritten by the sweep above, so
# without this it keeps a poll-page slot forever and starves newer batches.
retired: Final = await _managed_object_table(self.prisma_client).update_many(
where={
"file_purpose": "batch",
"batch_processed": False,
"status": {"in": ["complete", "completed"]},
"created_at": {"lt": cutoff},
},
data={"batch_processed": True},
)
if retired > 0:
verbose_proxy_logger.warning(
f"CheckBatchCost: gave up on {retired} completed managed objects older than "
f"{MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days that were never costed"
)
async def _fallback_find_jobs(self) -> "Sequence[ManagedObjectRow]":
"""Query batch jobs without the batch_processed filter (for older schemas)."""
return await _managed_object_table(self.prisma_client).find_many(
return await self._find_jobs(
where={
"file_purpose": "batch",
"status": {
@ -197,10 +184,59 @@ class CheckBatchCost:
]
},
},
take=MAX_OBJECTS_PER_POLL_CYCLE,
order={"created_at": "asc"},
)
async def _find_jobs(self, where: Mapping[str, object]) -> Sequence[ManagedObjectRow]:
arrivals: Final = self._poll_arrivals
page: Final = await self._find_job_page(where, arrivals=arrivals)
self._poll_arrivals = not arrivals if page else arrivals
if page:
return page
return await self._find_job_page(where, arrivals=not arrivals)
async def _find_job_page(self, where: Mapping[str, object], *, arrivals: bool) -> Sequence[ManagedObjectRow]:
cursor: Final = self._arrival_cursor if arrivals else self._poll_cursor
if cursor is None:
if arrivals:
self._arrival_before = datetime.now(timezone.utc)
else:
self._poll_before = datetime.now(timezone.utc) - RECENT_BATCH_POLL_WINDOW
before: Final = self._arrival_before if arrivals else self._poll_before
page: Final = await _managed_object_table(self.prisma_client).find_many(
where={
**where,
"created_at": {
"lte": before,
**({"gt": before - RECENT_BATCH_POLL_WINDOW} if arrivals else {}),
},
**(
{
"OR": [
{"created_at": {"gt": cursor[0]}},
{"created_at": cursor[0], "id": {"gt": cursor[1]}},
]
}
if cursor is not None
else {}
),
},
take=MAX_OBJECTS_PER_POLL_CYCLE,
order=[{"created_at": "asc"}, {"id": "asc"}],
)
next_cursor: Final = (page[-1].created_at, page[-1].id) if len(page) == MAX_OBJECTS_PER_POLL_CYCLE else None
if arrivals:
self._arrival_cursor = next_cursor
else:
self._poll_cursor = next_cursor
return page
def _unprocessed_snapshot(self, job: ManagedObjectRow) -> Mapping[str, object]:
return {
"id": job.id,
"updated_at": job.updated_at,
**({"batch_processed": False} if self._has_batch_processed_column else {}),
}
async def _retire_job(self, job: "ManagedObjectRow", reason: str) -> None:
"""
Take a row that can never be costed out of the poll page. Leaving it selectable
@ -208,24 +244,19 @@ class CheckBatchCost:
once enough such rows accumulate no newer batch is ever reached. Older schemas
without batch_processed can only be excluded through the status filter.
"""
data: Final = (
{"batch_processed": True}
if self._has_batch_processed_column
else {"status": "stale_expired"}
)
data: Final = {"batch_processed": True} if self._has_batch_processed_column else {"status": "stale_expired"}
try:
await _managed_object_table(self.prisma_client).update(
where={"id": job.id},
retired: Final = await _managed_object_table(self.prisma_client).update_many(
where=self._unprocessed_snapshot(job),
data=data,
)
except Exception as db_err:
verbose_proxy_logger.error(
f"CheckBatchCost: failed to retire uncostable job {job.id} ({reason}): {db_err}"
)
verbose_proxy_logger.error(f"CheckBatchCost: failed to retire uncostable job {job.id} ({reason}): {db_err}")
return
if retired == 0:
return
verbose_proxy_logger.warning(
f"CheckBatchCost: job {job.id} can never be costed ({reason}), "
"so it will no longer be polled"
f"CheckBatchCost: job {job.id} can never be costed ({reason}), so it will no longer be polled"
)
async def _claim_job_for_costing(self, job: "ManagedObjectRow", response: "LiteLLMBatch") -> bool:
@ -312,13 +343,13 @@ class CheckBatchCost:
from litellm.exceptions import NotFoundError
return isinstance(error, (NotFoundError, openai.NotFoundError)) and batch_id in str(error)
return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, batch_id)
def _batch_deployment_exists(self, model_id: str) -> bool:
"""A 404 only proves the batch is gone when it came from the batch's own
deployment. Once that deployment leaves the router, default fallbacks can
silently send the retrieve to a provider that never saw the batch, so its
404 must not retire the row; the staleness sweep bounds it instead."""
404 must not retire a potentially recoverable row."""
return self.llm_router.get_deployment(model_id=model_id) is not None
@staticmethod
@ -333,7 +364,7 @@ class CheckBatchCost:
if not output_file_id:
return False
return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, output_file_id)
async def _finalize_unbilled_terminal_job(
self, job: "ManagedObjectRow", response: "LiteLLMBatch"
@ -355,18 +386,18 @@ class CheckBatchCost:
db_batch_object=job,
unified_batch_id=_is_base64_encoded_unified_file_id(job.unified_object_id),
)
update_data: Final[dict] = {
update_data: Final[dict[str, object]] = {
"status": response.status,
"file_object": response.model_dump_json(),
**({"batch_processed": True} if self._has_batch_processed_column else {}),
}
await _managed_object_table(self.prisma_client).update(
where={"id": job.id},
finalized: Final = await _managed_object_table(self.prisma_client).update_many(
where=self._unprocessed_snapshot(job),
data=update_data,
)
verbose_proxy_logger.info(
f"CheckBatchCost: marked job {job.id} as {response.status} in DB"
)
if finalized == 0:
return False
verbose_proxy_logger.info(f"CheckBatchCost: marked job {job.id} as {response.status} in DB")
return True
except Exception as db_err:
verbose_proxy_logger.error(
@ -817,20 +848,26 @@ class CheckBatchCost:
)
return None
try:
await logging_obj.async_success_handler(
result=response,
batch_cost=batch_result.cost,
batch_usage=batch_result.usage,
batch_models=batch_result.models,
batch_successful_requests=batch_result.successful_requests,
batch_failed_requests=batch_result.failed_requests,
batch_prompt_cost=batch_result.prompt_cost,
batch_completion_cost=batch_result.completion_cost,
)
except Exception:
await self._release_job_claim(job, original_status, original_file_object)
raise
from litellm.proxy.hooks.proxy_track_cost_callback import track_batch_accounting
with track_batch_accounting(logging_obj.litellm_call_id) as accounting:
try:
await logging_obj.async_success_handler(
result=response,
batch_cost=batch_result.cost,
batch_usage=batch_result.usage,
batch_models=batch_result.models,
batch_successful_requests=batch_result.successful_requests,
batch_failed_requests=batch_result.failed_requests,
batch_prompt_cost=batch_result.prompt_cost,
batch_completion_cost=batch_result.completion_cost,
)
finally:
if accounting.retryable_failure:
await self._release_job_claim(job, original_status, original_file_object)
if accounting.retryable_failure:
return None
# Record batch duration (completed_at - created_at)
if prom_logger and response.completed_at and response.created_at:
@ -1003,8 +1040,8 @@ class CheckBatchCost:
from litellm.proxy.proxy_server import proxy_config
from litellm.secret_managers.main import str_to_bool
api_key: Final = getattr(job, "api_key", None)
team_id: Final = getattr(job, "team_id", None)
api_key: Final = job.api_key if hasattr(job, "api_key") else None
team_id: Final = job.team_id if hasattr(job, "team_id") else None
has_key_row: Final = api_key is not None and is_valid_sha256_hash(api_key)
try:
key_row: Final = (
@ -1022,7 +1059,9 @@ class CheckBatchCost:
if (has_key_row and key_row is None) or (team_id and team_row is None):
return True
settings: Final = get_dynamic_logging_metadata(
UserAPIKeyAuth(metadata=_row_metadata(key_row), team_metadata=_row_metadata(team_row), team_id=team_id),
UserAPIKeyAuth(
metadata=dict(row_metadata(key_row)), team_metadata=dict(row_metadata(team_row)), team_id=team_id
),
proxy_config,
)
callback_vars: Final = (settings.callback_vars if settings is not None else None) or {}
@ -1047,6 +1086,7 @@ class CheckBatchCost:
"""
try:
from litellm.integrations.prometheus import PrometheusLogger
prom_logger = PrometheusLogger.get_instance()
except Exception as e:
verbose_proxy_logger.error(f"CheckBatchCost: could not get Prometheus logger: {e}")
@ -1071,7 +1111,7 @@ class CheckBatchCost:
# endpoint may transition a batch to "complete" before
# CheckBatchCost runs. The batch_processed=False filter
# already prevents reprocessing finished batches.
jobs = await _managed_object_table(self.prisma_client).find_many(
jobs = await self._find_jobs(
where={
"file_purpose": "batch",
"batch_processed": False,
@ -1084,8 +1124,6 @@ class CheckBatchCost:
]
},
},
take=MAX_OBJECTS_PER_POLL_CYCLE,
order={"created_at": "asc"},
)
self.batch_processed_support_confirmed = True
except Exception as query_err:
@ -1152,7 +1190,7 @@ class CheckBatchCost:
continue
verbose_proxy_logger.error(
f"CheckBatchCost: failed to track cost for batch {batch_id} "
f"(job {job.id}); leaving it unprocessed so the next poll retries: {tracking_err}"
f"(job {job.id}): {tracking_err}"
)
self._record_error(prom_logger, "cost_tracking_error")
continue

View file

@ -7,6 +7,12 @@ attribution, priced against the deployment that served it. The poll's status che
with BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN and client reads are marked managed by the
managed files hook, so both stay free. The spend event carries the id the client holds, the row's
unified_object_id, so its spend log is found under that id with the _background_cost suffix.
The spend event reaches the creating key's or team's own logging callbacks the way that creator's
requests do. Its message content is redacted when the creator's key or team logging turns message
logging off, else as the proxy is set, the way the batch poll decides it. The row is claimed before
the creator is read, so reads that lose the race skip the lookups. A creator whose lookup fails, or
whose logging settings the logger rejects, is still billed, through the proxy's own callbacks only
and with its message content redacted.
A read that finds no row, because the create could not store one, stores it under the reading key
and bills it the same way, so a response the create left untracked is still billed once. A streamed
background create is free on its own; its stream stores the row under the creating key and bills
@ -29,6 +35,7 @@ from typing import TYPE_CHECKING, Final, cast
from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
ManagedObjectCreatorAttribution,
ManagedObjectRow,
ResponseCreator,
)
from pydantic import BaseModel, ConfigDict, TypeAdapter
@ -54,12 +61,29 @@ from litellm.types.utils import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.proxy.proxy_server import ProxyConfig
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.repositories.prisma_protocols import TableActions
from litellm.router import Router
_STORED_RESPONSE_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
_CALLBACK_LIST_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...])
class _RoutedModelInfo(BaseModel):
model_config = ConfigDict(frozen=True)
id: str | None = None
class _RoutedDeployment(BaseModel):
model_config = ConfigDict(frozen=True)
model_info: _RoutedModelInfo = _RoutedModelInfo()
_ROUTED_DEPLOYMENTS: Final = TypeAdapter(tuple[_RoutedDeployment, ...])
_RETRIEVE_RESPONSE_CALL_TYPE: Final = "aget_responses"
_CALLBACK_LIST_FIELDS: Final = frozenset({"success_callback", "failure_callback", "litellm_disabled_callbacks"})
_REJECTED_CREATOR_LOGGING: Final[Mapping[str, object]] = MappingProxyType({"turn_off_message_logging": True})
@dataclass(frozen=True, slots=True)
@ -76,6 +100,7 @@ class BilledBackgroundResponse:
custom_llm_provider: str | None
litellm_params: Mapping[str, object]
metadata: Mapping[str, object]
creator_logging: Mapping[str, object]
@dataclass(frozen=True, slots=True)
@ -94,11 +119,51 @@ def _without_read_price(response: ResponsesAPIResponse) -> ResponsesAPIResponse:
return unpriced
async def log_billed_background_response(billed: BilledBackgroundResponse) -> BilledCharge:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
def _string_callbacks(callbacks: object) -> tuple[str, ...]:
if not isinstance(callbacks, list):
return ()
return tuple(
callback for callback in _CALLBACK_LIST_ADAPTER.validate_python(callbacks) if isinstance(callback, str)
)
logging_obj: Final = LiteLLMLogging(
def _running_proxy_config() -> "ProxyConfig":
from litellm.proxy.proxy_server import proxy_config
return proxy_config
def creator_logging_fields(creator: ResponseCreator, proxy_config: "ProxyConfig | None") -> Mapping[str, object]:
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.litellm_pre_call_utils import key_and_team_logging_fields
if not creator.settings_read:
return _REJECTED_CREATOR_LOGGING
try:
return key_and_team_logging_fields(
user_api_key_dict=UserAPIKeyAuth(
metadata=dict(creator.key_metadata),
team_id=creator.team_id,
team_metadata=dict(creator.team_metadata),
),
proxy_config=proxy_config if proxy_config is not None else _running_proxy_config(),
)
except Exception as e:
verbose_proxy_logger.error(
f"CheckResponsesCost: could not read the creator's logging settings, so only the proxy's "
f"callbacks get the billed event: {e}"
)
return _REJECTED_CREATOR_LOGGING
def _billed_logging_obj(billed: BilledBackgroundResponse, creator_logging: Mapping[str, object]) -> "LiteLLMLogging":
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.utils import split_dynamic_success_callbacks
sync_callbacks, async_callbacks = split_dynamic_success_callbacks(
_string_callbacks(creator_logging.get("success_callback"))
)
return LiteLLMLogging(
model=billed.model,
messages=[{"role": "user", "content": "<retrieve_response>"}],
stream=False,
@ -106,7 +171,27 @@ async def log_billed_background_response(billed: BilledBackgroundResponse) -> Bi
start_time=datetime.now(),
litellm_call_id=str(uuid.uuid4()),
function_id=str(uuid.uuid4()),
dynamic_success_callbacks=list(sync_callbacks),
dynamic_async_success_callbacks=list(async_callbacks) or None,
kwargs={key: value for key, value in creator_logging.items() if key not in _CALLBACK_LIST_FIELDS},
)
def _creator_or_proxy_logging_obj(billed: BilledBackgroundResponse) -> "LiteLLMLogging":
try:
return _billed_logging_obj(billed, billed.creator_logging)
except Exception as e:
verbose_proxy_logger.error(
f"CheckResponsesCost: the creator's logging settings were rejected, so only the proxy's "
f"callbacks get the billed event: {e}"
)
return _billed_logging_obj(billed, _REJECTED_CREATOR_LOGGING)
async def log_billed_background_response(billed: BilledBackgroundResponse) -> BilledCharge:
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
logging_obj: Final = _creator_or_proxy_logging_obj(billed)
logging_obj.update_environment_variables(
litellm_params={**billed.litellm_params, "metadata": dict(billed.metadata)},
optional_params={},
@ -174,6 +259,7 @@ class CheckResponsesCost:
prisma_client: "PrismaClient",
llm_router: "Router",
log_billed_response: BilledResponseLogger = log_billed_background_response,
proxy_config: "ProxyConfig | None" = None,
):
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.router import Router
@ -183,6 +269,7 @@ class CheckResponsesCost:
self.llm_router: Router = llm_router
self._creator_attribution: Final = ManagedObjectCreatorAttribution(prisma_client, "CheckResponsesCost")
self._log_billed_response: Final = log_billed_response
self._proxy_config: Final = proxy_config
def _resolve_deployment(self, response_id: str) -> Deployment | None:
model_id: str | None = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
@ -294,11 +381,12 @@ class CheckResponsesCost:
data={"status": job.status or PENDING_RESPONSE_STATUSES[0]},
)
async def _billed_response(
def _billed_response(
self,
job: ManagedObjectRow,
target: _PollTarget,
response: ResponsesAPIResponse,
creator: ResponseCreator,
) -> BilledBackgroundResponse:
deployment: Final = target.deployment
return BilledBackgroundResponse(
@ -310,12 +398,26 @@ class CheckResponsesCost:
litellm_params=_deployment_litellm_params(deployment) if deployment is not None else MappingProxyType({}),
metadata=MappingProxyType(
{
**(await self._creator_attribution.response_metadata(job, job.unified_object_id)),
**creator.metadata,
INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
**target.model_metadata,
**(_deployment_metadata(deployment) if deployment is not None else {}),
**self._requested_model_group(target, job.team_id),
}
),
creator_logging=creator_logging_fields(creator, self._proxy_config),
)
def _requested_model_group(self, target: _PollTarget, team_id: str | None) -> Mapping[str, object]:
requested: Final = target.model_metadata.get("model_group")
if target.deployment is None or not isinstance(requested, str):
return MappingProxyType({})
routed: Final = _ROUTED_DEPLOYMENTS.validate_python(
self.llm_router.get_model_list(model_name=requested, team_id=team_id) or ()
)
serving_id: Final = target.deployment.model_info.id
return MappingProxyType(
{"model_group": requested} if any(row.model_info.id == serving_id for row in routed) else {}
)
async def _bill_once(
@ -328,7 +430,8 @@ class CheckResponsesCost:
verbose_proxy_logger.info(f"Response {job.unified_object_id} was already billed elsewhere, skipping")
return None
try:
charge: Final = await self._log_billed_response(await self._billed_response(job, target, response))
creator: Final = await self._creator_attribution.response_creator(job, job.unified_object_id)
charge: Final = await self._log_billed_response(self._billed_response(job, target, response, creator))
except BaseException:
await self._release_billing_claim(job)
raise

View file

@ -9,12 +9,17 @@ from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final, Protocol, cast
from datetime import datetime
from typing import TYPE_CHECKING, Final, Protocol, TypeGuard, cast
from pydantic import TypeAdapter
from litellm._logging import verbose_proxy_logger
from litellm.constants import CLI_SESSION_KEY_PREFIX
from litellm.repositories.table_repositories import (
DeletedVerificationTokenRepository,
DeprecatedVerificationTokenRepository,
)
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
@ -27,6 +32,12 @@ if TYPE_CHECKING:
class ManagedObjectRow(Protocol):
@property
def created_at(self) -> datetime: ...
@property
def updated_at(self) -> datetime: ...
@property
def id(self) -> str: ...
@ -63,6 +74,7 @@ class ManagedObjectRow(Protocol):
_TAG_LIST_ADAPTER: Final[TypeAdapter[list[object]]] = TypeAdapter(list[object])
_KEY_OWNER_SCOPE_PREFIX: Final = "key:"
_MAX_ROTATION_HOPS: Final = 16
def _string_tags(request_tags: object) -> list[str] | None:
@ -89,6 +101,20 @@ def _team_table(prisma_client: PrismaClient) -> TableActions[prisma_models.LiteL
return TeamRepository(prisma_client).table
def _rotated_token_table(
prisma_client: PrismaClient,
) -> TableActions[prisma_models.LiteLLM_DeprecatedVerificationToken]:
return DeprecatedVerificationTokenRepository(prisma_client).table
def _deleted_token_table(prisma_client: PrismaClient) -> TableActions[prisma_models.LiteLLM_DeletedVerificationToken]:
return DeletedVerificationTokenRepository(prisma_client).table
def _deleted_team_table(prisma_client: PrismaClient) -> TableActions[prisma_models.LiteLLM_DeletedTeamTable]:
return TeamRepository(prisma_client).deleted_table
@dataclass(frozen=True, slots=True)
class _CreatorRows:
key: prisma_models.LiteLLM_VerificationToken | None
@ -96,6 +122,15 @@ class _CreatorRows:
user: prisma_models.LiteLLM_UserTable | None
@dataclass(frozen=True, slots=True)
class ResponseCreator:
metadata: Mapping[str, object]
key_metadata: Mapping[str, object]
team_id: str | None
team_metadata: Mapping[str, object]
settings_read: bool = True
class ManagedObjectCreatorAttribution:
"""Rebuilds the spend-tracking metadata of whoever created a managed object, so the poll
job's spend row is attributed like the creating request: key hash, team, organization,
@ -107,8 +142,15 @@ class ManagedObjectCreatorAttribution:
user's alias, because a resolvable name is more useful on the spend row than a null.
user_api_key_org_id is resolved too: the spend update writer reads it off this metadata
to increment organization spend, so leaving it out drops the cost from org accounting.
A background response also carries its end user and the key, team, and user budgets, the
way the read that billed it before did, and never names a key-scoped creator as its user.
A background response also carries its end user, the creating key's project and its alias,
and the key, team, and user budgets and per-model budgets, the way the read that billed it
before did, and never names a key-scoped creator as its user. A key that sets no budget of
its own takes its budget tier's, the same rule auth applies. Its creator's key and team
metadata come along too, so the billed event reaches the key's and team's logging callbacks
like that read did. A rotated key's metadata is read from the live key its rotations lead
to, and a deleted key's or team's from its deleted row. A lookup that fails leaves the
creator's settings unread and the attribution to the rows that still load, so the
response still bills and still counts toward their budgets.
"""
def __init__(self, prisma_client: PrismaClient, poller_name: str) -> None:
@ -119,17 +161,70 @@ class ManagedObjectCreatorAttribution:
rows: Final = await self._rows(job, object_id)
return {"user_api_key_user_id": job.created_by, **_shared_metadata(job, rows)}
async def response_metadata(self, job: ManagedObjectRow, object_id: str) -> Mapping[str, object]:
rows: Final = await self._rows(job, object_id)
end_user_id: Final = getattr(job, "end_user_id", None)
return {
"user_api_key_user_id": _creator_user_id(job.created_by),
**_shared_metadata(job, rows),
**({"user_api_key_end_user_id": end_user_id} if end_user_id else {}),
**_budget_fields(rows.key, "user_api_key_spend", "user_api_key_max_budget"),
**_budget_fields(rows.team, "user_api_key_team_spend", "user_api_key_team_max_budget"),
**_budget_fields(rows.user, "user_api_key_user_spend", "user_api_key_user_max_budget"),
}
async def response_creator(self, job: ManagedObjectRow, object_id: str) -> ResponseCreator:
try:
return await self._read_response_creator(job, object_id)
except Exception as e:
verbose_proxy_logger.error(
f"{self._poller_name}: could not read the creator of {object_id}, so it bills without their "
f"settings: {e}"
)
return ResponseCreator(
metadata=_response_creator_metadata(job, await self._rows(job, object_id)),
key_metadata={},
team_id=getattr(job, "team_id", None),
team_metadata={},
settings_read=False,
)
async def _read_response_creator(self, job: ManagedObjectRow, object_id: str) -> ResponseCreator:
api_key: Final = getattr(job, "api_key", None)
team_id: Final = getattr(job, "team_id", None)
rows: Final = _CreatorRows(
key=await self._live_key_row(api_key, job.created_by),
team=await self._live_team_row(team_id),
user=await self._user_row(object_id, _creator_user_id(job.created_by)),
)
return ResponseCreator(
metadata=_response_creator_metadata(job, rows),
key_metadata=row_metadata(rows.key) if rows.key is not None else await self._retired_key_metadata(api_key),
team_id=team_id,
team_metadata=(
row_metadata(rows.team) if rows.team is not None else await self._deleted_team_metadata(team_id)
),
)
async def _retired_key_metadata(self, api_key: str | None) -> Mapping[str, object]:
if not _names_a_key_row(api_key):
return {}
current_token: Final = await self._token_rotated_to(api_key, _MAX_ROTATION_HOPS)
current_key: Final = (
await _token_table(self._prisma_client).find_unique(where={"token": current_token})
if current_token != api_key
else None
)
if current_key is not None:
return row_metadata(current_key)
return row_metadata(
await _deleted_token_table(self._prisma_client).find_first(
where={"token": current_token}, order={"deleted_at": "desc"}
)
)
async def _token_rotated_to(self, token: str, hops_left: int) -> str:
rotation: Final = await _rotated_token_table(self._prisma_client).find_unique(where={"token": token})
if rotation is None or hops_left == 0:
return token
return await self._token_rotated_to(rotation.active_token_id, hops_left - 1)
async def _deleted_team_metadata(self, team_id: str | None) -> Mapping[str, object]:
if not team_id:
return {}
return row_metadata(
await _deleted_team_table(self._prisma_client).find_first(
where={"team_id": team_id}, order={"deleted_at": "desc"}
)
)
async def _rows(self, job: ManagedObjectRow, object_id: str) -> _CreatorRows:
return _CreatorRows(
@ -152,23 +247,57 @@ class ManagedObjectCreatorAttribution:
async def _key_row(
self, object_id: str, api_key: str | None, created_by: str | None
) -> prisma_models.LiteLLM_VerificationToken | None:
if not api_key or _is_cli_session_key(api_key, created_by):
return None
try:
return await _token_table(self._prisma_client).find_unique(where={"token": api_key})
return await self._live_key_row(api_key, created_by)
except Exception as e:
verbose_proxy_logger.error(f"{self._poller_name}: could not look up the creating key for {object_id}: {e}")
return None
async def _team_row(self, object_id: str, team_id: str | None) -> prisma_models.LiteLLM_TeamTable | None:
if not team_id:
async def _live_key_row(
self, api_key: str | None, created_by: str | None
) -> prisma_models.LiteLLM_VerificationToken | None:
if not api_key or _is_cli_session_key(api_key, created_by):
return None
return await _token_table(self._prisma_client).find_unique(
where={"token": api_key}, include={"litellm_budget_table": True, "litellm_project_table": True}
)
async def _team_row(self, object_id: str, team_id: str | None) -> prisma_models.LiteLLM_TeamTable | None:
try:
return await _team_table(self._prisma_client).find_unique(where={"team_id": team_id})
return await self._live_team_row(team_id)
except Exception as e:
verbose_proxy_logger.error(f"{self._poller_name}: could not look up team {team_id} for {object_id}: {e}")
return None
async def _live_team_row(self, team_id: str | None) -> prisma_models.LiteLLM_TeamTable | None:
if not team_id:
return None
return await _team_table(self._prisma_client).find_unique(where={"team_id": team_id})
def _response_creator_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str, object]:
end_user_id: Final = getattr(job, "end_user_id", None)
return {
"user_api_key_user_id": _creator_user_id(job.created_by),
**_shared_metadata(job, rows),
**({"user_api_key_end_user_id": end_user_id} if end_user_id else {}),
"user_api_key_project_id": getattr(rows.key, "project_id", None),
"user_api_key_project_alias": getattr(getattr(rows.key, "litellm_project_table", None), "project_alias", None),
**_key_budget_fields(rows.key),
**_budget_fields(
rows.team,
"user_api_key_team_spend",
"user_api_key_team_max_budget",
"user_api_key_team_model_max_budget",
),
**_budget_fields(
rows.user,
"user_api_key_user_spend",
"user_api_key_user_max_budget",
"user_api_key_user_model_max_budget",
),
}
def _shared_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str, object]:
api_key: Final = getattr(job, "api_key", None)
@ -196,6 +325,19 @@ def _shared_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str,
}
def row_metadata(row: object) -> Mapping[str, object]:
metadata: Final = getattr(row, "metadata", None)
if not isinstance(metadata, dict):
return {}
return cast(Mapping[str, object], metadata) # cast-ok: prisma Json object column
def _names_a_key_row(api_key: object) -> TypeGuard[str]:
from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash
return isinstance(api_key, str) and is_valid_sha256_hash(api_key)
def _is_cli_session_key(api_key: str, created_by: str | None) -> bool:
return bool(created_by) and api_key == f"{CLI_SESSION_KEY_PREFIX}-{created_by}"
@ -212,7 +354,28 @@ def _organization_id(row: object) -> str | None:
return cast("str | None", getattr(row, "organization_id", None)) if row is not None else None
def _budget_fields(row: object, spend_key: str, max_budget_key: str) -> Mapping[str, object]:
def _budget_fields(row: object, spend_key: str, max_budget_key: str, model_max_budget_key: str) -> Mapping[str, object]:
if row is None:
return {}
return {spend_key: getattr(row, "spend", None), max_budget_key: getattr(row, "max_budget", None)}
return {
spend_key: getattr(row, "spend", None),
max_budget_key: getattr(row, "max_budget", None),
model_max_budget_key: getattr(row, "model_max_budget", None),
}
def _key_budget_fields(key: object) -> Mapping[str, object]:
if key is None:
return {}
tier: Final = getattr(key, "litellm_budget_table", None)
return {
"user_api_key_spend": getattr(key, "spend", None),
"user_api_key_max_budget": _own_or_tier_budget(key, tier, "max_budget"),
"user_api_key_model_max_budget": _own_or_tier_budget(key, tier, "model_max_budget"),
}
def _own_or_tier_budget(key: object, tier: object, field: str) -> object:
own: Final = getattr(key, field, None)
tier_value: Final = getattr(tier, field, None)
return own if tier_value is None or own not in (None, {}) else tier_value

View file

@ -44,6 +44,7 @@ from litellm.constants import (
)
from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.litellm_core_utils.internal_call_metadata import requests_background
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -133,7 +134,7 @@ if TYPE_CHECKING:
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
from litellm.proxy.utils import PrismaClient as _PrismaClient
Span = Union[_Span, Any]
Span = _Span
InternalUsageCache = _InternalUsageCache
PrismaClient = _PrismaClient
else:
@ -490,6 +491,14 @@ _PROVIDER_FILE_ID_KEYS: Final = tuple(dict.fromkeys(chain.from_iterable(_PROVIDE
_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS: Final = (0.5, 1.0, 2.0)
class _SupportsKeyedGet(Protocol):
def get(self, key: str, default: object, /) -> object: ...
def _keyed_get(source: object, key: str, default: object = None) -> object:
return require_attributes(source, _SupportsKeyedGet, "get").get(key, default)
class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Class variables or attributes
def __init__(
@ -1611,7 +1620,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
return holders if holders or "input_file_id" in request else healthy_deployments
async def async_pre_call_deployment_hook(
self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
self, kwargs: dict[str, object], call_type: Optional[CallTypes]
) -> Optional[dict]:
"""
Allow modifying the request just before it's sent to the deployment.
@ -1623,11 +1632,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping"))
# model_info may be at top-level or nested under litellm_metadata
# (batch/file operations use litellm_metadata)
model_id = cast(Optional[str], kwargs.get("model_info", {}).get("id", None))
model_id = cast(Optional[str], _keyed_get(kwargs.get("model_info", {}), "id"))
if model_id is None:
model_id = cast(
Optional[str],
kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None),
_keyed_get(_keyed_get(kwargs.get("litellm_metadata", {}), "model_info", {}), "id"),
)
for accessor_key in accessor_keys:
input_file_id = cast(Optional[str], kwargs.get(accessor_key))
@ -1824,7 +1833,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
),
)
async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span) -> dict:
async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span | None) -> dict:
"""
Get model-specific file IDs for a list of proxy file IDs.
Returns a dictionary mapping litellm_proxy/ file_id -> model_id -> model_file_id
@ -1866,7 +1875,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
llm_router: Optional[Router],
_create_file_request: CreateFileRequest,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
) -> List[OpenAIFileObject]:
if llm_router is None:
raise Exception("LLM Router not initialized. Ensure models added to proxy.")
@ -1882,7 +1891,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
create_file_request: CreateFileRequest,
llm_router: Router,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
user_api_key_dict: UserAPIKeyAuth,
) -> OpenAIFileObject:
responses = await self.create_file_for_each_model(
@ -1949,7 +1958,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
file_objects: List[OpenAIFileObject],
create_file_request: CreateFileRequest,
internal_usage_cache: InternalUsageCache,
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
target_model_names_list: List[str],
) -> OpenAIFileObject:
## GET THE FILE TYPE FROM THE CREATE FILE REQUEST
@ -2652,7 +2661,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Continue with other files even if one fails
continue
def _get_content_type_from_file_object(self, file_object: Optional[Any]) -> str:
def _get_content_type_from_file_object(self, file_object: object) -> str:
"""
Determine content type from file object.

View file

@ -29,7 +29,7 @@ if TYPE_CHECKING:
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
from litellm.proxy.utils import PrismaClient as _PrismaClient
Span = Union[_Span, Any]
Span = _Span
InternalUsageCache = _InternalUsageCache
PrismaClient = _PrismaClient
else:
@ -107,7 +107,7 @@ class _PROXY_LiteLLMManagedVectorStores(
llm_router: Router,
model: str,
request_data: dict[str, object] | VectorStoreCreateOptionalRequestParams,
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
) -> VectorStoreCreateResponse:
"""
Create a vector store for a specific model.
@ -134,7 +134,7 @@ class _PROXY_LiteLLMManagedVectorStores(
create_request: VectorStoreCreateOptionalRequestParams,
llm_router: Router,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
user_api_key_dict: UserAPIKeyAuth,
) -> VectorStoreCreateResponse:
"""

View file

@ -36,6 +36,7 @@ from litellm.repositories.budget_repository import BudgetRepository
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.project_repository import ProjectRepository
from litellm.repositories.table_repositories import PrismaTableRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
@ -73,6 +74,16 @@ def _verification_token_table(
return VerificationTokenRepository(prisma_client).table
class _WriterVerificationTokenRepository(PrismaTableRepository["prisma_models.LiteLLM_VerificationToken"]):
table_name = "litellm_verificationtoken"
def _writer_verification_token_table(
prisma_client: PrismaClient,
) -> TableActions["prisma_models.LiteLLM_VerificationToken"]:
return _WriterVerificationTokenRepository(prisma_client, use_writer=True).table
def _budget_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_BudgetTable"]:
return BudgetRepository(prisma_client).table
@ -849,7 +860,7 @@ async def update_project(
else None
)
if current_project is not None and data.team_id != current_project.team_id:
mismatched_key_count: Final = await prisma_client.writer_db.litellm_verificationtoken.count(
mismatched_key_count: Final = await _writer_verification_token_table(prisma_client).count(
where={
"project_id": data.project_id,
"OR": [{"team_id": {"not": data.team_id}}, {"team_id": None}],

View file

@ -14,6 +14,7 @@ from collections.abc import Mapping
from typing import TYPE_CHECKING, Final, List, Optional, Protocol
from fastapi import APIRouter, Depends, HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -37,6 +38,7 @@ if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
router = APIRouter()
_DECODED_JSON: Final = TypeAdapter(object)
class ManagedVectorStoreRow(Protocol):
@ -348,11 +350,11 @@ async def update_vector_store(
raise HTTPException(status_code=500, detail="Database not connected")
try:
update_data = data.model_dump(exclude_unset=True)
vector_store_id: Final[str] = update_data.pop("vector_store_id")
update_data = data.model_dump(exclude_unset=True, exclude={"vector_store_id"})
vector_store_id: Final = data.vector_store_id
if update_data.get("vector_store_metadata") is not None:
update_data["vector_store_metadata"] = safe_dumps(
update_data["vector_store_metadata"]
_DECODED_JSON.validate_python(update_data["vector_store_metadata"])
)
updated = await managed_vector_store_table(prisma_client).update(

View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.78"
version = "0.1.79"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.78"
version = "0.1.79"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
ARG PGBOUNCER_VERSION=1.26.0
@ -59,7 +59,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
@ -125,7 +125,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN for i in 1 2 3; do \
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic libevent && break; \
apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic libevent && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
@ -140,7 +140,12 @@ ENV HOME=/home/nonroot \
PYTHONUNBUFFERED=1 \
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
COPY --from=builder --chown=nonroot:nonroot /app /app
COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
COPY --from=builder /opt/prisma /opt/prisma
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer

View file

@ -1,7 +1,7 @@
"""Path allowlist for the gateway component.
The gateway exposes the LLM data-plane surface: chat/completions, embeddings,
audio, batches, files, fine-tuning, rerank, decisions, ocr, rag, video, search, image,
audio, batches, files, fine-tuning, rerank, decisions, extraction, ocr, rag, video, search, image,
responses, vector stores, passthrough providers, realtime websockets, MCP
tool-call endpoints, and operational endpoints (/health, /metrics, and the
/debug/memory/summary read of the serving worker's RSS).
@ -64,6 +64,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
"/decisions",
"/v1/systemone",
"/systemone",
"/v1/extraction",
"/extraction",
"/v1/ocr",
"/ocr",
"/v1/rag/",

View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.110"
version = "0.4.111"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.110"
version = "0.4.111"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

View file

@ -8202,18 +8202,18 @@ dependencies = [
[[package]]
name = "webpki-root-certs"
version = "1.0.9"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "webpki-roots"
version = "1.0.9"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a"
checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9"
dependencies = [
"rustls-pki-types",
]

View file

@ -124,7 +124,7 @@ moka = { version = "0.12.16", features = ["future"] }
strum = { version = "0.28.0", features = ["derive"] }
url = "2.5.8"
percent-encoding = "2.3"
webpki-roots = "1"
webpki-roots = "1.0.10"
time = { version = "0.3.53", features = ["parsing"] }
criterion = "0.8.2"
fancy-regex = "0.19.2"

View file

@ -72,26 +72,26 @@ fn duration(seconds: f64) -> Result<Duration, Error> {
#[derive(Clone, Default)]
pub(crate) struct CacheHeaders(std::sync::Arc<std::sync::OnceLock<String>>);
impl litellm_host::interceptors::Interceptors<litellm_inference::RouteError> for CacheHeaders {
impl litellm_host::interceptors::Interceptors<litellm_inference::Error> for CacheHeaders {
async fn before_provider_request(
&self,
wire: litellm_host::interceptors::WireRequest,
_: litellm_host::interceptors::RequestContext,
) -> Result<litellm_host::interceptors::WireRequest, litellm_inference::RouteError> {
) -> Result<litellm_host::interceptors::WireRequest, litellm_inference::Error> {
Ok(wire)
}
async fn after_provider_response(
&self,
_: litellm_host::interceptors::RawResponse,
) -> Result<(), litellm_inference::RouteError> {
) -> Result<(), litellm_inference::Error> {
Ok(())
}
async fn result_ready(
&self,
facts: litellm_host::interceptors::ExecutionFacts,
) -> Result<(), litellm_inference::RouteError> {
) -> Result<(), litellm_inference::Error> {
if let litellm_host::interceptors::ResultSource::Cache { key } = facts.source {
let _ = self.0.set(key);
}

View file

@ -4,7 +4,7 @@ use axum::{
response::{IntoResponse, Response},
};
use litellm_http::transport::Error as TransportError;
use litellm_inference::RouteError;
use litellm_inference::Error as InferenceError;
use litellm_llms::base_llm::ocr::error::Error as OcrError;
use serde_json::{Map, Value, json};
@ -19,7 +19,7 @@ pub enum Error {
)]
UnknownModel(String),
#[error(transparent)]
Route(#[from] RouteError),
Route(#[from] InferenceError),
#[error(transparent)]
Ocr(#[from] OcrError),
#[error("{0} is not implemented by the Rust gateway")]
@ -36,8 +36,8 @@ impl IntoResponse for Error {
}
}
impl From<litellm_host_http::Error<RouteError>> for Error {
fn from(error: litellm_host_http::Error<RouteError>) -> Self {
impl From<litellm_host_http::Error<InferenceError>> for Error {
fn from(error: litellm_host_http::Error<InferenceError>) -> Self {
match error {
litellm_host_http::Error::Call(error) => Self::Route(error),
litellm_host_http::Error::Protocol => Self::Internal(error.to_string()),
@ -50,7 +50,7 @@ impl Error {
match self {
Self::Auth(error) => error.status(),
Self::Unsupported(_)
| Self::Route(RouteError::Unsupported(_))
| Self::Route(InferenceError::Unsupported(_))
| Self::Ocr(OcrError::Unsupported(_)) => StatusCode::NOT_IMPLEMENTED,
Self::BodyTooLarge => StatusCode::PAYLOAD_TOO_LARGE,
Self::Ocr(
@ -64,10 +64,10 @@ impl Error {
.and_then(|status| StatusCode::from_u16(status).ok())
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
Self::InvalidBody(_) | Self::UnknownModel(_) => StatusCode::BAD_REQUEST,
Self::Route(RouteError::Transport(TransportError::Http { status, .. })) => {
Self::Route(InferenceError::Transport(TransportError::Http { status, .. })) => {
StatusCode::from_u16(*status).unwrap_or(StatusCode::BAD_GATEWAY)
}
Self::Route(RouteError::Auth(litellm_auth::Error::MissingApiKey { .. })) => {
Self::Route(InferenceError::Auth(litellm_auth::Error::MissingApiKey { .. })) => {
StatusCode::UNAUTHORIZED
}
Self::Route(error) if error.is_request() => StatusCode::BAD_REQUEST,
@ -97,7 +97,9 @@ impl Error {
/// body already in that shape passes through, any other has its message extracted.
pub fn body(&self, request_id: Option<&str>) -> Value {
let raw = match self {
Self::Route(RouteError::Transport(TransportError::Http { body, .. })) => body.clone(),
Self::Route(InferenceError::Transport(TransportError::Http { body, .. })) => {
body.clone()
}
other => other.to_string(),
};
let parsed = serde_json::from_str::<Value>(&raw).ok();
@ -175,7 +177,7 @@ mod tests {
use super::*;
fn upstream(status: u16, body: &str) -> Error {
Error::Route(RouteError::Transport(TransportError::Http {
Error::Route(InferenceError::Transport(TransportError::Http {
status,
body: body.into(),
}))
@ -225,16 +227,16 @@ mod tests {
#[rstest]
#[case::upstream_status(upstream(429, ""), StatusCode::TOO_MANY_REQUESTS)]
#[case::rejected_request(Error::Route(RouteError::InvalidRequest("top_k".into())), StatusCode::BAD_REQUEST)]
#[case::rejected_request(Error::Route(InferenceError::InvalidRequest("top_k".into())), StatusCode::BAD_REQUEST)]
#[case::missing_key(
Error::Route(RouteError::Auth(litellm_auth::Error::MissingApiKey {
Error::Route(InferenceError::Auth(litellm_auth::Error::MissingApiKey {
provider: "Anthropic",
environment_variable: "ANTHROPIC_API_KEY",
})),
StatusCode::UNAUTHORIZED,
)]
#[case::lost_connection(
Error::Route(RouteError::Transport(TransportError::Network("reset".into()))),
Error::Route(InferenceError::Transport(TransportError::Network("reset".into()))),
StatusCode::INTERNAL_SERVER_ERROR,
)]
fn status_follows_who_is_at_fault(#[case] error: Error, #[case] status: StatusCode) {

View file

@ -1,7 +1,7 @@
use litellm_host::observation::ObservationSender;
pub mod route;
pub mod types;
pub use litellm_inference::RouteError as Error;
pub use litellm_inference::Error;
mod common_utils;
pub mod constants;
pub(crate) mod handler;

View file

@ -21,7 +21,7 @@ use litellm_host::{
lifecycle::{CallEvent, ExecutionEvent},
observation::observation_channel,
};
use litellm_inference::RouteError;
use litellm_inference::Error;
use rstest::{fixture, rstest};
use serde_json::{Value, json};
@ -166,12 +166,12 @@ struct ChangingHooks {
facts: std::sync::Mutex<Vec<ExecutionFacts>>,
}
impl Interceptors<RouteError> for ChangingHooks {
impl Interceptors<Error> for ChangingHooks {
async fn before_provider_request(
&self,
mut wire: WireRequest,
_: RequestContext,
) -> Result<WireRequest, RouteError> {
) -> Result<WireRequest, Error> {
let call = self.calls.fetch_add(1, Ordering::SeqCst);
if self.rewrite {
wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 });
@ -179,11 +179,11 @@ impl Interceptors<RouteError> for ChangingHooks {
Ok(wire)
}
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
Ok(())
}
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
self.facts.lock().unwrap().push(facts);
Ok(())
}

View file

@ -14,7 +14,7 @@ use std::sync::Arc;
use crate::common_utils::messages_provider;
pub use litellm_inference::RouteError as Error;
pub use litellm_inference::Error;
pub use types::{
MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params,
messages_body,

View file

@ -17,7 +17,7 @@ use litellm_host::interceptors::{
ResultSource, WireRequest,
};
use litellm_inference::{
RouteError,
Error,
caching::{CacheRequest, execute_unary},
};
use rstest::{fixture, rstest};
@ -133,12 +133,12 @@ struct ChangingHooks {
facts: std::sync::Mutex<Vec<ExecutionFacts>>,
}
impl Interceptors<RouteError> for ChangingHooks {
impl Interceptors<Error> for ChangingHooks {
async fn before_provider_request(
&self,
mut wire: WireRequest,
_: RequestContext,
) -> Result<WireRequest, RouteError> {
) -> Result<WireRequest, Error> {
let call = self.calls.fetch_add(1, Ordering::SeqCst);
if self.rewrite {
wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 });
@ -146,11 +146,11 @@ impl Interceptors<RouteError> for ChangingHooks {
Ok(wire)
}
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
Ok(())
}
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
self.facts.lock().unwrap().push(facts);
Ok(())
}

View file

@ -3,4 +3,4 @@
- The route returns its completed response directly
- Route closures supply OCR-specific host capabilities, such as the caller's Azure AD token provider (`src/route.rs`)
- Provider code reaches the caller's hooks mid-call only through `litellm_llms::base_llm::ocr::handler::CallHooks`, which this crate implements over its host until it folds into `litellm_host::interceptors::Interceptors`
- OCR failures use `base_llm/ocr/error.rs`, the recorded exception to `litellm_inference::RouteError` until OCR folds into `litellm_llms::Error`
- OCR failures use `base_llm/ocr/error.rs`, the recorded exception to `litellm_inference::Error` until OCR folds into `litellm_llms::Error`

View file

@ -249,7 +249,7 @@ pub(crate) fn resolve_provider_config<'a>(
provider: LlmProviders::Mistral,
}
}
Err(litellm_inference::RouteError::InvalidProvider(provider)) => {
Err(litellm_inference::Error::InvalidProvider(provider)) => {
return Err(Error::InvalidProvider(provider));
}
Err(error) => return Err(Error::InvalidProvider(error.to_string())),

View file

@ -1,5 +1,5 @@
use litellm_host::observation::ObservationSender;
pub use litellm_inference::RouteError as Error;
pub use litellm_inference::Error;
pub mod ids;
pub mod route;

View file

@ -18,7 +18,7 @@ use litellm_host::interceptors::{
WireRequest,
};
use litellm_inference::{
RouteError,
Error,
caching::{CacheRequest, execute_unary},
};
use rstest::{fixture, rstest};
@ -124,12 +124,12 @@ struct ChangingHooks {
facts: std::sync::Mutex<Vec<ExecutionFacts>>,
}
impl Interceptors<RouteError> for ChangingHooks {
impl Interceptors<Error> for ChangingHooks {
async fn before_provider_request(
&self,
mut wire: WireRequest,
_: RequestContext,
) -> Result<WireRequest, RouteError> {
) -> Result<WireRequest, Error> {
let call = self.calls.fetch_add(1, Ordering::SeqCst);
if self.rewrite {
wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 });
@ -137,11 +137,11 @@ impl Interceptors<RouteError> for ChangingHooks {
Ok(wire)
}
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
Ok(())
}
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
self.facts.lock().unwrap().push(facts);
Ok(())
}

View file

@ -1,5 +1,5 @@
pub mod types;
pub use litellm_inference::RouteError as Error;
pub use litellm_inference::Error;
mod constants;
mod handler;

View file

@ -2,7 +2,7 @@
- `gateway-inference` (serving) -> `inference-<fmt>` (one API's call) -> `inference` (shared base)
- This crate is the shared base every format crate builds on
- `RouteError` (`src/error.rs`), `CallOptions` (`src/lib.rs`), `CallContext` (`src/context.rs`)
- `Error` (`src/error.rs`), `CallOptions` (`src/lib.rs`), `CallContext` (`src/context.rs`)
- diagnostic spans (`src/diagnostic.rs`), outbound send and signing (`src/outbound.rs`), provider resolution (`src/provider.rs`), `CoreResources` (`src/resources.rs`)
- response caching (`src/caching.rs`): `Cachable`, `StreamCachable`, `CacheRequest`, `CallCache`, `execute_unary`, `execute_streaming`, stream capture
- Shared test helpers belong in litellm-inference-testing, used only as a dev dependency
@ -49,14 +49,14 @@
- `litellm-inference-<fmt>` mirrors the route packages (`litellm/ocr/`, `litellm/messages/`, ...)
- Provider code never imports from `inference` or `inference-*`
- Handlers belong in `inference-*` or `llms`, never in a host crate
- Import every item from its canonical path. Never re-export another crate's items or give an item a second public path. The only allowed re-exports are a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`) and each format crate's `pub use litellm_inference::RouteError as Error;`, which keeps the pre-split `<fmt>::Error` name
- Import every item from its canonical path. Never re-export another crate's items or give an item a second public path. The only allowed re-exports are a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`) and each format crate's `pub use litellm_inference::Error;`, which keeps the pre-split `<fmt>::Error` name
## Error placement
- The workspace `Error definitions` rules shape each crate's error. This section decides which crate and module a failure belongs to
- A failure is declared once, by the lowest crate that raises it
- every crate above nests it unchanged (`#[error(transparent)] Auth(#[from] litellm_auth::Error)`) or maps it once at its boundary, as `src/error.rs` does for `litellm_llms::Error`
- `RouteError` collects route failures for every format crate and never re-declares a variant a lower crate raises
- `Error` collects route failures for every format crate and never re-declares a variant a lower crate raises
- Scope follows the concept, not the first caller
- an error type under `litellm-llms`'s `<provider>/` is private to that provider: no other provider and nothing in `base_llm` may import it
- a failure two providers or two routes can hit (wire framing, stream event decoding, a malformed provider response) belongs to the crate that owns the concept: `litellm-framer` for framing, `litellm_llms::Error` for the transformation layer

View file

@ -24,9 +24,9 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned};
use serde_json::Value;
use tokio_util::codec::Decoder;
use crate::RouteError;
use crate::Error;
pub trait Cachable: Protocol<Error = RouteError> {
pub trait Cachable: Protocol<Error = Error> {
const SURFACE: &'static str;
fn reusable(_response: &Self::Response) -> bool {
@ -145,15 +145,15 @@ pub async fn execute_unary<P, F, Fut>(
request: CacheRequest,
cache: Option<Arc<dyn ResponseCacheService>>,
options: Option<CacheOptions>,
interceptors: &impl Interceptors<RouteError>,
interceptors: &impl Interceptors<Error>,
observers: Option<&ObservationSender>,
provider: F,
) -> Result<P::Response, RouteError>
) -> Result<P::Response, Error>
where
P: Cachable,
P::Response: Serialize + DeserializeOwned,
F: FnOnce() -> Fut,
Fut: Future<Output = Result<P::Response, RouteError>>,
Fut: Future<Output = Result<P::Response, Error>>,
{
let identity = request.identity.clone();
crate::diagnostic::provider(&identity.model, &identity.provider);
@ -191,15 +191,15 @@ pub async fn execute_streaming<P, F, Fut>(
request: CacheRequest,
cache: Option<Arc<dyn ResponseCacheService>>,
options: Option<CacheOptions>,
interceptors: &impl Interceptors<RouteError>,
interceptors: &impl Interceptors<Error>,
observers: Option<&ObservationSender>,
provider: F,
) -> Result<OutputOf<P>, RouteError>
) -> Result<OutputOf<P>, Error>
where
P: StreamCachable,
P::Response: Serialize + DeserializeOwned,
F: FnOnce() -> Fut,
Fut: Future<Output = Result<OutputOf<P>, RouteError>>,
Fut: Future<Output = Result<OutputOf<P>, Error>>,
{
let identity = request.identity.clone();
crate::diagnostic::provider(&identity.model, &identity.provider);
@ -299,9 +299,9 @@ impl<P: StreamCachable> CallCache<P> {
}
fn capture_stream<P: StreamCachable>(
chunks: futures_util::stream::BoxStream<'static, Result<P::Chunk, RouteError>>,
chunks: futures_util::stream::BoxStream<'static, Result<P::Chunk, Error>>,
session: CacheSession,
) -> futures_util::stream::BoxStream<'static, Result<P::Chunk, RouteError>> {
) -> futures_util::stream::BoxStream<'static, Result<P::Chunk, Error>> {
stream::try_unfold(
(chunks, Some(Vec::<u8>::new()), session),
|(mut chunks, captured, session)| async move {
@ -330,7 +330,7 @@ fn capture_stream<P: StreamCachable>(
{
session.store(entry).await;
}
Ok::<_, RouteError>(None)
Ok::<_, Error>(None)
}
}
},
@ -369,9 +369,9 @@ fn successful_stream(text: &str, terminal: &str) -> bool {
async fn publish(
facts: ExecutionFacts,
interceptors: &impl Interceptors<RouteError>,
interceptors: &impl Interceptors<Error>,
observers: Option<&ObservationSender>,
) -> Result<(), RouteError> {
) -> Result<(), Error> {
if let Some(observers) = observers {
observers.emit(CallEvent::Execution(ExecutionEvent::ResultReady {
facts: facts.clone(),

View file

@ -5,7 +5,7 @@ use litellm_host::{
observation::ObservationSender,
};
use crate::{CallOptions, RouteError};
use crate::{CallOptions, Error};
pub struct CallContext<'a, I> {
pub interceptors: &'a I,
@ -13,7 +13,7 @@ pub struct CallContext<'a, I> {
pub cache: CachePolicy,
}
impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
impl<'a, I: Interceptors<Error>> CallContext<'a, I> {
pub fn new(interceptors: &'a I, options: CallOptions) -> Self {
Self {
interceptors,
@ -22,7 +22,7 @@ impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
}
}
pub async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
pub async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
if let Some(observers) = &self.observers {
observers.emit(CallEvent::Execution(ExecutionEvent::ResultReady {
facts: facts.clone(),
@ -31,7 +31,7 @@ impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
self.interceptors.result_ready(facts).await
}
pub async fn response_received(&self, body: &str) -> Result<(), RouteError> {
pub async fn response_received(&self, body: &str) -> Result<(), Error> {
let raw = RawResponse {
body: body.to_owned(),
};
@ -43,6 +43,6 @@ impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
self.interceptors
.after_provider_response(raw)
.await
.map_err(RouteError::post_call)
.map_err(Error::post_call)
}
}

View file

@ -11,7 +11,7 @@ use litellm_http::transport::Error as TransportError;
use litellm_llms::Error as LlmError;
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum RouteError {
pub enum Error {
#[error("expected {expected}, got {actual}")]
InvalidType {
expected: &'static str,
@ -38,10 +38,10 @@ pub enum RouteError {
#[error(transparent)]
Secret(#[from] SecretError),
#[error("post-call hook failed: {0}")]
PostCallHook(#[source] Arc<RouteError>),
PostCallHook(#[source] Arc<Error>),
}
impl From<litellm_host::machine::MachineFault> for RouteError {
impl From<litellm_host::machine::MachineFault> for Error {
fn from(fault: litellm_host::machine::MachineFault) -> Self {
use litellm_host::machine::MachineFault;
Self::InvalidRequest(match fault {
@ -51,7 +51,7 @@ impl From<litellm_host::machine::MachineFault> for RouteError {
}
}
impl RouteError {
impl Error {
pub fn post_call(error: Self) -> Self {
Self::PostCallHook(Arc::new(error))
}
@ -76,7 +76,7 @@ impl RouteError {
}
}
impl From<LlmError> for RouteError {
impl From<LlmError> for Error {
fn from(error: LlmError) -> Self {
match error {
LlmError::InvalidType { expected, actual } => Self::InvalidType { expected, actual },
@ -99,7 +99,7 @@ impl SecretError {
}
}
impl From<litellm_secrets::Error> for RouteError {
impl From<litellm_secrets::Error> for Error {
fn from(error: litellm_secrets::Error) -> Self {
Self::Secret(SecretError(Arc::new(error)))
}
@ -115,22 +115,22 @@ impl Eq for SecretError {}
#[cfg(test)]
mod tests {
use super::RouteError;
use super::Error;
use litellm_llms::{Error as LlmError, ErrorDetail};
use rstest::rstest;
#[test]
fn a_missing_api_key_is_the_environment_not_the_request() {
assert!(
!RouteError::Auth(litellm_auth::Error::MissingApiKey {
!Error::Auth(litellm_auth::Error::MissingApiKey {
provider: "Anthropic",
environment_variable: "ANTHROPIC_API_KEY",
})
.is_request()
);
assert!(RouteError::Auth(litellm_auth::Error::InvalidHeader).is_request());
assert!(RouteError::InvalidRequest("top_k".into()).is_request());
assert!(!RouteError::InvalidResponse("bad json".into()).is_request());
assert!(Error::Auth(litellm_auth::Error::InvalidHeader).is_request());
assert!(Error::InvalidRequest("top_k".into()).is_request());
assert!(!Error::InvalidResponse("bad json".into()).is_request());
}
#[rstest]
#[case::request(true)]
@ -139,7 +139,7 @@ mod tests {
let source = serde_json::from_str::<serde_json::Value>("{").unwrap_err();
let source_message = source.to_string();
let detail = ErrorDetail::invalid("test payload", source);
let error = RouteError::from(if request {
let error = Error::from(if request {
LlmError::InvalidRequest(detail)
} else {
LlmError::InvalidResponse(detail)

View file

@ -7,7 +7,7 @@ pub mod outbound;
pub mod provider;
pub mod resources;
pub use error::RouteError;
pub use error::Error;
#[derive(Clone, Default)]
pub struct CallOptions {

View file

@ -2,7 +2,7 @@ use litellm_core_utils::get_llm_provider_logic::{
CustomLlmProvider, LlmProviders, get_custom_llm_provider,
};
use crate::error::RouteError as Error;
use crate::error::Error;
#[derive(Debug)]
pub struct ResolvedProvider<'a> {

View file

@ -25,7 +25,7 @@ use litellm_host::{
protocol::Protocol,
};
use litellm_inference::{
RouteError,
Error,
caching::{Cachable, CacheRequest, StreamCachable, execute_streaming, execute_unary},
};
use rstest::{fixture, rstest};
@ -36,7 +36,7 @@ struct TestRoute;
impl Protocol for TestRoute {
type Request = Value;
type Response = Value;
type Error = RouteError;
type Error = Error;
type HostCall = Infallible;
type Chunk = Bytes;
type StreamHead = ();
@ -204,7 +204,7 @@ async fn streamed(
.chunks(3)
.map(|bytes| Ok(Bytes::copy_from_slice(bytes)))
.collect::<Vec<_>>();
let ending = fail.then_some(Err(RouteError::Unsupported("test transport failure")));
let ending = fail.then_some(Err(Error::Unsupported("test transport failure")));
Ok(CallOutput::Stream {
head: (),
chunks: stream::iter(chunks.into_iter().chain(ending)).boxed(),
@ -215,7 +215,7 @@ async fn streamed(
.unwrap()
}
async fn consume(output: OutputOf<TestRoute>) -> Result<Vec<u8>, RouteError> {
async fn consume(output: OutputOf<TestRoute>) -> Result<Vec<u8>, Error> {
let CallOutput::Stream { chunks, .. } = output else {
panic!("expected a stream");
};
@ -315,7 +315,7 @@ async fn a_provider_failure_never_populates_the_cache(cache: Arc<dyn ResponseCac
None,
|| async {
calls.fetch_add(1, Ordering::SeqCst);
Err(RouteError::Unsupported("test provider failure"))
Err(Error::Unsupported("test provider failure"))
},
)
.await;
@ -467,8 +467,8 @@ struct CacheHitAccounting {
reject: bool,
}
impl Interceptors<RouteError> for CacheHitAccounting {
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
impl Interceptors<Error> for CacheHitAccounting {
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
self.calls.fetch_add(1, Ordering::SeqCst);
let ResultSource::Cache { key } = facts.source else {
panic!("expected cache source")
@ -482,7 +482,7 @@ impl Interceptors<RouteError> for CacheHitAccounting {
);
self.key.set(key).unwrap();
if self.reject {
return Err(RouteError::Unsupported("cache accounting rejected"));
return Err(Error::Unsupported("cache accounting rejected"));
}
Ok(())
}
@ -491,11 +491,11 @@ impl Interceptors<RouteError> for CacheHitAccounting {
&self,
wire: WireRequest,
_: RequestContext,
) -> Result<WireRequest, RouteError> {
) -> Result<WireRequest, Error> {
Ok(wire)
}
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
Ok(())
}
}
@ -571,7 +571,7 @@ async fn cache_hits_notify_accounting_once_and_propagate_its_failure(
if reject {
assert!(matches!(
result,
Err(RouteError::Unsupported("cache accounting rejected"))
Err(Error::Unsupported("cache accounting rejected"))
));
} else {
assert_eq!(result.unwrap(), expected);
@ -590,7 +590,7 @@ async fn cache_hits_notify_accounting_once_and_propagate_its_failure(
impl Protocol for UnaryTestRoute {
type Request = Value;
type Response = Value;
type Error = RouteError;
type Error = Error;
type HostCall = Infallible;
type Chunk = Infallible;
type StreamHead = Infallible;

View file

@ -235,7 +235,10 @@ impl AnthropicBeta {
BetaProvider::Databricks,
],
Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
Self::ComputerUse20250124
| Self::ComputerUse20251124
| Self::Context1m20250807
| Self::ThinkingBindingControls20260801 => &[],
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
Self::Effort20251124 => &[BetaProvider::VertexAi],
Self::FastMode20260201 | Self::Oauth20250420 => &[
@ -272,7 +275,6 @@ impl AnthropicBeta {
BetaProvider::Databricks,
],
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
Self::ThinkingBindingControls20260801 => &[BetaProvider::AzureAi],
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
Self::ToolSearchTool20251019 => &[
BetaProvider::Anthropic,

View file

@ -4,7 +4,7 @@ mod selection;
pub(crate) use native::NativeCacheHandle;
pub(crate) use python::{CacheCall, PythonCache};
pub(crate) use selection::{Cached, Selection, admit_native, configure, configured_native};
pub(crate) use selection::{Cached, Selection, configure, configured_native};
use litellm_cache::Error;
use pyo3::{

View file

@ -279,7 +279,7 @@ pub(in crate::cache) fn configured(
)> {
let handle = native_handle(configured)?.ok_or_else(|| {
pyo3::exceptions::PyRuntimeError::new_err(
"the configured cache changed to a Python cache after native admission",
"native inference needs a v2 cache, the catalog keeps Python caches on Python",
)
})?;
let cache = handle.extract::<PyRef<'_, NativeCacheHandle>>()?;

View file

@ -76,21 +76,6 @@ fn selected_cache<'py>(
Ok(Some(configured))
}
pub(crate) fn admit_native(
py: Python<'_>,
kwargs: &Bound<'_, PyDict>,
call_type: &str,
) -> PyResult<()> {
if let Some(configured) = selected_cache(py, kwargs, call_type)?
&& native::v2::native_handle(&configured)?.is_none()
{
return Err(crate::errors::RustBridgeDeclined::new_err(
"the configured cache requires Python inference",
));
}
Ok(())
}
pub(crate) fn configured_native(
py: Python<'_>,
kwargs: &Bound<'_, PyDict>,

View file

@ -1,5 +1,5 @@
use litellm_http::transport::Error as TransportError;
use litellm_inference::RouteError;
use litellm_inference::Error;
use pyo3::{
exceptions::{PyRuntimeError, PyValueError},
prelude::*,
@ -19,9 +19,9 @@ pyo3::create_exception!(
"The provider call was already issued and failed. Args are (status, message); status is 0 when there was no HTTP response."
);
pub(crate) fn route_error_to_pyerr(error: RouteError) -> PyErr {
pub(crate) fn route_error_to_pyerr(error: Error) -> PyErr {
match error {
RouteError::Transport(TransportError::Http { status, body }) => {
Error::Transport(TransportError::Http { status, body }) => {
RustUpstreamError::new_err((status, body))
}
other => by_fault(other.is_request(), other.to_string()),
@ -42,13 +42,13 @@ mod tests {
use super::*;
#[rstest::rstest]
#[case::unsupported(RouteError::Unsupported("test capability"), true)]
#[case::invalid_provider(RouteError::InvalidProvider("unknown".into()), true)]
#[case::invalid_request(RouteError::InvalidRequest("empty messages".into()), true)]
#[case::unsupported(Error::Unsupported("test capability"), true)]
#[case::invalid_provider(Error::InvalidProvider("unknown".into()), true)]
#[case::invalid_request(Error::InvalidRequest("empty messages".into()), true)]
#[case::connection(TransportError::Connect("unreachable".into()).into(), false)]
#[case::network(TransportError::Network("timed out".into()).into(), false)]
#[case::invalid_response(RouteError::InvalidResponse("missing usage".into()), false)]
fn route_failures_are_terminal(#[case] error: RouteError, #[case] is_request: bool) {
#[case::invalid_response(Error::InvalidResponse("missing usage".into()), false)]
fn route_failures_are_terminal(#[case] error: Error, #[case] is_request: bool) {
Python::initialize();
Python::attach(|py| {
let failure = route_error_to_pyerr(error);
@ -86,14 +86,12 @@ mod tests {
fn missing_api_key_stays_a_runtime_error_while_other_auth_failures_are_value_errors() {
Python::initialize();
Python::attach(|py| {
let missing =
route_error_to_pyerr(RouteError::Auth(litellm_auth::Error::MissingApiKey {
provider: "Anthropic",
environment_variable: "ANTHROPIC_API_KEY",
}));
let missing = route_error_to_pyerr(Error::Auth(litellm_auth::Error::MissingApiKey {
provider: "Anthropic",
environment_variable: "ANTHROPIC_API_KEY",
}));
assert!(missing.is_instance_of::<PyRuntimeError>(py));
let invalid =
route_error_to_pyerr(RouteError::Auth(litellm_auth::Error::InvalidHeader));
let invalid = route_error_to_pyerr(Error::Auth(litellm_auth::Error::InvalidHeader));
assert!(invalid.is_instance_of::<PyValueError>(py));
});
}

View file

@ -7,7 +7,7 @@ use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
use litellm_host::{call::HostedMachine, protocol::Protocol};
use litellm_host_python::{PythonBinding, PythonHostCalls, from_py};
use litellm_http::transport::Error as TransportError;
use litellm_inference::RouteError;
use litellm_inference::Error;
use litellm_secrets::source::SecretSource;
use pyo3::{
exceptions::PyValueError,
@ -119,14 +119,14 @@ impl InferenceHost {
public_response(py, self.module, response)
}
pub fn error(&self, py: Python<'_>, error: RouteError) -> PyResult<PyErr> {
if let RouteError::Secret(source) = &error
pub fn error(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
if let Error::Secret(source) = &error
&& let Some(original) = crate::secrets::python_error(py, source.source_error())
{
return Ok(original);
}
let native = match error {
RouteError::Transport(TransportError::Http { status, body }) => {
Error::Transport(TransportError::Http { status, body }) => {
let error = RustUpstreamError::new_err((status, body));
error
.value(py)
@ -144,7 +144,7 @@ impl InferenceHost {
}
pub(super) trait InferenceRoute: Sized + 'static {
type Protocol: Protocol<Error = RouteError>;
type Protocol: Protocol<Error = Error>;
const OPERATION: LoggingOperation;
const SYNC_CALL_TYPE: &'static str;
const ASYNC_CALL_TYPE: &'static str;
@ -177,7 +177,6 @@ where
} else {
R::SYNC_CALL_TYPE
};
crate::cache::admit_native(py, &call.kwargs, call_type)?;
let (arguments, hooks) = super::call_hooks(py, R::OPERATION, &call, asynchronous)?;
super::run_public_call(
py,

View file

@ -8,59 +8,19 @@ use litellm_auth::AuthServices;
use litellm_cache_response::{CachePolicy, ScopedCache};
use litellm_callbacks_legacy_python::LoggingOperation;
use litellm_host::{call::HostedMachine, protocol::Protocol};
use litellm_host_python::present;
use litellm_inference_responses::{ResponsesRoute, route::Responses};
use litellm_secrets::source::SecretSource;
use pyo3::prelude::*;
use serde_json::Value;
pub(crate) use websocket::ResponsesWebSocketConnection;
use super::{
NativeCall,
inference::{InferenceHost, InferenceRoute, run_inference},
};
use crate::errors::RustBridgeDeclined;
const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.responses";
fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyResult<Py<PyAny>> {
let resolved = call.resolved()?;
if let Some(reason) = py
.import(ROUTE_HOST_MODULE)?
.getattr("decline_reason")?
.call1((&resolved,))?
.extract::<Option<String>>()?
{
return Err(RustBridgeDeclined::new_err(reason));
}
let argument = |name: &str| present(&call.kwargs, &call.base, name);
let model = argument("model")?
.ok_or_else(|| pyo3::exceptions::PyValueError::new_err("model is required"))?
.extract::<String>()?;
let provider = argument("custom_llm_provider")?
.map(|value| value.extract::<String>())
.transpose()?;
if provider
.as_deref()
.is_some_and(|provider| provider != "openai")
|| model
.strip_prefix("openai/")
.unwrap_or(&model)
.contains('/')
{
return Err(RustBridgeDeclined::new_err(
"native HTTP responses provider",
));
}
if argument("stream")?
.map(|value| litellm_host_python::from_py::<Value>(&value))
.transpose()?
.is_some_and(|value| value == Value::Bool(true))
{
return Err(RustBridgeDeclined::new_err(
"native Python responses streaming",
));
}
let host = InferenceHost::new(call.view()?, ROUTE_HOST_MODULE);
run_inference::<ResponsesRoute, _>(py, call, asynchronous, ResponsesPythonHost::new(host))
}

View file

@ -175,7 +175,7 @@ argilla_batch_size: Optional[int] = None
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload.
gcs_pub_sub_use_v1: Optional[bool] = False # if you want to use v1 gcs pubsub logged payload
generic_api_use_v1: Optional[bool] = False # if you want to use v1 generic api logged payload
argilla_transformation_object: Optional[Dict[str, Any]] = None
argilla_transformation_object: Mapping[str, str] | None = None
_async_input_callback: List[Union[str, Callable[..., object], "CustomLogger"]] = ( # CustomLogger is lazy-loaded
[]
) # internal variable - async custom callbacks are routed here.
@ -333,7 +333,7 @@ in_memory_llm_clients_cache: "LLMClientCache"
safe_memory_mode: bool = False
enable_azure_ad_token_refresh: Optional[bool] = False
# Proxy Authentication - auto-obtain/refresh OAuth2/JWT tokens for LiteLLM Proxy
proxy_auth: Optional[Any] = None
proxy_auth: _ProxyAuthHandlerType | None = None
### DEFAULT AZURE API VERSION ###
AZURE_DEFAULT_API_VERSION = "2025-02-01-preview" # this is updated to the latest
### DEFAULT WATSONX API VERSION ###
@ -503,7 +503,7 @@ skill_search_embedding_model: Optional[str] = None
# Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]])
# New format: { "displayName": { "url": "...", "index": 0 } }
# Old format: { "displayName": "url" } (for backward compatibility)
public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {}
public_model_groups_links: dict[str, str | dict[str, object]] = {}
#### REQUEST PRIORITIZATION #######
priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]]] = None
# priority_reservation_settings is lazy-loaded via __getattr__
@ -551,6 +551,7 @@ from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, ma
model_cost = get_model_cost_map(url=model_cost_map_url)
cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
apply_search_free_tier: bool = False
cost_margin_config: Dict[
str, Union[float, Dict[str, float]]
] = {} # Provider-specific or global cost margins. Examples:
@ -681,6 +682,7 @@ darkbloom_models: Set = set()
openinfer_models: Final[Set[str]] = set()
v0_models: Set = set()
morph_models: Set = set()
opencode_go_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads
lambda_ai_models: Set = set()
inception_models: Set = set()
hyperbolic_models: Set = set()
@ -969,6 +971,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
v0_models.add(key)
elif value.get("litellm_provider") == "morph":
morph_models.add(key)
elif value.get("litellm_provider") == "opencode_go":
opencode_go_models.add(key)
elif value.get("litellm_provider") == "lambda_ai":
lambda_ai_models.add(key)
elif value.get("litellm_provider") == "inception":
@ -1136,6 +1140,7 @@ model_list = list(
| openinfer_models
| v0_models
| morph_models
| opencode_go_models
| lambda_ai_models
| inception_models
| black_forest_labs_models
@ -1253,6 +1258,7 @@ def _build_models_by_provider() -> dict:
"openinfer": openinfer_models,
"v0": v0_models,
"morph": morph_models,
"opencode_go": opencode_go_models,
"lambda_ai": lambda_ai_models,
"inception": inception_models,
"hyperbolic": hyperbolic_models,
@ -1504,6 +1510,7 @@ from .rag.main import *
from .sandbox.main import *
from .decisions.main import *
from .systemone.main import *
from .extraction.main import *
from .tool_loop import ToolLoopMaxRoundsExceeded, arun_tool_loop, run_tool_loop
from .search.main import *
from .realtime_api.main import (
@ -1710,6 +1717,9 @@ if TYPE_CHECKING:
from .llms.cloudflare.decisions.transformation import (
CloudflareDecisionsConfig as CloudflareDecisionsConfig,
)
from .llms.cloudflare.extraction.transformation import (
CloudflareExtractionConfig as CloudflareExtractionConfig,
)
from .llms.strands_decider.decisions.transformation import (
StrandsDeciderDecisionsConfig as StrandsDeciderDecisionsConfig,
)
@ -2284,6 +2294,10 @@ if TYPE_CHECKING:
LiteLLM_UpperboundKeyGenerateParams,
)
from litellm.utils import ModelResponseListIterator as _ModelResponseListIterator
from collections.abc import Awaitable
from litellm.types.utils import ModelResponse as _ModelResponseType
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper as _CustomStreamWrapperType
from litellm.proxy_auth import ProxyAuthHandler as _ProxyAuthHandlerType
# Cost calculator functions
cost_per_token: Callable[..., Tuple[float, float]]
@ -2307,7 +2321,7 @@ if TYPE_CHECKING:
supports_audio_output: Callable[..., bool]
supports_system_messages: Callable[..., bool]
supports_reasoning: Callable[..., bool]
acreate: Callable[..., Any]
acreate: Callable[..., Awaitable[_ModelResponseType | _CustomStreamWrapperType]]
get_max_tokens: Callable[..., int]
get_model_info: Callable[..., _ModelInfoType]
register_prompt_template: Callable[..., None]
@ -2605,8 +2619,10 @@ def __getattr__(name: str) -> Any:
cancel_run,
delete_run,
)
from types import MappingProxyType
return locals()[name]
evals_exports: Final[Mapping[str, object]] = MappingProxyType(locals())
return evals_exports[name]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

View file

@ -19,7 +19,7 @@ import importlib
import sys
from collections.abc import Callable, Mapping
from types import ModuleType
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final
from typing_extensions import ReadOnly, TypedDict
@ -456,7 +456,7 @@ def _lazy_import_http_handlers(name: str) -> object:
params: Final = {"timeout": async_timeout, "client_alias": "module level aclient"}
# Create the client instance
provider_id: Final = cast(Any, "litellm_module_level_client")
provider_id: Final = "litellm_module_level_client"
async_client: Final = get_async_httpx_client(
llm_provider=provider_id,
params=params,

View file

@ -166,6 +166,7 @@ LLM_CONFIG_NAMES: Final = (
"OpenRouterDecisionsConfig",
"FireworksAIDecisionsConfig",
"CloudflareDecisionsConfig",
"CloudflareExtractionConfig",
"StrandsDeciderDecisionsConfig",
"DatabricksDecisionsConfig",
"HostedVLLMDecisionsConfig",
@ -735,6 +736,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
"FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),
"CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"),
"CloudflareExtractionConfig": (".llms.cloudflare.extraction.transformation", "CloudflareExtractionConfig"),
"StrandsDeciderDecisionsConfig": (
".llms.strands_decider.decisions.transformation",
"StrandsDeciderDecisionsConfig",

View file

@ -5,7 +5,10 @@ Supports dynamic cost parameters that allow platform owners
to define custom costs per agent query or per token.
"""
from typing import TYPE_CHECKING, Any, Final
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, SupportsFloat, SupportsIndex
from typing_extensions import Buffer
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import (
@ -15,6 +18,15 @@ else:
LitellmLoggingObject = Any
def _custom_response_cost(model_call_details: Mapping[str, object]) -> float | None:
response_cost: Final = model_call_details.get("response_cost", None)
if response_cost is None:
return None
if isinstance(response_cost, (str, Buffer, SupportsFloat, SupportsIndex)):
return float(response_cost)
raise TypeError(f"float() argument must be a string or a real number, not {type(response_cost).__name__!r}")
class A2ACostCalculator:
@staticmethod
def calculate_a2a_cost(
@ -45,9 +57,9 @@ class A2ACostCalculator:
model_call_details: Final = litellm_logging_obj.model_call_details
# Check if user set a custom response cost (backward compatibility)
response_cost: Final = model_call_details.get("response_cost", None)
if response_cost is not None:
return float(response_cost)
custom_response_cost: Final = _custom_response_cost(model_call_details)
if custom_response_cost is not None:
return custom_response_cost
# Get litellm_params for cost parameters
litellm_params: Final = model_call_details.get("litellm_params", {}) or {}

View file

@ -80,7 +80,7 @@ class A2ACompletionBridgeHandler:
@staticmethod
def _build_completion_params(
params: dict[str, Any],
litellm_params: Mapping[str, Any],
litellm_params: Mapping[str, object],
api_base: str | None,
agent_extra_headers: Mapping[str, str] | None,
*,

View file

@ -602,9 +602,9 @@ def _build_streaming_logging_obj(
litellm_params: dict[str, object] | None,
metadata: dict[str, object] | None,
proxy_server_request: dict[str, object] | None,
start_time: datetime.datetime,
) -> Logging:
"""Build logging object for streaming A2A requests."""
start_time: Final = datetime.datetime.now()
model: Final = f"a2a_agent/{agent_name}"
logging_obj: Final = Logging(
@ -718,6 +718,7 @@ async def asend_message_streaming(
if request is None:
raise ValueError("request is required")
called_at: Final = datetime.datetime.now()
_raw_logging_obj: Final = kwargs.get("litellm_logging_obj")
logging_obj: Logging | None = _raw_logging_obj if isinstance(_raw_logging_obj, Logging) else None
@ -751,6 +752,7 @@ async def asend_message_streaming(
litellm_params=litellm_params,
metadata=metadata,
proxy_server_request=proxy_server_request,
start_time=called_at,
)
verbose_logger.info("A2A send_message_streaming request_id=%s, agent=%s", request.id, agent_name)

View file

@ -84,14 +84,23 @@ def _filter_reserved_headers(
return filtered or None
def _calling_key_hash(litellm_params: Mapping[str, object]) -> str | None:
principal: Final = litellm_params.get(A2A_USER_API_KEY_HASH_PARAM)
if principal is None or isinstance(principal, str):
return principal
if not principal:
return None
raise AttributeError(f"{type(principal).__name__!r} object has no attribute 'encode'", name="encode", obj=principal)
def _request_scoped_runtime_session_id(
params: Mapping[str, object],
litellm_params: Mapping[str, Any],
litellm_params: Mapping[str, object],
) -> str | None:
context_id: Final = get_session_id_from_a2a_params(params)
if not isinstance(context_id, str) or not context_id:
return None
return scope_session_to_principal(context_id, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM))
return scope_session_to_principal(context_id, _calling_key_hash(litellm_params))
def _validate_runtime_session_id(session_id: str, model: str) -> str:
@ -109,6 +118,13 @@ def _validate_runtime_session_id(session_id: str, model: str) -> str:
)
def _agentcore_model_name(litellm_params: Mapping[str, object]) -> str:
model: Final = litellm_params.get("model", "")
if isinstance(model, str):
return model
raise AttributeError(f"{type(model).__name__!r} object has no attribute 'startswith'", name="startswith", obj=model)
class BedrockAgentCoreA2ATransformation:
"""
Request/response transformation for Bedrock AgentCore A2A agents.
@ -152,11 +168,8 @@ class BedrockAgentCoreA2ATransformation:
"""
# Extract model and strip the "bedrock/" prefix
# "bedrock/agentcore/arn:aws:..." → "agentcore/arn:aws:..."
model: Final = litellm_params.get("model", "")
if model.startswith("bedrock/"):
agentcore_model = model[len("bedrock/") :]
else:
agentcore_model = model
model: Final = _agentcore_model_name(litellm_params)
agentcore_model: Final = model.removeprefix("bedrock/")
# Build optional_params from litellm_params (everything except model and custom_llm_provider)
optional_params: Final = {k: v for k, v in litellm_params.items() if k not in ("model", "custom_llm_provider")}

View file

@ -8,11 +8,31 @@ WXO uses a REST API (not A2A/JSON-RPC) with an async-poll execution model:
"""
import asyncio
from collections.abc import AsyncIterator, Mapping
from typing import Any, Final
from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
from itertools import chain
from typing import Final
from uuid import uuid4
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
from litellm.llms.openai.common_utils import decoded_json_object
def _a2a_text_part(part: object) -> Sequence[object]:
part_fields: Final = as_str_mapping(part)
if not isinstance(part_fields, dict):
return ()
if part_fields.get("kind") in (None, "", "text") and part_fields.get("text"):
return (part_fields["text"],)
return ()
def _join_with_spaces(texts: Sequence[object]) -> str:
strings: Final = [text for text in texts if isinstance(text, str)]
if len(strings) == len(texts):
return " ".join(strings)
first_bad: Final = next(position for position, text in enumerate(texts) if not isinstance(text, str))
raise TypeError(f"sequence item {first_bad}: expected str instance, {type(texts[first_bad]).__name__} found")
class WatsonxOrchestrateTransformation:
@ -29,22 +49,16 @@ class WatsonxOrchestrateTransformation:
return f"{cp4d_host.rstrip('/')}/orchestrate/cpd/instances/{instance_id}"
@staticmethod
def extract_text_from_a2a_params(params: dict[str, Any]) -> str:
def extract_text_from_a2a_params(params: Mapping[str, object]) -> str:
"""
Extract user message text from A2A MessageSendParams.
A2A format: params.message.parts[*] where part.kind == "text"
"""
message: Final = params.get("message", {})
parts: Final = message.get("parts", [])
texts: Final = []
for part in parts:
if not isinstance(part, dict):
continue
kind = part.get("kind")
if kind in (None, "", "text") and part.get("text"):
texts.append(part["text"])
return " ".join(texts) or ""
message: Final = decoded_json_object(params.get("message", {}))
parts: Final[Iterable[object]] = iter_items(message.get("parts", []))
texts: Final = list(chain.from_iterable(map(_a2a_text_part, parts)))
return _join_with_spaces(texts) or ""
@staticmethod
def build_wxo_run_body(

View file

@ -41,7 +41,7 @@ class A2AStreamingIterator:
self.request = request
self.logging_obj = logging_obj
self.agent_name = agent_name
self.start_time = datetime.now()
self.start_time = logging_obj.start_time
# Collect chunks for token counting
self.chunks: list[SendStreamingMessageResponse] = []
@ -54,6 +54,8 @@ class A2AStreamingIterator:
async def __anext__(self) -> "SendStreamingMessageResponse":
try:
chunk: Final = await self.stream.__anext__()
if self.logging_obj.completion_start_time is None:
self.logging_obj.update_completion_start_time(completion_start_time=datetime.now())
# Store chunk
self.chunks.append(chunk)

View file

@ -4,15 +4,20 @@ Utility functions for A2A protocol.
import hashlib
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final, Protocol
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import has_attributes
if TYPE_CHECKING:
from a2a.types import SendMessageRequest, SendStreamingMessageRequest
class _SupportsModelDump(Protocol):
def model_dump(self, *, mode: str) -> object: ...
class A2ARequestUtils:
"""Utility class for A2A request/response processing."""
@ -74,7 +79,7 @@ class A2ARequestUtils:
@staticmethod
def get_input_message_from_request(
request: "SendMessageRequest | SendStreamingMessageRequest",
) -> Any:
) -> object:
"""
Extract the input message from an A2A request.
@ -84,7 +89,7 @@ class A2ARequestUtils:
Returns:
The message object/dict or None
"""
params: Final = getattr(request, "params", None)
params: Final[object] = getattr(request, "params", None)
if params is None:
return None
return getattr(params, "message", None)
@ -127,10 +132,12 @@ class A2ARequestUtils:
# the dict branch — request-side parts are a2a-sdk Part RootModels whose
# kind/text live on part.root, which the object branch cannot read. This
# mirrors how the response side already works (it operates on model_dump).
input_message = A2ARequestUtils.get_input_message_from_request(request)
if input_message is not None and hasattr(input_message, "model_dump"):
input_message = input_message.model_dump(mode="json")
input_text: Final = A2ARequestUtils.extract_text_from_message(input_message)
input_message: Final = A2ARequestUtils.get_input_message_from_request(request)
input_text: Final = A2ARequestUtils.extract_text_from_message(
input_message.model_dump(mode="json")
if has_attributes(input_message, _SupportsModelDump, "model_dump")
else input_message
)
prompt_tokens: Final = A2ARequestUtils.count_tokens(input_text)
# Count output tokens

View file

@ -67,7 +67,7 @@
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
"text_editor_20241022": null,
"text_editor_20250124": null,
"thinking-binding-controls-2026-08-01": null,
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
"token-efficient-tools-2025-02-19": null,
"web-fetch-2025-09-10": "web-fetch-2025-09-10",

View file

@ -36,6 +36,7 @@ import httpx
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
# Cache for the loaded configuration
_BETA_HEADERS_CONFIG: dict | None = None
@ -44,6 +45,7 @@ _BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_na
BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
_DECODED_JSON: Final = TypeAdapter(object)
class GetAnthropicBetaHeadersConfig:
@ -75,7 +77,7 @@ class GetAnthropicBetaHeadersConfig:
}
@staticmethod
def _check_is_valid_dict(fetched_config: dict) -> bool:
def _check_is_valid_dict(fetched_config: object) -> bool:
"""Check if fetched config is a non-empty dict with expected structure."""
if not isinstance(fetched_config, dict):
verbose_logger.warning(
@ -109,7 +111,7 @@ class GetAnthropicBetaHeadersConfig:
return True
@classmethod
def validate_beta_headers_config(cls, fetched_config: dict) -> bool:
def validate_beta_headers_config(cls, fetched_config: object) -> bool:
"""
Validate the integrity of a fetched beta headers config.
@ -118,7 +120,7 @@ class GetAnthropicBetaHeadersConfig:
return cls._check_is_valid_dict(fetched_config)
@staticmethod
def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> dict:
def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> object:
"""
Fetch the beta headers config from a remote URL.
@ -127,7 +129,7 @@ class GetAnthropicBetaHeadersConfig:
"""
response: Final = httpx.get(url, timeout=timeout)
response.raise_for_status()
return response.json()
return _DECODED_JSON.validate_python(response.json())
def get_beta_headers_config(url: str) -> dict:
@ -145,7 +147,7 @@ def get_beta_headers_config(url: str) -> dict:
Dict containing the beta headers configuration
"""
# Check if local-only mode is enabled
if os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
# verbose_logger.debug("Using local Anthropic beta headers config (LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True)")
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
@ -160,7 +162,9 @@ def get_beta_headers_config(url: str) -> dict:
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
# Validate the fetched config
if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content):
if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content) or not isinstance(
content, dict
):
verbose_logger.warning(
"LiteLLM: Fetched beta headers config failed integrity check. Using local backup instead. url=%s",
url,

View file

@ -7,10 +7,14 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop
import json
from typing import Final
from pydantic import TypeAdapter
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicErrorType
_PARSED_ERROR_BODY: Final = TypeAdapter(object)
# HTTP status code -> Anthropic error type
# Source: https://docs.anthropic.com/en/api/errors
ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = {
@ -79,7 +83,7 @@ class AnthropicExceptionMapping:
- Generic: {"message": "..."}
- Plain strings
"""
parsed: Final = safe_json_loads(raw_message)
parsed: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message))
if isinstance(parsed, dict):
# Bedrock format
if "detail" in parsed and isinstance(parsed["detail"], dict):
@ -145,9 +149,8 @@ class AnthropicExceptionMapping:
AnthropicErrorResponse dict
"""
# Try to parse as JSON once
parsed: dict | None = safe_json_loads(raw_message)
if not isinstance(parsed, dict):
parsed = None
decoded: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message))
parsed: Final[dict[str, object] | None] = decoded if isinstance(decoded, dict) else None
# If parsed and already in Anthropic format - passthrough
if parsed is not None and AnthropicExceptionMapping._is_anthropic_error_dict(parsed):

View file

@ -77,7 +77,7 @@ def get_assistants(
api_base: str | None = None,
api_version: str | None = None,
**kwargs,
) -> SyncCursorPage[Assistant]:
) -> SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]]:
aget_assistants: Final[bool | None] = kwargs.pop("aget_assistants", None)
if aget_assistants is not None and not isinstance(aget_assistants, bool):
raise Exception("Invalid value passed in for aget_assistants. Only bool or None allowed")
@ -100,7 +100,7 @@ def get_assistants(
elif timeout is None:
timeout = 600.0
response: SyncCursorPage[Assistant] | None = None
response: SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -564,7 +564,7 @@ def create_thread(
tool_resources: OpenAICreateThreadParamsToolResources | None = None,
client: OpenAI | None = None,
**kwargs,
) -> Thread:
) -> Thread | Coroutine[None, None, Thread]:
"""
- get the llm provider
- if openai - route it there
@ -610,7 +610,7 @@ def create_thread(
api_base: str | None = None
api_key: str | None = None
response: Thread | None = None
response: Thread | Coroutine[None, None, Thread] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -735,7 +735,7 @@ def get_thread(
thread_id: str,
client=None,
**kwargs,
) -> Thread:
) -> Thread | Coroutine[None, None, Thread]:
"""Get the thread object, given a thread_id"""
aget_thread: Final = kwargs.pop("aget_thread", None)
optional_params: Final = GenericLiteLLMParams(**kwargs)
@ -757,7 +757,7 @@ def get_thread(
timeout = 600.0
api_base: str | None = None
api_key: str | None = None
response: Thread | None = None
response: Thread | Coroutine[None, None, Thread] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -903,7 +903,7 @@ def add_message(
metadata: dict | None = None,
client=None,
**kwargs,
) -> OpenAIMessage:
) -> OpenAIMessage | Coroutine[None, None, OpenAIMessage]:
### COMMON OBJECTS ###
a_add_message: Final = kwargs.pop("a_add_message", None)
_message_data: Final = MessageData(role=role, content=content, attachments=attachments, metadata=metadata)
@ -935,7 +935,7 @@ def add_message(
timeout = 600.0
api_key: str | None = None
api_base: str | None = None
response: OpenAIMessage | None = None
response: OpenAIMessage | Coroutine[None, None, OpenAIMessage] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -1065,7 +1065,7 @@ def get_messages(
thread_id: str,
client: Any | None = None,
**kwargs,
) -> SyncCursorPage[OpenAIMessage]:
) -> SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]:
aget_messages: Final = kwargs.pop("aget_messages", None)
optional_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params_dict: Final = get_litellm_params(**kwargs)
@ -1086,7 +1086,7 @@ def get_messages(
elif timeout is None:
timeout = 600.0
response: SyncCursorPage[OpenAIMessage] | None = None
response: SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]] | None = None
api_key: str | None = None
api_base: str | None = None
if custom_llm_provider == "openai":
@ -1256,7 +1256,7 @@ def run_thread(
client: object | None = None,
event_handler: AssistantEventHandler | None = None, # for stream=True calls
**kwargs,
) -> Run:
) -> Run | Coroutine[None, None, Run]:
"""Run a given thread + assistant."""
arun_thread: Final = kwargs.pop("arun_thread", None)
optional_params: Final = GenericLiteLLMParams(**kwargs)
@ -1278,7 +1278,7 @@ def run_thread(
elif timeout is None:
timeout = 600.0
response: Run | None = None
response: Run | Coroutine[None, None, Run] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there

View file

@ -13,7 +13,7 @@ This module is dependency-injected: callers pass the proxy ``llm_router`` and
from __future__ import annotations
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final
import litellm
from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
@ -25,12 +25,12 @@ if TYPE_CHECKING:
def resolve_embedding_router(
embedding_model: str,
llm_router: Router | None,
llm_model_list: list[dict[str, Any]] | None,
llm_model_list: Sequence[Mapping[str, object]] | None,
) -> Router | None:
"""Return ``llm_router`` iff it serves ``embedding_model`` as a deployment."""
if llm_router is None:
return None
router_model_names: Final[list[str]] = (
router_model_names: Final[Sequence[object]] = (
[m["model_name"] for m in llm_model_list if "model_name" in m] if llm_model_list is not None else []
)
if embedding_model in router_model_names:

View file

@ -24,6 +24,16 @@ if current == false then
end
if ARGV[3] then
local decoded, stored = pcall(cjson.decode, current)
if ARGV[4] then
local legacy = current
if decoded and type(stored) == 'string' then legacy = stored end
for _, eligible in ipairs(cjson.decode(ARGV[3])) do
if eligible['model_id'] == legacy then
redis.call('EXPIRE', KEYS[1], ARGV[2])
return current
end
end
end
if decoded and type(stored) == 'table' then
for _, eligible in ipairs(cjson.decode(ARGV[3])) do
local matches = true
@ -68,6 +78,7 @@ def claim_affinity_pin_in_memory(
ttl_seconds: int,
*,
eligible_values: tuple[Mapping[str, str], ...] | None = None,
preserve_legacy_model_ids: bool = False,
) -> object:
"""No await between read and write, so same-loop claims agree during a Redis outage."""
existing: Final[object] = cache.in_memory_cache.get_cache(cache_key)
@ -75,7 +86,12 @@ def claim_affinity_pin_in_memory(
if _legacy_pin_matches(existing, pin_value):
set_local_affinity_pin(cache, cache_key, pin_value, ttl_seconds)
return existing
winner: Final = existing if existing is not None and existing in (eligible_values or ()) else pin_value
eligible_existing: Final = existing in (eligible_values or ()) or (
preserve_legacy_model_ids
and isinstance(existing, str)
and any("model_id" in value and _legacy_pin_matches(existing, value) for value in (eligible_values or ()))
)
winner: Final = existing if existing is not None and eligible_existing else pin_value
set_local_affinity_pin(cache, cache_key, winner, ttl_seconds)
return winner
@ -94,6 +110,7 @@ async def claim_affinity_pin(
ttl_seconds: int,
*,
eligible_values: tuple[Mapping[str, str], ...] | None = None,
preserve_legacy_model_ids: bool = False,
) -> object:
"""Return the authoritative first writer, replacing it only when it becomes ineligible.
@ -112,6 +129,7 @@ async def claim_affinity_pin(
if eligible_values is not None
else ()
),
*(("1",) if preserve_legacy_model_ids and eligible_values is not None else ()),
)
raw: Final = cast( # cast-ok: Redis scripts return heterogeneous values; only object is asserted here
object, await claim_script(keys=(cache_key,), args=args)
@ -124,4 +142,11 @@ async def claim_affinity_pin(
return winner
except Exception as error: # noqa: BLE001 # Redis/Lua faults retain same-pod affinity through local claims
verbose_router_logger.debug("Affinity cache: Redis claim failed, using pod-local claim. error=%s", error)
return claim_affinity_pin_in_memory(cache, cache_key, pin_value, ttl_seconds, eligible_values=eligible_values)
return claim_affinity_pin_in_memory(
cache,
cache_key,
pin_value,
ttl_seconds,
eligible_values=eligible_values,
preserve_legacy_model_ids=preserve_legacy_model_ids,
)

View file

@ -13,10 +13,14 @@ import json
from contextlib import suppress
from typing import Final
from pydantic import TypeAdapter
from litellm._logging import print_verbose, verbose_logger
from .base_cache import BaseCache
_DECODED_BLOB: Final = TypeAdapter(object)
class AzureBlobCache(BaseCache):
def __init__(self, account_url, container) -> None:
@ -65,7 +69,7 @@ class AzureBlobCache(BaseCache):
print_verbose(f"Get Azure Blob Cache: key: {key}")
as_bytes: Final = self.container_client.download_blob(key).readall()
as_str: Final = as_bytes.decode("utf-8")
cached_response: Final[object] = json.loads(as_str)
cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str))
verbose_logger.debug(
"Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s",
@ -86,7 +90,7 @@ class AzureBlobCache(BaseCache):
blob: Final = await self.async_container_client.download_blob(key)
as_bytes: Final = await blob.readall()
as_str: Final = as_bytes.decode("utf-8")
cached_response: Final[object] = json.loads(as_str)
cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str))
verbose_logger.debug(
"Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s",
key,

View file

@ -9,14 +9,7 @@ Has 4 methods:
"""
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, Final
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span | Any
else:
Span = Any
from typing import Final
class BaseCache(ABC):

View file

@ -1,15 +1,19 @@
import json
from typing import TYPE_CHECKING, Any, Final
from pydantic import TypeAdapter
from .base_cache import BaseCache
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span | Any
Span = _Span
else:
Span = Any
_DECODED_DISK_VALUE: Final = TypeAdapter(object)
class DiskCache(BaseCache):
def __init__(self, disk_cache_dir: str | None = None):
@ -44,7 +48,7 @@ class DiskCache(BaseCache):
original_cached_response: Final = self.disk_cache.get(key)
if original_cached_response:
try:
cached_response = json.loads(original_cached_response)
cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response))
except Exception:
cached_response = original_cached_response
return cached_response

View file

@ -59,16 +59,6 @@ class PendingBatchRead:
previous_access_times: dict[str, float | None]
@dataclass(frozen=True, slots=True)
class DeclaredBatchRead:
"""A ``async_batch_get_cache`` split in two: the memory half done, the Redis half declared on a ``RedisBatch``
so it rides that batch's next round trip, resolved later with ``async_resolve_batch_get``."""
keys: tuple[str, ...]
pending: PendingBatchRead
result: BatchResult[Mapping[str, object]] | None
def _log_deferred_increment_failure(future: asyncio.Future[float]) -> None:
if future.cancelled():
return
@ -382,22 +372,6 @@ class DualCache(BaseCache):
await self.in_memory_cache.async_set_cache(key, value, **self._backfill_kwargs(kwargs))
return merged
async def declare_batch_get(self, keys: Sequence[str], batch: RedisBatch) -> DeclaredBatchRead:
pending: Final = await self._prepare_batch_get(
list(keys),
local_only=False,
throttle_redis=False,
)
return DeclaredBatchRead(
keys=tuple(keys),
pending=pending,
result=batch.mget(pending.redis_keys) if pending.redis_keys else None,
)
async def async_resolve_batch_get(self, declared: DeclaredBatchRead) -> list[object | None]:
redis_result: Final = None if declared.result is None else await declared.result
return await self._apply_batch_get(declared.pending, redis_result)
async def async_batch_get_cache(
self,
keys: list,

View file

@ -95,7 +95,10 @@ class InMemoryCache(BaseCache):
"""
Check if a specific key is expired
"""
return key in self.ttl_dict and self._clock() > self.ttl_dict[key]
try:
return self._clock() > self.ttl_dict[key]
except KeyError:
return False
def _cache_keys(self) -> tuple[str, ...]:
return tuple(self.cache_dict) # pyright: ignore[reportUnknownVariableType] # cache_dict is untyped legacy state
@ -227,7 +230,10 @@ class InMemoryCache(BaseCache):
if key in self.cache_dict:
if self.evict_element_if_expired(key):
return None
original_cached_response: Final = self.cache_dict[key]
try:
original_cached_response: Final = self.cache_dict[key]
except KeyError:
return None
try:
cached_response = json.loads(original_cached_response)
except Exception:

View file

@ -134,7 +134,12 @@ _INCREMENT_WITH_FLOOR_LUA: Final = (
"return count"
)
_INCREMENT_IF_PRESENT_LUA: Final = (
"if redis.call('EXISTS', KEYS[1]) == 0 then return false end return redis.call('INCRBYFLOAT', KEYS[1], ARGV[1])"
)
_LUA_COUNT: Final = TypeAdapter(int)
_LUA_OPTIONAL_TOTAL: Final[TypeAdapter[float | None]] = TypeAdapter(float | None)
_OPTIONAL_COUNTS: Final = TypeAdapter(tuple[int | None, ...])
@ -675,7 +680,7 @@ class RedisCache(BaseCache):
# redis namespaces
self.namespace = namespace
# for high traffic, we store the redis results in memory and then batch write to redis
self.redis_batch_writing_buffer: list = []
self.redis_batch_writing_buffer: list[tuple[str, object]] = []
if redis_flush_size is None:
self.redis_flush_size: int = 100
else:
@ -1568,6 +1573,46 @@ class RedisCache(BaseCache):
count: Final = await _redis_client.eval(_INCREMENT_WITH_FLOOR_LUA, 1, namespaced_key, value, ttl)
return _LUA_COUNT.validate_python(count)
@_redis_circuit_breaker_guard
async def async_increment_if_present(
self, key: str, value: float, parent_otel_span: Span | None = None
) -> float | None:
_redis_client: Final = self._async_commands()
namespaced_key: Final = self.check_and_fix_namespace(key=key)
start_time: Final = time.time()
try:
total: Final = _LUA_OPTIONAL_TOTAL.validate_python(
await _redis_client.eval(_INCREMENT_IF_PRESENT_LUA, 1, namespaced_key, value)
)
except Exception as e:
failed_at: Final = time.time()
asyncio.create_task(
self.service_logger_obj.async_service_failure_hook(
service=ServiceTypes.REDIS,
duration=failed_at - start_time,
error=e,
call_type="async_increment_if_present",
caller=_get_call_stack_info(),
start_time=start_time,
end_time=failed_at,
parent_otel_span=parent_otel_span,
)
)
raise
end_time: Final = time.time()
asyncio.create_task(
self.service_logger_obj.async_service_success_hook(
service=ServiceTypes.REDIS,
duration=end_time - start_time,
call_type="async_increment_if_present",
caller=_get_call_stack_info(),
start_time=start_time,
end_time=end_time,
parent_otel_span=parent_otel_span,
)
)
return total
async def flush_cache_buffer(self):
print_verbose(f"flushing to redis....reached size of buffer {len(self.redis_batch_writing_buffer)}")
await self.async_set_cache_pipeline(self.redis_batch_writing_buffer)
@ -1693,7 +1738,9 @@ class RedisCache(BaseCache):
return key_value_dict
@_redis_circuit_breaker_guard
async def async_get_cache(self, key, parent_otel_span: Span | None = None, **kwargs):
async def async_get_cache(
self, key, parent_otel_span: Span | None = None, *, raise_on_error: bool = False, **kwargs
):
from redis.asyncio import Redis
_redis_client: Final[Redis] = self.init_async_client()
@ -1736,6 +1783,8 @@ class RedisCache(BaseCache):
)
)
print_verbose(f"litellm.caching.caching: async get() - Got exception from REDIS: {e}")
if raise_on_error:
raise
_record_swallowed_redis_failure(self._circuit_breaker, e)
@_redis_circuit_breaker_guard

View file

@ -1,15 +1,19 @@
import inspect
from collections.abc import Awaitable, Callable, Coroutine, Mapping
from types import MappingProxyType
from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable
from litellm import main
from litellm.rust_bridge.catalog import Route, RouteContext
from litellm.rust_bridge import detectors
from litellm.rust_bridge.catalog import Feature, Route, RouteContext
from litellm.rust_bridge.chat_completions import detectors as route_detectors
from litellm.rust_bridge.chat_completions.entrypoints import (
NATIVE_ACOMPLETION,
NATIVE_COMPLETION,
)
from litellm.rust_bridge.dispatch import PublicDispatch
from litellm.rust_bridge.public_call import (
AsyncFlag,
NativeCall,
bind,
native_call,
@ -44,6 +48,7 @@ def _python_acompletion() -> PythonAcompletion:
_PYTHON_COMPLETION: Final = _python_completion()
_COMPLETION: Final = signature(_PYTHON_COMPLETION)
_FLAG: Final = AsyncFlag("acompletion", sync=_COMPLETION)
_PYTHON_ACOMPLETION: Final = _python_acompletion()
_ACOMPLETION: Final = signature(_PYTHON_ACOMPLETION)
@ -61,11 +66,32 @@ def _public_request(
return native_call(legacy, args, kwargs)
_FEATURES: Final[detectors.Detectors] = MappingProxyType(
{
**detectors.COMMON,
Feature.MOCK_RESPONSE: route_detectors.chat_mock_response,
Feature.AGENTIC_LOOP_HOOK: route_detectors.chat_agentic_loop_hook,
Feature.RESPONSE_CACHE: detectors.response_cache,
Feature.STREAMING: detectors.streaming,
Feature.DROP_PARAMS: detectors.drop_params,
Feature.MODIFY_PARAMS: detectors.modify_params,
Feature.EXTRA_BODY: detectors.extra_body,
Feature.RETRIES_FALLBACKS: route_detectors.chat_retries_fallbacks,
Feature.PROMPT_MANAGEMENT: detectors.prompt_management,
Feature.CACHE_CONTROL_INJECTION: detectors.cache_control_injection,
Feature.MODEL_PARAM_GATING: route_detectors.chat_model_param_gating,
Feature.MODEL_LIST: route_detectors.model_list,
Feature.MODEL_ALIAS_MAP: route_detectors.model_alias_map,
}
)
def _context(request: NativeCall) -> RouteContext:
return RouteContext(
Route.CHAT_COMPLETIONS,
provider=optional_str(request.resolved.get("custom_llm_provider")),
model=str(request.resolved["model"]),
features=detectors.detected(_FEATURES, request),
)
@ -73,7 +99,6 @@ _DISPATCH: Final = PublicDispatch(
route=Route.CHAT_COMPLETIONS,
request=lambda args, kwargs: _public_request(_COMPLETION, args, kwargs),
context=_context,
bypass=lambda request: request.kwargs.get("acompletion") is True,
)
_ADISPATCH: Final = PublicDispatch(
@ -88,9 +113,11 @@ def completion(
**kwargs: object, # kwargs-ok: preserve the public chat completions call shape
) -> ChatResult | Coroutine[object, object, ChatResult]:
python: Final = _PYTHON_COMPLETION
if _FLAG.requested(kwargs):
return acompletion(**_FLAG.keywords(args, kwargs))
return _DISPATCH.run(
args,
kwargs,
_FLAG.dropped(kwargs),
python=python,
binding=NATIVE_COMPLETION,
native=native_call_hook,
@ -101,7 +128,7 @@ async def acompletion(*args: object, **kwargs: object) -> ChatResult: # kwargs-
python: Final = _PYTHON_ACOMPLETION
return await _ADISPATCH.arun(
args,
kwargs,
_FLAG.dropped(kwargs),
python=python,
binding=NATIVE_ACOMPLETION,
native=native_call_hook,

View file

@ -25,6 +25,8 @@ CLAUDE_CODE_CLIENT: Final = "claude-code"
MICROSOFT_365_COPILOT_DEFAULT_TIME_ZONE: Final = "UTC"
GITHUB_COPILOT_PER_USER_AUTH_TYPE: Final = "per_user_oauth"
GITHUB_COPILOT_AUTH_TYPE_KEY: Final = "github_copilot_auth_type"
GITHUB_COPILOT_HOST_KEY: Final = "github_copilot_host"
GITHUB_COPILOT_DEFAULT_HOST: Final = "github.com"
GITHUB_COPILOT_USER_TOKEN_SAFETY_MARGIN_SECONDS: Final = 60
GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS: Final = 60
GITHUB_COPILOT_DEVICE_FLOW_CACHE_PREFIX: Final = "github_copilot_device_flow"
@ -154,6 +156,9 @@ DEFAULT_REPLICATE_POLLING_RETRIES: Final = get_env_int("DEFAULT_REPLICATE_POLLIN
DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)
DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250)
HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS: Final = get_env_int_in_range(
"HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 300, 1, 86400
)
# Maximum wall-clock seconds a streaming response is allowed to run.
# Streams exceeding this duration are terminated with a Timeout error.
@ -256,6 +261,9 @@ MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS: Final = get_env_int("MCP_PER_USER_TOKE
# MCP timeout defaults (seconds). Override via env vars for slow/custom MCP servers.
MCP_CLIENT_TIMEOUT: Final = get_env_float("LITELLM_MCP_CLIENT_TIMEOUT", 60.0)
MCP_MAX_SSE_EVENT_SIZE_BYTES: Final = get_env_int_in_range(
"LITELLM_MCP_MAX_SSE_EVENT_SIZE_BYTES", 64 * 1024 * 1024, 1, sys.maxsize
)
MCP_TOOL_LISTING_TIMEOUT: Final = get_env_float("LITELLM_MCP_TOOL_LISTING_TIMEOUT", 30.0)
MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES: Final = get_env_int("LITELLM_MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES", 3)
MCP_UPSTREAM_RATE_LIMIT_BASE_DELAY_SECONDS: Final = get_env_float(
@ -472,6 +480,7 @@ REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_end_user_sp
REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_agent_spend_update_buffer"
REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_tag_spend_update_buffer"
REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_window_spend_update_buffer"
REDIS_TOOL_DISCOVERY_BUFFER_KEY: Final = "litellm_tool_discovery_buffer"
MAX_REDIS_BUFFER_DEQUEUE_COUNT: Final = get_env_int("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100)
REDIS_SPEND_LOGS_BUFFER_KEY: Final = "litellm_spend_logs_buffer"
REDIS_SPEND_LOGS_BUFFER_MAX_ROWS: Final = 100000
@ -603,6 +612,7 @@ NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int(
) # timeout for adjacent services (e.g. jwt auth)
MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000)
MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048)
MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024
BEDROCK_MAX_POLICY_SIZE: Final = get_env_int("BEDROCK_MAX_POLICY_SIZE", 75)
# One entry per distinct AWS credential-argument set. Per-user cost attribution passes the attributed
# identity as aws_session_name, so this bounds how many attributed identities keep a cached STS session.
@ -852,6 +862,7 @@ LITELLM_CHAT_PROVIDERS: Final = [
"heroku",
"oci",
"morph",
"opencode_go",
"lambda_ai",
"inception",
"vercel_ai_gateway",
@ -1107,6 +1118,7 @@ openai_compatible_providers: Final[list[str]] = [
"v0",
"helicone",
"morph",
"opencode_go",
"lambda_ai",
"inception",
"hyperbolic",
@ -1649,6 +1661,7 @@ BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
@ -1812,6 +1825,7 @@ LITELLM_EXECUTED_BATCH_CONCURRENCY: Final = max(1, get_env_int("LITELLM_EXECUTED
########################### CLI SSO AUTHENTICATION CONSTANTS ###########################
LITELLM_CLI_SOURCE_IDENTIFIER: Final = "litellm-cli"
LITELLM_CLI_SESSION_TOKEN_PREFIX: Final = "litellm-session-token"
CP_RETURN_TO_COOKIE_PREFIX: Final = "return_to.v1."
CLI_SSO_SESSION_CACHE_KEY_PREFIX: Final = "cli_sso_session"
CLI_SSO_SESSION_TTL_SECONDS: Final = 600
CLI_SESSION_KEY_PREFIX: Final = "cli-session"
@ -1831,6 +1845,7 @@ LITELLM_UI_SESSION_DURATION: Final = os.getenv("LITELLM_UI_SESSION_DURATION", "2
########################### DB CRON JOB NAMES ###########################
DB_SPEND_UPDATE_JOB_NAME: Final = "db_spend_update_job"
TOOL_DISCOVERY_COMMIT_JOB_NAME: Final = "tool_discovery_commit_job"
DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME: Final = "db_daily_tag_spend_update_job"
PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME: Final = "prometheus_emit_budget_metrics"
CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME: Final = "cloudzero_export_usage_data"
@ -1855,6 +1870,9 @@ SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS: Final = get_env_float(
SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS: Final = get_env_float("SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS", 300.0)
SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS: Final = get_env_float("SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS", 30.0)
SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP: Final = get_env_int("SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP", 100000)
SPEND_LOG_SIZE_CHECK_JOB_NAME: Final = "spend_log_size_check"
SPEND_LOG_SIZE_CHECK_INTERVAL: Final = os.getenv("SPEND_LOG_SIZE_CHECK_INTERVAL", "1d")
SPEND_LOG_SIZE_CLEANUP_RECHECK_BATCHES: Final = get_env_int("SPEND_LOG_SIZE_CLEANUP_RECHECK_BATCHES", 10)
SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS: Final = get_env_float(
"SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS", 5.0
)
@ -1976,6 +1994,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
"public_model_groups",
"public_model_groups_links",
"cost_discount_config",
"apply_search_free_tier",
"cost_margin_config",
"block_requests_for_models_without_pricing",
"budget_exceeded_throttle_percentage",
@ -2316,6 +2335,7 @@ HARNESS_STDERR_TAIL_LINES: Final = 40
HARNESS_STREAM_READ_CHUNK_BYTES: Final = 64 * 1024
HARNESS_EVENT_QUEUE_MAX_SIZE: Final = 1024
HARNESS_PROCESS_KILL_GRACE_SECONDS: Final = 5.0
HARNESS_RESUME_COMMAND_TIMEOUT_SECONDS: Final = 30.0
HARNESS_SNAPSHOT_SKIP_DIRS: Final = frozenset(
{
".git",

View file

@ -37,6 +37,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
_SERVICE_TIER_TO_COST_KEY_SUFFIX,
BilledTokenRates,
CostCalculatorUtils,
calculate_cache_writing_cost,
calculate_cost_component,
generic_cost_per_character,
generic_cost_per_token,
@ -112,6 +113,7 @@ from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_toke
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.agents import LiteLLMSendMessageResponse
from litellm.types.decisions import DecisionsResponse, DecisionsUsage, OpenAIDecisionResponse, OpenAIDecisionUsage
from litellm.types.extraction import ExtractionResponse, ExtractionUsage
from litellm.types.llms.base import CachedTokensDetails, LiteLLMBaseModel
from litellm.types.llms.openai import (
HttpxBinaryResponseContent,
@ -906,11 +908,15 @@ def _bedrock_pricing_info(
return model_info_with_pricing_overrides(profile, registration[1])
def _declares_rates(model_info: ModelInfo | None) -> bool:
entry: Final = _raw_cost_map_entry(model_info["key"]) if model_info is not None else None
def cost_map_key_prices_anything(key: str) -> bool:
entry: Final = _raw_cost_map_entry(key)
return entry is not None and _cost_map_entry_prices_anything(entry)
def _declares_rates(model_info: ModelInfo | None) -> bool:
return model_info is not None and cost_map_key_prices_anything(model_info["key"])
def _base_model_pricing_provider(base_model: str, custom_llm_provider: str | None) -> str | None:
"""The provider to price an explicit base_model under, when the deployment's own provider has
no rates for it (e.g. chatgpt/ subscription rows) but the base model's own row does"""
@ -1167,6 +1173,8 @@ def get_usage_object(
if isinstance(completion_response, (DecisionsResponse, OpenAIDecisionResponse)):
return None if completion_response.usage is None else _decisions_usage(completion_response.usage)
if isinstance(completion_response, ExtractionResponse):
return _extraction_usage(completion_response.usage)
if isinstance(completion_response, HttpxBinaryResponseContent):
return completion_response.usage
if usage_obj is None:
@ -1193,12 +1201,16 @@ def get_usage_object(
elif isinstance(usage_obj, dict):
return Usage(**usage_obj)
elif isinstance(usage_obj, BaseModel):
return Usage(**usage_obj.model_dump())
return _usage_from_model(usage_obj)
else:
verbose_logger.debug("Unknown usage object type: %s, usage_obj: %s", type(usage_obj), usage_obj)
return None
def _usage_from_model(usage_model: BaseModel) -> Usage:
return Usage(**usage_model.model_dump())
def _decisions_prompt_tokens_details(usage: DecisionsUsage | OpenAIDecisionUsage) -> PromptTokensDetailsWrapper:
match usage:
case DecisionsUsage():
@ -1223,6 +1235,14 @@ def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage:
)
def _extraction_usage(usage: ExtractionUsage) -> Usage:
return Usage(
prompt_tokens=usage.input_tokens,
completion_tokens=usage.output_tokens,
total_tokens=usage.input_tokens + usage.output_tokens,
)
def _is_known_usage_objects(usage_obj):
"""Returns True if the usage obj is a known Usage type"""
return (
@ -1230,6 +1250,7 @@ def _is_known_usage_objects(usage_obj):
or isinstance(usage_obj, ResponseAPIUsage)
or isinstance(usage_obj, DecisionsUsage)
or isinstance(usage_obj, OpenAIDecisionUsage)
or isinstance(usage_obj, ExtractionUsage)
or TranscriptionUsageObjectTransformation.is_transcription_usage_object(usage_obj)
)
@ -1259,6 +1280,8 @@ def infer_call_type(call_type: CallTypesLiteral | None, completion_response: obj
return "send_message"
elif isinstance(completion_response, (DecisionsResponse, OpenAIDecisionResponse)):
return "adecisions"
elif isinstance(completion_response, ExtractionResponse):
return "aextraction"
return call_type
@ -1507,6 +1530,7 @@ def completion_cost(
litellm_model_name: str | None = None,
router_model_id: str | None = None,
litellm_logging_obj: LitellmLoggingObject | None = None,
pricing_provider: str | None = None,
### SERVICE TIER ###
service_tier: str | None = None, # for OpenAI service tier pricing
### DATA RESIDENCY ###
@ -1527,6 +1551,10 @@ def completion_cost(
"""
try:
call_type = infer_call_type(call_type, completion_response) or "completion"
if isinstance(completion_response, DecisionsResponse) and completion_response.usage is None:
raise ValueError("Cannot calculate Decisions cost: provider did not report usage")
if pricing_provider is not None:
custom_llm_provider = pricing_provider # rebind-ok: model_info.pricing_provider overrides the routing provider for cost-map lookups
if call_type == CallTypes.aresponses_websocket.value and isinstance(
completion_response, LiteLLMRealtimeStreamLoggingObject
@ -1552,6 +1580,7 @@ def completion_cost(
optional_params=optional_params,
custom_pricing=custom_pricing,
base_model=base_model,
pricing_provider=pricing_provider,
standard_built_in_tools_params=standard_built_in_tools_params,
litellm_model_name=litellm_model_name,
router_model_id=router_model_id,
@ -1642,17 +1671,19 @@ def completion_cost(
*(((model, request_pricing_provider),) if model is not None else ()),
)
potential_model_names: Final = (
raw_candidates if explicit_pricing else _priced_candidates_before_unpriced_rules(raw_candidates)
raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates)
)
for idx, (model, pricing_provider) in enumerate(potential_model_names):
for idx, (model, candidate_pricing_provider) in enumerate(potential_model_names):
try:
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug("selected model name for cost calculation: %s", model)
if completion_response is not None and (
isinstance(completion_response, BaseModel) or isinstance(completion_response, dict)
): # tts returns a custom class
isinstance(completion_response, (BaseModel, dict))
or isinstance(completion_response, HttpxBinaryResponseContent)
and completion_response.usage is not None
):
if isinstance(completion_response, dict):
usage_obj: dict | Usage | None = completion_response.get("usage", {})
else:
@ -1662,10 +1693,12 @@ def completion_cost(
setattr(
completion_response,
"usage",
litellm.Usage(**_usage_for_dump.model_dump()),
_usage_from_model(_usage_for_dump),
)
if isinstance(usage_obj, (DecisionsUsage, OpenAIDecisionUsage)):
_usage = _decisions_usage(usage_obj).model_dump()
elif isinstance(usage_obj, ExtractionUsage):
_usage = _extraction_usage(usage_obj).model_dump()
elif usage_obj is None:
_usage = {}
elif isinstance(usage_obj, BaseModel):
@ -1712,7 +1745,9 @@ def completion_cost(
hidden_params = getattr(completion_response, "_hidden_params", None)
if hidden_params is not None:
custom_llm_provider = hidden_params.get("custom_llm_provider", custom_llm_provider or None)
custom_llm_provider = pricing_provider or hidden_params.get(
"custom_llm_provider", custom_llm_provider or None
)
region_name = hidden_params.get("region_name", region_name)
else:
@ -1998,7 +2033,7 @@ def completion_cost(
prompt_tokens=prompt_tokens or 0,
completion_tokens=completion_tokens or 0,
custom_llm_provider=_pricing_provider_for_cost_candidate(
candidate_provider=pricing_provider,
candidate_provider=candidate_pricing_provider,
request_provider=request_pricing_provider,
current_provider=custom_llm_provider,
),
@ -2102,7 +2137,7 @@ def completion_cost(
_token_type_breakdown = get_token_type_cost_breakdown(
model=model,
custom_llm_provider=_pricing_provider_for_cost_candidate(
candidate_provider=pricing_provider,
candidate_provider=candidate_pricing_provider,
request_provider=request_pricing_provider,
current_provider=custom_llm_provider,
),
@ -2189,7 +2224,8 @@ def response_cost_calculator(
| Response
| SearchResponse
| DecisionsResponse
| OpenAIDecisionResponse,
| OpenAIDecisionResponse
| ExtractionResponse,
model: str,
custom_llm_provider: str | None,
call_type: Literal[
@ -2215,6 +2251,8 @@ def response_cost_calculator(
"adecisions",
"systemone",
"asystemone",
"extraction",
"aextraction",
],
optional_params: dict,
cache_hit: bool | None = None,
@ -2225,6 +2263,7 @@ def response_cost_calculator(
litellm_model_name: str | None = None,
router_model_id: str | None = None,
litellm_logging_obj: LitellmLoggingObject | None = None,
pricing_provider: str | None = None,
### SERVICE TIER ###
service_tier: str | None = None, # for OpenAI service tier pricing
### DATA RESIDENCY ###
@ -2232,6 +2271,8 @@ def response_cost_calculator(
### VERTEX LOCATION ###
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
region_name: str | None = None,
custom_cost_per_token: CostPerToken | None = None,
custom_cost_per_second: float | None = None,
) -> float:
"""
Returns
@ -2260,11 +2301,14 @@ def response_cost_calculator(
optional_params=optional_params,
custom_pricing=custom_pricing,
base_model=base_model,
custom_cost_per_token=custom_cost_per_token,
custom_cost_per_second=custom_cost_per_second,
prompt=prompt,
standard_built_in_tools_params=standard_built_in_tools_params,
litellm_model_name=litellm_model_name,
router_model_id=router_model_id,
litellm_logging_obj=litellm_logging_obj,
pricing_provider=pricing_provider,
service_tier=service_tier,
data_residency=data_residency,
vertex_location=vertex_location,
@ -2842,6 +2886,7 @@ def _batch_rate(
model_info: ModelInfo,
key: Literal[
"input_cost_per_audio_token_batches",
"cache_read_input_audio_token_cost_batches",
"input_cost_per_image_token_batches",
"input_cost_per_video_token_batches",
],
@ -2927,7 +2972,12 @@ def batch_cost_calculator(
total_completion_cost = 0.0
if batch_rates.input is not None:
total_prompt_cost = _batch_prompt_cost(
usage, model_info, batch_rates.input, batch_rates.cache_read, batch_rates.cache_creation
usage,
model_info,
batch_rates.input,
batch_rates.cache_read,
batch_rates.cache_creation,
batch_rates.cache_creation_rates_by_ttl,
)
elif input_cost_per_token:
details: Final = parse_prompt_tokens_details(usage)
@ -2972,10 +3022,13 @@ def _batch_prompt_cost(
input_rate: float,
cache_read_rate: float | None,
cache_creation_rate: float | None,
cache_creation_rates_by_ttl: Mapping[str, float],
) -> float:
details: Final = parse_prompt_tokens_details(usage)
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl)
written_tokens: Final = details["cache_creation_tokens"] if bills_cache_writes else 0
audio_tokens, image_tokens, video_tokens = (
details["audio_tokens"],
details["image_tokens"],
@ -2989,13 +3042,27 @@ def _batch_prompt_cost(
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
+ cached_tokens * (cache_read_rate or 0.0)
+ written_tokens * (cache_creation_rate or 0.0)
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
+ cached_audio_tokens
* _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0)
+ (
calculate_cache_writing_cost(
written_tokens,
details["cache_creation_token_details"],
input_rate if cache_creation_rate is None else cache_creation_rate,
cache_creation_rates_by_ttl,
)
if bills_cache_writes
else 0.0
)
)
_ATTRIBUTE_VALUE: Final = TypeAdapter(object)
def _attribute_value(obj: object, name: str) -> object:
return getattr(obj, name)
return _ATTRIBUTE_VALUE.validate_python(getattr(obj, name))
def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]:
@ -3630,7 +3697,26 @@ def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo |
return None
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...]) -> float | None:
_TranscriptionRateKey: TypeAlias = Literal[
"input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
]
def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None:
match key:
case "input_cost_per_second":
return info.get("input_cost_per_second")
case "input_cost_per_audio_token":
return info.get("input_cost_per_audio_token")
case "input_cost_per_token":
return info.get("input_cost_per_token")
case "output_cost_per_token":
return info.get("output_cost_per_token")
case _:
assert_never(key)
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
"""First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry
because ``get_model_info`` synthesizes zero token rates for entries that omit them."""
if info is None:
@ -3639,12 +3725,18 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...])
if declared is None:
return None
return next(
(float(value) for key in keys if declared.get(key) is not None and (value := info.get(key)) is not None),
(
float(value)
for key in keys
if declared.get(key) is not None and (value := _model_info_rate(info, key)) is not None
),
None,
)
def _transcription_rate(keys: tuple[str, ...], override: ModelInfo | None, base: ModelInfo | None) -> float:
def _transcription_rate(
keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None
) -> float:
rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base))
return next((rate for rate in rates if rate is not None), 0.0)

View file

@ -11,6 +11,7 @@ from pydantic_core import to_jsonable_python
from typing_extensions import assert_never
import litellm
from litellm.cost_calculator import cost_map_key_prices_anything
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -21,6 +22,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.decisions.transformation import (
BaseDecisionsConfig,
DecisionsResponseValidationError,
image_media_type,
image_param,
input_images,
@ -427,7 +429,7 @@ def _jsonable_fallback(value: object) -> object:
def _logged_request_message(caller_request: object) -> ChatCompletionUserMessage:
caller_fields: Final = _LOGGED_FIELDS_ADAPTER.validate_python(
to_jsonable_python(caller_request, fallback=_jsonable_fallback)
to_jsonable_python(caller_request, fallback=_jsonable_fallback, bytes_mode="base64")
)
sent_fields: Final = json.dumps(
dict((name, value) for name, value in caller_fields.items() if value not in (None, [], {}))
@ -687,6 +689,13 @@ def _request_body(call: DecisionsCall) -> Mapping[str, object]:
return body
def _pricing_model(response: DecisionsIRResponse, call: DecisionsCall) -> str:
returned: Final = get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
if returned is not None and cost_map_key_prices_anything(returned):
return returned
return f"{call.custom_llm_provider}/{call.model}"
def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
formatted: Final = _formatted_response(response, call)
provider_cost: Final = call.provider_config.provider_reported_cost(response)
@ -694,10 +703,7 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci
{
"model": f"{call.custom_llm_provider}/{call.model}",
"custom_llm_provider": call.custom_llm_provider,
"provider_response_model": (
get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
or f"{call.custom_llm_provider}/{call.model}"
),
"provider_response_model": _pricing_model(response, call),
**({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}),
}
)
@ -727,18 +733,21 @@ def _formatted_response(
def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception:
if isinstance(error, BaseLLMException) and error.status_code_is_synthesized:
provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception"
return litellm.APIConnectionError(
message=f"{provider_label} - {error.message}",
llm_provider=call.custom_llm_provider,
model=f"{call.custom_llm_provider}/{call.model}",
)
return litellm.exception_type(
model=f"{call.custom_llm_provider}/{call.model}",
custom_llm_provider=call.custom_llm_provider,
original_exception=error,
)
provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception"
model: Final = f"{call.custom_llm_provider}/{call.model}"
match error:
case DecisionsResponseValidationError():
return litellm.InternalServerError(
message=f"{provider_label} - {error.message}", llm_provider=call.custom_llm_provider, model=model
)
case BaseLLMException(status_code_is_synthesized=True):
return litellm.APIConnectionError(
message=f"{provider_label} - {error.message}", llm_provider=call.custom_llm_provider, model=model
)
case _:
return litellm.exception_type(
model=model, custom_llm_provider=call.custom_llm_provider, original_exception=error
)
async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:

View file

@ -9,7 +9,15 @@ from litellm.rust_bridge.embeddings.entrypoints import (
NATIVE_AEMBEDDING,
NATIVE_EMBEDDING,
)
from litellm.rust_bridge.public_call import NativeCall, bind, native_call, native_call_hook, optional_str, signature
from litellm.rust_bridge.public_call import (
AsyncFlag,
NativeCall,
bind,
native_call,
native_call_hook,
optional_str,
signature,
)
from litellm.types.utils import EmbeddingResponse
__all__ = ("aembedding", "embedding")
@ -24,6 +32,7 @@ _PYTHON_AEMBEDDING: Final = cast( # cast-ok: [LIT006] preserve the legacy publi
PythonAembedding, main.aembedding
)
_EMBEDDING_SIGNATURE: Final = signature(_PYTHON_EMBEDDING)
_FLAG: Final = AsyncFlag("aembedding", sync=_EMBEDDING_SIGNATURE)
def _public_request(
@ -50,7 +59,6 @@ _DISPATCH: Final = PublicDispatch(
route=Route.EMBEDDINGS,
request=lambda args, kwargs: _public_request(_EMBEDDING_SIGNATURE, args, kwargs),
context=_context,
bypass=lambda request: request.kwargs.get("aembedding") is True,
)
_ADISPATCH: Final = PublicDispatch(
@ -64,9 +72,11 @@ def embedding(
*args: object,
**kwargs: object, # kwargs-ok: preserve the public embedding call shape
) -> EmbeddingResponse | Coroutine[object, object, EmbeddingResponse]:
if _FLAG.requested(kwargs):
return aembedding(**_FLAG.keywords(args, kwargs))
return _DISPATCH.run(
args,
kwargs,
_FLAG.dropped(kwargs),
python=_PYTHON_EMBEDDING,
binding=NATIVE_EMBEDDING,
native=native_call_hook,
@ -76,7 +86,7 @@ def embedding(
async def aembedding(*args: object, **kwargs: object) -> EmbeddingResponse: # kwargs-ok: preserve the public call shape
return await _ADISPATCH.arun(
args,
kwargs,
_FLAG.dropped(kwargs),
python=_PYTHON_AEMBEDDING,
binding=NATIVE_AEMBEDDING,
native=native_call_hook,

View file

@ -122,6 +122,7 @@ class SpeechToCompletionBridgeHandler:
if isinstance(result, ModelResponse):
return self.transformation_handler.transform_response(
model_response=result,
model=model,
response_format=requested_response_format if isinstance(requested_response_format, str) else None,
)
else:

View file

@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Final, cast
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache, set_hidden_param
if TYPE_CHECKING:
@ -38,6 +39,11 @@ class ChatAudioParam(TypedDict):
format: ReadOnly[NotRequired[str]]
class ChatSpeechConfigAudioParam(TypedDict):
speech_config: ReadOnly[Mapping[str, object]]
format: ReadOnly[str]
class SpeechToCompletionBridgeTransformationHandler:
def _validate_response_format(
self, model: str, custom_llm_provider: str, optional_params: Mapping[str, object]
@ -49,6 +55,20 @@ class SpeechToCompletionBridgeTransformationHandler:
return
from litellm.exceptions import BadRequestError
if custom_llm_provider.startswith("vertex_ai"):
from litellm.llms.vertex_ai.text_to_speech.transformation import VertexAITextToSpeechConfig
vertex_supported: Final = ", ".join(
sorted({*GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS, *VertexAITextToSpeechConfig.GEMINI_FORMAT_MAPPINGS})
)
raise BadRequestError(
message=(
f"Vertex AI Gemini TTS does not support response_format='{response_format}'."
f" Supported response formats: {vertex_supported}."
),
model=model,
llm_provider=custom_llm_provider,
)
supported: Final = ", ".join(sorted(GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS))
raise BadRequestError(
message=(
@ -75,8 +95,27 @@ class SpeechToCompletionBridgeTransformationHandler:
return response_format if isinstance(response_format, str) else None
def _chat_audio_param(
self, model: str, voice: str | Mapping[str, object] | None, optional_params: Mapping[str, object]
) -> ChatAudioParam | None:
self,
model: str,
voice: str | Mapping[str, object] | None,
optional_params: Mapping[str, object],
drop_params: bool,
custom_llm_provider: str,
) -> ChatAudioParam | ChatSpeechConfigAudioParam | None:
import litellm
if isinstance(voice, Mapping) and self._is_gemini_tts_model(litellm.model_alias_map.get(model, model)):
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
normalize_gemini_speech_config,
reject_unknown_gemini_tts_voice_keys,
)
reject_unknown_gemini_tts_voice_keys(model, voice, drop_params, custom_llm_provider)
speech_audio: Final[ChatSpeechConfigAudioParam] = {
"speech_config": normalize_gemini_speech_config(voice, drops_unsupported_params(drop_params)),
"format": GEMINI_TTS_CHAT_AUDIO_FORMAT,
}
return speech_audio
if not isinstance(voice, str):
return None
audio_format: Final = self._chat_audio_format(model, optional_params)
@ -90,9 +129,9 @@ class SpeechToCompletionBridgeTransformationHandler:
self,
model: str,
input: str,
voice: str | dict | None,
optional_params: dict,
litellm_params: dict,
voice: str | Mapping[str, object] | None,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
headers: dict,
litellm_logging_obj: "LiteLLMLoggingObj",
custom_llm_provider: str,
@ -104,7 +143,13 @@ class SpeechToCompletionBridgeTransformationHandler:
"messages": [user_message],
"modalities": ["audio"],
**self._chat_completion_params(optional_params),
"audio": self._chat_audio_param(model, voice, optional_params),
"audio": self._chat_audio_param(
model,
voice,
optional_params,
drops_unsupported_params(litellm_params.get("drop_params")),
custom_llm_provider,
),
**litellm_params,
"headers": headers,
"litellm_logging_obj": litellm_logging_obj,
@ -153,8 +198,9 @@ class SpeechToCompletionBridgeTransformationHandler:
return wav_header + pcm_data
def _is_gemini_tts_model(self, model: str) -> bool:
"""Check if the model is a Gemini TTS model that returns PCM16 data."""
return "gemini" in model.lower() and ("tts" in model.lower() or "preview-tts" in model.lower())
from litellm.utils import is_gemini_tts_model
return is_gemini_tts_model(model)
def _gemini_tts_response_body(self, decoded_audio: bytes, response_format: str | None) -> tuple[bytes, str]:
if response_format == GEMINI_TTS_RAW_RESPONSE_FORMAT:
@ -162,7 +208,7 @@ class SpeechToCompletionBridgeTransformationHandler:
return self._convert_pcm16_to_wav(decoded_audio), "audio/wav"
def transform_response(
self, model_response: "ModelResponse", response_format: str | None
self, model_response: "ModelResponse", model: str, response_format: str | None
) -> "HttpxBinaryResponseContent":
import base64
@ -176,10 +222,9 @@ class SpeechToCompletionBridgeTransformationHandler:
raise ValueError("No audio part found in the response")
decoded_audio: Final = base64.b64decode(audio_part.data)
model: Final = getattr(model_response, "model", "")
content, content_type = (
self._gemini_tts_response_body(decoded_audio, response_format)
if self._is_gemini_tts_model(model)
if self._is_gemini_tts_model(model) or self._is_gemini_tts_model(model_response.model or "")
else (decoded_audio, "audio/mpeg")
)
response: Final = httpx.Response(

View file

@ -1273,7 +1273,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
self.message = _saved_message
self.args = (_saved_message,)
original_headers: Final = getattr(original_exception, "litellm_response_headers", None)
original_headers: Final[object] = getattr(original_exception, "litellm_response_headers", None)
if original_headers is not None:
self.litellm_response_headers = original_headers

View file

@ -95,6 +95,7 @@ from pydantic import AnyUrl, TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import (
MCP_CLIENT_TIMEOUT,
MCP_MAX_SSE_EVENT_SIZE_BYTES,
MCP_NPM_CACHE_DIR,
MCP_TOOL_LISTING_MAX_PAGES,
MCP_TOOL_LISTING_TIMEOUT,
@ -292,6 +293,7 @@ class _MCPResponseStream(httpx2.AsyncByteStream):
class _MCPHTTPClient(httpx2.AsyncClient):
cleanup_scope: anyio.CancelScope | None = None
cleanup_errors: tuple[Exception, ...] = ()
post_failure: asyncio.Future[Exception] | None = None
def _record_cleanup_error(self, error: Exception) -> None:
if self.cleanup_scope is not None and self.cleanup_scope.shield:
@ -319,7 +321,12 @@ class _MCPHTTPClient(httpx2.AsyncClient):
response: Final = await super().send(request, stream=stream, auth=auth, follow_redirects=follow_redirects)
if request.method == "POST" and response.is_error and response.status_code != 404:
await response.aclose()
response.raise_for_status()
try:
response.raise_for_status()
except httpx2.HTTPStatusError as error:
if self.post_failure is not None and not self.post_failure.done():
self.post_failure.set_result(error)
raise
if stream:
response.stream = _MCPResponseStream(
cast(httpx2.AsyncByteStream, response.stream), self._record_cleanup_error
@ -641,6 +648,7 @@ class MCPClient:
transport_ctx: Final = streamable_http_client(
url=self.server_url,
http_client=http_client,
max_sse_event_size=MCP_MAX_SSE_EVENT_SIZE_BYTES,
)
return transport_ctx, http_client
@ -757,6 +765,8 @@ class MCPClient:
read_stream: Final = transport[0]
write_stream: Final = transport[1]
stream_error: Final[asyncio.Future[Exception]] = asyncio.get_running_loop().create_future()
if isinstance(http_client, _MCPHTTPClient):
http_client.post_failure = stream_error # rebind-ok: operation-scoped stream error
_notify_stream_error(on_stream_error, stream_error)
session_kwargs: Final = (
@ -823,6 +833,7 @@ class MCPClient:
if isinstance(http_client, _MCPHTTPClient):
http_client.cleanup_errors = ()
http_client.cleanup_scope = None
http_client.post_failure = None # rebind-ok: clear operation-scoped stream error
await anyio.lowlevel.checkpoint_if_cancelled()
if cleanup_scope.cancel_called:
raise (

View file

@ -10,6 +10,7 @@ from mcp.types import Tool as MCPTool
from openai.types.chat import ChatCompletionToolParam
from openai.types.responses.function_tool_param import FunctionToolParam
from openai.types.shared_params.function_definition import FunctionDefinition
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import (
@ -20,6 +21,8 @@ from litellm.constants import (
from litellm.types.llms.anthropic import AnthropicMessagesTool
from litellm.types.utils import ChatCompletionMessageToolCall
_DECODED_JSON: Final = TypeAdapter(object)
########################################################
# List MCP Tool functions
@ -200,15 +203,18 @@ async def call_mcp_tool(
return tool_result
def _decode_json_arguments(arguments: str) -> object:
try:
return _DECODED_JSON.validate_python(json.loads(arguments))
except json.JSONDecodeError:
return {}
def _get_function_arguments(function: FunctionDefinition) -> dict:
"""Helper to safely get and parse function arguments."""
arguments = function.get("arguments", {})
if isinstance(arguments, str):
try:
arguments = json.loads(arguments)
except json.JSONDecodeError:
arguments = {}
return arguments if isinstance(arguments, dict) else {}
arguments: Final = function.get("arguments", {})
decoded: Final = _decode_json_arguments(arguments) if isinstance(arguments, str) else arguments
return decoded if isinstance(decoded, dict) else {}
def transform_openai_tool_call_request_to_mcp_tool_call_request(

View file

@ -0,0 +1,3 @@
from litellm.extraction.main import aextraction, extraction
__all__ = ["aextraction", "extraction"]

351
litellm/extraction/main.py Normal file
View file

@ -0,0 +1,351 @@
import asyncio
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from functools import partial
from types import MappingProxyType
from typing import Final, TypeAlias
import httpx
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.decisions.call import log_caller_request
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.tool_fields import DropPolicy, drop_policy
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.extraction.transformation import BaseExtractionConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.types.extraction import (
EXTRACTION_STATE_REWRITTEN_METADATA_KEY,
ExtractionAnswer,
ExtractionBase64Image,
ExtractionField,
ExtractionOptions,
ExtractionRequestBody,
ExtractionResponse,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders, is_litellm_owned_kwarg
from litellm.utils import (
ProviderConfigManager,
client, # pyright: ignore[reportUnknownVariableType] # client is an untyped decorator
get_model_cost_key,
)
ExtractionFields: TypeAlias = Mapping[str, ExtractionField | Mapping[str, object]]
ExtractionImages: TypeAlias = Sequence[str | ExtractionBase64Image | Mapping[str, object]]
ExtractionOptionsParam: TypeAlias = ExtractionOptions | Mapping[str, object]
_REQUEST_ADAPTER: Final[TypeAdapter[ExtractionRequestBody]] = TypeAdapter(ExtractionRequestBody)
_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
_HANDLER: Final = BaseLLMHTTPHandler()
_LOCAL_KWARGS: Final = frozenset(
(
*GenericLiteLLMParams.model_fields,
"extra_headers",
"base_url",
"default_headers",
"deployment_id",
"additional_drop_params",
)
)
@dataclass(frozen=True, slots=True, repr=False)
class _ExtractionCall:
model: str
requested_model: str
custom_llm_provider: str
provider_config: BaseExtractionConfig
body: Mapping[str, object]
api_base: str
api_key: str | None
logging_obj: LiteLLMLoggingObj | None
headers: Mapping[str, str]
timeout: float | httpx.Timeout | None
litellm_params: Mapping[str, object]
client: HTTPHandler | AsyncHTTPHandler | None
def supported_extraction_providers() -> tuple[str, ...]:
return tuple(
provider.value
for provider in LlmProviders
if ProviderConfigManager.get_provider_extraction_config(model="", provider=provider) is not None
)
def _provider_config(model: str, custom_llm_provider: str) -> BaseExtractionConfig:
provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None)
provider_config: Final = (
None if provider is None else ProviderConfigManager.get_provider_extraction_config(model, provider)
)
if provider_config is None:
raise litellm.BadRequestError(
message=(
f"Unknown Extraction provider '{custom_llm_provider}'. "
f"Supported providers: {', '.join(supported_extraction_providers())}"
),
model=model,
llm_provider=custom_llm_provider,
)
return provider_config
def _request_fields(
*,
state: str | None,
fields: ExtractionFields | None,
images: ExtractionImages | None,
reference_date: str | None,
options: ExtractionOptionsParam | None,
) -> Mapping[str, object]:
sent: Final[Mapping[str, object | None]] = MappingProxyType(
{"state": state, "fields": fields, "images": images, "reference_date": reference_date, "options": options}
)
return MappingProxyType({name: value for name, value in sent.items() if value is not None})
def _is_unsupported(name: str, value: object, policy: DropPolicy) -> bool:
return (
value is not None
and value is not False
and name not in _LOCAL_KWARGS
and not is_litellm_owned_kwarg(name)
and not policy.drops(name)
)
def _unsupported_params(kwargs: Mapping[str, object]) -> tuple[str, ...]:
policy: Final = drop_policy(kwargs)
return tuple(sorted(name for name, value in kwargs.items() if _is_unsupported(name, value, policy)))
def _prepare_call(
*,
model: str,
request_fields: Mapping[str, object],
api_key: str | None,
api_base: str | None,
timeout: float | httpx.Timeout | None,
custom_llm_provider: str | None,
extra_headers: Mapping[str, str] | None,
kwargs: Mapping[str, object],
) -> _ExtractionCall:
log_caller_request(kwargs, request_fields)
upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
api_key=api_key,
)
provider_config: Final = _provider_config(model, provider)
unsupported: Final = _unsupported_params(kwargs)
if unsupported:
raise litellm.UnsupportedParamsError(
message=(
f"{provider} does not support parameters: {list(unsupported)}, for model={model}. "
"To drop these, set `litellm.drop_params=True` or for proxy:\n\n"
"`litellm_settings:\n drop_params: true`\n"
),
model=model,
llm_provider=provider,
)
try:
request: Final = _REQUEST_ADAPTER.validate_python(request_fields)
except ValidationError as error:
raise litellm.BadRequestError(
message=f"Invalid Extraction request: {error}", model=model, llm_provider=provider
) from error
canonical_model: Final = provider_config.canonical_model(upstream_model)
resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base)
if resolved_api_base is None:
raise litellm.BadRequestError(
message=provider_config.missing_api_base_message(provider), model=model, llm_provider=provider
)
resolved_api_key: Final = provider_config.resolve_api_key(dynamic_api_key or api_key)
if resolved_api_key is None:
raise litellm.BadRequestError(
message=f"Missing API key for Extraction provider '{provider}'", model=model, llm_provider=provider
)
logging_obj: Final = kwargs.get("litellm_logging_obj")
http_client: Final = kwargs.get("client")
if isinstance(logging_obj, LiteLLMLoggingObj):
logging_obj.update_from_kwargs( # pyright: ignore[reportUnknownMemberType] # Logging takes untyped dicts
kwargs=dict(kwargs),
model=canonical_model,
litellm_params={
"litellm_call_id": kwargs.get("litellm_call_id"),
"api_base": provider_config.get_complete_url(resolved_api_base, canonical_model),
},
custom_llm_provider=provider,
)
return _ExtractionCall(
model=canonical_model,
requested_model=model,
custom_llm_provider=provider,
provider_config=provider_config,
body=provider_config.transform_extraction_request(canonical_model, request),
api_base=resolved_api_base,
api_key=resolved_api_key,
logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None,
headers=MappingProxyType(dict(extra_headers or {})),
timeout=timeout,
litellm_params=kwargs,
client=http_client if isinstance(http_client, HTTPHandler | AsyncHTTPHandler) else None,
)
def _state_rewritten_by_guardrail(litellm_params: Mapping[str, object]) -> bool:
metadata: Final = litellm_params.get(get_metadata_variable_name_from_kwargs(litellm_params))
return (
isinstance(metadata, Mapping)
and _METADATA_ADAPTER.validate_python(metadata).get(EXTRACTION_STATE_REWRITTEN_METADATA_KEY) is True
)
def _without_spans(answers: Mapping[str, ExtractionAnswer]) -> Mapping[str, ExtractionAnswer]:
return {name: answer.model_copy(update={"span": None, "spans": None}) for name, answer in answers.items()}
def _formatted_response(response: ExtractionResponse, call: _ExtractionCall) -> ExtractionResponse:
spans_index_hidden_text: Final = response.state_text is None and _state_rewritten_by_guardrail(call.litellm_params)
formatted: Final = response.model_copy(
update={
"model": call.requested_model,
**({"answers": _without_spans(response.answers)} if spans_index_hidden_text else {}),
}
)
formatted.set_hidden_params(
{
"model": f"{call.custom_llm_provider}/{call.model}",
"custom_llm_provider": call.custom_llm_provider,
"provider_response_model": (
get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
or f"{call.custom_llm_provider}/{call.model}"
),
}
)
return formatted
def _upstream_exception(error: Exception, call: _ExtractionCall) -> Exception:
if isinstance(error, BaseLLMException) and error.status_code_is_synthesized:
return litellm.APIConnectionError(
message=f"{call.custom_llm_provider.capitalize()}Exception - {error.message}",
llm_provider=call.custom_llm_provider,
model=f"{call.custom_llm_provider}/{call.model}",
)
return litellm.exception_type( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped
model=f"{call.custom_llm_provider}/{call.model}",
custom_llm_provider=call.custom_llm_provider,
original_exception=error,
)
async def _asend(call: _ExtractionCall) -> ExtractionResponse:
try:
response: Final = await _HANDLER.aextraction(
model=call.model,
custom_llm_provider=call.custom_llm_provider,
logging_obj=call.logging_obj,
provider_config=call.provider_config,
body=call.body,
api_base=call.api_base,
api_key=call.api_key,
headers=call.headers,
timeout=call.timeout,
litellm_params=call.litellm_params,
client=call.client if isinstance(call.client, AsyncHTTPHandler) else None,
)
except Exception as error:
raise _upstream_exception(error, call) from error
return _formatted_response(response, call)
def _send(call: _ExtractionCall) -> ExtractionResponse:
try:
response: Final = _HANDLER.extraction(
model=call.model,
custom_llm_provider=call.custom_llm_provider,
logging_obj=call.logging_obj,
provider_config=call.provider_config,
body=call.body,
api_base=call.api_base,
api_key=call.api_key,
headers=call.headers,
timeout=call.timeout,
litellm_params=call.litellm_params,
client=call.client if isinstance(call.client, HTTPHandler) else None,
)
except Exception as error:
raise _upstream_exception(error, call) from error
return _formatted_response(response, call)
@client
async def aextraction(
model: str,
fields: ExtractionFields,
state: str | None = None,
images: ExtractionImages | None = None,
reference_date: str | None = None,
options: ExtractionOptionsParam | None = None,
api_key: str | None = None,
api_base: str | None = None,
timeout: float | httpx.Timeout | None = None,
custom_llm_provider: str | None = None,
extra_headers: Mapping[str, str] | None = None,
**kwargs: object, # kwargs-ok: litellm_params forwarded by @client
) -> ExtractionResponse:
call: Final = await asyncio.to_thread(
partial(
_prepare_call,
model=model,
request_fields=_request_fields(
state=state, fields=fields, images=images, reference_date=reference_date, options=options
),
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
)
)
return await _asend(call)
@client
def extraction(
model: str,
fields: ExtractionFields,
state: str | None = None,
images: ExtractionImages | None = None,
reference_date: str | None = None,
options: ExtractionOptionsParam | None = None,
api_key: str | None = None,
api_base: str | None = None,
timeout: float | httpx.Timeout | None = None,
custom_llm_provider: str | None = None,
extra_headers: Mapping[str, str] | None = None,
**kwargs: object, # kwargs-ok: litellm_params forwarded by @client
) -> ExtractionResponse:
call: Final = _prepare_call(
model=model,
request_fields=_request_fields(
state=state, fields=fields, images=images, reference_date=reference_date, options=options
),
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
)
return _send(call)
__all__ = ["aextraction", "extraction"]

View file

@ -9,7 +9,7 @@ import asyncio
import contextvars
import time
import uuid as uuid_module
from collections.abc import Coroutine
from collections.abc import Awaitable, Coroutine
from functools import partial
from typing import Final, Literal, cast
@ -293,6 +293,14 @@ def create_file(
raise e
async def _await_retrieved_file(pending: Awaitable[FileObject | OpenAIFileObject]) -> FileObject | OpenAIFileObject:
return await pending
async def _await_deleted_file(pending: Awaitable[object]) -> object:
return await pending
@client
async def afile_retrieve(
file_id: str,
@ -325,7 +333,7 @@ async def afile_retrieve(
func_with_context: Final = partial(ctx.run, func)
init_response: Final = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
response = await _await_retrieved_file(init_response)
else:
response = init_response
@ -505,7 +513,7 @@ async def afile_delete(
func_with_context: Final = partial(ctx.run, func)
init_response: Final = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
response = await _await_deleted_file(init_response)
else:
response = init_response

View file

@ -1,7 +1,7 @@
import datetime
import traceback
from collections.abc import AsyncIterator, Iterator
from typing import TYPE_CHECKING, Any, Final, Optional, cast
from typing import TYPE_CHECKING, Final, Optional, cast
import anyio
@ -163,7 +163,7 @@ class FileContentStreamingResponse:
merged_hidden_params: Final = cast(
"StandardLoggingHiddenParams",
{
**cast(dict[str, Any], payload.get("hidden_params") or {}),
**(payload.get("hidden_params") or {}),
**self._hidden_params,
},
)

View file

@ -71,7 +71,7 @@ class BaseGoogleGenAIGenerateContentStreamingIterator:
):
self.litellm_logging_obj = litellm_logging_obj
self.request_body = request_body
self.start_time = datetime.now()
self.start_time = litellm_logging_obj.start_time
self.collected_chunks: list[bytes] = []
self.model = model
self.custom_llm_provider = custom_llm_provider
@ -192,6 +192,8 @@ class AsyncGoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateCo
async def __anext__(self):
try:
chunk: Final = await _anext_google_genai_sse_chunk(self.stream_iterator)
if self.litellm_logging_obj.completion_start_time is None:
self.litellm_logging_obj.update_completion_start_time(completion_start_time=datetime.now())
self.collected_chunks.append(chunk)
return chunk
except StopAsyncIteration:

View file

@ -37,6 +37,7 @@ from litellm.harness.context import GatewayTarget
from litellm.harness.errors import HarnessError, HarnessInstallFailed
from litellm.harness.types import Harness, Usage
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
from litellm.types.llms.custom_http import httpxSpecialProvider
if TYPE_CHECKING:
@ -690,18 +691,21 @@ class ModelEndpoint:
async def _invoke_sdk(
self,
route: str,
kwargs: dict[str, Any], # mutable-ok: injects stream_options into the SDK kwargs
kwargs: dict[str, Any], # mutable-ok: SDK call kwargs built by _sdk_kwargs
) -> object:
if route == ROUTE_MESSAGES:
return await litellm.anthropic.messages.acreate(**kwargs)
if route == ROUTE_CHAT:
if kwargs.get("stream"):
stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field
kwargs["stream_options"] = { # mutable-ok: JSON field sent to litellm.acompletion
"include_usage": True,
**stream_options,
if not kwargs.get("stream"):
return await litellm.acompletion(**kwargs)
stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field
return await litellm.acompletion(
**{
**kwargs,
"stream_options": {"include_usage": True, **stream_options},
STREAM_USAGE_INJECTED_FIELD: "include_usage" not in stream_options,
}
return await litellm.acompletion(**kwargs)
)
return await litellm.aresponses(**kwargs)
async def _call_sdk(self, route: str, body: Mapping[str, object]) -> Response:

View file

@ -4,7 +4,6 @@ from __future__ import annotations
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
from typing import Any
from litellm.harness.context import SessionContext
from litellm.harness.errors import CapabilityUnsupported
@ -38,5 +37,5 @@ class BaseHarnessHandler(ABC):
async def resume(self, ctx: SessionContext, native_session_id: str) -> None:
"""Continue the runtime's own session on the next turn."""
async def history(self, ctx: SessionContext) -> list[dict[str, Any]]: # mutable-ok: public history() API shape
async def history(self, ctx: SessionContext) -> list[dict[str, object]]: # mutable-ok: public history() API shape
raise CapabilityUnsupported(f"Harness.{self.config.harness.name} does not expose history")

View file

@ -12,17 +12,27 @@ from __future__ import annotations
import asyncio
import os
from collections import deque
from collections.abc import AsyncIterator, Sequence
from collections.abc import AsyncIterator, Mapping, Sequence
from dataclasses import dataclass
from typing import Final
from litellm._logging import verbose_logger
from litellm.constants import HARNESS_STDERR_TAIL_LINES, HARNESS_STREAM_READ_CHUNK_BYTES
from litellm.constants import (
HARNESS_RESUME_COMMAND_TIMEOUT_SECONDS,
HARNESS_STDERR_TAIL_LINES,
HARNESS_STREAM_READ_CHUNK_BYTES,
)
from litellm.harness.context import SessionContext
from litellm.harness.errors import HarnessInstallFailed, SandboxError
from litellm.harness.errors import HarnessError, HarnessInstallFailed, SandboxError
from litellm.harness.handlers.base import BaseHarnessHandler
from litellm.harness.sandbox.base import Process, Sandbox
from litellm.harness.sandbox.base import CompletedRun, Process, Sandbox
from litellm.harness.types import Event
from litellm.llms.base_llm.harness.transformation import BaseCLIHarnessConfig, HarnessSessionSetup
from litellm.llms.base_llm.harness.transformation import (
BaseCLIHarnessConfig,
HarnessSessionCopy,
HarnessSessionSetup,
HarnessTurnError,
)
from litellm.llms.base_llm.harness.utils import decode_json_line, read_skill_files
# Link <private_dir>/<dir> to a LiteLLM-owned cache dir so a later session can resume.
@ -60,10 +70,10 @@ async def send_stdin(proc: Process, data: str) -> None:
async def private_dir_for(sandbox: Sandbox) -> str:
tempdir = getattr(sandbox, "tempdir", None)
tempdir: Final = sandbox.tempdir if hasattr(sandbox, "tempdir") else None
if tempdir is None:
raise SandboxError(f"{type(sandbox).__name__} has no tempdir(); CLI harnesses need a private config dir")
path: str = await tempdir()
path: Final = await tempdir()
return path
@ -81,6 +91,52 @@ async def persist_dir(sandbox: Sandbox, link_path: str, cache_subpath: str) -> N
)
@dataclass(frozen=True, slots=True)
class ResumeFailed:
reason: str
async def bounded_resume_run(
sandbox: Sandbox, cmd: Sequence[str], env: Mapping[str, str]
) -> CompletedRun | ResumeFailed:
try:
run: Final = await sandbox.run(cmd, env=env, timeout=HARNESS_RESUME_COMMAND_TIMEOUT_SECONDS)
except SandboxError as exc:
return ResumeFailed(str(exc))
return run if run.exit_code == 0 else ResumeFailed(run.stderr.strip() or f"exit code {run.exit_code}")
async def import_session_copy(sandbox: Sandbox, copy: HarnessSessionCopy, env: Mapping[str, str]) -> str | ResumeFailed:
try:
await sandbox.write(copy.path, copy.data)
except SandboxError as exc:
return ResumeFailed(str(exc))
imported: Final = await bounded_resume_run(sandbox, copy.import_command, env)
return imported if isinstance(imported, ResumeFailed) else copy.session_id
async def resumable_session_id(
config: BaseCLIHarnessConfig[object, object],
sandbox: Sandbox,
setup_env: Mapping[str, str],
private_dir: str,
native_session_id: str,
) -> str | ResumeFailed:
export_cmd: Final = config.transform_resume_export_command(native_session_id)
if export_cmd is None:
return native_session_id
exported: Final = await bounded_resume_run(sandbox, export_cmd, setup_env)
if isinstance(exported, ResumeFailed):
return exported
try:
copy: Final = config.transform_resume_copy(exported.stdout, private_dir, sandbox.workdir)
except HarnessError as exc:
return ResumeFailed(str(exc))
if copy is None:
return native_session_id
return await import_session_copy(sandbox, copy, setup_env)
async def copy_skills(sandbox: Sandbox, skills: Sequence[str], skills_root: str) -> None:
for skill in skills:
name = os.path.basename(os.path.realpath(os.fspath(skill)))
@ -89,13 +145,14 @@ async def copy_skills(sandbox: Sandbox, skills: Sequence[str], skills_root: str)
class CLIHarnessHandler(BaseHarnessHandler):
config: BaseCLIHarnessConfig
config: BaseCLIHarnessConfig[object, object]
def __init__(self, config: BaseCLIHarnessConfig) -> None:
super().__init__(config)
self._private_dir: str | None = None
self._setup: HarnessSessionSetup | None = None
self._native_id: str | None = None
self._pending_resume: str | None = None
self._proc: Process | None = None
async def start(self, ctx: SessionContext) -> None:
@ -119,6 +176,9 @@ class CLIHarnessHandler(BaseHarnessHandler):
async def turn(self, ctx: SessionContext, prompt: str) -> AsyncIterator[Event]:
if self._setup is None or self._private_dir is None:
raise RuntimeError("CLIHarnessHandler.turn() called before start()")
if self._pending_resume is not None:
self._native_id = await self._prepared_session_id(ctx, self._setup, self._private_dir, self._pending_resume)
self._pending_resume = None
request = self.config.transform_turn_request(ctx, self._setup, self._private_dir, prompt, self._native_id)
argv: Final = list(request.argv) # mutable-ok: Sandbox.exec takes list[str]
proc = await ctx.sandbox.exec(argv, env=request.env, cwd=request.cwd)
@ -158,4 +218,16 @@ class CLIHarnessHandler(BaseHarnessHandler):
return self._native_id
async def resume(self, ctx: SessionContext, native_session_id: str) -> None:
self._native_id = native_session_id
if native_session_id != self._native_id:
self._native_id = native_session_id
self._pending_resume = native_session_id
async def _prepared_session_id(
self, ctx: SessionContext, setup: HarnessSessionSetup, private_dir: str, saved_id: str
) -> str:
target: Final = await resumable_session_id(self.config, ctx.sandbox, setup.env, private_dir, saved_id)
match target:
case ResumeFailed(reason=reason):
raise HarnessTurnError(f"could not prepare session {saved_id} for resume: {reason}")
case session_id:
return session_id

Some files were not shown because too many files have changed in this diff Show more