mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge branch 'litellm_guardrail_choice_isolation' into litellm_guardrail_stream_inspection
# Conflicts: # litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py
This commit is contained in:
commit
b783f41a6a
2292 changed files with 111812 additions and 26076 deletions
|
|
@ -1291,7 +1291,7 @@ jobs:
|
|||
name: Install MCP SDK1 peer
|
||||
command: |
|
||||
uv venv --python 3.12 .venv-mcp-peer
|
||||
uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
|
||||
uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
|
||||
echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
|
||||
- when:
|
||||
condition:
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ while IFS= read -r file || [ -n "$file" ]; do
|
|||
*) has_backend=true ;;
|
||||
esac
|
||||
case "$file" in
|
||||
model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json | model_prices_and_context_window.schema.json)
|
||||
model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json)
|
||||
has_cost_map=true ;;
|
||||
tests/unit/proxy/*) : ;;
|
||||
*) outside_cost_map_set=true ;;
|
||||
|
|
|
|||
4
.github/CODEOWNERS
vendored
4
.github/CODEOWNERS
vendored
|
|
@ -1,4 +1,4 @@
|
|||
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
|
||||
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
|
||||
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/helm/ @yassin-berriai
|
||||
/terraform/ @yassin-berriai
|
||||
|
|
|
|||
9
.github/PULL_REQUEST_TEMPLATE/general.md
vendored
9
.github/PULL_REQUEST_TEMPLATE/general.md
vendored
|
|
@ -4,6 +4,13 @@
|
|||
Drop every section you have nothing to put in, heading included: a bare "## Relevant issues" or
|
||||
"## Affected release" with nothing under it must not appear in the final description -->
|
||||
|
||||
<!-- Two or three plain sentences a teammate can read in 10 seconds, written the way you'd explain it in Slack
|
||||
Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
|
||||
No headings, bullets, tables, or code names here. Everything else goes inside the Details block below -->
|
||||
|
||||
<details>
|
||||
<summary>Details</summary>
|
||||
|
||||
## TLDR
|
||||
|
||||
<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
|
||||
|
|
@ -164,3 +171,5 @@ Example checklists:
|
|||
## Final Attestation
|
||||
|
||||
- [ ] The tests check the right things, including the edge cases, and regressions in the respective real-world customer use-cases are not possible after this PR
|
||||
|
||||
</details>
|
||||
|
|
|
|||
91
.github/workflows/interactions-spec-canary.yml
vendored
Normal file
91
.github/workflows/interactions-spec-canary.yml
vendored
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
name: Interactions spec canary
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "17 7 * * *"
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/workflows/interactions-spec-canary.yml
|
||||
- tests/unit/interactions/**
|
||||
- litellm/types/interactions/**
|
||||
workflow_dispatch:
|
||||
|
||||
permissions: {}
|
||||
|
||||
concurrency: interactions-spec-canary
|
||||
|
||||
jobs:
|
||||
interactions-spec-canary:
|
||||
name: interactions-spec-canary
|
||||
if: github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
compliance: ${{ steps.compliance.outcome }}
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Install dependencies
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy --no-install-project
|
||||
|
||||
- name: Run the compliance tests against Google's live spec
|
||||
id: compliance
|
||||
run: |
|
||||
uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \
|
||||
--interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \
|
||||
--tb=short -vv -p no:rerunfailures -p no:pytest-retry \
|
||||
--allow-hosts=127.0.0.1,::1,localhost,ai.google.dev
|
||||
|
||||
tracking-issue:
|
||||
name: tracking-issue
|
||||
needs: interactions-spec-canary
|
||||
if: always() && github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
issues: write
|
||||
steps:
|
||||
- name: File the tracking issue unless the compliance step passed, close it when it did
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
env:
|
||||
COMPLIANCE_OUTCOME: ${{ needs.interactions-spec-canary.outputs.compliance }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
with:
|
||||
script: |
|
||||
const title = "Interactions spec canary is red";
|
||||
const outcome = process.env.COMPLIANCE_OUTCOME;
|
||||
const runUrl = process.env.RUN_URL;
|
||||
const query = `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open in:title "${title}"`;
|
||||
const { data: found } = await github.rest.search.issuesAndPullRequests({ q: query, per_page: 20 });
|
||||
const open = found.items.find((issue) => issue.title === title);
|
||||
if (outcome === "success") {
|
||||
if (open === undefined) return;
|
||||
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Green again: ${runUrl}` });
|
||||
await github.rest.issues.update({ ...context.repo, issue_number: open.number, state: "closed" });
|
||||
return;
|
||||
}
|
||||
const state = outcome === "failure" ? "red" : `not run (compliance step outcome: ${outcome || "none"})`;
|
||||
if (open !== undefined) {
|
||||
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Still ${state}: ${runUrl}` });
|
||||
return;
|
||||
}
|
||||
await github.rest.issues.create({
|
||||
...context.repo,
|
||||
title,
|
||||
assignees: ["mateo-berri"],
|
||||
body: `The scheduled run of tests/unit/interactions/test_openapi_compliance.py against Google's live Interactions spec is ${state}: ${runUrl}\n\nlitellm/types/interactions/README.md says how to refresh the snapshot and regenerate the types.`,
|
||||
});
|
||||
11
.github/workflows/test-unit.yml
vendored
11
.github/workflows/test-unit.yml
vendored
|
|
@ -139,6 +139,7 @@ jobs:
|
|||
test-path: |-
|
||||
tests/unit/batch_completion
|
||||
tests/unit/decisions
|
||||
tests/unit/extraction
|
||||
tests/unit/litellm_core_utils
|
||||
tests/unit/systemone
|
||||
python-version: "3.12"
|
||||
|
|
@ -215,10 +216,12 @@ jobs:
|
|||
|
||||
- shard: llms-providers
|
||||
test-path: |-
|
||||
tests/unit/llms/test_api_base_trailing_slash_routes.py
|
||||
tests/unit/llms/test_cache_control_and_reasoning.py
|
||||
tests/unit/llms/test_custom_llm.py
|
||||
tests/unit/llms/test_file_content_block.py
|
||||
tests/unit/llms/test_file_search_responses.py
|
||||
tests/unit/llms/test_get_complete_url_trailing_slash.py
|
||||
tests/unit/llms/test_lifecycle_fix.py
|
||||
tests/unit/llms/test_oss_decision.py
|
||||
tests/unit/llms/test_polling_url_origin_match.py
|
||||
|
|
@ -422,6 +425,7 @@ jobs:
|
|||
tests/unit/realtime_api
|
||||
tests/unit/rerank_api
|
||||
tests/unit/search
|
||||
tests/unit/text_completions
|
||||
tests/unit/vector_stores
|
||||
tests/unit/videos
|
||||
python-version: "3.12"
|
||||
|
|
@ -433,6 +437,13 @@ jobs:
|
|||
|
||||
- shard: rust-bridge-harness
|
||||
test-path: >-
|
||||
tests/test_litellm_rust/test_dispatch.py
|
||||
tests/test_litellm_rust/test_public_entries.py
|
||||
tests/test_litellm_rust/chat_completions/test_dispatch.py
|
||||
tests/test_litellm_rust/embeddings/test_dispatch.py
|
||||
tests/test_litellm_rust/messages/test_dispatch.py
|
||||
tests/test_litellm_rust/ocr/test_dispatch.py
|
||||
tests/test_litellm_rust/responses/test_dispatch.py
|
||||
tests/unit/compression
|
||||
tests/unit/harness
|
||||
tests/unit/rust_bridge
|
||||
|
|
|
|||
11
Dockerfile
11
Dockerfile
|
|
@ -1,10 +1,10 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base image for building
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
|
||||
# Runtime image
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
@ -61,6 +61,7 @@ RUN apk add --no-cache \
|
|||
python-3.13-dev \
|
||||
rust \
|
||||
openssl \
|
||||
libssl3 \
|
||||
openssl-dev \
|
||||
nodejs \
|
||||
npm \
|
||||
|
|
@ -120,8 +121,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
|
|||
FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder
|
||||
COPY --from=uvbin /uv /usr/local/bin/uv
|
||||
RUN apk add --no-cache python-3.13
|
||||
ADD --checksum=sha256:2f7ae5cdd9d91731c0990e74a58239dc3e3fd2bf28dab23b55eafcdc47aaf87e \
|
||||
https://github.com/BerriAI/litellm-admin-agent/archive/ef501e94bc9fbacb9233b922abf71427f030408c.tar.gz /tmp/liteadmin.tar.gz
|
||||
ADD --checksum=sha256:4fb1847e28e2be30ba76dac94f02c4903291326516a0f4ba772f6ffb8b25d05c \
|
||||
https://github.com/BerriAI/litellm-admin-agent/archive/5661c805ce479a0cf11b842ea133cf8d22cd7e34.tar.gz /tmp/liteadmin.tar.gz
|
||||
RUN mkdir /tmp/liteadmin && tar xzf /tmp/liteadmin.tar.gz --strip-components=1 -C /tmp/liteadmin && \
|
||||
uv venv /opt/liteadmin --python python3.13 && \
|
||||
uv pip install --python /opt/liteadmin/bin/python --require-hashes -r /tmp/liteadmin/requirements.txt && \
|
||||
|
|
@ -141,7 +142,7 @@ USER root
|
|||
RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories
|
||||
|
||||
# node (without npm) is required by the prisma CLI at runtime
|
||||
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
|
||||
RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
|
||||
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
|
||||
|
||||
WORKDIR /app
|
||||
|
|
|
|||
|
|
@ -616,15 +616,15 @@ The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws
|
|||
|
||||
### Verify Docker Image Signatures
|
||||
|
||||
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
|
||||
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Releases are signed with the key introduced in [commit `e82a3c6`](https://github.com/BerriAI/litellm/commit/e82a3c646908ba0a6ee0beddc4310d7a6cac4065).
|
||||
|
||||
**Verify using the pinned commit hash (recommended):**
|
||||
|
||||
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
|
||||
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the published signing key:
|
||||
|
||||
```bash
|
||||
cosign verify \
|
||||
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
|
||||
--key https://raw.githubusercontent.com/BerriAI/litellm/e82a3c646908ba0a6ee0beddc4310d7a6cac4065/cosign.pub \
|
||||
ghcr.io/berriai/litellm:<release-tag>
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ To qualify for a bounty, reports must include clear reproduction steps, a reprod
|
|||
- Audit Logs with retention policy
|
||||
- Control Allowed IP Addresses that can access your Cloud LiteLLM Instance
|
||||
|
||||
For security inquiries, please contact us at support@berri.ai
|
||||
For security inquiries, please contact us at security@berri.ai
|
||||
|
||||
#### Supported data regions for LiteLLM Cloud
|
||||
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
|
||||
FROM $UV_IMAGE AS uvbin
|
||||
|
|
@ -43,7 +43,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
|
|||
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
|
||||
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
|
||||
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
@ -109,7 +109,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
|
|||
USER root
|
||||
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
|
||||
apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
@ -124,7 +124,12 @@ ENV HOME=/home/nonroot \
|
|||
PYTHONUNBUFFERED=1 \
|
||||
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
|
||||
|
||||
COPY --from=builder --chown=nonroot:nonroot /app /app
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/backend /app/backend
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
|
||||
COPY --from=builder /opt/prisma /opt/prisma
|
||||
|
||||
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
|
||||
|
|
|
|||
|
|
@ -4,8 +4,7 @@ Every pull request whose diff against its merge base touches one of the two cost
|
|||
checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
|
||||
skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
|
||||
branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
|
||||
get the file checks and additionally may only touch those two files or the legacy schema
|
||||
path and may only add or update models.
|
||||
get the file checks and additionally may only touch those two files and may only add or update models.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -21,7 +20,6 @@ from typing import Final
|
|||
|
||||
COST_MAP_PATH: Final = "model_prices_and_context_window.json"
|
||||
BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
|
||||
LEGACY_SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
|
||||
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
|
||||
BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
|
||||
SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
|
||||
|
|
@ -110,7 +108,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
|
|||
*(
|
||||
f"bot PRs may only change the cost map files, not {path}"
|
||||
for path in changed_files
|
||||
if path not in GUARDED_PATHS and path != LEGACY_SCHEMA_PATH
|
||||
if path not in GUARDED_PATHS
|
||||
),
|
||||
*(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map),
|
||||
*(f"bot PRs may not remove fields: {ref}" for ref in removed_fields),
|
||||
|
|
|
|||
41
cookbook/litellm_proxy_server/batch_api/bedrock/README.md
Normal file
41
cookbook/litellm_proxy_server/batch_api/bedrock/README.md
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
# Bedrock batch endpoints
|
||||
|
||||
Set `aws_bedrock_control_plane_endpoint` on a deployment when batch create, retrieve, and cancel requests must use a custom Bedrock control-plane endpoint. The proxy signs these requests for the configured URL. Without this setting, batch requests use the standard regional Bedrock endpoint
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: bedrock-batch
|
||||
litellm_params:
|
||||
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
|
||||
aws_region_name: us-east-1
|
||||
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
|
||||
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
|
||||
aws_batch_role_arn: os.environ/AWS_BATCH_ROLE_ARN
|
||||
s3_output_bucket_name: os.environ/AWS_S3_OUTPUT_BUCKET_NAME
|
||||
aws_bedrock_control_plane_endpoint: os.environ/BEDROCK_CONTROL_PLANE_ENDPOINT
|
||||
```
|
||||
|
||||
Set `BEDROCK_CONTROL_PLANE_ENDPOINT` to your Bedrock control-plane VPC endpoint or compatible gateway base URL, without `/model-invocation-job`. The proxy adds the operation path. The endpoint must be reachable from the proxy, and the configured AWS region must match the endpoint's signing region
|
||||
|
||||
`api_base` and `aws_bedrock_runtime_endpoint` configure runtime inference separately. They do not override the batch control-plane endpoint. `s3_endpoint_url` configures S3 file access separately
|
||||
|
||||
The control-plane setting belongs in deployment configuration. Sending `aws_bedrock_control_plane_endpoint` in an API request body is rejected. There is no automatic `AWS_BEDROCK_CONTROL_PLANE_ENDPOINT` environment fallback, so reference the desired environment variable explicitly as shown above
|
||||
|
||||
Start the proxy with this configuration and set `LITELLM_API_BASE` to its URL. Set `BEDROCK_BATCH_INPUT_URI` to a valid input JSONL file in S3 that the batch service role can read
|
||||
|
||||
```bash
|
||||
curl -sS "$LITELLM_API_BASE/v1/batches" \
|
||||
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"model\":\"bedrock-batch\",\"input_file_id\":\"$BEDROCK_BATCH_INPUT_URI\",\"endpoint\":\"/v1/chat/completions\",\"completion_window\":\"24h\"}"
|
||||
```
|
||||
|
||||
Use the returned `id` as `BATCH_ID` to retrieve or cancel the same job through the deployment's configured endpoint
|
||||
|
||||
```bash
|
||||
curl -sS "$LITELLM_API_BASE/v1/batches/$BATCH_ID?model=bedrock-batch" \
|
||||
-H "Authorization: Bearer $LITELLM_MASTER_KEY"
|
||||
|
||||
curl -sS -X POST "$LITELLM_API_BASE/v1/batches/$BATCH_ID/cancel?model=bedrock-batch" \
|
||||
-H "Authorization: Bearer $LITELLM_MASTER_KEY"
|
||||
```
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
-----BEGIN PUBLIC KEY-----
|
||||
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb
|
||||
POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g==
|
||||
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEG06dpKM+vEDnmp2zcjyXgVCXxzRg
|
||||
JA5H2LnmwBKQafZGka6OOrWujCEWATQewiAgyD4WQdJ0pvU9RC4mvNAjvQ==
|
||||
-----END PUBLIC KEY-----
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base image for building
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
|
||||
# Runtime image
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
@ -59,6 +59,7 @@ RUN apk add --no-cache \
|
|||
python-3.13 \
|
||||
python-3.13-dev \
|
||||
openssl \
|
||||
libssl3 \
|
||||
openssl-dev \
|
||||
nodejs \
|
||||
npm \
|
||||
|
|
@ -122,7 +123,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
|
|||
USER root
|
||||
|
||||
# node (without npm) is required by the prisma CLI at runtime
|
||||
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
|
||||
RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
|
||||
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
|
||||
|
||||
WORKDIR /app
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base images
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
@ -60,6 +60,7 @@ RUN for i in 1 2 3; do \
|
|||
coreutils \
|
||||
curl \
|
||||
openssl \
|
||||
libssl3 \
|
||||
libsndfile \
|
||||
nodejs \
|
||||
npm && break || sleep 5; \
|
||||
|
|
@ -134,7 +135,7 @@ RUN for i in 1 2 3; do \
|
|||
apk upgrade --no-cache && break || sleep 5; \
|
||||
done && \
|
||||
for i in 1 2 3; do \
|
||||
apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs libevent && break || sleep 5; \
|
||||
apk add --no-cache python-3.13 bash openssl libssl3 tzdata libsndfile nodejs libevent && break || sleep 5; \
|
||||
done
|
||||
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ RUN uv venv --python python && \
|
|||
"google-cloud-aiplatform==1.133.0" \
|
||||
"google-genai==1.37.0" \
|
||||
"anthropic[vertex]==0.84.0" \
|
||||
"grpcio==1.78.0" \
|
||||
"grpcio==1.84.1" \
|
||||
"prometheus-client==0.20.0" \
|
||||
"langfuse==2.59.7" \
|
||||
"opentelemetry-api==1.28.0" \
|
||||
|
|
|
|||
|
|
@ -15,9 +15,10 @@ sys.path.insert(
|
|||
) # Adds the parent directory to the system path
|
||||
import json
|
||||
import sys
|
||||
from typing import Any, List, Literal, Optional
|
||||
from typing import Final, List, Literal, Optional
|
||||
|
||||
from fastapi import HTTPException
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -29,11 +30,13 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
)
|
||||
from litellm.llms.openai.common_utils import decoded_json_object
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
||||
GUARDRAIL_NAME = "aporia"
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class AporiaGuardrail(CustomGuardrail):
|
||||
|
|
@ -68,7 +71,7 @@ class AporiaGuardrail(CustomGuardrail):
|
|||
async def prepare_aporia_request(
|
||||
self, new_messages: List[dict], response_string: Optional[str] = None
|
||||
) -> dict:
|
||||
data: dict[str, Any] = {}
|
||||
data: dict[str, object] = {}
|
||||
if new_messages is not None:
|
||||
data["messages"] = new_messages
|
||||
if response_string is not None:
|
||||
|
|
@ -122,8 +125,10 @@ class AporiaGuardrail(CustomGuardrail):
|
|||
verbose_proxy_logger.debug("Aporia AI response: %s", response.text)
|
||||
if response.status_code == 200:
|
||||
# check if the response was flagged
|
||||
_json_response = response.json()
|
||||
action: str = _json_response.get(
|
||||
_json_response: Final = decoded_json_object(
|
||||
_DECODED_JSON.validate_python(response.json())
|
||||
)
|
||||
action: Final = _json_response.get(
|
||||
"action"
|
||||
) # possible values are modify, passthrough, block, rephrase
|
||||
if action == "block":
|
||||
|
|
|
|||
|
|
@ -7,9 +7,10 @@
|
|||
## This accepts a list of user id's for whom calls will be rejected
|
||||
|
||||
|
||||
from typing import Literal, Optional
|
||||
from typing import Final, Literal, Optional
|
||||
|
||||
from fastapi import HTTPException
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._internal_context import with_service_target
|
||||
|
|
@ -19,6 +20,9 @@ from litellm.integrations.custom_logger import CustomLogger
|
|||
from litellm.proxy._types import LiteLLM_EndUserTable, UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
from litellm.repositories.table_repositories import EndUserRepository
|
||||
|
||||
_REQUEST_BODY_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class ENTERPRISE_BlockedUserList(CustomLogger):
|
||||
|
|
@ -94,15 +98,14 @@ class ENTERPRISE_BlockedUserList(CustomLogger):
|
|||
)
|
||||
if end_user_cache_obj is None and self.prisma_client is not None:
|
||||
# check db
|
||||
end_user_obj = (
|
||||
await self.prisma_client.db.litellm_endusertable.find_unique(
|
||||
where={"user_id": user}
|
||||
)
|
||||
)
|
||||
end_user_id: Final = _REQUEST_BODY_VALUE.validate_python(user)
|
||||
end_user_obj = await EndUserRepository(
|
||||
self.prisma_client
|
||||
).table.find_unique(where={"user_id": end_user_id})
|
||||
if end_user_obj is None: # user not in db - assume not blocked
|
||||
end_user_obj = LiteLLM_EndUserTable(user_id=user, blocked=False)
|
||||
cache.set_cache(key=cache_key, value=end_user_obj, ttl=60)
|
||||
if end_user_obj is not None and end_user_obj.blocked is True:
|
||||
if end_user_obj.blocked is True:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
|
|
|
|||
|
|
@ -6,7 +6,10 @@
|
|||
# +-----------------------------------------------+
|
||||
# Thank you users! We ❤️ you! - Krrish & Ishaan
|
||||
|
||||
from typing import Final
|
||||
|
||||
from fastapi import HTTPException
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -15,6 +18,8 @@ from litellm.proxy._types import UserAPIKeyAuth
|
|||
from litellm.proxy.guardrails._content_utils import iter_message_text
|
||||
from litellm.types.utils import CallTypesLiteral
|
||||
|
||||
_THRESHOLD_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class ENTERPRISE_GoogleTextModeration(CustomLogger):
|
||||
user_api_key_cache = None
|
||||
|
|
@ -69,9 +74,11 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
|
|||
f"{category}_confidence_threshold",
|
||||
default_confidence_threshold,
|
||||
)
|
||||
set_confidence_value = getattr(
|
||||
self,
|
||||
f"{category}_confidence_threshold",
|
||||
set_confidence_value = _THRESHOLD_VALUE.validate_python(
|
||||
getattr(
|
||||
self,
|
||||
f"{category}_confidence_threshold",
|
||||
)
|
||||
)
|
||||
verbose_proxy_logger.info(
|
||||
f"Google Text Moderation: {category}_confidence_threshold: {set_confidence_value}"
|
||||
|
|
|
|||
|
|
@ -1,15 +1,20 @@
|
|||
# this is an example endpoint to receive data from litellm
|
||||
from typing import Final
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
_LOGGED_EVENT: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
@app.post("/log-event")
|
||||
async def log_event(request: Request):
|
||||
try:
|
||||
print("Received /log-event request") # noqa
|
||||
# Assuming the incoming request has JSON data
|
||||
data = await request.json()
|
||||
data: Final = _LOGGED_EVENT.validate_python(await request.json())
|
||||
print("Received request data:") # noqa
|
||||
print(data) # noqa
|
||||
|
||||
|
|
|
|||
|
|
@ -8,25 +8,30 @@
|
|||
## This provides an LLM Guard Integration for content moderation on the proxy
|
||||
|
||||
import asyncio
|
||||
from typing import Final, Optional
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import aiohttp
|
||||
from fastapi import HTTPException
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.llms.openai.common_utils import decoded_json_object
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.utils import CallTypesLiteral
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class _ENTERPRISE_LLMGuard(CustomLogger):
|
||||
# Class variables or attributes
|
||||
def __init__(
|
||||
self,
|
||||
mock_testing: bool = False,
|
||||
mock_redacted_text: Optional[dict] = None,
|
||||
mock_redacted_text: Mapping[str, object] | None = None,
|
||||
):
|
||||
self.mock_redacted_text = mock_redacted_text
|
||||
self.llm_guard_mode = litellm.llm_guard_mode
|
||||
|
|
@ -66,7 +71,9 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
|
|||
async with session.post(
|
||||
analyze_url, json={"prompt": text}
|
||||
) as response:
|
||||
redacted_text = await response.json()
|
||||
redacted_text = _DECODED_JSON.validate_python(
|
||||
await response.json()
|
||||
)
|
||||
verbose_proxy_logger.debug(
|
||||
f"LLM Guard: Received response - {redacted_text}"
|
||||
)
|
||||
|
|
@ -77,12 +84,13 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
|
|||
"error": f"Invalid content moderation response: {redacted_text}"
|
||||
},
|
||||
)
|
||||
if redacted_text.get("is_valid", None) is False:
|
||||
redacted_response: Final = decoded_json_object(redacted_text)
|
||||
if redacted_response.get("is_valid", None) is False:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"error": "Violated content safety policy"},
|
||||
)
|
||||
sanitized_prompt = redacted_text.get("sanitized_prompt")
|
||||
sanitized_prompt: Final = redacted_response.get("sanitized_prompt")
|
||||
return sanitized_prompt if isinstance(sanitized_prompt, str) else text
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception(
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ import os
|
|||
|
||||
from fastapi import HTTPException, status
|
||||
|
||||
from litellm.proxy._types import MODEL_LIST_ROUTES
|
||||
|
||||
|
||||
class EnterpriseRouteChecks:
|
||||
@staticmethod
|
||||
|
|
@ -43,7 +45,7 @@ class EnterpriseRouteChecks:
|
|||
|
||||
# Routes that should remain accessible even when LLM API endpoints are disabled.
|
||||
# These are read-only model listing routes needed by the Admin UI.
|
||||
LLM_API_EXEMPT_ROUTES = ["/models", "/v1/models"]
|
||||
LLM_API_EXEMPT_ROUTES = MODEL_LIST_ROUTES
|
||||
|
||||
@staticmethod
|
||||
def should_call_route(route: str):
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t
|
|||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import replace as dataclasses_replace
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
|
@ -13,6 +14,7 @@ from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tupl
|
|||
from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
|
||||
ManagedObjectCreatorAttribution,
|
||||
ManagedObjectRow,
|
||||
row_metadata,
|
||||
)
|
||||
|
||||
import litellm
|
||||
|
|
@ -39,7 +41,12 @@ if TYPE_CHECKING:
|
|||
from litellm.types.utils import LiteLLMBatch
|
||||
|
||||
|
||||
def _error_names_resource(error: Exception, resource_id: str) -> bool:
|
||||
return resource_id in re.findall(r"""[^\s"'`]+""", str(error))
|
||||
|
||||
|
||||
CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost"
|
||||
RECENT_BATCH_POLL_WINDOW: Final = timedelta(hours=48)
|
||||
|
||||
PROVIDER_TERMINAL_BATCH_STATUSES: Final[Tuple[str, ...]] = (
|
||||
"completed",
|
||||
|
|
@ -84,11 +91,6 @@ def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.Li
|
|||
return TeamRepository(prisma_client).table
|
||||
|
||||
|
||||
def _row_metadata(row: object) -> dict[str, object]:
|
||||
metadata: Final = getattr(row, "metadata", None)
|
||||
return cast(dict[str, object], metadata) if isinstance(metadata, dict) else {} # cast-ok: prisma Json object column
|
||||
|
||||
|
||||
class CheckBatchCost:
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -109,6 +111,11 @@ class CheckBatchCost:
|
|||
# the guaranteed-failing primary query on every subsequent cycle.
|
||||
self._has_batch_processed_column: bool = True
|
||||
self.batch_processed_support_confirmed: bool = False
|
||||
self._poll_cursor: tuple[datetime, str] | None = None
|
||||
self._poll_before: datetime = datetime.now(timezone.utc)
|
||||
self._arrival_cursor: tuple[datetime, str] | None = None
|
||||
self._arrival_before: datetime = self._poll_before
|
||||
self._poll_arrivals: bool = False
|
||||
self._line_item_stores: Final[set[asyncio.Task[None]]] = set()
|
||||
self._line_item_store_turn: Final = asyncio.Lock()
|
||||
self._poll_idle: Final = asyncio.Event()
|
||||
|
|
@ -161,29 +168,9 @@ class CheckBatchCost:
|
|||
f"(older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days) as stale_expired"
|
||||
)
|
||||
|
||||
if not self._has_batch_processed_column:
|
||||
return
|
||||
|
||||
# A row already in a terminal status is never rewritten by the sweep above, so
|
||||
# without this it keeps a poll-page slot forever and starves newer batches.
|
||||
retired: Final = await _managed_object_table(self.prisma_client).update_many(
|
||||
where={
|
||||
"file_purpose": "batch",
|
||||
"batch_processed": False,
|
||||
"status": {"in": ["complete", "completed"]},
|
||||
"created_at": {"lt": cutoff},
|
||||
},
|
||||
data={"batch_processed": True},
|
||||
)
|
||||
if retired > 0:
|
||||
verbose_proxy_logger.warning(
|
||||
f"CheckBatchCost: gave up on {retired} completed managed objects older than "
|
||||
f"{MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days that were never costed"
|
||||
)
|
||||
|
||||
async def _fallback_find_jobs(self) -> "Sequence[ManagedObjectRow]":
|
||||
"""Query batch jobs without the batch_processed filter (for older schemas)."""
|
||||
return await _managed_object_table(self.prisma_client).find_many(
|
||||
return await self._find_jobs(
|
||||
where={
|
||||
"file_purpose": "batch",
|
||||
"status": {
|
||||
|
|
@ -197,10 +184,59 @@ class CheckBatchCost:
|
|||
]
|
||||
},
|
||||
},
|
||||
take=MAX_OBJECTS_PER_POLL_CYCLE,
|
||||
order={"created_at": "asc"},
|
||||
)
|
||||
|
||||
async def _find_jobs(self, where: Mapping[str, object]) -> Sequence[ManagedObjectRow]:
|
||||
arrivals: Final = self._poll_arrivals
|
||||
page: Final = await self._find_job_page(where, arrivals=arrivals)
|
||||
self._poll_arrivals = not arrivals if page else arrivals
|
||||
if page:
|
||||
return page
|
||||
return await self._find_job_page(where, arrivals=not arrivals)
|
||||
|
||||
async def _find_job_page(self, where: Mapping[str, object], *, arrivals: bool) -> Sequence[ManagedObjectRow]:
|
||||
cursor: Final = self._arrival_cursor if arrivals else self._poll_cursor
|
||||
if cursor is None:
|
||||
if arrivals:
|
||||
self._arrival_before = datetime.now(timezone.utc)
|
||||
else:
|
||||
self._poll_before = datetime.now(timezone.utc) - RECENT_BATCH_POLL_WINDOW
|
||||
before: Final = self._arrival_before if arrivals else self._poll_before
|
||||
page: Final = await _managed_object_table(self.prisma_client).find_many(
|
||||
where={
|
||||
**where,
|
||||
"created_at": {
|
||||
"lte": before,
|
||||
**({"gt": before - RECENT_BATCH_POLL_WINDOW} if arrivals else {}),
|
||||
},
|
||||
**(
|
||||
{
|
||||
"OR": [
|
||||
{"created_at": {"gt": cursor[0]}},
|
||||
{"created_at": cursor[0], "id": {"gt": cursor[1]}},
|
||||
]
|
||||
}
|
||||
if cursor is not None
|
||||
else {}
|
||||
),
|
||||
},
|
||||
take=MAX_OBJECTS_PER_POLL_CYCLE,
|
||||
order=[{"created_at": "asc"}, {"id": "asc"}],
|
||||
)
|
||||
next_cursor: Final = (page[-1].created_at, page[-1].id) if len(page) == MAX_OBJECTS_PER_POLL_CYCLE else None
|
||||
if arrivals:
|
||||
self._arrival_cursor = next_cursor
|
||||
else:
|
||||
self._poll_cursor = next_cursor
|
||||
return page
|
||||
|
||||
def _unprocessed_snapshot(self, job: ManagedObjectRow) -> Mapping[str, object]:
|
||||
return {
|
||||
"id": job.id,
|
||||
"updated_at": job.updated_at,
|
||||
**({"batch_processed": False} if self._has_batch_processed_column else {}),
|
||||
}
|
||||
|
||||
async def _retire_job(self, job: "ManagedObjectRow", reason: str) -> None:
|
||||
"""
|
||||
Take a row that can never be costed out of the poll page. Leaving it selectable
|
||||
|
|
@ -208,24 +244,19 @@ class CheckBatchCost:
|
|||
once enough such rows accumulate no newer batch is ever reached. Older schemas
|
||||
without batch_processed can only be excluded through the status filter.
|
||||
"""
|
||||
data: Final = (
|
||||
{"batch_processed": True}
|
||||
if self._has_batch_processed_column
|
||||
else {"status": "stale_expired"}
|
||||
)
|
||||
data: Final = {"batch_processed": True} if self._has_batch_processed_column else {"status": "stale_expired"}
|
||||
try:
|
||||
await _managed_object_table(self.prisma_client).update(
|
||||
where={"id": job.id},
|
||||
retired: Final = await _managed_object_table(self.prisma_client).update_many(
|
||||
where=self._unprocessed_snapshot(job),
|
||||
data=data,
|
||||
)
|
||||
except Exception as db_err:
|
||||
verbose_proxy_logger.error(
|
||||
f"CheckBatchCost: failed to retire uncostable job {job.id} ({reason}): {db_err}"
|
||||
)
|
||||
verbose_proxy_logger.error(f"CheckBatchCost: failed to retire uncostable job {job.id} ({reason}): {db_err}")
|
||||
return
|
||||
if retired == 0:
|
||||
return
|
||||
verbose_proxy_logger.warning(
|
||||
f"CheckBatchCost: job {job.id} can never be costed ({reason}), "
|
||||
"so it will no longer be polled"
|
||||
f"CheckBatchCost: job {job.id} can never be costed ({reason}), so it will no longer be polled"
|
||||
)
|
||||
|
||||
async def _claim_job_for_costing(self, job: "ManagedObjectRow", response: "LiteLLMBatch") -> bool:
|
||||
|
|
@ -312,13 +343,13 @@ class CheckBatchCost:
|
|||
|
||||
from litellm.exceptions import NotFoundError
|
||||
|
||||
return isinstance(error, (NotFoundError, openai.NotFoundError)) and batch_id in str(error)
|
||||
return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, batch_id)
|
||||
|
||||
def _batch_deployment_exists(self, model_id: str) -> bool:
|
||||
"""A 404 only proves the batch is gone when it came from the batch's own
|
||||
deployment. Once that deployment leaves the router, default fallbacks can
|
||||
silently send the retrieve to a provider that never saw the batch, so its
|
||||
404 must not retire the row; the staleness sweep bounds it instead."""
|
||||
404 must not retire a potentially recoverable row."""
|
||||
return self.llm_router.get_deployment(model_id=model_id) is not None
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -333,7 +364,7 @@ class CheckBatchCost:
|
|||
|
||||
if not output_file_id:
|
||||
return False
|
||||
return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
|
||||
return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, output_file_id)
|
||||
|
||||
async def _finalize_unbilled_terminal_job(
|
||||
self, job: "ManagedObjectRow", response: "LiteLLMBatch"
|
||||
|
|
@ -355,18 +386,18 @@ class CheckBatchCost:
|
|||
db_batch_object=job,
|
||||
unified_batch_id=_is_base64_encoded_unified_file_id(job.unified_object_id),
|
||||
)
|
||||
update_data: Final[dict] = {
|
||||
update_data: Final[dict[str, object]] = {
|
||||
"status": response.status,
|
||||
"file_object": response.model_dump_json(),
|
||||
**({"batch_processed": True} if self._has_batch_processed_column else {}),
|
||||
}
|
||||
await _managed_object_table(self.prisma_client).update(
|
||||
where={"id": job.id},
|
||||
finalized: Final = await _managed_object_table(self.prisma_client).update_many(
|
||||
where=self._unprocessed_snapshot(job),
|
||||
data=update_data,
|
||||
)
|
||||
verbose_proxy_logger.info(
|
||||
f"CheckBatchCost: marked job {job.id} as {response.status} in DB"
|
||||
)
|
||||
if finalized == 0:
|
||||
return False
|
||||
verbose_proxy_logger.info(f"CheckBatchCost: marked job {job.id} as {response.status} in DB")
|
||||
return True
|
||||
except Exception as db_err:
|
||||
verbose_proxy_logger.error(
|
||||
|
|
@ -817,20 +848,26 @@ class CheckBatchCost:
|
|||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
await logging_obj.async_success_handler(
|
||||
result=response,
|
||||
batch_cost=batch_result.cost,
|
||||
batch_usage=batch_result.usage,
|
||||
batch_models=batch_result.models,
|
||||
batch_successful_requests=batch_result.successful_requests,
|
||||
batch_failed_requests=batch_result.failed_requests,
|
||||
batch_prompt_cost=batch_result.prompt_cost,
|
||||
batch_completion_cost=batch_result.completion_cost,
|
||||
)
|
||||
except Exception:
|
||||
await self._release_job_claim(job, original_status, original_file_object)
|
||||
raise
|
||||
from litellm.proxy.hooks.proxy_track_cost_callback import track_batch_accounting
|
||||
|
||||
with track_batch_accounting(logging_obj.litellm_call_id) as accounting:
|
||||
try:
|
||||
await logging_obj.async_success_handler(
|
||||
result=response,
|
||||
batch_cost=batch_result.cost,
|
||||
batch_usage=batch_result.usage,
|
||||
batch_models=batch_result.models,
|
||||
batch_successful_requests=batch_result.successful_requests,
|
||||
batch_failed_requests=batch_result.failed_requests,
|
||||
batch_prompt_cost=batch_result.prompt_cost,
|
||||
batch_completion_cost=batch_result.completion_cost,
|
||||
)
|
||||
finally:
|
||||
if accounting.retryable_failure:
|
||||
await self._release_job_claim(job, original_status, original_file_object)
|
||||
|
||||
if accounting.retryable_failure:
|
||||
return None
|
||||
|
||||
# Record batch duration (completed_at - created_at)
|
||||
if prom_logger and response.completed_at and response.created_at:
|
||||
|
|
@ -1003,8 +1040,8 @@ class CheckBatchCost:
|
|||
from litellm.proxy.proxy_server import proxy_config
|
||||
from litellm.secret_managers.main import str_to_bool
|
||||
|
||||
api_key: Final = getattr(job, "api_key", None)
|
||||
team_id: Final = getattr(job, "team_id", None)
|
||||
api_key: Final = job.api_key if hasattr(job, "api_key") else None
|
||||
team_id: Final = job.team_id if hasattr(job, "team_id") else None
|
||||
has_key_row: Final = api_key is not None and is_valid_sha256_hash(api_key)
|
||||
try:
|
||||
key_row: Final = (
|
||||
|
|
@ -1022,7 +1059,9 @@ class CheckBatchCost:
|
|||
if (has_key_row and key_row is None) or (team_id and team_row is None):
|
||||
return True
|
||||
settings: Final = get_dynamic_logging_metadata(
|
||||
UserAPIKeyAuth(metadata=_row_metadata(key_row), team_metadata=_row_metadata(team_row), team_id=team_id),
|
||||
UserAPIKeyAuth(
|
||||
metadata=dict(row_metadata(key_row)), team_metadata=dict(row_metadata(team_row)), team_id=team_id
|
||||
),
|
||||
proxy_config,
|
||||
)
|
||||
callback_vars: Final = (settings.callback_vars if settings is not None else None) or {}
|
||||
|
|
@ -1047,6 +1086,7 @@ class CheckBatchCost:
|
|||
"""
|
||||
try:
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
|
||||
prom_logger = PrometheusLogger.get_instance()
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(f"CheckBatchCost: could not get Prometheus logger: {e}")
|
||||
|
|
@ -1071,7 +1111,7 @@ class CheckBatchCost:
|
|||
# endpoint may transition a batch to "complete" before
|
||||
# CheckBatchCost runs. The batch_processed=False filter
|
||||
# already prevents reprocessing finished batches.
|
||||
jobs = await _managed_object_table(self.prisma_client).find_many(
|
||||
jobs = await self._find_jobs(
|
||||
where={
|
||||
"file_purpose": "batch",
|
||||
"batch_processed": False,
|
||||
|
|
@ -1084,8 +1124,6 @@ class CheckBatchCost:
|
|||
]
|
||||
},
|
||||
},
|
||||
take=MAX_OBJECTS_PER_POLL_CYCLE,
|
||||
order={"created_at": "asc"},
|
||||
)
|
||||
self.batch_processed_support_confirmed = True
|
||||
except Exception as query_err:
|
||||
|
|
@ -1152,7 +1190,7 @@ class CheckBatchCost:
|
|||
continue
|
||||
verbose_proxy_logger.error(
|
||||
f"CheckBatchCost: failed to track cost for batch {batch_id} "
|
||||
f"(job {job.id}); leaving it unprocessed so the next poll retries: {tracking_err}"
|
||||
f"(job {job.id}): {tracking_err}"
|
||||
)
|
||||
self._record_error(prom_logger, "cost_tracking_error")
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -7,6 +7,12 @@ attribution, priced against the deployment that served it. The poll's status che
|
|||
with BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN and client reads are marked managed by the
|
||||
managed files hook, so both stay free. The spend event carries the id the client holds, the row's
|
||||
unified_object_id, so its spend log is found under that id with the _background_cost suffix.
|
||||
The spend event reaches the creating key's or team's own logging callbacks the way that creator's
|
||||
requests do. Its message content is redacted when the creator's key or team logging turns message
|
||||
logging off, else as the proxy is set, the way the batch poll decides it. The row is claimed before
|
||||
the creator is read, so reads that lose the race skip the lookups. A creator whose lookup fails, or
|
||||
whose logging settings the logger rejects, is still billed, through the proxy's own callbacks only
|
||||
and with its message content redacted.
|
||||
A read that finds no row, because the create could not store one, stores it under the reading key
|
||||
and bills it the same way, so a response the create left untracked is still billed once. A streamed
|
||||
background create is free on its own; its stream stores the row under the creating key and bills
|
||||
|
|
@ -29,6 +35,7 @@ from typing import TYPE_CHECKING, Final, cast
|
|||
from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
|
||||
ManagedObjectCreatorAttribution,
|
||||
ManagedObjectRow,
|
||||
ResponseCreator,
|
||||
)
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
|
|
@ -54,12 +61,29 @@ from litellm.types.utils import (
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.router import Router
|
||||
|
||||
_STORED_RESPONSE_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
|
||||
_CALLBACK_LIST_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
class _RoutedModelInfo(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
id: str | None = None
|
||||
|
||||
|
||||
class _RoutedDeployment(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
model_info: _RoutedModelInfo = _RoutedModelInfo()
|
||||
|
||||
|
||||
_ROUTED_DEPLOYMENTS: Final = TypeAdapter(tuple[_RoutedDeployment, ...])
|
||||
_RETRIEVE_RESPONSE_CALL_TYPE: Final = "aget_responses"
|
||||
_CALLBACK_LIST_FIELDS: Final = frozenset({"success_callback", "failure_callback", "litellm_disabled_callbacks"})
|
||||
_REJECTED_CREATOR_LOGGING: Final[Mapping[str, object]] = MappingProxyType({"turn_off_message_logging": True})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -76,6 +100,7 @@ class BilledBackgroundResponse:
|
|||
custom_llm_provider: str | None
|
||||
litellm_params: Mapping[str, object]
|
||||
metadata: Mapping[str, object]
|
||||
creator_logging: Mapping[str, object]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -94,11 +119,51 @@ def _without_read_price(response: ResponsesAPIResponse) -> ResponsesAPIResponse:
|
|||
return unpriced
|
||||
|
||||
|
||||
async def log_billed_background_response(billed: BilledBackgroundResponse) -> BilledCharge:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
def _string_callbacks(callbacks: object) -> tuple[str, ...]:
|
||||
if not isinstance(callbacks, list):
|
||||
return ()
|
||||
return tuple(
|
||||
callback for callback in _CALLBACK_LIST_ADAPTER.validate_python(callbacks) if isinstance(callback, str)
|
||||
)
|
||||
|
||||
logging_obj: Final = LiteLLMLogging(
|
||||
|
||||
def _running_proxy_config() -> "ProxyConfig":
|
||||
from litellm.proxy.proxy_server import proxy_config
|
||||
|
||||
return proxy_config
|
||||
|
||||
|
||||
def creator_logging_fields(creator: ResponseCreator, proxy_config: "ProxyConfig | None") -> Mapping[str, object]:
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.litellm_pre_call_utils import key_and_team_logging_fields
|
||||
|
||||
if not creator.settings_read:
|
||||
return _REJECTED_CREATOR_LOGGING
|
||||
try:
|
||||
return key_and_team_logging_fields(
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
metadata=dict(creator.key_metadata),
|
||||
team_id=creator.team_id,
|
||||
team_metadata=dict(creator.team_metadata),
|
||||
),
|
||||
proxy_config=proxy_config if proxy_config is not None else _running_proxy_config(),
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(
|
||||
f"CheckResponsesCost: could not read the creator's logging settings, so only the proxy's "
|
||||
f"callbacks get the billed event: {e}"
|
||||
)
|
||||
return _REJECTED_CREATOR_LOGGING
|
||||
|
||||
|
||||
def _billed_logging_obj(billed: BilledBackgroundResponse, creator_logging: Mapping[str, object]) -> "LiteLLMLogging":
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.utils import split_dynamic_success_callbacks
|
||||
|
||||
sync_callbacks, async_callbacks = split_dynamic_success_callbacks(
|
||||
_string_callbacks(creator_logging.get("success_callback"))
|
||||
)
|
||||
return LiteLLMLogging(
|
||||
model=billed.model,
|
||||
messages=[{"role": "user", "content": "<retrieve_response>"}],
|
||||
stream=False,
|
||||
|
|
@ -106,7 +171,27 @@ async def log_billed_background_response(billed: BilledBackgroundResponse) -> Bi
|
|||
start_time=datetime.now(),
|
||||
litellm_call_id=str(uuid.uuid4()),
|
||||
function_id=str(uuid.uuid4()),
|
||||
dynamic_success_callbacks=list(sync_callbacks),
|
||||
dynamic_async_success_callbacks=list(async_callbacks) or None,
|
||||
kwargs={key: value for key, value in creator_logging.items() if key not in _CALLBACK_LIST_FIELDS},
|
||||
)
|
||||
|
||||
|
||||
def _creator_or_proxy_logging_obj(billed: BilledBackgroundResponse) -> "LiteLLMLogging":
|
||||
try:
|
||||
return _billed_logging_obj(billed, billed.creator_logging)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(
|
||||
f"CheckResponsesCost: the creator's logging settings were rejected, so only the proxy's "
|
||||
f"callbacks get the billed event: {e}"
|
||||
)
|
||||
return _billed_logging_obj(billed, _REJECTED_CREATOR_LOGGING)
|
||||
|
||||
|
||||
async def log_billed_background_response(billed: BilledBackgroundResponse) -> BilledCharge:
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
||||
logging_obj: Final = _creator_or_proxy_logging_obj(billed)
|
||||
logging_obj.update_environment_variables(
|
||||
litellm_params={**billed.litellm_params, "metadata": dict(billed.metadata)},
|
||||
optional_params={},
|
||||
|
|
@ -174,6 +259,7 @@ class CheckResponsesCost:
|
|||
prisma_client: "PrismaClient",
|
||||
llm_router: "Router",
|
||||
log_billed_response: BilledResponseLogger = log_billed_background_response,
|
||||
proxy_config: "ProxyConfig | None" = None,
|
||||
):
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.router import Router
|
||||
|
|
@ -183,6 +269,7 @@ class CheckResponsesCost:
|
|||
self.llm_router: Router = llm_router
|
||||
self._creator_attribution: Final = ManagedObjectCreatorAttribution(prisma_client, "CheckResponsesCost")
|
||||
self._log_billed_response: Final = log_billed_response
|
||||
self._proxy_config: Final = proxy_config
|
||||
|
||||
def _resolve_deployment(self, response_id: str) -> Deployment | None:
|
||||
model_id: str | None = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
|
||||
|
|
@ -294,11 +381,12 @@ class CheckResponsesCost:
|
|||
data={"status": job.status or PENDING_RESPONSE_STATUSES[0]},
|
||||
)
|
||||
|
||||
async def _billed_response(
|
||||
def _billed_response(
|
||||
self,
|
||||
job: ManagedObjectRow,
|
||||
target: _PollTarget,
|
||||
response: ResponsesAPIResponse,
|
||||
creator: ResponseCreator,
|
||||
) -> BilledBackgroundResponse:
|
||||
deployment: Final = target.deployment
|
||||
return BilledBackgroundResponse(
|
||||
|
|
@ -310,12 +398,26 @@ class CheckResponsesCost:
|
|||
litellm_params=_deployment_litellm_params(deployment) if deployment is not None else MappingProxyType({}),
|
||||
metadata=MappingProxyType(
|
||||
{
|
||||
**(await self._creator_attribution.response_metadata(job, job.unified_object_id)),
|
||||
**creator.metadata,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
|
||||
**target.model_metadata,
|
||||
**(_deployment_metadata(deployment) if deployment is not None else {}),
|
||||
**self._requested_model_group(target, job.team_id),
|
||||
}
|
||||
),
|
||||
creator_logging=creator_logging_fields(creator, self._proxy_config),
|
||||
)
|
||||
|
||||
def _requested_model_group(self, target: _PollTarget, team_id: str | None) -> Mapping[str, object]:
|
||||
requested: Final = target.model_metadata.get("model_group")
|
||||
if target.deployment is None or not isinstance(requested, str):
|
||||
return MappingProxyType({})
|
||||
routed: Final = _ROUTED_DEPLOYMENTS.validate_python(
|
||||
self.llm_router.get_model_list(model_name=requested, team_id=team_id) or ()
|
||||
)
|
||||
serving_id: Final = target.deployment.model_info.id
|
||||
return MappingProxyType(
|
||||
{"model_group": requested} if any(row.model_info.id == serving_id for row in routed) else {}
|
||||
)
|
||||
|
||||
async def _bill_once(
|
||||
|
|
@ -328,7 +430,8 @@ class CheckResponsesCost:
|
|||
verbose_proxy_logger.info(f"Response {job.unified_object_id} was already billed elsewhere, skipping")
|
||||
return None
|
||||
try:
|
||||
charge: Final = await self._log_billed_response(await self._billed_response(job, target, response))
|
||||
creator: Final = await self._creator_attribution.response_creator(job, job.unified_object_id)
|
||||
charge: Final = await self._log_billed_response(self._billed_response(job, target, response, creator))
|
||||
except BaseException:
|
||||
await self._release_billing_claim(job)
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -9,12 +9,17 @@ from __future__ import annotations
|
|||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Final, Protocol, cast
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Final, Protocol, TypeGuard, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import CLI_SESSION_KEY_PREFIX
|
||||
from litellm.repositories.table_repositories import (
|
||||
DeletedVerificationTokenRepository,
|
||||
DeprecatedVerificationTokenRepository,
|
||||
)
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
from litellm.repositories.verification_token_repository import VerificationTokenRepository
|
||||
|
|
@ -27,6 +32,12 @@ if TYPE_CHECKING:
|
|||
|
||||
|
||||
class ManagedObjectRow(Protocol):
|
||||
@property
|
||||
def created_at(self) -> datetime: ...
|
||||
|
||||
@property
|
||||
def updated_at(self) -> datetime: ...
|
||||
|
||||
@property
|
||||
def id(self) -> str: ...
|
||||
|
||||
|
|
@ -63,6 +74,7 @@ class ManagedObjectRow(Protocol):
|
|||
|
||||
_TAG_LIST_ADAPTER: Final[TypeAdapter[list[object]]] = TypeAdapter(list[object])
|
||||
_KEY_OWNER_SCOPE_PREFIX: Final = "key:"
|
||||
_MAX_ROTATION_HOPS: Final = 16
|
||||
|
||||
|
||||
def _string_tags(request_tags: object) -> list[str] | None:
|
||||
|
|
@ -89,6 +101,20 @@ def _team_table(prisma_client: PrismaClient) -> TableActions[prisma_models.LiteL
|
|||
return TeamRepository(prisma_client).table
|
||||
|
||||
|
||||
def _rotated_token_table(
|
||||
prisma_client: PrismaClient,
|
||||
) -> TableActions[prisma_models.LiteLLM_DeprecatedVerificationToken]:
|
||||
return DeprecatedVerificationTokenRepository(prisma_client).table
|
||||
|
||||
|
||||
def _deleted_token_table(prisma_client: PrismaClient) -> TableActions[prisma_models.LiteLLM_DeletedVerificationToken]:
|
||||
return DeletedVerificationTokenRepository(prisma_client).table
|
||||
|
||||
|
||||
def _deleted_team_table(prisma_client: PrismaClient) -> TableActions[prisma_models.LiteLLM_DeletedTeamTable]:
|
||||
return TeamRepository(prisma_client).deleted_table
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _CreatorRows:
|
||||
key: prisma_models.LiteLLM_VerificationToken | None
|
||||
|
|
@ -96,6 +122,15 @@ class _CreatorRows:
|
|||
user: prisma_models.LiteLLM_UserTable | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ResponseCreator:
|
||||
metadata: Mapping[str, object]
|
||||
key_metadata: Mapping[str, object]
|
||||
team_id: str | None
|
||||
team_metadata: Mapping[str, object]
|
||||
settings_read: bool = True
|
||||
|
||||
|
||||
class ManagedObjectCreatorAttribution:
|
||||
"""Rebuilds the spend-tracking metadata of whoever created a managed object, so the poll
|
||||
job's spend row is attributed like the creating request: key hash, team, organization,
|
||||
|
|
@ -107,8 +142,15 @@ class ManagedObjectCreatorAttribution:
|
|||
user's alias, because a resolvable name is more useful on the spend row than a null.
|
||||
user_api_key_org_id is resolved too: the spend update writer reads it off this metadata
|
||||
to increment organization spend, so leaving it out drops the cost from org accounting.
|
||||
A background response also carries its end user and the key, team, and user budgets, the
|
||||
way the read that billed it before did, and never names a key-scoped creator as its user.
|
||||
A background response also carries its end user, the creating key's project and its alias,
|
||||
and the key, team, and user budgets and per-model budgets, the way the read that billed it
|
||||
before did, and never names a key-scoped creator as its user. A key that sets no budget of
|
||||
its own takes its budget tier's, the same rule auth applies. Its creator's key and team
|
||||
metadata come along too, so the billed event reaches the key's and team's logging callbacks
|
||||
like that read did. A rotated key's metadata is read from the live key its rotations lead
|
||||
to, and a deleted key's or team's from its deleted row. A lookup that fails leaves the
|
||||
creator's settings unread and the attribution to the rows that still load, so the
|
||||
response still bills and still counts toward their budgets.
|
||||
"""
|
||||
|
||||
def __init__(self, prisma_client: PrismaClient, poller_name: str) -> None:
|
||||
|
|
@ -119,17 +161,70 @@ class ManagedObjectCreatorAttribution:
|
|||
rows: Final = await self._rows(job, object_id)
|
||||
return {"user_api_key_user_id": job.created_by, **_shared_metadata(job, rows)}
|
||||
|
||||
async def response_metadata(self, job: ManagedObjectRow, object_id: str) -> Mapping[str, object]:
|
||||
rows: Final = await self._rows(job, object_id)
|
||||
end_user_id: Final = getattr(job, "end_user_id", None)
|
||||
return {
|
||||
"user_api_key_user_id": _creator_user_id(job.created_by),
|
||||
**_shared_metadata(job, rows),
|
||||
**({"user_api_key_end_user_id": end_user_id} if end_user_id else {}),
|
||||
**_budget_fields(rows.key, "user_api_key_spend", "user_api_key_max_budget"),
|
||||
**_budget_fields(rows.team, "user_api_key_team_spend", "user_api_key_team_max_budget"),
|
||||
**_budget_fields(rows.user, "user_api_key_user_spend", "user_api_key_user_max_budget"),
|
||||
}
|
||||
async def response_creator(self, job: ManagedObjectRow, object_id: str) -> ResponseCreator:
|
||||
try:
|
||||
return await self._read_response_creator(job, object_id)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(
|
||||
f"{self._poller_name}: could not read the creator of {object_id}, so it bills without their "
|
||||
f"settings: {e}"
|
||||
)
|
||||
return ResponseCreator(
|
||||
metadata=_response_creator_metadata(job, await self._rows(job, object_id)),
|
||||
key_metadata={},
|
||||
team_id=getattr(job, "team_id", None),
|
||||
team_metadata={},
|
||||
settings_read=False,
|
||||
)
|
||||
|
||||
async def _read_response_creator(self, job: ManagedObjectRow, object_id: str) -> ResponseCreator:
|
||||
api_key: Final = getattr(job, "api_key", None)
|
||||
team_id: Final = getattr(job, "team_id", None)
|
||||
rows: Final = _CreatorRows(
|
||||
key=await self._live_key_row(api_key, job.created_by),
|
||||
team=await self._live_team_row(team_id),
|
||||
user=await self._user_row(object_id, _creator_user_id(job.created_by)),
|
||||
)
|
||||
return ResponseCreator(
|
||||
metadata=_response_creator_metadata(job, rows),
|
||||
key_metadata=row_metadata(rows.key) if rows.key is not None else await self._retired_key_metadata(api_key),
|
||||
team_id=team_id,
|
||||
team_metadata=(
|
||||
row_metadata(rows.team) if rows.team is not None else await self._deleted_team_metadata(team_id)
|
||||
),
|
||||
)
|
||||
|
||||
async def _retired_key_metadata(self, api_key: str | None) -> Mapping[str, object]:
|
||||
if not _names_a_key_row(api_key):
|
||||
return {}
|
||||
current_token: Final = await self._token_rotated_to(api_key, _MAX_ROTATION_HOPS)
|
||||
current_key: Final = (
|
||||
await _token_table(self._prisma_client).find_unique(where={"token": current_token})
|
||||
if current_token != api_key
|
||||
else None
|
||||
)
|
||||
if current_key is not None:
|
||||
return row_metadata(current_key)
|
||||
return row_metadata(
|
||||
await _deleted_token_table(self._prisma_client).find_first(
|
||||
where={"token": current_token}, order={"deleted_at": "desc"}
|
||||
)
|
||||
)
|
||||
|
||||
async def _token_rotated_to(self, token: str, hops_left: int) -> str:
|
||||
rotation: Final = await _rotated_token_table(self._prisma_client).find_unique(where={"token": token})
|
||||
if rotation is None or hops_left == 0:
|
||||
return token
|
||||
return await self._token_rotated_to(rotation.active_token_id, hops_left - 1)
|
||||
|
||||
async def _deleted_team_metadata(self, team_id: str | None) -> Mapping[str, object]:
|
||||
if not team_id:
|
||||
return {}
|
||||
return row_metadata(
|
||||
await _deleted_team_table(self._prisma_client).find_first(
|
||||
where={"team_id": team_id}, order={"deleted_at": "desc"}
|
||||
)
|
||||
)
|
||||
|
||||
async def _rows(self, job: ManagedObjectRow, object_id: str) -> _CreatorRows:
|
||||
return _CreatorRows(
|
||||
|
|
@ -152,23 +247,57 @@ class ManagedObjectCreatorAttribution:
|
|||
async def _key_row(
|
||||
self, object_id: str, api_key: str | None, created_by: str | None
|
||||
) -> prisma_models.LiteLLM_VerificationToken | None:
|
||||
if not api_key or _is_cli_session_key(api_key, created_by):
|
||||
return None
|
||||
try:
|
||||
return await _token_table(self._prisma_client).find_unique(where={"token": api_key})
|
||||
return await self._live_key_row(api_key, created_by)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(f"{self._poller_name}: could not look up the creating key for {object_id}: {e}")
|
||||
return None
|
||||
|
||||
async def _team_row(self, object_id: str, team_id: str | None) -> prisma_models.LiteLLM_TeamTable | None:
|
||||
if not team_id:
|
||||
async def _live_key_row(
|
||||
self, api_key: str | None, created_by: str | None
|
||||
) -> prisma_models.LiteLLM_VerificationToken | None:
|
||||
if not api_key or _is_cli_session_key(api_key, created_by):
|
||||
return None
|
||||
return await _token_table(self._prisma_client).find_unique(
|
||||
where={"token": api_key}, include={"litellm_budget_table": True, "litellm_project_table": True}
|
||||
)
|
||||
|
||||
async def _team_row(self, object_id: str, team_id: str | None) -> prisma_models.LiteLLM_TeamTable | None:
|
||||
try:
|
||||
return await _team_table(self._prisma_client).find_unique(where={"team_id": team_id})
|
||||
return await self._live_team_row(team_id)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(f"{self._poller_name}: could not look up team {team_id} for {object_id}: {e}")
|
||||
return None
|
||||
|
||||
async def _live_team_row(self, team_id: str | None) -> prisma_models.LiteLLM_TeamTable | None:
|
||||
if not team_id:
|
||||
return None
|
||||
return await _team_table(self._prisma_client).find_unique(where={"team_id": team_id})
|
||||
|
||||
|
||||
def _response_creator_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str, object]:
|
||||
end_user_id: Final = getattr(job, "end_user_id", None)
|
||||
return {
|
||||
"user_api_key_user_id": _creator_user_id(job.created_by),
|
||||
**_shared_metadata(job, rows),
|
||||
**({"user_api_key_end_user_id": end_user_id} if end_user_id else {}),
|
||||
"user_api_key_project_id": getattr(rows.key, "project_id", None),
|
||||
"user_api_key_project_alias": getattr(getattr(rows.key, "litellm_project_table", None), "project_alias", None),
|
||||
**_key_budget_fields(rows.key),
|
||||
**_budget_fields(
|
||||
rows.team,
|
||||
"user_api_key_team_spend",
|
||||
"user_api_key_team_max_budget",
|
||||
"user_api_key_team_model_max_budget",
|
||||
),
|
||||
**_budget_fields(
|
||||
rows.user,
|
||||
"user_api_key_user_spend",
|
||||
"user_api_key_user_max_budget",
|
||||
"user_api_key_user_model_max_budget",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _shared_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str, object]:
|
||||
api_key: Final = getattr(job, "api_key", None)
|
||||
|
|
@ -196,6 +325,19 @@ def _shared_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str,
|
|||
}
|
||||
|
||||
|
||||
def row_metadata(row: object) -> Mapping[str, object]:
|
||||
metadata: Final = getattr(row, "metadata", None)
|
||||
if not isinstance(metadata, dict):
|
||||
return {}
|
||||
return cast(Mapping[str, object], metadata) # cast-ok: prisma Json object column
|
||||
|
||||
|
||||
def _names_a_key_row(api_key: object) -> TypeGuard[str]:
|
||||
from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash
|
||||
|
||||
return isinstance(api_key, str) and is_valid_sha256_hash(api_key)
|
||||
|
||||
|
||||
def _is_cli_session_key(api_key: str, created_by: str | None) -> bool:
|
||||
return bool(created_by) and api_key == f"{CLI_SESSION_KEY_PREFIX}-{created_by}"
|
||||
|
||||
|
|
@ -212,7 +354,28 @@ def _organization_id(row: object) -> str | None:
|
|||
return cast("str | None", getattr(row, "organization_id", None)) if row is not None else None
|
||||
|
||||
|
||||
def _budget_fields(row: object, spend_key: str, max_budget_key: str) -> Mapping[str, object]:
|
||||
def _budget_fields(row: object, spend_key: str, max_budget_key: str, model_max_budget_key: str) -> Mapping[str, object]:
|
||||
if row is None:
|
||||
return {}
|
||||
return {spend_key: getattr(row, "spend", None), max_budget_key: getattr(row, "max_budget", None)}
|
||||
return {
|
||||
spend_key: getattr(row, "spend", None),
|
||||
max_budget_key: getattr(row, "max_budget", None),
|
||||
model_max_budget_key: getattr(row, "model_max_budget", None),
|
||||
}
|
||||
|
||||
|
||||
def _key_budget_fields(key: object) -> Mapping[str, object]:
|
||||
if key is None:
|
||||
return {}
|
||||
tier: Final = getattr(key, "litellm_budget_table", None)
|
||||
return {
|
||||
"user_api_key_spend": getattr(key, "spend", None),
|
||||
"user_api_key_max_budget": _own_or_tier_budget(key, tier, "max_budget"),
|
||||
"user_api_key_model_max_budget": _own_or_tier_budget(key, tier, "model_max_budget"),
|
||||
}
|
||||
|
||||
|
||||
def _own_or_tier_budget(key: object, tier: object, field: str) -> object:
|
||||
own: Final = getattr(key, field, None)
|
||||
tier_value: Final = getattr(tier, field, None)
|
||||
return own if tier_value is None or own not in (None, {}) else tier_value
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.core_helpers import require_attributes
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
|
||||
from litellm.litellm_core_utils.internal_call_metadata import requests_background
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
|
|
@ -133,7 +134,7 @@ if TYPE_CHECKING:
|
|||
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
|
||||
from litellm.proxy.utils import PrismaClient as _PrismaClient
|
||||
|
||||
Span = Union[_Span, Any]
|
||||
Span = _Span
|
||||
InternalUsageCache = _InternalUsageCache
|
||||
PrismaClient = _PrismaClient
|
||||
else:
|
||||
|
|
@ -490,6 +491,14 @@ _PROVIDER_FILE_ID_KEYS: Final = tuple(dict.fromkeys(chain.from_iterable(_PROVIDE
|
|||
_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS: Final = (0.5, 1.0, 2.0)
|
||||
|
||||
|
||||
class _SupportsKeyedGet(Protocol):
|
||||
def get(self, key: str, default: object, /) -> object: ...
|
||||
|
||||
|
||||
def _keyed_get(source: object, key: str, default: object = None) -> object:
|
||||
return require_attributes(source, _SupportsKeyedGet, "get").get(key, default)
|
||||
|
||||
|
||||
class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
||||
# Class variables or attributes
|
||||
def __init__(
|
||||
|
|
@ -1611,7 +1620,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
return holders if holders or "input_file_id" in request else healthy_deployments
|
||||
|
||||
async def async_pre_call_deployment_hook(
|
||||
self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
|
||||
self, kwargs: dict[str, object], call_type: Optional[CallTypes]
|
||||
) -> Optional[dict]:
|
||||
"""
|
||||
Allow modifying the request just before it's sent to the deployment.
|
||||
|
|
@ -1623,11 +1632,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping"))
|
||||
# model_info may be at top-level or nested under litellm_metadata
|
||||
# (batch/file operations use litellm_metadata)
|
||||
model_id = cast(Optional[str], kwargs.get("model_info", {}).get("id", None))
|
||||
model_id = cast(Optional[str], _keyed_get(kwargs.get("model_info", {}), "id"))
|
||||
if model_id is None:
|
||||
model_id = cast(
|
||||
Optional[str],
|
||||
kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None),
|
||||
_keyed_get(_keyed_get(kwargs.get("litellm_metadata", {}), "model_info", {}), "id"),
|
||||
)
|
||||
for accessor_key in accessor_keys:
|
||||
input_file_id = cast(Optional[str], kwargs.get(accessor_key))
|
||||
|
|
@ -1824,7 +1833,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
),
|
||||
)
|
||||
|
||||
async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span) -> dict:
|
||||
async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span | None) -> dict:
|
||||
"""
|
||||
Get model-specific file IDs for a list of proxy file IDs.
|
||||
Returns a dictionary mapping litellm_proxy/ file_id -> model_id -> model_file_id
|
||||
|
|
@ -1866,7 +1875,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
llm_router: Optional[Router],
|
||||
_create_file_request: CreateFileRequest,
|
||||
target_model_names_list: List[str],
|
||||
litellm_parent_otel_span: Span,
|
||||
litellm_parent_otel_span: Span | None,
|
||||
) -> List[OpenAIFileObject]:
|
||||
if llm_router is None:
|
||||
raise Exception("LLM Router not initialized. Ensure models added to proxy.")
|
||||
|
|
@ -1882,7 +1891,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
create_file_request: CreateFileRequest,
|
||||
llm_router: Router,
|
||||
target_model_names_list: List[str],
|
||||
litellm_parent_otel_span: Span,
|
||||
litellm_parent_otel_span: Span | None,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
) -> OpenAIFileObject:
|
||||
responses = await self.create_file_for_each_model(
|
||||
|
|
@ -1949,7 +1958,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
file_objects: List[OpenAIFileObject],
|
||||
create_file_request: CreateFileRequest,
|
||||
internal_usage_cache: InternalUsageCache,
|
||||
litellm_parent_otel_span: Span,
|
||||
litellm_parent_otel_span: Span | None,
|
||||
target_model_names_list: List[str],
|
||||
) -> OpenAIFileObject:
|
||||
## GET THE FILE TYPE FROM THE CREATE FILE REQUEST
|
||||
|
|
@ -2652,7 +2661,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
# Continue with other files even if one fails
|
||||
continue
|
||||
|
||||
def _get_content_type_from_file_object(self, file_object: Optional[Any]) -> str:
|
||||
def _get_content_type_from_file_object(self, file_object: object) -> str:
|
||||
"""
|
||||
Determine content type from file object.
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ if TYPE_CHECKING:
|
|||
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
|
||||
from litellm.proxy.utils import PrismaClient as _PrismaClient
|
||||
|
||||
Span = Union[_Span, Any]
|
||||
Span = _Span
|
||||
InternalUsageCache = _InternalUsageCache
|
||||
PrismaClient = _PrismaClient
|
||||
else:
|
||||
|
|
@ -107,7 +107,7 @@ class _PROXY_LiteLLMManagedVectorStores(
|
|||
llm_router: Router,
|
||||
model: str,
|
||||
request_data: dict[str, object] | VectorStoreCreateOptionalRequestParams,
|
||||
litellm_parent_otel_span: Span,
|
||||
litellm_parent_otel_span: Span | None,
|
||||
) -> VectorStoreCreateResponse:
|
||||
"""
|
||||
Create a vector store for a specific model.
|
||||
|
|
@ -134,7 +134,7 @@ class _PROXY_LiteLLMManagedVectorStores(
|
|||
create_request: VectorStoreCreateOptionalRequestParams,
|
||||
llm_router: Router,
|
||||
target_model_names_list: List[str],
|
||||
litellm_parent_otel_span: Span,
|
||||
litellm_parent_otel_span: Span | None,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
) -> VectorStoreCreateResponse:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ from litellm.repositories.budget_repository import BudgetRepository
|
|||
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.repositories.project_repository import ProjectRepository
|
||||
from litellm.repositories.table_repositories import PrismaTableRepository
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
from litellm.repositories.verification_token_repository import VerificationTokenRepository
|
||||
|
|
@ -73,6 +74,16 @@ def _verification_token_table(
|
|||
return VerificationTokenRepository(prisma_client).table
|
||||
|
||||
|
||||
class _WriterVerificationTokenRepository(PrismaTableRepository["prisma_models.LiteLLM_VerificationToken"]):
|
||||
table_name = "litellm_verificationtoken"
|
||||
|
||||
|
||||
def _writer_verification_token_table(
|
||||
prisma_client: PrismaClient,
|
||||
) -> TableActions["prisma_models.LiteLLM_VerificationToken"]:
|
||||
return _WriterVerificationTokenRepository(prisma_client, use_writer=True).table
|
||||
|
||||
|
||||
def _budget_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_BudgetTable"]:
|
||||
return BudgetRepository(prisma_client).table
|
||||
|
||||
|
|
@ -849,7 +860,7 @@ async def update_project(
|
|||
else None
|
||||
)
|
||||
if current_project is not None and data.team_id != current_project.team_id:
|
||||
mismatched_key_count: Final = await prisma_client.writer_db.litellm_verificationtoken.count(
|
||||
mismatched_key_count: Final = await _writer_verification_token_table(prisma_client).count(
|
||||
where={
|
||||
"project_id": data.project_id,
|
||||
"OR": [{"team_id": {"not": data.team_id}}, {"team_id": None}],
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from collections.abc import Mapping
|
|||
from typing import TYPE_CHECKING, Final, List, Optional, Protocol
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -37,6 +38,7 @@ if TYPE_CHECKING:
|
|||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
router = APIRouter()
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class ManagedVectorStoreRow(Protocol):
|
||||
|
|
@ -348,11 +350,11 @@ async def update_vector_store(
|
|||
raise HTTPException(status_code=500, detail="Database not connected")
|
||||
|
||||
try:
|
||||
update_data = data.model_dump(exclude_unset=True)
|
||||
vector_store_id: Final[str] = update_data.pop("vector_store_id")
|
||||
update_data = data.model_dump(exclude_unset=True, exclude={"vector_store_id"})
|
||||
vector_store_id: Final = data.vector_store_id
|
||||
if update_data.get("vector_store_metadata") is not None:
|
||||
update_data["vector_store_metadata"] = safe_dumps(
|
||||
update_data["vector_store_metadata"]
|
||||
_DECODED_JSON.validate_python(update_data["vector_store_metadata"])
|
||||
)
|
||||
|
||||
updated = await managed_vector_store_table(prisma_client).update(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.78"
|
||||
version = "0.1.79"
|
||||
description = "Package for LiteLLM Enterprise features"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.1.78"
|
||||
version = "0.1.79"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-enterprise==",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
|
||||
ARG PGBOUNCER_VERSION=1.26.0
|
||||
|
|
@ -59,7 +59,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
|
|||
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
|
||||
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
|
||||
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
@ -125,7 +125,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
|||
USER root
|
||||
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic libevent && break; \
|
||||
apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic libevent && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
@ -140,7 +140,12 @@ ENV HOME=/home/nonroot \
|
|||
PYTHONUNBUFFERED=1 \
|
||||
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
|
||||
|
||||
COPY --from=builder --chown=nonroot:nonroot /app /app
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
|
||||
COPY --from=builder /opt/prisma /opt/prisma
|
||||
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""Path allowlist for the gateway component.
|
||||
|
||||
The gateway exposes the LLM data-plane surface: chat/completions, embeddings,
|
||||
audio, batches, files, fine-tuning, rerank, decisions, ocr, rag, video, search, image,
|
||||
audio, batches, files, fine-tuning, rerank, decisions, extraction, ocr, rag, video, search, image,
|
||||
responses, vector stores, passthrough providers, realtime websockets, MCP
|
||||
tool-call endpoints, and operational endpoints (/health, /metrics, and the
|
||||
/debug/memory/summary read of the serving worker's RSS).
|
||||
|
|
@ -64,6 +64,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
|
|||
"/decisions",
|
||||
"/v1/systemone",
|
||||
"/systemone",
|
||||
"/v1/extraction",
|
||||
"/extraction",
|
||||
"/v1/ocr",
|
||||
"/ocr",
|
||||
"/v1/rag/",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.110"
|
||||
version = "0.4.111"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.110"
|
||||
version = "0.4.111"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-proxy-extras==",
|
||||
|
|
|
|||
8
litellm-rust/Cargo.lock
generated
8
litellm-rust/Cargo.lock
generated
|
|
@ -8202,18 +8202,18 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "webpki-root-certs"
|
||||
version = "1.0.9"
|
||||
version = "1.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
|
||||
checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "1.0.9"
|
||||
version = "1.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a"
|
||||
checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -124,7 +124,7 @@ moka = { version = "0.12.16", features = ["future"] }
|
|||
strum = { version = "0.28.0", features = ["derive"] }
|
||||
url = "2.5.8"
|
||||
percent-encoding = "2.3"
|
||||
webpki-roots = "1"
|
||||
webpki-roots = "1.0.10"
|
||||
time = { version = "0.3.53", features = ["parsing"] }
|
||||
criterion = "0.8.2"
|
||||
fancy-regex = "0.19.2"
|
||||
|
|
|
|||
|
|
@ -72,26 +72,26 @@ fn duration(seconds: f64) -> Result<Duration, Error> {
|
|||
#[derive(Clone, Default)]
|
||||
pub(crate) struct CacheHeaders(std::sync::Arc<std::sync::OnceLock<String>>);
|
||||
|
||||
impl litellm_host::interceptors::Interceptors<litellm_inference::RouteError> for CacheHeaders {
|
||||
impl litellm_host::interceptors::Interceptors<litellm_inference::Error> for CacheHeaders {
|
||||
async fn before_provider_request(
|
||||
&self,
|
||||
wire: litellm_host::interceptors::WireRequest,
|
||||
_: litellm_host::interceptors::RequestContext,
|
||||
) -> Result<litellm_host::interceptors::WireRequest, litellm_inference::RouteError> {
|
||||
) -> Result<litellm_host::interceptors::WireRequest, litellm_inference::Error> {
|
||||
Ok(wire)
|
||||
}
|
||||
|
||||
async fn after_provider_response(
|
||||
&self,
|
||||
_: litellm_host::interceptors::RawResponse,
|
||||
) -> Result<(), litellm_inference::RouteError> {
|
||||
) -> Result<(), litellm_inference::Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn result_ready(
|
||||
&self,
|
||||
facts: litellm_host::interceptors::ExecutionFacts,
|
||||
) -> Result<(), litellm_inference::RouteError> {
|
||||
) -> Result<(), litellm_inference::Error> {
|
||||
if let litellm_host::interceptors::ResultSource::Cache { key } = facts.source {
|
||||
let _ = self.0.set(key);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ use axum::{
|
|||
response::{IntoResponse, Response},
|
||||
};
|
||||
use litellm_http::transport::Error as TransportError;
|
||||
use litellm_inference::RouteError;
|
||||
use litellm_inference::Error as InferenceError;
|
||||
use litellm_llms::base_llm::ocr::error::Error as OcrError;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
|
|
@ -19,7 +19,7 @@ pub enum Error {
|
|||
)]
|
||||
UnknownModel(String),
|
||||
#[error(transparent)]
|
||||
Route(#[from] RouteError),
|
||||
Route(#[from] InferenceError),
|
||||
#[error(transparent)]
|
||||
Ocr(#[from] OcrError),
|
||||
#[error("{0} is not implemented by the Rust gateway")]
|
||||
|
|
@ -36,8 +36,8 @@ impl IntoResponse for Error {
|
|||
}
|
||||
}
|
||||
|
||||
impl From<litellm_host_http::Error<RouteError>> for Error {
|
||||
fn from(error: litellm_host_http::Error<RouteError>) -> Self {
|
||||
impl From<litellm_host_http::Error<InferenceError>> for Error {
|
||||
fn from(error: litellm_host_http::Error<InferenceError>) -> Self {
|
||||
match error {
|
||||
litellm_host_http::Error::Call(error) => Self::Route(error),
|
||||
litellm_host_http::Error::Protocol => Self::Internal(error.to_string()),
|
||||
|
|
@ -50,7 +50,7 @@ impl Error {
|
|||
match self {
|
||||
Self::Auth(error) => error.status(),
|
||||
Self::Unsupported(_)
|
||||
| Self::Route(RouteError::Unsupported(_))
|
||||
| Self::Route(InferenceError::Unsupported(_))
|
||||
| Self::Ocr(OcrError::Unsupported(_)) => StatusCode::NOT_IMPLEMENTED,
|
||||
Self::BodyTooLarge => StatusCode::PAYLOAD_TOO_LARGE,
|
||||
Self::Ocr(
|
||||
|
|
@ -64,10 +64,10 @@ impl Error {
|
|||
.and_then(|status| StatusCode::from_u16(status).ok())
|
||||
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
|
||||
Self::InvalidBody(_) | Self::UnknownModel(_) => StatusCode::BAD_REQUEST,
|
||||
Self::Route(RouteError::Transport(TransportError::Http { status, .. })) => {
|
||||
Self::Route(InferenceError::Transport(TransportError::Http { status, .. })) => {
|
||||
StatusCode::from_u16(*status).unwrap_or(StatusCode::BAD_GATEWAY)
|
||||
}
|
||||
Self::Route(RouteError::Auth(litellm_auth::Error::MissingApiKey { .. })) => {
|
||||
Self::Route(InferenceError::Auth(litellm_auth::Error::MissingApiKey { .. })) => {
|
||||
StatusCode::UNAUTHORIZED
|
||||
}
|
||||
Self::Route(error) if error.is_request() => StatusCode::BAD_REQUEST,
|
||||
|
|
@ -97,7 +97,9 @@ impl Error {
|
|||
/// body already in that shape passes through, any other has its message extracted.
|
||||
pub fn body(&self, request_id: Option<&str>) -> Value {
|
||||
let raw = match self {
|
||||
Self::Route(RouteError::Transport(TransportError::Http { body, .. })) => body.clone(),
|
||||
Self::Route(InferenceError::Transport(TransportError::Http { body, .. })) => {
|
||||
body.clone()
|
||||
}
|
||||
other => other.to_string(),
|
||||
};
|
||||
let parsed = serde_json::from_str::<Value>(&raw).ok();
|
||||
|
|
@ -175,7 +177,7 @@ mod tests {
|
|||
use super::*;
|
||||
|
||||
fn upstream(status: u16, body: &str) -> Error {
|
||||
Error::Route(RouteError::Transport(TransportError::Http {
|
||||
Error::Route(InferenceError::Transport(TransportError::Http {
|
||||
status,
|
||||
body: body.into(),
|
||||
}))
|
||||
|
|
@ -225,16 +227,16 @@ mod tests {
|
|||
|
||||
#[rstest]
|
||||
#[case::upstream_status(upstream(429, ""), StatusCode::TOO_MANY_REQUESTS)]
|
||||
#[case::rejected_request(Error::Route(RouteError::InvalidRequest("top_k".into())), StatusCode::BAD_REQUEST)]
|
||||
#[case::rejected_request(Error::Route(InferenceError::InvalidRequest("top_k".into())), StatusCode::BAD_REQUEST)]
|
||||
#[case::missing_key(
|
||||
Error::Route(RouteError::Auth(litellm_auth::Error::MissingApiKey {
|
||||
Error::Route(InferenceError::Auth(litellm_auth::Error::MissingApiKey {
|
||||
provider: "Anthropic",
|
||||
environment_variable: "ANTHROPIC_API_KEY",
|
||||
})),
|
||||
StatusCode::UNAUTHORIZED,
|
||||
)]
|
||||
#[case::lost_connection(
|
||||
Error::Route(RouteError::Transport(TransportError::Network("reset".into()))),
|
||||
Error::Route(InferenceError::Transport(TransportError::Network("reset".into()))),
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
)]
|
||||
fn status_follows_who_is_at_fault(#[case] error: Error, #[case] status: StatusCode) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use litellm_host::observation::ObservationSender;
|
||||
pub mod route;
|
||||
pub mod types;
|
||||
pub use litellm_inference::RouteError as Error;
|
||||
pub use litellm_inference::Error;
|
||||
mod common_utils;
|
||||
pub mod constants;
|
||||
pub(crate) mod handler;
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ use litellm_host::{
|
|||
lifecycle::{CallEvent, ExecutionEvent},
|
||||
observation::observation_channel,
|
||||
};
|
||||
use litellm_inference::RouteError;
|
||||
use litellm_inference::Error;
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
|
|
@ -166,12 +166,12 @@ struct ChangingHooks {
|
|||
facts: std::sync::Mutex<Vec<ExecutionFacts>>,
|
||||
}
|
||||
|
||||
impl Interceptors<RouteError> for ChangingHooks {
|
||||
impl Interceptors<Error> for ChangingHooks {
|
||||
async fn before_provider_request(
|
||||
&self,
|
||||
mut wire: WireRequest,
|
||||
_: RequestContext,
|
||||
) -> Result<WireRequest, RouteError> {
|
||||
) -> Result<WireRequest, Error> {
|
||||
let call = self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
if self.rewrite {
|
||||
wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 });
|
||||
|
|
@ -179,11 +179,11 @@ impl Interceptors<RouteError> for ChangingHooks {
|
|||
Ok(wire)
|
||||
}
|
||||
|
||||
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
|
||||
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
|
||||
self.facts.lock().unwrap().push(facts);
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ use std::sync::Arc;
|
|||
|
||||
use crate::common_utils::messages_provider;
|
||||
|
||||
pub use litellm_inference::RouteError as Error;
|
||||
pub use litellm_inference::Error;
|
||||
pub use types::{
|
||||
MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params,
|
||||
messages_body,
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ use litellm_host::interceptors::{
|
|||
ResultSource, WireRequest,
|
||||
};
|
||||
use litellm_inference::{
|
||||
RouteError,
|
||||
Error,
|
||||
caching::{CacheRequest, execute_unary},
|
||||
};
|
||||
use rstest::{fixture, rstest};
|
||||
|
|
@ -133,12 +133,12 @@ struct ChangingHooks {
|
|||
facts: std::sync::Mutex<Vec<ExecutionFacts>>,
|
||||
}
|
||||
|
||||
impl Interceptors<RouteError> for ChangingHooks {
|
||||
impl Interceptors<Error> for ChangingHooks {
|
||||
async fn before_provider_request(
|
||||
&self,
|
||||
mut wire: WireRequest,
|
||||
_: RequestContext,
|
||||
) -> Result<WireRequest, RouteError> {
|
||||
) -> Result<WireRequest, Error> {
|
||||
let call = self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
if self.rewrite {
|
||||
wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 });
|
||||
|
|
@ -146,11 +146,11 @@ impl Interceptors<RouteError> for ChangingHooks {
|
|||
Ok(wire)
|
||||
}
|
||||
|
||||
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
|
||||
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
|
||||
self.facts.lock().unwrap().push(facts);
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,4 +3,4 @@
|
|||
- The route returns its completed response directly
|
||||
- Route closures supply OCR-specific host capabilities, such as the caller's Azure AD token provider (`src/route.rs`)
|
||||
- Provider code reaches the caller's hooks mid-call only through `litellm_llms::base_llm::ocr::handler::CallHooks`, which this crate implements over its host until it folds into `litellm_host::interceptors::Interceptors`
|
||||
- OCR failures use `base_llm/ocr/error.rs`, the recorded exception to `litellm_inference::RouteError` until OCR folds into `litellm_llms::Error`
|
||||
- OCR failures use `base_llm/ocr/error.rs`, the recorded exception to `litellm_inference::Error` until OCR folds into `litellm_llms::Error`
|
||||
|
|
|
|||
|
|
@ -249,7 +249,7 @@ pub(crate) fn resolve_provider_config<'a>(
|
|||
provider: LlmProviders::Mistral,
|
||||
}
|
||||
}
|
||||
Err(litellm_inference::RouteError::InvalidProvider(provider)) => {
|
||||
Err(litellm_inference::Error::InvalidProvider(provider)) => {
|
||||
return Err(Error::InvalidProvider(provider));
|
||||
}
|
||||
Err(error) => return Err(Error::InvalidProvider(error.to_string())),
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use litellm_host::observation::ObservationSender;
|
||||
pub use litellm_inference::RouteError as Error;
|
||||
pub use litellm_inference::Error;
|
||||
|
||||
pub mod ids;
|
||||
pub mod route;
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ use litellm_host::interceptors::{
|
|||
WireRequest,
|
||||
};
|
||||
use litellm_inference::{
|
||||
RouteError,
|
||||
Error,
|
||||
caching::{CacheRequest, execute_unary},
|
||||
};
|
||||
use rstest::{fixture, rstest};
|
||||
|
|
@ -124,12 +124,12 @@ struct ChangingHooks {
|
|||
facts: std::sync::Mutex<Vec<ExecutionFacts>>,
|
||||
}
|
||||
|
||||
impl Interceptors<RouteError> for ChangingHooks {
|
||||
impl Interceptors<Error> for ChangingHooks {
|
||||
async fn before_provider_request(
|
||||
&self,
|
||||
mut wire: WireRequest,
|
||||
_: RequestContext,
|
||||
) -> Result<WireRequest, RouteError> {
|
||||
) -> Result<WireRequest, Error> {
|
||||
let call = self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
if self.rewrite {
|
||||
wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 });
|
||||
|
|
@ -137,11 +137,11 @@ impl Interceptors<RouteError> for ChangingHooks {
|
|||
Ok(wire)
|
||||
}
|
||||
|
||||
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
|
||||
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
|
||||
self.facts.lock().unwrap().push(facts);
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
pub mod types;
|
||||
pub use litellm_inference::RouteError as Error;
|
||||
pub use litellm_inference::Error;
|
||||
mod constants;
|
||||
mod handler;
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
- `gateway-inference` (serving) -> `inference-<fmt>` (one API's call) -> `inference` (shared base)
|
||||
- This crate is the shared base every format crate builds on
|
||||
- `RouteError` (`src/error.rs`), `CallOptions` (`src/lib.rs`), `CallContext` (`src/context.rs`)
|
||||
- `Error` (`src/error.rs`), `CallOptions` (`src/lib.rs`), `CallContext` (`src/context.rs`)
|
||||
- diagnostic spans (`src/diagnostic.rs`), outbound send and signing (`src/outbound.rs`), provider resolution (`src/provider.rs`), `CoreResources` (`src/resources.rs`)
|
||||
- response caching (`src/caching.rs`): `Cachable`, `StreamCachable`, `CacheRequest`, `CallCache`, `execute_unary`, `execute_streaming`, stream capture
|
||||
- Shared test helpers belong in litellm-inference-testing, used only as a dev dependency
|
||||
|
|
@ -49,14 +49,14 @@
|
|||
- `litellm-inference-<fmt>` mirrors the route packages (`litellm/ocr/`, `litellm/messages/`, ...)
|
||||
- Provider code never imports from `inference` or `inference-*`
|
||||
- Handlers belong in `inference-*` or `llms`, never in a host crate
|
||||
- Import every item from its canonical path. Never re-export another crate's items or give an item a second public path. The only allowed re-exports are a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`) and each format crate's `pub use litellm_inference::RouteError as Error;`, which keeps the pre-split `<fmt>::Error` name
|
||||
- Import every item from its canonical path. Never re-export another crate's items or give an item a second public path. The only allowed re-exports are a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`) and each format crate's `pub use litellm_inference::Error;`, which keeps the pre-split `<fmt>::Error` name
|
||||
|
||||
## Error placement
|
||||
|
||||
- The workspace `Error definitions` rules shape each crate's error. This section decides which crate and module a failure belongs to
|
||||
- A failure is declared once, by the lowest crate that raises it
|
||||
- every crate above nests it unchanged (`#[error(transparent)] Auth(#[from] litellm_auth::Error)`) or maps it once at its boundary, as `src/error.rs` does for `litellm_llms::Error`
|
||||
- `RouteError` collects route failures for every format crate and never re-declares a variant a lower crate raises
|
||||
- `Error` collects route failures for every format crate and never re-declares a variant a lower crate raises
|
||||
- Scope follows the concept, not the first caller
|
||||
- an error type under `litellm-llms`'s `<provider>/` is private to that provider: no other provider and nothing in `base_llm` may import it
|
||||
- a failure two providers or two routes can hit (wire framing, stream event decoding, a malformed provider response) belongs to the crate that owns the concept: `litellm-framer` for framing, `litellm_llms::Error` for the transformation layer
|
||||
|
|
|
|||
|
|
@ -24,9 +24,9 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned};
|
|||
use serde_json::Value;
|
||||
use tokio_util::codec::Decoder;
|
||||
|
||||
use crate::RouteError;
|
||||
use crate::Error;
|
||||
|
||||
pub trait Cachable: Protocol<Error = RouteError> {
|
||||
pub trait Cachable: Protocol<Error = Error> {
|
||||
const SURFACE: &'static str;
|
||||
|
||||
fn reusable(_response: &Self::Response) -> bool {
|
||||
|
|
@ -145,15 +145,15 @@ pub async fn execute_unary<P, F, Fut>(
|
|||
request: CacheRequest,
|
||||
cache: Option<Arc<dyn ResponseCacheService>>,
|
||||
options: Option<CacheOptions>,
|
||||
interceptors: &impl Interceptors<RouteError>,
|
||||
interceptors: &impl Interceptors<Error>,
|
||||
observers: Option<&ObservationSender>,
|
||||
provider: F,
|
||||
) -> Result<P::Response, RouteError>
|
||||
) -> Result<P::Response, Error>
|
||||
where
|
||||
P: Cachable,
|
||||
P::Response: Serialize + DeserializeOwned,
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: Future<Output = Result<P::Response, RouteError>>,
|
||||
Fut: Future<Output = Result<P::Response, Error>>,
|
||||
{
|
||||
let identity = request.identity.clone();
|
||||
crate::diagnostic::provider(&identity.model, &identity.provider);
|
||||
|
|
@ -191,15 +191,15 @@ pub async fn execute_streaming<P, F, Fut>(
|
|||
request: CacheRequest,
|
||||
cache: Option<Arc<dyn ResponseCacheService>>,
|
||||
options: Option<CacheOptions>,
|
||||
interceptors: &impl Interceptors<RouteError>,
|
||||
interceptors: &impl Interceptors<Error>,
|
||||
observers: Option<&ObservationSender>,
|
||||
provider: F,
|
||||
) -> Result<OutputOf<P>, RouteError>
|
||||
) -> Result<OutputOf<P>, Error>
|
||||
where
|
||||
P: StreamCachable,
|
||||
P::Response: Serialize + DeserializeOwned,
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: Future<Output = Result<OutputOf<P>, RouteError>>,
|
||||
Fut: Future<Output = Result<OutputOf<P>, Error>>,
|
||||
{
|
||||
let identity = request.identity.clone();
|
||||
crate::diagnostic::provider(&identity.model, &identity.provider);
|
||||
|
|
@ -299,9 +299,9 @@ impl<P: StreamCachable> CallCache<P> {
|
|||
}
|
||||
|
||||
fn capture_stream<P: StreamCachable>(
|
||||
chunks: futures_util::stream::BoxStream<'static, Result<P::Chunk, RouteError>>,
|
||||
chunks: futures_util::stream::BoxStream<'static, Result<P::Chunk, Error>>,
|
||||
session: CacheSession,
|
||||
) -> futures_util::stream::BoxStream<'static, Result<P::Chunk, RouteError>> {
|
||||
) -> futures_util::stream::BoxStream<'static, Result<P::Chunk, Error>> {
|
||||
stream::try_unfold(
|
||||
(chunks, Some(Vec::<u8>::new()), session),
|
||||
|(mut chunks, captured, session)| async move {
|
||||
|
|
@ -330,7 +330,7 @@ fn capture_stream<P: StreamCachable>(
|
|||
{
|
||||
session.store(entry).await;
|
||||
}
|
||||
Ok::<_, RouteError>(None)
|
||||
Ok::<_, Error>(None)
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -369,9 +369,9 @@ fn successful_stream(text: &str, terminal: &str) -> bool {
|
|||
|
||||
async fn publish(
|
||||
facts: ExecutionFacts,
|
||||
interceptors: &impl Interceptors<RouteError>,
|
||||
interceptors: &impl Interceptors<Error>,
|
||||
observers: Option<&ObservationSender>,
|
||||
) -> Result<(), RouteError> {
|
||||
) -> Result<(), Error> {
|
||||
if let Some(observers) = observers {
|
||||
observers.emit(CallEvent::Execution(ExecutionEvent::ResultReady {
|
||||
facts: facts.clone(),
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use litellm_host::{
|
|||
observation::ObservationSender,
|
||||
};
|
||||
|
||||
use crate::{CallOptions, RouteError};
|
||||
use crate::{CallOptions, Error};
|
||||
|
||||
pub struct CallContext<'a, I> {
|
||||
pub interceptors: &'a I,
|
||||
|
|
@ -13,7 +13,7 @@ pub struct CallContext<'a, I> {
|
|||
pub cache: CachePolicy,
|
||||
}
|
||||
|
||||
impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
|
||||
impl<'a, I: Interceptors<Error>> CallContext<'a, I> {
|
||||
pub fn new(interceptors: &'a I, options: CallOptions) -> Self {
|
||||
Self {
|
||||
interceptors,
|
||||
|
|
@ -22,7 +22,7 @@ impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
|
|||
}
|
||||
}
|
||||
|
||||
pub async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
|
||||
pub async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
|
||||
if let Some(observers) = &self.observers {
|
||||
observers.emit(CallEvent::Execution(ExecutionEvent::ResultReady {
|
||||
facts: facts.clone(),
|
||||
|
|
@ -31,7 +31,7 @@ impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
|
|||
self.interceptors.result_ready(facts).await
|
||||
}
|
||||
|
||||
pub async fn response_received(&self, body: &str) -> Result<(), RouteError> {
|
||||
pub async fn response_received(&self, body: &str) -> Result<(), Error> {
|
||||
let raw = RawResponse {
|
||||
body: body.to_owned(),
|
||||
};
|
||||
|
|
@ -43,6 +43,6 @@ impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
|
|||
self.interceptors
|
||||
.after_provider_response(raw)
|
||||
.await
|
||||
.map_err(RouteError::post_call)
|
||||
.map_err(Error::post_call)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ use litellm_http::transport::Error as TransportError;
|
|||
use litellm_llms::Error as LlmError;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum RouteError {
|
||||
pub enum Error {
|
||||
#[error("expected {expected}, got {actual}")]
|
||||
InvalidType {
|
||||
expected: &'static str,
|
||||
|
|
@ -38,10 +38,10 @@ pub enum RouteError {
|
|||
#[error(transparent)]
|
||||
Secret(#[from] SecretError),
|
||||
#[error("post-call hook failed: {0}")]
|
||||
PostCallHook(#[source] Arc<RouteError>),
|
||||
PostCallHook(#[source] Arc<Error>),
|
||||
}
|
||||
|
||||
impl From<litellm_host::machine::MachineFault> for RouteError {
|
||||
impl From<litellm_host::machine::MachineFault> for Error {
|
||||
fn from(fault: litellm_host::machine::MachineFault) -> Self {
|
||||
use litellm_host::machine::MachineFault;
|
||||
Self::InvalidRequest(match fault {
|
||||
|
|
@ -51,7 +51,7 @@ impl From<litellm_host::machine::MachineFault> for RouteError {
|
|||
}
|
||||
}
|
||||
|
||||
impl RouteError {
|
||||
impl Error {
|
||||
pub fn post_call(error: Self) -> Self {
|
||||
Self::PostCallHook(Arc::new(error))
|
||||
}
|
||||
|
|
@ -76,7 +76,7 @@ impl RouteError {
|
|||
}
|
||||
}
|
||||
|
||||
impl From<LlmError> for RouteError {
|
||||
impl From<LlmError> for Error {
|
||||
fn from(error: LlmError) -> Self {
|
||||
match error {
|
||||
LlmError::InvalidType { expected, actual } => Self::InvalidType { expected, actual },
|
||||
|
|
@ -99,7 +99,7 @@ impl SecretError {
|
|||
}
|
||||
}
|
||||
|
||||
impl From<litellm_secrets::Error> for RouteError {
|
||||
impl From<litellm_secrets::Error> for Error {
|
||||
fn from(error: litellm_secrets::Error) -> Self {
|
||||
Self::Secret(SecretError(Arc::new(error)))
|
||||
}
|
||||
|
|
@ -115,22 +115,22 @@ impl Eq for SecretError {}
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::RouteError;
|
||||
use super::Error;
|
||||
use litellm_llms::{Error as LlmError, ErrorDetail};
|
||||
use rstest::rstest;
|
||||
|
||||
#[test]
|
||||
fn a_missing_api_key_is_the_environment_not_the_request() {
|
||||
assert!(
|
||||
!RouteError::Auth(litellm_auth::Error::MissingApiKey {
|
||||
!Error::Auth(litellm_auth::Error::MissingApiKey {
|
||||
provider: "Anthropic",
|
||||
environment_variable: "ANTHROPIC_API_KEY",
|
||||
})
|
||||
.is_request()
|
||||
);
|
||||
assert!(RouteError::Auth(litellm_auth::Error::InvalidHeader).is_request());
|
||||
assert!(RouteError::InvalidRequest("top_k".into()).is_request());
|
||||
assert!(!RouteError::InvalidResponse("bad json".into()).is_request());
|
||||
assert!(Error::Auth(litellm_auth::Error::InvalidHeader).is_request());
|
||||
assert!(Error::InvalidRequest("top_k".into()).is_request());
|
||||
assert!(!Error::InvalidResponse("bad json".into()).is_request());
|
||||
}
|
||||
#[rstest]
|
||||
#[case::request(true)]
|
||||
|
|
@ -139,7 +139,7 @@ mod tests {
|
|||
let source = serde_json::from_str::<serde_json::Value>("{").unwrap_err();
|
||||
let source_message = source.to_string();
|
||||
let detail = ErrorDetail::invalid("test payload", source);
|
||||
let error = RouteError::from(if request {
|
||||
let error = Error::from(if request {
|
||||
LlmError::InvalidRequest(detail)
|
||||
} else {
|
||||
LlmError::InvalidResponse(detail)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ pub mod outbound;
|
|||
pub mod provider;
|
||||
pub mod resources;
|
||||
|
||||
pub use error::RouteError;
|
||||
pub use error::Error;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct CallOptions {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ use litellm_core_utils::get_llm_provider_logic::{
|
|||
CustomLlmProvider, LlmProviders, get_custom_llm_provider,
|
||||
};
|
||||
|
||||
use crate::error::RouteError as Error;
|
||||
use crate::error::Error;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ResolvedProvider<'a> {
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ use litellm_host::{
|
|||
protocol::Protocol,
|
||||
};
|
||||
use litellm_inference::{
|
||||
RouteError,
|
||||
Error,
|
||||
caching::{Cachable, CacheRequest, StreamCachable, execute_streaming, execute_unary},
|
||||
};
|
||||
use rstest::{fixture, rstest};
|
||||
|
|
@ -36,7 +36,7 @@ struct TestRoute;
|
|||
impl Protocol for TestRoute {
|
||||
type Request = Value;
|
||||
type Response = Value;
|
||||
type Error = RouteError;
|
||||
type Error = Error;
|
||||
type HostCall = Infallible;
|
||||
type Chunk = Bytes;
|
||||
type StreamHead = ();
|
||||
|
|
@ -204,7 +204,7 @@ async fn streamed(
|
|||
.chunks(3)
|
||||
.map(|bytes| Ok(Bytes::copy_from_slice(bytes)))
|
||||
.collect::<Vec<_>>();
|
||||
let ending = fail.then_some(Err(RouteError::Unsupported("test transport failure")));
|
||||
let ending = fail.then_some(Err(Error::Unsupported("test transport failure")));
|
||||
Ok(CallOutput::Stream {
|
||||
head: (),
|
||||
chunks: stream::iter(chunks.into_iter().chain(ending)).boxed(),
|
||||
|
|
@ -215,7 +215,7 @@ async fn streamed(
|
|||
.unwrap()
|
||||
}
|
||||
|
||||
async fn consume(output: OutputOf<TestRoute>) -> Result<Vec<u8>, RouteError> {
|
||||
async fn consume(output: OutputOf<TestRoute>) -> Result<Vec<u8>, Error> {
|
||||
let CallOutput::Stream { chunks, .. } = output else {
|
||||
panic!("expected a stream");
|
||||
};
|
||||
|
|
@ -315,7 +315,7 @@ async fn a_provider_failure_never_populates_the_cache(cache: Arc<dyn ResponseCac
|
|||
None,
|
||||
|| async {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
Err(RouteError::Unsupported("test provider failure"))
|
||||
Err(Error::Unsupported("test provider failure"))
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
|
@ -467,8 +467,8 @@ struct CacheHitAccounting {
|
|||
reject: bool,
|
||||
}
|
||||
|
||||
impl Interceptors<RouteError> for CacheHitAccounting {
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
|
||||
impl Interceptors<Error> for CacheHitAccounting {
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
let ResultSource::Cache { key } = facts.source else {
|
||||
panic!("expected cache source")
|
||||
|
|
@ -482,7 +482,7 @@ impl Interceptors<RouteError> for CacheHitAccounting {
|
|||
);
|
||||
self.key.set(key).unwrap();
|
||||
if self.reject {
|
||||
return Err(RouteError::Unsupported("cache accounting rejected"));
|
||||
return Err(Error::Unsupported("cache accounting rejected"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -491,11 +491,11 @@ impl Interceptors<RouteError> for CacheHitAccounting {
|
|||
&self,
|
||||
wire: WireRequest,
|
||||
_: RequestContext,
|
||||
) -> Result<WireRequest, RouteError> {
|
||||
) -> Result<WireRequest, Error> {
|
||||
Ok(wire)
|
||||
}
|
||||
|
||||
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
|
||||
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -571,7 +571,7 @@ async fn cache_hits_notify_accounting_once_and_propagate_its_failure(
|
|||
if reject {
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(RouteError::Unsupported("cache accounting rejected"))
|
||||
Err(Error::Unsupported("cache accounting rejected"))
|
||||
));
|
||||
} else {
|
||||
assert_eq!(result.unwrap(), expected);
|
||||
|
|
@ -590,7 +590,7 @@ async fn cache_hits_notify_accounting_once_and_propagate_its_failure(
|
|||
impl Protocol for UnaryTestRoute {
|
||||
type Request = Value;
|
||||
type Response = Value;
|
||||
type Error = RouteError;
|
||||
type Error = Error;
|
||||
type HostCall = Infallible;
|
||||
type Chunk = Infallible;
|
||||
type StreamHead = Infallible;
|
||||
|
|
|
|||
|
|
@ -235,7 +235,10 @@ impl AnthropicBeta {
|
|||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
|
||||
Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
|
||||
Self::ComputerUse20250124
|
||||
| Self::ComputerUse20251124
|
||||
| Self::Context1m20250807
|
||||
| Self::ThinkingBindingControls20260801 => &[],
|
||||
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
|
||||
Self::Effort20251124 => &[BetaProvider::VertexAi],
|
||||
Self::FastMode20260201 | Self::Oauth20250420 => &[
|
||||
|
|
@ -272,7 +275,6 @@ impl AnthropicBeta {
|
|||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
|
||||
Self::ThinkingBindingControls20260801 => &[BetaProvider::AzureAi],
|
||||
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
|
||||
Self::ToolSearchTool20251019 => &[
|
||||
BetaProvider::Anthropic,
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ mod selection;
|
|||
|
||||
pub(crate) use native::NativeCacheHandle;
|
||||
pub(crate) use python::{CacheCall, PythonCache};
|
||||
pub(crate) use selection::{Cached, Selection, admit_native, configure, configured_native};
|
||||
pub(crate) use selection::{Cached, Selection, configure, configured_native};
|
||||
|
||||
use litellm_cache::Error;
|
||||
use pyo3::{
|
||||
|
|
|
|||
|
|
@ -279,7 +279,7 @@ pub(in crate::cache) fn configured(
|
|||
)> {
|
||||
let handle = native_handle(configured)?.ok_or_else(|| {
|
||||
pyo3::exceptions::PyRuntimeError::new_err(
|
||||
"the configured cache changed to a Python cache after native admission",
|
||||
"native inference needs a v2 cache, the catalog keeps Python caches on Python",
|
||||
)
|
||||
})?;
|
||||
let cache = handle.extract::<PyRef<'_, NativeCacheHandle>>()?;
|
||||
|
|
|
|||
|
|
@ -76,21 +76,6 @@ fn selected_cache<'py>(
|
|||
Ok(Some(configured))
|
||||
}
|
||||
|
||||
pub(crate) fn admit_native(
|
||||
py: Python<'_>,
|
||||
kwargs: &Bound<'_, PyDict>,
|
||||
call_type: &str,
|
||||
) -> PyResult<()> {
|
||||
if let Some(configured) = selected_cache(py, kwargs, call_type)?
|
||||
&& native::v2::native_handle(&configured)?.is_none()
|
||||
{
|
||||
return Err(crate::errors::RustBridgeDeclined::new_err(
|
||||
"the configured cache requires Python inference",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn configured_native(
|
||||
py: Python<'_>,
|
||||
kwargs: &Bound<'_, PyDict>,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use litellm_http::transport::Error as TransportError;
|
||||
use litellm_inference::RouteError;
|
||||
use litellm_inference::Error;
|
||||
use pyo3::{
|
||||
exceptions::{PyRuntimeError, PyValueError},
|
||||
prelude::*,
|
||||
|
|
@ -19,9 +19,9 @@ pyo3::create_exception!(
|
|||
"The provider call was already issued and failed. Args are (status, message); status is 0 when there was no HTTP response."
|
||||
);
|
||||
|
||||
pub(crate) fn route_error_to_pyerr(error: RouteError) -> PyErr {
|
||||
pub(crate) fn route_error_to_pyerr(error: Error) -> PyErr {
|
||||
match error {
|
||||
RouteError::Transport(TransportError::Http { status, body }) => {
|
||||
Error::Transport(TransportError::Http { status, body }) => {
|
||||
RustUpstreamError::new_err((status, body))
|
||||
}
|
||||
other => by_fault(other.is_request(), other.to_string()),
|
||||
|
|
@ -42,13 +42,13 @@ mod tests {
|
|||
use super::*;
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::unsupported(RouteError::Unsupported("test capability"), true)]
|
||||
#[case::invalid_provider(RouteError::InvalidProvider("unknown".into()), true)]
|
||||
#[case::invalid_request(RouteError::InvalidRequest("empty messages".into()), true)]
|
||||
#[case::unsupported(Error::Unsupported("test capability"), true)]
|
||||
#[case::invalid_provider(Error::InvalidProvider("unknown".into()), true)]
|
||||
#[case::invalid_request(Error::InvalidRequest("empty messages".into()), true)]
|
||||
#[case::connection(TransportError::Connect("unreachable".into()).into(), false)]
|
||||
#[case::network(TransportError::Network("timed out".into()).into(), false)]
|
||||
#[case::invalid_response(RouteError::InvalidResponse("missing usage".into()), false)]
|
||||
fn route_failures_are_terminal(#[case] error: RouteError, #[case] is_request: bool) {
|
||||
#[case::invalid_response(Error::InvalidResponse("missing usage".into()), false)]
|
||||
fn route_failures_are_terminal(#[case] error: Error, #[case] is_request: bool) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let failure = route_error_to_pyerr(error);
|
||||
|
|
@ -86,14 +86,12 @@ mod tests {
|
|||
fn missing_api_key_stays_a_runtime_error_while_other_auth_failures_are_value_errors() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let missing =
|
||||
route_error_to_pyerr(RouteError::Auth(litellm_auth::Error::MissingApiKey {
|
||||
provider: "Anthropic",
|
||||
environment_variable: "ANTHROPIC_API_KEY",
|
||||
}));
|
||||
let missing = route_error_to_pyerr(Error::Auth(litellm_auth::Error::MissingApiKey {
|
||||
provider: "Anthropic",
|
||||
environment_variable: "ANTHROPIC_API_KEY",
|
||||
}));
|
||||
assert!(missing.is_instance_of::<PyRuntimeError>(py));
|
||||
let invalid =
|
||||
route_error_to_pyerr(RouteError::Auth(litellm_auth::Error::InvalidHeader));
|
||||
let invalid = route_error_to_pyerr(Error::Auth(litellm_auth::Error::InvalidHeader));
|
||||
assert!(invalid.is_instance_of::<PyValueError>(py));
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
|
|||
use litellm_host::{call::HostedMachine, protocol::Protocol};
|
||||
use litellm_host_python::{PythonBinding, PythonHostCalls, from_py};
|
||||
use litellm_http::transport::Error as TransportError;
|
||||
use litellm_inference::RouteError;
|
||||
use litellm_inference::Error;
|
||||
use litellm_secrets::source::SecretSource;
|
||||
use pyo3::{
|
||||
exceptions::PyValueError,
|
||||
|
|
@ -119,14 +119,14 @@ impl InferenceHost {
|
|||
public_response(py, self.module, response)
|
||||
}
|
||||
|
||||
pub fn error(&self, py: Python<'_>, error: RouteError) -> PyResult<PyErr> {
|
||||
if let RouteError::Secret(source) = &error
|
||||
pub fn error(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
|
||||
if let Error::Secret(source) = &error
|
||||
&& let Some(original) = crate::secrets::python_error(py, source.source_error())
|
||||
{
|
||||
return Ok(original);
|
||||
}
|
||||
let native = match error {
|
||||
RouteError::Transport(TransportError::Http { status, body }) => {
|
||||
Error::Transport(TransportError::Http { status, body }) => {
|
||||
let error = RustUpstreamError::new_err((status, body));
|
||||
error
|
||||
.value(py)
|
||||
|
|
@ -144,7 +144,7 @@ impl InferenceHost {
|
|||
}
|
||||
|
||||
pub(super) trait InferenceRoute: Sized + 'static {
|
||||
type Protocol: Protocol<Error = RouteError>;
|
||||
type Protocol: Protocol<Error = Error>;
|
||||
const OPERATION: LoggingOperation;
|
||||
const SYNC_CALL_TYPE: &'static str;
|
||||
const ASYNC_CALL_TYPE: &'static str;
|
||||
|
|
@ -177,7 +177,6 @@ where
|
|||
} else {
|
||||
R::SYNC_CALL_TYPE
|
||||
};
|
||||
crate::cache::admit_native(py, &call.kwargs, call_type)?;
|
||||
let (arguments, hooks) = super::call_hooks(py, R::OPERATION, &call, asynchronous)?;
|
||||
super::run_public_call(
|
||||
py,
|
||||
|
|
|
|||
|
|
@ -8,59 +8,19 @@ use litellm_auth::AuthServices;
|
|||
use litellm_cache_response::{CachePolicy, ScopedCache};
|
||||
use litellm_callbacks_legacy_python::LoggingOperation;
|
||||
use litellm_host::{call::HostedMachine, protocol::Protocol};
|
||||
use litellm_host_python::present;
|
||||
use litellm_inference_responses::{ResponsesRoute, route::Responses};
|
||||
use litellm_secrets::source::SecretSource;
|
||||
use pyo3::prelude::*;
|
||||
use serde_json::Value;
|
||||
pub(crate) use websocket::ResponsesWebSocketConnection;
|
||||
|
||||
use super::{
|
||||
NativeCall,
|
||||
inference::{InferenceHost, InferenceRoute, run_inference},
|
||||
};
|
||||
use crate::errors::RustBridgeDeclined;
|
||||
|
||||
const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.responses";
|
||||
|
||||
fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyResult<Py<PyAny>> {
|
||||
let resolved = call.resolved()?;
|
||||
if let Some(reason) = py
|
||||
.import(ROUTE_HOST_MODULE)?
|
||||
.getattr("decline_reason")?
|
||||
.call1((&resolved,))?
|
||||
.extract::<Option<String>>()?
|
||||
{
|
||||
return Err(RustBridgeDeclined::new_err(reason));
|
||||
}
|
||||
let argument = |name: &str| present(&call.kwargs, &call.base, name);
|
||||
let model = argument("model")?
|
||||
.ok_or_else(|| pyo3::exceptions::PyValueError::new_err("model is required"))?
|
||||
.extract::<String>()?;
|
||||
let provider = argument("custom_llm_provider")?
|
||||
.map(|value| value.extract::<String>())
|
||||
.transpose()?;
|
||||
if provider
|
||||
.as_deref()
|
||||
.is_some_and(|provider| provider != "openai")
|
||||
|| model
|
||||
.strip_prefix("openai/")
|
||||
.unwrap_or(&model)
|
||||
.contains('/')
|
||||
{
|
||||
return Err(RustBridgeDeclined::new_err(
|
||||
"native HTTP responses provider",
|
||||
));
|
||||
}
|
||||
if argument("stream")?
|
||||
.map(|value| litellm_host_python::from_py::<Value>(&value))
|
||||
.transpose()?
|
||||
.is_some_and(|value| value == Value::Bool(true))
|
||||
{
|
||||
return Err(RustBridgeDeclined::new_err(
|
||||
"native Python responses streaming",
|
||||
));
|
||||
}
|
||||
let host = InferenceHost::new(call.view()?, ROUTE_HOST_MODULE);
|
||||
run_inference::<ResponsesRoute, _>(py, call, asynchronous, ResponsesPythonHost::new(host))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ argilla_batch_size: Optional[int] = None
|
|||
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload.
|
||||
gcs_pub_sub_use_v1: Optional[bool] = False # if you want to use v1 gcs pubsub logged payload
|
||||
generic_api_use_v1: Optional[bool] = False # if you want to use v1 generic api logged payload
|
||||
argilla_transformation_object: Optional[Dict[str, Any]] = None
|
||||
argilla_transformation_object: Mapping[str, str] | None = None
|
||||
_async_input_callback: List[Union[str, Callable[..., object], "CustomLogger"]] = ( # CustomLogger is lazy-loaded
|
||||
[]
|
||||
) # internal variable - async custom callbacks are routed here.
|
||||
|
|
@ -333,7 +333,7 @@ in_memory_llm_clients_cache: "LLMClientCache"
|
|||
safe_memory_mode: bool = False
|
||||
enable_azure_ad_token_refresh: Optional[bool] = False
|
||||
# Proxy Authentication - auto-obtain/refresh OAuth2/JWT tokens for LiteLLM Proxy
|
||||
proxy_auth: Optional[Any] = None
|
||||
proxy_auth: _ProxyAuthHandlerType | None = None
|
||||
### DEFAULT AZURE API VERSION ###
|
||||
AZURE_DEFAULT_API_VERSION = "2025-02-01-preview" # this is updated to the latest
|
||||
### DEFAULT WATSONX API VERSION ###
|
||||
|
|
@ -503,7 +503,7 @@ skill_search_embedding_model: Optional[str] = None
|
|||
# Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]])
|
||||
# New format: { "displayName": { "url": "...", "index": 0 } }
|
||||
# Old format: { "displayName": "url" } (for backward compatibility)
|
||||
public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {}
|
||||
public_model_groups_links: dict[str, str | dict[str, object]] = {}
|
||||
#### REQUEST PRIORITIZATION #######
|
||||
priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]]] = None
|
||||
# priority_reservation_settings is lazy-loaded via __getattr__
|
||||
|
|
@ -551,6 +551,7 @@ from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, ma
|
|||
|
||||
model_cost = get_model_cost_map(url=model_cost_map_url)
|
||||
cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
|
||||
apply_search_free_tier: bool = False
|
||||
cost_margin_config: Dict[
|
||||
str, Union[float, Dict[str, float]]
|
||||
] = {} # Provider-specific or global cost margins. Examples:
|
||||
|
|
@ -681,6 +682,7 @@ darkbloom_models: Set = set()
|
|||
openinfer_models: Final[Set[str]] = set()
|
||||
v0_models: Set = set()
|
||||
morph_models: Set = set()
|
||||
opencode_go_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads
|
||||
lambda_ai_models: Set = set()
|
||||
inception_models: Set = set()
|
||||
hyperbolic_models: Set = set()
|
||||
|
|
@ -969,6 +971,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
v0_models.add(key)
|
||||
elif value.get("litellm_provider") == "morph":
|
||||
morph_models.add(key)
|
||||
elif value.get("litellm_provider") == "opencode_go":
|
||||
opencode_go_models.add(key)
|
||||
elif value.get("litellm_provider") == "lambda_ai":
|
||||
lambda_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "inception":
|
||||
|
|
@ -1136,6 +1140,7 @@ model_list = list(
|
|||
| openinfer_models
|
||||
| v0_models
|
||||
| morph_models
|
||||
| opencode_go_models
|
||||
| lambda_ai_models
|
||||
| inception_models
|
||||
| black_forest_labs_models
|
||||
|
|
@ -1253,6 +1258,7 @@ def _build_models_by_provider() -> dict:
|
|||
"openinfer": openinfer_models,
|
||||
"v0": v0_models,
|
||||
"morph": morph_models,
|
||||
"opencode_go": opencode_go_models,
|
||||
"lambda_ai": lambda_ai_models,
|
||||
"inception": inception_models,
|
||||
"hyperbolic": hyperbolic_models,
|
||||
|
|
@ -1504,6 +1510,7 @@ from .rag.main import *
|
|||
from .sandbox.main import *
|
||||
from .decisions.main import *
|
||||
from .systemone.main import *
|
||||
from .extraction.main import *
|
||||
from .tool_loop import ToolLoopMaxRoundsExceeded, arun_tool_loop, run_tool_loop
|
||||
from .search.main import *
|
||||
from .realtime_api.main import (
|
||||
|
|
@ -1710,6 +1717,9 @@ if TYPE_CHECKING:
|
|||
from .llms.cloudflare.decisions.transformation import (
|
||||
CloudflareDecisionsConfig as CloudflareDecisionsConfig,
|
||||
)
|
||||
from .llms.cloudflare.extraction.transformation import (
|
||||
CloudflareExtractionConfig as CloudflareExtractionConfig,
|
||||
)
|
||||
from .llms.strands_decider.decisions.transformation import (
|
||||
StrandsDeciderDecisionsConfig as StrandsDeciderDecisionsConfig,
|
||||
)
|
||||
|
|
@ -2284,6 +2294,10 @@ if TYPE_CHECKING:
|
|||
LiteLLM_UpperboundKeyGenerateParams,
|
||||
)
|
||||
from litellm.utils import ModelResponseListIterator as _ModelResponseListIterator
|
||||
from collections.abc import Awaitable
|
||||
from litellm.types.utils import ModelResponse as _ModelResponseType
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper as _CustomStreamWrapperType
|
||||
from litellm.proxy_auth import ProxyAuthHandler as _ProxyAuthHandlerType
|
||||
|
||||
# Cost calculator functions
|
||||
cost_per_token: Callable[..., Tuple[float, float]]
|
||||
|
|
@ -2307,7 +2321,7 @@ if TYPE_CHECKING:
|
|||
supports_audio_output: Callable[..., bool]
|
||||
supports_system_messages: Callable[..., bool]
|
||||
supports_reasoning: Callable[..., bool]
|
||||
acreate: Callable[..., Any]
|
||||
acreate: Callable[..., Awaitable[_ModelResponseType | _CustomStreamWrapperType]]
|
||||
get_max_tokens: Callable[..., int]
|
||||
get_model_info: Callable[..., _ModelInfoType]
|
||||
register_prompt_template: Callable[..., None]
|
||||
|
|
@ -2605,8 +2619,10 @@ def __getattr__(name: str) -> Any:
|
|||
cancel_run,
|
||||
delete_run,
|
||||
)
|
||||
from types import MappingProxyType
|
||||
|
||||
return locals()[name]
|
||||
evals_exports: Final[Mapping[str, object]] = MappingProxyType(locals())
|
||||
return evals_exports[name]
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import importlib
|
|||
import sys
|
||||
from collections.abc import Callable, Mapping
|
||||
from types import ModuleType
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
|
|
@ -456,7 +456,7 @@ def _lazy_import_http_handlers(name: str) -> object:
|
|||
params: Final = {"timeout": async_timeout, "client_alias": "module level aclient"}
|
||||
|
||||
# Create the client instance
|
||||
provider_id: Final = cast(Any, "litellm_module_level_client")
|
||||
provider_id: Final = "litellm_module_level_client"
|
||||
async_client: Final = get_async_httpx_client(
|
||||
llm_provider=provider_id,
|
||||
params=params,
|
||||
|
|
|
|||
|
|
@ -166,6 +166,7 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"OpenRouterDecisionsConfig",
|
||||
"FireworksAIDecisionsConfig",
|
||||
"CloudflareDecisionsConfig",
|
||||
"CloudflareExtractionConfig",
|
||||
"StrandsDeciderDecisionsConfig",
|
||||
"DatabricksDecisionsConfig",
|
||||
"HostedVLLMDecisionsConfig",
|
||||
|
|
@ -735,6 +736,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
"OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
|
||||
"FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),
|
||||
"CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"),
|
||||
"CloudflareExtractionConfig": (".llms.cloudflare.extraction.transformation", "CloudflareExtractionConfig"),
|
||||
"StrandsDeciderDecisionsConfig": (
|
||||
".llms.strands_decider.decisions.transformation",
|
||||
"StrandsDeciderDecisionsConfig",
|
||||
|
|
|
|||
|
|
@ -5,7 +5,10 @@ Supports dynamic cost parameters that allow platform owners
|
|||
to define custom costs per agent query or per token.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final, SupportsFloat, SupportsIndex
|
||||
|
||||
from typing_extensions import Buffer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
|
|
@ -15,6 +18,15 @@ else:
|
|||
LitellmLoggingObject = Any
|
||||
|
||||
|
||||
def _custom_response_cost(model_call_details: Mapping[str, object]) -> float | None:
|
||||
response_cost: Final = model_call_details.get("response_cost", None)
|
||||
if response_cost is None:
|
||||
return None
|
||||
if isinstance(response_cost, (str, Buffer, SupportsFloat, SupportsIndex)):
|
||||
return float(response_cost)
|
||||
raise TypeError(f"float() argument must be a string or a real number, not {type(response_cost).__name__!r}")
|
||||
|
||||
|
||||
class A2ACostCalculator:
|
||||
@staticmethod
|
||||
def calculate_a2a_cost(
|
||||
|
|
@ -45,9 +57,9 @@ class A2ACostCalculator:
|
|||
model_call_details: Final = litellm_logging_obj.model_call_details
|
||||
|
||||
# Check if user set a custom response cost (backward compatibility)
|
||||
response_cost: Final = model_call_details.get("response_cost", None)
|
||||
if response_cost is not None:
|
||||
return float(response_cost)
|
||||
custom_response_cost: Final = _custom_response_cost(model_call_details)
|
||||
if custom_response_cost is not None:
|
||||
return custom_response_cost
|
||||
|
||||
# Get litellm_params for cost parameters
|
||||
litellm_params: Final = model_call_details.get("litellm_params", {}) or {}
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ class A2ACompletionBridgeHandler:
|
|||
@staticmethod
|
||||
def _build_completion_params(
|
||||
params: dict[str, Any],
|
||||
litellm_params: Mapping[str, Any],
|
||||
litellm_params: Mapping[str, object],
|
||||
api_base: str | None,
|
||||
agent_extra_headers: Mapping[str, str] | None,
|
||||
*,
|
||||
|
|
|
|||
|
|
@ -602,9 +602,9 @@ def _build_streaming_logging_obj(
|
|||
litellm_params: dict[str, object] | None,
|
||||
metadata: dict[str, object] | None,
|
||||
proxy_server_request: dict[str, object] | None,
|
||||
start_time: datetime.datetime,
|
||||
) -> Logging:
|
||||
"""Build logging object for streaming A2A requests."""
|
||||
start_time: Final = datetime.datetime.now()
|
||||
model: Final = f"a2a_agent/{agent_name}"
|
||||
|
||||
logging_obj: Final = Logging(
|
||||
|
|
@ -718,6 +718,7 @@ async def asend_message_streaming(
|
|||
if request is None:
|
||||
raise ValueError("request is required")
|
||||
|
||||
called_at: Final = datetime.datetime.now()
|
||||
_raw_logging_obj: Final = kwargs.get("litellm_logging_obj")
|
||||
logging_obj: Logging | None = _raw_logging_obj if isinstance(_raw_logging_obj, Logging) else None
|
||||
|
||||
|
|
@ -751,6 +752,7 @@ async def asend_message_streaming(
|
|||
litellm_params=litellm_params,
|
||||
metadata=metadata,
|
||||
proxy_server_request=proxy_server_request,
|
||||
start_time=called_at,
|
||||
)
|
||||
|
||||
verbose_logger.info("A2A send_message_streaming request_id=%s, agent=%s", request.id, agent_name)
|
||||
|
|
|
|||
|
|
@ -84,14 +84,23 @@ def _filter_reserved_headers(
|
|||
return filtered or None
|
||||
|
||||
|
||||
def _calling_key_hash(litellm_params: Mapping[str, object]) -> str | None:
|
||||
principal: Final = litellm_params.get(A2A_USER_API_KEY_HASH_PARAM)
|
||||
if principal is None or isinstance(principal, str):
|
||||
return principal
|
||||
if not principal:
|
||||
return None
|
||||
raise AttributeError(f"{type(principal).__name__!r} object has no attribute 'encode'", name="encode", obj=principal)
|
||||
|
||||
|
||||
def _request_scoped_runtime_session_id(
|
||||
params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, Any],
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> str | None:
|
||||
context_id: Final = get_session_id_from_a2a_params(params)
|
||||
if not isinstance(context_id, str) or not context_id:
|
||||
return None
|
||||
return scope_session_to_principal(context_id, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM))
|
||||
return scope_session_to_principal(context_id, _calling_key_hash(litellm_params))
|
||||
|
||||
|
||||
def _validate_runtime_session_id(session_id: str, model: str) -> str:
|
||||
|
|
@ -109,6 +118,13 @@ def _validate_runtime_session_id(session_id: str, model: str) -> str:
|
|||
)
|
||||
|
||||
|
||||
def _agentcore_model_name(litellm_params: Mapping[str, object]) -> str:
|
||||
model: Final = litellm_params.get("model", "")
|
||||
if isinstance(model, str):
|
||||
return model
|
||||
raise AttributeError(f"{type(model).__name__!r} object has no attribute 'startswith'", name="startswith", obj=model)
|
||||
|
||||
|
||||
class BedrockAgentCoreA2ATransformation:
|
||||
"""
|
||||
Request/response transformation for Bedrock AgentCore A2A agents.
|
||||
|
|
@ -152,11 +168,8 @@ class BedrockAgentCoreA2ATransformation:
|
|||
"""
|
||||
# Extract model and strip the "bedrock/" prefix
|
||||
# "bedrock/agentcore/arn:aws:..." → "agentcore/arn:aws:..."
|
||||
model: Final = litellm_params.get("model", "")
|
||||
if model.startswith("bedrock/"):
|
||||
agentcore_model = model[len("bedrock/") :]
|
||||
else:
|
||||
agentcore_model = model
|
||||
model: Final = _agentcore_model_name(litellm_params)
|
||||
agentcore_model: Final = model.removeprefix("bedrock/")
|
||||
|
||||
# Build optional_params from litellm_params (everything except model and custom_llm_provider)
|
||||
optional_params: Final = {k: v for k, v in litellm_params.items() if k not in ("model", "custom_llm_provider")}
|
||||
|
|
|
|||
|
|
@ -8,11 +8,31 @@ WXO uses a REST API (not A2A/JSON-RPC) with an async-poll execution model:
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator, Mapping
|
||||
from typing import Any, Final
|
||||
from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from uuid import uuid4
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
|
||||
from litellm.llms.openai.common_utils import decoded_json_object
|
||||
|
||||
|
||||
def _a2a_text_part(part: object) -> Sequence[object]:
|
||||
part_fields: Final = as_str_mapping(part)
|
||||
if not isinstance(part_fields, dict):
|
||||
return ()
|
||||
if part_fields.get("kind") in (None, "", "text") and part_fields.get("text"):
|
||||
return (part_fields["text"],)
|
||||
return ()
|
||||
|
||||
|
||||
def _join_with_spaces(texts: Sequence[object]) -> str:
|
||||
strings: Final = [text for text in texts if isinstance(text, str)]
|
||||
if len(strings) == len(texts):
|
||||
return " ".join(strings)
|
||||
first_bad: Final = next(position for position, text in enumerate(texts) if not isinstance(text, str))
|
||||
raise TypeError(f"sequence item {first_bad}: expected str instance, {type(texts[first_bad]).__name__} found")
|
||||
|
||||
|
||||
class WatsonxOrchestrateTransformation:
|
||||
|
|
@ -29,22 +49,16 @@ class WatsonxOrchestrateTransformation:
|
|||
return f"{cp4d_host.rstrip('/')}/orchestrate/cpd/instances/{instance_id}"
|
||||
|
||||
@staticmethod
|
||||
def extract_text_from_a2a_params(params: dict[str, Any]) -> str:
|
||||
def extract_text_from_a2a_params(params: Mapping[str, object]) -> str:
|
||||
"""
|
||||
Extract user message text from A2A MessageSendParams.
|
||||
|
||||
A2A format: params.message.parts[*] where part.kind == "text"
|
||||
"""
|
||||
message: Final = params.get("message", {})
|
||||
parts: Final = message.get("parts", [])
|
||||
texts: Final = []
|
||||
for part in parts:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
kind = part.get("kind")
|
||||
if kind in (None, "", "text") and part.get("text"):
|
||||
texts.append(part["text"])
|
||||
return " ".join(texts) or ""
|
||||
message: Final = decoded_json_object(params.get("message", {}))
|
||||
parts: Final[Iterable[object]] = iter_items(message.get("parts", []))
|
||||
texts: Final = list(chain.from_iterable(map(_a2a_text_part, parts)))
|
||||
return _join_with_spaces(texts) or ""
|
||||
|
||||
@staticmethod
|
||||
def build_wxo_run_body(
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ class A2AStreamingIterator:
|
|||
self.request = request
|
||||
self.logging_obj = logging_obj
|
||||
self.agent_name = agent_name
|
||||
self.start_time = datetime.now()
|
||||
self.start_time = logging_obj.start_time
|
||||
|
||||
# Collect chunks for token counting
|
||||
self.chunks: list[SendStreamingMessageResponse] = []
|
||||
|
|
@ -54,6 +54,8 @@ class A2AStreamingIterator:
|
|||
async def __anext__(self) -> "SendStreamingMessageResponse":
|
||||
try:
|
||||
chunk: Final = await self.stream.__anext__()
|
||||
if self.logging_obj.completion_start_time is None:
|
||||
self.logging_obj.update_completion_start_time(completion_start_time=datetime.now())
|
||||
|
||||
# Store chunk
|
||||
self.chunks.append(chunk)
|
||||
|
|
|
|||
|
|
@ -4,15 +4,20 @@ Utility functions for A2A protocol.
|
|||
|
||||
import hashlib
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import has_attributes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from a2a.types import SendMessageRequest, SendStreamingMessageRequest
|
||||
|
||||
|
||||
class _SupportsModelDump(Protocol):
|
||||
def model_dump(self, *, mode: str) -> object: ...
|
||||
|
||||
|
||||
class A2ARequestUtils:
|
||||
"""Utility class for A2A request/response processing."""
|
||||
|
||||
|
|
@ -74,7 +79,7 @@ class A2ARequestUtils:
|
|||
@staticmethod
|
||||
def get_input_message_from_request(
|
||||
request: "SendMessageRequest | SendStreamingMessageRequest",
|
||||
) -> Any:
|
||||
) -> object:
|
||||
"""
|
||||
Extract the input message from an A2A request.
|
||||
|
||||
|
|
@ -84,7 +89,7 @@ class A2ARequestUtils:
|
|||
Returns:
|
||||
The message object/dict or None
|
||||
"""
|
||||
params: Final = getattr(request, "params", None)
|
||||
params: Final[object] = getattr(request, "params", None)
|
||||
if params is None:
|
||||
return None
|
||||
return getattr(params, "message", None)
|
||||
|
|
@ -127,10 +132,12 @@ class A2ARequestUtils:
|
|||
# the dict branch — request-side parts are a2a-sdk Part RootModels whose
|
||||
# kind/text live on part.root, which the object branch cannot read. This
|
||||
# mirrors how the response side already works (it operates on model_dump).
|
||||
input_message = A2ARequestUtils.get_input_message_from_request(request)
|
||||
if input_message is not None and hasattr(input_message, "model_dump"):
|
||||
input_message = input_message.model_dump(mode="json")
|
||||
input_text: Final = A2ARequestUtils.extract_text_from_message(input_message)
|
||||
input_message: Final = A2ARequestUtils.get_input_message_from_request(request)
|
||||
input_text: Final = A2ARequestUtils.extract_text_from_message(
|
||||
input_message.model_dump(mode="json")
|
||||
if has_attributes(input_message, _SupportsModelDump, "model_dump")
|
||||
else input_message
|
||||
)
|
||||
prompt_tokens: Final = A2ARequestUtils.count_tokens(input_text)
|
||||
|
||||
# Count output tokens
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@
|
|||
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
|
||||
"text_editor_20241022": null,
|
||||
"text_editor_20250124": null,
|
||||
"thinking-binding-controls-2026-08-01": null,
|
||||
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
|
||||
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
|
||||
"token-efficient-tools-2025-02-19": null,
|
||||
"web-fetch-2025-09-10": "web-fetch-2025-09-10",
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ import httpx
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
|
||||
# Cache for the loaded configuration
|
||||
_BETA_HEADERS_CONFIG: dict | None = None
|
||||
|
|
@ -44,6 +45,7 @@ _BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_na
|
|||
|
||||
BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
|
||||
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class GetAnthropicBetaHeadersConfig:
|
||||
|
|
@ -75,7 +77,7 @@ class GetAnthropicBetaHeadersConfig:
|
|||
}
|
||||
|
||||
@staticmethod
|
||||
def _check_is_valid_dict(fetched_config: dict) -> bool:
|
||||
def _check_is_valid_dict(fetched_config: object) -> bool:
|
||||
"""Check if fetched config is a non-empty dict with expected structure."""
|
||||
if not isinstance(fetched_config, dict):
|
||||
verbose_logger.warning(
|
||||
|
|
@ -109,7 +111,7 @@ class GetAnthropicBetaHeadersConfig:
|
|||
return True
|
||||
|
||||
@classmethod
|
||||
def validate_beta_headers_config(cls, fetched_config: dict) -> bool:
|
||||
def validate_beta_headers_config(cls, fetched_config: object) -> bool:
|
||||
"""
|
||||
Validate the integrity of a fetched beta headers config.
|
||||
|
||||
|
|
@ -118,7 +120,7 @@ class GetAnthropicBetaHeadersConfig:
|
|||
return cls._check_is_valid_dict(fetched_config)
|
||||
|
||||
@staticmethod
|
||||
def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> dict:
|
||||
def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> object:
|
||||
"""
|
||||
Fetch the beta headers config from a remote URL.
|
||||
|
||||
|
|
@ -127,7 +129,7 @@ class GetAnthropicBetaHeadersConfig:
|
|||
"""
|
||||
response: Final = httpx.get(url, timeout=timeout)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
return _DECODED_JSON.validate_python(response.json())
|
||||
|
||||
|
||||
def get_beta_headers_config(url: str) -> dict:
|
||||
|
|
@ -145,7 +147,7 @@ def get_beta_headers_config(url: str) -> dict:
|
|||
Dict containing the beta headers configuration
|
||||
"""
|
||||
# Check if local-only mode is enabled
|
||||
if os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
|
||||
if is_offline_mode() or os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
|
||||
# verbose_logger.debug("Using local Anthropic beta headers config (LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True)")
|
||||
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
|
||||
|
||||
|
|
@ -160,7 +162,9 @@ def get_beta_headers_config(url: str) -> dict:
|
|||
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
|
||||
|
||||
# Validate the fetched config
|
||||
if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content):
|
||||
if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content) or not isinstance(
|
||||
content, dict
|
||||
):
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Fetched beta headers config failed integrity check. Using local backup instead. url=%s",
|
||||
url,
|
||||
|
|
|
|||
|
|
@ -7,10 +7,14 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
|
||||
from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicErrorType
|
||||
|
||||
_PARSED_ERROR_BODY: Final = TypeAdapter(object)
|
||||
|
||||
# HTTP status code -> Anthropic error type
|
||||
# Source: https://docs.anthropic.com/en/api/errors
|
||||
ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = {
|
||||
|
|
@ -79,7 +83,7 @@ class AnthropicExceptionMapping:
|
|||
- Generic: {"message": "..."}
|
||||
- Plain strings
|
||||
"""
|
||||
parsed: Final = safe_json_loads(raw_message)
|
||||
parsed: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message))
|
||||
if isinstance(parsed, dict):
|
||||
# Bedrock format
|
||||
if "detail" in parsed and isinstance(parsed["detail"], dict):
|
||||
|
|
@ -145,9 +149,8 @@ class AnthropicExceptionMapping:
|
|||
AnthropicErrorResponse dict
|
||||
"""
|
||||
# Try to parse as JSON once
|
||||
parsed: dict | None = safe_json_loads(raw_message)
|
||||
if not isinstance(parsed, dict):
|
||||
parsed = None
|
||||
decoded: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message))
|
||||
parsed: Final[dict[str, object] | None] = decoded if isinstance(decoded, dict) else None
|
||||
|
||||
# If parsed and already in Anthropic format - passthrough
|
||||
if parsed is not None and AnthropicExceptionMapping._is_anthropic_error_dict(parsed):
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ def get_assistants(
|
|||
api_base: str | None = None,
|
||||
api_version: str | None = None,
|
||||
**kwargs,
|
||||
) -> SyncCursorPage[Assistant]:
|
||||
) -> SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]]:
|
||||
aget_assistants: Final[bool | None] = kwargs.pop("aget_assistants", None)
|
||||
if aget_assistants is not None and not isinstance(aget_assistants, bool):
|
||||
raise Exception("Invalid value passed in for aget_assistants. Only bool or None allowed")
|
||||
|
|
@ -100,7 +100,7 @@ def get_assistants(
|
|||
elif timeout is None:
|
||||
timeout = 600.0
|
||||
|
||||
response: SyncCursorPage[Assistant] | None = None
|
||||
response: SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]] | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
api_base = (
|
||||
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
|
|
@ -564,7 +564,7 @@ def create_thread(
|
|||
tool_resources: OpenAICreateThreadParamsToolResources | None = None,
|
||||
client: OpenAI | None = None,
|
||||
**kwargs,
|
||||
) -> Thread:
|
||||
) -> Thread | Coroutine[None, None, Thread]:
|
||||
"""
|
||||
- get the llm provider
|
||||
- if openai - route it there
|
||||
|
|
@ -610,7 +610,7 @@ def create_thread(
|
|||
api_base: str | None = None
|
||||
api_key: str | None = None
|
||||
|
||||
response: Thread | None = None
|
||||
response: Thread | Coroutine[None, None, Thread] | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
api_base = (
|
||||
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
|
|
@ -735,7 +735,7 @@ def get_thread(
|
|||
thread_id: str,
|
||||
client=None,
|
||||
**kwargs,
|
||||
) -> Thread:
|
||||
) -> Thread | Coroutine[None, None, Thread]:
|
||||
"""Get the thread object, given a thread_id"""
|
||||
aget_thread: Final = kwargs.pop("aget_thread", None)
|
||||
optional_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
|
|
@ -757,7 +757,7 @@ def get_thread(
|
|||
timeout = 600.0
|
||||
api_base: str | None = None
|
||||
api_key: str | None = None
|
||||
response: Thread | None = None
|
||||
response: Thread | Coroutine[None, None, Thread] | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
api_base = (
|
||||
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
|
|
@ -903,7 +903,7 @@ def add_message(
|
|||
metadata: dict | None = None,
|
||||
client=None,
|
||||
**kwargs,
|
||||
) -> OpenAIMessage:
|
||||
) -> OpenAIMessage | Coroutine[None, None, OpenAIMessage]:
|
||||
### COMMON OBJECTS ###
|
||||
a_add_message: Final = kwargs.pop("a_add_message", None)
|
||||
_message_data: Final = MessageData(role=role, content=content, attachments=attachments, metadata=metadata)
|
||||
|
|
@ -935,7 +935,7 @@ def add_message(
|
|||
timeout = 600.0
|
||||
api_key: str | None = None
|
||||
api_base: str | None = None
|
||||
response: OpenAIMessage | None = None
|
||||
response: OpenAIMessage | Coroutine[None, None, OpenAIMessage] | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
api_base = (
|
||||
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
|
|
@ -1065,7 +1065,7 @@ def get_messages(
|
|||
thread_id: str,
|
||||
client: Any | None = None,
|
||||
**kwargs,
|
||||
) -> SyncCursorPage[OpenAIMessage]:
|
||||
) -> SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]:
|
||||
aget_messages: Final = kwargs.pop("aget_messages", None)
|
||||
optional_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
litellm_params_dict: Final = get_litellm_params(**kwargs)
|
||||
|
|
@ -1086,7 +1086,7 @@ def get_messages(
|
|||
elif timeout is None:
|
||||
timeout = 600.0
|
||||
|
||||
response: SyncCursorPage[OpenAIMessage] | None = None
|
||||
response: SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]] | None = None
|
||||
api_key: str | None = None
|
||||
api_base: str | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
|
|
@ -1256,7 +1256,7 @@ def run_thread(
|
|||
client: object | None = None,
|
||||
event_handler: AssistantEventHandler | None = None, # for stream=True calls
|
||||
**kwargs,
|
||||
) -> Run:
|
||||
) -> Run | Coroutine[None, None, Run]:
|
||||
"""Run a given thread + assistant."""
|
||||
arun_thread: Final = kwargs.pop("arun_thread", None)
|
||||
optional_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
|
|
@ -1278,7 +1278,7 @@ def run_thread(
|
|||
elif timeout is None:
|
||||
timeout = 600.0
|
||||
|
||||
response: Run | None = None
|
||||
response: Run | Coroutine[None, None, Run] | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
api_base = (
|
||||
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ This module is dependency-injected: callers pass the proxy ``llm_router`` and
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import litellm
|
||||
from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
|
||||
|
|
@ -25,12 +25,12 @@ if TYPE_CHECKING:
|
|||
def resolve_embedding_router(
|
||||
embedding_model: str,
|
||||
llm_router: Router | None,
|
||||
llm_model_list: list[dict[str, Any]] | None,
|
||||
llm_model_list: Sequence[Mapping[str, object]] | None,
|
||||
) -> Router | None:
|
||||
"""Return ``llm_router`` iff it serves ``embedding_model`` as a deployment."""
|
||||
if llm_router is None:
|
||||
return None
|
||||
router_model_names: Final[list[str]] = (
|
||||
router_model_names: Final[Sequence[object]] = (
|
||||
[m["model_name"] for m in llm_model_list if "model_name" in m] if llm_model_list is not None else []
|
||||
)
|
||||
if embedding_model in router_model_names:
|
||||
|
|
|
|||
|
|
@ -24,6 +24,16 @@ if current == false then
|
|||
end
|
||||
if ARGV[3] then
|
||||
local decoded, stored = pcall(cjson.decode, current)
|
||||
if ARGV[4] then
|
||||
local legacy = current
|
||||
if decoded and type(stored) == 'string' then legacy = stored end
|
||||
for _, eligible in ipairs(cjson.decode(ARGV[3])) do
|
||||
if eligible['model_id'] == legacy then
|
||||
redis.call('EXPIRE', KEYS[1], ARGV[2])
|
||||
return current
|
||||
end
|
||||
end
|
||||
end
|
||||
if decoded and type(stored) == 'table' then
|
||||
for _, eligible in ipairs(cjson.decode(ARGV[3])) do
|
||||
local matches = true
|
||||
|
|
@ -68,6 +78,7 @@ def claim_affinity_pin_in_memory(
|
|||
ttl_seconds: int,
|
||||
*,
|
||||
eligible_values: tuple[Mapping[str, str], ...] | None = None,
|
||||
preserve_legacy_model_ids: bool = False,
|
||||
) -> object:
|
||||
"""No await between read and write, so same-loop claims agree during a Redis outage."""
|
||||
existing: Final[object] = cache.in_memory_cache.get_cache(cache_key)
|
||||
|
|
@ -75,7 +86,12 @@ def claim_affinity_pin_in_memory(
|
|||
if _legacy_pin_matches(existing, pin_value):
|
||||
set_local_affinity_pin(cache, cache_key, pin_value, ttl_seconds)
|
||||
return existing
|
||||
winner: Final = existing if existing is not None and existing in (eligible_values or ()) else pin_value
|
||||
eligible_existing: Final = existing in (eligible_values or ()) or (
|
||||
preserve_legacy_model_ids
|
||||
and isinstance(existing, str)
|
||||
and any("model_id" in value and _legacy_pin_matches(existing, value) for value in (eligible_values or ()))
|
||||
)
|
||||
winner: Final = existing if existing is not None and eligible_existing else pin_value
|
||||
set_local_affinity_pin(cache, cache_key, winner, ttl_seconds)
|
||||
return winner
|
||||
|
||||
|
|
@ -94,6 +110,7 @@ async def claim_affinity_pin(
|
|||
ttl_seconds: int,
|
||||
*,
|
||||
eligible_values: tuple[Mapping[str, str], ...] | None = None,
|
||||
preserve_legacy_model_ids: bool = False,
|
||||
) -> object:
|
||||
"""Return the authoritative first writer, replacing it only when it becomes ineligible.
|
||||
|
||||
|
|
@ -112,6 +129,7 @@ async def claim_affinity_pin(
|
|||
if eligible_values is not None
|
||||
else ()
|
||||
),
|
||||
*(("1",) if preserve_legacy_model_ids and eligible_values is not None else ()),
|
||||
)
|
||||
raw: Final = cast( # cast-ok: Redis scripts return heterogeneous values; only object is asserted here
|
||||
object, await claim_script(keys=(cache_key,), args=args)
|
||||
|
|
@ -124,4 +142,11 @@ async def claim_affinity_pin(
|
|||
return winner
|
||||
except Exception as error: # noqa: BLE001 # Redis/Lua faults retain same-pod affinity through local claims
|
||||
verbose_router_logger.debug("Affinity cache: Redis claim failed, using pod-local claim. error=%s", error)
|
||||
return claim_affinity_pin_in_memory(cache, cache_key, pin_value, ttl_seconds, eligible_values=eligible_values)
|
||||
return claim_affinity_pin_in_memory(
|
||||
cache,
|
||||
cache_key,
|
||||
pin_value,
|
||||
ttl_seconds,
|
||||
eligible_values=eligible_values,
|
||||
preserve_legacy_model_ids=preserve_legacy_model_ids,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,10 +13,14 @@ import json
|
|||
from contextlib import suppress
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import print_verbose, verbose_logger
|
||||
|
||||
from .base_cache import BaseCache
|
||||
|
||||
_DECODED_BLOB: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class AzureBlobCache(BaseCache):
|
||||
def __init__(self, account_url, container) -> None:
|
||||
|
|
@ -65,7 +69,7 @@ class AzureBlobCache(BaseCache):
|
|||
print_verbose(f"Get Azure Blob Cache: key: {key}")
|
||||
as_bytes: Final = self.container_client.download_blob(key).readall()
|
||||
as_str: Final = as_bytes.decode("utf-8")
|
||||
cached_response: Final[object] = json.loads(as_str)
|
||||
cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str))
|
||||
|
||||
verbose_logger.debug(
|
||||
"Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s",
|
||||
|
|
@ -86,7 +90,7 @@ class AzureBlobCache(BaseCache):
|
|||
blob: Final = await self.async_container_client.download_blob(key)
|
||||
as_bytes: Final = await blob.readall()
|
||||
as_str: Final = as_bytes.decode("utf-8")
|
||||
cached_response: Final[object] = json.loads(as_str)
|
||||
cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str))
|
||||
verbose_logger.debug(
|
||||
"Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s",
|
||||
key,
|
||||
|
|
|
|||
|
|
@ -9,14 +9,7 @@ Has 4 methods:
|
|||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
Span = _Span | Any
|
||||
else:
|
||||
Span = Any
|
||||
from typing import Final
|
||||
|
||||
|
||||
class BaseCache(ABC):
|
||||
|
|
|
|||
|
|
@ -1,15 +1,19 @@
|
|||
import json
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from .base_cache import BaseCache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
Span = _Span | Any
|
||||
Span = _Span
|
||||
else:
|
||||
Span = Any
|
||||
|
||||
_DECODED_DISK_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class DiskCache(BaseCache):
|
||||
def __init__(self, disk_cache_dir: str | None = None):
|
||||
|
|
@ -44,7 +48,7 @@ class DiskCache(BaseCache):
|
|||
original_cached_response: Final = self.disk_cache.get(key)
|
||||
if original_cached_response:
|
||||
try:
|
||||
cached_response = json.loads(original_cached_response)
|
||||
cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response))
|
||||
except Exception:
|
||||
cached_response = original_cached_response
|
||||
return cached_response
|
||||
|
|
|
|||
|
|
@ -59,16 +59,6 @@ class PendingBatchRead:
|
|||
previous_access_times: dict[str, float | None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeclaredBatchRead:
|
||||
"""A ``async_batch_get_cache`` split in two: the memory half done, the Redis half declared on a ``RedisBatch``
|
||||
so it rides that batch's next round trip, resolved later with ``async_resolve_batch_get``."""
|
||||
|
||||
keys: tuple[str, ...]
|
||||
pending: PendingBatchRead
|
||||
result: BatchResult[Mapping[str, object]] | None
|
||||
|
||||
|
||||
def _log_deferred_increment_failure(future: asyncio.Future[float]) -> None:
|
||||
if future.cancelled():
|
||||
return
|
||||
|
|
@ -382,22 +372,6 @@ class DualCache(BaseCache):
|
|||
await self.in_memory_cache.async_set_cache(key, value, **self._backfill_kwargs(kwargs))
|
||||
return merged
|
||||
|
||||
async def declare_batch_get(self, keys: Sequence[str], batch: RedisBatch) -> DeclaredBatchRead:
|
||||
pending: Final = await self._prepare_batch_get(
|
||||
list(keys),
|
||||
local_only=False,
|
||||
throttle_redis=False,
|
||||
)
|
||||
return DeclaredBatchRead(
|
||||
keys=tuple(keys),
|
||||
pending=pending,
|
||||
result=batch.mget(pending.redis_keys) if pending.redis_keys else None,
|
||||
)
|
||||
|
||||
async def async_resolve_batch_get(self, declared: DeclaredBatchRead) -> list[object | None]:
|
||||
redis_result: Final = None if declared.result is None else await declared.result
|
||||
return await self._apply_batch_get(declared.pending, redis_result)
|
||||
|
||||
async def async_batch_get_cache(
|
||||
self,
|
||||
keys: list,
|
||||
|
|
|
|||
|
|
@ -95,7 +95,10 @@ class InMemoryCache(BaseCache):
|
|||
"""
|
||||
Check if a specific key is expired
|
||||
"""
|
||||
return key in self.ttl_dict and self._clock() > self.ttl_dict[key]
|
||||
try:
|
||||
return self._clock() > self.ttl_dict[key]
|
||||
except KeyError:
|
||||
return False
|
||||
|
||||
def _cache_keys(self) -> tuple[str, ...]:
|
||||
return tuple(self.cache_dict) # pyright: ignore[reportUnknownVariableType] # cache_dict is untyped legacy state
|
||||
|
|
@ -227,7 +230,10 @@ class InMemoryCache(BaseCache):
|
|||
if key in self.cache_dict:
|
||||
if self.evict_element_if_expired(key):
|
||||
return None
|
||||
original_cached_response: Final = self.cache_dict[key]
|
||||
try:
|
||||
original_cached_response: Final = self.cache_dict[key]
|
||||
except KeyError:
|
||||
return None
|
||||
try:
|
||||
cached_response = json.loads(original_cached_response)
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -134,7 +134,12 @@ _INCREMENT_WITH_FLOOR_LUA: Final = (
|
|||
"return count"
|
||||
)
|
||||
|
||||
_INCREMENT_IF_PRESENT_LUA: Final = (
|
||||
"if redis.call('EXISTS', KEYS[1]) == 0 then return false end return redis.call('INCRBYFLOAT', KEYS[1], ARGV[1])"
|
||||
)
|
||||
|
||||
_LUA_COUNT: Final = TypeAdapter(int)
|
||||
_LUA_OPTIONAL_TOTAL: Final[TypeAdapter[float | None]] = TypeAdapter(float | None)
|
||||
_OPTIONAL_COUNTS: Final = TypeAdapter(tuple[int | None, ...])
|
||||
|
||||
|
||||
|
|
@ -675,7 +680,7 @@ class RedisCache(BaseCache):
|
|||
# redis namespaces
|
||||
self.namespace = namespace
|
||||
# for high traffic, we store the redis results in memory and then batch write to redis
|
||||
self.redis_batch_writing_buffer: list = []
|
||||
self.redis_batch_writing_buffer: list[tuple[str, object]] = []
|
||||
if redis_flush_size is None:
|
||||
self.redis_flush_size: int = 100
|
||||
else:
|
||||
|
|
@ -1568,6 +1573,46 @@ class RedisCache(BaseCache):
|
|||
count: Final = await _redis_client.eval(_INCREMENT_WITH_FLOOR_LUA, 1, namespaced_key, value, ttl)
|
||||
return _LUA_COUNT.validate_python(count)
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_increment_if_present(
|
||||
self, key: str, value: float, parent_otel_span: Span | None = None
|
||||
) -> float | None:
|
||||
_redis_client: Final = self._async_commands()
|
||||
namespaced_key: Final = self.check_and_fix_namespace(key=key)
|
||||
start_time: Final = time.time()
|
||||
try:
|
||||
total: Final = _LUA_OPTIONAL_TOTAL.validate_python(
|
||||
await _redis_client.eval(_INCREMENT_IF_PRESENT_LUA, 1, namespaced_key, value)
|
||||
)
|
||||
except Exception as e:
|
||||
failed_at: Final = time.time()
|
||||
asyncio.create_task(
|
||||
self.service_logger_obj.async_service_failure_hook(
|
||||
service=ServiceTypes.REDIS,
|
||||
duration=failed_at - start_time,
|
||||
error=e,
|
||||
call_type="async_increment_if_present",
|
||||
caller=_get_call_stack_info(),
|
||||
start_time=start_time,
|
||||
end_time=failed_at,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
)
|
||||
raise
|
||||
end_time: Final = time.time()
|
||||
asyncio.create_task(
|
||||
self.service_logger_obj.async_service_success_hook(
|
||||
service=ServiceTypes.REDIS,
|
||||
duration=end_time - start_time,
|
||||
call_type="async_increment_if_present",
|
||||
caller=_get_call_stack_info(),
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
)
|
||||
return total
|
||||
|
||||
async def flush_cache_buffer(self):
|
||||
print_verbose(f"flushing to redis....reached size of buffer {len(self.redis_batch_writing_buffer)}")
|
||||
await self.async_set_cache_pipeline(self.redis_batch_writing_buffer)
|
||||
|
|
@ -1693,7 +1738,9 @@ class RedisCache(BaseCache):
|
|||
return key_value_dict
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_get_cache(self, key, parent_otel_span: Span | None = None, **kwargs):
|
||||
async def async_get_cache(
|
||||
self, key, parent_otel_span: Span | None = None, *, raise_on_error: bool = False, **kwargs
|
||||
):
|
||||
from redis.asyncio import Redis
|
||||
|
||||
_redis_client: Final[Redis] = self.init_async_client()
|
||||
|
|
@ -1736,6 +1783,8 @@ class RedisCache(BaseCache):
|
|||
)
|
||||
)
|
||||
print_verbose(f"litellm.caching.caching: async get() - Got exception from REDIS: {e}")
|
||||
if raise_on_error:
|
||||
raise
|
||||
_record_swallowed_redis_failure(self._circuit_breaker, e)
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
|
|
|
|||
|
|
@ -1,15 +1,19 @@
|
|||
import inspect
|
||||
from collections.abc import Awaitable, Callable, Coroutine, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable
|
||||
|
||||
from litellm import main
|
||||
from litellm.rust_bridge.catalog import Route, RouteContext
|
||||
from litellm.rust_bridge import detectors
|
||||
from litellm.rust_bridge.catalog import Feature, Route, RouteContext
|
||||
from litellm.rust_bridge.chat_completions import detectors as route_detectors
|
||||
from litellm.rust_bridge.chat_completions.entrypoints import (
|
||||
NATIVE_ACOMPLETION,
|
||||
NATIVE_COMPLETION,
|
||||
)
|
||||
from litellm.rust_bridge.dispatch import PublicDispatch
|
||||
from litellm.rust_bridge.public_call import (
|
||||
AsyncFlag,
|
||||
NativeCall,
|
||||
bind,
|
||||
native_call,
|
||||
|
|
@ -44,6 +48,7 @@ def _python_acompletion() -> PythonAcompletion:
|
|||
|
||||
_PYTHON_COMPLETION: Final = _python_completion()
|
||||
_COMPLETION: Final = signature(_PYTHON_COMPLETION)
|
||||
_FLAG: Final = AsyncFlag("acompletion", sync=_COMPLETION)
|
||||
_PYTHON_ACOMPLETION: Final = _python_acompletion()
|
||||
_ACOMPLETION: Final = signature(_PYTHON_ACOMPLETION)
|
||||
|
||||
|
|
@ -61,11 +66,32 @@ def _public_request(
|
|||
return native_call(legacy, args, kwargs)
|
||||
|
||||
|
||||
_FEATURES: Final[detectors.Detectors] = MappingProxyType(
|
||||
{
|
||||
**detectors.COMMON,
|
||||
Feature.MOCK_RESPONSE: route_detectors.chat_mock_response,
|
||||
Feature.AGENTIC_LOOP_HOOK: route_detectors.chat_agentic_loop_hook,
|
||||
Feature.RESPONSE_CACHE: detectors.response_cache,
|
||||
Feature.STREAMING: detectors.streaming,
|
||||
Feature.DROP_PARAMS: detectors.drop_params,
|
||||
Feature.MODIFY_PARAMS: detectors.modify_params,
|
||||
Feature.EXTRA_BODY: detectors.extra_body,
|
||||
Feature.RETRIES_FALLBACKS: route_detectors.chat_retries_fallbacks,
|
||||
Feature.PROMPT_MANAGEMENT: detectors.prompt_management,
|
||||
Feature.CACHE_CONTROL_INJECTION: detectors.cache_control_injection,
|
||||
Feature.MODEL_PARAM_GATING: route_detectors.chat_model_param_gating,
|
||||
Feature.MODEL_LIST: route_detectors.model_list,
|
||||
Feature.MODEL_ALIAS_MAP: route_detectors.model_alias_map,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _context(request: NativeCall) -> RouteContext:
|
||||
return RouteContext(
|
||||
Route.CHAT_COMPLETIONS,
|
||||
provider=optional_str(request.resolved.get("custom_llm_provider")),
|
||||
model=str(request.resolved["model"]),
|
||||
features=detectors.detected(_FEATURES, request),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -73,7 +99,6 @@ _DISPATCH: Final = PublicDispatch(
|
|||
route=Route.CHAT_COMPLETIONS,
|
||||
request=lambda args, kwargs: _public_request(_COMPLETION, args, kwargs),
|
||||
context=_context,
|
||||
bypass=lambda request: request.kwargs.get("acompletion") is True,
|
||||
)
|
||||
|
||||
_ADISPATCH: Final = PublicDispatch(
|
||||
|
|
@ -88,9 +113,11 @@ def completion(
|
|||
**kwargs: object, # kwargs-ok: preserve the public chat completions call shape
|
||||
) -> ChatResult | Coroutine[object, object, ChatResult]:
|
||||
python: Final = _PYTHON_COMPLETION
|
||||
if _FLAG.requested(kwargs):
|
||||
return acompletion(**_FLAG.keywords(args, kwargs))
|
||||
return _DISPATCH.run(
|
||||
args,
|
||||
kwargs,
|
||||
_FLAG.dropped(kwargs),
|
||||
python=python,
|
||||
binding=NATIVE_COMPLETION,
|
||||
native=native_call_hook,
|
||||
|
|
@ -101,7 +128,7 @@ async def acompletion(*args: object, **kwargs: object) -> ChatResult: # kwargs-
|
|||
python: Final = _PYTHON_ACOMPLETION
|
||||
return await _ADISPATCH.arun(
|
||||
args,
|
||||
kwargs,
|
||||
_FLAG.dropped(kwargs),
|
||||
python=python,
|
||||
binding=NATIVE_ACOMPLETION,
|
||||
native=native_call_hook,
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ CLAUDE_CODE_CLIENT: Final = "claude-code"
|
|||
MICROSOFT_365_COPILOT_DEFAULT_TIME_ZONE: Final = "UTC"
|
||||
GITHUB_COPILOT_PER_USER_AUTH_TYPE: Final = "per_user_oauth"
|
||||
GITHUB_COPILOT_AUTH_TYPE_KEY: Final = "github_copilot_auth_type"
|
||||
GITHUB_COPILOT_HOST_KEY: Final = "github_copilot_host"
|
||||
GITHUB_COPILOT_DEFAULT_HOST: Final = "github.com"
|
||||
GITHUB_COPILOT_USER_TOKEN_SAFETY_MARGIN_SECONDS: Final = 60
|
||||
GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS: Final = 60
|
||||
GITHUB_COPILOT_DEVICE_FLOW_CACHE_PREFIX: Final = "github_copilot_device_flow"
|
||||
|
|
@ -154,6 +156,9 @@ DEFAULT_REPLICATE_POLLING_RETRIES: Final = get_env_int("DEFAULT_REPLICATE_POLLIN
|
|||
DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)
|
||||
DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250)
|
||||
HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
|
||||
HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS: Final = get_env_int_in_range(
|
||||
"HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 300, 1, 86400
|
||||
)
|
||||
|
||||
# Maximum wall-clock seconds a streaming response is allowed to run.
|
||||
# Streams exceeding this duration are terminated with a Timeout error.
|
||||
|
|
@ -256,6 +261,9 @@ MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS: Final = get_env_int("MCP_PER_USER_TOKE
|
|||
|
||||
# MCP timeout defaults (seconds). Override via env vars for slow/custom MCP servers.
|
||||
MCP_CLIENT_TIMEOUT: Final = get_env_float("LITELLM_MCP_CLIENT_TIMEOUT", 60.0)
|
||||
MCP_MAX_SSE_EVENT_SIZE_BYTES: Final = get_env_int_in_range(
|
||||
"LITELLM_MCP_MAX_SSE_EVENT_SIZE_BYTES", 64 * 1024 * 1024, 1, sys.maxsize
|
||||
)
|
||||
MCP_TOOL_LISTING_TIMEOUT: Final = get_env_float("LITELLM_MCP_TOOL_LISTING_TIMEOUT", 30.0)
|
||||
MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES: Final = get_env_int("LITELLM_MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES", 3)
|
||||
MCP_UPSTREAM_RATE_LIMIT_BASE_DELAY_SECONDS: Final = get_env_float(
|
||||
|
|
@ -472,6 +480,7 @@ REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_end_user_sp
|
|||
REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_agent_spend_update_buffer"
|
||||
REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_tag_spend_update_buffer"
|
||||
REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_window_spend_update_buffer"
|
||||
REDIS_TOOL_DISCOVERY_BUFFER_KEY: Final = "litellm_tool_discovery_buffer"
|
||||
MAX_REDIS_BUFFER_DEQUEUE_COUNT: Final = get_env_int("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100)
|
||||
REDIS_SPEND_LOGS_BUFFER_KEY: Final = "litellm_spend_logs_buffer"
|
||||
REDIS_SPEND_LOGS_BUFFER_MAX_ROWS: Final = 100000
|
||||
|
|
@ -603,6 +612,7 @@ NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int(
|
|||
) # timeout for adjacent services (e.g. jwt auth)
|
||||
MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000)
|
||||
MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048)
|
||||
MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024
|
||||
BEDROCK_MAX_POLICY_SIZE: Final = get_env_int("BEDROCK_MAX_POLICY_SIZE", 75)
|
||||
# One entry per distinct AWS credential-argument set. Per-user cost attribution passes the attributed
|
||||
# identity as aws_session_name, so this bounds how many attributed identities keep a cached STS session.
|
||||
|
|
@ -852,6 +862,7 @@ LITELLM_CHAT_PROVIDERS: Final = [
|
|||
"heroku",
|
||||
"oci",
|
||||
"morph",
|
||||
"opencode_go",
|
||||
"lambda_ai",
|
||||
"inception",
|
||||
"vercel_ai_gateway",
|
||||
|
|
@ -1107,6 +1118,7 @@ openai_compatible_providers: Final[list[str]] = [
|
|||
"v0",
|
||||
"helicone",
|
||||
"morph",
|
||||
"opencode_go",
|
||||
"lambda_ai",
|
||||
"inception",
|
||||
"hyperbolic",
|
||||
|
|
@ -1649,6 +1661,7 @@ BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
|
|||
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
|
||||
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
|
||||
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
|
||||
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
|
||||
SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
|
||||
|
|
@ -1812,6 +1825,7 @@ LITELLM_EXECUTED_BATCH_CONCURRENCY: Final = max(1, get_env_int("LITELLM_EXECUTED
|
|||
########################### CLI SSO AUTHENTICATION CONSTANTS ###########################
|
||||
LITELLM_CLI_SOURCE_IDENTIFIER: Final = "litellm-cli"
|
||||
LITELLM_CLI_SESSION_TOKEN_PREFIX: Final = "litellm-session-token"
|
||||
CP_RETURN_TO_COOKIE_PREFIX: Final = "return_to.v1."
|
||||
CLI_SSO_SESSION_CACHE_KEY_PREFIX: Final = "cli_sso_session"
|
||||
CLI_SSO_SESSION_TTL_SECONDS: Final = 600
|
||||
CLI_SESSION_KEY_PREFIX: Final = "cli-session"
|
||||
|
|
@ -1831,6 +1845,7 @@ LITELLM_UI_SESSION_DURATION: Final = os.getenv("LITELLM_UI_SESSION_DURATION", "2
|
|||
|
||||
########################### DB CRON JOB NAMES ###########################
|
||||
DB_SPEND_UPDATE_JOB_NAME: Final = "db_spend_update_job"
|
||||
TOOL_DISCOVERY_COMMIT_JOB_NAME: Final = "tool_discovery_commit_job"
|
||||
DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME: Final = "db_daily_tag_spend_update_job"
|
||||
PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME: Final = "prometheus_emit_budget_metrics"
|
||||
CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME: Final = "cloudzero_export_usage_data"
|
||||
|
|
@ -1855,6 +1870,9 @@ SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS: Final = get_env_float(
|
|||
SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS: Final = get_env_float("SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS", 300.0)
|
||||
SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS: Final = get_env_float("SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS", 30.0)
|
||||
SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP: Final = get_env_int("SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP", 100000)
|
||||
SPEND_LOG_SIZE_CHECK_JOB_NAME: Final = "spend_log_size_check"
|
||||
SPEND_LOG_SIZE_CHECK_INTERVAL: Final = os.getenv("SPEND_LOG_SIZE_CHECK_INTERVAL", "1d")
|
||||
SPEND_LOG_SIZE_CLEANUP_RECHECK_BATCHES: Final = get_env_int("SPEND_LOG_SIZE_CLEANUP_RECHECK_BATCHES", 10)
|
||||
SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS: Final = get_env_float(
|
||||
"SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS", 5.0
|
||||
)
|
||||
|
|
@ -1976,6 +1994,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
|
|||
"public_model_groups",
|
||||
"public_model_groups_links",
|
||||
"cost_discount_config",
|
||||
"apply_search_free_tier",
|
||||
"cost_margin_config",
|
||||
"block_requests_for_models_without_pricing",
|
||||
"budget_exceeded_throttle_percentage",
|
||||
|
|
@ -2316,6 +2335,7 @@ HARNESS_STDERR_TAIL_LINES: Final = 40
|
|||
HARNESS_STREAM_READ_CHUNK_BYTES: Final = 64 * 1024
|
||||
HARNESS_EVENT_QUEUE_MAX_SIZE: Final = 1024
|
||||
HARNESS_PROCESS_KILL_GRACE_SECONDS: Final = 5.0
|
||||
HARNESS_RESUME_COMMAND_TIMEOUT_SECONDS: Final = 30.0
|
||||
HARNESS_SNAPSHOT_SKIP_DIRS: Final = frozenset(
|
||||
{
|
||||
".git",
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
|||
_SERVICE_TIER_TO_COST_KEY_SUFFIX,
|
||||
BilledTokenRates,
|
||||
CostCalculatorUtils,
|
||||
calculate_cache_writing_cost,
|
||||
calculate_cost_component,
|
||||
generic_cost_per_character,
|
||||
generic_cost_per_token,
|
||||
|
|
@ -112,6 +113,7 @@ from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_toke
|
|||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.types.agents import LiteLLMSendMessageResponse
|
||||
from litellm.types.decisions import DecisionsResponse, DecisionsUsage, OpenAIDecisionResponse, OpenAIDecisionUsage
|
||||
from litellm.types.extraction import ExtractionResponse, ExtractionUsage
|
||||
from litellm.types.llms.base import CachedTokensDetails, LiteLLMBaseModel
|
||||
from litellm.types.llms.openai import (
|
||||
HttpxBinaryResponseContent,
|
||||
|
|
@ -906,11 +908,15 @@ def _bedrock_pricing_info(
|
|||
return model_info_with_pricing_overrides(profile, registration[1])
|
||||
|
||||
|
||||
def _declares_rates(model_info: ModelInfo | None) -> bool:
|
||||
entry: Final = _raw_cost_map_entry(model_info["key"]) if model_info is not None else None
|
||||
def cost_map_key_prices_anything(key: str) -> bool:
|
||||
entry: Final = _raw_cost_map_entry(key)
|
||||
return entry is not None and _cost_map_entry_prices_anything(entry)
|
||||
|
||||
|
||||
def _declares_rates(model_info: ModelInfo | None) -> bool:
|
||||
return model_info is not None and cost_map_key_prices_anything(model_info["key"])
|
||||
|
||||
|
||||
def _base_model_pricing_provider(base_model: str, custom_llm_provider: str | None) -> str | None:
|
||||
"""The provider to price an explicit base_model under, when the deployment's own provider has
|
||||
no rates for it (e.g. chatgpt/ subscription rows) but the base model's own row does"""
|
||||
|
|
@ -1167,6 +1173,8 @@ def get_usage_object(
|
|||
|
||||
if isinstance(completion_response, (DecisionsResponse, OpenAIDecisionResponse)):
|
||||
return None if completion_response.usage is None else _decisions_usage(completion_response.usage)
|
||||
if isinstance(completion_response, ExtractionResponse):
|
||||
return _extraction_usage(completion_response.usage)
|
||||
if isinstance(completion_response, HttpxBinaryResponseContent):
|
||||
return completion_response.usage
|
||||
if usage_obj is None:
|
||||
|
|
@ -1193,12 +1201,16 @@ def get_usage_object(
|
|||
elif isinstance(usage_obj, dict):
|
||||
return Usage(**usage_obj)
|
||||
elif isinstance(usage_obj, BaseModel):
|
||||
return Usage(**usage_obj.model_dump())
|
||||
return _usage_from_model(usage_obj)
|
||||
else:
|
||||
verbose_logger.debug("Unknown usage object type: %s, usage_obj: %s", type(usage_obj), usage_obj)
|
||||
return None
|
||||
|
||||
|
||||
def _usage_from_model(usage_model: BaseModel) -> Usage:
|
||||
return Usage(**usage_model.model_dump())
|
||||
|
||||
|
||||
def _decisions_prompt_tokens_details(usage: DecisionsUsage | OpenAIDecisionUsage) -> PromptTokensDetailsWrapper:
|
||||
match usage:
|
||||
case DecisionsUsage():
|
||||
|
|
@ -1223,6 +1235,14 @@ def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage:
|
|||
)
|
||||
|
||||
|
||||
def _extraction_usage(usage: ExtractionUsage) -> Usage:
|
||||
return Usage(
|
||||
prompt_tokens=usage.input_tokens,
|
||||
completion_tokens=usage.output_tokens,
|
||||
total_tokens=usage.input_tokens + usage.output_tokens,
|
||||
)
|
||||
|
||||
|
||||
def _is_known_usage_objects(usage_obj):
|
||||
"""Returns True if the usage obj is a known Usage type"""
|
||||
return (
|
||||
|
|
@ -1230,6 +1250,7 @@ def _is_known_usage_objects(usage_obj):
|
|||
or isinstance(usage_obj, ResponseAPIUsage)
|
||||
or isinstance(usage_obj, DecisionsUsage)
|
||||
or isinstance(usage_obj, OpenAIDecisionUsage)
|
||||
or isinstance(usage_obj, ExtractionUsage)
|
||||
or TranscriptionUsageObjectTransformation.is_transcription_usage_object(usage_obj)
|
||||
)
|
||||
|
||||
|
|
@ -1259,6 +1280,8 @@ def infer_call_type(call_type: CallTypesLiteral | None, completion_response: obj
|
|||
return "send_message"
|
||||
elif isinstance(completion_response, (DecisionsResponse, OpenAIDecisionResponse)):
|
||||
return "adecisions"
|
||||
elif isinstance(completion_response, ExtractionResponse):
|
||||
return "aextraction"
|
||||
|
||||
return call_type
|
||||
|
||||
|
|
@ -1507,6 +1530,7 @@ def completion_cost(
|
|||
litellm_model_name: str | None = None,
|
||||
router_model_id: str | None = None,
|
||||
litellm_logging_obj: LitellmLoggingObject | None = None,
|
||||
pricing_provider: str | None = None,
|
||||
### SERVICE TIER ###
|
||||
service_tier: str | None = None, # for OpenAI service tier pricing
|
||||
### DATA RESIDENCY ###
|
||||
|
|
@ -1527,6 +1551,10 @@ def completion_cost(
|
|||
"""
|
||||
try:
|
||||
call_type = infer_call_type(call_type, completion_response) or "completion"
|
||||
if isinstance(completion_response, DecisionsResponse) and completion_response.usage is None:
|
||||
raise ValueError("Cannot calculate Decisions cost: provider did not report usage")
|
||||
if pricing_provider is not None:
|
||||
custom_llm_provider = pricing_provider # rebind-ok: model_info.pricing_provider overrides the routing provider for cost-map lookups
|
||||
|
||||
if call_type == CallTypes.aresponses_websocket.value and isinstance(
|
||||
completion_response, LiteLLMRealtimeStreamLoggingObject
|
||||
|
|
@ -1552,6 +1580,7 @@ def completion_cost(
|
|||
optional_params=optional_params,
|
||||
custom_pricing=custom_pricing,
|
||||
base_model=base_model,
|
||||
pricing_provider=pricing_provider,
|
||||
standard_built_in_tools_params=standard_built_in_tools_params,
|
||||
litellm_model_name=litellm_model_name,
|
||||
router_model_id=router_model_id,
|
||||
|
|
@ -1642,17 +1671,19 @@ def completion_cost(
|
|||
*(((model, request_pricing_provider),) if model is not None else ()),
|
||||
)
|
||||
potential_model_names: Final = (
|
||||
raw_candidates if explicit_pricing else _priced_candidates_before_unpriced_rules(raw_candidates)
|
||||
raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates)
|
||||
)
|
||||
|
||||
for idx, (model, pricing_provider) in enumerate(potential_model_names):
|
||||
for idx, (model, candidate_pricing_provider) in enumerate(potential_model_names):
|
||||
try:
|
||||
if verbose_logger.isEnabledFor(logging.DEBUG):
|
||||
verbose_logger.debug("selected model name for cost calculation: %s", model)
|
||||
|
||||
if completion_response is not None and (
|
||||
isinstance(completion_response, BaseModel) or isinstance(completion_response, dict)
|
||||
): # tts returns a custom class
|
||||
isinstance(completion_response, (BaseModel, dict))
|
||||
or isinstance(completion_response, HttpxBinaryResponseContent)
|
||||
and completion_response.usage is not None
|
||||
):
|
||||
if isinstance(completion_response, dict):
|
||||
usage_obj: dict | Usage | None = completion_response.get("usage", {})
|
||||
else:
|
||||
|
|
@ -1662,10 +1693,12 @@ def completion_cost(
|
|||
setattr(
|
||||
completion_response,
|
||||
"usage",
|
||||
litellm.Usage(**_usage_for_dump.model_dump()),
|
||||
_usage_from_model(_usage_for_dump),
|
||||
)
|
||||
if isinstance(usage_obj, (DecisionsUsage, OpenAIDecisionUsage)):
|
||||
_usage = _decisions_usage(usage_obj).model_dump()
|
||||
elif isinstance(usage_obj, ExtractionUsage):
|
||||
_usage = _extraction_usage(usage_obj).model_dump()
|
||||
elif usage_obj is None:
|
||||
_usage = {}
|
||||
elif isinstance(usage_obj, BaseModel):
|
||||
|
|
@ -1712,7 +1745,9 @@ def completion_cost(
|
|||
|
||||
hidden_params = getattr(completion_response, "_hidden_params", None)
|
||||
if hidden_params is not None:
|
||||
custom_llm_provider = hidden_params.get("custom_llm_provider", custom_llm_provider or None)
|
||||
custom_llm_provider = pricing_provider or hidden_params.get(
|
||||
"custom_llm_provider", custom_llm_provider or None
|
||||
)
|
||||
region_name = hidden_params.get("region_name", region_name)
|
||||
|
||||
else:
|
||||
|
|
@ -1998,7 +2033,7 @@ def completion_cost(
|
|||
prompt_tokens=prompt_tokens or 0,
|
||||
completion_tokens=completion_tokens or 0,
|
||||
custom_llm_provider=_pricing_provider_for_cost_candidate(
|
||||
candidate_provider=pricing_provider,
|
||||
candidate_provider=candidate_pricing_provider,
|
||||
request_provider=request_pricing_provider,
|
||||
current_provider=custom_llm_provider,
|
||||
),
|
||||
|
|
@ -2102,7 +2137,7 @@ def completion_cost(
|
|||
_token_type_breakdown = get_token_type_cost_breakdown(
|
||||
model=model,
|
||||
custom_llm_provider=_pricing_provider_for_cost_candidate(
|
||||
candidate_provider=pricing_provider,
|
||||
candidate_provider=candidate_pricing_provider,
|
||||
request_provider=request_pricing_provider,
|
||||
current_provider=custom_llm_provider,
|
||||
),
|
||||
|
|
@ -2189,7 +2224,8 @@ def response_cost_calculator(
|
|||
| Response
|
||||
| SearchResponse
|
||||
| DecisionsResponse
|
||||
| OpenAIDecisionResponse,
|
||||
| OpenAIDecisionResponse
|
||||
| ExtractionResponse,
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
call_type: Literal[
|
||||
|
|
@ -2215,6 +2251,8 @@ def response_cost_calculator(
|
|||
"adecisions",
|
||||
"systemone",
|
||||
"asystemone",
|
||||
"extraction",
|
||||
"aextraction",
|
||||
],
|
||||
optional_params: dict,
|
||||
cache_hit: bool | None = None,
|
||||
|
|
@ -2225,6 +2263,7 @@ def response_cost_calculator(
|
|||
litellm_model_name: str | None = None,
|
||||
router_model_id: str | None = None,
|
||||
litellm_logging_obj: LitellmLoggingObject | None = None,
|
||||
pricing_provider: str | None = None,
|
||||
### SERVICE TIER ###
|
||||
service_tier: str | None = None, # for OpenAI service tier pricing
|
||||
### DATA RESIDENCY ###
|
||||
|
|
@ -2232,6 +2271,8 @@ def response_cost_calculator(
|
|||
### VERTEX LOCATION ###
|
||||
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
|
||||
region_name: str | None = None,
|
||||
custom_cost_per_token: CostPerToken | None = None,
|
||||
custom_cost_per_second: float | None = None,
|
||||
) -> float:
|
||||
"""
|
||||
Returns
|
||||
|
|
@ -2260,11 +2301,14 @@ def response_cost_calculator(
|
|||
optional_params=optional_params,
|
||||
custom_pricing=custom_pricing,
|
||||
base_model=base_model,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
prompt=prompt,
|
||||
standard_built_in_tools_params=standard_built_in_tools_params,
|
||||
litellm_model_name=litellm_model_name,
|
||||
router_model_id=router_model_id,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
pricing_provider=pricing_provider,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
|
|
@ -2842,6 +2886,7 @@ def _batch_rate(
|
|||
model_info: ModelInfo,
|
||||
key: Literal[
|
||||
"input_cost_per_audio_token_batches",
|
||||
"cache_read_input_audio_token_cost_batches",
|
||||
"input_cost_per_image_token_batches",
|
||||
"input_cost_per_video_token_batches",
|
||||
],
|
||||
|
|
@ -2927,7 +2972,12 @@ def batch_cost_calculator(
|
|||
total_completion_cost = 0.0
|
||||
if batch_rates.input is not None:
|
||||
total_prompt_cost = _batch_prompt_cost(
|
||||
usage, model_info, batch_rates.input, batch_rates.cache_read, batch_rates.cache_creation
|
||||
usage,
|
||||
model_info,
|
||||
batch_rates.input,
|
||||
batch_rates.cache_read,
|
||||
batch_rates.cache_creation,
|
||||
batch_rates.cache_creation_rates_by_ttl,
|
||||
)
|
||||
elif input_cost_per_token:
|
||||
details: Final = parse_prompt_tokens_details(usage)
|
||||
|
|
@ -2972,10 +3022,13 @@ def _batch_prompt_cost(
|
|||
input_rate: float,
|
||||
cache_read_rate: float | None,
|
||||
cache_creation_rate: float | None,
|
||||
cache_creation_rates_by_ttl: Mapping[str, float],
|
||||
) -> float:
|
||||
details: Final = parse_prompt_tokens_details(usage)
|
||||
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
|
||||
written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0
|
||||
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
|
||||
bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl)
|
||||
written_tokens: Final = details["cache_creation_tokens"] if bills_cache_writes else 0
|
||||
audio_tokens, image_tokens, video_tokens = (
|
||||
details["audio_tokens"],
|
||||
details["image_tokens"],
|
||||
|
|
@ -2989,13 +3042,27 @@ def _batch_prompt_cost(
|
|||
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
|
||||
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
|
||||
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
|
||||
+ cached_tokens * (cache_read_rate or 0.0)
|
||||
+ written_tokens * (cache_creation_rate or 0.0)
|
||||
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
|
||||
+ cached_audio_tokens
|
||||
* _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0)
|
||||
+ (
|
||||
calculate_cache_writing_cost(
|
||||
written_tokens,
|
||||
details["cache_creation_token_details"],
|
||||
input_rate if cache_creation_rate is None else cache_creation_rate,
|
||||
cache_creation_rates_by_ttl,
|
||||
)
|
||||
if bills_cache_writes
|
||||
else 0.0
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
_ATTRIBUTE_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def _attribute_value(obj: object, name: str) -> object:
|
||||
return getattr(obj, name)
|
||||
return _ATTRIBUTE_VALUE.validate_python(getattr(obj, name))
|
||||
|
||||
|
||||
def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]:
|
||||
|
|
@ -3630,7 +3697,26 @@ def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo |
|
|||
return None
|
||||
|
||||
|
||||
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...]) -> float | None:
|
||||
_TranscriptionRateKey: TypeAlias = Literal[
|
||||
"input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
|
||||
]
|
||||
|
||||
|
||||
def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None:
|
||||
match key:
|
||||
case "input_cost_per_second":
|
||||
return info.get("input_cost_per_second")
|
||||
case "input_cost_per_audio_token":
|
||||
return info.get("input_cost_per_audio_token")
|
||||
case "input_cost_per_token":
|
||||
return info.get("input_cost_per_token")
|
||||
case "output_cost_per_token":
|
||||
return info.get("output_cost_per_token")
|
||||
case _:
|
||||
assert_never(key)
|
||||
|
||||
|
||||
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
|
||||
"""First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry
|
||||
because ``get_model_info`` synthesizes zero token rates for entries that omit them."""
|
||||
if info is None:
|
||||
|
|
@ -3639,12 +3725,18 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...])
|
|||
if declared is None:
|
||||
return None
|
||||
return next(
|
||||
(float(value) for key in keys if declared.get(key) is not None and (value := info.get(key)) is not None),
|
||||
(
|
||||
float(value)
|
||||
for key in keys
|
||||
if declared.get(key) is not None and (value := _model_info_rate(info, key)) is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _transcription_rate(keys: tuple[str, ...], override: ModelInfo | None, base: ModelInfo | None) -> float:
|
||||
def _transcription_rate(
|
||||
keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None
|
||||
) -> float:
|
||||
rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base))
|
||||
return next((rate for rate in rates if rate is not None), 0.0)
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from pydantic_core import to_jsonable_python
|
|||
from typing_extensions import assert_never
|
||||
|
||||
import litellm
|
||||
from litellm.cost_calculator import cost_map_key_prices_anything
|
||||
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params
|
||||
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -21,6 +22,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
|||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.decisions.transformation import (
|
||||
BaseDecisionsConfig,
|
||||
DecisionsResponseValidationError,
|
||||
image_media_type,
|
||||
image_param,
|
||||
input_images,
|
||||
|
|
@ -427,7 +429,7 @@ def _jsonable_fallback(value: object) -> object:
|
|||
|
||||
def _logged_request_message(caller_request: object) -> ChatCompletionUserMessage:
|
||||
caller_fields: Final = _LOGGED_FIELDS_ADAPTER.validate_python(
|
||||
to_jsonable_python(caller_request, fallback=_jsonable_fallback)
|
||||
to_jsonable_python(caller_request, fallback=_jsonable_fallback, bytes_mode="base64")
|
||||
)
|
||||
sent_fields: Final = json.dumps(
|
||||
dict((name, value) for name, value in caller_fields.items() if value not in (None, [], {}))
|
||||
|
|
@ -687,6 +689,13 @@ def _request_body(call: DecisionsCall) -> Mapping[str, object]:
|
|||
return body
|
||||
|
||||
|
||||
def _pricing_model(response: DecisionsIRResponse, call: DecisionsCall) -> str:
|
||||
returned: Final = get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
|
||||
if returned is not None and cost_map_key_prices_anything(returned):
|
||||
return returned
|
||||
return f"{call.custom_llm_provider}/{call.model}"
|
||||
|
||||
|
||||
def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
|
||||
formatted: Final = _formatted_response(response, call)
|
||||
provider_cost: Final = call.provider_config.provider_reported_cost(response)
|
||||
|
|
@ -694,10 +703,7 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci
|
|||
{
|
||||
"model": f"{call.custom_llm_provider}/{call.model}",
|
||||
"custom_llm_provider": call.custom_llm_provider,
|
||||
"provider_response_model": (
|
||||
get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
|
||||
or f"{call.custom_llm_provider}/{call.model}"
|
||||
),
|
||||
"provider_response_model": _pricing_model(response, call),
|
||||
**({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}),
|
||||
}
|
||||
)
|
||||
|
|
@ -727,18 +733,21 @@ def _formatted_response(
|
|||
|
||||
|
||||
def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception:
|
||||
if isinstance(error, BaseLLMException) and error.status_code_is_synthesized:
|
||||
provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception"
|
||||
return litellm.APIConnectionError(
|
||||
message=f"{provider_label} - {error.message}",
|
||||
llm_provider=call.custom_llm_provider,
|
||||
model=f"{call.custom_llm_provider}/{call.model}",
|
||||
)
|
||||
return litellm.exception_type(
|
||||
model=f"{call.custom_llm_provider}/{call.model}",
|
||||
custom_llm_provider=call.custom_llm_provider,
|
||||
original_exception=error,
|
||||
)
|
||||
provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception"
|
||||
model: Final = f"{call.custom_llm_provider}/{call.model}"
|
||||
match error:
|
||||
case DecisionsResponseValidationError():
|
||||
return litellm.InternalServerError(
|
||||
message=f"{provider_label} - {error.message}", llm_provider=call.custom_llm_provider, model=model
|
||||
)
|
||||
case BaseLLMException(status_code_is_synthesized=True):
|
||||
return litellm.APIConnectionError(
|
||||
message=f"{provider_label} - {error.message}", llm_provider=call.custom_llm_provider, model=model
|
||||
)
|
||||
case _:
|
||||
return litellm.exception_type(
|
||||
model=model, custom_llm_provider=call.custom_llm_provider, original_exception=error
|
||||
)
|
||||
|
||||
|
||||
async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
|
||||
|
|
|
|||
|
|
@ -9,7 +9,15 @@ from litellm.rust_bridge.embeddings.entrypoints import (
|
|||
NATIVE_AEMBEDDING,
|
||||
NATIVE_EMBEDDING,
|
||||
)
|
||||
from litellm.rust_bridge.public_call import NativeCall, bind, native_call, native_call_hook, optional_str, signature
|
||||
from litellm.rust_bridge.public_call import (
|
||||
AsyncFlag,
|
||||
NativeCall,
|
||||
bind,
|
||||
native_call,
|
||||
native_call_hook,
|
||||
optional_str,
|
||||
signature,
|
||||
)
|
||||
from litellm.types.utils import EmbeddingResponse
|
||||
|
||||
__all__ = ("aembedding", "embedding")
|
||||
|
|
@ -24,6 +32,7 @@ _PYTHON_AEMBEDDING: Final = cast( # cast-ok: [LIT006] preserve the legacy publi
|
|||
PythonAembedding, main.aembedding
|
||||
)
|
||||
_EMBEDDING_SIGNATURE: Final = signature(_PYTHON_EMBEDDING)
|
||||
_FLAG: Final = AsyncFlag("aembedding", sync=_EMBEDDING_SIGNATURE)
|
||||
|
||||
|
||||
def _public_request(
|
||||
|
|
@ -50,7 +59,6 @@ _DISPATCH: Final = PublicDispatch(
|
|||
route=Route.EMBEDDINGS,
|
||||
request=lambda args, kwargs: _public_request(_EMBEDDING_SIGNATURE, args, kwargs),
|
||||
context=_context,
|
||||
bypass=lambda request: request.kwargs.get("aembedding") is True,
|
||||
)
|
||||
|
||||
_ADISPATCH: Final = PublicDispatch(
|
||||
|
|
@ -64,9 +72,11 @@ def embedding(
|
|||
*args: object,
|
||||
**kwargs: object, # kwargs-ok: preserve the public embedding call shape
|
||||
) -> EmbeddingResponse | Coroutine[object, object, EmbeddingResponse]:
|
||||
if _FLAG.requested(kwargs):
|
||||
return aembedding(**_FLAG.keywords(args, kwargs))
|
||||
return _DISPATCH.run(
|
||||
args,
|
||||
kwargs,
|
||||
_FLAG.dropped(kwargs),
|
||||
python=_PYTHON_EMBEDDING,
|
||||
binding=NATIVE_EMBEDDING,
|
||||
native=native_call_hook,
|
||||
|
|
@ -76,7 +86,7 @@ def embedding(
|
|||
async def aembedding(*args: object, **kwargs: object) -> EmbeddingResponse: # kwargs-ok: preserve the public call shape
|
||||
return await _ADISPATCH.arun(
|
||||
args,
|
||||
kwargs,
|
||||
_FLAG.dropped(kwargs),
|
||||
python=_PYTHON_AEMBEDDING,
|
||||
binding=NATIVE_AEMBEDDING,
|
||||
native=native_call_hook,
|
||||
|
|
|
|||
|
|
@ -122,6 +122,7 @@ class SpeechToCompletionBridgeHandler:
|
|||
if isinstance(result, ModelResponse):
|
||||
return self.transformation_handler.transform_response(
|
||||
model_response=result,
|
||||
model=model,
|
||||
response_format=requested_response_format if isinstance(requested_response_format, str) else None,
|
||||
)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Final, cast
|
|||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache, set_hidden_param
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -38,6 +39,11 @@ class ChatAudioParam(TypedDict):
|
|||
format: ReadOnly[NotRequired[str]]
|
||||
|
||||
|
||||
class ChatSpeechConfigAudioParam(TypedDict):
|
||||
speech_config: ReadOnly[Mapping[str, object]]
|
||||
format: ReadOnly[str]
|
||||
|
||||
|
||||
class SpeechToCompletionBridgeTransformationHandler:
|
||||
def _validate_response_format(
|
||||
self, model: str, custom_llm_provider: str, optional_params: Mapping[str, object]
|
||||
|
|
@ -49,6 +55,20 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
return
|
||||
from litellm.exceptions import BadRequestError
|
||||
|
||||
if custom_llm_provider.startswith("vertex_ai"):
|
||||
from litellm.llms.vertex_ai.text_to_speech.transformation import VertexAITextToSpeechConfig
|
||||
|
||||
vertex_supported: Final = ", ".join(
|
||||
sorted({*GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS, *VertexAITextToSpeechConfig.GEMINI_FORMAT_MAPPINGS})
|
||||
)
|
||||
raise BadRequestError(
|
||||
message=(
|
||||
f"Vertex AI Gemini TTS does not support response_format='{response_format}'."
|
||||
f" Supported response formats: {vertex_supported}."
|
||||
),
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
)
|
||||
supported: Final = ", ".join(sorted(GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS))
|
||||
raise BadRequestError(
|
||||
message=(
|
||||
|
|
@ -75,8 +95,27 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
return response_format if isinstance(response_format, str) else None
|
||||
|
||||
def _chat_audio_param(
|
||||
self, model: str, voice: str | Mapping[str, object] | None, optional_params: Mapping[str, object]
|
||||
) -> ChatAudioParam | None:
|
||||
self,
|
||||
model: str,
|
||||
voice: str | Mapping[str, object] | None,
|
||||
optional_params: Mapping[str, object],
|
||||
drop_params: bool,
|
||||
custom_llm_provider: str,
|
||||
) -> ChatAudioParam | ChatSpeechConfigAudioParam | None:
|
||||
import litellm
|
||||
|
||||
if isinstance(voice, Mapping) and self._is_gemini_tts_model(litellm.model_alias_map.get(model, model)):
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
normalize_gemini_speech_config,
|
||||
reject_unknown_gemini_tts_voice_keys,
|
||||
)
|
||||
|
||||
reject_unknown_gemini_tts_voice_keys(model, voice, drop_params, custom_llm_provider)
|
||||
speech_audio: Final[ChatSpeechConfigAudioParam] = {
|
||||
"speech_config": normalize_gemini_speech_config(voice, drops_unsupported_params(drop_params)),
|
||||
"format": GEMINI_TTS_CHAT_AUDIO_FORMAT,
|
||||
}
|
||||
return speech_audio
|
||||
if not isinstance(voice, str):
|
||||
return None
|
||||
audio_format: Final = self._chat_audio_format(model, optional_params)
|
||||
|
|
@ -90,9 +129,9 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
self,
|
||||
model: str,
|
||||
input: str,
|
||||
voice: str | dict | None,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
voice: str | Mapping[str, object] | None,
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object],
|
||||
headers: dict,
|
||||
litellm_logging_obj: "LiteLLMLoggingObj",
|
||||
custom_llm_provider: str,
|
||||
|
|
@ -104,7 +143,13 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
"messages": [user_message],
|
||||
"modalities": ["audio"],
|
||||
**self._chat_completion_params(optional_params),
|
||||
"audio": self._chat_audio_param(model, voice, optional_params),
|
||||
"audio": self._chat_audio_param(
|
||||
model,
|
||||
voice,
|
||||
optional_params,
|
||||
drops_unsupported_params(litellm_params.get("drop_params")),
|
||||
custom_llm_provider,
|
||||
),
|
||||
**litellm_params,
|
||||
"headers": headers,
|
||||
"litellm_logging_obj": litellm_logging_obj,
|
||||
|
|
@ -153,8 +198,9 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
return wav_header + pcm_data
|
||||
|
||||
def _is_gemini_tts_model(self, model: str) -> bool:
|
||||
"""Check if the model is a Gemini TTS model that returns PCM16 data."""
|
||||
return "gemini" in model.lower() and ("tts" in model.lower() or "preview-tts" in model.lower())
|
||||
from litellm.utils import is_gemini_tts_model
|
||||
|
||||
return is_gemini_tts_model(model)
|
||||
|
||||
def _gemini_tts_response_body(self, decoded_audio: bytes, response_format: str | None) -> tuple[bytes, str]:
|
||||
if response_format == GEMINI_TTS_RAW_RESPONSE_FORMAT:
|
||||
|
|
@ -162,7 +208,7 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
return self._convert_pcm16_to_wav(decoded_audio), "audio/wav"
|
||||
|
||||
def transform_response(
|
||||
self, model_response: "ModelResponse", response_format: str | None
|
||||
self, model_response: "ModelResponse", model: str, response_format: str | None
|
||||
) -> "HttpxBinaryResponseContent":
|
||||
import base64
|
||||
|
||||
|
|
@ -176,10 +222,9 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
raise ValueError("No audio part found in the response")
|
||||
decoded_audio: Final = base64.b64decode(audio_part.data)
|
||||
|
||||
model: Final = getattr(model_response, "model", "")
|
||||
content, content_type = (
|
||||
self._gemini_tts_response_body(decoded_audio, response_format)
|
||||
if self._is_gemini_tts_model(model)
|
||||
if self._is_gemini_tts_model(model) or self._is_gemini_tts_model(model_response.model or "")
|
||||
else (decoded_audio, "audio/mpeg")
|
||||
)
|
||||
response: Final = httpx.Response(
|
||||
|
|
|
|||
|
|
@ -1273,7 +1273,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
|
|||
self.message = _saved_message
|
||||
self.args = (_saved_message,)
|
||||
|
||||
original_headers: Final = getattr(original_exception, "litellm_response_headers", None)
|
||||
original_headers: Final[object] = getattr(original_exception, "litellm_response_headers", None)
|
||||
if original_headers is not None:
|
||||
self.litellm_response_headers = original_headers
|
||||
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@ from pydantic import AnyUrl, TypeAdapter
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import (
|
||||
MCP_CLIENT_TIMEOUT,
|
||||
MCP_MAX_SSE_EVENT_SIZE_BYTES,
|
||||
MCP_NPM_CACHE_DIR,
|
||||
MCP_TOOL_LISTING_MAX_PAGES,
|
||||
MCP_TOOL_LISTING_TIMEOUT,
|
||||
|
|
@ -292,6 +293,7 @@ class _MCPResponseStream(httpx2.AsyncByteStream):
|
|||
class _MCPHTTPClient(httpx2.AsyncClient):
|
||||
cleanup_scope: anyio.CancelScope | None = None
|
||||
cleanup_errors: tuple[Exception, ...] = ()
|
||||
post_failure: asyncio.Future[Exception] | None = None
|
||||
|
||||
def _record_cleanup_error(self, error: Exception) -> None:
|
||||
if self.cleanup_scope is not None and self.cleanup_scope.shield:
|
||||
|
|
@ -319,7 +321,12 @@ class _MCPHTTPClient(httpx2.AsyncClient):
|
|||
response: Final = await super().send(request, stream=stream, auth=auth, follow_redirects=follow_redirects)
|
||||
if request.method == "POST" and response.is_error and response.status_code != 404:
|
||||
await response.aclose()
|
||||
response.raise_for_status()
|
||||
try:
|
||||
response.raise_for_status()
|
||||
except httpx2.HTTPStatusError as error:
|
||||
if self.post_failure is not None and not self.post_failure.done():
|
||||
self.post_failure.set_result(error)
|
||||
raise
|
||||
if stream:
|
||||
response.stream = _MCPResponseStream(
|
||||
cast(httpx2.AsyncByteStream, response.stream), self._record_cleanup_error
|
||||
|
|
@ -641,6 +648,7 @@ class MCPClient:
|
|||
transport_ctx: Final = streamable_http_client(
|
||||
url=self.server_url,
|
||||
http_client=http_client,
|
||||
max_sse_event_size=MCP_MAX_SSE_EVENT_SIZE_BYTES,
|
||||
)
|
||||
return transport_ctx, http_client
|
||||
|
||||
|
|
@ -757,6 +765,8 @@ class MCPClient:
|
|||
read_stream: Final = transport[0]
|
||||
write_stream: Final = transport[1]
|
||||
stream_error: Final[asyncio.Future[Exception]] = asyncio.get_running_loop().create_future()
|
||||
if isinstance(http_client, _MCPHTTPClient):
|
||||
http_client.post_failure = stream_error # rebind-ok: operation-scoped stream error
|
||||
_notify_stream_error(on_stream_error, stream_error)
|
||||
|
||||
session_kwargs: Final = (
|
||||
|
|
@ -823,6 +833,7 @@ class MCPClient:
|
|||
if isinstance(http_client, _MCPHTTPClient):
|
||||
http_client.cleanup_errors = ()
|
||||
http_client.cleanup_scope = None
|
||||
http_client.post_failure = None # rebind-ok: clear operation-scoped stream error
|
||||
await anyio.lowlevel.checkpoint_if_cancelled()
|
||||
if cleanup_scope.cancel_called:
|
||||
raise (
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from mcp.types import Tool as MCPTool
|
|||
from openai.types.chat import ChatCompletionToolParam
|
||||
from openai.types.responses.function_tool_param import FunctionToolParam
|
||||
from openai.types.shared_params.function_definition import FunctionDefinition
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import (
|
||||
|
|
@ -20,6 +21,8 @@ from litellm.constants import (
|
|||
from litellm.types.llms.anthropic import AnthropicMessagesTool
|
||||
from litellm.types.utils import ChatCompletionMessageToolCall
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
########################################################
|
||||
# List MCP Tool functions
|
||||
|
|
@ -200,15 +203,18 @@ async def call_mcp_tool(
|
|||
return tool_result
|
||||
|
||||
|
||||
def _decode_json_arguments(arguments: str) -> object:
|
||||
try:
|
||||
return _DECODED_JSON.validate_python(json.loads(arguments))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
|
||||
|
||||
def _get_function_arguments(function: FunctionDefinition) -> dict:
|
||||
"""Helper to safely get and parse function arguments."""
|
||||
arguments = function.get("arguments", {})
|
||||
if isinstance(arguments, str):
|
||||
try:
|
||||
arguments = json.loads(arguments)
|
||||
except json.JSONDecodeError:
|
||||
arguments = {}
|
||||
return arguments if isinstance(arguments, dict) else {}
|
||||
arguments: Final = function.get("arguments", {})
|
||||
decoded: Final = _decode_json_arguments(arguments) if isinstance(arguments, str) else arguments
|
||||
return decoded if isinstance(decoded, dict) else {}
|
||||
|
||||
|
||||
def transform_openai_tool_call_request_to_mcp_tool_call_request(
|
||||
|
|
|
|||
3
litellm/extraction/__init__.py
Normal file
3
litellm/extraction/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from litellm.extraction.main import aextraction, extraction
|
||||
|
||||
__all__ = ["aextraction", "extraction"]
|
||||
351
litellm/extraction/main.py
Normal file
351
litellm/extraction/main.py
Normal file
|
|
@ -0,0 +1,351 @@
|
|||
import asyncio
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.decisions.call import log_caller_request
|
||||
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tool_fields import DropPolicy, drop_policy
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.extraction.transformation import BaseExtractionConfig
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.types.extraction import (
|
||||
EXTRACTION_STATE_REWRITTEN_METADATA_KEY,
|
||||
ExtractionAnswer,
|
||||
ExtractionBase64Image,
|
||||
ExtractionField,
|
||||
ExtractionOptions,
|
||||
ExtractionRequestBody,
|
||||
ExtractionResponse,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders, is_litellm_owned_kwarg
|
||||
from litellm.utils import (
|
||||
ProviderConfigManager,
|
||||
client, # pyright: ignore[reportUnknownVariableType] # client is an untyped decorator
|
||||
get_model_cost_key,
|
||||
)
|
||||
|
||||
ExtractionFields: TypeAlias = Mapping[str, ExtractionField | Mapping[str, object]]
|
||||
ExtractionImages: TypeAlias = Sequence[str | ExtractionBase64Image | Mapping[str, object]]
|
||||
ExtractionOptionsParam: TypeAlias = ExtractionOptions | Mapping[str, object]
|
||||
|
||||
_REQUEST_ADAPTER: Final[TypeAdapter[ExtractionRequestBody]] = TypeAdapter(ExtractionRequestBody)
|
||||
_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
_HANDLER: Final = BaseLLMHTTPHandler()
|
||||
_LOCAL_KWARGS: Final = frozenset(
|
||||
(
|
||||
*GenericLiteLLMParams.model_fields,
|
||||
"extra_headers",
|
||||
"base_url",
|
||||
"default_headers",
|
||||
"deployment_id",
|
||||
"additional_drop_params",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, repr=False)
|
||||
class _ExtractionCall:
|
||||
model: str
|
||||
requested_model: str
|
||||
custom_llm_provider: str
|
||||
provider_config: BaseExtractionConfig
|
||||
body: Mapping[str, object]
|
||||
api_base: str
|
||||
api_key: str | None
|
||||
logging_obj: LiteLLMLoggingObj | None
|
||||
headers: Mapping[str, str]
|
||||
timeout: float | httpx.Timeout | None
|
||||
litellm_params: Mapping[str, object]
|
||||
client: HTTPHandler | AsyncHTTPHandler | None
|
||||
|
||||
|
||||
def supported_extraction_providers() -> tuple[str, ...]:
|
||||
return tuple(
|
||||
provider.value
|
||||
for provider in LlmProviders
|
||||
if ProviderConfigManager.get_provider_extraction_config(model="", provider=provider) is not None
|
||||
)
|
||||
|
||||
|
||||
def _provider_config(model: str, custom_llm_provider: str) -> BaseExtractionConfig:
|
||||
provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None)
|
||||
provider_config: Final = (
|
||||
None if provider is None else ProviderConfigManager.get_provider_extraction_config(model, provider)
|
||||
)
|
||||
if provider_config is None:
|
||||
raise litellm.BadRequestError(
|
||||
message=(
|
||||
f"Unknown Extraction provider '{custom_llm_provider}'. "
|
||||
f"Supported providers: {', '.join(supported_extraction_providers())}"
|
||||
),
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
)
|
||||
return provider_config
|
||||
|
||||
|
||||
def _request_fields(
|
||||
*,
|
||||
state: str | None,
|
||||
fields: ExtractionFields | None,
|
||||
images: ExtractionImages | None,
|
||||
reference_date: str | None,
|
||||
options: ExtractionOptionsParam | None,
|
||||
) -> Mapping[str, object]:
|
||||
sent: Final[Mapping[str, object | None]] = MappingProxyType(
|
||||
{"state": state, "fields": fields, "images": images, "reference_date": reference_date, "options": options}
|
||||
)
|
||||
return MappingProxyType({name: value for name, value in sent.items() if value is not None})
|
||||
|
||||
|
||||
def _is_unsupported(name: str, value: object, policy: DropPolicy) -> bool:
|
||||
return (
|
||||
value is not None
|
||||
and value is not False
|
||||
and name not in _LOCAL_KWARGS
|
||||
and not is_litellm_owned_kwarg(name)
|
||||
and not policy.drops(name)
|
||||
)
|
||||
|
||||
|
||||
def _unsupported_params(kwargs: Mapping[str, object]) -> tuple[str, ...]:
|
||||
policy: Final = drop_policy(kwargs)
|
||||
return tuple(sorted(name for name, value in kwargs.items() if _is_unsupported(name, value, policy)))
|
||||
|
||||
|
||||
def _prepare_call(
|
||||
*,
|
||||
model: str,
|
||||
request_fields: Mapping[str, object],
|
||||
api_key: str | None,
|
||||
api_base: str | None,
|
||||
timeout: float | httpx.Timeout | None,
|
||||
custom_llm_provider: str | None,
|
||||
extra_headers: Mapping[str, str] | None,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> _ExtractionCall:
|
||||
log_caller_request(kwargs, request_fields)
|
||||
upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
)
|
||||
provider_config: Final = _provider_config(model, provider)
|
||||
unsupported: Final = _unsupported_params(kwargs)
|
||||
if unsupported:
|
||||
raise litellm.UnsupportedParamsError(
|
||||
message=(
|
||||
f"{provider} does not support parameters: {list(unsupported)}, for model={model}. "
|
||||
"To drop these, set `litellm.drop_params=True` or for proxy:\n\n"
|
||||
"`litellm_settings:\n drop_params: true`\n"
|
||||
),
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
try:
|
||||
request: Final = _REQUEST_ADAPTER.validate_python(request_fields)
|
||||
except ValidationError as error:
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Invalid Extraction request: {error}", model=model, llm_provider=provider
|
||||
) from error
|
||||
canonical_model: Final = provider_config.canonical_model(upstream_model)
|
||||
resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base)
|
||||
if resolved_api_base is None:
|
||||
raise litellm.BadRequestError(
|
||||
message=provider_config.missing_api_base_message(provider), model=model, llm_provider=provider
|
||||
)
|
||||
resolved_api_key: Final = provider_config.resolve_api_key(dynamic_api_key or api_key)
|
||||
if resolved_api_key is None:
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Missing API key for Extraction provider '{provider}'", model=model, llm_provider=provider
|
||||
)
|
||||
logging_obj: Final = kwargs.get("litellm_logging_obj")
|
||||
http_client: Final = kwargs.get("client")
|
||||
if isinstance(logging_obj, LiteLLMLoggingObj):
|
||||
logging_obj.update_from_kwargs( # pyright: ignore[reportUnknownMemberType] # Logging takes untyped dicts
|
||||
kwargs=dict(kwargs),
|
||||
model=canonical_model,
|
||||
litellm_params={
|
||||
"litellm_call_id": kwargs.get("litellm_call_id"),
|
||||
"api_base": provider_config.get_complete_url(resolved_api_base, canonical_model),
|
||||
},
|
||||
custom_llm_provider=provider,
|
||||
)
|
||||
return _ExtractionCall(
|
||||
model=canonical_model,
|
||||
requested_model=model,
|
||||
custom_llm_provider=provider,
|
||||
provider_config=provider_config,
|
||||
body=provider_config.transform_extraction_request(canonical_model, request),
|
||||
api_base=resolved_api_base,
|
||||
api_key=resolved_api_key,
|
||||
logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None,
|
||||
headers=MappingProxyType(dict(extra_headers or {})),
|
||||
timeout=timeout,
|
||||
litellm_params=kwargs,
|
||||
client=http_client if isinstance(http_client, HTTPHandler | AsyncHTTPHandler) else None,
|
||||
)
|
||||
|
||||
|
||||
def _state_rewritten_by_guardrail(litellm_params: Mapping[str, object]) -> bool:
|
||||
metadata: Final = litellm_params.get(get_metadata_variable_name_from_kwargs(litellm_params))
|
||||
return (
|
||||
isinstance(metadata, Mapping)
|
||||
and _METADATA_ADAPTER.validate_python(metadata).get(EXTRACTION_STATE_REWRITTEN_METADATA_KEY) is True
|
||||
)
|
||||
|
||||
|
||||
def _without_spans(answers: Mapping[str, ExtractionAnswer]) -> Mapping[str, ExtractionAnswer]:
|
||||
return {name: answer.model_copy(update={"span": None, "spans": None}) for name, answer in answers.items()}
|
||||
|
||||
|
||||
def _formatted_response(response: ExtractionResponse, call: _ExtractionCall) -> ExtractionResponse:
|
||||
spans_index_hidden_text: Final = response.state_text is None and _state_rewritten_by_guardrail(call.litellm_params)
|
||||
formatted: Final = response.model_copy(
|
||||
update={
|
||||
"model": call.requested_model,
|
||||
**({"answers": _without_spans(response.answers)} if spans_index_hidden_text else {}),
|
||||
}
|
||||
)
|
||||
formatted.set_hidden_params(
|
||||
{
|
||||
"model": f"{call.custom_llm_provider}/{call.model}",
|
||||
"custom_llm_provider": call.custom_llm_provider,
|
||||
"provider_response_model": (
|
||||
get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
|
||||
or f"{call.custom_llm_provider}/{call.model}"
|
||||
),
|
||||
}
|
||||
)
|
||||
return formatted
|
||||
|
||||
|
||||
def _upstream_exception(error: Exception, call: _ExtractionCall) -> Exception:
|
||||
if isinstance(error, BaseLLMException) and error.status_code_is_synthesized:
|
||||
return litellm.APIConnectionError(
|
||||
message=f"{call.custom_llm_provider.capitalize()}Exception - {error.message}",
|
||||
llm_provider=call.custom_llm_provider,
|
||||
model=f"{call.custom_llm_provider}/{call.model}",
|
||||
)
|
||||
return litellm.exception_type( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped
|
||||
model=f"{call.custom_llm_provider}/{call.model}",
|
||||
custom_llm_provider=call.custom_llm_provider,
|
||||
original_exception=error,
|
||||
)
|
||||
|
||||
|
||||
async def _asend(call: _ExtractionCall) -> ExtractionResponse:
|
||||
try:
|
||||
response: Final = await _HANDLER.aextraction(
|
||||
model=call.model,
|
||||
custom_llm_provider=call.custom_llm_provider,
|
||||
logging_obj=call.logging_obj,
|
||||
provider_config=call.provider_config,
|
||||
body=call.body,
|
||||
api_base=call.api_base,
|
||||
api_key=call.api_key,
|
||||
headers=call.headers,
|
||||
timeout=call.timeout,
|
||||
litellm_params=call.litellm_params,
|
||||
client=call.client if isinstance(call.client, AsyncHTTPHandler) else None,
|
||||
)
|
||||
except Exception as error:
|
||||
raise _upstream_exception(error, call) from error
|
||||
return _formatted_response(response, call)
|
||||
|
||||
|
||||
def _send(call: _ExtractionCall) -> ExtractionResponse:
|
||||
try:
|
||||
response: Final = _HANDLER.extraction(
|
||||
model=call.model,
|
||||
custom_llm_provider=call.custom_llm_provider,
|
||||
logging_obj=call.logging_obj,
|
||||
provider_config=call.provider_config,
|
||||
body=call.body,
|
||||
api_base=call.api_base,
|
||||
api_key=call.api_key,
|
||||
headers=call.headers,
|
||||
timeout=call.timeout,
|
||||
litellm_params=call.litellm_params,
|
||||
client=call.client if isinstance(call.client, HTTPHandler) else None,
|
||||
)
|
||||
except Exception as error:
|
||||
raise _upstream_exception(error, call) from error
|
||||
return _formatted_response(response, call)
|
||||
|
||||
|
||||
@client
|
||||
async def aextraction(
|
||||
model: str,
|
||||
fields: ExtractionFields,
|
||||
state: str | None = None,
|
||||
images: ExtractionImages | None = None,
|
||||
reference_date: str | None = None,
|
||||
options: ExtractionOptionsParam | None = None,
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
extra_headers: Mapping[str, str] | None = None,
|
||||
**kwargs: object, # kwargs-ok: litellm_params forwarded by @client
|
||||
) -> ExtractionResponse:
|
||||
call: Final = await asyncio.to_thread(
|
||||
partial(
|
||||
_prepare_call,
|
||||
model=model,
|
||||
request_fields=_request_fields(
|
||||
state=state, fields=fields, images=images, reference_date=reference_date, options=options
|
||||
),
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
timeout=timeout,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
extra_headers=extra_headers,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
)
|
||||
return await _asend(call)
|
||||
|
||||
|
||||
@client
|
||||
def extraction(
|
||||
model: str,
|
||||
fields: ExtractionFields,
|
||||
state: str | None = None,
|
||||
images: ExtractionImages | None = None,
|
||||
reference_date: str | None = None,
|
||||
options: ExtractionOptionsParam | None = None,
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
extra_headers: Mapping[str, str] | None = None,
|
||||
**kwargs: object, # kwargs-ok: litellm_params forwarded by @client
|
||||
) -> ExtractionResponse:
|
||||
call: Final = _prepare_call(
|
||||
model=model,
|
||||
request_fields=_request_fields(
|
||||
state=state, fields=fields, images=images, reference_date=reference_date, options=options
|
||||
),
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
timeout=timeout,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
extra_headers=extra_headers,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
return _send(call)
|
||||
|
||||
|
||||
__all__ = ["aextraction", "extraction"]
|
||||
|
|
@ -9,7 +9,7 @@ import asyncio
|
|||
import contextvars
|
||||
import time
|
||||
import uuid as uuid_module
|
||||
from collections.abc import Coroutine
|
||||
from collections.abc import Awaitable, Coroutine
|
||||
from functools import partial
|
||||
from typing import Final, Literal, cast
|
||||
|
||||
|
|
@ -293,6 +293,14 @@ def create_file(
|
|||
raise e
|
||||
|
||||
|
||||
async def _await_retrieved_file(pending: Awaitable[FileObject | OpenAIFileObject]) -> FileObject | OpenAIFileObject:
|
||||
return await pending
|
||||
|
||||
|
||||
async def _await_deleted_file(pending: Awaitable[object]) -> object:
|
||||
return await pending
|
||||
|
||||
|
||||
@client
|
||||
async def afile_retrieve(
|
||||
file_id: str,
|
||||
|
|
@ -325,7 +333,7 @@ async def afile_retrieve(
|
|||
func_with_context: Final = partial(ctx.run, func)
|
||||
init_response: Final = await loop.run_in_executor(None, func_with_context)
|
||||
if asyncio.iscoroutine(init_response):
|
||||
response = await init_response
|
||||
response = await _await_retrieved_file(init_response)
|
||||
else:
|
||||
response = init_response
|
||||
|
||||
|
|
@ -505,7 +513,7 @@ async def afile_delete(
|
|||
func_with_context: Final = partial(ctx.run, func)
|
||||
init_response: Final = await loop.run_in_executor(None, func_with_context)
|
||||
if asyncio.iscoroutine(init_response):
|
||||
response = await init_response
|
||||
response = await _await_deleted_file(init_response)
|
||||
else:
|
||||
response = init_response
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import datetime
|
||||
import traceback
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from typing import TYPE_CHECKING, Any, Final, Optional, cast
|
||||
from typing import TYPE_CHECKING, Final, Optional, cast
|
||||
|
||||
import anyio
|
||||
|
||||
|
|
@ -163,7 +163,7 @@ class FileContentStreamingResponse:
|
|||
merged_hidden_params: Final = cast(
|
||||
"StandardLoggingHiddenParams",
|
||||
{
|
||||
**cast(dict[str, Any], payload.get("hidden_params") or {}),
|
||||
**(payload.get("hidden_params") or {}),
|
||||
**self._hidden_params,
|
||||
},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ class BaseGoogleGenAIGenerateContentStreamingIterator:
|
|||
):
|
||||
self.litellm_logging_obj = litellm_logging_obj
|
||||
self.request_body = request_body
|
||||
self.start_time = datetime.now()
|
||||
self.start_time = litellm_logging_obj.start_time
|
||||
self.collected_chunks: list[bytes] = []
|
||||
self.model = model
|
||||
self.custom_llm_provider = custom_llm_provider
|
||||
|
|
@ -192,6 +192,8 @@ class AsyncGoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateCo
|
|||
async def __anext__(self):
|
||||
try:
|
||||
chunk: Final = await _anext_google_genai_sse_chunk(self.stream_iterator)
|
||||
if self.litellm_logging_obj.completion_start_time is None:
|
||||
self.litellm_logging_obj.update_completion_start_time(completion_start_time=datetime.now())
|
||||
self.collected_chunks.append(chunk)
|
||||
return chunk
|
||||
except StopAsyncIteration:
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ from litellm.harness.context import GatewayTarget
|
|||
from litellm.harness.errors import HarnessError, HarnessInstallFailed
|
||||
from litellm.harness.types import Harness, Usage
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
|
||||
from litellm.types.llms.custom_http import httpxSpecialProvider
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -690,18 +691,21 @@ class ModelEndpoint:
|
|||
async def _invoke_sdk(
|
||||
self,
|
||||
route: str,
|
||||
kwargs: dict[str, Any], # mutable-ok: injects stream_options into the SDK kwargs
|
||||
kwargs: dict[str, Any], # mutable-ok: SDK call kwargs built by _sdk_kwargs
|
||||
) -> object:
|
||||
if route == ROUTE_MESSAGES:
|
||||
return await litellm.anthropic.messages.acreate(**kwargs)
|
||||
if route == ROUTE_CHAT:
|
||||
if kwargs.get("stream"):
|
||||
stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field
|
||||
kwargs["stream_options"] = { # mutable-ok: JSON field sent to litellm.acompletion
|
||||
"include_usage": True,
|
||||
**stream_options,
|
||||
if not kwargs.get("stream"):
|
||||
return await litellm.acompletion(**kwargs)
|
||||
stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field
|
||||
return await litellm.acompletion(
|
||||
**{
|
||||
**kwargs,
|
||||
"stream_options": {"include_usage": True, **stream_options},
|
||||
STREAM_USAGE_INJECTED_FIELD: "include_usage" not in stream_options,
|
||||
}
|
||||
return await litellm.acompletion(**kwargs)
|
||||
)
|
||||
return await litellm.aresponses(**kwargs)
|
||||
|
||||
async def _call_sdk(self, route: str, body: Mapping[str, object]) -> Response:
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ from __future__ import annotations
|
|||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
from litellm.harness.context import SessionContext
|
||||
from litellm.harness.errors import CapabilityUnsupported
|
||||
|
|
@ -38,5 +37,5 @@ class BaseHarnessHandler(ABC):
|
|||
async def resume(self, ctx: SessionContext, native_session_id: str) -> None:
|
||||
"""Continue the runtime's own session on the next turn."""
|
||||
|
||||
async def history(self, ctx: SessionContext) -> list[dict[str, Any]]: # mutable-ok: public history() API shape
|
||||
async def history(self, ctx: SessionContext) -> list[dict[str, object]]: # mutable-ok: public history() API shape
|
||||
raise CapabilityUnsupported(f"Harness.{self.config.harness.name} does not expose history")
|
||||
|
|
|
|||
|
|
@ -12,17 +12,27 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import os
|
||||
from collections import deque
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import HARNESS_STDERR_TAIL_LINES, HARNESS_STREAM_READ_CHUNK_BYTES
|
||||
from litellm.constants import (
|
||||
HARNESS_RESUME_COMMAND_TIMEOUT_SECONDS,
|
||||
HARNESS_STDERR_TAIL_LINES,
|
||||
HARNESS_STREAM_READ_CHUNK_BYTES,
|
||||
)
|
||||
from litellm.harness.context import SessionContext
|
||||
from litellm.harness.errors import HarnessInstallFailed, SandboxError
|
||||
from litellm.harness.errors import HarnessError, HarnessInstallFailed, SandboxError
|
||||
from litellm.harness.handlers.base import BaseHarnessHandler
|
||||
from litellm.harness.sandbox.base import Process, Sandbox
|
||||
from litellm.harness.sandbox.base import CompletedRun, Process, Sandbox
|
||||
from litellm.harness.types import Event
|
||||
from litellm.llms.base_llm.harness.transformation import BaseCLIHarnessConfig, HarnessSessionSetup
|
||||
from litellm.llms.base_llm.harness.transformation import (
|
||||
BaseCLIHarnessConfig,
|
||||
HarnessSessionCopy,
|
||||
HarnessSessionSetup,
|
||||
HarnessTurnError,
|
||||
)
|
||||
from litellm.llms.base_llm.harness.utils import decode_json_line, read_skill_files
|
||||
|
||||
# Link <private_dir>/<dir> to a LiteLLM-owned cache dir so a later session can resume.
|
||||
|
|
@ -60,10 +70,10 @@ async def send_stdin(proc: Process, data: str) -> None:
|
|||
|
||||
|
||||
async def private_dir_for(sandbox: Sandbox) -> str:
|
||||
tempdir = getattr(sandbox, "tempdir", None)
|
||||
tempdir: Final = sandbox.tempdir if hasattr(sandbox, "tempdir") else None
|
||||
if tempdir is None:
|
||||
raise SandboxError(f"{type(sandbox).__name__} has no tempdir(); CLI harnesses need a private config dir")
|
||||
path: str = await tempdir()
|
||||
path: Final = await tempdir()
|
||||
return path
|
||||
|
||||
|
||||
|
|
@ -81,6 +91,52 @@ async def persist_dir(sandbox: Sandbox, link_path: str, cache_subpath: str) -> N
|
|||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ResumeFailed:
|
||||
reason: str
|
||||
|
||||
|
||||
async def bounded_resume_run(
|
||||
sandbox: Sandbox, cmd: Sequence[str], env: Mapping[str, str]
|
||||
) -> CompletedRun | ResumeFailed:
|
||||
try:
|
||||
run: Final = await sandbox.run(cmd, env=env, timeout=HARNESS_RESUME_COMMAND_TIMEOUT_SECONDS)
|
||||
except SandboxError as exc:
|
||||
return ResumeFailed(str(exc))
|
||||
return run if run.exit_code == 0 else ResumeFailed(run.stderr.strip() or f"exit code {run.exit_code}")
|
||||
|
||||
|
||||
async def import_session_copy(sandbox: Sandbox, copy: HarnessSessionCopy, env: Mapping[str, str]) -> str | ResumeFailed:
|
||||
try:
|
||||
await sandbox.write(copy.path, copy.data)
|
||||
except SandboxError as exc:
|
||||
return ResumeFailed(str(exc))
|
||||
imported: Final = await bounded_resume_run(sandbox, copy.import_command, env)
|
||||
return imported if isinstance(imported, ResumeFailed) else copy.session_id
|
||||
|
||||
|
||||
async def resumable_session_id(
|
||||
config: BaseCLIHarnessConfig[object, object],
|
||||
sandbox: Sandbox,
|
||||
setup_env: Mapping[str, str],
|
||||
private_dir: str,
|
||||
native_session_id: str,
|
||||
) -> str | ResumeFailed:
|
||||
export_cmd: Final = config.transform_resume_export_command(native_session_id)
|
||||
if export_cmd is None:
|
||||
return native_session_id
|
||||
exported: Final = await bounded_resume_run(sandbox, export_cmd, setup_env)
|
||||
if isinstance(exported, ResumeFailed):
|
||||
return exported
|
||||
try:
|
||||
copy: Final = config.transform_resume_copy(exported.stdout, private_dir, sandbox.workdir)
|
||||
except HarnessError as exc:
|
||||
return ResumeFailed(str(exc))
|
||||
if copy is None:
|
||||
return native_session_id
|
||||
return await import_session_copy(sandbox, copy, setup_env)
|
||||
|
||||
|
||||
async def copy_skills(sandbox: Sandbox, skills: Sequence[str], skills_root: str) -> None:
|
||||
for skill in skills:
|
||||
name = os.path.basename(os.path.realpath(os.fspath(skill)))
|
||||
|
|
@ -89,13 +145,14 @@ async def copy_skills(sandbox: Sandbox, skills: Sequence[str], skills_root: str)
|
|||
|
||||
|
||||
class CLIHarnessHandler(BaseHarnessHandler):
|
||||
config: BaseCLIHarnessConfig
|
||||
config: BaseCLIHarnessConfig[object, object]
|
||||
|
||||
def __init__(self, config: BaseCLIHarnessConfig) -> None:
|
||||
super().__init__(config)
|
||||
self._private_dir: str | None = None
|
||||
self._setup: HarnessSessionSetup | None = None
|
||||
self._native_id: str | None = None
|
||||
self._pending_resume: str | None = None
|
||||
self._proc: Process | None = None
|
||||
|
||||
async def start(self, ctx: SessionContext) -> None:
|
||||
|
|
@ -119,6 +176,9 @@ class CLIHarnessHandler(BaseHarnessHandler):
|
|||
async def turn(self, ctx: SessionContext, prompt: str) -> AsyncIterator[Event]:
|
||||
if self._setup is None or self._private_dir is None:
|
||||
raise RuntimeError("CLIHarnessHandler.turn() called before start()")
|
||||
if self._pending_resume is not None:
|
||||
self._native_id = await self._prepared_session_id(ctx, self._setup, self._private_dir, self._pending_resume)
|
||||
self._pending_resume = None
|
||||
request = self.config.transform_turn_request(ctx, self._setup, self._private_dir, prompt, self._native_id)
|
||||
argv: Final = list(request.argv) # mutable-ok: Sandbox.exec takes list[str]
|
||||
proc = await ctx.sandbox.exec(argv, env=request.env, cwd=request.cwd)
|
||||
|
|
@ -158,4 +218,16 @@ class CLIHarnessHandler(BaseHarnessHandler):
|
|||
return self._native_id
|
||||
|
||||
async def resume(self, ctx: SessionContext, native_session_id: str) -> None:
|
||||
self._native_id = native_session_id
|
||||
if native_session_id != self._native_id:
|
||||
self._native_id = native_session_id
|
||||
self._pending_resume = native_session_id
|
||||
|
||||
async def _prepared_session_id(
|
||||
self, ctx: SessionContext, setup: HarnessSessionSetup, private_dir: str, saved_id: str
|
||||
) -> str:
|
||||
target: Final = await resumable_session_id(self.config, ctx.sandbox, setup.env, private_dir, saved_id)
|
||||
match target:
|
||||
case ResumeFailed(reason=reason):
|
||||
raise HarnessTurnError(f"could not prepare session {saved_id} for resume: {reason}")
|
||||
case session_id:
|
||||
return session_id
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue