scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 199

Merge main into litellm_chat_prompt_cache_breakpoint_to_cache_control

Browse source 
Take the chaos test's upstream port from allocate_port(), the TQ012 rule main added
This commit is contained in:
mateo-berri 2026-10-17 04:35:02 +00:00
commit b272e193df
797 changed files with 74893 additions and 8283 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
.circleci/config.yml
View file

@ -1293,6 +1293,19 @@ jobs:
uv venv --python 3.12 .venv-mcp-peer
uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
- when:
condition:
equal: [sdk, << parameters.suite >>]
steps:
- run:
name: Install redis-server 7.2.16
command: |
mkdir -p /tmp/redis-bin
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
docker cp "${cid}:/usr/local/bin/redis-server" /tmp/redis-bin/redis-server
docker rm "$cid"
/tmp/redis-bin/redis-server --version | grep -q 'v=7.2.16'
echo 'export PATH="/tmp/redis-bin:$PATH"' >> "$BASH_ENV"
- start_postgres:
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
server_args: "-c max_connections=300 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000"

58
.github/scripts/assert_ci_coverage.py vendored
View file

@ -32,6 +32,7 @@ IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)")
DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*")
COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE)
GLOB_CHARS = frozenset("*?")
DEDICATED_INTEGRATION_WORKFLOWS: Final = MappingProxyType({"image-scan.yml": "tests/integration/database/image/"})
# Trees whose jobs are sharded with no catch-all bucket, so every child that holds
# tests has to be named by some shard or it runs nowhere. A child listed here is
@ -560,6 +561,17 @@ def _integration_github_files(runner: pathlib.Path) -> frozenset[str]:
return frozenset(ast.literal_eval(values))
def _owns(workflow: str, relative_path: str) -> bool:
owned_root: Final = DEDICATED_INTEGRATION_WORKFLOWS.get(workflow)
return owned_root is not None and relative_path.startswith(owned_root)
def _tokens_outside_owner(relative_path: str, workflow_tokens: Mapping[str, frozenset[str]]) -> frozenset[str]:
return frozenset[str]().union(
*(tokens for workflow, tokens in workflow_tokens.items() if not _owns(workflow, relative_path))
)
def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]:
runner: Final = repo_root / "tests/integration/run.py"
if not runner.exists():
@ -611,15 +623,20 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
)
!= 1
)
gha_tokens: Final = _invoked_test_tokens(
scalar
for path in (repo_root / ".github/workflows").glob("*.y*ml")
for scalar in _scalars(yaml.safe_load(path.read_text()), path.name)
workflow_tokens: Final = MappingProxyType(
{
path.name: _invoked_test_tokens(_scalars(yaml.safe_load(path.read_text()), path.name))
for path in (repo_root / ".github/workflows").glob("*.y*ml")
}
)
gha_tokens: Final = frozenset[str]().union(*workflow_tokens.values())
circle_owned: Final = (
frozenset(str(path.relative_to(repo_root)) for path in integration_root.rglob("test_*.py")) - github_files
)
findings: Final = (
tuple(
Finding(path, "integration contract is also selected by GitHub Actions")
for path in paths
Finding(path, "integration file outside GITHUB_FILES is selected by GitHub Actions")
for path in sorted(circle_owned)
if any(_token_covers(token, path) for token in gha_tokens)
)
+ tuple(
@ -632,6 +649,11 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
for path in sorted(github_files)
if not (repo_root / path).is_file()
)
+ tuple(
Finding(path, "GitHub-owned integration contract runs outside a dedicated integration workflow")
for path in sorted(github_files)
if any(_token_covers(token, path) for token in _tokens_outside_owner(path, workflow_tokens))
)
)
browser_commands: Final = tuple(
scalar.value
@ -688,13 +710,9 @@ def main() -> int:
scalars = _all_scalars()
integration_paths, ownership_findings = _integration_ownership()
test_findings = (
_uncovered_tests(
allowlist,
_invoked_selections(scalars)
+ (Selection(included=integration_paths, ignored=frozenset()),),
)
+ ownership_findings
test_findings = _uncovered_tests(
allowlist,
_invoked_selections(scalars) + (Selection(included=integration_paths, ignored=frozenset()),),
)
dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles())
@ -711,13 +729,25 @@ def main() -> int:
test_findings,
"Add each to a job's test path, or list it in .github/ci-coverage-allowlist.yml with a reason.",
)
if ownership_findings:
browser_remedy = (
" CircleCI's browser group owns the integrationCritical Playwright contracts, so no GitHub workflow may "
"select them and the default Playwright config must exclude them."
if any("browser" in finding.detail or "Playwright" in finding.detail for finding in ownership_findings)
else ""
)
_report(
"integration contracts with the wrong CI owner",
ownership_findings,
"Follow tests/integration/AGENTS.md, which says which CI job owns each integration file." + browser_remedy,
)
if dockerfile_findings:
_report(
"Dockerfiles that no CI job builds",
dockerfile_findings,
"Build each in a workflow, or list it in .github/ci-coverage-allowlist.yml with a reason.",
)
if stale_findings or test_findings or dockerfile_findings:
if stale_findings or test_findings or ownership_findings or dockerfile_findings:
return 1
_write(

8
.github/workflows/test-rust.yml vendored
View file

@ -16,6 +16,10 @@ on:
- "litellm/proxy/guardrails/**"
- "litellm/utils.py"
- "litellm/ocr/**"
- "litellm/chat_completions/dispatch.py"
- "litellm/embeddings/dispatch.py"
- "litellm/messages/dispatch.py"
- "litellm/responses/dispatch.py"
- "litellm/llms/base_llm/ocr/**"
- "litellm/llms/custom_httpx/llm_http_handler.py"
- "Makefile"
@ -46,6 +50,10 @@ on:
- "litellm/proxy/guardrails/**"
- "litellm/utils.py"
- "litellm/ocr/**"
- "litellm/chat_completions/dispatch.py"
- "litellm/embeddings/dispatch.py"
- "litellm/messages/dispatch.py"
- "litellm/responses/dispatch.py"
- "litellm/llms/base_llm/ocr/**"
- "litellm/llms/custom_httpx/llm_http_handler.py"
- "Makefile"

14
.github/workflows/test-unit.yml vendored
View file

@ -565,7 +565,6 @@ jobs:
- shard: mcp-elicitation
test-path: |-
--override-ini=pythonpath=tests
tests/unit/experimental_mcp_client/test_mcp_client.py
tests/unit/proxy/_experimental/mcp_server/test_capabilities.py
tests/unit/proxy/_experimental/mcp_server/test_interactions.py
@ -580,17 +579,6 @@ jobs:
job-timeout-minutes: 60
dist: loadscope
- shard: mcp-interactions
test-path: |-
--override-ini=pythonpath=tests
tests/integration/mcp/test_interactions.py
python-version: "3.12"
workers: 4
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
dist: load
- shard: proxy-infra
test-path: |-
tests/unit/proxy/db
@ -708,7 +696,6 @@ jobs:
tests/unit/responses/litellm_completion_transformation
tests/unit/responses/test_additional_tools.py
tests/unit/responses/test_custom_tool_call.py
tests/unit/responses/test_dispatch.py
tests/unit/responses/test_metadata_codex_callback.py
tests/unit/responses/test_no_duplicate_spend_logs.py
tests/unit/responses/test_null_test_fix.py
@ -1463,7 +1450,6 @@ jobs:
tests/unit/caching/test_redis_connection_pool.py \
tests/unit/caching/test_redis_cluster_cache.py \
tests/unit/caching/test_evicted_client_closer.py \
tests/integration/sdk/test_redis_cluster_iam_auth.py \
--tb=short -vv -rs -p no:rerunfailures -p no:pytest-retry --timeout=120 --durations=20 \
--cov=./litellm --cov-report=xml:coverage.xml

4
AGENTS.md
View file

@ -29,7 +29,7 @@ A test must only fail when litellm code changes. Never pin facts we don't own (a
`tests/unit/` mirrors `litellm/` in a parallel path (see `tests/unit/AGENTS.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `AGENTS.md`
End-to-end tests belong in `tests/e2e/`. A rule about one directory's files, like its harness or which CI runs it, goes in that directory's `AGENTS.md`, never only in a README or here. Read a directory's `AGENTS.md` before editing files in it
When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD`
@ -72,7 +72,7 @@ When referencing or running models (coding, QA'ing, writing docs, writing tests,
Always pull before starting any work. The checkout or worktree may be sitting on a stale branch
If you're an internal contributor, when creating a new PR, the typical flow is to branch off the repository's current default branch and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
Internal contributors branch a new PR off the repository's current default branch, with a branch name prefixed with litellm_
Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch

1
README.md
View file

@ -384,6 +384,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [ModelScope (`modelscope`)](https://docs.litellm.ai/docs/providers/modelscope) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Moonshot (`moonshot`)](https://docs.litellm.ai/docs/providers/moonshot) | ✅ | ✅ | ✅ | | | | | | | |
| [Morph (`morph`)](https://docs.litellm.ai/docs/providers/morph) | ✅ | ✅ | ✅ | | | | | | | |
| [Nace (`nace`)](https://console.nace.ai/docs/drex) | | | | | | | | | | |
| [Nebius AI Studio (`nebius`)](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [NLP Cloud (`nlp_cloud`)](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | | | | | | | |
| [Novita AI (`novita`)](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | | | | | | | |

32
backend/Dockerfile
View file

@ -4,6 +4,33 @@ ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2
FROM $UV_IMAGE AS uvbin
# ---------- Rust extension ----------
# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes
FROM $LITELLM_BUILD_IMAGE AS rust-native
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
ENV UV_PYTHON_DOWNLOADS=0
COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \
touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
python -c "import maturin; maturin.build_wheel('/wheels')"
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
@ -54,7 +81,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
uv sync --frozen --no-install-project --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
@ -63,6 +90,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra saml \
--python python3.13
RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

15
ci_cd/generate_model_prices_schema.py
View file

@ -19,6 +19,7 @@ NONNEG_NUMBER: JsonSchema = {"type": "number", "minimum": 0}
NONNEG_INTEGER: JsonSchema = {"type": "integer", "minimum": 0}
BOOLEAN: JsonSchema = {"type": "boolean"}
STRING: JsonSchema = {"type": "string"}
NON_BLANK_STRING: Final[JsonSchema] = {"type": "string", "pattern": r"\S"}
TIME_WINDOW: Final[JsonSchema] = {"type": "string", "pattern": r"^([01]\d|2[0-3]):[0-5]\d-([01]\d|2[0-3]):[0-5]\d$"}
WEEKDAY_PATTERN: Final = (
r"(?i)^(mon|monday|tue|tues|tuesday|wed|wednesday|thu|thur|thurs|thursday|fri|friday|sat|saturday|sun|sunday)$"
@ -103,6 +104,16 @@ OBJECT_KEYS: dict[str, JsonSchema] = {
"type": "object",
"description": "Provider-internal routing hints (e.g. bedrock_invocation_schema).",
},
"token_count_model": {
"type": "object",
"required": ["model"],
"additionalProperties": False,
"properties": {
"model": NON_BLANK_STRING,
"region": NON_BLANK_STRING,
},
"description": "Model that counts tokens for this one, and the AWS region its count API runs in.",
},
}
ARRAY_KEYS: dict[str, JsonSchema] = {
@ -189,6 +200,10 @@ INTEGER_KEYS: dict[str, JsonSchema] = {
**NONNEG_INTEGER,
"description": "Smallest prefix the provider will actually cache; absent means the provider default applies.",
},
"token_count_framing": {
**NONNEG_INTEGER,
"description": "Tokens added once per request to a local count estimate, for the provider's chat framing.",
},
"tpm": {**NONNEG_INTEGER, "description": "Provider default tokens-per-minute limit."},
"rpm": {**NONNEG_INTEGER, "description": "Provider default requests-per-minute limit."},
}

81
cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json
View file

@ -6563,6 +6563,87 @@
],
"title": "litellm_project_model_rate_limit_used_metric",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 447
},
"id": 115,
"panels": [],
"title": "Agent tracing export (needs LITELLM_LENS_URL)",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Request records the Lens exporter delivered to the Lens service against the records it dropped before the service accepted them",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"showPoints": "never",
"spanNulls": false
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 448
},
"id": 116,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(rate(litellm_lens_export_rows_written_total[$__rate_interval]))",
"legendFormat": "written",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(rate(litellm_lens_export_rows_dropped_total[$__rate_interval]))",
"legendFormat": "dropped",
"range": true,
"refId": "B"
}
],
"title": "litellm_lens_export_rows_written and litellm_lens_export_rows_dropped rate",
"type": "timeseries"
}
],
"preload": false,

4
cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/readme.md
View file

@ -1,10 +1,10 @@
# LiteLLM All Prometheus Metrics dashboard
Every `litellm_*` metric family the proxy can expose on `/metrics` (141 families across 97 panels), grouped into rows: proxy traffic, latency, spend and tokens, cache, LLM API deployments, key and team rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, the Redis circuit breaker, the spend log cleanup job, and the `prometheus_system` service callback metrics (per-service latency, request and failure rates, spend update queue sizes). Panel titles are the metric names so you can grep the JSON for the metric you care about
Every `litellm_*` metric family the proxy can expose on `/metrics` (143 families across 98 panels), grouped into rows: proxy traffic, latency, spend and tokens, cache, LLM API deployments, key and team rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, the Redis circuit breaker, the spend log cleanup job, the `prometheus_system` service callback metrics (per-service latency, request and failure rates, spend update queue sizes), project model rate limits, and the agent tracing export to Lens. Panel titles are the metric names so you can grep the JSON for the metric you care about
Import `grafana_dashboard.json` from **Dashboards > New > Import** and pick your Prometheus data source when prompted (the `DS_PROMETHEUS` variable). Counters are plotted as `rate()` over `$__rate_interval`, histograms as p50 / p95 / p99, gauges as the raw value grouped by the most useful label. Every query names the metric exactly as the proxy emits it (counters carry the `_total` suffix the Prometheus client adds), and `tests/unit/integrations/test_prometheus_metric_name_consistency.py` fails if a metric is renamed without updating this dashboard
The first eleven rows need only `callbacks: ["prometheus"]`. The last three rows and the `litellm_admission_*` panels are emitted by other subsystems and stay empty until those are on: the service callback row needs `service_callback: ["prometheus_system"]` in `litellm_settings`, the circuit breaker row needs a Redis cache, the cleanup row needs spend log retention, and admission control needs its middleware enabled. Within the base rows, many panels only fill in once the matching feature is in use: budgets need keys, teams, users or orgs with `max_budget` set, cache panels need caching on, guardrail and MCP panels need those features configured, deployment health needs the router with more than one deployment or a failure to record, and `litellm_in_flight_requests` needs traffic at scrape time. An empty panel for a feature you do not use is expected
The first eleven rows and the project model rate limits row need only `callbacks: ["prometheus"]`. The other rows and the `litellm_admission_*` panels are emitted by other subsystems and stay empty until those are on: the service callback row needs `service_callback: ["prometheus_system"]` in `litellm_settings`, the circuit breaker row needs a Redis cache, the cleanup row needs spend log retention, the agent tracing export row needs `LITELLM_LENS_URL`, and admission control needs its middleware enabled. Within the base rows, many panels only fill in once the matching feature is in use: budgets need keys, teams, users or orgs with `max_budget` set, cache panels need caching on, guardrail and MCP panels need those features configured, deployment health needs the router with more than one deployment or a failure to record, and `litellm_in_flight_requests` needs traffic at scrape time. An empty panel for a feature you do not use is expected
## Pre-requisites

34
gateway/Dockerfile
View file

@ -20,6 +20,33 @@ RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${
make -j"$(nproc)" pgbouncer && \
install -m 0755 pgbouncer /usr/local/bin/pgbouncer
# ---------- Rust extension ----------
# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes
FROM $LITELLM_BUILD_IMAGE AS rust-native
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
ENV UV_PYTHON_DOWNLOADS=0
COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \
touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
python -c "import maturin; maturin.build_wheel('/wheels')"
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
@ -66,11 +93,11 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra bedrock-realtime \
--python python3.13
# Stage 2 — copy source and install the project + workspace members.
# Stage 2 — copy source and install the workspace members.
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
uv sync --frozen --no-install-project --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
@ -79,6 +106,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra bedrock-realtime \
--python python3.13
RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
# PYTHONPATH=/app makes the source tree shadow the installed package, so the
# compiled Rust extension must live next to the source or it is never imported.
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/

23
litellm-proxy-extras/litellm_proxy_extras/migrations/20261015120000_add_environment_variables_revision_trigger/migration.sql Normal file
View file

@ -0,0 +1,23 @@
SET LOCAL lock_timeout = '10s';
CREATE SEQUENCE IF NOT EXISTS "litellm_environment_variables_revision_seq";
GRANT USAGE ON SEQUENCE "litellm_environment_variables_revision_seq" TO PUBLIC;
CREATE OR REPLACE FUNCTION litellm_bump_environment_variables_revision() RETURNS TRIGGER AS $$
DECLARE
next_revision BIGINT := nextval(format('%I.litellm_environment_variables_revision_seq', TG_TABLE_SCHEMA)::regclass);
BEGIN
IF TG_OP = 'UPDATE' THEN
NEW."reload_revision" := GREATEST(next_revision, OLD."reload_revision" + 1);
ELSE
NEW."reload_revision" := next_revision;
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS "litellm_environment_variables_revision" ON "LiteLLM_Config";
CREATE TRIGGER "litellm_environment_variables_revision"
BEFORE INSERT OR UPDATE ON "LiteLLM_Config"
FOR EACH ROW WHEN (NEW."param_name" = 'environment_variables')
EXECUTE PROCEDURE litellm_bump_environment_variables_revision();

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20261016000000_background_interaction_settlement_settled_at_index/migration.sql Normal file
View file

@ -0,0 +1,2 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "idx_background_interaction_settlement_settled_at"
ON "LiteLLM_BackgroundInteractionSettlement" ("settled_at");

1
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -1981,6 +1981,7 @@ model LiteLLM_BackgroundInteractionSettlement {
outcome String?
@@index([claimed_at], map: "idx_background_interaction_settlement_claimed_at")
@@index([settled_at], map: "idx_background_interaction_settlement_settled_at")
}
model LiteLLM_Lens {

2
litellm-rust/crates/gateway-inference/src/messages.rs
View file

@ -24,7 +24,7 @@ const OPENROUTER_API_HEADERS: [&str; 2] = [
"x-openrouter-experimental-metadata",
];
const ANTHROPIC_API_HEADER_PROVIDERS: &str =
"anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter";
"anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter,deepseek";
pub async fn create(
State(gateway): State<Arc<Gateway>>,

2
litellm-rust/crates/gateway-inference/tests/messages.rs
View file

@ -16,7 +16,7 @@ use wiremock::{
#[rstest]
#[case::anthropic("anthropic/test-model", "/v1/messages", true)]
#[case::azure("azure_ai/test-model", "/anthropic/v1/messages", false)]
#[case::deepseek("deepseek/test-model", "/anthropic/v1/messages", false)]
#[case::deepseek("deepseek/test-model", "/anthropic/v1/messages", true)]
#[tokio::test]
async fn messages_reaches_the_provider_and_preserves_json_or_sse(
#[case] model: &str,

37
litellm-rust/crates/inference-messages/src/prepare.rs
View file

@ -124,7 +124,7 @@ fn prepare_provider_request(
url,
body: transformed,
environment,
timeout,
timeout: timeout.or_else(|| shaping.settings.request_timeout()),
api_key: api_key.map(SecretValue::new),
})
}
@ -253,6 +253,41 @@ mod tests {
);
}
#[rstest]
#[case::nothing_configured(None, None, None)]
#[case::configured_setting(None, Some(1800.0), Some(Duration::from_secs(1800)))]
#[case::call_timeout_wins(
Some(Duration::from_secs(30)),
Some(1800.0),
Some(Duration::from_secs(30))
)]
#[case::call_timeout_only(Some(Duration::from_secs(30)), None, Some(Duration::from_secs(30)))]
#[case::unusable_setting_ignored(None, Some(-1.0), None)]
#[case::zero_setting_ignored(None, Some(0.0), None)]
fn timeout_falls_back_to_the_configured_request_timeout(
shaping: MessagesShaping,
#[case] call_timeout: Option<Duration>,
#[case] request_timeout: Option<f64>,
#[case] expected: Option<Duration>,
) {
let prepared = prepare(MessagesCall {
body: body(json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})),
api_key: Some("sk-test".into()),
api_base: Some("https://anthropic.test".into()),
custom_llm_provider: Some("anthropic".into()),
litellm_params: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: call_timeout,
shaping: MessagesShaping {
settings: MessagesSettings { request_timeout, ..shaping.settings },
..shaping
},
})
.unwrap();
assert_eq!(prepared.timeout, expected);
}
fn prepared_body(fields: Value, shaping: MessagesShaping) -> Result<Value, Error> {
prepare(MessagesCall {
body: body(fields),

14
litellm-rust/crates/inference-messages/src/types.rs
View file

@ -60,6 +60,16 @@ pub struct MessagesSettings {
pub reasoning_auto_summary: bool,
#[serde(default)]
pub additional_drop_params: Vec<String>,
#[serde(default)]
pub request_timeout: Option<f64>,
}
impl MessagesSettings {
pub fn request_timeout(&self) -> Option<Duration> {
self.request_timeout
.and_then(|secs| Duration::try_from_secs_f64(secs).ok())
.filter(|timeout| !timeout.is_zero())
}
}
#[cfg(test)]
@ -120,13 +130,15 @@ mod tests {
},
"drop_params": true,
"reasoning_auto_summary": true,
"additional_drop_params": ["metadata.user_id", "thinking"]
"additional_drop_params": ["metadata.user_id", "thinking"],
"request_timeout": 1800.0
}),
MessagesShaping {
settings: MessagesSettings {
drop_params: true,
reasoning_auto_summary: true,
additional_drop_params: vec!["metadata.user_id".to_string(), "thinking".to_string()],
request_timeout: Some(1800.0),
},
capabilities: MessagesModelCapabilities {
supports_reasoning: true,

2
litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs
View file

@ -23,7 +23,7 @@ fn run_chat_completions(
) -> PyResult<Py<PyAny>> {
let host = InferenceHost::new(
call.view()?,
"litellm.rust_bridge.chat_completions.route_host",
"litellm.rust_bridge.host.routes.chat_completions",
);
run_inference::<ChatCompletionsRoute, _>(
py,

10
litellm-rust/crates/python-bridge/src/routes/inference.rs
View file

@ -21,10 +21,7 @@ use serde_json::{Map, Value};
use super::{NativeCall, RequestView};
use crate::{
errors::{RustUpstreamError, route_error_to_pyerr},
marshal::{
RouteOptions, optional_timeout, project_optional_fields, public_response,
python_timeout_seconds,
},
marshal::{RouteOptions, optional_timeout, public_response, python_timeout_seconds},
};
pub(super) struct InferenceHost {
@ -115,10 +112,7 @@ impl InferenceHost {
py: Python<'_>,
arguments: &Bound<'_, PyDict>,
) -> PyResult<Map<String, Value>> {
let names: Vec<String> = py.import(self.module)?.getattr("PARAMETERS")?.extract()?;
project_optional_fields(names.iter().map(String::as_str), |name| {
self.argument(py, arguments, name)
})
self.view.body_params(py, arguments, self.module)
}
pub fn response(&self, py: Python<'_>, response: &impl Serialize) -> PyResult<Py<PyAny>> {

29
litellm-rust/crates/python-bridge/src/routes/messages/host.rs
View file

@ -26,34 +26,9 @@ use crate::{
routes::RequestView,
};
const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.messages.route_host";
const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.messages";
const REQUEST_ERROR_MARKER: &str = "messages_request_error";
const BODY_FIELDS: [&str; 22] = [
"max_tokens",
"metadata",
"stop_sequences",
"stream",
"system",
"temperature",
"thinking",
"tool_choice",
"tools",
"top_k",
"inference_geo",
"top_p",
"mcp_servers",
"context_management",
"compaction",
"container",
"output_format",
"speed",
"output_config",
"cache_control",
"reasoning_effort",
"safeguards",
];
fn merge_headers(
forwarded: Option<Map<String, Value>>,
extra_headers: Option<Map<String, Value>>,
@ -143,7 +118,7 @@ impl MessagesPythonHost {
let model = string("model")?.ok_or_else(|| PyValueError::new_err("model is required"))?;
let messages =
argument("messages")?.ok_or_else(|| PyValueError::new_err("messages is required"))?;
let fields = project_optional_fields(BODY_FIELDS, argument)?;
let fields = self.view.body_params(py, arguments, ROUTE_HOST_MODULE)?;
let body = [
("model".to_string(), Value::String(model.clone())),
("messages".to_string(), from_py(&messages)?),

18
litellm-rust/crates/python-bridge/src/routes/mod.rs
View file

@ -13,6 +13,7 @@ use litellm_callbacks_legacy_python::{LegacyLogging, LoggingOperation, PublicCal
use litellm_host::{call::HostedCompletion, machine::Machine, protocol::Protocol};
use litellm_host_python::{
CallOptions, HookChain, PythonBinding, PythonCallHooks, PythonHostCalls, effective_py_args,
from_py,
};
use litellm_host_python::{Error as HostError, present};
use pyo3::{
@ -20,6 +21,7 @@ use pyo3::{
prelude::*,
types::{PyDict, PyMapping, PyTuple},
};
use serde_json::{Map, Value};
/// The public call as Python bound it: `base` holds the positional arguments by name plus
/// the signature defaults, `kwargs` the caller's keyword dict.
@ -92,6 +94,22 @@ impl RequestView {
present(arguments, self.request(py)?, name)
}
/// The body params `module.body_params` keeps from the keywords the call would send, so
/// each route's Python host decides which params reach the provider.
pub(crate) fn body_params(
&self,
py: Python<'_>,
arguments: &Bound<'_, PyDict>,
module: &str,
) -> PyResult<Map<String, Value>> {
let request = effective_py_args(self.request(py)?, arguments)?;
from_py(
&py.import(module)?
.getattr("body_params")?
.call1((request,))?,
)
}
pub(crate) fn close(&mut self) {
self.0 = None;
}

4
litellm-rust/crates/python-bridge/src/routes/ocr/host.rs
View file

@ -77,7 +77,7 @@ impl OcrPythonHost {
_ => "",
};
let mapped = py
.import("litellm.rust_bridge.ocr.route_host")
.import("litellm.rust_bridge.host.routes.ocr")
.and_then(|module| module.getattr("map_failure"))
.and_then(|map| map.call1((error.value(py), self.view.request(py)?, provider)))
.and_then(|mapped| mapped.extract::<Py<PyBaseException>>().map_err(PyErr::from));
@ -115,7 +115,7 @@ impl PythonBinding for OcrPythonHost {
py: Python<'_>,
response: LiteLLMOcrResponse,
) -> PyResult<Py<PyAny>> {
public_response(py, "litellm.rust_bridge.ocr.route_host", &response)
public_response(py, "litellm.rust_bridge.host.routes.ocr", &response)
}
fn encode_stream_head(

2
litellm-rust/crates/python-bridge/src/routes/responses/mod.rs
View file

@ -21,7 +21,7 @@ use super::{
};
use crate::errors::RustBridgeDeclined;
const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.responses.route_host";
const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.responses";
fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyResult<Py<PyAny>> {
let resolved = call.resolved()?;

11
litellm/__init__.py
View file

@ -207,7 +207,7 @@ _async_failure_callback: List[Union[str, Callable[..., object], "CustomLogger"]]
[]
) # internal variable - async custom callbacks are routed here.
pre_call_rules: List[Callable] = []
post_call_rules: List[Callable] = []
post_call_rules: List[Callable[..., object]] = []
turn_off_message_logging: Optional[bool] = False
standard_logging_payload_excluded_fields: Optional[List[str]] = (
None # Fields to exclude from StandardLoggingPayload before callbacks receive it
@ -393,6 +393,7 @@ key_alias_pattern: str | None = None
enable_gemini_default_thinking_level_low: bool = (
False # opt-in: force thinkingLevel low/minimal for Gemini 3 thinking param mapping
)
gemini_include_thoughts_default: bool = True
####################
logging: bool = True
enable_loadbalancing_on_batch_endpoints: Optional[bool] = None
@ -1176,7 +1177,9 @@ def _build_models_by_provider() -> dict:
"edenai": edenai_models,
"datarobot": datarobot_models,
"vertex_ai": vertex_chat_models
| vertex_code_chat_models
| vertex_text_models
| vertex_code_text_models
| vertex_anthropic_models
| vertex_vision_models
| vertex_language_models
@ -1184,7 +1187,7 @@ def _build_models_by_provider() -> dict:
| vertex_minimax_models
| vertex_moonshot_models
| vertex_zai_models,
"ai21": ai21_models,
"ai21": ai21_models | ai21_chat_models,
"bedrock": bedrock_models | bedrock_converse_models,
"petals": petals_models,
"ollama": ollama_models,
@ -1415,6 +1418,7 @@ from .exceptions import (
InvalidRequestError,
BadRequestError,
ImageFetchError,
MalformedRequestFieldError as MalformedRequestFieldError,
VectorStoreSearchError,
NotFoundError,
PermissionDeniedError,
@ -1690,6 +1694,9 @@ if TYPE_CHECKING:
from .llms.perplexity.decisions.transformation import (
PerplexityDecisionsConfig as PerplexityDecisionsConfig,
)
from .llms.nace.decisions.transformation import (
NaceDecisionsConfig as NaceDecisionsConfig,
)
from .llms.typesafe.decisions.transformation import (
TypeSafeDecisionsConfig as TypeSafeDecisionsConfig,
)

10
litellm/_internal_context.py
View file

@ -37,6 +37,16 @@ REDIS_FAMILIES_METADATA_KEY: Final = "families"
_service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None)
@contextmanager
def internal_sub_call() -> Generator[None]:
"""A nested call the parent call logs and bills, so the parent must fold this call's cost into its own."""
token: Final = is_internal_call.set(True)
try:
yield
finally:
is_internal_call.reset(token)
@contextmanager
def post_response_phase() -> Generator[None]:
"""Work the caller no longer waits for (success callbacks, response-cache writes), including tasks it spawns."""

2
litellm/_lazy_imports_registry.py
View file

@ -162,6 +162,7 @@ LLM_CONFIG_NAMES: Final = (
"HostedVLLMRerankConfig",
"PerplexityDecisionsConfig",
"TypeSafeDecisionsConfig",
"NaceDecisionsConfig",
"OpenRouterDecisionsConfig",
"FireworksAIDecisionsConfig",
"CloudflareDecisionsConfig",
@ -729,6 +730,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"HostedVLLMRerankConfig",
),
"PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
"NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"),
"TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
"OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
"FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),

8
litellm/assistants/main.py
View file

@ -129,6 +129,7 @@ def get_assistants(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
aget_assistants=aget_assistants,
)
@ -312,6 +313,7 @@ def create_assistants(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
create_assistant_data=create_assistant_data,
client=client,
async_create_assistants=async_create_assistants,
@ -461,6 +463,7 @@ def delete_assistant(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
assistant_id=assistant_id,
client=client,
async_delete_assistants=async_delete_assistants,
@ -637,6 +640,7 @@ def create_thread(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
acreate_thread=acreate_thread,
)
@ -783,6 +787,7 @@ def get_thread(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
aget_thread=aget_thread,
)
@ -960,6 +965,7 @@ def add_message(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
a_add_message=a_add_message,
)
@ -1111,6 +1117,7 @@ def get_messages(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
aget_messages=aget_messages,
)
@ -1308,6 +1315,7 @@ def run_thread(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
arun_thread=arun_thread,
event_handler=event_handler,

3
litellm/batches/batch_utils.py
View file

@ -19,7 +19,7 @@ from litellm.llms.vertex_ai.batches.transformation import (
)
from litellm.types.llms.openai import Batch
from litellm.types.utils import ModelInfo, Usage
from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS
from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS
from litellm.utils import token_counter
@ -569,6 +569,7 @@ def extract_file_access_credentials(
# A federated deployment holds no api_key, so without these the fetch that reads a
# finished batch's output has nothing to authenticate with and its cost is never billed.
*sorted(ANTHROPIC_WIF_KWARGS_KEYS),
*sorted(OPENAI_WIF_KWARGS_KEYS),
)
for key in credential_keys:
if key in litellm_params:

4
litellm/batches/main.py
View file

@ -309,6 +309,7 @@ def create_batch(
timeout=timeout,
max_retries=optional_params.max_retries,
_is_async=_is_async,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@ -471,6 +472,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
organization=organization,
timeout=timeout,
max_retries=optional_params.max_retries,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@ -819,6 +821,7 @@ def list_batches(
organization=organization,
timeout=timeout,
max_retries=optional_params.max_retries,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@ -1016,6 +1019,7 @@ def cancel_batch(
organization=organization,
timeout=timeout,
max_retries=optional_params.max_retries,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")

13
litellm/caching/caching.py
View file

@ -25,7 +25,7 @@ from litellm._internal_context import current_service_target, service_target
from litellm._logging import verbose_logger
from litellm.constants import CACHED_STREAMING_CHUNK_DELAY
from litellm.integrations.otel.runtime import phase_span
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper, peek_reasoning_summary_aliases
from litellm.types.caching import *
from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
@ -376,7 +376,7 @@ class Cache:
)
return "".join(f"{field}: {value}" for field, value in scope_values if value is not None)
def get_cache_key(self, **kwargs) -> str:
def get_cache_key(self, **kwargs: object) -> str:
"""
Get the cache key for the given arguments.
@ -410,13 +410,16 @@ class Cache:
if litellm.enable_caching_on_provider_specific_optional_params is True: # feature flagged for now
if kwargs[param] is None:
continue # ignore None params
param_value = kwargs[param]
cache_key += f"{param}: {param_value}"
cache_key += f"{param}: {kwargs[param]}"
if is_semantic_cache:
cache_key += self._get_semantic_cache_tenant_scope(kwargs)
hashed_cache_key = Cache._get_hashed_cache_key(cache_key)
reasoning_summary: Final = peek_reasoning_summary_aliases(kwargs)
reasoning_summary_material: Final = (
"" if reasoning_summary is None else f"reasoning_summary: {reasoning_summary}"
)
hashed_cache_key = Cache._get_hashed_cache_key(cache_key + reasoning_summary_material)
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
verbose_logger.debug(
"\nCreated cache key: %s (source material length: %d)",

11
litellm/caching/caching_handler.py
View file

@ -24,7 +24,7 @@ from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar
from pydantic import ConfigDict, SkipValidation, TypeAdapter, ValidationError
import litellm
from litellm._internal_context import post_response_phase
from litellm._internal_context import is_internal_call, post_response_phase
from litellm._logging import print_verbose, verbose_logger
from litellm.caching import InMemoryCache
from litellm.caching.caching import S3Cache, response_cache_phase
@ -511,7 +511,9 @@ class LLMCachingHandler:
custom_llm_provider=custom_llm_provider,
)
if not _should_defer_streaming_cache_hit_callbacks(cached_result=cached_result):
if not is_internal_call.get() and not _should_defer_streaming_cache_hit_callbacks(
cached_result=cached_result
):
logging_obj.handle_sync_success_callbacks_for_async_calls(
result=cached_result,
start_time=start_time,
@ -820,7 +822,12 @@ class LLMCachingHandler:
start_time (datetime): The start time of the operation.
end_time (datetime): The end time of the operation.
cache_hit (bool): Whether it was a cache hit.
An internal sub-call's hit fires nothing, since the parent call that folds it in logs it.
"""
if is_internal_call.get():
return
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(

6
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -197,7 +197,7 @@ def _drops_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool
return not _supports_audio_input(model, litellm_params)
def _provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
def responses_provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
return MappingProxyType(
{
key: value
@ -1455,7 +1455,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
model_response.id = _upstream_response_id(raw_response.id) or raw_response.id
for key, value in _provider_metadata(raw_response.model_extra).items():
for key, value in responses_provider_metadata(raw_response.model_extra).items():
setattr(model_response, key, value)
# Preserve hidden params from the ResponsesAPIResponse, especially the headers
@ -2139,7 +2139,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
from litellm.responses.utils import ResponseAPILoggingUtils
usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response_data.get("usage"))
provider_metadata: Final = _provider_metadata(response_data)
provider_metadata: Final = responses_provider_metadata(response_data)
served_service_tier: Final = response_data.get("service_tier")
return ModelResponseStream(
choices=[

11
litellm/constants.py
View file

@ -57,6 +57,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
"model_group_retry_policy",
"model_group_alias",
"enable_weighted_failover",
"enable_mid_stream_fallback_continuation",
"enable_tag_filtering",
"tag_routing_prefix",
"optional_pre_call_checks",
@ -146,6 +147,7 @@ HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
# Streams exceeding this duration are terminated with a Timeout error.
# None (default) = no limit. Set env var to a number of seconds to enable globally.
LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = get_env_float_or_none("LITELLM_MAX_STREAMING_DURATION_SECONDS")
MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0
# Maximum number of base64 characters to keep in logging payloads.
# Data URIs exceeding this are replaced with a size placeholder.
@ -380,6 +382,7 @@ REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES: Final = get_env_int_or_none("REALTIME
REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float(
"REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0
)
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0)
# RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code
WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
@ -388,7 +391,7 @@ DEEPGRAM_DEFAULT_API_BASE: Final = "https://api.deepgram.com/v1"
NADIR_DEFAULT_API_BASE: Final = "https://api.getnadir.com/v1"
DEEPGRAM_LISTEN_DEFAULT_MODEL: Final = "nova-3"
BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_session_update"
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_opening_message"
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: Final = "litellm.bedrock_realtime.committed_failure"
BEDROCK_REALTIME_SDK_DISTRIBUTION: Final = "aws-sdk-bedrock-runtime"
@ -972,7 +975,7 @@ DEFAULT_CHAT_COMPLETION_PARAM_VALUES: Final = {
"context_management": None,
}
openai_compatible_endpoints: Final[list] = [
openai_compatible_endpoints: Final[list[str]] = [
"api.perplexity.ai",
"api.endpoints.anyscale.com/v1",
"api.deepinfra.com/v1/openai",
@ -1632,6 +1635,7 @@ INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
CALLER_SESSION_ID_METADATA_KEY: Final = "litellm_caller_session_id"
LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated"
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = (
"Truncation is a DB storage safeguard. "
@ -1708,6 +1712,7 @@ AZURE_SPEECH_FAST_TRANSCRIPTION_PATH: Final = "/speechtotext/transcriptions:tran
AZURE_SPEECH_STT_DOMAIN: Final = "stt.speech.microsoft.com"
AZURE_SPEECH_COGNITIVE_SERVICES_DOMAIN: Final = "api.cognitive.microsoft.com"
AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER: Final = "Ocp-Apim-Subscription-Key"
AWS_ERROR_TYPE_HEADER: Final = "x-amzn-errortype"
AZURE_SPEECH_SHORT_AUDIO_MODEL: Final = "short-audio"
AZURE_SPEECH_BATCH_MODEL: Final = "batch-transcription"
AZURE_SPEECH_FAST_TRANSCRIPTION_MODEL: Final = "fast-transcription"
@ -1858,6 +1863,7 @@ SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = get_env_int("SPEND_COUNTER_RESEED_L
PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, get_env_int("PROXY_DB_LOOKUP_MAX_CONCURRENCY", 25))
PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, get_env_float("PROXY_DB_LOOKUP_DEADLINE_SECONDS", 10.0))
PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, get_env_float("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", 30.0))
TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS: Final = 30
DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = get_env_int("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60) # 1 minute
PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597)
RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, get_env_int("RESET_BUDGET_JOB_BATCH_SIZE", 500))
@ -1897,6 +1903,7 @@ _background_interaction_cost_polling_env: Final = os.getenv(
"BACKGROUND_INTERACTION_COST_POLLING_ENABLED", "true"
).lower()
BACKGROUND_INTERACTION_COST_POLLING_ENABLED: Final = _background_interaction_cost_polling_env == "true"
BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS: Final = 300.0
PROXY_BUDGET_RESCHEDULER_MAX_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605)
PROXY_BATCH_WRITE_AT: Final = get_env_int("PROXY_BATCH_WRITE_AT", 10) # in seconds, increased from 10
PROXY_CONFIG_RELOAD_INTERVAL_SECONDS: Final = get_env_int("PROXY_CONFIG_RELOAD_INTERVAL_SECONDS", 30)

1
litellm/containers/README.md
View file

@ -120,6 +120,7 @@ class AzureContainerConfig(BaseContainerConfig):
self,
headers: dict,
api_key: Optional[str] = None,
litellm_params: Optional[GenericLiteLLMParams] = None,
) -> dict:
"""Azure uses api-key header instead of Bearer token."""
import litellm

84
litellm/cost_calculator.py
View file

@ -86,6 +86,9 @@ from litellm.llms.openai.cost_calculation import (
from litellm.llms.openai.cost_calculation import (
video_output_cost_per_second,
)
from litellm.llms.openai.transcriptions.gpt_transformation import (
OpenAIGPTAudioTranscriptionConfig,
)
from litellm.llms.perplexity.cost_calculator import (
cost_per_token as perplexity_cost_per_token,
)
@ -325,6 +328,15 @@ def _transcription_usage_has_token_details(
return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool:
if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
return False
transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI
)
return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig)
OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"]
@ -1426,32 +1438,15 @@ def completion_cost(
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
) -> float:
"""
Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm.
Return the USD cost of a LiteLLM call.
Parameters:
completion_response (litellm.ModelResponses): [Required] The response received from a LiteLLM completion request.
Pass `completion_response` to price a response LiteLLM returned. Without one, pass `model` plus
`messages` / `prompt` and `completion`, and token counts are computed from the text. Prices come from
`litellm.model_cost` unless `custom_cost_per_token` or `custom_cost_per_second` is given. `total_time`
is in milliseconds and only matters for per-second pricing such as Replicate and SageMaker.
[OPTIONAL PARAMS]
model (str): Optional. The name of the language model used in the completion calls
prompt (str): Optional. The input prompt passed to the llm
completion (str): Optional. The output completion text from the llm
total_time (float, int): Optional. (Only used for Replicate LLMs) The total time used for the request in seconds
custom_cost_per_token: Optional[CostPerToken]: the cost per input + output token for the llm api call.
custom_cost_per_second: Optional[float]: the cost per second for the llm api call.
Returns:
float: The cost in USD dollars for the completion based on the provided parameters.
Exceptions:
Raises exception if model not in the litellm model cost map. Register model, via custom pricing or PR - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json
Note:
- If completion_response is provided, the function extracts token information and the model name from it.
- If completion_response is not provided, the function calculates token counts based on the model and input text.
- The cost is calculated based on the model, prompt tokens, and completion tokens.
- For certain models containing "togethercomputer" in the name, prices are based on the model size.
- For un-mapped Replicate models, the cost is calculated based on the total time used for the request.
Raises if no candidate model name resolves to pricing. Register the model via `litellm.register_model`
or add it to model_prices_and_context_window.json.
"""
try:
call_type = infer_call_type(call_type, completion_response) or "completion"
@ -1739,6 +1734,12 @@ def completion_cost(
audio_transcription_file_duration = _hidden.get(
"audio_transcription_duration",
getattr(completion_response, "duration", 0.0),
) or (
completion_response.usage.seconds
if isinstance(completion_response, TranscriptionResponse)
and isinstance(completion_response.usage, TranscriptionUsageDurationObject)
and _bills_provider_transcription_seconds(requested_model, custom_llm_provider)
else 0.0
)
elif call_type in _RERANK_CALL_TYPES:
if completion_response is not None and isinstance(completion_response, RerankResponse):
@ -2048,7 +2049,7 @@ def completion_cost(
)
if idx == len(potential_model_names) - 1:
raise e
raise Exception(f"Unable to calculat cost for received potential model names - {potential_model_names}")
raise Exception(f"Unable to calculate cost for received potential model names - {potential_model_names}")
except Exception as e:
raise e
@ -2063,13 +2064,20 @@ def get_response_cost_from_hidden_params(
additional_headers: Final = _hidden_params_dict.get("additional_headers", {})
if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers:
response_cost: Final = additional_headers["llm_provider-x-litellm-response-cost"]
if response_cost is None:
return None
return float(additional_headers["llm_provider-x-litellm-response-cost"])
return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"])
return None
def _billable_header_cost(value: object) -> float | None:
if not isinstance(value, (str, int, float)):
return None
try:
cost: Final = float(value)
except ValueError:
return None
return cost if math.isfinite(cost) and cost >= 0 else None
def response_cost_calculator(
response_object: ModelResponse
| EmbeddingResponse
@ -3008,6 +3016,7 @@ _RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "re
class _ResponsesWsEventResponse(LiteLLMBaseModel):
id: str | None = None
usage: Mapping[str, object] | None = None
service_tier: str | None = None
@ -3059,6 +3068,23 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
)
return ResponsesWebSocketTokenUsageProcessor.combine_usage_objects(list(collected_usage_objects))
@staticmethod
def response_id_from_responses_ws_results(results: Sequence[Mapping[str, object]]) -> str | None:
response_ids: Final = tuple(
response.id for _, response in _billable_responses_ws_events(results) if response.id
)
return response_ids[-1] if response_ids else None
@staticmethod
def create_logging_object(results: Sequence[Mapping[str, object]]) -> LiteLLMRealtimeStreamLoggingObject:
tier_partition: Final = ResponsesWebSocketTokenUsageProcessor.partition_results_by_service_tier(results)
return LiteLLMRealtimeStreamLoggingObject(
id=ResponsesWebSocketTokenUsageProcessor.response_id_from_responses_ws_results(results),
usage=ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(results),
results=cast(OpenAIRealtimeStreamList, list(results)), # cast-ok: raw WS frames, unvalidated field
service_tier=next(iter(tier_partition)) if len(tier_partition) == 1 else None,
)
_TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"

10
litellm/decisions/call.py
View file

@ -57,7 +57,7 @@ from litellm.types.decisions import (
UnsupportedDecisionsRequest,
)
from litellm.types.utils import LlmProviders, ProviderSpecificHeader
from litellm.utils import ProviderConfigManager
from litellm.utils import ProviderConfigManager, get_model_cost_key
SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]]
@ -458,7 +458,7 @@ def prepare_call(
) from error
try:
resolved_api_base, resolved_api_key = provider_config.resolve_connection(
resolved_api_base, static_api_key = provider_config.resolve_connection(
dynamic_api_base or api_base, dynamic_api_key or api_key
)
except ValueError as error:
@ -469,6 +469,7 @@ def prepare_call(
model=model,
llm_provider=provider,
)
resolved_api_key: Final = provider_config.resolve_credential(static_api_key, resolved_api_base, kwargs)
if resolved_api_key is None and provider_config.api_key_required:
raise litellm.AuthenticationError(
message=f"Missing API key for Decisions provider '{provider}'",
@ -666,7 +667,10 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci
{
"model": f"{call.custom_llm_provider}/{call.model}",
"custom_llm_provider": call.custom_llm_provider,
"provider_response_model": f"{call.custom_llm_provider}/{call.model}",
"provider_response_model": (
get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
or f"{call.custom_llm_provider}/{call.model}"
),
**({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}),
}
)

31
litellm/decisions/main.py
View file

@ -1,5 +1,7 @@
import asyncio
import warnings
from collections.abc import Mapping
from functools import partial
from typing import Final
import httpx
@ -104,19 +106,22 @@ async def adecisions(
videos: DecisionsVideoClips | None = None,
**kwargs: object,
) -> OpenAIDecisionResponse | DecisionsResponse:
call: Final = _prepare_decisions_call(
model=model,
decision_input=input,
questions=questions,
safety_identifier=safety_identifier,
audio=audio,
videos=videos,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
call: Final = await asyncio.to_thread(
partial(
_prepare_decisions_call,
model=model,
decision_input=input,
questions=questions,
safety_identifier=safety_identifier,
audio=audio,
videos=videos,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
)
)
return await asend(call)

6
litellm/exceptions.py
View file

@ -313,6 +313,10 @@ class VectorStoreSearchError(BadRequestError):
)
class MalformedRequestFieldError(BadRequestError):
pass
class UnprocessableEntityError(openai.UnprocessableEntityError):
def __init__(
self,
@ -1217,6 +1221,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
num_retries: int | None = None,
generated_content: str = "",
is_pre_first_chunk: bool = False,
emitted_disqualifying_content: bool = False,
):
original_status: Final = getattr(original_exception, "status_code", None)
self.status_code = int(original_status) if original_status is not None else 503
@ -1229,6 +1234,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
self.num_retries = num_retries
self.generated_content = generated_content
self.is_pre_first_chunk = is_pre_first_chunk
self.emitted_disqualifying_content = emitted_disqualifying_content
# Create a response if one wasn't provided
if response is None:

14
litellm/experimental_mcp_client/client.py
View file

@ -498,6 +498,7 @@ class MCPClient:
elicitation_callback: "ElicitationCallback | None" = None,
logging_callback: Callable | None = None,
protocol_version: MCPUpstreamProtocol = "auto",
upstream_transport: httpx2.AsyncBaseTransport | None = None,
):
self.protocol_version: MCPUpstreamProtocol = TypeAdapter[MCPUpstreamProtocol](
MCPUpstreamProtocol
@ -524,6 +525,7 @@ class MCPClient:
self._sampling_callback: Callable | None = sampling_callback
self._elicitation_callback: ElicitationCallback | None = elicitation_callback
self._logging_callback: Callable | None = logging_callback
self._upstream_transport: httpx2.AsyncBaseTransport | None = upstream_transport
self._session_callbacks_override: _SessionCallbackMap | None = None
# handle the basic auth value if provided
if auth_value:
@ -610,7 +612,11 @@ class MCPClient:
return stdio_client(server_params), None
if self.transport_type == MCPTransport.sse:
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
httpx_client_factory = (
self._create_httpx_client_factory()
if self._upstream_transport is None
else self._create_httpx_client_factory(transport=self._upstream_transport)
)
return (
sse_client(
url=self.server_url,
@ -622,7 +628,11 @@ class MCPClient:
)
# HTTP transport (default)
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
httpx_client_factory = (
self._create_httpx_client_factory()
if self._upstream_transport is None
else self._create_httpx_client_factory(transport=self._upstream_transport)
)
verbose_logger.debug("litellm headers for streamable_http_client: %s", headers)
http_client = httpx_client_factory(
headers=headers,

6
litellm/files/main.py
View file

@ -259,6 +259,7 @@ def create_file(
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
create_file_data=_create_file_request,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -388,6 +389,7 @@ def file_retrieve(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=optional_params.model_dump(exclude_none=True),
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -569,6 +571,7 @@ def file_delete(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -775,6 +778,7 @@ def file_list(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=optional_params.model_dump(exclude_none=True),
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -972,6 +976,7 @@ def file_content(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -1093,6 +1098,7 @@ def file_content_streaming(
organization=openai_creds.organization,
chunk_size=chunk_size,
client=client if isinstance(client, (OpenAI, AsyncOpenAI)) else None,
litellm_params=optional_params.model_dump(exclude_none=True),
)
elif custom_llm_provider == LlmProviders.VERTEX_AI.value:
if not _is_async:

4
litellm/fine_tuning/main.py
View file

@ -239,6 +239,7 @@ def create_fine_tuning_job(
client=kwargs.get(
"client", None
), # note, when we add this to `GenericLiteLLMParams` it impacts a lot of other tests + linting
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":
@ -436,6 +437,7 @@ def cancel_fine_tuning_job(
max_retries=optional_params.max_retries,
_is_async=_is_async,
client=kwargs.get("client", None),
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":
@ -591,6 +593,7 @@ def list_fine_tuning_jobs(
max_retries=optional_params.max_retries,
_is_async=_is_async,
client=kwargs.get("client", None),
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":
@ -730,6 +733,7 @@ def retrieve_fine_tuning_job(
max_retries=optional_params.max_retries,
_is_async=_is_async,
client=kwargs.get("client", None),
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":

9
litellm/images/main.py
View file

@ -39,6 +39,7 @@ from openai.types.audio.transcription_create_params import FileTypes
# BFL handlers
from litellm.llms.black_forest_labs.image_edit.handler import bfl_image_edit
from litellm.llms.black_forest_labs.image_generation.handler import bfl_image_generation
from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config
from litellm.main import (
azure_chat_completions,
base_llm_aiohttp_handler,
@ -505,6 +506,7 @@ def image_generation(
aimg_generation=aimg_generation,
client=client,
headers=headers,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "bedrock":
if model is None:
@ -681,10 +683,13 @@ def image_variation(
api_base = provider_config.get_api_base(litellm_params.get("api_base", None))
if image_variation_provider == LITELLM_IMAGE_VARIATION_PROVIDERS.OPENAI:
if api_key is None:
raise ValueError("API key is required for OpenAI image variations")
if api_base is None:
raise ValueError("API base is required for OpenAI image variations")
if api_key is None and (
resolve_openai_workload_identity_config(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
is None
):
raise ValueError("API key is required for OpenAI image variations")
response = openai_image_variations.image_variations(
model_response=model_response,

19
litellm/integrations/custom_guardrail.py
View file

@ -695,9 +695,7 @@ class CustomGuardrail(CustomLogger):
supported_event_hooks: list[GuardrailEventHooks],
) -> None:
allowed_hooks: Final = frozenset(supported_event_hooks) | (
frozenset((GuardrailEventHooks.logging_only,))
if self.uses_apply_guardrail_interface() and not self.use_native_lifecycle_hooks
else frozenset()
frozenset((GuardrailEventHooks.logging_only,)) if self.accepts_logging_only_mode() else frozenset()
)
def _validate_event_hook_list_is_in_supported_event_hooks(
@ -869,8 +867,13 @@ class CustomGuardrail(CustomLogger):
return True
return False
def uses_apply_guardrail_interface(self) -> bool:
return type(self).apply_guardrail is not CustomGuardrail.apply_guardrail
@classmethod
def uses_apply_guardrail_interface(cls) -> bool:
return cls.apply_guardrail is not CustomGuardrail.apply_guardrail
@classmethod
def accepts_logging_only_mode(cls) -> bool:
return cls.uses_apply_guardrail_interface() and not cls.use_native_lifecycle_hooks
@classmethod
def supports_logging_only_scope(cls) -> bool:
@ -1029,7 +1032,7 @@ class CustomGuardrail(CustomLogger):
)
return kwargs, result
litellm_params: Final = kwargs.get("litellm_params") or {}
scratch_metadata: Final = {
scratch_metadata: Final[dict[str, object]] = { # mutable-ok: apply_guardrail records its verdict here
key: value
for key, value in (litellm_params.get("metadata") or {}).items()
if key != "standard_logging_guardrail_information"
@ -1083,11 +1086,11 @@ class CustomGuardrail(CustomLogger):
async def _scan_logged_call(
self,
kwargs: dict, # mutable-ok: CustomLogger.async_logging_hook contract
kwargs: dict[str, object], # mutable-ok: CustomLogger.async_logging_hook contract
response: object | None,
translation: "BaseTranslation",
output_translation: "BaseTranslation",
scratch_metadata: dict, # mutable-ok: apply_guardrail records its verdict into request metadata
scratch_metadata: dict[str, object], # mutable-ok: apply_guardrail records its verdict into request metadata
) -> None:
scratch_fields: Final = self._copy_scratch_request_fields(kwargs)
scratch_input, scratch_tools = scratch_fields or (None, None)

2
litellm/integrations/gitlab/gitlab_prompt_manager.py
View file

@ -219,7 +219,7 @@ class GitLabTemplateManager:
elif value.replace(".", "").isdigit():
try:
result[key] = float(value)
except Exception:
except ValueError:
result[key] = value
else:
result[key] = value.strip("\"'")

71
litellm/integrations/langfuse/langfuse.py
View file

@ -103,6 +103,43 @@ def _is_session_header_trace(trace_id: object, session_id: object, proxy_server_
)
def langfuse_trace_metadata(metadata: object, proxy_server_request: object) -> Mapping[str, object]:
entries: Final = _object_mapping(metadata) or _NO_METADATA
request: Final = _object_mapping(proxy_server_request)
headers: Final = _object_mapping(request.get("headers")) if request is not None else None
if headers is None:
return entries
header_entries: Final = {
key.removeprefix("langfuse_"): value
for key, value in _widened_items(headers)
if isinstance(key, str) and key.startswith("langfuse_")
}
return MappingProxyType({**entries, **header_entries})
def langfuse_trace_id(
metadata: Mapping[str, object],
standard_trace_id: object,
litellm_trace_id: object,
litellm_call_id: object,
proxy_server_request: object,
) -> object:
existing_trace_id: Final = metadata.get("existing_trace_id")
if existing_trace_id is not None:
return existing_trace_id
metadata_trace_id: Final = metadata.get("trace_id")
trace_id: Final = (
metadata_trace_id
if metadata_trace_id is not None
else standard_trace_id
if standard_trace_id is not None
else litellm_trace_id or litellm_call_id
)
if _is_session_header_trace(trace_id, metadata.get("session_id"), proxy_server_request):
return litellm_call_id or trace_id
return trace_id
class _UsageObject(Protocol):
"""Token-count surface the Langfuse logger reads off a response usage payload."""
@ -680,33 +717,19 @@ class LangFuseLogger:
# Add default langfuse tags
tags = self.add_default_langfuse_tags(tags=tags, kwargs=kwargs, metadata=metadata)
call_trace_id: Final = langfuse_trace_id(
metadata=clean_metadata,
standard_trace_id=standard_logging_object.get("trace_id")
if standard_logging_object is not None
else None,
litellm_trace_id=kwargs.get("litellm_trace_id"),
litellm_call_id=litellm_call_id,
proxy_server_request=litellm_params.get("proxy_server_request"),
)
session_id: Final = clean_metadata.pop("session_id", None)
trace_name = cast(str | None, clean_metadata.pop("trace_name", None))
trace_id = clean_metadata.pop("trace_id", None)
# Use standard_logging_object.trace_id if available (when trace_id from metadata is None)
# This allows standard trace_id to be used when provided in standard_logging_object
if trace_id is None and standard_logging_object is not None:
trace_id = cast(str | None, standard_logging_object.get("trace_id"))
# Fallback to litellm_call_id if no trace_id found
if trace_id is None:
trace_id = kwargs.get("litellm_trace_id") or litellm_call_id
clean_metadata.pop("trace_id", None)
existing_trace_id: Final = clean_metadata.pop("existing_trace_id", None)
# If existing_trace_id is provided, use it as the trace_id to return
# This allows continuing an existing trace while still returning the correct trace_id
if existing_trace_id is not None:
trace_id = existing_trace_id
call_trace_id: Final = (
litellm_call_id or trace_id
if existing_trace_id is None
and _is_session_header_trace(trace_id, session_id, litellm_params.get("proxy_server_request"))
else trace_id
)
if call_trace_id != trace_id:
verbose_logger.debug(
"Langfuse: trace_id %s came from a session header; using call id %s so each call gets its own trace",
trace_id,
call_trace_id,
)
requested_trace_keys: Final = _as_steering_key_sequence(clean_metadata.pop("update_trace_keys", ()))
update_trace_keys: Final = (
requested_trace_keys if _as_steering_flag(litellm.langfuse_enable_update_trace_keys) else ()

11
litellm/integrations/prometheus.py
View file

@ -27,6 +27,7 @@ from litellm.constants import (
PROXY_REJECTED_BEFORE_ROUTING_KEY,
)
from litellm.exceptions import (
MalformedRequestFieldError,
validate_rate_limit_category,
validate_rate_limit_type,
)
@ -3104,7 +3105,11 @@ class PrometheusLogger(CustomLogger):
# On LiteLLM-side rejects (no deployment picked), route request_kwargs["model"]
# into requested_model and leave deployment-scoped labels empty.
deployment_selected: Final = bool(model_id) and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
deployment_selected: Final = (
bool(model_id)
and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
and not isinstance(exception, MalformedRequestFieldError)
)
if deployment_selected:
label_litellm_model_name = litellm_model_name
label_model_id = model_id
@ -3368,7 +3373,7 @@ class PrometheusLogger(CustomLogger):
label_context=label_context,
)
if remaining_requests:
if isinstance(remaining_requests, int):
"""
"model_group",
"api_provider",
@ -3382,7 +3387,7 @@ class PrometheusLogger(CustomLogger):
)
self.litellm_remaining_requests_metric.labels(**_labels).set(remaining_requests)
if remaining_tokens:
if isinstance(remaining_tokens, int):
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_remaining_tokens_metric"),
enum_values=enum_values,

289
litellm/interactions/background_cost_polling.py
View file

@ -32,19 +32,24 @@ task to settle. A ``BackgroundSettlementStore`` makes the pending settlement
durable across processes: the create registers the request context that
billing needs (never provider credentials), the settlement is claimed
exactly once through the store, and a delete on any replica rebuilds the
billing context from the store when the poll task is not local. Rows left
unclaimed by a process that died are resumed at startup. The default store is
in-memory, which keeps the SDK and single-process behavior unchanged; the
proxy installs a database-backed one.
billing context from the store when the poll task is not local. Every row
still unsettled when a process starts is resumed there, including one whose
settler died between its claim and its outcome: the database-backed store
treats a claim as a lease that its settler renews until it records the
outcome, so such a row is claimable again once the lease runs out. The
default store is in-memory, which keeps the SDK and single-process behavior
unchanged; the proxy installs a database-backed one.
"""
import asyncio
import itertools
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass, field
from datetime import datetime, timezone
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias
from openai import APIStatusError
from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError
from pydantic_core import PydanticSerializationError, to_jsonable_python
@ -54,10 +59,12 @@ from litellm.constants import (
BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS,
BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS,
BACKGROUND_INTERACTION_COST_POLLING_ENABLED,
BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS,
)
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
from litellm.types.interactions import InteractionsAPIResponse
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router import CredentialLiteLLMParams
from litellm.types.utils import CustomPricingLiteLLMParams
if TYPE_CHECKING:
@ -101,6 +108,7 @@ class PendingBackgroundInteraction:
custom_llm_provider: str
create_context: BackgroundInteractionCreateContext
created_at: datetime
claim_expires_at: datetime | None = None
class BackgroundSettlementStore(Protocol):
@ -112,9 +120,11 @@ class BackgroundSettlementStore(Protocol):
async def claim(self, interaction_id: str) -> bool: ...
async def renew_claim(self, interaction_id: str) -> None: ...
async def record_outcome(self, interaction_id: str, outcome: SettlementOutcome) -> None: ...
async def unclaimed(self) -> Sequence[PendingBackgroundInteraction]: ...
async def unsettled(self) -> Sequence[PendingBackgroundInteraction]: ...
@dataclass(frozen=True, slots=True)
@ -145,10 +155,13 @@ class InMemoryBackgroundSettlementStore:
self._rows[interaction_id] = None
return True
async def renew_claim(self, interaction_id: str) -> None:
return None
async def record_outcome(self, interaction_id: str, outcome: SettlementOutcome) -> None:
return None
async def unclaimed(self) -> Sequence[PendingBackgroundInteraction]:
async def unsettled(self) -> Sequence[PendingBackgroundInteraction]:
return tuple(row for row in self._rows.values() if row is not None)
@ -169,11 +182,15 @@ class BackgroundInteractionPollContext:
interaction_id: str
custom_llm_provider: str
logging_obj: "LiteLLMLoggingObj"
api_key: str | None = None
api_base: str | None = None
credentials: CredentialLiteLLMParams = field(default_factory=CredentialLiteLLMParams)
deployment_id: str | None = None
model_group: str | None = None
initial_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS
max_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS
timeout_seconds: float = BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS
start_delay_seconds: float = 0.0
outcome_write_timeout_seconds: float = BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS / 2
claim_renewal_interval_seconds: float = BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS / 3
store: BackgroundSettlementStore = field(default_factory=InMemoryBackgroundSettlementStore)
resumed: bool = False
@ -184,15 +201,41 @@ FetchInteraction: TypeAlias = Callable[[BackgroundInteractionPollContext], Await
async def fetch_background_interaction(context: BackgroundInteractionPollContext) -> InteractionsAPIResponse:
from litellm.interactions import aget
credentials: Final = context.credentials
return await aget(
interaction_id=context.interaction_id,
custom_llm_provider=context.custom_llm_provider,
api_key=context.api_key,
api_base=context.api_base,
api_key=credentials.api_key,
api_base=credentials.api_base,
vertex_project=credentials.vertex_project,
vertex_location=credentials.vertex_location,
vertex_credentials=credentials.vertex_credentials,
**{"no-log": True},
)
_FETCH_CREDENTIAL_FIELDS: Final = ("api_key", "api_base", "vertex_project", "vertex_location", "vertex_credentials")
def _fetch_credentials(request_kwargs: Mapping[str, object]) -> CredentialLiteLLMParams:
try:
return CredentialLiteLLMParams.model_validate(
{field_name: request_kwargs.get(field_name) for field_name in _FETCH_CREDENTIAL_FIELDS}
)
except ValidationError:
verbose_logger.debug(
"Fetching a background interaction with environment credentials: the request's are malformed"
)
return CredentialLiteLLMParams()
_PERMANENT_FETCH_STATUS_CODES: Final = frozenset({400, 401, 403, 404})
def _is_permanent_fetch_error(error: Exception) -> bool:
return isinstance(error, APIStatusError) and error.status_code in _PERMANENT_FETCH_STATUS_CODES
def _poll_intervals(initial: float, maximum: float, timeout: float) -> Iterator[float]:
elapsed = 0.0 # rebind-ok: the schedule accumulates the time it has already yielded
interval = initial # rebind-ok: the schedule doubles the interval up to the cap
@ -231,6 +274,7 @@ _UNCARRIED_METADATA_KEY: Final = "user_api_key_auth"
_JSON_VALUE: Final = TypeAdapter(JsonValue)
_STRING: Final = TypeAdapter(str)
_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object])
_OBJECT_DICT: Final = TypeAdapter(dict[str, object])
def _carries(key: str) -> bool:
@ -256,10 +300,36 @@ def _as_datetime(start_time: datetime | float) -> datetime:
return start_time if isinstance(start_time, datetime) else datetime.fromtimestamp(start_time, tz=timezone.utc)
def _spend_metadata(logging_obj: "LiteLLMLoggingObj") -> Mapping[str, object]:
"""
The metadata the creating worker's own spend row is built from, with the
request tags computed once, header-derived ones included, because a
rebuilt logging object has no request headers to derive them from.
"""
litellm_params: Final = _OBJECT_DICT.validate_python(logging_obj.model_call_details.get("litellm_params") or {})
return {
**_OBJECT_MAPPING.validate_python(get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)),
**_request_tags(litellm_params),
}
def _request_tags(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
try:
return {
"tags": StandardLoggingPayloadSetup.get_request_tags(
litellm_params=_OBJECT_DICT.validate_python(litellm_params),
proxy_server_request=_OBJECT_DICT.validate_python(litellm_params.get("proxy_server_request") or {}),
)
}
except AttributeError:
verbose_logger.debug("Keeping a background interaction's request tags as sent: they are not a list")
return {}
def _create_context(logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str) -> BackgroundInteractionCreateContext:
metadata: Final = _OBJECT_MAPPING.validate_python(
get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
)
metadata: Final = _spend_metadata(logging_obj)
litellm_params: Final = _OBJECT_MAPPING.validate_python(logging_obj.litellm_params)
model: Final = logging_obj.model_call_details.get("model")
return BackgroundInteractionCreateContext(
@ -313,6 +383,16 @@ async def _settled_elsewhere(context: BackgroundInteractionPollContext) -> bool:
return False
async def _settled_elsewhere_before_the_window_ends(
context: BackgroundInteractionPollContext, intervals: Iterator[float]
) -> bool:
for interval in intervals:
await asyncio.sleep(interval)
if await _settled_elsewhere(context):
return True
return False
async def _claim(context: BackgroundInteractionPollContext) -> bool | None:
"""
Exactly-once gate between every settler of one interaction, on every
@ -326,30 +406,118 @@ async def _claim(context: BackgroundInteractionPollContext) -> bool | None:
return None
async def _record(context: BackgroundInteractionPollContext, outcome: SettlementOutcome) -> SettlementOutcome:
async def _wrote_outcome(context: BackgroundInteractionPollContext, outcome: SettlementOutcome) -> bool:
try:
await context.store.record_outcome(context.interaction_id, outcome)
except Exception: # noqa: BLE001 # the outcome is an audit trail; the claim already made the settlement exclusive
verbose_logger.exception("Could not record the settlement of background interaction %s", context.interaction_id)
except Exception as e: # noqa: BLE001 # a failed write is retried while the claim still shuts out takeovers
verbose_logger.warning(
"Could not record the settlement of background interaction %s: %s", context.interaction_id, e
)
return False
return True
async def _recorded(context: BackgroundInteractionPollContext, outcome: SettlementOutcome) -> bool:
"""
Retries a failed write for half the claim lease, while no other settler
can take the row over, so a database blip does not leave the row to a
takeover or hold the budget reservation that only a recorded release frees.
"""
retries: Final = _poll_intervals(
initial=context.initial_interval_seconds,
maximum=context.max_interval_seconds,
timeout=context.outcome_write_timeout_seconds,
)
for delay in itertools.chain((0.0,), retries):
await asyncio.sleep(delay)
if await _wrote_outcome(context, outcome):
return True
verbose_logger.error(
"Could not record the settlement of background interaction %s within %ss; the next process to take "
"over its lapsed claim settles it",
context.interaction_id,
context.outcome_write_timeout_seconds,
)
return False
async def _bill(context: BackgroundInteractionPollContext, response: InteractionsAPIResponse) -> SettlementOutcome:
await _bill_settled_interaction(logging_obj=context.logging_obj, response=response)
await _recorded(context, "billed")
return "billed"
async def _release(
context: BackgroundInteractionPollContext, outcome: Literal["released", "unsettled"]
) -> SettlementOutcome:
"""
A release writes no spend row, so a takeover cannot tell whether the
settler before it released. The outcome is recorded first and the
reservation released only once it is, which leaves a takeover with
nothing to release twice.
"""
if await _recorded(context, outcome):
await _release_open_budget_reservation(logging_obj=context.logging_obj)
return outcome
async def _renew_claim_until_cancelled(context: BackgroundInteractionPollContext) -> None:
while True:
await asyncio.sleep(context.claim_renewal_interval_seconds)
try:
await context.store.renew_claim(context.interaction_id)
except Exception as e: # noqa: BLE001 # a missed renewal is retried next interval, well inside the lease
verbose_logger.warning(
"Could not renew the settlement claim of background interaction %s: %s", context.interaction_id, e
)
async def _settle(
context: BackgroundInteractionPollContext, response: InteractionsAPIResponse | None
) -> SettlementOutcome:
"""
Billing runs every success callback in turn, and a slow one can outlast
the claim lease, so the claim is renewed until the outcome is recorded.
A settler that stops renewing has died, and only then can another take
the row over.
"""
renewal: Final = asyncio.create_task(_renew_claim_until_cancelled(context))
try:
if response is not None and response.status in _TERMINAL_STATUSES and response.usage is not None:
return await _bill(context, response)
return await _release(context, "released")
finally:
renewal.cancel()
async def poll_and_log_background_interaction_cost(
context: BackgroundInteractionPollContext,
fetch_interaction: FetchInteraction = fetch_background_interaction,
) -> SettlementOutcome | None:
last_response: InteractionsAPIResponse | None = None # rebind-ok: the give-up path settles from the last poll
for interval in _poll_intervals(
await asyncio.sleep(context.start_delay_seconds)
intervals: Final = _poll_intervals(
initial=context.initial_interval_seconds,
maximum=context.max_interval_seconds,
timeout=context.timeout_seconds,
):
)
for interval in intervals:
await asyncio.sleep(interval)
if await _settled_elsewhere(context):
return None
try:
response = await fetch_interaction(context)
except Exception as e: # noqa: BLE001 # any fetch error must not kill the billing poll loop
if context.resumed and _is_permanent_fetch_error(e):
verbose_logger.warning(
"Stopped fetching resumed background interaction %s: the provider answered %s, so this "
"process cannot read it and leaves it to the replica that created it until the poll window ends",
context.interaction_id,
e,
)
if await _settled_elsewhere_before_the_window_ends(context, intervals):
return None
break
verbose_logger.debug(
"Background interaction cost poll for %s failed, will retry: %s",
context.interaction_id,
@ -363,11 +531,11 @@ async def poll_and_log_background_interaction_cost(
continue
if not claimed:
return None
return await _record(context, await _settle_terminal(logging_obj=context.logging_obj, response=response))
return await _settle(context, response)
if not await _claim(context):
return None
if last_response is not None and last_response.status in _TERMINAL_STATUSES:
return await _record(context, await _settle_terminal(logging_obj=context.logging_obj, response=last_response))
return await _settle(context, last_response)
if last_response is not None and last_response.status not in _POLLABLE_STATUSES:
verbose_logger.error(
"Gave up cost polling for background interaction %s after %ss: its last status %r is in neither "
@ -379,20 +547,10 @@ async def poll_and_log_background_interaction_cost(
)
else:
verbose_logger.warning(
"Gave up cost polling for background interaction %s after %ss; its usage will not be tracked",
"Gave up cost polling for background interaction %s; its usage will not be tracked",
context.interaction_id,
context.timeout_seconds,
)
await _release_open_budget_reservation(logging_obj=context.logging_obj)
return await _record(context, "unsettled")
async def _settle_terminal(logging_obj: "LiteLLMLoggingObj", response: InteractionsAPIResponse) -> SettlementOutcome:
if response.status in _TERMINAL_STATUSES and response.usage is not None:
await _bill_settled_interaction(logging_obj=logging_obj, response=response)
return "billed"
await _release_open_budget_reservation(logging_obj=logging_obj)
return "released"
return await _release(context, "unsettled")
async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") -> None:
@ -405,7 +563,9 @@ async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") ->
settlement must release the reservation here or the spend counters stay
pinned at the estimated cost.
"""
metadata: Final = get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
metadata: Final = _OBJECT_MAPPING.validate_python(
get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
)
budget_reservation: Final = metadata.get("user_api_key_budget_reservation")
if not isinstance(budget_reservation, dict):
return
@ -518,13 +678,18 @@ class _UnverifiedRegistrationStore:
return False
return await self.local.claim(interaction_id)
async def renew_claim(self, interaction_id: str) -> None:
if await self.local.is_claimed(interaction_id):
return
await self.durable.renew_claim(interaction_id)
async def record_outcome(self, interaction_id: str, outcome: SettlementOutcome) -> None:
if await self.local.is_claimed(interaction_id):
return
await self.durable.record_outcome(interaction_id, outcome)
async def unclaimed(self) -> Sequence[PendingBackgroundInteraction]:
return await self.durable.unclaimed()
async def unsettled(self) -> Sequence[PendingBackgroundInteraction]:
return await self.durable.unsettled()
async def _registered_store(
@ -559,8 +724,6 @@ async def maybe_schedule_background_interaction_cost_polling(
logging_obj: Final = create_kwargs.get("litellm_logging_obj")
if not isinstance(logging_obj, Logging):
return None
api_key: Final = create_kwargs.get("api_key")
api_base: Final = create_kwargs.get("api_base")
pending: Final = PendingBackgroundInteraction(
interaction_id=response.id,
custom_llm_provider=custom_llm_provider,
@ -571,8 +734,7 @@ async def maybe_schedule_background_interaction_cost_polling(
interaction_id=response.id,
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
api_key=api_key if isinstance(api_key, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
credentials=_fetch_credentials(create_kwargs),
store=await _registered_store(store or _STORE.store, pending),
)
return _track_poll(context, fetch_interaction)
@ -603,10 +765,7 @@ async def _settle_before_delete(
) -> SettlementOutcome | None:
if not await _claim(context):
return None
if response is None:
await _release_open_budget_reservation(logging_obj=context.logging_obj)
return await _record(context, "released")
return await _record(context, await _settle_terminal(logging_obj=context.logging_obj, response=response))
return await _settle(context, response)
async def maybe_settle_background_interaction_before_delete(
@ -622,14 +781,11 @@ async def maybe_settle_background_interaction_before_delete(
pending: Final = await _pending(settlement_store, interaction_id)
if pending is None:
return None
api_key: Final = delete_kwargs.get("api_key")
api_base: Final = delete_kwargs.get("api_base")
context: Final = BackgroundInteractionPollContext(
interaction_id=interaction_id,
custom_llm_provider=pending.custom_llm_provider,
logging_obj=_rebuild_logging_obj(pending.create_context),
api_key=api_key if isinstance(api_key, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
credentials=_fetch_credentials(delete_kwargs),
store=settlement_store,
)
try:
@ -644,9 +800,9 @@ async def maybe_settle_background_interaction_before_delete(
return await _settle_before_delete(context, response)
async def _unclaimed(store: BackgroundSettlementStore) -> Sequence[PendingBackgroundInteraction]:
async def _unsettled(store: BackgroundSettlementStore) -> Sequence[PendingBackgroundInteraction]:
try:
return await store.unclaimed()
return await store.unsettled()
except Exception: # noqa: BLE001 # an unreadable store at startup leaves its rows for the next boot
verbose_logger.exception("Could not list the unsettled background interactions")
return ()
@ -662,17 +818,36 @@ class PollSchedule:
DEFAULT_POLL_SCHEDULE: Final = PollSchedule()
def _deployment_id(create_context: BackgroundInteractionCreateContext) -> str | None:
model_info: Final = create_context.metadata.get("model_info")
deployment_id: Final = model_info.get("id") if isinstance(model_info, dict) else None
return deployment_id if isinstance(deployment_id, str) else None
def _model_group(create_context: BackgroundInteractionCreateContext) -> str | None:
model_group: Final = create_context.metadata.get("model_group")
return model_group if isinstance(model_group, str) else None
def _seconds_until(moment: datetime | None) -> float:
if moment is None:
return 0.0
return max((moment - datetime.now(timezone.utc)).total_seconds(), 0.0)
def _resumed_context(
row: PendingBackgroundInteraction, store: BackgroundSettlementStore, schedule: PollSchedule
) -> BackgroundInteractionPollContext:
age_seconds: Final = (datetime.now(timezone.utc) - row.created_at).total_seconds()
return BackgroundInteractionPollContext(
interaction_id=row.interaction_id,
custom_llm_provider=row.custom_llm_provider,
logging_obj=_rebuild_logging_obj(row.create_context),
deployment_id=_deployment_id(row.create_context),
model_group=_model_group(row.create_context),
initial_interval_seconds=schedule.initial_interval_seconds,
max_interval_seconds=schedule.max_interval_seconds,
timeout_seconds=max(schedule.timeout_seconds - age_seconds, schedule.initial_interval_seconds),
timeout_seconds=schedule.timeout_seconds,
start_delay_seconds=_seconds_until(row.claim_expires_at),
store=store,
resumed=True,
)
@ -684,13 +859,15 @@ async def resume_unsettled_background_interactions(
schedule: PollSchedule = DEFAULT_POLL_SCHEDULE,
) -> tuple["asyncio.Task[SettlementOutcome | None]", ...]:
"""
Pick up every settlement no process has claimed, which is what a replica
that died mid-poll leaves behind. Each resumed poll keeps the remaining
share of the original timeout and gets at least one fetch, so a completed
interaction is still billed however late the resume comes.
Pick up every settlement no process has finished, which is what a replica
that died mid-poll or mid-settlement leaves behind. A row whose claim is
still within its lease waits until the lease runs out, then polls like any
other. Each resumed poll gets the full poll window however old its row is,
so a fetch that fails right after a late restart is retried rather than
ending the row as unsettled.
"""
return tuple(
_track_poll(_resumed_context(row, store, schedule), fetch_interaction)
for row in await _unclaimed(store)
for row in await _unsettled(store)
if row.interaction_id not in _ACTIVE_POLLS
)

4
litellm/litellm_core_utils/core_helpers.py
View file

@ -318,7 +318,7 @@ def remove_items_at_indices(items: _PoppableByIndex | None, indices: Iterable[in
items.pop(index)
def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metadata: dict) -> dict:
def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metadata: Mapping[str, object]) -> dict:
"""
Helper to get litellm metadata for spend tracking
@ -326,7 +326,7 @@ def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metad
and user_api_key values are in 'metadata'.
"""
potential_spend_tracking_metadata_substring: Final = "user_api_key"
for key, value in metadata.items():
for key, value in dict(metadata).items():
if potential_spend_tracking_metadata_substring in key:
litellm_metadata[key] = value
return litellm_metadata

1
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -78,6 +78,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"github_copilot_auth_type",
"github_copilot_user_session",
"fireworks_forward_user_id",
"responses_api_version",
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
}
)

4
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -61,6 +61,10 @@ def _endpoint_matches_api_base(endpoint: str, api_base: str) -> bool:
return url_path == endpoint_path or url_path.startswith(endpoint_path + "/")
def is_openai_compatible_endpoint(api_base: str) -> bool:
return any(_endpoint_matches_api_base(endpoint, api_base) for endpoint in litellm.openai_compatible_endpoints)
def is_non_openai_azure_model(model: str) -> bool:
try:
model_name: Final = model.split("/", 1)[1]

79
litellm/litellm_core_utils/get_model_cost_map.py
View file

@ -17,6 +17,7 @@ import random
import sys
import threading
import time
import urllib.parse
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass, replace
from datetime import datetime, timezone
@ -24,11 +25,11 @@ from importlib.resources import files
from itertools import chain
from pathlib import Path
from types import MappingProxyType
from typing import Final, Protocol
from typing import Final, Protocol, cast # noqa: TID251 # adapts boto3 and botocore's dynamic client responses
import httpx
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm import verbose_logger
from litellm.constants import (
@ -266,6 +267,19 @@ class _SyncGetClient(Protocol):
def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: ...
class _S3Body(Protocol):
def read(self) -> bytes: ...
class _S3GetObjectResponse(TypedDict):
Body: ReadOnly[_S3Body]
ETag: NotRequired[ReadOnly[str]]
class _S3Client(Protocol):
def get_object(self, *, Bucket: str, Key: str) -> _S3GetObjectResponse: ...
_FetchAttemptOutcome = ModelCostMapReloaded | ModelCostMapReloadUnavailable | _FetchAttemptRetryable
@ -276,6 +290,20 @@ def _default_reload_client() -> _AsyncGetClient:
return get_async_httpx_client(llm_provider=httpxSpecialProvider.ModelCostMap)
def _default_s3_client(timeout: int) -> _S3Client:
import boto3
from botocore.config import Config
return cast( # cast-ok: boto3's generic client factory does not expose its generated S3 response type
_S3Client,
boto3.client( # pyright: ignore[reportUnknownMemberType] # generic factory returns Unknown service clients
"s3",
region_name=os.getenv("AWS_REGION_NAME") or None,
config=Config(connect_timeout=timeout, read_timeout=timeout, retries={"total_max_attempts": 1}),
),
)
def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -> _FetchAttemptOutcome:
reason: Final = f"{type(error).__name__} fetching {url}: {error}"
if isinstance(error, (httpx.InvalidURL, httpx.UnsupportedProtocol)):
@ -283,7 +311,41 @@ def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -
return _FetchAttemptRetryable(reason=reason, retry_after_seconds=None)
def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome:
try:
parsed: Final[object] = json.loads(content)
except ValueError as e:
return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
if not isinstance(parsed, dict):
return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}")
model_cost_map: Final = cast( # cast-ok: JSON object keys are strings by definition
dict[str, object], parsed
)
return ModelCostMapReloaded(model_cost_map=model_cost_map, revision=git_blob_id(content), etag=etag)
def _attempt_fetch_s3(url: str, timeout: int) -> _FetchAttemptOutcome:
from botocore.exceptions import BotoCoreError, ClientError
parts: Final = urllib.parse.urlsplit(url)
bucket: Final = parts.netloc
key: Final = parts.path.lstrip("/")
if not bucket or not key:
return ModelCostMapReloadUnavailable(reason=f"invalid S3 URL {url}: expected s3://bucket/key")
try:
client: Final = _default_s3_client(timeout)
response: Final = client.get_object(Bucket=bucket, Key=key)
content: Final = response["Body"].read()
except (BotoCoreError, ClientError) as error:
return ModelCostMapReloadUnavailable(reason=f"S3 {type(error).__name__} fetching {url}: {error}")
return _classify_fetch_content(content, url, response.get("ETag"))
async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
parts: Final = urllib.parse.urlsplit(url)
if parts.scheme == "s3":
return await asyncio.to_thread(_attempt_fetch_s3, url, timeout)
try:
response: Final = await client.get(url, timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL) as e:
@ -292,6 +354,9 @@ async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _Fe
def _attempt_fetch_sync(client: _SyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
parts: Final = urllib.parse.urlsplit(url)
if parts.scheme == "s3":
return _attempt_fetch_s3(url, timeout)
try:
response: Final = client.get(url, timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL) as e:
@ -307,15 +372,7 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp
)
if response.is_error:
return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}")
try:
parsed: Final = response.json()
except ValueError as e:
return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
if not isinstance(parsed, dict):
return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}")
return ModelCostMapReloaded(
model_cost_map=parsed, revision=git_blob_id(response.content), etag=response.headers.get("etag")
)
return _classify_fetch_content(response.content, url, response.headers.get("etag"))
def _next_retry_wait(

5
litellm/litellm_core_utils/hidden_params.py
View file

@ -77,6 +77,11 @@ def get_hidden_params(obj: object) -> MutableMapping[str, object] | None:
return _as_hidden_params_mapping(hidden_params)
def served_from_cache(obj: object) -> bool:
hidden_params: Final = get_hidden_params(obj)
return hidden_params is not None and hidden_params.get("cache_hit") is True
def set_hidden_params(obj: object, hidden_params: dict[str, object] | HiddenParams) -> None:
if isinstance(obj, dict):
obj[_HIDDEN_PARAMS_ATTR] = hidden_params

9
litellm/litellm_core_utils/internal_call_metadata.py
View file

@ -157,7 +157,10 @@ def sanitize_user_api_key_auth(auth: object) -> object:
return auth
def _sanitized(parent_metadata: Mapping[str, object]) -> dict[str, object]: # mutable-ok: SDK metadata kwarg
def without_budget_reservation(
parent_metadata: Mapping[str, object],
) -> dict[str, object]: # mutable-ok: SDK metadata kwarg
"""Parent metadata minus the admission reservation, also inside ``user_api_key_auth``."""
return {
k: sanitize_user_api_key_auth(v) if k == _USER_API_KEY_AUTH_KEY else v
for k, v in parent_metadata.items()
@ -176,7 +179,7 @@ def forwarded_internal_call_metadata(
"""
if not parent_metadata:
return {}
return _sanitized(parent_metadata) | {INTERNAL_CALL_ORIGIN_METADATA_KEY: call_origin}
return without_budget_reservation(parent_metadata) | {INTERNAL_CALL_ORIGIN_METADATA_KEY: call_origin}
def parent_session_kwargs(request_kwargs: Mapping[str, object] | None) -> Mapping[str, str]:
@ -202,4 +205,4 @@ def sanitized_forwardable_call_metadata(
must not inherit per-request state such as its routing decision or logging payload.
"""
identity: Final = {k: v for k, v in parent_metadata.items() if k in FORWARDABLE_IDENTITY_METADATA_KEYS}
return _sanitized(identity) | {INTERNAL_CALL_ORIGIN_METADATA_KEY: call_origin}
return without_budget_reservation(identity) | {INTERNAL_CALL_ORIGIN_METADATA_KEY: call_origin}

187
litellm/litellm_core_utils/litellm_logging.py
View file

@ -19,7 +19,7 @@ from types import MappingProxyType, TracebackType
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
from httpx import Response
from pydantic import BaseModel, JsonValue
from pydantic import BaseModel, JsonValue, TypeAdapter
import litellm
from litellm._internal_context import post_response_phase
@ -79,7 +79,12 @@ from litellm.litellm_core_utils.core_helpers import (
)
from litellm.litellm_core_utils.error_normalization import normalize_error
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_param
from litellm.litellm_core_utils.hidden_params import (
HIDDEN_PARAMS_ATTR,
get_hidden_params,
served_from_cache,
set_hidden_param,
)
from litellm.litellm_core_utils.internal_call_metadata import (
MODEL_ACCESS_GROUP_METADATA_KEY,
is_unbilled_non_inference_call,
@ -536,6 +541,10 @@ def _resolve_mantle_region_for_cost(
return resolve_mantle_region(litellm_params or MappingProxyType({}))
_PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
def _provider_response_id(source: object) -> str | None:
candidate: Final = source.get("id") if isinstance(source, dict) else getattr(source, "id", None)
return candidate if isinstance(candidate, str) and candidate else None
@ -771,6 +780,36 @@ class Logging(LiteLLMLoggingBaseClass):
self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
def copy_for_new_call(self, litellm_call_id: str, start_time: datetime.datetime) -> "Logging":
"""One turn of a long-lived connection: the connection's deployment, key and metadata under its own call id,
start time and chunk buffers, so the turn logs and bills as a request of its own."""
connection_params: Final = _LITELLM_PARAMS.validate_python(
self.litellm_params # pyright: ignore[reportUnknownMemberType] # Logging.litellm_params is untyped
)
litellm_params: Final[dict[str, object]] = {
**connection_params,
**{
key: _LITELLM_PARAMS.validate_python(value)
for key, value in connection_params.items()
if key in _PER_CALL_LITELLM_PARAM_DICTS and isinstance(value, dict)
},
"litellm_call_id": litellm_call_id,
}
new_call: Final = copy.copy(self)
new_call.litellm_call_id = litellm_call_id
new_call.start_time = start_time
new_call.litellm_params = litellm_params
new_call.model_call_details = {
**self.model_call_details,
"litellm_call_id": litellm_call_id,
"start_time": start_time,
"first_api_call_start_time": None,
"litellm_params": litellm_params,
}
new_call.streaming_chunks = []
new_call.sync_streaming_chunks = []
return new_call
def set_response_timing_metrics(self, timing_metrics: Mapping[str, float]) -> None:
"""Keep ``_response_ms`` / ``litellm_overhead_time_ms`` for a result that has no ``_hidden_params``."""
self.response_timing_metrics = dict(timing_metrics)
@ -2022,7 +2061,7 @@ class Logging(LiteLLMLoggingBaseClass):
) -> None:
if response_cost is None and not calculation_failed:
return
if self.model_call_details.get("cache_hit") is True:
if self.model_call_details.get("cache_hit") is True or _served_from_cache_at_no_charge(result):
self.model_call_details["zero_cost_diagnostic"] = None
return
try:
@ -2261,6 +2300,54 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["combined_usage_object"] = usage
self.model_call_details["response_cost"] = response_cost
def attempt_scoped_copy(self) -> "Logging":
"""A copy with its own model_call_details, so logging one attempt of a request through it leaves
the state the request's other attempts share untouched."""
attempt: Final = copy.copy(self)
attempt.model_call_details = dict(self.model_call_details)
return attempt
def carry_failed_attempt_usage(self) -> None:
"""Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its
own usage while the request's failure log still bills every attempt."""
usage: Final = self.model_call_details.pop("combined_usage_object", None)
response_cost: Final = self.model_call_details.pop("response_cost", None)
if isinstance(usage, Usage):
self.model_call_details["failed_attempts_usage"] = (
*self.model_call_details.get("failed_attempts_usage", ()),
(usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0),
)
def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
"""Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
self.model_call_details["continued_partial_usage"] = usage
self.model_call_details["continued_partial_cost"] = response_cost
self.model_call_details["continued_response_id"] = response_id
def _add_continued_partial_spend(self, response: object) -> None:
partial_usage: Final = self.model_call_details.get("continued_partial_usage")
partial_cost: Final = self.model_call_details.get("continued_partial_cost")
own_usage: Final = getattr(response, "usage", None)
own_cost: Final = self.model_call_details.get("response_cost")
if not (isinstance(partial_usage, Usage) and isinstance(partial_cost, float) and isinstance(own_usage, Usage)):
return
from litellm.cost_calculator import BaseTokenUsageProcessor
self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
usage_objects=[own_usage, partial_usage]
)
if not isinstance(own_cost, (int, float)):
return
self.model_call_details["response_cost"] = own_cost + partial_cost
if self.cost_breakdown is not None:
self.cost_breakdown = self.cost_breakdown | CostBreakdown(
total_cost=own_cost + partial_cost,
additional_costs={
**self.cost_breakdown.get("additional_costs", {}),
"mid_stream_partial_cost": partial_cost,
},
)
def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
"""Bill a fully streamed response on the failure log when a post-call hook rejects it."""
usage: Final = getattr(assembled, "usage", None)
@ -2384,20 +2471,9 @@ class Logging(LiteLLMLoggingBaseClass):
)
elif self.call_type == CallTypes.aresponses_websocket.value and isinstance(result, list): # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
combined_ws_usage: Final = (
ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(
results=result # pyright: ignore[reportUnknownArgumentType] # raw event dicts from the WS stream
)
)
ws_tier_partition: Final = ResponsesWebSocketTokenUsageProcessor.partition_results_by_service_tier(
logging_result = ResponsesWebSocketTokenUsageProcessor.create_logging_object(
results=result # pyright: ignore[reportUnknownArgumentType] # raw event dicts from the WS stream
)
ws_service_tier: Final = next(iter(ws_tier_partition)) if len(ws_tier_partition) == 1 else None
logging_result = LiteLLMRealtimeStreamLoggingObject(
usage=combined_ws_usage,
results=result, # pyright: ignore[reportUnknownArgumentType] # raw event dicts from the WS stream
service_tier=ws_service_tier,
)
elif self.relays_llm_passthrough and isinstance(result, Response):
from litellm.utils import ProviderConfigManager
@ -2567,6 +2643,15 @@ class Logging(LiteLLMLoggingBaseClass):
return response
return response.model_copy(update={"model": model})
def _with_continued_response_id(
self,
response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
response_id: Final = self.model_call_details.get("continued_response_id")
if not isinstance(response_id, str) or getattr(response, "id", None) == response_id:
return response
return response.model_copy(update={"id": response_id})
def _success_handler_helper_fn(
self,
result: object = None,
@ -2600,6 +2685,8 @@ class Logging(LiteLLMLoggingBaseClass):
result = self._handle_a2a_response_logging(result=result)
logging_result: Final = self.normalize_logging_result(result=result)
if cache_hit is None and _served_from_cache_at_no_charge(logging_result):
self.model_call_details["cache_hit"] = True
if isinstance(result, Response) and isinstance(
logging_result, (ModelResponse, EmbeddingResponse, TextCompletionResponse)
@ -2885,8 +2972,11 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["response_cost"] = self.response_cost_calculator(
result=complete_streaming_response
)
self._add_continued_partial_spend(complete_streaming_response)
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
logged_streaming_response: Final = self._with_client_facing_stream_model(complete_streaming_response)
logged_streaming_response: Final = self._with_continued_response_id(
self._with_client_facing_stream_model(complete_streaming_response)
)
self.model_call_details["complete_streaming_response"] = logged_streaming_response
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
@ -3422,12 +3512,15 @@ class Logging(LiteLLMLoggingBaseClass):
self.model,
)
self.model_call_details["response_cost"] = None
self._add_continued_partial_spend(complete_streaming_response)
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
await self._prepare_baseline_cache_estimate(complete_streaming_response)
logged_streaming_response: Final = self._with_client_facing_stream_model(complete_streaming_response)
logged_streaming_response: Final = self._with_continued_response_id(
self._with_client_facing_stream_model(complete_streaming_response)
)
self.model_call_details["async_complete_streaming_response"] = logged_streaming_response
## STANDARDIZED LOGGING PAYLOAD
@ -4259,6 +4352,9 @@ class Logging(LiteLLMLoggingBaseClass):
row is built from this response rather than re-parsed from the served body."""
self.bridged_model_response = response
def record_bridged_responses_api_response(self, response: ResponsesAPIResponse) -> None:
self.record_bridged_model_response(self._translate_responses_api_response_to_model_response(response))
def billable_result(self, result: object) -> object:
"""The response the cost calculator prices: a /v1/messages body served through the chat
bridge is replaced by the ModelResponse its spend row is built from, anything else is
@ -4359,6 +4455,7 @@ class Logging(LiteLLMLoggingBaseClass):
"""
from litellm.completion_extras.litellm_responses_transformation.transformation import (
LiteLLMResponsesTransformationHandler,
responses_provider_metadata,
)
served_id: Final = _provider_response_id(result)
@ -4383,7 +4480,9 @@ class Logging(LiteLLMLoggingBaseClass):
"usage-only ModelResponse to keep the spend_logs row.",
str(e),
)
model_response: Final = litellm.ModelResponse(id=served_id)
model_response: Final = litellm.ModelResponse(
id=served_id, **responses_provider_metadata(result.model_extra)
)
model_response.model = self.model
usage: Final = result.usage
if usage is not None and ResponseAPILoggingUtils.is_response_api_usage(usage):
@ -4507,6 +4606,32 @@ def get_masked_values(
_get_masked_values = get_masked_values
@lru_cache(maxsize=1)
def start_sentry(dsn: str) -> None:
try:
import sentry_sdk
except ImportError:
print_verbose("Package 'sentry_sdk' is missing. Installing it...")
subprocess.check_call([sys.executable, "-m", "pip", "install", "sentry_sdk"])
import sentry_sdk
from litellm.litellm_core_utils.sentry_scrubbing import build_sentry_init_options
sentry_sdk.init(**build_sentry_init_options({**os.environ, "SENTRY_DSN": dsn}))
def initialize_sentry() -> None:
global sentry_sdk_instance, capture_exception, add_breadcrumb
dsn: Final = os.environ.get("SENTRY_DSN")
if not dsn:
return
start_sentry(dsn)
import sentry_sdk
sentry_sdk_instance = sentry_sdk # rebind-ok: process-wide Sentry SDK handle read by every logging path
capture_exception = sentry_sdk.capture_exception # rebind-ok: process-wide Sentry SDK handle
add_breadcrumb = sentry_sdk.add_breadcrumb # rebind-ok: process-wide Sentry SDK handle
def set_callbacks(
callback_list: Iterable[str | Callable[..., object] | CustomLogger],
function_id: str | None = None,
@ -4515,9 +4640,6 @@ def set_callbacks(
Globally sets the callback client
"""
global \
sentry_sdk_instance, \
capture_exception, \
add_breadcrumb, \
slack_app, \
alerts_channel, \
traceloopLogger, \
@ -4541,18 +4663,7 @@ def set_callbacks(
try:
for callback in callback_list:
if callback == "sentry":
try:
import sentry_sdk
except ImportError:
print_verbose("Package 'sentry_sdk' is missing. Installing it...")
subprocess.check_call([sys.executable, "-m", "pip", "install", "sentry_sdk"])
import sentry_sdk
from litellm.litellm_core_utils.sentry_scrubbing import build_sentry_init_options
sentry_sdk_instance = sentry_sdk
sentry_sdk_instance.init(**build_sentry_init_options(os.environ))
capture_exception = sentry_sdk_instance.capture_exception
add_breadcrumb = sentry_sdk_instance.add_breadcrumb
initialize_sentry()
elif callback == "slack":
try:
from slack_bolt import App
@ -6599,6 +6710,11 @@ def _get_status_fields(
return StandardLoggingPayloadStatusFields(llm_api_status=llm_api_status, guardrail_status=guardrail_status)
def _served_from_cache_at_no_charge(result: object) -> bool:
hidden_params: Final = get_hidden_params(result)
return served_from_cache(result) and hidden_params is not None and hidden_params.get("response_cost") == 0
def _extract_response_obj_and_hidden_params(
init_response_obj: object,
original_exception: Exception | None,
@ -6714,7 +6830,7 @@ def get_standard_logging_object_payload(
else raw_usage_dict
)
id = response_obj.get("id", kwargs.get("litellm_call_id"))
id = response_obj.get("id") or kwargs.get("litellm_call_id")
_model_id: Final = metadata.get("model_info", {}).get("id", "")
_model_group: Final = metadata.get("model_group", "")
@ -6761,8 +6877,9 @@ def get_standard_logging_object_payload(
) # maintain backwards compatibility with old request body check
saved_cache_cost: float = 0.0
if cache_hit is True:
if cache_hit is True or served_from_cache(init_response_obj):
id = f"{id}_cache_hit{time.time()}" # do not duplicate the request id
if cache_hit is True:
saved_cache_cost = (
logging_obj.response_cost_calculator(
result=init_response_obj,

175
litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
View file

@ -2,12 +2,13 @@
Helper utilities for tracking the cost of built-in tools.
"""
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import Final, Literal, cast
from pydantic import ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
from litellm.litellm_core_utils.llm_cost_calc.utils import (
get_web_search_requests_from_usage,
@ -30,20 +31,36 @@ from litellm.types.utils import (
)
def _output_item_field(output_item: object, field: str) -> object:
if isinstance(output_item, dict):
fields: Final[Mapping[str, object]] = cast(Mapping[str, object], output_item) # cast-ok: narrowed by isinstance
return fields.get(field)
return getattr(output_item, field, None)
def _output_item_type(output_item: object) -> str | None:
item_type: Final = output_item.get("type") if isinstance(output_item, dict) else getattr(output_item, "type", None)
item_type: Final = _output_item_field(output_item, "type")
return item_type if isinstance(item_type, str) else None
def _reported_web_search_requests(response_object: ResponsesAPIResponse) -> int | None:
tool_usage: Final = getattr(response_object, "tool_usage", None)
if tool_usage is None:
def _responses_tool_usage(response_object: ResponsesAPIResponse) -> ResponsesToolUsage | None:
top_level: Final = getattr(response_object, "tool_usage", None)
raw: Final = (
top_level if top_level is not None else getattr(getattr(response_object, "usage", None), "tool_usage", None)
)
if raw is None:
return None
try:
web_search: Final = ResponsesToolUsage.model_validate(tool_usage).web_search
return ResponsesToolUsage.model_validate(raw)
except ValidationError:
return None
return None if web_search is None else web_search.num_requests
def _reported_web_search_requests(response_object: ResponsesAPIResponse) -> int | None:
tool_usage: Final = _responses_tool_usage(response_object)
if tool_usage is None:
return None
return None if tool_usage.web_search is None else tool_usage.web_search.num_requests
def _usage_reports_server_side_web_search_calls(usage: Usage) -> bool:
@ -87,31 +104,48 @@ class StandardBuiltInToolCostTracking:
usage=usage,
)
image_generation_cost: Final = StandardBuiltInToolCostTracking._handle_image_generation_cost(
response_object=response_object,
custom_llm_provider=custom_llm_provider,
)
# Handle web search
if StandardBuiltInToolCostTracking.response_object_includes_web_search_call(
response_object=response_object, usage=usage
):
return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_web_search_cost(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
standard_built_in_tools_params=standard_built_in_tools_params,
response_object=response_object,
return (
google_maps_grounding_cost
+ image_generation_cost
+ StandardBuiltInToolCostTracking._handle_web_search_cost(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
standard_built_in_tools_params=standard_built_in_tools_params,
response_object=response_object,
)
)
# Handle file search
if StandardBuiltInToolCostTracking.response_object_includes_file_search_call(response_object=response_object):
return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_file_search_cost(
return (
google_maps_grounding_cost
+ image_generation_cost
+ StandardBuiltInToolCostTracking._handle_file_search_cost(
model=model,
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=standard_built_in_tools_params,
)
)
# Handle Azure assistant features
return (
google_maps_grounding_cost
+ image_generation_cost
+ StandardBuiltInToolCostTracking._handle_azure_assistant_costs(
model=model,
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=standard_built_in_tools_params,
)
# Handle Azure assistant features
return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_azure_assistant_costs(
model=model,
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=standard_built_in_tools_params,
)
@staticmethod
@ -210,6 +244,87 @@ class StandardBuiltInToolCostTracking:
)
return max(count, 1)
@staticmethod
def _image_generation_tool_model(response_object: ResponsesAPIResponse) -> str | None:
tools: Final[Sequence[object]] = cast(Sequence[object], response_object.tools or ()) # cast-ok: dict tool union
for tool in tools:
if _output_item_field(tool, "type") != "image_generation":
continue
if isinstance(model := _output_item_field(tool, "model"), str) and model:
return model
return None
@staticmethod
def _image_generation_call_cost(output_item: object, tool_model: str, custom_llm_provider: str | None) -> float:
from litellm.cost_calculator import (
default_image_cost_calculator, # pyright: ignore[reportUnknownVariableType] # optional_params param is untyped
)
quality: Final = _output_item_field(output_item, "quality")
size: Final = _output_item_field(output_item, "size")
try:
return default_image_cost_calculator(
model=tool_model,
custom_llm_provider=custom_llm_provider,
quality=quality if isinstance(quality, str) and quality != "auto" else None,
n=1,
size=size if isinstance(size, str) and size != "auto" else None,
)
except Exception as e: # noqa: BLE001 # pricing helpers raise bare Exception for unmapped models; bill 0.0
verbose_logger.debug("Could not price Responses API image_generation_call item: %s", e)
return 0.0
@staticmethod
def _image_generation_token_cost(
response_object: ResponsesAPIResponse, tool_model: str, custom_llm_provider: str | None
) -> float | None:
tool_usage: Final = _responses_tool_usage(response_object)
if tool_usage is None or tool_usage.image_gen is None or tool_usage.image_gen.total_tokens <= 0:
return None
try:
model_info: Final = litellm.get_model_info(model=tool_model, custom_llm_provider=custom_llm_provider)
except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; fall back
verbose_logger.debug("Could not resolve pricing for image tool model %s: %s", tool_model, e)
return None
image_gen: Final = tool_usage.image_gen
input_details: Final = image_gen.input_tokens_details
output_details: Final = image_gen.output_tokens_details
token_cost: Final = (
(input_details.text_tokens if input_details else 0) * (model_info.get("input_cost_per_token") or 0)
+ (input_details.image_tokens if input_details else 0) * (model_info.get("input_cost_per_image_token") or 0)
+ (output_details.image_tokens if output_details else 0)
* (model_info.get("output_cost_per_image_token") or 0)
+ (output_details.text_tokens if output_details else 0) * (model_info.get("output_cost_per_token") or 0)
)
return token_cost if token_cost > 0 else None
@staticmethod
def _handle_image_generation_cost(response_object: object, custom_llm_provider: str | None) -> float:
if not isinstance(response_object, ResponsesAPIResponse):
return 0.0
output: Final[Sequence[object]] = cast(Sequence[object], response_object.output) # cast-ok: isinstance-narrowed
completed_items: Final = tuple(
output_item
for output_item in output
if _output_item_type(output_item) == "image_generation_call"
and _output_item_field(output_item, "status") == "completed"
)
if not completed_items:
return 0.0
tool_model: Final = StandardBuiltInToolCostTracking._image_generation_tool_model(response_object)
if tool_model is None:
verbose_logger.debug("Responses API image_generation_call has no tools[].model to price, billing 0.0")
return 0.0
token_cost: Final = StandardBuiltInToolCostTracking._image_generation_token_cost(
response_object, tool_model, custom_llm_provider
)
if token_cost is not None:
return token_cost
return sum(
StandardBuiltInToolCostTracking._image_generation_call_cost(output_item, tool_model, custom_llm_provider)
for output_item in completed_items
)
@staticmethod
def _handle_file_search_cost(
model: str,
@ -445,9 +560,12 @@ class StandardBuiltInToolCostTracking:
return False
elif isinstance(response_object, ResponsesAPIResponse):
# response api explicitly includes web_search_call in the output
return StandardBuiltInToolCostTracking.response_includes_output_type(
if StandardBuiltInToolCostTracking.response_includes_output_type(
response_object=response_object, output_type="web_search_call"
)
):
return True
reported: Final = _reported_web_search_requests(response_object)
return reported is not None and reported > 0
elif usage is not None:
if get_web_search_requests_from_usage(usage) is not None or (
hasattr(usage, "prompt_tokens_details")
@ -740,9 +858,9 @@ class StandardBuiltInToolCostTracking:
@staticmethod
def get_web_search_options(kwargs: Mapping[str, object]) -> WebSearchOptions | None:
if "web_search_options" in kwargs:
web_search_options: Final = cast(WebSearchOptions, kwargs.get("web_search_options", {}))
return WebSearchOptions(**web_search_options)
web_search_options: Final = kwargs.get("web_search_options")
if isinstance(web_search_options, dict):
return WebSearchOptions(**cast(WebSearchOptions, web_search_options))
tools: Final = StandardBuiltInToolCostTracking._get_tools_from_kwargs(
kwargs=kwargs, tool_type="web_search_preview"
@ -758,10 +876,9 @@ class StandardBuiltInToolCostTracking:
_get_web_search_options = get_web_search_options
@staticmethod
def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> list[object] | None:
if "tools" in kwargs:
return cast(list[object], kwargs.get("tools", []))
return None
def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> Sequence[object] | None:
tools: Final = kwargs.get("tools")
return cast(Sequence[object], tools) if isinstance(tools, (list, tuple)) else None
@staticmethod
def get_file_search_tool_call(kwargs: Mapping[str, object]) -> FileSearchTool | None:

5
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -16,6 +16,7 @@ from litellm.litellm_core_utils.hidden_params import get_hidden_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_reasoning_content,
)
from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
from litellm.types.llms.databricks import DatabricksTool
from litellm.types.llms.openai import (
ChatCompletionReasoningItem,
@ -833,7 +834,9 @@ def convert_to_model_response_object(
if k not in _MODEL_RESPONSE_FIELDS:
setattr(model_response_object, k, v)
return model_response_object
ollama_ids: Final = OllamaResponseIdNormalizer()
ollama_ids.observe(model_response_object)
return ollama_ids.apply(model_response_object)
elif response_type == "embedding" and (
model_response_object is None or isinstance(model_response_object, EmbeddingResponse)
):

13
litellm/litellm_core_utils/llm_response_utils/get_headers.py
View file

@ -1,8 +1,11 @@
from collections.abc import Mapping
from typing import Final
import httpx
from pydantic import TypeAdapter, ValidationError
from litellm.constants import PROVIDER_REQUEST_ID_HEADERS
_HEADERS: Final = TypeAdapter(Mapping[str, str])
_REQUEST_ID_HEADERS: Final = ("request-id", "x-request-id", "llm_provider-request-id", "llm_provider-x-request-id")
@ -16,6 +19,16 @@ def get_provider_request_id(headers: object) -> str | None:
return next((normalized[name] for name in _REQUEST_ID_HEADERS if normalized.get(name)), None)
def provider_request_id_headers(headers: Mapping[str, object] | None) -> httpx.Headers | None:
"""The provider request id headers in `headers` whose values are ASCII strings, or None when there are none."""
request_id_headers: Final = tuple(
(name, value)
for name, value in (headers or {}).items()
if name.lower() in PROVIDER_REQUEST_ID_HEADERS and isinstance(value, str) and value.isascii()
)
return httpx.Headers(request_id_headers) if request_id_headers else None
def get_response_headers(_response_headers: Mapping[str, str] | None = None) -> dict:
"""

19
litellm/litellm_core_utils/model_param_helper.py
View file

@ -1,3 +1,4 @@
from collections.abc import Mapping
from functools import lru_cache
from typing import ClassVar, Final
@ -22,6 +23,24 @@ from litellm.types.llms.anthropic import AnthropicMessagesRequest
from litellm.types.rerank import RerankRequest
def peek_reasoning_summary_aliases(optional_params: Mapping[str, object]) -> object | None:
"""Read AI-SDK-style reasoning summary from optional_params or nested extra_body.
Uses key membership (not ``or`` chains) so falsy values like ``""`` are not skipped.
"""
if "reasoningSummary" in optional_params:
return optional_params["reasoningSummary"]
if "reasoning_summary" in optional_params:
return optional_params["reasoning_summary"]
extra_body: Final = optional_params.get("extra_body")
if isinstance(extra_body, dict):
if "reasoningSummary" in extra_body:
return extra_body["reasoningSummary"]
if "reasoning_summary" in extra_body:
return extra_body["reasoning_summary"]
return None
class ModelParamHelper:
# Cached at class level — deterministic set built from static OpenAI type annotations
relevant_logging_args: ClassVar[frozenset[str]] = frozenset()

125
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -9,7 +9,8 @@ import json
import mimetypes
import re
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from itertools import groupby, islice
from graphlib import CycleError, TopologicalSorter
from itertools import chain, groupby, islice
from os import PathLike
from pathlib import Path
from types import MappingProxyType
@ -1426,6 +1427,123 @@ def _schema_levels(schema: Mapping[str, object]) -> Iterator[tuple[Mapping[str,
frontier = tuple(child for node in frontier for child in _subschemas(node))
def schema_fragment_ref(ref: str) -> str:
return f"#{ref}" if ref.startswith(("/$defs/", "/definitions/")) else ref
def schema_has_unresolvable_ref(schema: Mapping[str, object]) -> bool:
definitions: Final = MappingProxyType(dict(_root_definitions(schema)))
if not _refs_in(schema) <= definitions.keys():
return True
try:
tuple(TopologicalSorter({pointer: _refs_in(body) for pointer, body in definitions.items()}).static_order())
except CycleError:
return True
return False
def schema_union_property_count(schema: Mapping[str, object]) -> int:
return sum(1 for _, node in _property_schemas(schema) if _is_union(node))
def schema_optional_property_count(schema: Mapping[str, object]) -> int:
return sum(len(_optional_property_names(node)) for node in _schema_nodes(schema))
def schema_property_count(schema: Mapping[str, object]) -> int:
return sum(len(properties) for node in _schema_nodes(schema) if (properties := _schema_at(node, "properties")))
def schema_grammar_branch_count(schema: Mapping[str, object]) -> int:
return sum(_grammar_branches(node) for node in _schema_nodes(schema))
def schema_has_open_object(schema: Mapping[str, object]) -> bool:
return any(_is_open_object(node) for node in _schema_nodes(schema))
def schema_has_lookaround_pattern(schema: Mapping[str, object]) -> bool:
return any(
isinstance(pattern := node.get("pattern"), str) and _uses_regex_lookaround(pattern)
for node in _schema_nodes(schema)
)
def schema_string_formats(schema: Mapping[str, object]) -> frozenset[str]:
return frozenset(
value for node in _schema_nodes(schema) if _may_be_string(node) and isinstance(value := node.get("format"), str)
)
def _may_be_string(node: Mapping[str, object]) -> bool:
node_type: Final = node.get("type")
return node_type is None or node_type == "string" or "string" in _list_at(node, "type")
def is_object_schema(node: Mapping[str, object]) -> bool:
node_type: Final = node.get("type")
return node_type == "object" or (isinstance(node_type, list) and "object" in node_type)
def _schema_nodes(schema: Mapping[str, object]) -> Iterator[Mapping[str, object]]:
return chain.from_iterable(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING))
_SCHEMA_LIST: Final = TypeAdapter(tuple[object, ...])
def _list_at(container: Mapping[str, object], key: str) -> tuple[object, ...]:
value: Final = container.get(key)
return _SCHEMA_LIST.validate_python(value) if isinstance(value, list) else ()
def _named_schemas(container: Mapping[str, object] | None) -> Iterator[tuple[str, Mapping[str, object]]]:
if container is None:
return iter(())
return ((key, value) for key in container if (value := _schema_at(container, key)) is not None)
def _root_definitions(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]:
for defs_key in ("$defs", "definitions"):
yield from ((f"#/{defs_key}/{name}", body) for name, body in _named_schemas(_schema_at(schema, defs_key)))
def _refs_in(schema: Mapping[str, object]) -> frozenset[str]:
return frozenset(
schema_fragment_ref(ref) for node in _schema_nodes(schema) if isinstance(ref := node.get("$ref"), str)
)
def _property_schemas(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]:
return chain.from_iterable(_named_schemas(_schema_at(node, "properties")) for node in _schema_nodes(schema))
def _is_union(node: Mapping[str, object]) -> bool:
return isinstance(node.get("anyOf"), list) or isinstance(node.get("oneOf"), list) or len(_list_at(node, "type")) > 1
def _grammar_branches(node: Mapping[str, object]) -> int:
properties: Final = _schema_at(node, "properties")
if properties is None:
return 0
optional: Final = len(_optional_property_names(node))
return 2**optional * (len(properties) - optional + 1)
def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]:
properties: Final = _schema_at(node, "properties")
if properties is None:
return frozenset()
return frozenset(properties) - frozenset(name for name in _list_at(node, "required") if isinstance(name, str))
def _is_open_object(node: Mapping[str, object]) -> bool:
additional_properties: Final = node.get("additionalProperties")
if additional_properties is True or isinstance(additional_properties, dict) or "patternProperties" in node:
return True
return is_object_schema(node) and "properties" not in node and additional_properties is not False
def _subschemas(node: Mapping[str, object]) -> Iterator[Mapping[str, object]]:
for key, value in node.items():
if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict):
@ -1894,6 +2012,11 @@ def strip_litellm_internal_message_fields(message: AllMessageValues) -> AllMessa
return _message_without_fields(message, LITELLM_INTERNAL_MESSAGE_FIELDS)
def strip_prefill_prefix_field(message: AllMessageValues) -> AllMessageValues:
"""Drop the ``prefix`` flag Mistral and DeepSeek read off a prefill, for endpoints that reject unknown keys."""
return _message_without_fields(message, frozenset({"prefix"}))
def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any:
"""
Filters a value from a dictionary

194
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -9,10 +9,11 @@ import xml.etree.ElementTree as ET
from collections.abc import Container, Iterable, Iterator, Mapping, Sequence
from enum import Enum
from types import MappingProxyType
from typing import Any, Final, TypeAlias, TypedDict, cast, overload
from typing import Annotated, Any, Final, TypeAlias, TypedDict, cast, overload
from jinja2.sandbox import ImmutableSandboxedEnvironment
from pydantic import BaseModel, JsonValue, TypeAdapter
from pydantic import BaseModel, BeforeValidator, ConfigDict, JsonValue, TypeAdapter, ValidationError, model_validator
from pydantic_core import ErrorDetails, PydanticCustomError
import litellm
import litellm.types
@ -2582,6 +2583,48 @@ def _with_native_redacted_data(
_AnthropicMessageList: TypeAlias = list[AllAnthropicPassThroughMessageValues]
def _client_tool_result(message: AllMessageValues, replaced_server_tool_answers: frozenset[str]) -> bool:
return (
message_field(message, "role") in ("tool", "function")
and message_field(message, "tool_call_id") not in replaced_server_tool_answers
)
def _tool_result_later_in_turn(
messages: Sequence[AllMessageValues], replaced_server_tool_answers: frozenset[str]
) -> tuple[bool, ...]:
from_the_end: Final = itertools.accumulate(
reversed(messages),
lambda later, message: (
message_field(message, "role") in ("user", "tool", "function")
and (later or _client_tool_result(message, replaced_server_tool_answers))
),
initial=False,
)
return tuple(from_the_end)[::-1][1:]
def _empty_text_placeholder(message: AllMessageValues) -> AnthropicMessagesTextParam:
placeholder: Final = AnthropicMessagesTextParam(type="text", text=_EMPTY_TEXT_PLACEHOLDER)
add_cache_control_to_content(anthropic_content_element=placeholder, original_content_element=message)
return placeholder
def _cache_breakpoint_placeholder(
message: AllMessageValues, tool_result_follows: bool
) -> tuple[AnthropicMessagesTextParam, ...]:
if tool_result_follows or not message_field(message, "cache_control"):
return ()
return (_empty_text_placeholder(message),)
def _empty_user_turn_placeholder(span: Sequence[AllMessageValues]) -> AnthropicMessagesTextParam | None:
empty_user_message: Final = next(
(message for message in reversed(span) if message["role"] == "user" and not message.get("content")), None
)
return None if empty_user_message is None else _empty_text_placeholder(empty_user_message)
def anthropic_messages_pt(
messages: list[AllMessageValues],
model: str,
@ -2636,6 +2679,7 @@ def anthropic_messages_pt(
is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False
force_base64 = is_bedrock_invoke or is_vertex_ai
tool_result_later_in_turn: Final = _tool_result_later_in_turn(messages, replaced_server_tool_answers)
msg_i = 0
while msg_i < len(messages):
user_content: list[AnthropicMessagesUserMessageValues] = []
@ -2648,8 +2692,12 @@ def anthropic_messages_pt(
ChatCompletionToolMessage | ChatCompletionUserMessage | ChatCompletionFunctionMessage
) = messages[msg_i]
if user_message_types_block["role"] == "user":
if isinstance(user_message_types_block["content"], list):
for m in user_message_types_block["content"]:
if not (user_block_content := user_message_types_block.get("content")):
user_content.extend(
_cache_breakpoint_placeholder(user_message_types_block, tool_result_later_in_turn[msg_i])
)
elif isinstance(user_block_content, list):
for m in user_block_content:
if m.get("type", "") == "image_url":
m = cast(ChatCompletionImageObject, m)
format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None
@ -2720,10 +2768,10 @@ def anthropic_messages_pt(
_file_content_element,
)
)
elif isinstance(user_message_types_block["content"], str):
elif isinstance(user_block_content, str):
_anthropic_content_text_element: AnthropicMessagesTextParam = {
"type": "text",
"text": user_message_types_block["content"],
"text": user_block_content,
}
_content_element = add_cache_control_to_content(
anthropic_content_element=_anthropic_content_text_element,
@ -2747,6 +2795,8 @@ def anthropic_messages_pt(
if user_content:
new_messages.append({"role": "user", "content": user_content})
elif (placeholder := _empty_user_turn_placeholder(messages[init_msg_i:msg_i])) is not None:
new_messages.append({"role": "user", "content": [placeholder]})
# Track unique tool IDs in this merge block to avoid duplication
unique_tool_ids: set[str] = set()
@ -4253,6 +4303,24 @@ def _with_text_when_document_only(message: BedrockMessageBlock) -> BedrockMessag
return BedrockMessageBlock(role="user", content=[*blocks[:cut], placeholder, *blocks[cut:]])
def _rstrip_text_block(block: BedrockContentBlock) -> BedrockContentBlock:
return BedrockContentBlock(text=block["text"].rstrip()) if "text" in block else block
def _assistant_turn_rstripped(message: BedrockMessageBlock) -> BedrockMessageBlock:
"""Claude on Converse rejects a final assistant turn that ends in whitespace, as the Anthropic path strips it."""
if message["role"] != "assistant":
return message
blocks: Final = message["content"]
last_text_index: Final = max((index for index, block in enumerate(blocks) if "text" in block), default=None)
return BedrockMessageBlock(
role="assistant",
content=[
_rstrip_text_block(block) if index == last_text_index else block for index, block in enumerate(blocks)
],
)
def _ensure_document_messages_have_text(
contents: list[BedrockMessageBlock],
) -> list[BedrockMessageBlock]:
@ -4844,7 +4912,12 @@ class BedrockConverseMessagesProcessor:
llm_provider=llm_provider,
)
return _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
converse_messages: Final = _ensure_document_messages_have_text(
_rename_duplicate_bedrock_document_names(contents)
)
if not (to_claude and converse_messages):
return converse_messages
return [*converse_messages[:-1], _assistant_turn_rstripped(converse_messages[-1])]
@staticmethod
def translate_thinking_blocks_to_reasoning_content_blocks(
@ -5253,7 +5326,10 @@ def _bedrock_converse_messages_pt(
llm_provider=llm_provider,
)
return _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
converse_messages: Final = _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
if not (to_claude and converse_messages):
return converse_messages
return [*converse_messages[:-1], _assistant_turn_rstripped(converse_messages[-1])]
def make_valid_bedrock_tool_name(input_tool_name: str) -> str:
@ -5428,14 +5504,14 @@ def _bedrock_tools_pt(
description = name
defs = parameters.pop("$defs", {})
defs_copy = copy.deepcopy(defs)
defs_copy = copy.deepcopy(defs) if isinstance(defs, dict) else {}
# Expand $ref references in parameters using the definitions
# Note: We don't pre-flatten defs as that causes exponential memory growth
# with circular references (see issue #19098). unpack_defs handles nested
# refs recursively and correctly detects/skips circular references.
unpack_defs(parameters, defs_copy)
normalize_json_schema_custom_types_to_object(parameters)
if parameters.get("type") not in _valid_json_schema_root_types:
if not (isinstance(root_type := parameters.get("type"), str) and root_type in _valid_json_schema_root_types):
parameters["type"] = "object"
tool_block = cast(
BedrockToolBlock,
@ -5516,7 +5592,7 @@ def function_call_prompt(
]
def response_schema_prompt(model: str, response_schema: dict) -> str:
def response_schema_prompt(model: str, response_schema: dict, llm_provider: str) -> str:
"""
Decides if a user-defined custom prompt or default needs to be used
@ -5532,11 +5608,8 @@ def response_schema_prompt(model: str, response_schema: dict) -> str:
custom_prompt_details = litellm.custom_prompt_dict["response_schema_prompt"]
if custom_prompt_details is not None:
return custom_prompt(
role_dict=custom_prompt_details["roles"],
initial_prompt_value=custom_prompt_details["initial_prompt_value"],
final_prompt_value=custom_prompt_details["final_prompt_value"],
messages=response_schema_as_message,
return render_custom_prompt_template(
template=custom_prompt_details, messages=response_schema_as_message, model=model, llm_provider=llm_provider
)
else:
return default_response_schema_prompt(response_schema=response_schema)
@ -5556,8 +5629,95 @@ def default_response_schema_prompt(response_schema: dict) -> str:
# Custom prompt template
def _mapping_or_none(value: object) -> object:
if value is None or issubclass(type(value), Mapping):
return value
verbose_logger.warning(
"Ignoring a custom prompt template roles value of type %s because it is not a mapping, "
"so those messages render without role markers",
type(value).__name__,
)
return None
class CustomPromptRole(BaseModel):
model_config = ConfigDict(frozen=True)
pre_message: str | None = None
post_message: str | None = None
class CustomPromptRoles(BaseModel):
model_config = ConfigDict(frozen=True, title="custom prompt template")
roles: Mapping[str, Annotated[CustomPromptRole | None, BeforeValidator(_mapping_or_none)]]
class CustomPromptTemplate(BaseModel):
model_config = ConfigDict(frozen=True, title="custom prompt template")
roles: Annotated[Mapping[object, object] | None, BeforeValidator(_mapping_or_none)] = None
initial_prompt_value: str | None = None
final_prompt_value: str | None = None
@model_validator(mode="after")
def _sets_a_rendered_field(self) -> "CustomPromptTemplate":
if self.roles is None and self.initial_prompt_value is None and self.final_prompt_value is None:
raise PydanticCustomError(
"empty_custom_prompt_template",
"it sets none of roles, initial_prompt_value, or final_prompt_value",
)
return self
def render_custom_prompt_template(
template: Mapping[str, object],
messages: Sequence[Mapping[str, object]],
model: str,
llm_provider: str,
) -> str:
try:
return _render_custom_prompt_template(template=template, messages=messages)
except ValidationError as error:
raise litellm.APIConnectionError(
message=f"The custom prompt template for {model} is invalid: {_describe_template_errors(error)}",
model=model,
llm_provider=llm_provider,
) from error
def _describe_template_errors(error: ValidationError) -> str:
return "; ".join(
_describe_template_error(detail) for detail in error.errors(include_url=False, include_input=False)
)
def _describe_template_error(detail: ErrorDetails) -> str:
location: Final = ".".join(str(part) for part in detail["loc"])
return f"{location}: {detail['msg']}" if location else detail["msg"]
def _render_custom_prompt_template(template: Mapping[str, object], messages: Sequence[Mapping[str, object]]) -> str:
validated: Final = CustomPromptTemplate.model_validate(template)
message_roles: Final = frozenset(message.get("role") for message in messages)
template_roles: Final = validated.roles or {}
used_roles: Final = CustomPromptRoles.model_validate(
{"roles": {role: entry for role, entry in template_roles.items() if role in message_roles}}
)
return custom_prompt(
role_dict={
role: {"pre_message": entry.pre_message or "", "post_message": entry.post_message or ""}
for role, entry in used_roles.roles.items()
if entry is not None
},
initial_prompt_value=validated.initial_prompt_value or "",
final_prompt_value=validated.final_prompt_value or "",
messages=list(messages),
)
def custom_prompt(
role_dict: dict,
role_dict: Mapping[str, Mapping[str, str]],
messages: list,
initial_prompt_value: str = "",
final_prompt_value: str = "",

15
litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py
View file

@ -414,10 +414,11 @@ def _assistant_renders(message: object, answered: frozenset[str]) -> bool:
def _renders(message: object, answered: frozenset[str]) -> bool:
"""Whether ``anthropic_messages_pt`` puts a block on the wire for this message.
A tool message always becomes a ``tool_result`` and a user message with string
content always becomes a text block (empty text gets a placeholder). A user list
renders only through parts of a type the converter emits; ``None``, an empty list,
and a list of other parts vanish. Assistant messages follow ``_assistant_renders``.
A tool message always becomes a ``tool_result``, and a user message with string or
missing content (``None``, an empty list) always puts its turn on the wire, an empty
turn getting a placeholder. A user list renders only through parts of a type the
converter emits, so a list of only other parts vanishes. Assistant messages follow
``_assistant_renders``.
"""
role: Final = message_field(message, "role")
if role in _TOOL_ROLES:
@ -425,8 +426,10 @@ def _renders(message: object, answered: frozenset[str]) -> bool:
if role == "assistant":
return _assistant_renders(message, answered)
content: Final = message_field(message, "content")
return isinstance(content, str) or any(
message_field(part, "type") in _RENDERED_PART_TYPES for part in parts_of(content)
return (
isinstance(content, str)
or not content
or any(message_field(part, "type") in _RENDERED_PART_TYPES for part in parts_of(content))
)

247
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -1,17 +1,20 @@
import asyncio
import json
import traceback
from collections.abc import Coroutine, Mapping, Sequence
from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from enum import Enum, auto
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypedDict, cast
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeAlias, TypedDict, cast, runtime_checkable
from pydantic import TypeAdapter
from typing_extensions import ReadOnly
import litellm
from litellm._logging import redact_internal_details_from_client_message, verbose_logger
from litellm._uuid import uuid
from litellm.constants import (
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
REALTIME_METERED_USAGE_KEY,
REALTIME_SESSION_FAILURE_LOGGED_KEY,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
@ -98,6 +101,16 @@ class ScopedWebSocket(Protocol):
def scope(self) -> _ASGIScope: ...
@runtime_checkable
class _ReplayScopeWebSocket(Protocol):
@property
def scope(self) -> MutableMapping[str, object]: ...
def client_websocket_scope(websocket: object) -> MutableMapping[str, object] | None:
return websocket.scope if isinstance(websocket, _ReplayScopeWebSocket) else None
class _ClientWebSocket(ScopedWebSocket, Protocol):
async def send_text(self, data: str) -> None: ...
async def receive_text(self) -> str: ...
@ -116,10 +129,87 @@ def _decode_json_object(payload: str) -> Mapping[str, object]:
return json.loads(payload)
_GUARDRAIL_TURN_DETECTION_GATE: Final[Mapping[str, object]] = MappingProxyType(
{"type": "server_vad", "create_response": False}
)
_EMPTY_SESSION: Final[Mapping[str, object]] = MappingProxyType({})
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
_GUARDRAIL_EVENT_ID_PREFIX: Final = "litellm_guardrail_"
def _json_object(value: object) -> Mapping[str, object] | None:
return _JSON_OBJECT.validate_python(value) if isinstance(value, Mapping) else None
def _mapping_field(container: Mapping[str, object], key: str) -> Mapping[str, object] | None:
return _json_object(container.get(key))
def _guardrail_event_id() -> str:
return f"{_GUARDRAIL_EVENT_ID_PREFIX}{uuid.uuid4().hex}"
def _error_event_id(event: Mapping[str, object]) -> str | None:
error: Final = _mapping_field(event, "error")
caused_by: Final = None if error is None else error.get("event_id")
return caused_by if isinstance(caused_by, str) else None
_TurnDetectionGate: TypeAlias = Callable[[object], Mapping[str, object]]
def _turn_detection_with_auto_response_off(turn_detection: object) -> Mapping[str, object]:
client_turn_detection: Final = _json_object(turn_detection)
if client_turn_detection is None:
return dict(_GUARDRAIL_TURN_DETECTION_GATE)
return {**client_turn_detection, "create_response": False}
def _nested_turn_detection_part(session: Mapping[str, object], gate: _TurnDetectionGate) -> Mapping[str, object]:
audio: Final = _mapping_field(session, "audio")
if audio is None:
return {}
audio_input: Final = _mapping_field(audio, "input")
if audio_input is None or "turn_detection" not in audio_input:
return {}
gated_input: Final = {
**audio_input,
"turn_detection": gate(audio_input["turn_detection"]),
}
return {"audio": {**audio, "input": gated_input}}
def _with_create_response_disabled(
session: Mapping[str, object], gate: _TurnDetectionGate
) -> Mapping[str, object] | None:
"""``session`` with every turn_detection the client sent unable to auto-respond, or None when it sent none."""
flat_part: Final = {"turn_detection": gate(session["turn_detection"])} if "turn_detection" in session else {}
nested_part: Final = _nested_turn_detection_part(session, gate)
if not flat_part and not nested_part:
return None
return {**session, **flat_part, **nested_part}
def _gate_session_update(
session: object, gate: _TurnDetectionGate, inject_gate_when_absent: bool
) -> Mapping[str, object] | None:
"""The session to forward under a transcript guardrail, or None when the frame needs no change."""
client_session: Final = _json_object(session)
if client_session is None:
return None
gated: Final = _with_create_response_disabled(client_session, gate)
if gated is not None or not inject_gate_when_absent:
return gated
return {**client_session, "turn_detection": gate(None)}
class RealtimeEventNormalizer(Protocol):
def should_drop(self, event: object) -> bool: ...
def normalize(self, event: dict) -> dict: ...
def patch_outgoing_session(self, session: dict) -> dict: ...
def gate_turn_detection(self, turn_detection: object, /) -> Mapping[str, object]: ...
@property
def deletes_conversation_items(self) -> bool: ...
class RealtimeUsageMeter(Protocol):
@ -154,6 +244,7 @@ class RealTimeStreaming:
logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
):
self.websocket: _ClientWebSocket = websocket
self._replay_scope: Final = client_websocket_scope(websocket)
self.backend_ws = backend_ws
self.logging_obj = logging_obj
self._logging_worker = logging_worker
@ -197,6 +288,8 @@ class RealTimeStreaming:
# Track whether we have already sent the guardrail turn-detection update
# that disables provider auto-response for transcription guardrails.
self._guardrail_turn_detection_update_sent: bool = False
self._session_gate_sent: bool = False
self._guardrail_cancel_event_ids: frozenset[str] = frozenset()
# Deferred Gemini Live setup: Pipecat may stream audio before session.update.
# Buffer client audio until the backend acknowledges setup (setupComplete).
self._backend_setup_complete: bool = provider_config is None or provider_config.requires_session_configuration()
@ -213,6 +306,12 @@ class RealTimeStreaming:
self._is_transcription_session: bool = force_transcription_model is not None
# Optional per-provider GA event normalizer (e.g. XAIRealtimeNormalizer).
self._event_normalizer = event_normalizer
self._gate_turn_detection: Final[_TurnDetectionGate] = (
_turn_detection_with_auto_response_off if event_normalizer is None else event_normalizer.gate_turn_detection
)
self._deletes_conversation_items: Final = (
event_normalizer is None or event_normalizer.deletes_conversation_items
)
self._usage_meter = usage_meter
@property
@ -734,20 +833,37 @@ class RealTimeStreaming:
except (json.JSONDecodeError, TypeError):
return
def _make_turn_detection_update(self, turn_detection: Mapping[str, object]) -> dict[str, object]:
session: Final[dict[str, object]] = (
{"turn_detection": turn_detection}
if self._backend_uses_beta_protocol
else {"type": "realtime", "audio": {"input": {"turn_detection": turn_detection}}}
)
return {"type": "session.update", "session": session}
def _make_disable_auto_response_message(self) -> str:
"""Return a session.update that disables VAD auto-response."""
turn_detection: Final[dict[str, str | bool]] = {
"type": "server_vad",
"create_response": False,
}
if self._backend_uses_beta_protocol:
session: dict[str, object] = {"turn_detection": turn_detection}
else:
session = {
"type": "realtime",
"audio": {"input": {"turn_detection": turn_detection}},
}
return json.dumps({"type": "session.update", "session": session})
return json.dumps(self._make_turn_detection_update(self._gate_turn_detection(None)))
async def _send_guardrail_cancel(self) -> None:
event_id: Final = _guardrail_event_id()
self._guardrail_cancel_event_ids = self._guardrail_cancel_event_ids | {event_id}
await self._send_to_backend(json.dumps({"type": "response.cancel", "event_id": event_id}))
async def _withdraw_blocked_turn(self, item_id: str | None) -> None:
await self._send_guardrail_cancel()
if item_id is not None and self._deletes_conversation_items:
await self._send_to_backend(json.dumps({"type": "conversation.item.delete", "item_id": item_id}))
def _is_guardrail_cancel_error(self, event: Mapping[str, object]) -> bool:
return _error_event_id(event) in self._guardrail_cancel_event_ids
async def _send_session_gate_once(self) -> None:
"""Gate the backend's default turn_detection ahead of the first client session.update, which overrides it."""
if self._session_gate_sent or not self._should_disable_vad_auto_response():
return
self._session_gate_sent = True
await self._send_to_backend(self._make_disable_auto_response_message())
async def _maybe_send_guardrail_turn_detection_update(self) -> None:
"""Disable provider auto-response once when transcription guardrails are enabled."""
@ -921,7 +1037,7 @@ class RealTimeStreaming:
if pre_block_backend_message is not None:
await self._send_to_backend(pre_block_backend_message)
if not self._is_transcription_session:
await self._send_to_backend(json.dumps({"type": "response.cancel"}))
await self._withdraw_blocked_turn(item_id)
# Send the policy violation hint (shows as small gray status text in UI).
await self.websocket.send_text(
json.dumps(
@ -1051,10 +1167,7 @@ class RealTimeStreaming:
self.store_message(event_str)
self._capture_transcription_usage(event)
await self._send_event_to_client(event, event_str)
blocked = await self.run_realtime_guardrails(
cast(str, transcript),
item_id=cast(str | None, event.get("item_id")),
)
blocked = await self.run_realtime_guardrails(cast(str, transcript))
if not blocked and self._should_disable_vad_auto_response():
await self._send_to_backend(json.dumps({"type": "response.create"}))
continue
@ -1086,7 +1199,7 @@ class RealTimeStreaming:
if event_type == "session.created" and self._should_disable_vad_auto_response():
self.store_message(event_obj)
await self.websocket.send_text(self._event_to_client_json(event_obj))
await self._send_to_backend(self._make_disable_auto_response_message())
await self._send_session_gate_once()
return True
if event_type == "conversation.item.input_audio_transcription.completed":
@ -1140,6 +1253,10 @@ class RealTimeStreaming:
if self._usage_meter is not None:
self._usage_meter.observe_backend_event(event)
if event.get("type") == "error" and self._is_guardrail_cancel_error(event):
verbose_logger.debug("Realtime: not forwarding an error caused by a guardrail event: %s", event)
continue
if await self._handle_raw_backend_message(event, raw_response):
continue
@ -1346,6 +1463,13 @@ class RealTimeStreaming:
return item
async def _receive_client_message(self) -> str | None:
refused_opening_message: Final = (
None
if self._replay_scope is None
else self._replay_scope.pop(BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY, None)
)
if isinstance(refused_opening_message, str):
return refused_opening_message
try:
return await self.websocket.receive_text()
except Exception as e: # noqa: BLE001 # whatever the client socket raises, the client is gone
@ -1472,19 +1596,21 @@ class RealTimeStreaming:
and self.session_configuration_request is None
and not self._guardrail_turn_detection_update_sent
and self._should_disable_vad_auto_response()
):
session: Mapping[str, object] | None = msg_obj.setdefault("session", {})
if isinstance(session, dict):
existing_td = session.get("turn_detection")
if not isinstance(existing_td, dict):
existing_td = {}
existing_td["create_response"] = False
session["turn_detection"] = existing_td
message = json.dumps(msg_obj)
guardrail_turn_detection_injected = True
verbose_logger.debug(
"Injected turn_detection into first session.update for audio transcription guardrails"
and (
gated_first := _gate_session_update(
client_event.get("session", _EMPTY_SESSION),
self._gate_turn_detection,
inject_gate_when_absent=True,
)
)
is not None
):
msg_obj["session"] = gated_first
message = json.dumps(msg_obj)
guardrail_turn_detection_injected = True
verbose_logger.debug(
"Injected turn_detection into first session.update for audio transcription guardrails"
)
## GUARDRAIL: Force ``create_response`` to False in any
# client-provided ``turn_detection`` so a later
@ -1492,52 +1618,25 @@ class RealTimeStreaming:
# and bypass the transcription guardrail after the
# initial disable. Covers both the flat beta key and the
# nested GA ``audio.input.turn_detection`` shape, since
# the GA remap below also accepts either form. Skipped
# when the injection block above already ran for this
# message, to avoid redundant double-serialization.
# the GA remap below also accepts either form. A later
# update with no turn_detection is forwarded as is: OpenAI
# keeps the earlier disable for an unset field, and xAI
# falls back to its manual default, which never auto-responds.
if (
msg_type == "session.update"
and not guardrail_turn_detection_injected
and self._should_disable_vad_auto_response()
and (
gated_later := _gate_session_update(
client_event.get("session"),
self._gate_turn_detection,
inject_gate_when_absent=False,
)
)
is not None
):
session = client_event.get("session")
if isinstance(session, dict):
td_overridden = False
flat_td = session.get("turn_detection")
flat_td_present = flat_td is not None
if flat_td_present:
if not isinstance(flat_td, dict):
flat_td = {}
if flat_td.get("create_response") is not False:
flat_td["create_response"] = False
session["turn_detection"] = flat_td
td_overridden = True
nested_td_present = False
audio = session.get("audio")
if isinstance(audio, dict):
audio_input = audio.get("input")
if isinstance(audio_input, dict):
nested_td = audio_input.get("turn_detection")
if nested_td is not None:
nested_td_present = True
if not isinstance(nested_td, dict):
nested_td = {}
if nested_td.get("create_response") is not False:
nested_td["create_response"] = False
audio_input["turn_detection"] = nested_td
td_overridden = True
# Symmetric with the first-update injection block:
# if the client omitted turn_detection entirely on
# a subsequent session.update, still inject the
# ``create_response: False`` override so the
# transcription guardrail cannot be re-enabled by
# any downstream merge that drops the original
# disable.
if not flat_td_present and not nested_td_present:
session["turn_detection"] = {"create_response": False}
td_overridden = True
if td_overridden:
message = json.dumps(msg_obj)
msg_obj["session"] = gated_later
message = json.dumps(msg_obj)
# GA compatibility: remap beta-style session fields only when
# the upstream is in GA mode. Beta upstreams expect the flat
@ -1590,6 +1689,8 @@ class RealTimeStreaming:
# would permanently disable the injection if ``_send_to_backend``
# raised — neither this loop nor
# ``_maybe_send_guardrail_turn_detection_update`` would retry.
if msg_type == "session.update" and self.provider_config is None:
await self._send_session_gate_once()
sent = await self._send_to_backend(message)
if guardrail_turn_detection_injected and sent:
self._guardrail_turn_detection_update_sent = True

6
litellm/litellm_core_utils/sentry_scrubbing.py
View file

@ -3,7 +3,7 @@ from __future__ import annotations
import re
from collections.abc import Callable, Mapping, Sequence
from functools import reduce
from typing import TYPE_CHECKING, Final, TypeAlias, cast
from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast
from pydantic import JsonValue
from sentry_sdk.scrubber import DEFAULT_DENYLIST, DEFAULT_PII_DENYLIST, EventScrubber
@ -65,6 +65,8 @@ class SentryInitOptions(TypedDict):
traces_sample_rate: ReadOnly[float]
sample_rate: ReadOnly[float]
send_default_pii: ReadOnly[bool]
max_request_body_size: ReadOnly[Literal["never"]]
include_local_variables: ReadOnly[bool]
event_scrubber: ReadOnly[EventScrubber]
before_send: ReadOnly[EventScrubFn]
before_send_transaction: ReadOnly[EventScrubFn]
@ -140,6 +142,8 @@ def build_sentry_init_options(env: Mapping[str, str]) -> SentryInitOptions:
traces_sample_rate=float(env.get("SENTRY_API_TRACE_RATE") or "1.0"),
sample_rate=float(env.get("SENTRY_API_SAMPLE_RATE") or "1.0"),
send_default_pii=send_default_pii,
max_request_body_size="never",
include_local_variables=send_default_pii,
event_scrubber=EventScrubber(
denylist=list(SECRET_FIELD_NAMES),
pii_denylist=list(PII_FIELD_NAMES),

7
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -1,4 +1,5 @@
import base64
import builtins
import time
from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass, replace
@ -118,6 +119,7 @@ class _BaseChunk(TypedDict, total=False):
created: ReadOnly[int]
model: ReadOnly[str]
system_fingerprint: ReadOnly[str | None]
provider_specific_fields: ReadOnly[Mapping[str, builtins.object] | None]
service_tier: ReadOnly[str | None]
choices: ReadOnly[Required[Sequence[StreamingChoices]]]
_hidden_params: ReadOnly[_ChunkHiddenParams]
@ -430,6 +432,11 @@ class ChunkProcessor:
"created": created,
"model": model,
"system_fingerprint": system_fingerprint,
**(
{"provider_specific_fields": chunks[0].get("provider_specific_fields")}
if system_fingerprint == "fp_ollama"
else {}
),
**(
MappingProxyType({"service_tier": service_tier})
if service_tier is not None

107
litellm/litellm_core_utils/streaming_handler.py
View file

@ -14,6 +14,7 @@ from typing import Any, Final, NoReturn, Protocol, TypeVar, cast
import anyio
import httpx
from openai.types.chat import ChatCompletionChunk
from pydantic import BaseModel, ValidationError
from typing_extensions import NotRequired, TypedDict
@ -27,6 +28,7 @@ from litellm.litellm_core_utils.model_response_utils import (
)
from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject
from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import OpenAIChatCompletionChunk
from litellm.types.router import GenericLiteLLMParams
@ -240,6 +242,7 @@ class CustomStreamWrapper:
self.completion_stream = completion_stream
self.sent_first_chunk = False
self.sent_last_chunk = False
self._failure_logging: asyncio.Task[None] | None = None
self._closed = False
self._stream_created_time: float = time.time()
@ -267,6 +270,7 @@ class CustomStreamWrapper:
self.holding_chunk = ""
self.complete_response = ""
self.response_uptil_now = ""
self._emitted_disqualifying_content = False
_model_info: Final[dict] = litellm_params.model_info or {}
_api_base: Final = get_api_base(
@ -285,6 +289,7 @@ class CustomStreamWrapper:
self._response_headers = _response_headers
self.response_id: str | None = None
self._ollama_response_ids = OllamaResponseIdNormalizer()
self.received_model_id: str | None = None
self.logging_loop = None
self.rules = Rules()
@ -824,7 +829,7 @@ class CustomStreamWrapper:
pass
else:
model_response.choices = [StreamingChoices(finish_reason=None)]
return model_response
return self._ollama_response_ids.apply(model_response)
def is_delta_empty(self, delta: Delta) -> bool:
is_empty = True
@ -1123,31 +1128,24 @@ class CustomStreamWrapper:
"""
if self.merge_reasoning_content_in_choices is True:
reasoning_content: Final = getattr(model_response.choices[0].delta, "reasoning_content", None)
if reasoning_content:
if self.sent_first_thinking_block is False:
# Ensure content is not None before concatenation
if model_response.choices[0].delta.content is None:
model_response.choices[0].delta.content = ""
model_response.choices[0].delta.content += "<think>" + reasoning_content
self.sent_first_thinking_block = True
elif (
self.sent_first_thinking_block is True
and hasattr(model_response.choices[0].delta, "reasoning_content")
and model_response.choices[0].delta.reasoning_content
):
model_response.choices[0].delta.content = reasoning_content
elif (
self.sent_first_thinking_block is True
and not self.sent_last_thinking_block
and model_response.choices[0].delta.content
):
model_response.choices[0].delta.content = "</think>" + (model_response.choices[0].delta.content or "")
self.sent_last_thinking_block = True
if self.merge_reasoning_content_in_choices is not True:
return
delta: Final = model_response.choices[0].delta
reasoning_content: Final = getattr(delta, "reasoning_content", None)
content: Final = delta.content or ""
if reasoning_content:
self._emitted_disqualifying_content = True
closes_thinking: Final = bool(content) and not self.sent_last_thinking_block
opening: Final = "" if self.sent_first_thinking_block else "<think>"
delta.content = opening + reasoning_content + ("</think>" + content if closes_thinking else "")
self.sent_first_thinking_block = True
self.sent_last_thinking_block = self.sent_last_thinking_block or closes_thinking
elif self.sent_first_thinking_block and not self.sent_last_thinking_block and content:
delta.content = "</think>" + content
self.sent_last_thinking_block = True
if hasattr(model_response.choices[0].delta, "reasoning_content"):
del model_response.choices[0].delta.reasoning_content
if hasattr(delta, "reasoning_content"):
del delta.reasoning_content
def _dispatch_provider_chunk(
self,
@ -1449,8 +1447,14 @@ class CustomStreamWrapper:
return _ProviderChunkParsed(response_obj)
def chunk_creator(self, chunk: Any):
if self.custom_llm_provider != "cached_response" and isinstance(
chunk, (ChatCompletionChunk, ModelResponseStream)
):
self._ollama_response_ids.observe(chunk)
if hasattr(chunk, "id"):
self.response_id = chunk.id
if self._ollama_response_ids.response_id is not None:
self.response_id = self._ollama_response_ids.response_id
provider_response_model: Final = _provider_response_model(chunk)
if provider_response_model is not None:
self._provider_response_model = provider_response_model
@ -1856,9 +1860,7 @@ class CustomStreamWrapper:
if response.choices:
choice = response.choices[0]
if isinstance(choice, StreamingChoices):
self.response_uptil_now += choice.delta.get("content", "") or ""
else:
self.response_uptil_now += ""
self._accumulate_streamed_delta(choice.delta)
self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
# HANDLE STREAM OPTIONS
self.chunks.append(response)
@ -2075,9 +2077,7 @@ class CustomStreamWrapper:
if processed_chunk.choices:
choice = processed_chunk.choices[0]
if isinstance(choice, StreamingChoices):
self.response_uptil_now += choice.delta.get("content", "") or ""
else:
self.response_uptil_now += ""
self._accumulate_streamed_delta(choice.delta)
self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
# Add mcp_list_tools to first chunk if present
if not self.sent_first_chunk and processed_chunk.choices:
@ -2147,9 +2147,7 @@ class CustomStreamWrapper:
choice = processed_chunk.choices[0]
if isinstance(choice, StreamingChoices):
self.response_uptil_now += choice.delta.get("content", "") or ""
else:
self.response_uptil_now += ""
self._accumulate_streamed_delta(choice.delta)
self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
# RETURN RESULT
self.chunks.append(processed_chunk)
@ -2163,7 +2161,7 @@ class CustomStreamWrapper:
if self.logging_obj is not None:
self._record_partial_usage_for_failure()
## LOGGING
asyncio.create_task(
self._failure_logging = asyncio.create_task(
self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True)
)
await self.aclose()
@ -2296,11 +2294,17 @@ class CustomStreamWrapper:
if self.logging_obj is not None:
self._record_partial_usage_for_failure()
## LOGGING
asyncio.create_task(
self._failure_logging = asyncio.create_task(
self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True)
)
self._handle_stream_fallback_error(e)
async def wait_for_failure_logging(self, timeout: float) -> None:
"""Let this stream's failure log read its own deployment before a fallback reuses the logging object."""
if self._failure_logging is None:
return
await asyncio.wait((self._failure_logging,), timeout=timeout)
def _record_partial_usage_for_failure(self) -> None:
"""
A stream that breaks mid-flight still billed the provider for the chunks
@ -2337,6 +2341,35 @@ class CustomStreamWrapper:
recover_error,
)
# Delta fields a text-only prefill continuation cannot carry, so a stream
# that emitted any of them is not eligible for mid-stream continuation.
_CONTINUATION_DISQUALIFYING_DELTA_FIELDS: Final = (
"tool_calls",
"function_call",
"reasoning_content",
"thinking_blocks",
"reasoning_items",
"audio",
"images",
"annotations",
)
@classmethod
def _delta_disqualifies_continuation(cls, delta: object) -> bool:
get: Final = getattr(delta, "get", None)
if not callable(get):
return False
return any(get(field) for field in cls._CONTINUATION_DISQUALIFYING_DELTA_FIELDS)
def _accumulate_streamed_delta(self, delta: object) -> None:
# Shared by the sync, async, and non-aiohttp iteration sites so answer
# text and the disqualifying-content latch stay in step across all three.
get: Final = getattr(delta, "get", None)
content: Final = get("content") if callable(get) else None
self.response_uptil_now += content if isinstance(content, str) else ""
if not self._emitted_disqualifying_content and self._delta_disqualifies_continuation(delta):
self._emitted_disqualifying_content = True
def _handle_stream_fallback_error(self, e: Exception) -> "NoReturn":
"""
Common error handling for both __next__ and __anext__.
@ -2368,6 +2401,7 @@ class CustomStreamWrapper:
original_exception=e.original_exception,
generated_content=self.response_uptil_now,
is_pre_first_chunk=not self.sent_first_chunk,
emitted_disqualifying_content=self._emitted_disqualifying_content or e.emitted_disqualifying_content,
)
# Map to OpenAI exception format. Some providers' mappers (e.g.
@ -2428,6 +2462,7 @@ class CustomStreamWrapper:
original_exception=mapped_exception,
generated_content=self.response_uptil_now,
is_pre_first_chunk=not self.sent_first_chunk,
emitted_disqualifying_content=self._emitted_disqualifying_content,
)
@staticmethod

164
litellm/litellm_core_utils/tool_fields.py Normal file
View file

@ -0,0 +1,164 @@
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from functools import reduce
from typing import Annotated, Final, TypeAlias
from pydantic import BeforeValidator, ConfigDict, Json, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
ROUTES_WITH_TOOLS: Final = frozenset(
{
"acompletion",
"aresponses",
"acompact_responses",
"anthropic_messages",
"agenerate_content",
"agenerate_content_stream",
"acreate_interaction",
}
)
TOOL_FIELDS: Final = frozenset({"tools", "tool_choice", "parallel_tool_calls", "toolConfig", "tool_config"})
_TOOLS: Final = TypeAdapter(list[object])
_FIELD_LIST: Final = TypeAdapter(list[object], config=ConfigDict(strict=True))
_JSON_OBJECT: Final = TypeAdapter(Json[dict[str, object]])
def _schema_as_read(value: object) -> object:
if not value:
return None
if not isinstance(value, str):
return value
try:
return _JSON_OBJECT.validate_python(value)
except ValidationError:
return value
_Schema: TypeAlias = Annotated[dict[str, object] | None, BeforeValidator(_schema_as_read)]
class _ToolFunction(TypedDict, total=False):
parameters: ReadOnly[_Schema]
class _ToolEntry(TypedDict, total=False):
function: ReadOnly[_ToolFunction | None]
parameters: ReadOnly[_Schema]
input_schema: ReadOnly[_Schema]
vector_store_ids: ReadOnly[list[str] | None]
_TOOL_ENTRY: Final = TypeAdapter(_ToolEntry)
_WEB_SEARCH_OPTIONS: Final = TypeAdapter(dict[str, object] | None, config=ConfigDict(strict=True))
@dataclass(frozen=True, slots=True)
class DropPolicy:
drop_params: bool
additional_drop_params: frozenset[str]
def drops(self, field: str) -> bool:
return self.drop_params or field in self.additional_drop_params
@dataclass(frozen=True, slots=True)
class MalformedField:
field: str
message: str
dropped: Callable[[Mapping[str, object]], Mapping[str, object]]
def _field_names(value: object) -> frozenset[str]:
try:
return frozenset(name for name in _FIELD_LIST.validate_python(value) if isinstance(name, str))
except ValidationError:
return frozenset()
def drop_policy(params: Mapping[str, object]) -> DropPolicy:
return DropPolicy(
drop_params=litellm.drop_params is True or normalize_drop_params(params.get("drop_params")) is True,
additional_drop_params=_field_names(params.get("additional_drop_params")),
)
def first_validation_error(e: ValidationError, prefix: tuple[str | int, ...] = ()) -> str:
error: Final = e.errors(include_url=False)[0]
return f"{'.'.join(str(part) for part in (*prefix, *error['loc']))}: {error['msg']}"
def _tool_entry_error(index: int, entry: object) -> str | None:
try:
_TOOL_ENTRY.validate_python(entry, strict=True)
except ValidationError as e:
return first_validation_error(e, prefix=("tools", index))
return None
def _without_tool_fields(data: Mapping[str, object]) -> Mapping[str, object]:
return {name: value for name, value in data.items() if name not in TOOL_FIELDS}
def _malformed_tools(data: Mapping[str, object]) -> MalformedField | None:
tools: Final = data.get("tools")
if tools is None:
return None
try:
entries: Final = _TOOLS.validate_python(tools)
except ValidationError as e:
return MalformedField("tools", first_validation_error(e, prefix=("tools",)), _without_tool_fields)
entry_errors: Final = tuple(_tool_entry_error(index, entry) for index, entry in enumerate(entries))
first_error: Final = next((error for error in entry_errors if error is not None), None)
if first_error is None:
return None
kept: Final = [entry for entry, error in zip(entries, entry_errors) if error is None]
return MalformedField(
"tools", first_error, lambda fields: {**fields, "tools": kept} if kept else _without_tool_fields(fields)
)
def _without_web_search_options(data: Mapping[str, object]) -> Mapping[str, object]:
return {name: value for name, value in data.items() if name != "web_search_options"}
def _malformed_web_search_options(data: Mapping[str, object]) -> MalformedField | None:
try:
_WEB_SEARCH_OPTIONS.validate_python(data.get("web_search_options"))
except ValidationError as e:
return MalformedField(
"web_search_options",
first_validation_error(e, prefix=("web_search_options",)),
_without_web_search_options,
)
return None
def resolve_tool_fields(
data: Mapping[str, object], candidates: Callable[[], tuple[DropPolicy, ...]]
) -> Mapping[str, object] | MalformedField:
checked: Final = (_malformed_tools(data), _malformed_web_search_options(data))
malformed: Final = tuple(field for field in checked if field is not None)
if not malformed:
return data
policies: Final = candidates()
rejected: Final = next(
(field for field in malformed if not any(policy.drops(field.field) for policy in policies)), None
)
if rejected is not None:
return rejected
return reduce(
lambda resolved, field: field.dropped(resolved),
(field for field in malformed if all(policy.drops(field.field) for policy in policies)),
data,
)
def resolve_deployment_tool_fields(
kwargs: Mapping[str, object], route: object
) -> Mapping[str, object] | MalformedField:
if "proxy_server_request" not in kwargs or route not in ROUTES_WITH_TOOLS:
return kwargs
return resolve_tool_fields(kwargs, lambda: (drop_policy(kwargs),))

5
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -45,6 +45,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
effective_scan_only_tool_results_for_guardrail,
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
list_field,
merge_guardrailed_scoped_messages,
merge_returned_tools_into_request_tools,
scoped_structured_message_indices,
@ -924,10 +925,10 @@ class AnthropicMessagesHandler(BaseTranslation):
images=(),
)
def extract_request_tool_names(self, data: dict) -> list[str]:
def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
"""Extract every tool name in an Anthropic messages request: tools[].name, plus
tools[].function.name for OpenAI-format tools the bridge forwards verbatim."""
return [name for tool in data.get("tools") or [] for name in anthropic_tool_names(tool)]
return [name for tool in list_field(data, "tools") for name in anthropic_tool_names(tool)]
@classmethod
def _extract_input_text_and_images(

114
litellm/llms/anthropic/chat/transformation.py
View file

@ -1,3 +1,4 @@
import copy
import json
import re
import time
@ -24,7 +25,18 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.prompt_templates.common_utils import (
DEFS_MAX_INLINED_BYTES,
is_object_schema,
sanitize_input_schema_for_anthropic,
schema_fragment_ref,
schema_grammar_branch_count,
schema_has_lookaround_pattern,
schema_has_open_object,
schema_has_unresolvable_ref,
schema_optional_property_count,
schema_string_formats,
schema_union_property_count,
unpack_defs,
)
from litellm.litellm_core_utils.prompt_templates.image_handling import (
RemoteMedia,
@ -43,7 +55,7 @@ from litellm.llms.base_llm.base_utils import (
type_to_response_format_param,
unwrap_response_format_envelope,
)
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS, BaseConfig, BaseLLMException
from litellm.types.llms.anthropic import (
ANTHROPIC_ADVISOR_TOOL_TYPE,
ANTHROPIC_BETA_HEADER_VALUES,
@ -150,6 +162,12 @@ _ANTHROPIC_TOOL_NAME_MAX_LEN: Final = 128
_DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
_TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_TOOLS_ADAPTER: Final = TypeAdapter(list[object])
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
)
NATIVE_OUTPUT_MAX_UNION_PARAMETERS: Final = 16
NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS: Final = 24
NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES: Final = 2048
def _tool_type(tool: object) -> str | None:
@ -753,7 +771,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
grammar Anthropic compiles the schema into, so the API rejects them with a
400 ``invalid_request_error`` (e.g. "output_format.schema: For 'array' type,
property 'uniqueItems' is not supported"):
- maxItems/minItems/uniqueItems/contains/minContains/maxContains/prefixItems: array constraints
- maxItems/minItems above 1/uniqueItems/contains/minContains/maxContains/prefixItems: array constraints
- minimum/maximum/exclusiveMinimum/exclusiveMaximum/multipleOf: numeric constraints
- minLength/maxLength: string constraints
- minProperties/maxProperties/patternProperties/propertyNames: object constraints
@ -810,7 +828,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"else": "conditional (else): {}",
"not": "must not match: {}",
}
unsupported_fields: Final = set(constraint_labels)
min_items: Final = schema.get("minItems")
enforced_min_items: Final = (
isinstance(min_items, int) and not isinstance(min_items, bool) and min_items in (0, 1)
)
unsupported_fields: Final = set(constraint_labels) - ({"minItems"} if enforced_min_items else set())
# Build description additions from removed constraints. Iterating
# constraint_labels (not the set) keeps the note order deterministic across
@ -818,7 +840,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# PYTHONHASHSEED and stay cache-friendly.
constraint_descriptions: Final[list] = []
for field, label in constraint_labels.items():
if field not in schema:
if field not in schema or field not in unsupported_fields:
continue
value = schema[field]
# A falsy boolean constraint (e.g. ``uniqueItems: false``) imposes no
@ -853,7 +875,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Already handled above
continue
if key in ("properties", "$defs") and isinstance(value, dict):
if key in ("properties", "$defs", "definitions") and isinstance(value, dict):
nested_schemas: dict[str, Any] = value # mutable-ok: filter_anthropic_output_schema takes a dict
result[key] = {k: AnthropicConfig.filter_anthropic_output_schema(v) for k, v in nested_schemas.items()}
elif key == "items" and isinstance(value, dict):
@ -865,6 +887,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
]
elif key == "allOf" and isinstance(value, list):
result[key] = [AnthropicConfig.filter_anthropic_output_schema(item) for item in value]
elif key == "$ref" and isinstance(value, str):
result[key] = schema_fragment_ref(value)
elif key == "oneOf" and isinstance(value, list):
# Anthropic rejects oneOf ("Schema type 'oneOf' is not supported");
# the Anthropic SDK rewrites it to anyOf, so do the same.
@ -876,7 +900,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Anthropic requires additionalProperties=false for object schemas
# See: https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs
if result.get("type") == "object":
if is_object_schema(result):
result["additionalProperties"] = False
return result
@ -1628,32 +1652,55 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return json_schema
@staticmethod
def _defs_unpacked(json_schema: Mapping[str, object]) -> Mapping[str, object]:
unpacked: Final = copy.deepcopy(dict(json_schema))
defs: Final = unpacked.pop("$defs", unpacked.pop("definitions", None))
if isinstance(defs, dict) and defs:
unpack_defs(unpacked, defs, max_inlined_bytes=DEFS_MAX_INLINED_BYTES)
return unpacked
@staticmethod
def native_output_schema(json_schema: Mapping[str, object]) -> Mapping[str, object] | None:
if (
schema_has_unresolvable_ref(json_schema)
or schema_has_open_object(json_schema)
or schema_has_lookaround_pattern(json_schema)
or not schema_string_formats(json_schema) <= _NATIVE_OUTPUT_STRING_FORMATS
or schema_union_property_count(json_schema) > NATIVE_OUTPUT_MAX_UNION_PARAMETERS
or schema_optional_property_count(json_schema) > NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS
or schema_grammar_branch_count(json_schema) > NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES
):
return None
return AnthropicConfig.filter_anthropic_output_schema(dict(json_schema))
def uses_native_structured_output_for_schema(
self,
model: str,
json_schema: Mapping[str, object] | None,
request_params: Mapping[str, object],
) -> bool:
open_object_takes_forced_json_tool: Final = (
json_schema is not None
and schema_has_open_object(json_schema)
and not self.is_thinking_requested(request_params)
and not self.is_thinking_enabled(dict(request_params))
and not request_params.get("tools")
and not AnthropicModelInfo.forced_tool_use_unsupported(model)
)
return self.uses_native_structured_output(model) and not open_object_takes_forced_json_tool
def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None:
return self.filter_anthropic_output_schema(dict(self._defs_unpacked(json_schema)))
def map_response_format_to_anthropic_output_format(self, value: dict | None) -> AnthropicOutputSchema | None:
json_schema: dict | None = self._extract_json_schema_from_response_format(value)
json_schema: Final = self._extract_json_schema_from_response_format(value)
if json_schema is None:
return None
# Resolve $ref/$defs before filtering — Anthropic doesn't support
# external schema references (e.g., /$defs/CalendarEvent).
import copy
from litellm.litellm_core_utils.prompt_templates.common_utils import (
DEFS_MAX_INLINED_BYTES,
unpack_defs,
)
json_schema = copy.deepcopy(json_schema)
defs: Final = json_schema.pop("$defs", json_schema.pop("definitions", {}))
if defs:
unpack_defs(json_schema, defs, max_inlined_bytes=DEFS_MAX_INLINED_BYTES)
# Filter out unsupported fields for Anthropic's output_format API
filtered_schema: Final = self.filter_anthropic_output_schema(json_schema)
return AnthropicOutputSchema(
type="json_schema",
schema=filtered_schema,
)
native_schema: Final = self.native_output_format_schema(json_schema)
if native_schema is None:
return None
return AnthropicOutputSchema(type="json_schema", schema=dict(native_schema))
def map_response_format_to_anthropic_tool(
self, value: dict | None, optional_params: dict, is_thinking_enabled: bool
@ -1820,7 +1867,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
output_key=param,
)
elif param == "response_format" and isinstance(value, dict):
if self.uses_native_structured_output(model):
if self.uses_native_structured_output_for_schema(
model,
self._extract_json_schema_from_response_format(value),
REQUEST_PARAMS.validate_python(non_default_params),
):
_output_format = self.map_response_format_to_anthropic_output_format(value)
if _output_format is not None:
optional_params["output_format"] = _output_format
@ -2979,6 +3030,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
completion_response["stop_reason"] = "stop"
if refusal_fields:
json_mode_message.provider_specific_fields = dict(refusal_fields)
if thinking_blocks:
json_mode_message.thinking_blocks = thinking_blocks
json_mode_message.reasoning_content = reasoning_content
_message = json_mode_message
model_response.choices[0].message = _message

9
litellm/llms/anthropic/completion/transformation.py
View file

@ -14,8 +14,8 @@ import httpx
import litellm
from litellm.constants import DEFAULT_MAX_TOKENS
from litellm.litellm_core_utils.prompt_templates.factory import (
custom_prompt,
prompt_factory,
render_custom_prompt_template,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import (
@ -244,11 +244,8 @@ class AnthropicTextConfig(BaseConfig):
if model in custom_prompt_dict:
# check if the model has a registered custom prompt
model_prompt_details: Final = custom_prompt_dict[model]
prompt = custom_prompt(
role_dict=model_prompt_details["roles"],
initial_prompt_value=model_prompt_details["initial_prompt_value"],
final_prompt_value=model_prompt_details["final_prompt_value"],
messages=messages,
prompt = render_custom_prompt_template(
template=model_prompt_details, messages=messages, model=model, llm_provider="anthropic"
)
else:
prompt = prompt_factory(model=model, messages=messages, custom_llm_provider="anthropic")

22
litellm/llms/anthropic/count_tokens/token_counter.py
View file

@ -6,11 +6,12 @@ from typing import Any, Final
from pydantic import JsonValue
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import AuthenticationError
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.types.utils import LlmProviders, TokenCountResponse
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
# Global handler instance - reuse across all token counting requests
anthropic_count_tokens_handler: Final = AnthropicCountTokensHandler()
@ -34,6 +35,9 @@ class AnthropicTokenCounter(BaseTokenCounter):
request_model: str = "",
tools: list[dict[str, JsonValue]] | None = None,
system: JsonValue = None,
messages_format: TokenCountMessagesFormat = "anthropic",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
) -> TokenCountResponse | None:
"""
Count tokens using Anthropic's CountTokens API.
@ -59,14 +63,24 @@ class AnthropicTokenCounter(BaseTokenCounter):
try:
auth_header: Final = await AnthropicModelInfo.aget_auth_header(
api_key=litellm_params.get("api_key"),
api_key=litellm_params.get("api_key") or litellm.anthropic_key or litellm.api_key,
api_base=api_base,
litellm_params=litellm_params,
allow_workload_identity=True,
)
if auth_header is None:
verbose_logger.warning("No Anthropic credential found for token counting")
return None
return TokenCountResponse(
total_tokens=0,
request_model=request_model,
model_used=model_to_use,
tokenizer_type="anthropic_api",
error=True,
error_message=(
f"No Anthropic API key is set for model {request_model or model_to_use}, so tokens can't be "
"counted. Set api_key on the deployment or ANTHROPIC_API_KEY in the proxy environment."
),
status_code=401,
)
result: Final = await anthropic_count_tokens_handler.handle_count_tokens_request(
model=model_to_use,

93
litellm/llms/anthropic/pass_through/adapters/handler.py
View file

@ -1,4 +1,5 @@
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
from dataclasses import dataclass
from typing import (
TYPE_CHECKING,
Final,
@ -62,6 +63,12 @@ class _CompletionKwargs(TypedDict, total=False, extra_items=object):
custom_llm_provider: str
@dataclass(frozen=True, slots=True)
class _ChatTargetReasoning:
effort: str
summary: str | None = None
def _messages_have_compaction_block(messages: _AnthropicMessages) -> bool:
"""Return True when any message carries a ``compaction`` content block."""
for msg in messages:
@ -132,7 +139,7 @@ def request_safeguards_evaluator(
)
async def _prepare_context_managed_request(
async def prepare_context_managed_request(
*,
model: str,
messages: _AnthropicMessages,
@ -145,6 +152,8 @@ async def _prepare_context_managed_request(
user_api_key_auth: "UserAPIKeyAuth | None" = None,
sub_call_logging_kwargs: ProxySubCallLoggingKwargs | None = None,
parent_trace_id: str | None = None,
count_only: bool = False,
drop_invalid_edits: bool = False,
) -> PolyfillResult | None:
"""Apply client compaction history, then optional context_management polyfill."""
from litellm.llms.anthropic.pass_through.context_management.editors.compact import (
@ -186,6 +195,8 @@ async def _prepare_context_managed_request(
user_api_key_auth=user_api_key_auth,
sub_call_logging_kwargs=sub_call_logging_kwargs,
parent_trace_id=parent_trace_id,
count_only=count_only,
drop_invalid_edits=drop_invalid_edits,
)
if polyfill_result is not None:
@ -259,16 +270,14 @@ def _spec_has_non_compact_edits(
return any(isinstance(edit.get("type"), str) and edit.get("type") != COMPACT_EDIT_TYPE for edit in edits)
def _context_management_explicitly_dropped(additional_drop_params: list[str] | None) -> bool:
def context_management_explicitly_dropped(additional_drop_params: object) -> bool:
"""True when the caller opted out of context_management via ``additional_drop_params``.
``drop_params`` deliberately does NOT gate the polyfill: ``context_management``
is a LiteLLM-supported param (native on Anthropic, polyfilled elsewhere), and
``drop_params`` only exists to drop genuinely unsupported params.
"""
if not isinstance(additional_drop_params, list):
return False
return "context_management" in additional_drop_params
return isinstance(additional_drop_params, list) and "context_management" in additional_drop_params
def _normalize_spec_edits(
@ -284,7 +293,7 @@ def _normalize_spec_edits(
if not context_management_spec:
return None
if _context_management_explicitly_dropped(additional_drop_params):
if context_management_explicitly_dropped(additional_drop_params):
return None
from litellm.llms.anthropic.pass_through.context_management.dispatcher import (
@ -310,20 +319,23 @@ async def _run_polyfill_if_enabled(
user_api_key_auth: "UserAPIKeyAuth | None" = None,
sub_call_logging_kwargs: ProxySubCallLoggingKwargs | None = None,
parent_trace_id: str | None = None,
count_only: bool = False,
drop_invalid_edits: bool = False,
) -> PolyfillResult | None:
"""Run the async context_management polyfill if a spec is present.
Returns ``None`` when the spec is empty or ``context_management`` is
listed in ``additional_drop_params`` (the explicit opt-out; ``drop_params``
does not disable the polyfill because context_management is a supported
does not disable a valid spec because context_management is a supported
param). Raises ``AnthropicContextManagementError`` so the /v1/messages
endpoint can emit an Anthropic-format 400. All other exceptions are
endpoint can emit an Anthropic-format 400, unless ``drop_invalid_edits``
(``drop_params``) drops the invalid spec. All other exceptions are
best-effort swallowed (matches v0 behavior).
"""
if not context_management_spec:
return None
if _context_management_explicitly_dropped(additional_drop_params):
if context_management_explicitly_dropped(additional_drop_params):
return None
try:
@ -338,13 +350,17 @@ async def _run_polyfill_if_enabled(
user_api_key_auth=user_api_key_auth,
sub_call_logging_kwargs=sub_call_logging_kwargs,
parent_trace_id=parent_trace_id,
count_only=count_only,
)
except AnthropicContextManagementError:
if drop_invalid_edits:
return None
# Surface validation errors so the endpoint can emit an Anthropic-format
# 400. Other exception types fall into the best-effort branch below.
raise
except Exception as e:
verbose_logger.exception("context_management polyfill: skipping edits due to error: %s", e)
log_failure: Final = verbose_logger.debug if count_only else verbose_logger.exception
log_failure("context_management polyfill: skipping edits due to error: %s", e)
# Best-effort swallow is only safe for compact-only specs, where the
# caller's compaction-block-slicing safety net produces a correct
# (if degraded) result. When the spec also requested non-compact
@ -441,15 +457,33 @@ class LiteLLMMessagesToCompletionTransformationHandler:
completion_kwargs["reasoning_effort"] = updated_reasoning_effort
@staticmethod
def _plain_effort_for_chat_target(
def _requested_reasoning_summary(thinking: Mapping[str, object]) -> str | None:
if thinking.get("type") == "disabled":
return None
summary: Final = thinking.get("summary")
if isinstance(summary, str):
return summary
display: Final = thinking.get("display")
return "auto" if display is not None and display != "omitted" else None
@staticmethod
def _chat_target_reasoning(
completion_kwargs: _CompletionKwargs,
*,
thinking: Mapping[str, object] | None,
) -> str | None:
reasoning_effort: Final = completion_kwargs.get("reasoning_effort")
if not thinking or not isinstance(reasoning_effort, dict) or "summary" not in reasoning_effort:
) -> _ChatTargetReasoning | None:
if not thinking:
return None
effort: Final = reasoning_effort.get("effort")
reasoning_effort: Final = completion_kwargs.get("reasoning_effort")
wraps_summary: Final = isinstance(reasoning_effort, dict) and "summary" in reasoning_effort
requested_summary: Final = LiteLLMMessagesToCompletionTransformationHandler._requested_reasoning_summary(
thinking
)
if not wraps_summary and requested_summary is None:
return None
effort: Final[object] = (
reasoning_effort.get("effort") if isinstance(reasoning_effort, dict) else reasoning_effort
)
model: Final = completion_kwargs.get("model")
if not isinstance(effort, str) or not isinstance(model, str) or not model:
return None
@ -467,7 +501,12 @@ class LiteLLMMessagesToCompletionTransformationHandler:
return None
if resolved_provider == "litellm_proxy":
return None
from litellm.responses.bridge_check import responses_api_bridge_check
from litellm.responses.bridge_check import chat_target_accepts_reasoning_summary, responses_api_bridge_check
if requested_summary is not None and chat_target_accepts_reasoning_summary(local_model, resolved_provider):
return _ChatTargetReasoning(effort=effort, summary=requested_summary)
if not wraps_summary or not isinstance(reasoning_effort, dict):
return None
web_search_options: Final = completion_kwargs.get("web_search_options")
tools: Final = completion_kwargs.get("tools")
@ -481,7 +520,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
reasoning_effort=reasoning_effort,
api_base=resolved_api_base,
)
return None if model_info.get("mode") == "responses" else effort
return None if model_info.get("mode") == "responses" else _ChatTargetReasoning(effort=effort)
@staticmethod
def _normalize_reasoning_effort(
@ -647,12 +686,14 @@ class LiteLLMMessagesToCompletionTransformationHandler:
thinking=thinking,
)
plain_effort: Final = LiteLLMMessagesToCompletionTransformationHandler._plain_effort_for_chat_target(
chat_reasoning: Final = LiteLLMMessagesToCompletionTransformationHandler._chat_target_reasoning(
completion_kwargs,
thinking=thinking,
)
if plain_effort is not None:
completion_kwargs["reasoning_effort"] = plain_effort
if chat_reasoning is not None:
completion_kwargs["reasoning_effort"] = chat_reasoning.effort
if chat_reasoning is not None and chat_reasoning.summary is not None:
completion_kwargs["reasoning_summary"] = chat_reasoning.summary
return completion_kwargs, tool_name_mapping
@ -677,6 +718,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
"""Handle non-Anthropic models asynchronously using the adapter"""
context_management: Final = kwargs.pop("context_management", None)
additional_drop_params: Final[list[str] | None] = kwargs.get("additional_drop_params", None)
drop_invalid_edits: Final = litellm.drop_params is True or (
"drop_params" in kwargs and kwargs["drop_params"] is True
)
litellm_router: Final[Router | None] = _router_or_proxy_fallback(kwargs.pop("litellm_router", None))
proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs)
@ -684,7 +728,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
sub_call_logging_kwargs: Final = proxy_sub_call_logging_kwargs(request_kwargs)
sub_call_trace_id: Final = proxy_sub_call_trace_id(request_kwargs)
polyfill_result: Final = await _prepare_context_managed_request(
polyfill_result: Final = await prepare_context_managed_request(
model=model,
messages=messages,
tools=tools,
@ -696,6 +740,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
user_api_key_auth=user_api_key_auth,
sub_call_logging_kwargs=sub_call_logging_kwargs,
parent_trace_id=sub_call_trace_id,
drop_invalid_edits=drop_invalid_edits,
)
effective_messages: Final = polyfill_result.messages if polyfill_result is not None else messages
@ -809,6 +854,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
# bridge to it via ``run_async_function``.
context_management: Final = kwargs.pop("context_management", None)
additional_drop_params: Final[list[str] | None] = kwargs.get("additional_drop_params", None)
drop_invalid_edits: Final = litellm.drop_params is True or (
"drop_params" in kwargs and kwargs["drop_params"] is True
)
# Deliberately do NOT auto-attach the proxy ``llm_router`` here:
# ``run_async_function`` spawns a new event loop in a worker thread
# to bridge to the async dispatcher, but the proxy router's httpx
@ -834,7 +882,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs)
request_kwargs: Final = _REQUEST_KWARGS.validate_python(kwargs)
polyfill_result = run_async_function(
_prepare_context_managed_request,
prepare_context_managed_request,
model=model,
messages=messages,
tools=tools,
@ -846,6 +894,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
user_api_key_auth=user_api_key_auth,
sub_call_logging_kwargs=proxy_sub_call_logging_kwargs(request_kwargs),
parent_trace_id=proxy_sub_call_trace_id(request_kwargs),
drop_invalid_edits=drop_invalid_edits,
)
effective_messages: Final = polyfill_result.messages if polyfill_result is not None else messages

75
litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py
View file

@ -24,6 +24,7 @@ from typing_extensions import assert_never
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.hidden_params import set_hidden_params
from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame
@ -39,6 +40,7 @@ from litellm.types.llms.anthropic import (
UsageDelta,
UsageIteration,
)
from litellm.types.llms.openai import ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock
from litellm.types.utils import AdapterCompletionStreamWrapper, ChatCompletionDeltaToolCall, Delta, Function
if TYPE_CHECKING:
@ -50,6 +52,7 @@ if TYPE_CHECKING:
_STREAMING_DELTA_TYPES: Final = frozenset(get_args(StreamingContentBlockDeltaType))
_SSE_EVENT: Final = TypeAdapter(Mapping[str, object])
_THINKING_BLOCK: Final = TypeAdapter(ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock)
class _UsageDeltaWithIterations(UsageDelta, total=False):
@ -106,6 +109,14 @@ def _delta_payload_field(delta_type: StreamingContentBlockDeltaType) -> str:
assert_never(delta_type)
def _first_thinking_block_index(chunk: "ModelResponseStream") -> int | None:
delta: Final = _optional_attr(chunk.choices[0], "delta") if chunk.choices else None
blocks: Final = _optional_attr_sequence(delta, "thinking_blocks")
first_block: Final = as_str_mapping(blocks[0]) if blocks else None
index: Final = first_block.get("index") if first_block is not None else None
return index if isinstance(index, int) else None
def _optional_str_attr(obj: object, name: str) -> str | None:
value: Final = _optional_attr(obj, name)
return value if isinstance(value, str) else None
@ -373,13 +384,44 @@ class _CombinedChunkSplitter:
finish_delta.thinking_blocks = None
return [content_chunk, finish_chunk]
@staticmethod
def _with_thinking_block(
chunk: "ModelResponseStream",
block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock,
*,
keep_role: bool,
) -> "ModelResponseStream":
piece: Final = copy.deepcopy(chunk)
role: Final = getattr(piece.choices[0].delta, "role", None) if keep_role else None
thinking: Final = block.get("thinking") if block["type"] == "thinking" else None
piece.choices[0].delta = Delta(role=role, reasoning_content=thinking or None, thinking_blocks=[block])
return piece
@staticmethod
def _split_thinking_blocks(chunk: "ModelResponseStream") -> "tuple[ModelResponseStream, ...]":
"""Anthropic streams one content block per thinking block, so a delta carrying several is split per block."""
choices: Final = _optional_attr_sequence(chunk, "choices")
if len(choices) != 1:
return (chunk,)
delta: Final = _optional_attr(choices[0], "delta")
blocks: Final = _optional_attr_sequence(delta, "thinking_blocks")
if len(blocks) < 2 or _optional_attr(delta, "content") or _optional_attr(delta, "tool_calls"):
return (chunk,)
return tuple(
_CombinedChunkSplitter._with_thinking_block(
chunk, _THINKING_BLOCK.validate_python(block), keep_role=position == 0
)
for position, block in enumerate(blocks)
)
def _expand(self, chunk: "ModelResponseStream") -> "tuple[ModelResponseStream, ...]":
split_chunks: Final = tuple(
split_chunk
for combined_chunk in self._split(chunk)
for split_chunk in self._split_by_payload_kind(combined_chunk) # comprehension-ok: flattens nested splits
)
return tuple(itertools.chain.from_iterable(self._route_tool_calls(split_chunk) for split_chunk in split_chunks))
block_chunks: Final = tuple(itertools.chain.from_iterable(map(self._split_thinking_blocks, split_chunks)))
return tuple(itertools.chain.from_iterable(map(self._route_tool_calls, block_chunks)))
def __iter__(self) -> "Iterator[ModelResponseStream]":
return self
@ -458,7 +500,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
sent_first_chunk: bool = False
sent_content_block_start: bool = False
sent_content_block_finish: bool = False
current_content_block_type: Literal["text", "tool_use", "thinking"] = "text"
current_content_block_type: Literal["text", "tool_use", "thinking", "redacted_thinking"] = "text"
current_thinking_block_index: int | None = None
sent_last_message: bool = False
holding_chunk: ContentBlockDelta | None = None
holding_stop_reason_chunk: MessageBlockDelta | None = None
@ -1254,13 +1297,14 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
Also gates re-emission after a block transition: when an upstream
chunk both *triggers* a new content block (its type differs from the
active block) and *carries* delta content, that content belongs to
the new block. The synthesized ``content_block_start`` only ever
carries an empty body — see
``_translate_streaming_openai_chunk_to_anthropic_content_block``,
which returns an empty ``TextBlock``/``ToolUseBlock``/thinking block —
so the trigger chunk's delta must be re-queued or the first token of
the new block (the first non-empty text/thinking delta, or bundled
tool arguments) is silently dropped.
the new block. The synthesized ``content_block_start`` from
``_translate_streaming_openai_chunk_to_anthropic_content_block``
carries an empty text, tool-use or unsigned thinking body, and only a
signed thinking block's text rides on the start because its delta is
a ``signature_delta``. So the trigger chunk's delta must be re-queued
or the first token of the new block (the first non-empty text or
unsigned thinking delta, or bundled tool arguments) is silently
dropped.
Delta types outside ``StreamingContentBlockDeltaType`` — the closed
set the translate layer can produce — are treated as empty. The
@ -1352,11 +1396,24 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
original_name: Final = self.tool_name_mapping.get(truncated_name, truncated_name)
tool_block["name"] = original_name
thinking_block_index: Final = _first_thinking_block_index(chunk)
previous_thinking_block_index: Final = self.current_thinking_block_index
if thinking_block_index is not None or block_type != self.current_content_block_type:
self.current_thinking_block_index = thinking_block_index
if block_type != self.current_content_block_type:
self.current_content_block_type = block_type
self.current_content_block_start = content_block_start
return True
if (
thinking_block_index is not None
and previous_thinking_block_index is not None
and thinking_block_index != previous_thinking_block_index
):
self.current_content_block_start = content_block_start
return True
# For parallel tool calls, we'll necessarily have a new content block
# if we get a function name since it signals a new tool call
if block_type == "tool_use":

10
litellm/llms/anthropic/pass_through/adapters/transformation.py
View file

@ -465,7 +465,7 @@ class LiteLLMAnthropicMessagesAdapter:
*,
custom_llm_provider: str | None = None,
preserve_midturn_system: bool = False,
) -> list:
) -> list[AllMessageValues]:
new_messages: Final[list[AllMessageValues]] = []
replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)
leading_count: Final = next(
@ -1627,7 +1627,7 @@ class LiteLLMAnthropicMessagesAdapter:
def translate_streaming_openai_chunk_to_anthropic_content_block(
self, choices: list[OpenAIStreamingChoice | StreamingChoices]
) -> tuple[
Literal["text", "tool_use", "thinking"],
Literal["text", "tool_use", "thinking", "redacted_thinking"],
"ContentBlockContentBlockDict",
]:
from litellm._uuid import uuid
@ -1664,6 +1664,10 @@ class LiteLLMAnthropicMessagesAdapter:
thinking_blocks = choice.delta.thinking_blocks or []
if len(thinking_blocks) > 0:
thinking_block = thinking_blocks[0]
if thinking_block["type"] == "redacted_thinking":
return "redacted_thinking", ChatCompletionRedactedThinkingBlock(
type="redacted_thinking", data=thinking_block.get("data") or ""
)
if thinking_block["type"] == "thinking":
thinking = thinking_block.get("thinking") or ""
signature = thinking_block.get("signature") or ""
@ -1672,7 +1676,7 @@ class LiteLLMAnthropicMessagesAdapter:
assert isinstance(signature, str)
return "thinking", ChatCompletionThinkingBlock(
type="thinking", thinking=thinking, signature=signature
type="thinking", thinking=thinking if signature else "", signature=signature
)
# OpenAI-compatible reasoning backends (e.g. vLLM/SGLang reasoning
# parsers) populate ``reasoning_content`` without ``thinking_blocks``.

2
litellm/llms/anthropic/pass_through/context_management/dispatcher.py
View file

@ -85,6 +85,7 @@ async def apply_context_management(
user_api_key_auth: "UserAPIKeyAuth | None" = None,
sub_call_logging_kwargs: "ProxySubCallLoggingKwargs | None" = None,
parent_trace_id: str | None = None,
count_only: bool = False,
) -> PolyfillResult:
"""Run edits in order; return a single ``PolyfillResult``.
@ -128,6 +129,7 @@ async def apply_context_management(
user_api_key_auth=user_api_key_auth,
sub_call_logging_kwargs=sub_call_logging_kwargs,
parent_trace_id=parent_trace_id,
count_only=count_only,
)
if editor_is_async
else await asyncify(editor)(

3
litellm/llms/anthropic/pass_through/context_management/editors/compact.py
View file

@ -642,6 +642,7 @@ async def apply_compact_20260112(
user_api_key_auth: Optional["UserAPIKeyAuth"] = None,
sub_call_logging_kwargs: ProxySubCallLoggingKwargs | None = None,
parent_trace_id: str | None = None,
count_only: bool = False,
) -> PolyfillResult:
"""Apply ``compact_20260112``; return a ``PolyfillResult``.
@ -682,7 +683,7 @@ async def apply_compact_20260112(
# Opt-in gate: no summary model configured → no-op (but still return the
# Phase A-sliced/stripped messages so compaction blocks don't leak).
summary_model: Final = _read_summary_model_setting()
summary_model: Final = None if count_only else _read_summary_model_setting()
if summary_model is None:
applied["error"] = "summary_model_not_configured"
# Slice-only forwarding: ``augmented_system`` already carries any prior

106
litellm/llms/anthropic/pass_through/convertible_blocks.py Normal file
View file

@ -0,0 +1,106 @@
from collections.abc import Iterator, Mapping, Sequence
from itertools import chain
from types import MappingProxyType
from typing import Final
from pydantic import JsonValue, TypeAdapter, ValidationError
from typing_extensions import assert_never
from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages
from litellm.types.utils import TokenCountMessagesFormat
_CONVERTIBLE_BLOCK_TYPES: Final = MappingProxyType(
{
"user": frozenset({"text", "image", "document", "tool_result"}),
"assistant": frozenset({"text", "tool_use", "thinking", "redacted_thinking"}),
}
)
_SOURCED_BLOCK_TYPES: Final = frozenset({"image", "document"})
_TOOL_RESULT_BLOCK_TYPES: Final = frozenset({"text", "image", "document"})
_JSON_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...])
def _json_messages(messages: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, JsonValue], ...]:
try:
return _JSON_MESSAGES.validate_python(messages)
except ValidationError:
return ()
def _source_label(block_type: str, source: JsonValue) -> str | None:
if not isinstance(source, dict):
return f"{block_type} without a source"
match source.get("type"):
case "base64" if source.get("data"):
return None
case "url" if source.get("url"):
return None
case "base64" | "url" as source_type:
return f"{block_type} with an empty {source_type} source"
case str() as source_type:
return f"{block_type} with a {source_type} source"
case _:
return f"{block_type} with an untyped source"
def _block_label(block: JsonValue, convertible: frozenset[str]) -> str | None:
if not isinstance(block, dict):
return None
block_type: Final = block.get("type")
if not isinstance(block_type, str):
return "a block without a type"
if block_type not in convertible:
return block_type
return _source_label(block_type, block.get("source")) if block_type in _SOURCED_BLOCK_TYPES else None
def _tool_result_labels(block: JsonValue) -> Iterator[str]:
if not isinstance(block, dict) or block.get("type") != "tool_result":
return iter(())
inner: Final = block.get("content")
if not isinstance(inner, list):
return iter(())
return (
f"{label} inside a tool_result"
for item in inner
if (label := _block_label(item, _TOOL_RESULT_BLOCK_TYPES)) is not None
)
def _content_labels(block: JsonValue, convertible: frozenset[str]) -> Iterator[str]:
label: Final = _block_label(block, convertible)
return chain(() if label is None else (label,), _tool_result_labels(block))
def _message_labels(message: Mapping[str, JsonValue]) -> Iterator[str]:
role: Final = message.get("role")
convertible: Final = _CONVERTIBLE_BLOCK_TYPES.get(role) if isinstance(role, str) else None
content: Final = message.get("content")
if convertible is None or not isinstance(content, list):
return iter(())
return chain.from_iterable(_content_labels(block, convertible) for block in content)
def unconvertible_anthropic_blocks(messages: Sequence[Mapping[str, object]]) -> tuple[str, ...]:
normalized: Final = _json_messages(normalize_native_messages(messages))
return tuple(sorted(set(chain.from_iterable(_message_labels(message) for message in normalized))))
def unconvertible_blocks_error(
messages: Sequence[Mapping[str, object]] | None, messages_format: TokenCountMessagesFormat
) -> str | None:
match messages_format:
case "openai":
return None
case "anthropic":
labels: Final = unconvertible_anthropic_blocks(messages or ())
return (
"count_tokens can't convert these content blocks for this model, so the count would leave them out: "
+ ", ".join(labels)
if labels
else None
)
case _:
assert_never(messages_format)

17
litellm/llms/anthropic/pass_through/messages/streaming_iterator.py
View file

@ -2,6 +2,7 @@ import asyncio
import json
import re
from collections.abc import AsyncGenerator, AsyncIterator, Mapping, Sequence
from contextvars import ContextVar
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final, Protocol, runtime_checkable
@ -352,6 +353,22 @@ async def aclose_if_supported(stream: object) -> None:
await stream.aclose()
_CLOSING_SUPERSEDED_ATTEMPT: Final[ContextVar[bool]] = ContextVar("closing_superseded_attempt", default=False)
async def aclose_superseded_attempt(stream: object) -> None:
"""Close an attempt a retry or fallback is taking over, so its end-of-stream logging records a failure."""
token: Final = _CLOSING_SUPERSEDED_ATTEMPT.set(True)
try:
await aclose_if_supported(stream)
finally:
_CLOSING_SUPERSEDED_ATTEMPT.reset(token)
def closing_superseded_attempt() -> bool:
return _CLOSING_SUPERSEDED_ATTEMPT.get()
_RESPONSE_HEADERS_ADAPTER: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])

15
litellm/llms/anthropic/pass_through/messages/utils.py
View file

@ -129,6 +129,13 @@ def anthropic_system_to_openai_message(system: object) -> ChatCompletionSystemMe
return ChatCompletionSystemMessage(role="system", content=system)
def normalize_native_messages(messages: Sequence[object]) -> list[dict[str, object]]:
"""Apply the block rewrites every /v1/messages request gets before it reaches a provider."""
return flatten_unencrypted_web_search_results_in_anthropic_messages(
sanitize_tool_use_ids_in_anthropic_messages(strip_empty_content_blocks_from_anthropic_messages(messages))
)
def prepare_native_messages(
messages: list[dict[str, JsonValue]],
system: str | list[dict[str, JsonValue]] | None,
@ -140,13 +147,7 @@ def prepare_native_messages(
api_base: str | None = None,
presanitized: bool = False,
) -> tuple[list[dict[str, JsonValue]], str | list[dict[str, JsonValue]] | None]:
normalized: Final = (
messages
if presanitized
else flatten_unencrypted_web_search_results_in_anthropic_messages(
sanitize_tool_use_ids_in_anthropic_messages(strip_empty_content_blocks_from_anthropic_messages(messages))
)
)
normalized: Final = messages if presanitized else normalize_native_messages(messages)
return cast( # cast-ok: legacy normalizers and injection preserve the JSON message and system shapes
tuple[list[dict[str, JsonValue]], str | list[dict[str, JsonValue]] | None],
AnthropicCacheControlHook.maybe_inject_cache_control(

10
litellm/llms/anthropic/pass_through/responses_adapters/handler.py
View file

@ -43,6 +43,12 @@ _BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]])
_ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter()
def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None:
logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs)
if logging_obj is not None:
logging_obj.record_bridged_responses_api_response(result)
def _forwarded_kwargs(extra_kwargs: Mapping[str, object] | None) -> Mapping[str, object]:
"""The litellm-specific kwargs forwarded verbatim onto the Responses API request."""
return extra_kwargs or {}
@ -301,7 +307,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
if not isinstance(result, ResponsesAPIResponse):
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
return await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
_record_bridged_response(responses_kwargs, result)
return served
@staticmethod
def anthropic_messages_handler(

12
litellm/llms/anthropic/pass_through/responses_adapters/transformation.py
View file

@ -579,6 +579,14 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
result["summary"] = "detailed"
return result
@classmethod
def system_blocks_to_instructions(cls, system: object) -> str:
return "\n".join(
text
for block in cls._content_blocks(system)
if block.get("type") == "text" and isinstance(text := block.get("text"), str) and text
)
def translate_request(
self,
anthropic_request: AnthropicMessagesRequest,
@ -628,9 +636,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if isinstance(system, str):
responses_kwargs["instructions"] = system
elif isinstance(system, list):
responses_kwargs["instructions"] = "\n".join(
filter(None, (b.get("text", "") for b in system if isinstance(b, dict) and b.get("type") == "text"))
)
responses_kwargs["instructions"] = self.system_blocks_to_instructions(system)
# max_tokens -> max_output_tokens
max_tokens: Final = anthropic_request.get("max_tokens")

1
litellm/llms/azure/chat/gpt_5_transformation.py
View file

@ -95,7 +95,6 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
status_code=400,
message=(
"Azure OpenAI does not support reasoning_effort='none' for this model. "
"Supported values are: 'low', 'medium', and 'high'. "
"To drop this parameter, set `litellm.drop_params=True` or for proxy:\n\n"
"`litellm_settings:\n drop_params: true`\n"
"Issue: https://github.com/BerriAI/litellm/issues/16704"

10
litellm/llms/azure/common_utils.py
View file

@ -2,6 +2,7 @@ import asyncio
import hashlib
import json
import os
import re
from collections.abc import Callable, Mapping
from functools import lru_cache
from types import MappingProxyType
@ -25,6 +26,7 @@ from litellm.types.router import GenericLiteLLMParams
from litellm.utils import add_path_to_api_base
azure_ad_cache: Final = DualCache()
_LAST_OPENAI_PATH_SEGMENT: Final = re.compile(r"^(.*/openai)(?=/|$)")
class _AzureAdTokenJson(TypedDict, total=False):
@ -798,6 +800,10 @@ class BaseAzureLLM(BaseOpenAILLM):
) -> dict[str, str]: # mutable-ok: mirrors override contract
return cls._base_validate_azure_environment(headers, litellm_params)
@staticmethod
def resolve_azure_api_base(api_base: str | None) -> str | None:
return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@staticmethod
def _get_base_azure_url(
api_base: str | None,
@ -815,7 +821,7 @@ class BaseAzureLLM(BaseOpenAILLM):
default_api_version: The default API version to use if no api_version is provided. If 'latest', it will use `openai/v1/...` route.
"""
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_base = BaseAzureLLM.resolve_azure_api_base(api_base)
if api_base is None:
raise ValueError(
f"api_base is required for Azure AI Studio. Please set the api_base parameter. Passed `api_base={api_base}`"
@ -843,7 +849,7 @@ class BaseAzureLLM(BaseOpenAILLM):
# ensure the request go to /openai/v1 and not just /openai
if "/openai/v1" not in new_url:
parsed_url: Final = httpx.URL(new_url)
new_url = str(parsed_url.copy_with(path=parsed_url.path.replace("/openai", "/openai/v1")))
new_url = str(parsed_url.copy_with(path=_LAST_OPENAI_PATH_SEGMENT.sub(r"\1/v1", parsed_url.path, 1)))
# Use the new query_params dictionary
final_url: Final = httpx.URL(new_url).copy_with(params=query_params)

1
litellm/llms/azure/containers/transformation.py
View file

@ -28,6 +28,7 @@ class AzureContainerConfig(OpenAIContainerConfig):
self,
headers: dict,
api_key: str | None = None,
litellm_params: GenericLiteLLMParams | None = None,
) -> dict:
return BaseAzureLLM.base_validate_azure_environment(
headers=headers,

12
litellm/llms/azure/fine_tuning/handler.py
View file

@ -1,4 +1,4 @@
from collections.abc import Coroutine
from collections.abc import Coroutine, Mapping
from typing import Final, cast
import httpx
@ -66,6 +66,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
self._ensure_training_type(create_fine_tuning_job_data)
@ -78,6 +79,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
client=client,
_is_async=_is_async,
api_version=api_version,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -109,6 +111,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -119,6 +122,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
client=client,
_is_async=_is_async,
api_version=api_version,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -149,6 +153,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -159,6 +164,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
client=client,
_is_async=_is_async,
api_version=api_version,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -188,14 +194,14 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
_is_async: bool = False,
api_version: str | None = None,
litellm_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None:
# Override to use Azure-specific client initialization
if isinstance(client, OpenAI) or isinstance(client, AsyncOpenAI):
client = None
return self.get_azure_openai_client(
litellm_params=litellm_params or {},
litellm_params=dict(litellm_params) if litellm_params is not None else None,
api_key=api_key,
api_base=api_base,
api_version=api_version,

39
litellm/llms/azure/responses/transformation.py
View file

@ -1,4 +1,7 @@
import re
from collections.abc import Mapping
from copy import deepcopy
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal
import httpx
@ -6,6 +9,7 @@ from openai.types.responses import ResponseReasoningItem
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config
from litellm.llms.azure.common_utils import BaseAzureLLM
@ -25,6 +29,34 @@ else:
_DECODED_JSON: Final = TypeAdapter(object)
_DATED_GA_API_VERSION: Final = re.compile(r"\d{4}-\d{2}-\d{2}")
def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None:
value: Final = litellm_params.get(key)
return value if isinstance(value, str) and value else None
def _resolve_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> str | None:
"""
Dated GA api_versions (e.g. 2024-10-21) predate the Responses API on Azure and 404 there, so a deployment
pinned to one for chat gets the default Responses version unless it sets responses_api_version.
"""
api_version: Final = _str_param(litellm_params, "api_version")
if api_base is not None and "api-version" in httpx.URL(api_base).params:
return api_version
responses_api_version: Final = _str_param(litellm_params, "responses_api_version")
if responses_api_version is not None:
return responses_api_version
if api_version is not None and _DATED_GA_API_VERSION.fullmatch(api_version):
return AZURE_DEFAULT_RESPONSES_API_VERSION
return api_version
def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> Mapping[str, object]:
return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)})
class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
# Parameters not supported by Azure Responses API
AZURE_UNSUPPORTED_PARAMS = ["context_management"]
@ -170,11 +202,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
- A complete URL string, e.g.,
"https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
"""
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
resolved_api_base: Final = BaseAzureLLM.resolve_azure_api_base(api_base)
return BaseAzureLLM.get_base_azure_url(
api_base=api_base,
litellm_params=litellm_params,
api_base=resolved_api_base,
litellm_params=_with_responses_api_version(resolved_api_base, litellm_params),
route="/openai/responses",
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
)

11
litellm/llms/azure_ai/anthropic/count_tokens/handler.py
View file

@ -30,7 +30,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
self,
model: str,
messages: list[dict[str, JsonValue]],
api_key: str,
api_key: str | None,
api_base: str,
litellm_params: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
@ -80,6 +80,15 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
api_key=api_key,
litellm_params=litellm_params,
)
if not {"x-api-key", "Authorization"} & headers.keys():
raise AnthropicError(
status_code=401,
message=(
f"No Azure AI API key or Entra ID credential is set for model {model}, so tokens can't be "
"counted. Set api_key or an Entra ID credential on the deployment, or AZURE_AI_API_KEY or "
"AZURE_API_KEY in the proxy environment."
),
)
# Use LiteLLM's async httpx client
async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.AZURE_AI)

38
litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py
View file

@ -7,17 +7,33 @@ from typing import Any, Final
from pydantic import JsonValue
import litellm
from litellm._logging import verbose_logger
from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
AzureAIAnthropicCountTokensHandler,
)
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.types.utils import LlmProviders, TokenCountResponse
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
# Global handler instance - reuse across all token counting requests
azure_ai_anthropic_count_tokens_handler: Final = AzureAIAnthropicCountTokensHandler()
def _missing_api_base_response(request_model: str, model_to_use: str) -> TokenCountResponse:
return TokenCountResponse(
total_tokens=0,
request_model=request_model,
model_used=model_to_use,
tokenizer_type="azure_ai_anthropic_api",
error=True,
error_message=(
f"No Azure AI API base is set for model {request_model or model_to_use}, so tokens can't be counted. "
"Set api_base on the deployment or AZURE_AI_API_BASE or AZURE_API_BASE in the proxy environment."
),
status_code=500,
)
class AzureAIAnthropicTokenCounter(BaseTokenCounter):
"""Token counter implementation for Azure AI Anthropic provider using the CountTokens API."""
@ -36,6 +52,9 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
request_model: str = "",
tools: list[dict[str, JsonValue]] | None = None,
system: object | None = None,
messages_format: TokenCountMessagesFormat = "anthropic",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
) -> TokenCountResponse | None:
"""
Count tokens using Azure AI Anthropic's CountTokens API.
@ -63,18 +82,15 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
if not api_key:
api_key = os.getenv("AZURE_AI_API_KEY")
# Get API base from deployment config or environment
api_base = litellm_params.get("api_base")
if not api_base:
api_base = os.getenv("AZURE_AI_API_BASE")
if not api_key:
verbose_logger.warning("No Azure AI API key found for token counting")
return None
api_base: Final = (
litellm_params.get("api_base")
or os.getenv("AZURE_AI_API_BASE")
or os.getenv("AZURE_API_BASE")
or litellm.api_base
)
if not api_base:
verbose_logger.warning("No Azure AI API base found for token counting")
return None
return _missing_api_base_response(request_model, model_to_use)
try:
result: Final = await azure_ai_anthropic_count_tokens_handler.handle_count_tokens_request(

9
litellm/llms/azure_ai/anthropic/count_tokens/transformation.py
View file

@ -24,7 +24,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
def get_required_headers(
self,
api_key: str,
api_key: str | None,
litellm_params: dict[str, object] | None = None,
) -> dict[str, str]:
"""
@ -45,12 +45,12 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
"Content-Type": "application/json",
"anthropic-version": "2023-06-01",
"anthropic-beta": ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
"x-api-key": api_key, # Azure AI Anthropic requires this header
**({"x-api-key": api_key} if api_key else {}),
}
# Also set up Azure auth headers for flexibility
litellm_params = litellm_params or {}
if "api_key" not in litellm_params:
if api_key and "api_key" not in litellm_params:
litellm_params["api_key"] = api_key
litellm_params_obj: Final = GenericLiteLLMParams.model_validate(litellm_params)
@ -58,8 +58,9 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
# Get Azure auth headers (api-key or Authorization)
azure_headers = BaseAzureLLM.base_validate_azure_environment(headers={}, litellm_params=litellm_params_obj)
# Merge Azure auth headers
headers.update(azure_headers)
if "api-key" in azure_headers:
headers.setdefault("x-api-key", azure_headers["api-key"])
return headers

29
litellm/llms/azure_ai/chat/transformation.py
View file

@ -1,6 +1,7 @@
import copy
import enum
import re
from collections.abc import AsyncIterator, Iterator
from typing import TYPE_CHECKING, Final, cast
import httpx
@ -21,12 +22,12 @@ from litellm.llms.azure_ai.common_utils import (
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
from litellm.llms.openai.common_utils import drop_params_from_unprocessable_entity_error
from litellm.llms.openai.openai import OpenAIConfig
from litellm.llms.openai.openai import OpenAIChatCompletionResponseIterator, OpenAIConfig
from litellm.llms.xai.chat.transformation import XAIChatConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ModelResponse, ProviderField
from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField
from litellm.utils import add_path_to_api_base, supports_tool_choice
if TYPE_CHECKING:
@ -45,6 +46,14 @@ NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = (
)
class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator):
def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract
usage: Final = chunk.get("usage")
if isinstance(usage, dict):
XAIChatConfig.fold_reasoning_tokens_into_completion(usage)
return super().chunk_parser(chunk)
class AzureAIGPT5Config(OpenAIGPT5Config):
@classmethod
def _model_map_lookup_name(cls, model: str) -> str:
@ -319,7 +328,7 @@ class AzureAIStudioConfig(OpenAIConfig):
json_mode: bool | None = None,
) -> ModelResponse:
model_response.model = f"azure_ai/{model}"
return super().transform_response(
response: Final = super().transform_response(
model=model,
raw_response=raw_response,
model_response=model_response,
@ -332,6 +341,20 @@ class AzureAIStudioConfig(OpenAIConfig):
api_key=api_key,
json_mode=json_mode,
)
XAIChatConfig.fold_reasoning_tokens_into_completion(response)
return response
def get_model_response_iterator(
self,
streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
sync_stream: bool,
json_mode: bool | None = False,
) -> "AzureAIChatCompletionResponseIterator":
return AzureAIChatCompletionResponseIterator(
streaming_response=streaming_response,
sync_stream=sync_stream,
json_mode=json_mode,
)
def should_retry_llm_api_inside_llm_translation_on_http_error(
self, e: httpx.HTTPStatusError, litellm_params: dict

26
litellm/llms/base_llm/base_model_iterator.py
View file

@ -6,16 +6,38 @@ import litellm
if TYPE_CHECKING:
import httpx
from litellm.types.llms.openai import ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock
from litellm.types.utils import (
Choices,
Delta,
GenericStreamingChunk,
Message,
ModelResponse,
ModelResponseStream,
StreamingChoices,
)
def _indexed_thinking_block(
block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, index: int
) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock:
if block["type"] == "redacted_thinking":
return {**block, "index": index}
return {**block, "index": index}
def _streaming_delta(message: Message) -> Delta:
thinking_blocks: Final = message.thinking_blocks if hasattr(message, "thinking_blocks") else None
return Delta(
**message.model_dump(exclude={"thinking_blocks"}),
thinking_blocks=(
None
if thinking_blocks is None
else [_indexed_thinking_block(block, index) for index, block in enumerate(thinking_blocks)]
),
)
def convert_model_response_to_streaming(
model_response: ModelResponse,
) -> ModelResponseStream:
@ -40,9 +62,7 @@ def convert_model_response_to_streaming(
streaming_choices.append(
StreamingChoices(
index=choice.index,
delta=Delta(
**cast(Choices, choice).message.model_dump(),
),
delta=_streaming_delta(cast(Choices, choice).message),
finish_reason=choice.finish_reason,
)
)

11
litellm/llms/base_llm/base_utils.py
View file

@ -16,7 +16,13 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk
from litellm.types.utils import Message, ProviderSpecificModelInfo, TokenCountResponse
from litellm.types.utils import (
CountContextManagement,
Message,
ProviderSpecificModelInfo,
TokenCountMessagesFormat,
TokenCountResponse,
)
class BaseTokenCounter(ABC):
@ -30,6 +36,9 @@ class BaseTokenCounter(ABC):
request_model: str = "",
tools: list[dict[str, Any]] | None = None,
system: Any | None = None,
messages_format: TokenCountMessagesFormat = "openai",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
) -> TokenCountResponse | None:
pass

20
litellm/llms/base_llm/chat/transformation.py
View file

@ -8,7 +8,7 @@ from collections.abc import AsyncIterator, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final, Union
import httpx
from pydantic import BaseModel
from pydantic import BaseModel, TypeAdapter
from litellm.constants import DEFAULT_MAX_TOKENS, RESPONSE_FORMAT_TOOL_NAME
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@ -38,6 +38,9 @@ else:
LiteLLMLoggingObj = Any
REQUEST_PARAMS: Final = TypeAdapter(Mapping[str, object])
class BaseLLMException(Exception):
def __init__(
self,
@ -126,6 +129,17 @@ class BaseConfig(ABC):
thinking_type: Final = thinking.get("type") if isinstance(thinking, dict) else None
return thinking is True or thinking_type == "enabled" or non_default_params.get("reasoning_effort") is not None
def is_thinking_requested(self, non_default_params: Mapping[str, object]) -> bool:
match non_default_params:
case {"thinking": True | {"type": "enabled" | "adaptive"}}:
return True
case {"reasoning_effort": None | "none"}:
return False
case {"reasoning_effort": object()}:
return True
case _:
return False
def is_max_tokens_in_request(self, non_default_params: dict) -> bool:
"""
OpenAI spec allows max_tokens or max_completion_tokens to be specified.
@ -488,6 +502,10 @@ class BaseConfig(ABC):
"""
return True
@property
def accepts_reasoning_summary_in_reasoning_effort(self) -> bool:
return False
def post_stream_processing(self, stream: object) -> object:
"""Hook for providers to post-process streaming responses. Default: pass-through."""
return stream

1
litellm/llms/base_llm/containers/transformation.py
View file

@ -81,6 +81,7 @@ class BaseContainerConfig(ABC):
self,
headers: dict,
api_key: str | None = None,
litellm_params: GenericLiteLLMParams | None = None,
) -> dict:
return {}

23
litellm/llms/base_llm/decisions/transformation.py
View file

@ -390,8 +390,24 @@ def systemone_response_to_ir(response: DecisionsResponse, request: DecisionsIRRe
)
_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
_LABELS: Final[TypeAdapter[tuple[str, ...]]] = TypeAdapter(tuple[str, ...])
def _normalize_answer(value: object) -> Mapping[str, object]:
answer: Final = _MAPPING.validate_python(value)
legend: Final = answer.get("legend")
if answer.get("type") != "score" or not isinstance(legend, list):
return answer
labels: Final = _LABELS.validate_python(legend)
return {**answer, "legend": {str(index): label for index, label in enumerate(labels)}}
def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
return systemone_response_to_ir(_SYSTEMONE_RESPONSE_ADAPTER.validate_python(payload), request)
response: Final = _MAPPING.validate_python(payload)
answers: Final = _MAPPING.validate_python(response.get("answers"))
normalized: Final = {**response, "answers": {name: _normalize_answer(answer) for name, answer in answers.items()}}
return systemone_response_to_ir(_SYSTEMONE_RESPONSE_ADAPTER.validate_python(normalized), request)
def _systemone_answer(answer: DecisionsIRAnswer) -> DecisionAnswer | None:
@ -495,6 +511,11 @@ class BaseDecisionsConfig(ABC):
def resolve_connection(self, api_base: str | None, api_key: str | None) -> tuple[str | None, str | None]:
return self.resolve_api_base(api_base), self.resolve_api_key(api_key)
def resolve_credential(
self, api_key: str | None, api_base: str, litellm_params: Mapping[str, object]
) -> str | None:
return api_key
@staticmethod
def _first_secret(names: tuple[str, ...]) -> str | None:
return next((value for value in (get_secret_str(name) for name in names) if value), None)

13
litellm/llms/base_llm/guardrail_translation/utils.py
View file

@ -4,7 +4,7 @@ import json
from collections.abc import Callable, Iterator, Mapping, Sequence
from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row has no typed constructor across roles
from pydantic import BaseModel
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage
@ -292,6 +292,17 @@ def openai_tool_name(tool: object) -> str | None:
return flat_name if isinstance(flat_name, str) else None
_FIELDS: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(strict=True))
_ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True))
def list_field(data: object, key: str) -> Sequence[object]:
try:
return _ITEMS.validate_python(_FIELDS.validate_python(data).get(key))
except ValidationError:
return ()
def anthropic_tool_names(tool: object) -> tuple[str, ...]:
"""Every name a /v1/messages tool dict can act under: the flat Anthropic ``name`` plus
``function.name`` for OpenAI-format tools the bridge forwards verbatim. Allowlist checks

10
litellm/llms/base_llm/realtime/transcription_protocol.py
View file

@ -102,6 +102,16 @@ def json_integer(
return value
def uses_server_vad(
update: TranscriptionSessionUpdate, backend: str, drop_params: bool, error: ProtocolErrorType
) -> bool:
if update.turn_detection_disabled:
return False
if update.turn_detection_type in (None, "server_vad") or drop_params:
return True
raise error(f"{backend} supports server_vad turn detection or null. Set drop_params to use server_vad instead")
def new_event_id() -> str:
return f"event_{uuid.uuid4().hex}"

123
litellm/llms/base_llm/videos/async_task_utils.py Normal file
View file

@ -0,0 +1,123 @@
import base64
from collections.abc import Mapping
from io import BufferedReader, BytesIO
from itertools import accumulate, repeat
from pathlib import Path, PurePath
from typing import Final
from pydantic import TypeAdapter, ValidationError
from litellm.images.utils import ImageEditRequestUtils
from litellm.types.videos.utils import decode_video_id_with_provider
_FIELDS_ADAPTER: Final = TypeAdapter(dict[str, object])
_FILE_TUPLE_ADAPTER: Final = TypeAdapter(tuple[object, ...])
def _fields(value: object) -> Mapping[str, object] | None:
try:
return _FIELDS_ADAPTER.validate_python(value)
except ValidationError:
return None
def _logged_litellm_params(logging_obj: object) -> Mapping[str, object]:
return _fields(getattr(logging_obj, "litellm_params", None)) or {}
def _id_safe_name(name: object) -> str | None:
return name if isinstance(name, str) and name and ";" not in name else None
def _stamped_models(metadata: object) -> tuple[str, str | None] | None:
fields: Final = _fields(metadata)
if fields is None:
return None
requested: Final = _id_safe_name(fields.get("model_group"))
served: Final = _id_safe_name(fields.get("deployment_model_name"))
if requested is None:
return None
return requested, None if served is None or served == requested or "*" in served else served
def routed_video_models(logging_obj: object) -> tuple[str, str | None] | None:
"""
A later status or content request checks the key's model access against the model encoded in the id, and
routes by it. One upstream model name can be served by several deployments (regions, accounts), so only the
group that served the task can send its polls back to the deployment that created it, while a key may be
scoped to the alias, routing group or wildcard name it asked for instead. The id carries both names: the one
the router was asked for on this hop, and the group that served it when that differs and is not a wildcard.
The router stamps both into litellm_metadata, and metadata carries a copy only when the caller sent none.
"""
params: Final = _logged_litellm_params(logging_obj)
stamped: Final = (_stamped_models(params.get(key)) for key in ("litellm_metadata", "metadata"))
return next((models for models in stamped if models), None)
def polled_video_models(logging_obj: object) -> tuple[str | None, str | None]:
"""
The served group survives into the returned id only when the router sent this poll to it, so an edited id
cannot write any name it likes back.
"""
video_id: Final = _logged_litellm_params(logging_obj).get("video_id")
if not isinstance(video_id, str):
return None, None
decoded: Final = decode_video_id_with_provider(video_id)
served: Final = decoded.get("served_model")
routed: Final = routed_video_models(logging_obj)
return decoded.get("model_id") or None, served if routed is None or served in routed else None
def _tuple_payload(file_obj: object) -> object:
if not isinstance(file_obj, tuple):
return file_obj
file_tuple: Final = _FILE_TUPLE_ADAPTER.validate_python(file_obj)
return file_tuple[1] if len(file_tuple) >= 2 else None
def _file_content(file_obj: object) -> object:
unwrapped: Final = accumulate(repeat(None), lambda current, _: _tuple_payload(current), initial=file_obj)
return next(item for item in unwrapped if not isinstance(item, tuple))
def input_reference_bytes(file_obj: object) -> bytes:
content: Final = _file_content(file_obj)
if isinstance(content, PurePath):
return Path(content).read_bytes()
if isinstance(content, (BytesIO, BufferedReader)):
current_position: Final = content.tell()
_ = content.seek(0)
data: Final = content.read()
_ = content.seek(current_position)
return data
if isinstance(content, bytes):
return content
if isinstance(content, bytearray):
return bytes(content)
read: Final = getattr(content, "read", None)
if callable(read):
read_data: Final[object] = read()
if isinstance(read_data, bytes):
return read_data
raise ValueError("input_reference must be a URL string, bytes, a path, a file object, or a file tuple")
def _reference_object_url(reference: Mapping[str, object]) -> str:
image_url: Final = reference.get("image_url")
if isinstance(image_url, str) and image_url:
return image_url
raise ValueError("input_reference as an object must carry an image_url, a file_id is not supported")
def input_reference_url(image: object) -> str:
"""A string is a URL the provider fetches itself and is never opened as a local path. OpenAI's object form
carries that URL, or a data URL, in ``image_url``."""
if isinstance(image, str):
return image
reference: Final = _fields(image)
if reference is not None:
return _reference_object_url(reference)
content: Final = input_reference_bytes(image)
content_type: Final = ImageEditRequestUtils.get_image_content_type(content)
encoded: Final = base64.b64encode(content).decode("utf-8")
return f"data:{content_type};base64,{encoded}"

28
litellm/llms/bedrock/base_aws_llm.py
View file

@ -99,11 +99,16 @@ def _assume_role_params(
)
_SecureTransportBool = TypedDict("_SecureTransportBool", {"aws:SecureTransport": ReadOnly[Literal["true"]]})
_SecureTransportBool: Final = TypedDict("_SecureTransportBool", {"aws:SecureTransport": ReadOnly[Literal["true"]]})
_CalledViaLastEquals: Final = TypedDict(
"_CalledViaLastEquals", {"aws:CalledViaLast": ReadOnly[Literal["aws-external-anthropic.amazonaws.com"]]}
)
class _SecureTransportCondition(TypedDict):
class _SessionPolicyCondition(TypedDict):
Bool: ReadOnly[_SecureTransportBool]
StringEquals: NotRequired[ReadOnly[_CalledViaLastEquals]]
class _SessionPolicyStatement(TypedDict):
@ -111,7 +116,7 @@ class _SessionPolicyStatement(TypedDict):
Effect: ReadOnly[Literal["Allow"]]
Action: ReadOnly[tuple[str, ...]]
Resource: ReadOnly[Literal["*"]]
Condition: ReadOnly[_SecureTransportCondition]
Condition: ReadOnly[_SessionPolicyCondition]
class WebIdentitySessionPolicy(TypedDict):
@ -147,11 +152,24 @@ _WEB_IDENTITY_SESSION_POLICY_ACTIONS: Final[Mapping[str, tuple[str, ...]]] = Map
"aws-external-anthropic:Get*",
"aws-external-anthropic:List*",
),
"ClaudePlatformOutboundIdentityLiteLLM": (
"sts:GetWebIdentityToken",
"sts:TagGetWebIdentityToken",
),
"BedrockMantleLiteLLM": ("bedrock-mantle:CreateInference",),
}
)
_SECURE_TRANSPORT_ONLY: Final = _SecureTransportCondition(Bool=_SecureTransportBool({"aws:SecureTransport": "true"}))
_SECURE_TRANSPORT_ONLY: Final = _SessionPolicyCondition(Bool=_SecureTransportBool({"aws:SecureTransport": "true"}))
_WEB_IDENTITY_SESSION_POLICY_CONDITIONS: Final[Mapping[str, _SessionPolicyCondition]] = MappingProxyType(
{
"ClaudePlatformOutboundIdentityLiteLLM": _SessionPolicyCondition(
Bool=_SecureTransportBool({"aws:SecureTransport": "true"}),
StringEquals=_CalledViaLastEquals({"aws:CalledViaLast": "aws-external-anthropic.amazonaws.com"}),
),
}
)
def build_web_identity_session_policy() -> WebIdentitySessionPolicy:
@ -163,7 +181,7 @@ def build_web_identity_session_policy() -> WebIdentitySessionPolicy:
Effect="Allow",
Action=actions,
Resource="*",
Condition=_SECURE_TRANSPORT_ONLY,
Condition=_WEB_IDENTITY_SESSION_POLICY_CONDITIONS.get(sid, _SECURE_TRANSPORT_ONLY),
)
for sid, actions in _WEB_IDENTITY_SESSION_POLICY_ACTIONS.items()
),

19
litellm/llms/bedrock/chat/converse_handler.py
View file

@ -1,5 +1,5 @@
import json
from collections.abc import Iterator
from collections.abc import Iterator, Mapping
from typing import Final
import httpx
@ -10,6 +10,7 @@ from litellm.anthropic_beta_headers_manager import (
)
from litellm.litellm_core_utils.get_litellm_params import stored_control_options
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.llms.base_llm import base_model_iterator
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -21,7 +22,7 @@ from litellm.utils import CustomStreamWrapper
from ..base_aws_llm import BaseAWSLLM, Credentials, bedrock_bearer_token, pop_aws_auth_params, run_aws_signing
from ..common_utils import BedrockError, error_response_text, get_all_bedrock_regions
from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call
from .invoke_handler import AWSEventStreamDecoder, make_call
def make_sync_call(
@ -36,7 +37,8 @@ def make_sync_call(
fake_stream: bool = False,
stream_chunk_size: int | None = None,
timeout: float | httpx.Timeout | None = None,
) -> tuple[MockResponseIterator | Iterator[object], httpx.Headers]:
optional_params: Mapping[str, object] | None = None,
) -> tuple[base_model_iterator.MockResponseIterator | Iterator[object], httpx.Headers]:
if client is None:
client = get_httpx_client() # Create a new client if none provided
@ -64,14 +66,15 @@ def make_sync_call(
model_response=litellm.ModelResponse(),
stream=True,
logging_obj=logging_obj,
optional_params={},
optional_params=dict(optional_params or {}),
api_key="",
data=data,
messages=messages,
encoding=litellm.encoding,
json_mode=json_mode,
)
completion_stream: MockResponseIterator | Iterator[object] = MockResponseIterator(
model_response=model_response, json_mode=json_mode
completion_stream: base_model_iterator.MockResponseIterator | Iterator[object] = (
base_model_iterator.MockResponseIterator(model_response=model_response)
)
else:
decoder: Final = AWSEventStreamDecoder(model=model, json_mode=json_mode)
@ -158,6 +161,7 @@ class BedrockConverseLLM(BaseAWSLLM):
json_mode=json_mode,
stream_chunk_size=stream_chunk_size,
timeout=timeout,
optional_params=optional_params,
)
streaming_response: Final = CustomStreamWrapper(
completion_stream=completion_stream,
@ -290,7 +294,7 @@ class BedrockConverseLLM(BaseAWSLLM):
stream_chunk_size: Final = stored_control_options(litellm_params).stream_chunk_size if stream is True else None
unencoded_model_id: Final = optional_params.pop("model_id", None)
fake_stream = optional_params.pop("fake_stream", False)
json_mode: Final = optional_params.get("json_mode", False)
json_mode: Final = optional_params.get("json_mode") is True
if unencoded_model_id is not None:
modelId = self.encode_model_id(model_id=unencoded_model_id)
else:
@ -463,6 +467,7 @@ class BedrockConverseLLM(BaseAWSLLM):
fake_stream=fake_stream,
stream_chunk_size=stream_chunk_size,
timeout=timeout,
optional_params=optional_params,
)
streaming_response: Final = CustomStreamWrapper(
completion_stream=completion_stream,

Some files were not shown because too many files have changed in this diff Show more