Merge remote-tracking branch 'origin/main' into beluga_decrease_anys_20261102
Some checks failed
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled

# Conflicts:
#	litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py
#	tests/unit/test_utils.py
This commit is contained in:
mateo-berri 2026-11-05 21:42:08 -08:00
commit fdd4b0028a
289 changed files with 10597 additions and 5697 deletions

View file

@ -53,7 +53,7 @@ jobs:
if: >-
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository
timeout-minutes: 30
timeout-minutes: 60
permissions:
contents: read
steps:

View file

@ -49,9 +49,9 @@ jobs:
id: compliance
run: |
uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \
--interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \
--interactions-spec-url=https://aistudio.google.com/api-reference/interactions.openapi.json \
--tb=short -vv -p no:rerunfailures -p no:pytest-retry \
--allow-hosts=127.0.0.1,::1,localhost,ai.google.dev
--allow-hosts=127.0.0.1,::1,localhost,ai.google.dev,aistudio.google.com
tracking-issue:
name: tracking-issue

View file

@ -21,7 +21,7 @@ jobs:
pull-requests: read
actions: read
runs-on: ubuntu-latest
timeout-minutes: 20
timeout-minutes: 30
services:
redis:
image: redis@sha256:e2debfb7956fa12c7ddc79d7e645c8cf26b30c99a6e9161ea9bf4171e1668a5f

View file

@ -45,7 +45,7 @@ jobs:
id: rust-bridge-key
if: steps.changes.outputs.decision != 'skip'
env:
RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml') }}
RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml', 'packaging/litellm-proxy/pyproject.toml') }}
run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT"
- name: Set up Python
@ -98,6 +98,18 @@ jobs:
mkdir -p rust-bridge-dist/litellm/rust_bridge
cp litellm/rust_bridge/_native.abi3.so rust-bridge-dist/litellm/rust_bridge/_native.abi3.so
- name: Build the gateway wheel once for test shards
if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true'
timeout-minutes: 15
working-directory: packaging/litellm-proxy
env:
RUSTC_WRAPPER: sccache
SCCACHE_GHA_ENABLED: "true"
SCCACHE_GHA_VERSION: litellm-ci-v1
SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }}
CARGO_INCREMENTAL: "0"
run: uv run --no-project --with maturin==1.15.0 maturin build --profile dev --strip --out ../../rust-bridge-dist
- name: Upload Rust bridge artifact
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 10
@ -1024,7 +1036,7 @@ jobs:
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
@ -1486,7 +1498,7 @@ jobs:
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
fi

View file

@ -76,6 +76,7 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
COPY pyproject.toml uv.lock ./
COPY enterprise/pyproject.toml enterprise/
COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/
# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
@ -101,7 +102,8 @@ COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Install project and workspace packages (fast - deps already cached)
RUN uv sync --frozen --no-default-groups --no-editable \
RUN uv build --package litellm-proxy --wheel --out-dir /wheels
RUN uv sync --frozen --no-install-package litellm-proxy --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
@ -111,6 +113,8 @@ RUN uv sync --frozen --no-default-groups --no-editable \
--extra bedrock-realtime \
--python python3.13
RUN uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm_proxy-*.whl
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma

View file

@ -74,6 +74,7 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
COPY pyproject.toml uv.lock ./
COPY enterprise/pyproject.toml enterprise/
COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/
# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
@ -99,7 +100,8 @@ COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Install project and workspace packages (fast - deps already cached)
RUN uv sync --frozen --no-default-groups --no-editable \
RUN uv build --package litellm-proxy --wheel --out-dir /wheels
RUN uv sync --frozen --no-install-package litellm-proxy --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
@ -109,6 +111,8 @@ RUN uv sync --frozen --no-default-groups --no-editable \
--extra bedrock-realtime \
--python python3.13
RUN uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm_proxy-*.whl
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma

View file

@ -8,7 +8,6 @@ import re
from collections.abc import Callable, Mapping, Sequence
from dataclasses import replace as dataclasses_replace
from datetime import datetime, timedelta
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tuple, cast
from pydantic import TypeAdapter, ValidationError
@ -766,7 +765,6 @@ class CheckBatchCost:
credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
_file_content = await afile_content(
file_id=cast(str, raw_output_file_id),
_litellm_internal_model_credentials=MappingProxyType(dict(credentials)),
**cast(FileContentCallOptions, credentials),
)
@ -867,10 +865,7 @@ class CheckBatchCost:
error_file_failed_requests: Final = await count_error_file_failed_requests(
response,
custom_llm_provider=batch_file_provider,
litellm_params={
**credentials,
"_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),
},
litellm_params=credentials,
)
batch_result: Final = (
output_file_result
@ -1029,11 +1024,7 @@ class CheckBatchCost:
custom_llm_provider=deployment_info.litellm_params.custom_llm_provider,
)
cost_provider: Final = cast(BatchFileProvider, llm_provider)
credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
file_access: Final = {
**credentials,
"_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),
}
file_access: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
output_content: Final = (
await fetch_batch_managed_file_content(
response.output_file_id, custom_llm_provider=cost_provider, litellm_params=file_access

View file

@ -127,7 +127,6 @@ class _ManagedFileRetrieve(Protocol):
self,
*,
file_id: str,
_litellm_internal_model_credentials: Mapping[str, object] | None = None,
**kwargs: Unpack[FileRetrieveCallOptions],
) -> OpenAIFileObject: ...
@ -400,7 +399,7 @@ class _RouterFileCallKwargs(TypedDict, total=False):
_MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str])
_FILE_UPLOAD_ONLY_FIELDS: Final = frozenset({"file", "purpose", "expires_after", "extra_body"})
_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id", "_litellm_internal_model_credentials"})
_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id"})
@dataclass(frozen=True, slots=True)
@ -459,15 +458,7 @@ async def _delete_provider_file(
provider_file_id: str,
request_data: Mapping[str, object],
) -> None:
credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
delete_data: Final = {
**{k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS},
**(
{"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
if credentials is not None
else {}
),
}
delete_data: Final = {k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS}
await llm_router.afile_delete(
model=model_id,
file_id=provider_file_id,
@ -1160,7 +1151,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
*,
provider_file_id: str,
call_options: Mapping[str, object],
internal_model_credentials: Mapping[str, object] | None = None,
) -> OpenAIFileObject:
"""Retrieve provider file details with transient retries and SDK retries disabled."""
retrieve_file: Final = cast(_ManagedFileRetrieve, litellm.afile_retrieve)
@ -1172,14 +1162,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if attempt > 0:
await self._sleep(_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS[attempt - 1])
try:
if internal_model_credentials is None:
return await retrieve_file(
file_id=provider_file_id,
**retrieve_options,
)
return await retrieve_file(
file_id=provider_file_id,
_litellm_internal_model_credentials=MappingProxyType(dict(internal_model_credentials)),
**retrieve_options,
)
except Exception as error:
@ -1221,7 +1205,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
self._afile_retrieve_with_retries(
provider_file_id=provider_file_id,
call_options=router_credentials,
internal_model_credentials=router_credentials,
),
timeout=BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS,
)
@ -2866,15 +2849,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
exception_dict = {}
for model_id, provider_file_id in specific_model_file_id_mapping.items():
try:
# Cloud-storage providers (e.g. Bedrock S3) validate file ids
# against the deployment's configured bucket, which they only
# trust from this immutable server-side snapshot, never from
# request params.
credentials = llm_router.registry().credentials(model_id, form="with_provider")
if credentials is not None:
data["_litellm_internal_model_credentials"] = cast(Dict, MappingProxyType(dict(credentials)))
else:
data.pop("_litellm_internal_model_credentials", None)
return cast(
HttpxBinaryResponseContent,
await llm_router.afile_content(

View file

@ -41,10 +41,14 @@ COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
COPY model_prices_and_context_window.json ./
COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \
touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
python -c "import maturin; maturin.build_wheel('/wheels')" && \
cd packaging/litellm-proxy && \
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
python -c "import maturin; maturin.build_wheel('/wheels')"
@ -85,6 +89,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,source=uv.lock,target=uv.lock \
--mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \
--mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \
--mount=type=bind,source=packaging/litellm-proxy/pyproject.toml,target=packaging/litellm-proxy/pyproject.toml \
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
@ -98,7 +103,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-install-project --no-default-groups --no-editable \
uv sync --frozen --no-install-project --no-install-package litellm-proxy --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
@ -108,7 +113,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--python python3.13
RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl /wheels/litellm_proxy-*.whl
# PYTHONPATH=/app makes the source tree shadow the installed package, so the
# compiled Rust extension must live next to the source or it is never imported.

File diff suppressed because one or more lines are too long

View file

@ -4,13 +4,7 @@
//! `should_retry_this_error` and `_time_to_sleep_before_retry` in `litellm/router.py`, and
//! `run_async_fallback` in `router_utils/fallback_event_handlers.py`.
use std::{
borrow::Cow,
future::Future,
pin::Pin,
sync::Arc,
time::{Duration, Instant},
};
use std::{borrow::Cow, future::Future, pin::Pin, sync::Arc, time::Duration};
use serde_json::{Map, Value};
@ -1231,7 +1225,7 @@ impl Core {
.unrouted_attempt(snapshot, op, request, target, (retry, max_retries), at)
.await;
}
let pick_started = Instant::now();
let pick_started = self.health.now_time();
let (deployment, candidates) = match self
.picked(snapshot, op, request, target, inputs, retry, at)
.await
@ -1248,7 +1242,11 @@ impl Core {
self.host.observe(&RouterEvent::Selected {
model_group: candidates.model_group.clone(),
deployment_id: deployment.id.clone(),
pick: pick_started.elapsed(),
pick: self
.health
.now_time()
.duration_since(pick_started)
.unwrap_or_default(),
});
if !self.host.supports(op, deployment.provider()) {
return Err(Error::Config(ConfigError::Unsupported(Unsupported::Op {
@ -1274,7 +1272,7 @@ impl Core {
},
None,
snapshot.settings.dynamic.default_ref(),
jiff::Timestamp::now(),
self.health.timestamp(),
),
Ok(None)
);

View file

@ -247,6 +247,10 @@ impl Health {
self.clock.now()
}
pub(crate) fn timestamp(&self) -> jiff::Timestamp {
jiff::Timestamp::try_from(self.clock.now()).unwrap_or(jiff::Timestamp::UNIX_EPOCH)
}
fn now(&self) -> f64 {
unix_seconds(self.clock.now())
}

View file

@ -433,6 +433,10 @@ impl<'a> Lookup<'a> {
.collect()
}
pub fn has_team_wildcards(&self, team_id: &str) -> bool {
self.host.team_patterns.iter().any(|(t, _)| t == team_id)
}
/// The deployments `team_id`'s own wildcard pattern routes `name` to
/// (`_core_team_pattern_routers`, keyed by `team_public_model_name`).
pub fn team_wildcard(&self, name: &str, team_id: &str) -> Vec<Arc<Deployment>> {

View file

@ -223,14 +223,20 @@ pub(crate) fn wildcard_name<'a>(
registry: &'a Registry,
name: &'a str,
specific_deployment: bool,
team_id: Option<&str>,
) -> Option<&'a str> {
let model = registry.alias_of(name).unwrap_or(name);
(registry.has_wildcards()
let lookup = crate::lookup::Lookup::new(registry);
let team_patterns = team_id.is_some_and(|team| lookup.has_team_wildcards(team));
let team_misses = || team_id.is_none_or(|team| lookup.team_wildcard(model, team).is_empty());
let global_misses = || registry.has_wildcards() && registry.wildcard_for(model).is_none();
((team_patterns || registry.has_wildcards())
&& !specific_deployment
&& !registry.has_name(model)
&& (registry.has_name(name) || registry.deployment(name).is_none())
&& registry.routing_group_named(model).is_none()
&& registry.wildcard_for(model).is_none())
&& team_misses()
&& (team_patterns || global_misses()))
.then_some(model)
}
@ -339,15 +345,21 @@ pub fn resolve_request(
let kept = reserved(model, deployments, caller, now)?;
return resolved(ResolveKind::Group, model, kept);
}
// `_core_wildcard_route_deployments`: the team's own pattern, else the proxy-wide one.
let team_owned = team_id
.map(|team| crate::lookup::Lookup::new(registry).team_wildcard(model, team))
.filter(|deployments| !deployments.is_empty());
if let Some(deployments) = team_owned {
// `_core_wildcard_route_deployments`: the team's own pattern, else the proxy-wide one,
// each tried as spelled and then under the name's provider (`get_pattern`).
let team_owned = team_id.and_then(|team| {
let lookup = crate::lookup::Lookup::new(registry);
[Some(model), caller.wildcard_request]
.into_iter()
.flatten()
.map(|request| (request, lookup.team_wildcard(request, team)))
.find(|(_, deployments)| !deployments.is_empty())
});
if let Some((request, deployments)) = team_owned {
let routed = deployments.iter().flat_map(|deployment| {
let public = deployment.model_info.team_public_model_name.as_deref();
let pattern = public.unwrap_or(model);
routed_wildcard(pattern, model, std::slice::from_ref(deployment))
let pattern = public.unwrap_or(request);
routed_wildcard(pattern, request, std::slice::from_ref(deployment))
});
let kept = reserved(model, routed.collect(), caller, now)?;
return resolved(ResolveKind::Wildcard, model, kept);
@ -428,6 +440,7 @@ pub fn resolve_request(
mod tests {
use std::collections::BTreeMap;
use rstest::rstest;
use serde_json::{Value, json};
use super::*;
@ -544,8 +557,11 @@ mod tests {
#[test]
fn a_name_no_pattern_matches_is_tried_under_its_provider() {
let registry = registry(wildcard_group());
assert_eq!(wildcard_name(&registry, "gpt-4o", false), Some("gpt-4o"));
assert_eq!(wildcard_name(&registry, "openai/gpt-4o", false), None);
assert_eq!(
wildcard_name(&registry, "gpt-4o", false, None),
Some("gpt-4o")
);
assert_eq!(wildcard_name(&registry, "openai/gpt-4o", false, None), None);
let caller = Caller {
wildcard_request: Some("openai/gpt-4o"),
..Caller::default()
@ -564,6 +580,48 @@ mod tests {
assert_eq!(resolution.deployments[0].params.model, "openai/gpt-4o");
}
#[rstest]
fn a_bare_name_is_tried_under_its_provider_against_the_teams_own_wildcard() {
let registry = registry(json!([
{"model_name": "model_name_tw_1", "litellm_params": {"model": "openai/*"},
"model_info": {"id": "tw", "team_id": "TW", "team_public_model_name": "openai/*"}},
{"model_name": "g", "litellm_params": {"model": "openai/g"}, "model_info": {"id": "g"}},
]));
// No proxy-wide wildcard: only the team's patterns call for the provider spelling.
assert_eq!(
wildcard_name(&registry, "gpt-4o-mini", false, Some("TW")),
Some("gpt-4o-mini")
);
assert_eq!(wildcard_name(&registry, "gpt-4o-mini", false, None), None);
assert_eq!(
wildcard_name(&registry, "openai/gpt-4o-mini", false, Some("TW")),
None
);
let resolve = |team_id| {
let caller = Caller {
team_id,
wildcard_request: Some("openai/gpt-4o-mini"),
..Caller::default()
};
resolve_request(
&registry,
"gpt-4o-mini",
false,
caller,
None,
None,
Timestamp::now(),
)
.unwrap()
};
let resolution = resolve(Some("TW")).unwrap();
assert_eq!(resolution.kind, ResolveKind::Wildcard);
assert_eq!(resolution.group, "gpt-4o-mini");
assert_eq!(ids(Some(resolution.clone())), ["tw"]);
assert_eq!(resolution.deployments[0].params.model, "openai/gpt-4o-mini");
assert!(resolve(Some("OTHER")).is_none());
}
#[test]
fn a_deployment_named_by_id_refuses_the_endpoint_like_any_other() {
let registry = registry(json!([

View file

@ -281,8 +281,12 @@ impl Core {
// No deployment for the name: route to the first default fallback.
let default_target = crate::resolve::default_target(&snapshot.settings.dynamic);
// `get_pattern`: a name no pattern matches is tried again under its inferred provider.
let wildcard_name =
crate::resolve::wildcard_name(registry, name, request.specific_deployment);
let wildcard_name = crate::resolve::wildcard_name(
registry,
name,
request.specific_deployment,
request.team_id.as_deref(),
);
let wildcard_request = match wildcard_name {
Some(model) => {
let scope = StrategyScope::Default;
@ -308,7 +312,7 @@ impl Core {
caller,
default_target,
snapshot.settings.dynamic.default_ref(),
jiff::Timestamp::now(),
self.health.timestamp(),
)?
.ok_or_else(|| Error::UnknownModel {
// The name `_common_checks_available_deployment` looked for, past the alias.

View file

@ -1755,3 +1755,57 @@ async fn a_listed_pass_through_pick_names_the_group_an_alias_resolves_to(config:
Error::NoPassThroughDeployments { model } if model == "primary"
));
}
struct StoppedClock(u64);
impl litellm_router_core::health::Clock for StoppedClock {
fn now(&self) -> std::time::SystemTime {
std::time::UNIX_EPOCH + Duration::from_secs(self.0)
}
}
fn utc_hour(hour: u64) -> u64 {
1_767_225_600 + hour * 3_600
}
#[rstest]
#[case::inside_the_window(utc_hour(10), vec!["open"])]
#[case::outside_the_window(utc_hour(20), vec!["open", "reserved"])]
#[tokio::test]
async fn access_windows_follow_the_routers_clock(#[case] now: u64, #[case] expected: Vec<&str>) {
let config: RouterConfig = serde_json::from_value(json!({
"model_list": [
{"model_name": "chat", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "open"}},
{"model_name": "chat", "litellm_params": {"model": "openai/b"}, "model_info": {
"id": "reserved",
"access_windows": [{"start": "09:00", "end": "17:00", "timezone": "UTC", "team_ids": ["batch"]}],
}},
]
}))
.unwrap();
let router = Router::builder(
config,
RouterDynamicConfig::default(),
Arc::new(ScriptedHost::default()),
)
.randomness(Arc::new(SeededRandomness::new(7)))
.clock(Arc::new(StoppedClock(now)))
.build()
.unwrap();
let mut picked = std::collections::BTreeSet::new();
for _ in 0..40 {
let deployment = router
.select(&Op::from("acompletion"), &request_for("chat"))
.await
.unwrap();
picked.insert(deployment.id.clone());
}
assert_eq!(picked.into_iter().collect::<Vec<_>>(), expected);
}
fn request_for(model: &str) -> RouterRequest {
RouterRequest {
model: model.to_owned(),
..RouterRequest::default()
}
}

View file

@ -641,9 +641,11 @@ def extract_file_access_credentials(
"bucket_name",
"s3_endpoint_url",
"s3_region_name",
"s3_bucket_name",
"s3_output_bucket_name",
"allow_legacy_cloud_file_ids",
"timeout",
"max_retries",
"_litellm_internal_model_credentials",
*AWS_CREDENTIAL_KWARGS_KEYS,
# A federated deployment holds no api_key, so without these the fetch that reads a
# finished batch's output has nothing to authenticate with and its cost is never billed.

View file

@ -22,7 +22,6 @@ from openai.types.batch import BatchRequestCounts
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.get_litellm_params import add_trusted_model_credentials_to_litellm_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.batches.handler import AnthropicBatchesHandler
from litellm.llms.azure.batches.handler import AzureBatchesAPI
@ -585,7 +584,6 @@ def retrieve_batch(
custom_llm_provider=custom_llm_provider,
**kwargs,
)
add_trusted_model_credentials_to_litellm_params(litellm_params, kwargs)
if litellm_logging_obj is not None:
litellm_logging_obj.update_from_kwargs(
kwargs=kwargs,

View file

@ -41,12 +41,14 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import (
)
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, iter_items
from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
prompt_cache_breakpoint_with_readable_mode,
responses_reasoning_items_from_thinking_blocks,
whole_prompt_cache_breakpoint_or_none,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
responses_reasoning_items_from_thinking_blocks,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.bridges.completion_transformation import (
CompletionTransformationBridge,

View file

@ -38,7 +38,6 @@ import litellm
from litellm import get_secret_str
from litellm.files.streaming import FileContentStreamingResponse
from litellm.files.types import FileContentProvider, FileContentStreamingResult, FileRetrieveProvider
from litellm.litellm_core_utils.get_litellm_params import add_trusted_model_credentials_to_litellm_params
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.azure.common_utils import get_azure_credentials
@ -422,10 +421,6 @@ def file_retrieve(
)
if provider_config is not None:
litellm_params_dict: Final = get_litellm_params(**kwargs)
add_trusted_model_credentials_to_litellm_params(
litellm_params_dict=litellm_params_dict,
kwargs=kwargs,
)
litellm_params_dict["api_key"] = optional_params.api_key
litellm_params_dict["api_base"] = optional_params.api_base
@ -544,10 +539,6 @@ def file_delete(
pass
optional_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params_dict: Final = get_litellm_params(**kwargs)
add_trusted_model_credentials_to_litellm_params(
litellm_params_dict=litellm_params_dict,
kwargs=kwargs,
)
### TIMEOUT LOGIC ###
timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600
# set timeout for 10 minutes by default
@ -737,10 +728,6 @@ def file_list(
)
if provider_config is not None:
litellm_params_dict: Final = get_litellm_params(**kwargs)
add_trusted_model_credentials_to_litellm_params(
litellm_params_dict=litellm_params_dict,
kwargs=kwargs,
)
litellm_params_dict["api_key"] = optional_params.api_key
litellm_params_dict["api_base"] = optional_params.api_base
@ -890,10 +877,6 @@ def file_content(
try:
optional_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params_dict: Final = get_litellm_params(**kwargs)
add_trusted_model_credentials_to_litellm_params(
litellm_params_dict=litellm_params_dict,
kwargs=kwargs,
)
### TIMEOUT LOGIC ###
timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600
client: Final = kwargs.get("client")

View file

@ -26,18 +26,18 @@ from litellm.exceptions import UnsupportedParamsError
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import PromptManagementClient
from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.prompt_templates.factory import (
tool_call_ids_answered_by_client,
tool_call_ids_answered_in,
)
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import string_field
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT,
cache_control_as_prompt_cache_breakpoint,
prompt_cache_breakpoint_as_cache_control,
with_prompt_cache_breakpoint,
without_prompt_cache_breakpoints,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
tool_call_ids_answered_by_client,
tool_call_ids_answered_in,
)
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import string_field
from litellm.llms.anthropic.common_utils import (
is_claude_code_one_shot_subagent_request,
supports_anthropic_cache_control,

View file

@ -898,6 +898,11 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
def redacts_messages_itself(self) -> bool:
return False
def exempt_from_message_redaction(self) -> bool:
"""Router pre-call checks that key routing off the request and never log it return True, so
`turn_off_message_logging` leaves the success event they receive unredacted."""
return False
def redact_standard_logging_payload_from_model_call_details(self, model_call_details: dict) -> dict:
"""
Redacts or excludes fields from StandardLoggingPayload before callbacks receive it.

View file

@ -71,12 +71,12 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
_TERMINAL_STATUSES: Final = frozenset(
{"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action"}
{"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action", "requires_continuation"}
)
_POLLABLE_STATUSES: Final = frozenset({"in_progress", "queued"})
_STATUSES_THAT_PRODUCED_OUTPUT: Final = frozenset({"completed", "requires_action"})
_STATUSES_THAT_PRODUCED_OUTPUT: Final = frozenset({"completed", "requires_action", "requires_continuation"})
SettlementOutcome: TypeAlias = Literal["billed", "released", "unsettled"]

View file

@ -95,6 +95,7 @@ async def acreate(
response_mime_type: str | None = None,
# Continuation
previous_interaction_id: str | None = None,
continuation_token: str | None = None,
# Extra params
extra_headers: dict[str, str] | None = None,
extra_body: Mapping[str, object] | None = None,
@ -129,6 +130,7 @@ async def acreate(
response_format: JSON schema for response format
response_mime_type: MIME type of the response
previous_interaction_id: ID of previous interaction for continuation
continuation_token: Token from a `requires_continuation` interaction to resume its long decode
extra_headers: Additional headers
extra_body: Additional body parameters
timeout: Request timeout
@ -164,6 +166,7 @@ async def acreate(
response_format=response_format,
response_mime_type=response_mime_type,
previous_interaction_id=previous_interaction_id,
continuation_token=continuation_token,
extra_headers=extra_headers,
extra_body=extra_body,
timeout=timeout,
@ -224,6 +227,7 @@ def create(
response_mime_type: str | None = None,
# Continuation
previous_interaction_id: str | None = None,
continuation_token: str | None = None,
# Extra params
extra_headers: dict[str, str] | None = None,
extra_body: Mapping[str, object] | None = None,
@ -262,6 +266,7 @@ def create(
response_format: JSON schema for response format
response_mime_type: MIME type of the response
previous_interaction_id: ID of previous interaction for continuation
continuation_token: Token from a `requires_continuation` interaction to resume its long decode
extra_headers: Additional headers
extra_body: Additional body parameters
timeout: Request timeout

View file

@ -20,6 +20,7 @@ INTERACTIONS_API_OPTIONAL_PARAMS: Final = {
"response_format",
"response_mime_type",
"previous_interaction_id",
"continuation_token",
"agent_config",
}

View file

@ -2,12 +2,10 @@ import posixpath
import re
from collections.abc import Mapping, Sequence
from itertools import accumulate, repeat
from types import MappingProxyType
from typing import Final
from urllib.parse import quote, unquote
from litellm._uuid import uuid
from litellm.litellm_core_utils.core_helpers import as_str_mapping
VERTEX_AI_MANAGED_GCS_PREFIX: Final = "litellm-vertex-files/"
BEDROCK_MANAGED_S3_BATCH_PREFIX: Final = "litellm-bedrock-files-"
@ -19,7 +17,6 @@ BEDROCK_MANAGED_S3_PREFIXES: Final = (
BEDROCK_MANAGED_S3_OUTPUT_PREFIX,
)
MANAGED_CLOUD_STORAGE_SCHEMES: Final = ("s3://", "gs://")
_MAPPING_PROXY_TYPE: Final[type] = type(MappingProxyType({}))
MAX_FILE_ID_DECODE_PASSES: Final = 8
@ -138,13 +135,7 @@ def encode_s3_object_key_for_url(object_key: str) -> str:
def should_allow_legacy_cloud_file_ids(
litellm_params: Mapping[str, object] | None = None,
) -> bool:
value = None
if isinstance(litellm_params, Mapping):
trusted_model_credentials: Final = litellm_params.get("_litellm_internal_model_credentials")
if isinstance(trusted_model_credentials, _MAPPING_PROXY_TYPE):
trusted_mapping: Final = as_str_mapping(trusted_model_credentials)
value = trusted_mapping.get("allow_legacy_cloud_file_ids") if trusted_mapping is not None else None
value: Final = litellm_params.get("allow_legacy_cloud_file_ids") if litellm_params is not None else None
if isinstance(value, bool):
return value
if isinstance(value, str):

View file

@ -1,7 +1,6 @@
import reprlib
from collections.abc import Mapping, MutableMapping
from collections.abc import Mapping
from dataclasses import dataclass, fields
from types import MappingProxyType
from typing import Final, cast
from pydantic import TypeAdapter, ValidationError
@ -61,6 +60,9 @@ OPTIONAL_KWARGS_KEYS: Final = (
"client_side_timeout",
"gcs_bucket_name",
"bucket_name",
"s3_bucket_name",
"s3_output_bucket_name",
"allow_legacy_cloud_file_ids",
"s3_endpoint_url",
"s3_region_name",
"s3_access_key_id",
@ -292,19 +294,3 @@ def get_litellm_params(
litellm_params[key] = kwargs[key]
return litellm_params
def add_trusted_model_credentials_to_litellm_params(
litellm_params_dict: MutableMapping[str, object], kwargs: Mapping[str, object]
) -> None:
"""
Carry the immutable server-side credential snapshot into litellm_params.
get_litellm_params has a fixed signature, so callers that need the snapshot to
survive into the logging object and the downstream file read have to re-add it. Only
a MappingProxyType is accepted, since providers resolve trusted configuration such
as a Bedrock file bucket from it and must not read a request-supplied mapping.
"""
trusted_model_credentials: Final = kwargs.get("_litellm_internal_model_credentials")
if isinstance(trusted_model_credentials, MappingProxyType):
litellm_params_dict["_litellm_internal_model_credentials"] = trusted_model_credentials

View file

@ -4440,7 +4440,11 @@ class Logging(LiteLLMLoggingBaseClass):
if isinstance(callback, CustomLogger): # custom logger class
from litellm.integrations.shadow_eval_logger import ShadowEvalLogger
model_call_details: dict = self.model_call_details
model_call_details: dict = (
self._model_call_details_with_request_messages()
if callback.exempt_from_message_redaction()
else self.model_call_details
)
##################################
# call redaction hook for custom logger
model_call_details = callback.redact_standard_logging_payload_from_model_call_details(
@ -5004,6 +5008,21 @@ class Logging(LiteLLMLoggingBaseClass):
) -> None:
executor.submit(self.success_handler, result, start_time, end_time, cache_hit)
def _model_call_details_with_request_messages(self) -> dict[str, object]: # mutable-ok: loggers take a dict
"""The success-event payload with the request messages put back where `turn_off_message_logging`
redacted them in place, for a callback that routes on the request and never logs it."""
request_messages: Final = cast(object, self.messages) # cast-ok: the request messages are only put back as-is
standard_logging_object: Final = self.model_call_details.get("standard_logging_object")
return {
**self.model_call_details,
"messages": request_messages,
**(
{"standard_logging_object": {**standard_logging_object, "messages": request_messages}}
if isinstance(standard_logging_object, dict)
else {}
),
}
def _get_completed_success_callbacks(
self, *, asynchronous: bool
) -> tuple[str | Callable[..., object] | CustomLogger, ...]:

View file

@ -19,7 +19,7 @@ from litellm.constants import (
VLLM_CHOICE_FIELDS,
)
from litellm.litellm_core_utils.hidden_params import get_hidden_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
extract_reasoning_content,
)
from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
@ -1072,9 +1072,10 @@ def convert_to_model_response_object(
if isinstance(e, APIError):
raise
hide_reply: Final = response_type == "audio_transcription"
received_args: Final = dict(
response_object=response_object,
model_response_object=model_response_object,
response_object="<hidden>" if hide_reply else response_object,
model_response_object="<hidden>" if hide_reply else model_response_object,
response_type=response_type,
stream=stream,
start_time=start_time,

File diff suppressed because it is too large Load diff

View file

@ -4291,7 +4291,7 @@ def _convert_to_bedrock_tool_call_invoke(
- extract name
- extract id
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import (
split_concatenated_json_objects,
)

View file

@ -0,0 +1,635 @@
import io
import mimetypes
import re
from collections.abc import Mapping
from os import PathLike
from pathlib import Path
from typing import Final, cast # noqa: TID251 # Existing casts retained unchanged in this pure move
from litellm import verbose_logger
from litellm.litellm_core_utils.batch_utils import InMemoryFile
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionFileObject,
)
from litellm.types.utils import (
ExtractedFileData,
FileTypes,
SpecialEnums,
)
def get_format_from_file_id(file_id: str | None) -> str | None:
"""
Gets format from file id
unified_file_id = litellm_proxy:{};unified_id,{}
If not a unified file id, returns 'file' as default format
"""
from litellm.proxy.openai_files_endpoints.common_utils import (
convert_b64_uid_to_unified_uid,
)
if not file_id:
return None
try:
transformed_file_id: Final = convert_b64_uid_to_unified_uid(file_id)
if transformed_file_id.startswith(SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value):
match: Final = re.match(
f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}:(.*?);unified_id",
transformed_file_id,
)
if match:
return match.group(1)
return None
except Exception:
return None
def update_messages_with_model_file_ids(
messages: list[AllMessageValues],
model_id: str | None,
model_file_id_mapping: dict[str, dict[str, str]],
) -> list[AllMessageValues]:
"""
Updates messages with model file ids.
For managed files (unified file IDs), uses model_file_id_mapping if it
resolves the id, otherwise decodes the base64-encoded unified file ID
and extracts the llm_output_file_id directly. Mirrors the Responses-API
sibling `update_responses_input_with_model_file_ids`.
model_file_id_mapping: Dict[str, Dict[str, str]] = {
"litellm_proxy/file_id": {
"model_id": "provider_file_id"
}
}
"""
from litellm.proxy.openai_files_endpoints.common_utils import (
convert_b64_uid_to_unified_uid,
get_original_file_id,
is_base64_encoded_unified_file_id,
is_model_embedded_id,
)
for message in messages:
if message.get("role") == "user":
content = message.get("content")
if content:
if isinstance(content, str):
continue
for c in content:
if not isinstance(c, dict):
# Content list items aren't always dicts. e.g.
# text_completion forwards a token-ids list/list-of-
# lists through this path. Skip non-dict items
# instead of indexing into them.
continue
if c.get("type") == "file":
file_object = cast(ChatCompletionFileObject, c)
file_object_file_field = file_object.get("file")
if not isinstance(file_object_file_field, dict):
# Content block has `type: "file"` but not the
# OpenAI Chat Completions shape (e.g. a LangChain
# v1 standardized file block, or a provider-native
# shape that also uses `type: "file"`). Nothing to
# remap here, so skip instead of crashing.
continue
file_id = file_object_file_field.get("file_id")
format = file_object_file_field.get("format", get_format_from_file_id(file_id))
if file_id:
provider_file_id = (
model_file_id_mapping.get(file_id, {}).get(model_id)
if model_file_id_mapping and model_id is not None
else None
)
if not provider_file_id and is_base64_encoded_unified_file_id(file_id):
unified_file_id = convert_b64_uid_to_unified_uid(file_id)
if "llm_output_file_id," in unified_file_id:
provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
if not provider_file_id and is_model_embedded_id(file_id):
provider_file_id = get_original_file_id(file_id)
file_object_file_field["file_id"] = provider_file_id or file_id
if format:
file_object_file_field["format"] = format
return messages
def update_responses_input_with_model_file_ids(
input: object,
model_id: str | None = None,
model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None,
) -> object:
"""
Updates responses API input with provider-specific file IDs.
File IDs are always inside the content array, not as direct input_file items.
For managed files (unified file IDs), uses model_file_id_mapping if provided,
otherwise decodes the base64-encoded unified file ID and extracts the llm_output_file_id directly.
Args:
input: The responses API input parameter
model_id: The model ID to use for looking up provider-specific file IDs
model_file_id_mapping: Dictionary mapping litellm file IDs to provider file IDs
Format: {"litellm_file_id": {"model_id": "provider_file_id"}}
"""
from litellm.proxy.openai_files_endpoints.common_utils import (
convert_b64_uid_to_unified_uid,
get_original_file_id,
is_base64_encoded_unified_file_id,
is_model_embedded_id,
)
if isinstance(input, str):
return input
if not isinstance(input, list):
return input
updated_input: Final = []
for item in input:
if not isinstance(item, dict):
updated_input.append(item)
continue
updated_item = item.copy()
content = item.get("content")
if isinstance(content, list):
updated_content = []
for content_item in content:
if isinstance(content_item, dict) and content_item.get("type") == "input_file":
file_id = content_item.get("file_id")
if isinstance(file_id, str) and file_id:
provider_file_id = file_id # Default to original
# Check if we have a mapping for this file ID
if model_file_id_mapping and model_id and file_id in model_file_id_mapping:
# Use the model-specific file ID from mapping
provider_file_id = model_file_id_mapping.get(file_id, {}).get(model_id) or file_id
updated_content_item = content_item.copy()
updated_content_item["file_id"] = provider_file_id
updated_content.append(updated_content_item)
else:
# Check if this is a base64-encoded unified file ID without mapping
is_unified_file_id = is_base64_encoded_unified_file_id(file_id)
if is_unified_file_id:
# Fallback: decode unified file ID
unified_file_id = convert_b64_uid_to_unified_uid(file_id)
if "llm_output_file_id," in unified_file_id:
provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
updated_content_item = content_item.copy()
updated_content_item["file_id"] = provider_file_id
updated_content.append(updated_content_item)
elif is_model_embedded_id(file_id):
updated_content_item = content_item.copy()
updated_content_item["file_id"] = get_original_file_id(file_id)
updated_content.append(updated_content_item)
else:
# Not a managed file, keep as-is
updated_content.append(content_item)
else:
updated_content.append(content_item)
else:
updated_content.append(content_item)
updated_item["content"] = updated_content
updated_input.append(updated_item)
return updated_input
def _decode_vector_store_ids_in_tools(
tools: list[dict[str, object]] | None,
) -> list[dict[str, object]] | None:
"""
Decodes unified (LiteLLM-managed) vector_store_ids in file_search tools to
provider-native IDs. Non-unified IDs are passed through unchanged.
This runs unconditionally — no file-ID mapping is required.
"""
if not tools or not isinstance(tools, list):
return tools
from litellm.llms.base_llm.managed_resources.utils import (
is_base64_encoded_unified_id,
parse_unified_id,
)
updated_tools: Final = []
for tool in tools:
if not isinstance(tool, dict) or tool.get("type") != "file_search":
updated_tools.append(tool)
continue
vector_store_ids = tool.get("vector_store_ids")
if not isinstance(vector_store_ids, list):
updated_tools.append(tool)
continue
decoded_ids = []
for vs_id in vector_store_ids:
if not isinstance(vs_id, str) or not is_base64_encoded_unified_id(vs_id):
decoded_ids.append(vs_id)
continue
parsed = parse_unified_id(vs_id)
provider_resource_id = parsed.get("provider_resource_id") if parsed else None
if not provider_resource_id:
verbose_logger.warning(
"file_search tool contains unified vector_store_id '%s' that could "
"not be decoded to a provider resource ID — passing original ID. "
"Ensure the vector store was created via LiteLLM.",
vs_id,
)
decoded_ids.append(vs_id)
else:
decoded_ids.append(provider_resource_id)
updated_tools.append({**tool, "vector_store_ids": decoded_ids})
return updated_tools
def update_responses_tools_with_model_file_ids(
tools: list[dict[str, object]] | None,
model_id: str | None = None,
model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None,
) -> list[dict[str, object]] | None:
"""
Updates responses API tools with provider-specific file IDs.
Pass 1 (always): decode unified vector_store_ids in file_search tools.
Pass 2 (needs mapping): map code_interpreter container file_ids to provider IDs.
Args:
tools: The responses API tools parameter
model_id: The model ID to use for looking up provider-specific file IDs
model_file_id_mapping: Dictionary mapping litellm file IDs to provider file IDs
Format: {"litellm_file_id": {"model_id": "provider_file_id"}}
"""
if not tools or not isinstance(tools, list):
return tools
# Pass 1: decode unified vector_store_ids (no mapping needed)
tools = _decode_vector_store_ids_in_tools(tools) or tools
# Pass 2: map code_interpreter file IDs (requires mapping)
if not model_file_id_mapping or not model_id:
return tools
updated_tools: Final = []
for tool in tools:
if not isinstance(tool, dict):
updated_tools.append(tool)
continue
updated_tool = tool.copy()
# Handle code_interpreter with container file_ids
if tool.get("type") == "code_interpreter":
container = tool.get("container")
if isinstance(container, dict):
container_file_ids = container.get("file_ids")
if isinstance(container_file_ids, list):
updated_file_ids = []
for file_id in container_file_ids:
if isinstance(file_id, str):
# Check if we have a mapping for this file ID
if file_id in model_file_id_mapping:
# Map to provider-specific file ID
provider_file_id = model_file_id_mapping.get(file_id, {}).get(model_id) or file_id
updated_file_ids.append(provider_file_id)
else:
updated_file_ids.append(file_id)
else:
updated_file_ids.append(file_id)
# Update the tool with new file IDs
updated_container = container.copy()
updated_container["file_ids"] = updated_file_ids
updated_tool["container"] = updated_container
updated_tools.append(updated_tool)
return updated_tools
def extract_file_metadata(file_data: FileTypes) -> tuple[str | None, str | None]:
"""
Resolve (filename, content_type) without reading the file body.
Mirrors extract_file_data's metadata resolution but never calls .read(), so
it stays O(1) on large uploads. Use this when only metadata is needed (batch
detection, GCS object naming) and the body must remain a streamable Path/handle.
"""
filename: str | None = None
content_type: str | None = None
file_content: object = None
if isinstance(file_data, tuple):
if len(file_data) == 2:
filename, file_content = file_data
elif len(file_data) == 3:
filename, file_content, content_type = file_data
elif len(file_data) == 4:
filename, file_content, content_type, _ = file_data
elif isinstance(file_data, InMemoryFile):
filename = file_data.name
content_type = file_data.content_type
else:
file_content = file_data
if filename is None:
if isinstance(file_content, PathLike):
filename = Path(file_content).name
elif isinstance(file_content, io.IOBase):
name_attr: Final = getattr(file_content, "name", None)
if isinstance(name_attr, str):
filename = Path(name_attr).name
if not content_type:
guessed: Final = mimetypes.guess_type(filename)[0] if filename else None
content_type = guessed or "application/octet-stream"
return filename, content_type
def extract_file_data(file_data: FileTypes) -> ExtractedFileData:
"""
Extracts and processes file data from various input formats.
Args:
file_data: Can be a tuple of (filename, content, [content_type], [headers]) or direct file content
Returns:
ExtractedFileData containing:
- filename: Name of the file if provided
- content: The file content in bytes
- content_type: MIME type of the file
- headers: Any additional headers
"""
# Parse the file_data based on its type
filename = None
file_content = None
content_type = None
file_headers: Mapping[str, str] = {}
if isinstance(file_data, tuple):
if len(file_data) == 2:
filename, file_content = file_data
elif len(file_data) == 3:
filename, file_content, content_type = file_data
elif len(file_data) == 4:
filename, file_content, content_type, file_headers = file_data
elif isinstance(file_data, InMemoryFile):
filename = file_data.name
file_content = file_data
content_type = file_data.content_type
else:
file_content = file_data
# Convert content to bytes
if isinstance(file_content, str):
# Bare string inputs are rejected: when this helper runs in a proxy
# request handler the string came from an attacker-controlled form
# field, and opening it as a path is an arbitrary file read on the
# proxy host. SDK callers who want to upload from a path should
# either pass a pathlib.Path (a PathLike instance — see the branch
# below) or open the file themselves and pass the handle / bytes.
raise ValueError(
"extract_file_data does not accept bare str inputs. Pass bytes, "
"an open file handle, a (filename, content) tuple, or a "
"pathlib.Path. To upload a local file from a path, call "
"open(path, 'rb') yourself."
)
if isinstance(file_content, PathLike):
# PathLike (pathlib.Path) is a Python-level type that HTTP form
# values can't fabricate. Treat as a local file path for SDK
# convenience.
if filename is None:
filename = Path(file_content).name
with open(file_content, "rb") as f:
content = f.read()
elif isinstance(file_content, io.IOBase):
# If it's a file-like object
# Try to get filename from file handle if not already set
if not filename and hasattr(file_content, "name"):
filename = Path(file_content.name).name
content = file_content.read()
if isinstance(content, str):
content = content.encode("utf-8")
# Reset file pointer to beginning
file_content.seek(0)
elif isinstance(file_content, bytes):
content = file_content
else:
raise ValueError(f"Unsupported file content type: {type(file_content)}")
# Use provided content type or guess based on filename
if not content_type:
if filename:
guessed_type: Final = mimetypes.guess_type(filename)[0]
content_type = guessed_type if guessed_type else "application/octet-stream"
else:
content_type = "application/octet-stream"
return ExtractedFileData(
filename=filename,
content=content,
content_type=content_type,
headers=file_headers,
)
def get_image_mime_type_from_url(url: str) -> str | None:
"""
Get mime type for common image URLs
See gemini mime types: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/image-understanding#image-requirements
Supported by Gemini:
application/pdf
audio/mpeg
audio/mp3
audio/wav
audio/ogg
image/png
image/jpeg
image/webp
text/plain
video/mov
video/mpeg
video/mp4
video/mpg
video/avi
video/wmv
video/mpegps
video/flv
"""
from urllib.parse import urlparse
url = url.lower()
# Parse URL to extract path without query parameters
# This handles URLs like: https://example.com/image.jpg?signature=...
parsed: Final = urlparse(url)
path: Final = parsed.path
# Map file extensions to mime types
mime_types: Final = {
# Images
(".jpg", ".jpeg"): "image/jpeg",
(".png",): "image/png",
(".webp",): "image/webp",
# Videos
(".mp4",): "video/mp4",
(".mov",): "video/mov",
(".mpeg", ".mpg"): "video/mpeg",
(".avi",): "video/avi",
(".wmv",): "video/wmv",
(".mpegps",): "video/mpegps",
(".flv",): "video/flv",
# Audio
(".mp3",): "audio/mp3",
(".wav",): "audio/wav",
(".mpeg",): "audio/mpeg",
(".ogg",): "audio/ogg",
# Documents
(".pdf",): "application/pdf",
(".txt",): "text/plain",
}
# Check each extension group against the URL
for extensions, mime_type in mime_types.items():
if any(path.endswith(ext) for ext in extensions):
return mime_type
return None
_get_image_mime_type_from_url = get_image_mime_type_from_url
def infer_content_type_from_url_and_content(
url: str,
content: bytes,
current_content_type: str | None = None,
) -> str:
"""
Infer content type from URL extension and binary content when content-type header is missing or generic.
This helper implements a fallback strategy for determining MIME types when HTTP headers
are missing or provide generic values (like binary/octet-stream). It's commonly used
when processing images and documents from various sources (S3, URLs, etc.).
Fallback Strategy:
1. If current_content_type is valid (not None and not generic octet-stream), return it
2. Try to infer from URL extension (handles query parameters)
3. Try to detect from binary content signature (magic bytes)
4. Raise ValueError if all methods fail
Args:
url: The URL of the content (used to extract file extension)
content: The binary content (first ~100 bytes are sufficient for detection)
current_content_type: The current content-type from headers (may be None or generic)
Returns:
str: The inferred MIME type (e.g., "image/png", "application/pdf")
Raises:
ValueError: If content type cannot be determined by any method
Example:
>>> content_type = infer_content_type_from_url_and_content(
... url="https://s3.amazonaws.com/bucket/image.png?AWSAccessKeyId=123",
... content=png_binary_data,
... current_content_type="binary/octet-stream"
... )
>>> print(content_type)
"image/png"
"""
from litellm.litellm_core_utils.token_counter import get_image_type
media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None
# If we have a valid content type that's not generic, use it
if media_type and media_type not in [
"binary/octet-stream",
"application/octet-stream",
]:
return media_type
# Extension to MIME type mapping
# Supports images, documents, and other common file types
extension_to_mime: Final = {
# Image formats
"jpg": "image/jpeg",
"jpeg": "image/jpeg",
"png": "image/png",
"gif": "image/gif",
"webp": "image/webp",
# Document formats
"pdf": "application/pdf",
"csv": "text/csv",
"doc": "application/msword",
"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"xls": "application/vnd.ms-excel",
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"html": "text/html",
"txt": "text/plain",
"md": "text/markdown",
}
# Try to infer from URL extension
if url:
extension: Final = url.split(".")[-1].lower().split("?")[0] # Remove query params
inferred_type: Final = extension_to_mime.get(extension)
if inferred_type:
return inferred_type
# Try to detect from binary content signature (magic bytes)
if content:
detected_type: Final = get_image_type(content[:100])
if detected_type:
type_to_mime: Final = {
"png": "image/png",
"jpeg": "image/jpeg",
"gif": "image/gif",
"webp": "image/webp",
"heic": "image/heic",
}
if detected_type in type_to_mime:
return type_to_mime[detected_type]
# If all fallbacks failed, raise error
raise ValueError(f"Unable to determine content type from URL: {url}. Response content-type: {current_content_type}")
def get_file_ids_from_messages(messages: list[AllMessageValues]) -> list[str]:
"""
Gets file ids from messages
"""
file_ids: Final = []
for message in messages:
if message.get("role") == "user":
content = message.get("content")
if content:
if isinstance(content, str):
continue
for c in content:
if isinstance(c, dict) and c["type"] == "file":
file_object = cast(ChatCompletionFileObject, c)
file_object_file_field = file_object.get("file")
if not isinstance(file_object_file_field, dict):
# Content block has `type: "file"` but not the
# OpenAI Chat Completions shape. No file_id to
# extract, so skip instead of raising KeyError.
continue
file_id = file_object_file_field.get("file_id")
if file_id:
file_ids.append(file_id)
return file_ids

View file

@ -21,7 +21,9 @@ import litellm
from litellm import verbose_logger
from litellm.caching.caching import InMemoryCache
from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB, NON_LLM_CONNECTION_TIMEOUT
from litellm.litellm_core_utils.prompt_templates.common_utils import infer_content_type_from_url_and_content
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
infer_content_type_from_url_and_content,
)
from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get
from litellm.types.llms.openai import AllMessageValues

View file

@ -0,0 +1,471 @@
import re
from collections.abc import Callable, Iterator, Mapping
from graphlib import CycleError, TopologicalSorter
from itertools import chain, islice
from types import MappingProxyType
from typing import Final
from pydantic import TypeAdapter
_TOP_LEVEL_SCHEMA_COMBINATORS: Final = ("allOf", "anyOf", "oneOf")
_OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS: Final = ("enum", "const", "not")
_LOCAL_SCHEMA_REF_PREFIXES: Final = (("#/$defs/", "$defs"), ("#/definitions/", "definitions"))
_MAX_SCHEMA_FLATTEN_DEPTH: Final = 32
_EMPTY_SCHEMA: Final[Mapping[str, object]] = MappingProxyType({})
def _schema_properties(schema: Mapping[str, object]) -> Mapping[str, object]:
properties: Final = schema.get("properties")
return properties if isinstance(properties, dict) else _EMPTY_SCHEMA
def _schema_branches(schema: Mapping[str, object], combinator: str) -> tuple[object, ...]:
branches: Final = schema.get(combinator)
return tuple(branches) if isinstance(branches, list) else ()
def _schema_required_names(schema: Mapping[str, object]) -> frozenset[str]:
required: Final = schema.get("required")
if not isinstance(required, list):
return frozenset()
return frozenset(name for name in required if isinstance(name, str))
def _combinator_required_names(combinator: str, branches: tuple[Mapping[str, object], ...]) -> frozenset[str]:
branch_names: Final = tuple(_schema_required_names(branch) for branch in branches)
if not branch_names:
return frozenset()
if combinator == "allOf":
return branch_names[0].union(*branch_names[1:])
return branch_names[0].intersection(*branch_names[1:])
def _resolve_local_schema_ref(root: Mapping[str, object], ref: str) -> Mapping[str, object] | None:
matched: Final = next(
((prefix, container) for prefix, container in _LOCAL_SCHEMA_REF_PREFIXES if ref.startswith(prefix)),
None,
)
if matched is None:
return None
prefix, container = matched
definitions: Final = root.get(container)
if not isinstance(definitions, dict):
return None
target: Final = definitions.get(ref[len(prefix) :])
return target if isinstance(target, dict) else None
def _mergeable_branch(
root: Mapping[str, object],
branch: object,
seen_refs: frozenset[str],
depth: int,
expanded_refs: dict[str, Mapping[str, object] | None], # mutable-ok: per-call memo bounding repeated $ref work
) -> Mapping[str, object] | None:
if not isinstance(branch, dict) or depth > _MAX_SCHEMA_FLATTEN_DEPTH:
return None
ref: Final = branch.get("$ref")
if not isinstance(ref, str):
flattened: Final = _flatten_schema_against_root(branch, root, seen_refs, depth, expanded_refs)
if any(combinator in flattened for combinator in _TOP_LEVEL_SCHEMA_COMBINATORS):
return None
return flattened
if ref in expanded_refs:
return expanded_refs[ref]
if ref in seen_refs:
return None
target: Final = _resolve_local_schema_ref(root, ref)
expanded: Final = (
None
if target is None
else _mergeable_branch(root, target, seen_refs | frozenset((ref,)), depth + 1, expanded_refs)
)
expanded_refs[ref] = expanded
return expanded
def _is_object_schema(schema: Mapping[str, object]) -> bool:
return schema.get("type") == "object" or ("type" not in schema and "properties" in schema)
def _flatten_schema_against_root(
schema: Mapping[str, object],
root: Mapping[str, object],
seen_refs: frozenset[str],
depth: int,
expanded_refs: dict[str, Mapping[str, object] | None], # mutable-ok: per-call memo bounding repeated $ref work
) -> Mapping[str, object]:
raw_branch_groups: Final = tuple(
(
combinator,
tuple(
_mergeable_branch(root, branch, seen_refs, depth + 1, expanded_refs)
for branch in _schema_branches(schema, combinator)
),
)
for combinator in _TOP_LEVEL_SCHEMA_COMBINATORS
if isinstance(schema.get(combinator), list)
)
dropped: Final = (
*(combinator for combinator, _ in raw_branch_groups),
*(key for key in _OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS if key in schema),
)
if not dropped:
return schema
if any(branch is None for _, group in raw_branch_groups for branch in group):
return schema
branch_groups: Final = tuple(
(combinator, tuple(branch for branch in group if branch is not None)) for combinator, group in raw_branch_groups
)
branches: Final = tuple(branch for _, group in branch_groups for branch in group)
is_object_schema: Final = _is_object_schema(schema) or (
"type" not in schema and branches != () and all(_is_object_schema(branch) for branch in branches)
)
if not is_object_schema:
return schema
merged_properties: Final = {
name: value for source in (*reversed(branches), schema) for name, value in _schema_properties(source).items()
}
required_names: Final = _schema_required_names(schema).union(
*(_combinator_required_names(combinator, group) for combinator, group in branch_groups)
)
kept: Final = MappingProxyType({key: value for key, value in schema.items() if key not in dropped})
required_update: Final = MappingProxyType({"required": sorted(required_names)}) if required_names else _EMPTY_SCHEMA
return {
**kept,
"type": "object",
"properties": merged_properties,
**required_update,
}
def flatten_top_level_schema_combinators(schema: Mapping[str, object]) -> Mapping[str, object]:
"""Merge top-level ``allOf``/``anyOf``/``oneOf`` branches into an object tool schema.
OpenAI's function-calling validator rejects tool ``parameters`` carrying
'oneOf'/'anyOf'/'allOf'/'enum'/'const'/'not' at the top level (nested uses
are accepted), while lenient backends such as the ChatGPT backend Codex
talks to natively accept them, so an MCP tool declaring a top-level union
400s through LiteLLM. Branch properties merge without clobbering (the
top-level schema wins, then earlier branches); ``required`` becomes the
top-level list plus the intersection of the branch lists for anyOf/oneOf
or their union for allOf. Branches that are local ``$ref``s
(``#/$defs/...`` or ``#/definitions/...``) are resolved first, each ref
at most once per call, and branches that are themselves combinators are
flattened recursively up to a fixed depth; a branch that cannot be fully
merged (a boolean schema, an external or cyclic ``$ref``, a non-object
union, or nesting past the depth cap) leaves the whole schema untouched so
OpenAI's own validation still applies. Non-object schemas pass through
unchanged and the input is never mutated.
"""
return _flatten_schema_against_root(schema, schema, frozenset(), 0, {})
_SUBSCHEMA_KEYWORDS: Final = frozenset(
{
"additionalItems",
"additionalProperties",
"contains",
"else",
"if",
"items",
"not",
"propertyNames",
"then",
"unevaluatedItems",
"unevaluatedProperties",
}
)
_SUBSCHEMA_LIST_KEYWORDS: Final = frozenset({"allOf", "anyOf", "items", "oneOf", "prefixItems"})
_SUBSCHEMA_MAP_KEYWORDS: Final = frozenset(
{"$defs", "definitions", "dependentSchemas", "patternProperties", "properties"}
)
_MAX_SCHEMA_NESTING: Final = 1024
def drop_non_python_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]:
"""Drop every regex in a schema position that Python's ``re`` cannot compile.
OpenAI validates tool ``parameters`` against the 2020-12 metaschema with
``jsonschema``'s format checker, which hands each ``pattern`` value and each
``patternProperties`` key to ``re.compile``, so a regex written for an
ECMA-262 engine (Unicode property escapes such as ``\\p{Cc}``, as in Claude
Code's ``Artifact`` tool) is refused with "'...' is not a 'regex'" by every
model family on both the chat and Responses wires. Only schema positions are
walked (properties, items, combinators, ``$defs`` and the other applicators),
so a ``pattern`` key inside ``default``, ``examples``, ``const`` or vendor
extensions is data and stays. Outside strict mode the keyword is only a
hint, so dropping it costs the model a constraint and the caller nothing.
Compilable regexes and everything else pass through, the input is never
mutated, and the same object comes back when nothing was dropped. The walk
is level-order rather than recursive, rebuilt deepest level first, and stops
at more schema levels than a JSON parser admits, so a cyclic schema built in
code cannot spin it.
"""
return _schema_without_rejected_regex(schema, _is_not_python_regex)
def drop_lookaround_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]:
"""Drop every regex in a schema position that uses a lookaround assertion.
Some Bedrock Converse families compile tool schema regexes with an engine that
has no lookahead or lookbehind and refuse the whole request over one. The ``(?=``,
``(?!``, ``(?<=`` and ``(?<!`` openers are matched textually, so an escaped literal
that spells one is dropped too, trading a hint for a request that goes through.
A ``patternProperties`` key dropped from an object closed by ``additionalProperties:
false`` leaves its value schema as that object's ``additionalProperties``, so the
names it allowed stay allowed; :func:`drop_non_python_regex_patterns` shares the walk.
"""
return _schema_without_rejected_regex(schema, _uses_regex_lookaround)
def _schema_without_rejected_regex(
schema: Mapping[str, object], rejected: Callable[[str], bool]
) -> Mapping[str, object]:
rebuilt: dict[int, Mapping[str, object]] = {} # mutable-ok: per-call memo of rewritten nodes, deepest level first
for level in reversed(tuple(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING))):
rebuilt.update(
(id(node), rewritten)
for node in level
if (rewritten := _node_without_rejected_regex(node, rebuilt, rejected)) is not node
)
return rebuilt.get(id(schema), schema)
def _schema_levels(schema: Mapping[str, object]) -> Iterator[tuple[Mapping[str, object], ...]]:
frontier: tuple[Mapping[str, object], ...] = (schema,) # rebind-ok: level-order cursor, one level a round
while frontier:
yield frontier
frontier = tuple(child for node in frontier for child in _subschemas(node))
def schema_fragment_ref(ref: str) -> str:
return f"#{ref}" if ref.startswith(("/$defs/", "/definitions/")) else ref
def schema_has_unresolvable_ref(schema: Mapping[str, object]) -> bool:
definitions: Final = MappingProxyType(dict(_root_definitions(schema)))
if not _refs_in(schema) <= definitions.keys():
return True
try:
tuple(TopologicalSorter({pointer: _refs_in(body) for pointer, body in definitions.items()}).static_order())
except CycleError:
return True
return False
def schema_union_property_count(schema: Mapping[str, object]) -> int:
return sum(1 for _, node in _property_schemas(schema) if _is_union(node))
def schema_optional_property_count(schema: Mapping[str, object]) -> int:
return sum(len(_optional_property_names(node)) for node in _schema_nodes(schema))
def schema_property_count(schema: Mapping[str, object]) -> int:
return sum(len(properties) for node in _schema_nodes(schema) if (properties := _schema_at(node, "properties")))
def schema_grammar_branch_count(schema: Mapping[str, object]) -> int:
return sum(_grammar_branches(node) for node in _schema_nodes(schema))
def schema_has_open_object(schema: Mapping[str, object]) -> bool:
return any(_is_open_object(node) for node in _schema_nodes(schema))
def schema_has_lookaround_pattern(schema: Mapping[str, object]) -> bool:
return any(
isinstance(pattern := node.get("pattern"), str) and _uses_regex_lookaround(pattern)
for node in _schema_nodes(schema)
)
def schema_string_formats(schema: Mapping[str, object]) -> frozenset[str]:
return frozenset(
value for node in _schema_nodes(schema) if _may_be_string(node) and isinstance(value := node.get("format"), str)
)
def _may_be_string(node: Mapping[str, object]) -> bool:
node_type: Final = node.get("type")
return node_type is None or node_type == "string" or "string" in _list_at(node, "type")
def is_object_schema(node: Mapping[str, object]) -> bool:
node_type: Final = node.get("type")
return node_type == "object" or (isinstance(node_type, list) and "object" in node_type)
def _schema_nodes(schema: Mapping[str, object]) -> Iterator[Mapping[str, object]]:
return chain.from_iterable(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING))
_SCHEMA_LIST: Final = TypeAdapter(tuple[object, ...])
def _list_at(container: Mapping[str, object], key: str) -> tuple[object, ...]:
value: Final = container.get(key)
return _SCHEMA_LIST.validate_python(value) if isinstance(value, list) else ()
def _named_schemas(container: Mapping[str, object] | None) -> Iterator[tuple[str, Mapping[str, object]]]:
if container is None:
return iter(())
return ((key, value) for key in container if (value := _schema_at(container, key)) is not None)
def _root_definitions(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]:
for defs_key in ("$defs", "definitions"):
yield from ((f"#/{defs_key}/{name}", body) for name, body in _named_schemas(_schema_at(schema, defs_key)))
def _refs_in(schema: Mapping[str, object]) -> frozenset[str]:
return frozenset(
schema_fragment_ref(ref) for node in _schema_nodes(schema) if isinstance(ref := node.get("$ref"), str)
)
def _property_schemas(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]:
return chain.from_iterable(_named_schemas(_schema_at(node, "properties")) for node in _schema_nodes(schema))
def _is_union(node: Mapping[str, object]) -> bool:
return isinstance(node.get("anyOf"), list) or isinstance(node.get("oneOf"), list) or len(_list_at(node, "type")) > 1
def _grammar_branches(node: Mapping[str, object]) -> int:
properties: Final = _schema_at(node, "properties")
if properties is None:
return 0
optional: Final = len(_optional_property_names(node))
return (1 << optional) * (len(properties) - optional + 1)
def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]:
properties: Final = _schema_at(node, "properties")
if properties is None:
return frozenset()
return frozenset(properties) - frozenset(name for name in _list_at(node, "required") if isinstance(name, str))
def _is_open_object(node: Mapping[str, object]) -> bool:
additional_properties: Final = node.get("additionalProperties")
if additional_properties is True or isinstance(additional_properties, dict) or "patternProperties" in node:
return True
return is_object_schema(node) and "properties" not in node and additional_properties is not False
def _subschemas(node: Mapping[str, object]) -> Iterator[Mapping[str, object]]:
for key, value in node.items():
if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict):
yield from (sub for sub in value.values() if isinstance(sub, dict))
elif key in _SUBSCHEMA_LIST_KEYWORDS and isinstance(value, list):
yield from (sub for sub in value if isinstance(sub, dict))
elif key in _SUBSCHEMA_KEYWORDS and isinstance(value, dict):
yield value
def _node_without_rejected_regex(
node: Mapping[str, object],
rebuilt: Mapping[int, Mapping[str, object]],
rejected: Callable[[str], bool],
) -> Mapping[str, object]:
kept: Final = {
key: _keyword_value_rebuilt(key, value, rebuilt, rejected)
for key, value in node.items()
if key != "pattern" or not isinstance(value, str) or not rejected(value)
}
if len(kept) == len(node) and all(kept[key] is node[key] for key in kept):
return node
dropped_pattern_properties: Final = _dropped_pattern_properties(node, kept, rebuilt)
if not dropped_pattern_properties or kept.get("additionalProperties") is not False:
return kept
return {**kept, "additionalProperties": _any_of(dropped_pattern_properties)}
def _dropped_pattern_properties(
node: Mapping[str, object],
kept: Mapping[str, object],
rebuilt: Mapping[int, Mapping[str, object]],
) -> tuple[object, ...]:
before: Final = _schema_at(node, "patternProperties")
after: Final = _schema_at(kept, "patternProperties")
if before is None or after is None:
return ()
return tuple(rebuilt.get(id(sub), sub) for name, sub in before.items() if name not in after)
def _schema_at(container: Mapping[str, object], key: str) -> Mapping[str, object] | None:
value: Final = container.get(key)
return value if isinstance(value, dict) else None
def _any_of(schemas: tuple[object, ...]) -> object:
return schemas[0] if len(schemas) == 1 else {"anyOf": list(schemas)}
def _keyword_value_rebuilt(
key: str,
value: object,
rebuilt: Mapping[int, Mapping[str, object]],
rejected: Callable[[str], bool],
) -> object:
if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict):
kept: Final = {
name: rebuilt.get(id(sub), sub)
for name, sub in value.items()
if key != "patternProperties" or not isinstance(name, str) or not rejected(name)
}
return value if len(kept) == len(value) and all(kept[name] is value[name] for name in kept) else kept
if key in _SUBSCHEMA_LIST_KEYWORDS and isinstance(value, list):
items: Final = [rebuilt.get(id(sub), sub) for sub in value]
return value if all(new is old for new, old in zip(items, value, strict=True)) else items
if key in _SUBSCHEMA_KEYWORDS and isinstance(value, dict):
return rebuilt.get(id(value), value)
return value
def _is_not_python_regex(pattern: str) -> bool:
try:
re.compile(pattern)
except (re.error, RecursionError):
return True
return False
_REGEX_LOOKAROUND_RE: Final = re.compile(r"\(\?<?[=!]")
def _uses_regex_lookaround(pattern: str) -> bool:
return _REGEX_LOOKAROUND_RE.search(pattern) is not None
def flatten_combinators_and_drop_non_python_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]:
return flatten_top_level_schema_combinators(drop_non_python_regex_patterns(schema))
def tool_with_sanitized_parameters(
tool: Mapping[str, object],
sanitize: Callable[[Mapping[str, object]], Mapping[str, object]],
) -> Mapping[str, object]:
"""Run the tool's JSON schema through ``sanitize``: ``function.parameters`` on an
OpenAI tool, ``input_schema`` on an Anthropic one. The same object comes back when
nothing changed."""
function: Final = _schema_at(tool, "function")
if function is not None:
parameters: Final = _schema_at(function, "parameters")
if parameters is None:
return tool
sanitized_parameters: Final = sanitize(parameters)
if sanitized_parameters is parameters:
return tool
return {**tool, "function": {**function, "parameters": sanitized_parameters}}
input_schema: Final = _schema_at(tool, "input_schema")
if input_schema is None:
return tool
sanitized_schema: Final = sanitize(input_schema)
return tool if sanitized_schema is input_schema else {**tool, "input_schema": sanitized_schema}

View file

@ -0,0 +1,212 @@
from collections.abc import Mapping
from typing import Final, TypeVar, cast # noqa: TID251 # Existing casts retained unchanged in this pure move
from pydantic import TypeAdapter, ValidationError
from litellm import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.types.llms.openai import (
ChatCompletionCachedContent,
ChatCompletionPromptCacheBreakpoint,
PromptCacheBreakpoint,
)
_MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object])
PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT: Final = (
"set `drop_params: true` on the request or in litellm_settings (`litellm.drop_params = True` in the SDK)"
)
def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
"""A breakpoint the client already wrote on the block wins, TTL included."""
if marker is None or target.get("prompt_cache_breakpoint") is not None:
return target
marked: Final = {**target, "prompt_cache_breakpoint": marker}
return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key
_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint)
_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(ChatCompletionPromptCacheBreakpoint)
_AcceptedBreakpointT: Final = TypeVar("_AcceptedBreakpointT", bound=Mapping[str, object])
def _accepted_prompt_cache_breakpoint(
accepted: TypeAdapter[_AcceptedBreakpointT], marker: object
) -> _AcceptedBreakpointT | None:
try:
return accepted.validate_python(marker)
except ValidationError:
return None
def prompt_cache_breakpoint_within(
accepted: TypeAdapter[_AcceptedBreakpointT], marker: object
) -> _AcceptedBreakpointT | None:
"""The client's marker cut down to the ``accepted`` shape: an unsupported ttl is dropped and the mode kept,
and a marker whose mode the target cannot read is dropped whole."""
whole: Final = _accepted_prompt_cache_breakpoint(accepted, marker)
if whole is not None:
return whole
fields: Final = _breakpoint_fields_or_none(marker)
without_ttl: Final = (
None
if fields is None
else _accepted_prompt_cache_breakpoint(accepted, {key: item for key, item in fields.items() if key != "ttl"})
)
verbose_logger.debug(
"Dropping malformed prompt_cache_breakpoint %r under drop_params"
if without_ttl is None
else "Dropping the unsupported ttl of prompt_cache_breakpoint %r under drop_params",
marker,
)
return without_ttl
def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
if marker is None or not drop_params:
return marker
return prompt_cache_breakpoint_within(_PROMPT_CACHE_BREAKPOINT, marker)
def prompt_cache_breakpoint_with_readable_mode(marker: object, drop_params: bool) -> object:
"""The client's marker as written, or None under drop_params when no target could read its mode, so the
target's own config decides which ttl and keys it keeps."""
return None if prompt_cache_breakpoint_for_wire(marker, drop_params) is None else marker
def whole_prompt_cache_breakpoint_or_none(marker: object) -> PromptCacheBreakpoint | None:
return _accepted_prompt_cache_breakpoint(_PROMPT_CACHE_BREAKPOINT, marker)
def chat_completion_prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
if marker is None or not drop_params:
return marker
return prompt_cache_breakpoint_within(_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT, marker)
def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
"""Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block."""
if block.get("cache_control") is None:
return block
unmarked: Final = {key: value for key, value in block.items() if key != "cache_control"}
return cast( # cast-ok: same block shape as the input with the marker key renamed
_MarkedT, with_prompt_cache_breakpoint(unmarked, PromptCacheBreakpoint(mode="explicit"))
)
def prompt_cache_breakpoint_as_cache_control(block: _MarkedT, drop_params: bool) -> _MarkedT:
"""OpenAI's ``prompt_cache_breakpoint`` mark rewritten as Anthropic's ``cache_control`` on the same block.
A ``cache_control`` the client already wrote on the block wins, TTL included."""
marker: Final = block.get("prompt_cache_breakpoint")
if marker is None:
return block
unmarked: Final = {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"}
control: Final = (
None
if block.get("cache_control") is not None
else _cache_control_for_prompt_cache_breakpoint(marker, drop_params)
)
marked: Final = unmarked if control is None else {**unmarked, "cache_control": control}
return cast(_MarkedT, marked) # cast-ok: same block shape as the input with the marker key renamed
def _cache_control_for_prompt_cache_breakpoint(marker: object, drop_params: bool) -> ChatCompletionCachedContent | None:
try:
breakpoint_mark: Final = _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
except ValidationError:
if not drop_params:
raise UnsupportedParamsError(
message=(
f"prompt_cache_breakpoint must be {{'mode': 'explicit'}} with an optional ttl of '30m' or '12h', got "
f"{marker!r}. Fix the marker, or "
f"{PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it."
),
status_code=400,
)
verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
return None
ttl: Final = breakpoint_mark.get("ttl")
if ttl is not None and not drop_params:
raise UnsupportedParamsError(
message=(
f"This model's prompt cache lasts 5m, or 1h where it offers one, so prompt_cache_breakpoint "
f"ttl={ttl!r} has no equivalent. Remove the ttl, {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to send "
"a 5m breakpoint instead, or send cache_control {'type': 'ephemeral', 'ttl': '1h'} on the block "
"where the model offers a 1h cache."
),
status_code=400,
)
return ChatCompletionCachedContent(type="ephemeral")
_BREAKPOINT_BLOCK_LIST_KEYS: Final = frozenset({"content", "tool_calls"})
_BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object])
_BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object])
def _breakpoint_fields_or_none(value: object) -> Mapping[str, object] | None:
try:
return _BREAKPOINT_FIELDS.validate_python(value)
except ValidationError:
return None
def _without_prompt_cache_breakpoint(value: object) -> object:
if not isinstance(value, Mapping):
return value
fields: Final = _BREAKPOINT_FIELDS.validate_python(value)
return {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"}
def _breakpoints_stripped_from(key: str, value: object) -> object:
if key in _BREAKPOINT_BLOCK_LIST_KEYS and isinstance(value, list):
return [_without_prompt_cache_breakpoint(block) for block in _BREAKPOINT_BLOCKS.validate_python(value)]
return _without_prompt_cache_breakpoint(value) if key == "function" else value
def without_prompt_cache_breakpoints(block: _MarkedT) -> _MarkedT:
"""A message or tool definition with every ``prompt_cache_breakpoint`` removed: its own, its content parts',
its tool calls' and its ``function``'s."""
stripped: Final = {
key: _breakpoints_stripped_from(key, value) for key, value in block.items() if key != "prompt_cache_breakpoint"
}
return cast(_MarkedT, stripped) # cast-ok: same block shape as the input minus the marker keys
def _part_with_chat_completion_prompt_cache_breakpoint(part: object) -> object:
if not isinstance(part, Mapping):
return part
fields: Final = _BREAKPOINT_FIELDS.validate_python(part)
marker: Final = fields.get("prompt_cache_breakpoint")
if marker is None:
return fields
unmarked: Final = {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"}
return with_prompt_cache_breakpoint(unmarked, chat_completion_prompt_cache_breakpoint_for_wire(marker, True))
def chat_completion_prompt_cache_breakpoints_for_wire(message: _MarkedT, drop_params: bool) -> _MarkedT:
content: Final = message.get("content")
if not drop_params or not isinstance(content, list):
return message
parts: Final = [
_part_with_chat_completion_prompt_cache_breakpoint(part) for part in _BREAKPOINT_BLOCKS.validate_python(content)
]
return cast(_MarkedT, {**message, "content": parts}) # cast-ok: same message shape with its parts' markers trimmed
LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS: Final = frozenset(
{"thinking_blocks", "reasoning_items", "provider_specific_fields"}
)
LITELLM_INTERNAL_MESSAGE_FIELDS: Final = LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS | {"reasoning_content"}

View file

@ -0,0 +1,274 @@
import base64
import binascii
import re
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from itertools import groupby
from typing import Final, cast # noqa: TID251 # Existing casts retained unchanged in this pure move
from litellm.types.llms.openai import (
ChatCompletionReasoningItem,
ChatCompletionReasoningSummaryTextBlock,
ChatCompletionRedactedThinkingBlock,
ChatCompletionThinkingBlock,
)
def _provider_text_or_none(value: object) -> str | None:
return cast(str | None, value) # cast-ok: reasoning fields arrive in untyped provider messages
def extract_reasoning_content(message: Mapping[str, object]) -> tuple[str | None, str | None]:
"""
Extract reasoning content and main content from a message.
Args:
message (dict): The message dictionary that may contain reasoning_content
Returns:
tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content)
"""
message_content: Final = message.get("content")
if "reasoning_content" in message:
return _provider_text_or_none(message["reasoning_content"]), _provider_text_or_none(message_content)
elif "reasoning" in message:
return _provider_text_or_none(message["reasoning"]), _provider_text_or_none(message_content)
elif isinstance(message_content, str):
return parse_content_for_reasoning(message_content)
return None, _provider_text_or_none(message_content)
_extract_reasoning_content = extract_reasoning_content
def _readable_thinking_text(block: Mapping[str, object]) -> str:
"""The text a chat model can read back, empty for redacted blocks and malformed ones."""
if block.get("type") != "thinking":
return ""
return str(block.get("thinking") or "")
def reasoning_content_from_thinking_blocks(
thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
) -> str:
"""Flatten Anthropic thinking blocks into the `reasoning_content` string chat models expect.
Redacted blocks carry no readable text, so they contribute nothing.
"""
return "\n".join(text for block in thinking_blocks if (text := _readable_thinking_text(block)))
ENCRYPTED_REASONING_SIGNATURE_PREFIX: Final = "litellm_encrypted_reasoning:"
def encrypted_reasoning_signature(encrypted_content: str) -> str:
"""The opaque value a Responses API reasoning item's `encrypted_content` travels in.
Anthropic clients echo a thinking block's `signature` and a redacted block's `data`
back verbatim, so either field can carry the encrypted reasoning across turns; the
prefix tells the two apart from a signature Anthropic minted.
"""
return f"{ENCRYPTED_REASONING_SIGNATURE_PREFIX}{encrypted_content}"
def _carries_encrypted_reasoning(signature: object) -> bool:
return isinstance(signature, str) and signature.startswith(ENCRYPTED_REASONING_SIGNATURE_PREFIX)
def encrypted_content_from_signature(signature: object) -> str | None:
if not isinstance(signature, str) or not _carries_encrypted_reasoning(signature):
return None
return signature.removeprefix(ENCRYPTED_REASONING_SIGNATURE_PREFIX) or None
def _encrypted_reasoning_field(block: Mapping[str, object]) -> object:
match block.get("type"):
case "thinking":
return block.get("signature")
case "redacted_thinking":
return block.get("data")
case _:
return None
def encrypted_content_of_block(block: Mapping[str, object]) -> str | None:
return encrypted_content_from_signature(_encrypted_reasoning_field(block))
def is_encrypted_reasoning_block(block: object) -> bool:
"""A thinking or redacted_thinking block carrying Responses API encrypted reasoning.
Only the Responses API that minted the content can read it back, so an Anthropic
backend has to drop such a block rather than fail signature verification on it.
"""
if not isinstance(block, Mapping):
return False
mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance
return _carries_encrypted_reasoning(_encrypted_reasoning_field(mapping))
def is_unsignable_thinking_block(block: object) -> bool:
"""A thinking block Anthropic cannot accept on input.
Anthropic verifies the signature cryptographically, so a block with a null,
empty, or missing signature (e.g. from an open-source reasoning model) is
rejected with a 400, and so is a block whose signature or data carries
another provider's encrypted reasoning. A signed block is kept whatever its
text, since ``display: omitted`` returns signed blocks with empty ``thinking``.
`redacted_thinking` blocks carry no signature and are always kept.
"""
if is_encrypted_reasoning_block(block):
return True
if not isinstance(block, Mapping):
return False
mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance
if mapping.get("type") != "thinking":
return False
signature: Final = mapping.get("signature")
return not (isinstance(signature, str) and len(signature) > 0)
def has_blank_redacted_data(block: object) -> bool:
match block:
case {"type": "redacted_thinking", "data": str() as data}:
return not data.strip()
case {"type": "redacted_thinking"}:
return True
case _:
return False
def native_redacted_thinking_data(data: str) -> str:
"""Claude's own `data` string for a redacted_thinking block.
Bedrock Converse carries it base64-wrapped as `redactedContent`, and LiteLLM passed that
wrapper through as `data` until it started decoding Converse replies, so histories saved
before then still hold it. Claude's own value never decodes twice, since its first byte
decodes to a protobuf tag outside the base64 alphabet.
"""
return _converse_wrapped_payload(data) or data
def converse_redacted_content(data: str) -> str:
"""The `redactedContent` Bedrock Converse expects for a redacted_thinking block's `data`."""
return base64.b64encode(native_redacted_thinking_data(data).encode()).decode()
def _converse_wrapped_payload(data: str) -> str | None:
try:
payload: Final = base64.b64decode(data, validate=True).decode("ascii")
_ = base64.b64decode(payload, validate=True)
except (binascii.Error, ValueError):
return None
else:
return payload
def strip_encrypted_reasoning_from_messages(
messages: object,
*,
should_strip: Callable[[Mapping[str, object]], bool] | None = None,
) -> None:
"""Drop the bridge-tagged reasoning blocks a routed deployment cannot decrypt from
Anthropic-shaped history.
The whole block goes, the way #40280 drops undecryptable Responses ``input`` items: a
provider that did not mint the block rejects it signed (a foreign signature) and unsigned
(a missing signature) alike, so keeping its text as an unsigned thinking block only moves
the 400 from the router to the provider.
Mutates the content lists in place: the router's fallback snapshot shares these
message objects, so a rebound list would replay the stripped blocks on the fallback hop.
"""
if not isinstance(messages, list):
return
for content in anthropic_content_lists(cast(list[object], messages)): # cast-ok: untyped client json
_strip_encrypted_reasoning_from_blocks(content, should_strip=should_strip)
def anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]:
return (
cast(list[object], content) # cast-ok: narrowed by isinstance
for message in messages
if isinstance(message, Mapping)
for content in (cast(Mapping[str, object], message).get("content"),) # cast-ok: narrowed by isinstance
if isinstance(content, list)
)
def _strip_encrypted_reasoning_from_blocks(
content: object,
*,
should_strip: Callable[[Mapping[str, object]], bool] | None = None,
) -> None:
blocks: Final = cast(list[object], content) # cast-ok: narrowed by the caller's isinstance
kept: Final = tuple(
block
for block in blocks
if not is_encrypted_reasoning_block(block)
or (should_strip is not None and not should_strip(cast(Mapping[str, object], block)))
)
blocks[:] = kept
def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str:
index, block = indexed_block
return f"encrypted:{index}" if is_encrypted_reasoning_block(block) else "summary"
def _reasoning_item_from_block_group(group: tuple[Mapping[str, object], ...]) -> ChatCompletionReasoningItem | None:
summary: Final[list[ChatCompletionReasoningSummaryTextBlock]] = [ # mutable-ok: API message payload
ChatCompletionReasoningSummaryTextBlock(type="summary_text", text=text)
for block in group
if (text := _readable_thinking_text(block))
]
encrypted_content: Final = encrypted_content_of_block(group[0])
if encrypted_content is not None:
return ChatCompletionReasoningItem(type="reasoning", summary=summary, encrypted_content=encrypted_content)
if not summary:
return None
return ChatCompletionReasoningItem(type="reasoning", summary=summary)
def responses_reasoning_items_from_thinking_blocks(
thinking_blocks: Iterable[Mapping[str, object]],
) -> tuple[ChatCompletionReasoningItem, ...]:
"""Build Responses API `reasoning` input items from Anthropic thinking blocks.
A block carrying encrypted reasoning replays the item it came from byte for byte;
a run of plain thinking blocks collapses into one summary-only item. No item carries
an `id`: the Responses API 404s on any id it did not mint itself and rejects an empty
one, while an item without an id is always accepted.
"""
return tuple(
item
for _, group in groupby(enumerate(thinking_blocks), key=_reasoning_replay_group_key)
if (item := _reasoning_item_from_block_group(tuple(block for _, block in group))) is not None
)
def parse_content_for_reasoning(
message_text: str | None,
) -> tuple[str | None, str | None]:
"""
Parse the content for reasoning
Returns:
- reasoning_content: The content of the reasoning
- content: The content of the message
"""
if not message_text:
return None, message_text
reasoning_match: Final = re.match(
r"<(?:think|thinking|budget:thinking)>(.*?)</(?:think|thinking|budget:thinking)>(.*)",
message_text,
re.DOTALL,
)
if reasoning_match:
return reasoning_match.group(1), reasoning_match.group(2)
return None, message_text
_parse_content_for_reasoning = parse_content_for_reasoning

View file

@ -0,0 +1,252 @@
import json
from typing import Any, Final # noqa: TID251 # Existing annotations retained unchanged in this pure move
from pydantic import TypeAdapter
from litellm import verbose_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
_DECODED_JSON: Final = TypeAdapter(object)
def _attempt_json_repair(s: str) -> object | None:
"""
Attempt to repair truncated JSON produced by LLM tool calls.
Handles the most common truncation patterns where the model generates
valid JSON that is cut short (missing closing brackets/braces).
Returns the parsed value on success, or None if repair fails.
"""
import json
stripped: Final = s.rstrip()
if not stripped:
return None
# Track the stack of unmatched openers to respect nesting order
opener_stack: Final[list] = []
in_string = False
escape_next = False
for ch in stripped:
if escape_next:
escape_next = False
continue
if ch == "\\":
if in_string:
escape_next = True
continue
if ch == '"':
in_string = not in_string
continue
if in_string:
continue
if ch == "{":
opener_stack.append("}")
elif ch == "[":
opener_stack.append("]")
elif ch in ("}", "]"):
if opener_stack and opener_stack[-1] == ch:
opener_stack.pop()
if not opener_stack:
return None
# Remove trailing comma before we close brackets
candidate = stripped.rstrip(",")
# Close in reverse order of opening (respects nesting)
candidate += "".join(reversed(opener_stack))
try:
return _DECODED_JSON.validate_python(json.loads(candidate))
except json.JSONDecodeError:
pass
return None
def parse_tool_call_arguments(
arguments: str | None,
tool_name: str | None = None,
context: str | None = None,
) -> Any:
"""
Parse tool call arguments from a JSON string.
When the JSON is malformed (e.g. truncated by the model), this function
attempts a lightweight repair (closing unmatched brackets/braces) before
raising an error. A warning is logged whenever repair succeeds so that
callers are aware the arguments were not perfectly formed.
Args:
arguments: The JSON string containing tool arguments, or None.
tool_name: Optional name of the tool (for error messages).
context: Optional context string (e.g., "Anthropic Messages API").
Returns:
Parsed arguments (usually a dict, but may be any JSON-deserializable
type such as list, str, int, float, or None). Returns empty dict if
arguments is None or empty.
Raises:
ValueError: If the arguments string is not valid JSON and cannot be repaired.
"""
import json
if not arguments or not arguments.strip():
return {}
try:
return _DECODED_JSON.validate_python(json.loads(arguments))
except json.JSONDecodeError as original_error:
repaired: Final = _attempt_json_repair(arguments)
if repaired is not None:
verbose_logger.warning(
"Repaired truncated tool call arguments for tool '%s' (%s). Original (%d chars): %.200s%s",
tool_name or "<unknown>",
context or "unknown context",
len(arguments),
arguments,
"..." if len(arguments) > 200 else "",
)
return repaired
error_parts: Final = ["Failed to parse tool call arguments"]
if tool_name:
error_parts.append(f"for tool '{tool_name}'")
if context:
error_parts.append(f"({context})")
error_message: Final = " ".join(error_parts) + f". Error: {original_error}. Arguments: {arguments}"
raise ValueError(error_message) from original_error
def split_concatenated_json_objects(raw: str) -> list[dict[str, object]]:
"""
Split a string that contains one or more concatenated JSON objects into
a list of parsed dicts.
LLM providers (notably Bedrock Claude Sonnet 4.5) sometimes return
multiple tool-call argument objects concatenated in a single
``arguments`` string, e.g.::
'{"command":["curl",...]}{"command":["curl",...]}{"command":["curl",...]}'
``json.loads()`` fails on this with ``JSONDecodeError: Extra data``.
This helper uses ``json.JSONDecoder.raw_decode()`` to walk the string
and extract each JSON object individually.
The walk degrades gracefully: if the string is malformed or truncated
(e.g. a stream that ended mid-tool-call), whatever complete objects were
parsed before the bad tail are returned and the remainder is discarded
with a warning, rather than raising. The sole caller
(``_convert_to_bedrock_tool_call_invoke``) treats an empty result as
``input={}`` so the conversation can continue instead of hard-failing.
Returns
-------
list[dict]
A list of parsed dicts – one per JSON object found. If *raw* is
empty, whitespace-only, or wholly unparseable, an empty list is
returned.
"""
import json
raw = raw.strip()
if not raw:
return []
decoder: Final = json.JSONDecoder()
results: Final[list[dict[str, object]]] = []
idx = 0
length: Final = len(raw)
while idx < length:
# Skip whitespace between objects
while idx < length and raw[idx] in " \t\n\r":
idx += 1
if idx >= length:
break
try:
decoded = decoder.raw_decode(raw, idx)
except json.JSONDecodeError as e:
verbose_logger.warning(
"split_concatenated_json_objects: discarding unparseable tool-call "
"arguments tail after %d complete object(s); decode_start=%d error=%s",
len(results),
idx,
e,
)
break
obj = as_str_mapping(_DECODED_JSON.validate_python(decoded[0]))
if obj is not None:
results.append(dict(obj))
else:
# Non-dict JSON value – wrap in empty dict (Bedrock requires
# toolUse.input to be an object).
results.append({})
idx = decoded[1]
return results
MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS: Final = 8
def salvage_concatenated_tool_arguments(raw: str) -> tuple[dict[str, object], ...]:
"""Return complete concatenated JSON objects that are safe to expand.
Identical objects collapse to the first one and are not capped. More than
``MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS`` objects that are not all identical
returns an empty tuple. Anything that is not a full concatenation of JSON
objects returns an empty tuple. Repeated copies of the first object are not
retained, and once the cap is passed the rest of the string is only checked.
"""
stripped: Final = raw.strip()
if not stripped:
return ()
decoder: Final = json.JSONDecoder()
length: Final = len(stripped)
idx = 0 # rebind-ok: cursor walks the concatenated JSON string
count = 0 # rebind-ok: counts complete objects without retaining duplicates
kept = () # rebind-ok: holds at most one object past the salvage cap
exceeded = False # rebind-ok: cap already passed, the tail is only validated
while idx < length:
while idx < length and stripped[idx] in " \t\n\r":
idx += 1
if idx >= length:
break
try:
decoded = decoder.raw_decode(stripped, idx)
except json.JSONDecodeError:
return ()
decoded_object = as_str_mapping(_DECODED_JSON.validate_python(decoded[0]))
if decoded_object is None:
return ()
idx = decoded[1]
if exceeded:
continue
obj = dict(decoded_object)
count += 1
if not kept:
kept = (obj,)
continue
if obj == kept[0] and len(kept) == 1:
continue
if len(kept) == 1 and count > 2 and count - 1 > MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS:
exceeded = True
continue
if len(kept) == 1 and count > 2:
kept = (kept[0],) * (count - 1)
if len(kept) >= MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS:
exceeded = True
continue
kept = (*kept, obj)
if exceeded:
return ()
return kept

View file

@ -0,0 +1,259 @@
import base64
import re
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from itertools import chain
from typing import Final, cast # noqa: TID251 # Existing casts retained unchanged in this pure move
from litellm.litellm_core_utils.core_helpers import as_str_mapping
@dataclass(frozen=True, slots=True)
class _TPMMediaProjection:
content: object
media: tuple[Mapping[str, object], ...] = ()
def _tpm_image_part(url: str, detail: object = "auto") -> Mapping[str, object]:
return {"type": "image_url", "image_url": {"url": url, "detail": detail}}
def _tpm_blank_fields(fields: Mapping[str, object], names: tuple[str, ...]) -> Mapping[str, object]:
return {key: "" if key in names and isinstance(value, str) else value for key, value in fields.items()}
def _tpm_file_media(part: Mapping[str, object]) -> _TPMMediaProjection:
from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type
nested: Final = part.get("type") == "file"
fields: Final = as_str_mapping(part.get("file")) if nested else part
if fields is None:
return _TPMMediaProjection(part)
raw_data: Final = fields.get("file_data")
data: Final = raw_data if isinstance(raw_data, str) else ""
declared: Final = next(
(
value
for key in ("format", "mime_type", "content_type")
if isinstance(value := fields.get(key), str) and value
),
"",
)
mime: Final = (declared or parse_mime_type(data[:256]) or "").split(";", 1)[0].strip().lower()
if not mime.startswith(("image/", "audio/")):
return _TPMMediaProjection(part)
raw_url: Final = fields.get("file_url") or fields.get("file_id")
url: Final = data or (raw_url if isinstance(raw_url, str) else "")
header_end: Final = data.find(",", 0, 256)
encoded: Final = (
data[header_end + 1 :]
if header_end >= 0 and data[:header_end].endswith(";base64")
else ""
if data.startswith(("data:", "https://", "http://", "gs://"))
else data
)
media: Final = (
_tpm_image_part(
url
if not data or url.startswith(("data:", "https://", "http://", "gs://"))
else f"data:{mime};base64,{url}",
fields.get("detail", "auto"),
)
if mime.startswith("image/")
else {"type": "input_audio", "input_audio": {"data": encoded, "format": mime.removeprefix("audio/")}}
)
projected: Final = _tpm_blank_fields(fields, ("file_data", "file_url"))
return _TPMMediaProjection({**part, "file": projected} if nested else projected, (media,))
def _tpm_media_leaf(value: object) -> _TPMMediaProjection:
part: Final = as_str_mapping(value)
if part is None:
return _TPMMediaProjection(value)
kind: Final = part.get("type")
if kind in ("file", "input_file"):
return _tpm_file_media(part)
if kind == "input_audio":
audio: Final = as_str_mapping(part.get("input_audio"))
return _TPMMediaProjection(
{**part, "input_audio": _tpm_blank_fields(audio, ("data",))} if audio is not None else value,
({"type": "input_audio", "input_audio": part.get("input_audio")},),
)
if kind in ("image_url", "input_image", "image", "computer_screenshot"):
image: Final = as_str_mapping(part.get("image_url"))
source: Final = as_str_mapping(part.get("source"))
canonical: Final = _tpm_image_part(
_image_part_url({**part, "type": "input_image"} if kind == "computer_screenshot" else part) or "",
(image or part).get("detail", part.get("detail", "auto")),
)
projected: Final = (
{**part, "image_url": _tpm_blank_fields(image, ("url",))}
if image is not None and kind in ("image_url", "input_image", "computer_screenshot")
else {**part, "source": _tpm_blank_fields(source, ("data", "url"))}
if source is not None and kind == "image" and source.get("type") in ("base64", "url")
else _tpm_blank_fields(part, ("image_url", "url"))
)
return _TPMMediaProjection(projected, (canonical,))
return _TPMMediaProjection(value)
def _tpm_media_list(value: object, split: Callable[[object], _TPMMediaProjection]) -> _TPMMediaProjection:
if not isinstance(value, (list, tuple)):
return _TPMMediaProjection(value)
parts: Final = cast(Sequence[object], value) # cast-ok: finite JSON content arrays
projected: Final = tuple(map(split, parts))
media: Final = tuple(chain.from_iterable(part.media for part in projected))
content: Final = tuple(part.content for part in projected)
return _TPMMediaProjection(parts if not media else content if isinstance(value, tuple) else list(content), media)
def _tpm_replace_field(part: Mapping[str, object], key: str, projection: _TPMMediaProjection) -> _TPMMediaProjection:
return _TPMMediaProjection({**part, key: projection.content} if projection.media else part, projection.media)
def _tpm_document_media(value: object) -> _TPMMediaProjection:
part: Final = as_str_mapping(value)
source: Final = as_str_mapping(part.get("source")) if part is not None else None
if part is None or part.get("type") != "document" or source is None or source.get("type") != "content":
return _tpm_media_leaf(value)
content: Final = _tpm_media_list(source.get("content"), _tpm_media_leaf)
return _tpm_replace_field(part, "source", _tpm_replace_field(source, "content", content))
def _tpm_tool_result_media(value: object) -> _TPMMediaProjection:
part: Final = as_str_mapping(value)
if part is None or part.get("type") != "tool_result":
return _tpm_document_media(value)
return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_document_media))
def _tpm_code_output_media(value: object) -> _TPMMediaProjection:
part: Final = as_str_mapping(value)
url: Final = part.get("url") if part is not None else None
if part is None or part.get("type") != "image" or not isinstance(url, str) or not url:
return _TPMMediaProjection(value)
return _TPMMediaProjection(_tpm_blank_fields(part, ("url",)), (_tpm_image_part(url),))
def _tpm_input_media(value: object) -> _TPMMediaProjection:
part: Final = as_str_mapping(value)
if part is None:
return _TPMMediaProjection(value)
kind: Final = part.get("type")
if kind in ("function_call_output", "custom_tool_call_output"):
return _tpm_replace_field(part, "output", _tpm_media_list(part.get("output"), _tpm_tool_result_media))
if kind == "computer_call_output":
return _tpm_replace_field(part, "output", _tpm_media_leaf(part.get("output")))
if kind == "code_interpreter_call":
return _tpm_replace_field(part, "outputs", _tpm_media_list(part.get("outputs"), _tpm_code_output_media))
result: Final = part.get("result")
if kind == "image_generation_call" and isinstance(result, str) and result:
return _TPMMediaProjection(
_tpm_blank_fields(part, ("result",)),
(_tpm_image_part("data:application/octet-stream;base64," + result),),
)
if kind == "message" or (kind is None and isinstance(part.get("role"), str)):
return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_tool_result_media))
return _tpm_tool_result_media(value)
def split_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]:
projection: Final = _tpm_media_list(value, _tpm_input_media)
return projection.content, projection.media
_TPM_EMBEDDING_FILE_MIME_PREFIXES: Final = ("application/pdf", "video/")
def _tpm_embedding_file(data_uri: str) -> tuple[Mapping[str, object], ...]:
from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type
mime: Final = (parse_mime_type(data_uri[:256]) or "").strip().lower()
if not mime.startswith(_TPM_EMBEDDING_FILE_MIME_PREFIXES):
return ()
return ({"type": "file", "file": {"file_data": data_uri, "format": mime}},)
_BASE64_PAYLOAD: Final = re.compile(r"[A-Za-z0-9+/]*={0,2}")
def _is_base64_data_uri(value: str) -> bool:
header_end: Final = value.find(",", 0, 256)
if header_end < 0 or not value[:header_end].endswith(";base64"):
return False
payload_start: Final = header_end + 1
if (len(value) - payload_start) % 4 or _BASE64_PAYLOAD.fullmatch(value, payload_start) is None:
return False
last_quantum: Final = value[max(payload_start, len(value) - 4) :]
return base64.b64encode(base64.b64decode(last_quantum)).decode("ascii") == last_quantum
def _tpm_embedding_file_block(part: Mapping[str, object]) -> _TPMMediaProjection:
projection: Final = _tpm_file_media(part)
fields: Final = as_str_mapping(part.get("file"))
data: Final = fields.get("file_data") if fields is not None else None
if projection.media or fields is None or not isinstance(data, str) or not _is_base64_data_uri(data):
return projection
media: Final = _tpm_embedding_file(data)
return _TPMMediaProjection({**part, "file": _tpm_blank_fields(fields, ("file_data",))} if media else part, media)
def _tpm_embedding_leaf(value: object) -> _TPMMediaProjection:
part: Final = as_str_mapping(value)
if part is not None and part.get("type") == "file":
return _tpm_embedding_file_block(part)
if not isinstance(value, str) or not value.startswith("data:"):
return _tpm_media_leaf(value)
if not _is_base64_data_uri(value):
return _TPMMediaProjection(value)
media: Final = _tpm_file_media({"file_data": value}).media or _tpm_embedding_file(value)
return _TPMMediaProjection("" if media else value, media)
def _is_json_array(value: object) -> bool:
return isinstance(value, (list, tuple))
def _tpm_embedding_item(value: object) -> _TPMMediaProjection:
return _tpm_media_list(value, _tpm_embedding_leaf) if _is_json_array(value) else _tpm_embedding_leaf(value)
def split_embedding_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]:
projection: Final = (
_tpm_media_list(value, _tpm_embedding_item) if _is_json_array(value) else _tpm_embedding_leaf(value)
)
return projection.content, projection.media
def embedding_input_is_all_media(value: object) -> bool:
items: Final = cast(Sequence[object], value) if _is_json_array(value) else (value,) # cast-ok: JSON input array
return all(_tpm_embedding_leaf(item).media for item in items)
def anthropic_image_source_to_openai_url(image_source: Mapping[str, object]) -> str | None:
"""Data or remote URL for an Anthropic ``source`` block, in the form chat completions expects."""
source_type: Final = image_source.get("type")
if source_type == "base64":
media_type: Final = image_source.get("media_type") or "image/jpeg"
image_data: Final = image_source.get("data") or ""
return f"data:{media_type};base64,{image_data}" if image_data else None
if source_type == "url":
url: Final = image_source.get("url")
return url if isinstance(url, str) else ""
return None
def _image_part_url(part: Mapping[str, object]) -> str | None:
"""The image URL carried by one content part, whichever of the three dialects wrote it."""
part_type: Final = part.get("type")
if part_type in ("image_url", "input_image"):
image_url: Final = part.get("image_url") or part.get("url")
if isinstance(image_url, str):
return image_url
image_fields: Final = as_str_mapping(image_url)
url: Final = image_fields.get("url") if image_fields is not None else None
return url if isinstance(url, str) else None
if part_type == "image":
source: Final = part.get("source")
return anthropic_image_source_to_openai_url(source) if isinstance(source, Mapping) else None
return None

View file

@ -409,7 +409,7 @@ def deployments_the_caller_may_use(
return ()
_WEB_SEARCH_TOOL_TYPES: Final = frozenset({"web_search", "web_search_preview"})
_WEB_SEARCH_TOOL_KEYS: Final = frozenset({"googleSearch", "google_search"})
def _deployment_supports_web_search(deployment: dict) -> bool:
@ -433,11 +433,18 @@ def _deployment_supports_web_search(deployment: dict) -> bool:
def is_web_search_request(request_kwargs: Mapping[str, object]) -> bool:
"""OpenAI ``web_search`` / ``web_search_preview``, Anthropic ``web_search_<version>``, Gemini
``googleSearch`` / ``google_search``."""
tools: Final[object] = request_kwargs.get("tools")
return isinstance(tools, (list, tuple)) and any(
isinstance(tool, dict)
and isinstance(tool_type := tool.get("type"), str)
and tool_type in _WEB_SEARCH_TOOL_TYPES
and (
any(key in tool for key in _WEB_SEARCH_TOOL_KEYS)
or (
isinstance(tool_type := tool.get("type"), str)
and (tool_type == "web_search" or tool_type.startswith("web_search_"))
)
)
for tool in tools
)

View file

@ -706,7 +706,7 @@ def _data_uri_media_scope(model: str, media: Sequence[Mapping[str, object]]) ->
def split_embedding_input_tokens(
value: _EmbeddingInputT, media_models: Callable[[], Sequence[str]], fallback_model: str | None = None
) -> tuple[_EmbeddingInputT, int]:
from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import (
embedding_input_is_all_media,
split_embedding_media_for_tpm,
)

View file

@ -37,18 +37,9 @@ from litellm.litellm_core_utils.core_helpers import (
from litellm.litellm_core_utils.prompt_templates.common_utils import (
DEFS_MAX_INLINED_BYTES,
invalid_type_message,
is_object_schema,
is_present_non_string,
is_unhashable_value_behind,
sanitize_input_schema_for_anthropic,
schema_fragment_ref,
schema_grammar_branch_count,
schema_has_lookaround_pattern,
schema_has_open_object,
schema_has_unresolvable_ref,
schema_optional_property_count,
schema_string_formats,
schema_union_property_count,
unpack_defs,
)
from litellm.litellm_core_utils.prompt_templates.image_handling import (
@ -59,6 +50,17 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
inline_remote_image_urls,
is_unformatted_pdf_link_image,
)
from litellm.litellm_core_utils.prompt_templates.json_schema_utils import (
is_object_schema,
schema_fragment_ref,
schema_grammar_branch_count,
schema_has_lookaround_pattern,
schema_has_open_object,
schema_has_unresolvable_ref,
schema_optional_property_count,
schema_string_formats,
schema_union_property_count,
)
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
message_field,
parts_of,

View file

@ -29,17 +29,19 @@ from litellm.constants import (
)
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping, lax_bool
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_file_ids_from_messages,
is_encrypted_reasoning_block,
native_redacted_thinking_data,
)
from litellm.litellm_core_utils.prompt_templates.factory import THOUGHT_SIGNATURE_SEPARATOR
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
get_file_ids_from_messages,
)
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
find_anthropic_server_tool_result,
message_field,
parts_of,
)
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
is_encrypted_reasoning_block,
native_redacted_thinking_data,
)
from litellm.litellm_core_utils.reasoning_effort_utils import DroppedThinkingBudget, non_numeric_thinking_budget_error
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.wif import (

View file

@ -21,7 +21,9 @@ import httpx
from openai.types.file_deleted import FileDeleted
from pydantic import ConfigDict, TypeAdapter
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
extract_file_data,
)
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.transformation import (

View file

@ -123,15 +123,23 @@ def create_tool_name_mapping(
from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingChoice
from litellm.litellm_core_utils.prompt_templates.common_utils import (
anthropic_image_source_to_openai_url,
anthropic_inline_document,
parse_tool_call_arguments,
reasoning_content_from_thinking_blocks,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
THOUGHT_SIGNATURE_SEPARATOR,
)
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
reasoning_content_from_thinking_blocks,
)
from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import (
parse_tool_call_arguments,
)
from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import (
anthropic_image_source_to_openai_url,
)
from litellm.litellm_core_utils.reasoning_effort_utils import (
reasoning_effort_from_thinking,
reasoning_effort_from_thinking_silently,

View file

@ -20,7 +20,7 @@ from litellm.litellm_core_utils.asyncify import (
run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
cache_control_as_prompt_cache_breakpoint,
prompt_cache_breakpoint_with_readable_mode,
with_prompt_cache_breakpoint,

View file

@ -13,7 +13,7 @@ from litellm._logging import redact_internal_details_from_client_message
from litellm._uuid import uuid
from litellm.exceptions import APIError, MidStreamFallbackError
from litellm.litellm_core_utils.core_helpers import has_attributes, iter_items
from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
encrypted_reasoning_signature,
)
from litellm.llms.anthropic.messages.native.utils import (

View file

@ -16,9 +16,13 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
TOOL_RESULT_IMAGE_BOUNDARY,
TOOL_RESULT_IMAGE_PLACEHOLDER,
anthropic_inline_document,
)
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
encrypted_reasoning_signature,
responses_reasoning_items_from_thinking_blocks,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.reasoning_effort_utils import (
reasoning_effort_from_thinking,

View file

@ -14,17 +14,21 @@ from litellm.integrations.anthropic_cache_control_hook import deployment_prompt_
from litellm.litellm_core_utils.allowed_tools_tool_choice import is_allowed_tools_tool_choice
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
chat_completion_prompt_cache_breakpoints_for_wire,
drop_tool_reference_parts_from_tool_messages,
flatten_combinators_and_drop_non_python_regex_patterns,
hoist_images_from_tool_messages,
system_messages_first,
tool_with_sanitized_parameters,
without_prompt_cache_breakpoints,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_azure_openai_messages,
)
from litellm.litellm_core_utils.prompt_templates.json_schema_utils import (
flatten_combinators_and_drop_non_python_regex_patterns,
tool_with_sanitized_parameters,
)
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
chat_completion_prompt_cache_breakpoints_for_wire,
without_prompt_cache_breakpoints,
)
from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.openai.chat.gpt_reasoning_transformation import GPT_REASONING_SERIES_MARKERS

View file

@ -11,7 +11,9 @@ from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
from litellm.litellm_core_utils.prompt_templates.common_utils import prompt_cache_breakpoint_within
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
prompt_cache_breakpoint_within,
)
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config
from litellm.llms.azure.common_utils import BaseAzureLLM

View file

@ -21,6 +21,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
drop_tool_reference_parts_from_tool_messages,
filter_value_from_dict,
is_pdf_file_part,
)
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
prompt_cache_breakpoint_for_wire,
with_prompt_cache_breakpoint,
)

View file

@ -38,25 +38,27 @@ from litellm.litellm_core_utils.core_helpers import (
safe_deep_copy,
)
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.prompt_templates.common_utils import (
drop_lookaround_regex_patterns,
is_object_schema,
native_redacted_thinking_data,
parse_content_for_reasoning,
tool_with_sanitized_parameters,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
BedrockConverseMessagesProcessor,
_bedrock_converse_messages_pt,
_bedrock_tools_pt,
make_valid_bedrock_tool_name,
)
from litellm.litellm_core_utils.prompt_templates.json_schema_utils import (
drop_lookaround_regex_patterns,
is_object_schema,
tool_with_sanitized_parameters,
)
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
CONVERTED_SYSTEM_NOTE,
is_system_message,
message_field,
parts_of,
)
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
native_redacted_thinking_data,
parse_content_for_reasoning,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
from litellm.llms.anthropic.chat.transformation import (
@ -77,6 +79,7 @@ from litellm.llms.bedrock.common_utils import (
bedrock_model_supports_regex_lookaround,
bedrock_native_output_format,
bedrock_native_output_is_opt_in,
bedrock_native_output_retry_requested,
bedrock_supports_tool_search,
deployment_model_info,
requests_output_config_format,
@ -1425,6 +1428,16 @@ class AmazonConverseConfig(BaseConfig):
deployment_model_info(litellm_params),
)
def uses_native_output(
self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
) -> bool:
return bedrock_native_output_retry_requested(
bedrock_capability_model(model, litellm_params.get("base_model")),
optional_params,
self.custom_llm_provider or "bedrock_converse",
deployment_model_info(litellm_params),
)
def update_optional_params_with_thinking_tokens(self, non_default_params: dict, optional_params: dict):
"""
Handles scenario where max tokens is not specified. For anthropic models (anthropic api/bedrock/vertex ai), this requires having the max tokens being set and being greater than the thinking token budget.

View file

@ -16,7 +16,9 @@ from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping
from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
from litellm.litellm_core_utils.prompt_templates.common_utils import native_redacted_thinking_data
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
native_redacted_thinking_data,
)
from litellm.llms.anthropic.chat.handler import (
ModelResponseIterator as AnthropicModelResponseIterator,
)

View file

@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Final, cast
from httpx import Response
from litellm import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
parse_content_for_reasoning,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator

View file

@ -34,6 +34,7 @@ from litellm.llms.bedrock.common_utils import (
bedrock_capability_model,
bedrock_chat_uses_native_structured_output,
bedrock_cleaned_output_schema,
bedrock_native_output_retry_requested,
bedrock_supports_tool_search,
deployment_model_info,
get_anthropic_beta_from_headers,
@ -161,6 +162,13 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
) -> bool:
return requests_opt_in_native_output(model, optional_params, deployment_model_info(litellm_params))
def uses_native_output(
self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
) -> bool:
return bedrock_native_output_retry_requested(
model, optional_params, model_info=deployment_model_info(litellm_params)
)
def map_openai_params(
self,
non_default_params: dict,

View file

@ -1,4 +1,5 @@
from collections.abc import Callable, Coroutine
from dataclasses import dataclass
from functools import partial
from typing import Final, TypeAlias
@ -6,6 +7,7 @@ from litellm.llms.anthropic.common_utils import is_claude_strict_tools_rejection
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.bedrock.common_utils import (
NativeOutputFallback,
bedrock_strict_tools_fallback,
is_bedrock_native_output_rejection,
warn_bedrock_native_output_rejected,
warn_bedrock_strict_tools_dropped,
@ -17,7 +19,15 @@ NativeOutputResponse: TypeAlias = ModelResponse | CustomStreamWrapper
NativeOutputAttemptResult: TypeAlias = NativeOutputResponse | Coroutine[object, object, NativeOutputResponse]
NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeOutputAttemptResult]
NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None]
_RetryFor: TypeAlias = Callable[[BaseLLMException], NativeOutputFallback | None]
@dataclass(frozen=True, slots=True)
class _SelectedFallback:
fallback: NativeOutputFallback
native_output_retry: bool
_RetryFor: TypeAlias = Callable[[BaseLLMException], _SelectedFallback | None]
def _native_output_retry(
@ -47,17 +57,42 @@ def _fallback_for(
fallback: NativeOutputFallbackFactory | None,
strict_tools_fallback: NativeOutputFallbackFactory | None,
model: str,
) -> NativeOutputFallback | None:
) -> _SelectedFallback | None:
native_output_retry: Final = _native_output_retry(error, fallback, model)
if native_output_retry is not None:
return native_output_retry
return _strict_tools_retry(error, strict_tools_fallback, model)
return _SelectedFallback(native_output_retry, native_output_retry=True)
strict_tools_retry: Final = _strict_tools_retry(error, strict_tools_fallback, model)
if strict_tools_retry is None:
return None
return _SelectedFallback(strict_tools_retry, native_output_retry=False)
def _strict_tools_retry_after_native_output_retry(
error: BaseLLMException,
retry: _SelectedFallback,
strict_tools_fallback: NativeOutputFallbackFactory | None,
model: str,
) -> NativeOutputFallback | None:
if (
not retry.native_output_retry
or strict_tools_fallback is None
or not is_claude_strict_tools_rejection(error.status_code, error.message)
):
return None
composed_retry: Final = bedrock_strict_tools_fallback(
model, retry.fallback.optional_params, retry.fallback.litellm_params
)
if composed_retry is not None:
warn_bedrock_strict_tools_dropped(model)
return composed_retry
async def _await_with_native_output_fallback(
first_attempt: Coroutine[object, object, NativeOutputResponse],
attempt: NativeOutputAttempt,
retry_for: _RetryFor,
strict_tools_fallback: NativeOutputFallbackFactory | None,
model: str,
) -> NativeOutputResponse:
try:
return await first_attempt
@ -65,8 +100,17 @@ async def _await_with_native_output_fallback(
retry: Final = retry_for(error)
if retry is None:
raise
second_attempt: Final = attempt(retry)
return await second_attempt if isinstance(second_attempt, Coroutine) else second_attempt
try:
second_attempt: Final = attempt(retry.fallback)
return await second_attempt if isinstance(second_attempt, Coroutine) else second_attempt
except BaseLLMException as error:
composed_retry: Final = _strict_tools_retry_after_native_output_retry(
error, retry, strict_tools_fallback, model
)
if composed_retry is None:
raise
third_attempt: Final = attempt(composed_retry)
return await third_attempt if isinstance(third_attempt, Coroutine) else third_attempt
def call_with_native_output_fallback(
@ -86,7 +130,15 @@ def call_with_native_output_fallback(
retry: Final = retry_for(error)
if retry is None:
raise
return attempt(retry)
try:
return attempt(retry.fallback)
except BaseLLMException as retry_error:
composed_retry: Final = _strict_tools_retry_after_native_output_retry(
retry_error, retry, strict_tools_fallback, model
)
if composed_retry is None:
raise
return attempt(composed_retry)
if isinstance(first_attempt, Coroutine):
return _await_with_native_output_fallback(first_attempt, attempt, retry_for)
return _await_with_native_output_fallback(first_attempt, attempt, retry_for, strict_tools_fallback, model)
return first_attempt

View file

@ -36,7 +36,9 @@ from litellm import verbose_logger
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.core_helpers import decoded_json_object, lax_bool, require_attributes
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count
from litellm.litellm_core_utils.prompt_templates.json_schema_utils import (
schema_property_count,
)
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.base_llm.anthropic_messages.transformation import (
@ -293,6 +295,15 @@ def _bedrock_model_supports(model: str, key: str, custom_llm_provider: str = "be
return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key=key)
def _bedrock_native_output_enabled(
model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None
) -> bool:
deployment_flag: Final = (model_info or {}).get("supports_native_structured_output")
if isinstance(deployment_flag, bool):
return deployment_flag
return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider)
_DEPLOYMENT_MODEL_INFO: Final = TypeAdapter(dict[str, object])
@ -313,9 +324,9 @@ def _bedrock_native_output_opt_in(
def bedrock_supports_native_structured_output(
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
) -> bool:
return _bedrock_model_supports(
model, "supports_native_structured_output", custom_llm_provider
) or _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
return _bedrock_native_output_enabled(model, custom_llm_provider, model_info) or _bedrock_native_output_opt_in(
model, custom_llm_provider, model_info
)
def _bedrock_map_native_output_flag(model: str) -> object:
@ -350,7 +361,7 @@ def bedrock_chat_opts_into_native_structured_output(
custom_llm_provider: str = "bedrock",
model_info: Mapping[str, object] | None = None,
) -> bool:
return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) or (
return _bedrock_native_output_enabled(model, custom_llm_provider, model_info) or (
thinking_requested and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
)
@ -371,15 +382,15 @@ def bedrock_chat_uses_native_structured_output(
def bedrock_native_output_is_deployment_opt_in(
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
) -> bool:
return not _bedrock_model_supports(
model, "supports_native_structured_output", custom_llm_provider
) and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
return not _bedrock_native_output_enabled(model, custom_llm_provider, model_info) and _bedrock_native_output_opt_in(
model, custom_llm_provider, model_info
)
def bedrock_native_output_is_opt_in(
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
) -> bool:
return not _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) and (
return not _bedrock_native_output_enabled(model, custom_llm_provider, model_info) and (
_bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
or _bedrock_native_output_replaces_unforceable_tool(model, model_info)
)
@ -388,7 +399,21 @@ def bedrock_native_output_is_opt_in(
def without_bedrock_native_output_opt_in(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
return {
**litellm_params,
"model_info": {**deployment_model_info(litellm_params), "supports_bedrock_native_structured_output": False},
"model_info": {
**deployment_model_info(litellm_params),
"supports_bedrock_native_structured_output": False,
},
}
def for_bedrock_json_tool_retry(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
opt_in_disabled: Final = without_bedrock_native_output_opt_in(litellm_params)
return {
**opt_in_disabled,
"model_info": {
**deployment_model_info(opt_in_disabled),
"supports_native_structured_output": False,
},
}
@ -396,6 +421,7 @@ _BEDROCK_NATIVE_OUTPUT_UNSUPPORTED: Final = "output_config.format: Extra inputs
_BEDROCK_NATIVE_OUTPUT_REJECTIONS: Final = (
_BEDROCK_NATIVE_OUTPUT_UNSUPPORTED,
"compilation exceeded limits: Schema is too complex",
"The compiled grammar is too large",
"output_config.format.schema:",
)
@ -476,8 +502,27 @@ def requests_output_config_format(optional_params: Mapping[str, object]) -> bool
def requests_native_output(anthropic_optional_params: Mapping[str, object]) -> bool:
return isinstance(anthropic_optional_params.get("output_format"), Mapping) or requests_output_config_format(
anthropic_optional_params
output_config: Final = json_object_or_none(anthropic_optional_params.get("outputConfig"))
return (
isinstance(anthropic_optional_params.get("output_format"), Mapping)
or requests_output_config_format(anthropic_optional_params)
or (
anthropic_optional_params.get("json_mode") is True
and output_config is not None
and isinstance(output_config.get("textFormat"), Mapping)
)
)
def bedrock_native_output_retry_requested(
model: str,
optional_params: Mapping[str, object],
custom_llm_provider: str = "bedrock",
model_info: Mapping[str, object] | None = None,
) -> bool:
return requests_native_output(optional_params) and (
bedrock_native_output_is_opt_in(model, custom_llm_provider, model_info)
or _bedrock_native_output_enabled(model, custom_llm_provider, model_info)
)
@ -519,7 +564,7 @@ def bedrock_native_output_schema(
model_info: Mapping[str, object] | None = None,
) -> Mapping[str, object] | None:
native_schema: Final = bedrock_cleaned_output_schema(model, schema, custom_llm_provider)
if _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider):
if _bedrock_native_output_enabled(model, custom_llm_provider, model_info):
return schema if native_schema is None else native_schema
if _bedrock_native_output_opt_in(model, custom_llm_provider, model_info):
return native_schema

View file

@ -41,9 +41,11 @@ from litellm.litellm_core_utils.cloud_storage_security import (
)
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_file_data,
text_completion_prompt_to_messages,
)
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
extract_file_data,
)
from litellm.llms.anthropic.common_utils import AnthropicError
from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -237,15 +239,6 @@ class _S3RequestTarget:
request_params: _BedrockS3RequestParams
class _TrustedS3ModelCredentials(LiteLLMBaseModel):
"""The S3 buckets the server trusts file ids against, from the deployment snapshot."""
model_config = ConfigDict(extra="ignore")
s3_bucket_name: str | None = None
s3_output_bucket_name: str | None = None
def extract_s3_uri_from_file_id(file_id: str) -> str:
"""
Resolve a Bedrock file id to its S3 URI.
@ -269,27 +262,31 @@ def extract_s3_uri_from_file_id(file_id: str) -> str:
raise ValueError("file_id must be a managed LiteLLM S3 file id")
_S3_BUCKET_REQUIRED_ERROR: Final = "S3 bucket_name is required. Set 's3_bucket_name' in proxy config or AWS_S3_BUCKET_NAME for Bedrock file content retrieval."
_S3_BUCKET_REQUIRED_ERROR: Final = (
"S3 bucket_name is required. Pass 's3_bucket_name', set it in proxy config, "
"or set AWS_S3_BUCKET_NAME for Bedrock file content retrieval."
)
def _trusted_s3_model_credentials(litellm_params: Mapping[str, object]) -> _TrustedS3ModelCredentials:
trusted_model_credentials: Final = litellm_params.get("_litellm_internal_model_credentials")
if not isinstance(trusted_model_credentials, MappingProxyType):
return _TrustedS3ModelCredentials()
snapshot: Final[dict[str, object]] = {}
snapshot.update(trusted_model_credentials) # any-ok: untyped snapshot
return _TrustedS3ModelCredentials.model_validate(snapshot)
def _s3_param(litellm_params: Mapping[str, object], key: str, env_var: str) -> str | None:
value: Final = litellm_params.get(key)
return value if isinstance(value, str) and value else os.getenv(env_var)
def _input_bucket_name(litellm_params: Mapping[str, object]) -> str | None:
return _s3_param(litellm_params, "s3_bucket_name", "AWS_S3_BUCKET_NAME")
def _output_bucket_name(litellm_params: Mapping[str, object]) -> str | None:
return _s3_param(litellm_params, "s3_output_bucket_name", "AWS_S3_OUTPUT_BUCKET_NAME")
def get_configured_s3_bucket_name(litellm_params: Mapping[str, object]) -> str:
"""
Resolve the server-configured S3 bucket for Bedrock file operations.
Only trusts the immutable server-side credential snapshot or the
environment; never a request-supplied param, since the bucket is what
`validate_managed_cloud_file_id` checks file ids against.
Resolve the S3 bucket for Bedrock file operations: the call's ``s3_bucket_name``,
else ``AWS_S3_BUCKET_NAME``. File ids are validated against this bucket.
"""
bucket_name: Final = _trusted_s3_model_credentials(litellm_params).s3_bucket_name or os.getenv("AWS_S3_BUCKET_NAME")
bucket_name: Final = _input_bucket_name(litellm_params)
if not bucket_name:
raise ValueError(_S3_BUCKET_REQUIRED_ERROR)
return bucket_name
@ -297,16 +294,13 @@ def get_configured_s3_bucket_name(litellm_params: Mapping[str, object]) -> str:
def get_configured_s3_bucket_names(litellm_params: Mapping[str, object]) -> tuple[str, ...]:
"""
Resolve the server-configured S3 buckets a Bedrock file id may live in.
Resolve the S3 buckets a Bedrock file id may live in.
Bedrock batch outputs land in ``s3_output_bucket_name`` when it differs from
the input bucket, so retrieval validates against both. Same trust rules as
``get_configured_s3_bucket_name``: only the immutable credential snapshot or
the environment, never a request param.
the input bucket, so retrieval validates against both.
"""
trusted: Final = _trusted_s3_model_credentials(litellm_params)
input_bucket: Final = trusted.s3_bucket_name or os.getenv("AWS_S3_BUCKET_NAME")
output_bucket: Final = trusted.s3_output_bucket_name or os.getenv("AWS_S3_OUTPUT_BUCKET_NAME")
input_bucket: Final = _input_bucket_name(litellm_params)
output_bucket: Final = _output_bucket_name(litellm_params)
buckets: Final = tuple(dict.fromkeys(bucket for bucket in (input_bucket, output_bucket) if bucket))
if not buckets:
raise ValueError(_S3_BUCKET_REQUIRED_ERROR)
@ -522,12 +516,7 @@ def _output_location_still_unlisted(litellm_params: Mapping[str, object]) -> boo
def _listing_bucket_name(litellm_params: Mapping[str, object], purpose: str | None) -> str:
if purpose != "batch_output":
return get_configured_s3_bucket_name(litellm_params)
trusted: Final = _trusted_s3_model_credentials(litellm_params)
return (
trusted.s3_output_bucket_name
or os.getenv("AWS_S3_OUTPUT_BUCKET_NAME")
or get_configured_s3_bucket_name(litellm_params)
)
return _output_bucket_name(litellm_params) or get_configured_s3_bucket_name(litellm_params)
def _listed_object_created_at(entry: ET.Element) -> int:

View file

@ -40,15 +40,17 @@ from litellm.llms.bedrock.common_utils import (
BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS,
BedrockError,
apply_bedrock_invoke_structured_output,
bedrock_native_output_is_deployment_opt_in,
bedrock_native_output_format,
bedrock_native_output_retry_requested,
bedrock_supports_tool_search,
ensure_bedrock_anthropic_messages_tool_names,
for_bedrock_json_tool_retry,
get_anthropic_beta_from_headers,
is_bedrock_native_output_rejection,
json_object_or_none,
normalize_bedrock_opus_output_config_effort,
normalize_custom_field_on_tools,
normalize_tool_input_schema_types_for_bedrock_invoke,
requests_native_output,
requests_output_config_format,
strip_unsupported_bedrock_invoke_output_config_keys,
supports_bedrock_prompt_cache_ttl,
@ -133,13 +135,24 @@ class AmazonAnthropicClaudeMessagesConfig(
) -> GenericLiteLLMParams | None:
model_info: Final = _MODEL_INFO.validate_python(litellm_params.model_info)
if not (
requests_native_output(anthropic_messages_optional_request_params)
and bedrock_native_output_is_deployment_opt_in(model, model_info=model_info)
bedrock_native_output_retry_requested(
model, anthropic_messages_optional_request_params, model_info=model_info
)
):
return None
return litellm_params.model_copy(
update={"model_info": {**(model_info or {}), "supports_bedrock_native_structured_output": False}}
legacy_output_format: Final = json_object_or_none(
anthropic_messages_optional_request_params.get("output_format")
)
output_config: Final = json_object_or_none(anthropic_messages_optional_request_params.get("output_config"))
output_config_format: Final = (
json_object_or_none(output_config.get("format")) if output_config is not None else None
)
output_format: Final = legacy_output_format if legacy_output_format is not None else output_config_format
if output_format is None or (bedrock_native_output_format(model, output_format, model_info=model_info) is None):
return None
retry_params: Final = for_bedrock_json_tool_retry({"model_info": model_info})
retry_model_info: Final = _MODEL_INFO.validate_python(retry_params.get("model_info"))
return litellm_params.model_copy(update={"model_info": retry_model_info})
def should_retry_anthropic_messages_without_native_output(
self, e: httpx.HTTPStatusError, request_body: Mapping[str, object], model: str

View file

@ -24,7 +24,9 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.prompt_templates.common_utils import chat_completion_prompt_cache_breakpoint_for_wire
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
chat_completion_prompt_cache_breakpoint_for_wire,
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items

View file

@ -210,7 +210,7 @@ def _prepare_multipart_file_upload(
Returns:
Tuple of (files_dict, headers_without_content_type)
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
extract_file_data,
)

View file

@ -19,12 +19,14 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
should_convert_tool_call_to_json_mode,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_reasoning_content,
merge_consecutive_system_messages,
strip_litellm_internal_message_fields,
strip_name_from_message,
strip_prefill_prefix_field,
)
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
extract_reasoning_content,
)
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator

View file

@ -158,7 +158,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
return response
except Exception as e:
raise ValueError(f"Error transforming Deepgram response: {e}\nResponse: {raw_response.text}")
raise ValueError(f"Error transforming Deepgram response: {e}") from None
def _reconstruct_diarized_transcript(self, words: list) -> str:
"""

View file

@ -160,7 +160,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
return response
except Exception as e:
raise ValueError(f"Error transforming ElevenLabs response: {e}\nResponse: {raw_response.text}")
raise ValueError(f"Error transforming ElevenLabs response: {e}") from None
def get_complete_url(
self,

View file

@ -138,13 +138,13 @@ class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
except ValueError:
raise GeminiError(
status_code=raw_response.status_code,
message=f"Received non-JSON response from Gemini Interactions API: {raw_response.text}",
message="Received non-JSON response from Gemini Interactions API",
)
parsed: Final = GeminiTranscriptionInteractionResponse.model_validate(response_json)
if parsed.status != "completed":
raise GeminiError(
status_code=raw_response.status_code,
message=f"Gemini transcription interaction did not complete (status={parsed.status}): {raw_response.text}",
message=f"Gemini transcription interaction did not complete (status={parsed.status})",
)
text_contents: Final = tuple(
content

View file

@ -14,7 +14,9 @@ from openai.types.file_deleted import FileDeleted
from typing_extensions import ReadOnly, Required
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
extract_file_data,
)
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.base_llm.files.transformation import (
BaseFilesConfig,

View file

@ -139,6 +139,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
"response_format",
"response_mime_type",
"previous_interaction_id",
"continuation_token",
]
def validate_environment(
@ -219,6 +220,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
"environment",
"response_modalities",
"previous_interaction_id",
"continuation_token",
]
dropped_keys: Final = (
frozenset({"agent_config"})
@ -314,7 +316,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
) -> InteractionsAPIStreamingResponse:
"""Parse streaming chunk."""
verbose_logger.debug("Google AI Interactions streaming chunk: %s", parsed_chunk)
return InteractionsAPIStreamingResponse(**parsed_chunk)
return InteractionsAPIStreamingResponse.model_validate(parsed_chunk)
# GET / DELETE / CANCEL - just build URLs, responses match spec directly

View file

@ -7,10 +7,10 @@ import json
from collections.abc import Coroutine
from typing import Final, Literal, cast, overload
from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
get_image_mime_type_from_url,
)
from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type
from litellm.litellm_core_utils.reasoning_effort_utils import (
reasoning_effort_from_thinking,
)

View file

@ -20,7 +20,9 @@ from pydantic import ConfigDict, TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
extract_file_data,
)
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.transformation import (

View file

@ -133,7 +133,7 @@ class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
response_json: Final = raw_response.json()
except Exception:
raise MistralAudioTranscriptionException(
message=raw_response.text,
message="Received non-JSON response from Mistral audio transcription",
status_code=raw_response.status_code,
headers=raw_response.headers,
)

View file

@ -17,7 +17,9 @@ from openai.types.file_deleted import FileDeleted
from pydantic import ConfigDict
from typing_extensions import ReadOnly, TypedDict
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
extract_file_data,
)
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.transformation import BaseFilesConfig, LiteLLMLoggingObj

View file

@ -12,6 +12,8 @@ from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
extract_images_from_message,
)
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
extract_reasoning_content,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
@ -353,7 +355,7 @@ class OllamaChatConfig(BaseConfig):
del response_json_message["thinking"]
elif response_json_message.get("content") is not None:
# parse reasoning content from content
from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
parse_content_for_reasoning,
)

View file

@ -75,7 +75,7 @@ class _OllamaGenerateReasoning(LiteLLMBaseModel):
def split(self) -> tuple[str | None, str | None]:
"""Reasoning reaches `/api/generate` either in the top-level `thinking` field or
inline in `<think>` tags, never both. The field wins, matching `ollama_chat`."""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
parse_content_for_reasoning,
)

View file

@ -21,20 +21,24 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
should_convert_tool_call_to_json_mode,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import (
chat_completion_prompt_cache_breakpoints_for_wire,
drop_non_python_regex_patterns,
drop_tool_reference_parts_from_tool_messages,
flatten_combinators_and_drop_non_python_regex_patterns,
get_tool_call_names,
hoist_images_from_tool_messages,
system_messages_first,
tool_with_sanitized_parameters,
without_prompt_cache_breakpoints,
)
from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_convert_url_to_base64,
convert_url_to_base64,
)
from litellm.litellm_core_utils.prompt_templates.json_schema_utils import (
drop_non_python_regex_patterns,
flatten_combinators_and_drop_non_python_regex_patterns,
tool_with_sanitized_parameters,
)
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
chat_completion_prompt_cache_breakpoints_for_wire,
without_prompt_cache_breakpoints,
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.base_utils import (

View file

@ -21,10 +21,12 @@ from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
safe_convert_created_field,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import (
cache_control_as_prompt_cache_breakpoint,
from litellm.litellm_core_utils.prompt_templates.json_schema_utils import (
drop_non_python_regex_patterns,
flatten_combinators_and_drop_non_python_regex_patterns,
)
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
cache_control_as_prompt_cache_breakpoint,
prompt_cache_breakpoint_for_wire,
whole_prompt_cache_breakpoint_or_none,
with_prompt_cache_breakpoint,

View file

@ -141,7 +141,4 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
if any(key in raw_response_json for key in TranscriptionResponse.model_fields):
return TranscriptionResponse(**raw_response_json)
else:
raise ValueError(
"Invalid response format. Received response does not match the expected format. Got: ",
raw_response_json,
)
raise ValueError("Invalid response format. Received response does not match the expected format.")

View file

@ -144,7 +144,7 @@ class OVHCloudAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
response_json: Final = raw_response.json()
except Exception:
raise OVHCloudException(
message=raw_response.text,
message="Received non-JSON response from OVHCloud audio transcription",
status_code=raw_response.status_code,
headers=raw_response.headers,
)

View file

@ -134,7 +134,7 @@ class ScalewayAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
response_json: Final = raw_response.json()
except Exception:
raise ScalewayAudioTranscriptionException(
message=raw_response.text,
message="Received non-JSON response from Scaleway audio transcription",
status_code=raw_response.status_code,
headers=raw_response.headers,
)

View file

@ -183,7 +183,7 @@ class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexB
except ValueError:
raise VertexAIError(
status_code=raw_response.status_code,
message=f"Received non-JSON response from Vertex AI Gemini transcription: {raw_response.text}",
message="Received non-JSON response from Vertex AI Gemini transcription",
)
parsed: Final = VertexGeminiTranscriptionResponse.model_validate(response_json)
texts: Final = tuple(

View file

@ -172,7 +172,7 @@ class VertexAIAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase)
except ValueError:
raise VertexAIError(
status_code=raw_response.status_code,
message=f"Received non-JSON response from Google Speech-to-Text: {raw_response.text}",
message="Received non-JSON response from Google Speech-to-Text",
)
parsed: Final = VertexSpeechToTextRecognizeResponse.model_validate(response_json)
transcripts: Final = tuple(

View file

@ -35,7 +35,7 @@ from litellm.litellm_core_utils.cloud_storage_security import (
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, iter_items
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
extract_file_data,
extract_file_metadata,
)

View file

@ -20,9 +20,6 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, has_attributes
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_image_mime_type_from_url,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
GeminiMediaSource,
convert_to_anthropic_image_obj,
@ -33,6 +30,9 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
get_thought_signature_from_tool,
response_schema_prompt,
)
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
get_image_mime_type_from_url,
)
from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia, async_inline_remote_media
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels

View file

@ -175,7 +175,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran
try:
raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json())
except Exception as e:
raise ValueError(f"Error transforming response to json: {e}\nResponse: {raw_response.text}")
raise ValueError(f"Error transforming response to json: {e}")
# Extract only valid fields for TranscriptionResponse.__init__()
# TranscriptionResponse only accepts 'text' and 'usage' in __init__()
@ -191,10 +191,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran
response_kwargs["usage"] = usage
if not response_kwargs:
raise ValueError(
"Invalid response format. Received response does not match the expected format. Got: ",
response_object,
)
raise ValueError("Invalid response format. Received response does not match the expected format.")
response: Final = TranscriptionResponse(**response_kwargs)

View file

@ -39,7 +39,7 @@ class _XAISttWord(LiteLLMBaseModel):
class _XAISttResponse(LiteLLMBaseModel):
model_config = ConfigDict(extra="allow")
model_config = ConfigDict(extra="allow", hide_input_in_errors=True)
text: str = ""
language: str = "unknown"
duration: float | None = None

View file

@ -13,7 +13,9 @@ from openai.types.file_deleted import FileDeleted
from pydantic import ConfigDict
from typing_extensions import ReadOnly, TypedDict
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
extract_file_data,
)
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.transformation import BaseFilesConfig, LiteLLMLoggingObj

View file

@ -144,7 +144,7 @@ from litellm.llms.bedrock.common_utils import (
bedrock_route_for_request,
bedrock_strict_tools_fallback,
deployment_model_info,
without_bedrock_native_output_opt_in,
for_bedrock_json_tool_retry,
without_bedrock_route_prefix,
)
from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config
@ -4374,7 +4374,7 @@ def _native_output_fallback(
first_attempt_params: Mapping[str, object],
first_attempt_litellm_params: Mapping[str, object],
) -> NativeOutputFallback:
litellm_params: Final = without_bedrock_native_output_opt_in(first_attempt_litellm_params)
litellm_params: Final = for_bedrock_json_tool_retry(first_attempt_litellm_params)
remapped_params: Final = ctx.optional_params_for_model_info(deployment_model_info(litellm_params))
return NativeOutputFallback(
optional_params={
@ -4505,7 +4505,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu
converse_attempt,
(
partial(_converse_native_output_fallback, ctx, converse_params, dict(litellm_params))
if AmazonConverseConfig().uses_opt_in_native_output(converse_model, converse_params, litellm_params)
if AmazonConverseConfig().uses_native_output(converse_model, converse_params, litellm_params)
else None
),
converse_model,
@ -4561,7 +4561,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu
(
partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params))
if isinstance(provider_config, AmazonAnthropicClaudeConfig)
and provider_config.uses_opt_in_native_output(model, invoke_params, litellm_params)
and provider_config.uses_native_output(model, invoke_params, litellm_params)
else None
),
model,

View file

@ -2242,7 +2242,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
@ -2331,7 +2331,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
@ -2420,7 +2420,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
@ -2504,7 +2504,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
@ -2593,7 +2593,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
@ -2715,7 +2715,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_native_structured_output": true,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
@ -2759,7 +2759,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_native_structured_output": true,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
@ -2803,7 +2803,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_native_structured_output": true,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
@ -2847,7 +2847,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_native_structured_output": true,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
@ -2891,7 +2891,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_native_structured_output": true,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
@ -4046,6 +4046,8 @@
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 2.5e-07,
"deprecation_date": "2027-09-01",
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
"litellm_provider": "azure_ai",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
@ -4068,7 +4070,7 @@
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": false,
"prompt_cache_min_tokens": 512,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models"
"source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement"
},
"azure_ai/claude-opus-5": {
"deprecation_date": "2027-07-08",
@ -13458,13 +13460,16 @@
"supports_reasoning": true
},
"azure_ai/mistral-document-ai-2512": {
"deprecation_date": "2027-02-15",
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
"litellm_provider": "azure_ai",
"ocr_cost_per_page": 0.003,
"mode": "ocr",
"supported_endpoints": [
"/v1/ocr"
],
"source": "https://ai.azure.com/catalog/models/mistral-document-ai-2512"
"source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
"successor_model": "azure_ai/mistral-ocr-4-0"
},
"azure_ai/mistral-ocr-4-0": {
"litellm_provider": "azure_ai",
@ -75499,7 +75504,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_native_structured_output": true,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
@ -75539,7 +75544,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
@ -76502,6 +76507,33 @@
"supports_tool_choice": true,
"supports_vision": true
},
"us-gov.xai.grok-4.7": {
"supports_regex_lookaround": false,
"supported_endpoints": ["/v1/chat/completions"],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"cache_read_input_token_cost": 6.6e-07,
"cache_read_input_token_cost_flex": 3.3e-07,
"cache_read_input_token_cost_priority": 1.155e-06,
"input_cost_per_token": 2.64e-06,
"input_cost_per_token_batches": 1.32e-06,
"input_cost_per_token_flex": 1.32e-06,
"input_cost_per_token_priority": 4.62e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 524288,
"max_output_tokens": 524288,
"mode": "chat",
"output_cost_per_token": 7.92e-06,
"output_cost_per_token_batches": 3.96e-06,
"output_cost_per_token_flex": 3.96e-06,
"output_cost_per_token_priority": 1.386e-05,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json",
"supports_function_calling": true,
"supports_prompt_caching": false,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"baseten/deepseek-ai/DeepSeek-V4.1-Flash-Fast": {
"cache_read_input_token_cost": 1.4e-08,
"input_cost_per_token": 6e-07,
@ -78470,7 +78502,7 @@
"cache_creation_input_token_cost": 1e-05,
"cache_creation_input_token_cost_above_1hr": 1.6e-05,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-10-01",
"deprecation_date": "2027-01-15",
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
"input_cost_per_token": 8e-06,
"litellm_provider": "azure_ai",
@ -78480,7 +78512,7 @@
"output_cost_per_token": 4e-05,
"prompt_cache_min_tokens": 512.0,
"search_context_cost_per_query": {"search_context_size_high":0.02,"search_context_size_low":0.02,"search_context_size_medium":0.02},
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models",
"source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -86471,5 +86503,32 @@
"supports_system_messages": false,
"supports_video_input": true,
"supports_vision": true
},
"databricks/databricks-kimi-k3-1-pro": {
"cache_creation_input_token_cost": 1.99997e-06,
"cache_read_input_token_cost": 1.9999e-07,
"input_cost_per_token": 1.99997e-06,
"input_dbu_cost_per_token": 2.8571e-05,
"litellm_provider": "databricks",
"max_input_tokens": 1048576,
"max_output_tokens": 1048576,
"metadata": {"notes":"Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference."},
"mode": "chat",
"output_cost_per_token": 9.99999e-06,
"output_dbu_cost_per_token": 0.000142857,
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"token_count_framing": 100.0
}
}

View file

@ -41,6 +41,14 @@ class PluginArchiveMetadata:
description: str | None = None
has_agent_skill_metadata: bool = False
updated_at: datetime | None = None
created_at: datetime | None = None
created_by: str | None = None
@dataclass(frozen=True, slots=True)
class PluginReview:
approved_by: str
security_scan_passed: bool
@dataclass(frozen=True, slots=True)
@ -51,6 +59,7 @@ class PluginPublication:
manifest: Mapping[str, object]
created_by: str | None
content: bytes | None = None
review: PluginReview | None = None
@dataclass(frozen=True, slots=True)
@ -66,6 +75,10 @@ class PublishedPlugin:
version: str
enabled: bool
updated_at: datetime
created_at: datetime | None = None
created_by: str | None = None
review: PluginReview | None = None
published_by: str | None = None
@dataclass(frozen=True, slots=True)

View file

@ -7683,6 +7683,17 @@
"PluginListItem": {
"description": "Plugin item in list responses.",
"properties": {
"approved_by": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Approved By"
},
"author": {
"anyOf": [
{
@ -7715,6 +7726,17 @@
],
"title": "Created At"
},
"created_by": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Created By"
},
"description": {
"anyOf": [
{
@ -7737,6 +7759,17 @@
],
"title": "Domain"
},
"download_count": {
"anyOf": [
{
"type": "integer"
},
{
"type": "null"
}
],
"title": "Download Count"
},
"enabled": {
"title": "Enabled",
"type": "boolean"
@ -7801,6 +7834,22 @@
],
"title": "Namespace"
},
"published_by": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Published By"
},
"security_scan_passed": {
"default": false,
"title": "Security Scan Passed",
"type": "boolean"
},
"source": {
"additionalProperties": {
"type": "string"
@ -59415,6 +59464,14 @@
},
"title": "Skills",
"type": "array"
},
"tags": {
"default": [],
"items": {
"type": "string"
},
"title": "Tags",
"type": "array"
}
},
"required": [
@ -59690,6 +59747,17 @@
},
"SkillHubSkill": {
"properties": {
"approved_by": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Approved By"
},
"content_url": {
"anyOf": [
{
@ -59701,6 +59769,29 @@
],
"title": "Content Url"
},
"created_at": {
"anyOf": [
{
"format": "date-time",
"type": "string"
},
{
"type": "null"
}
],
"title": "Created At"
},
"created_by": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Created By"
},
"description": {
"anyOf": [
{
@ -59732,7 +59823,7 @@
"type": "null"
}
],
"description": "Gateway ZIP downloads, included in proxy-admin catalog listings",
"description": "Gateway ZIP downloads, including repeat downloads",
"title": "Download Count"
},
"download_url": {
@ -59779,6 +59870,22 @@
],
"title": "Namespace"
},
"published_by": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Published By"
},
"security_scan_passed": {
"default": false,
"title": "Security Scan Passed",
"type": "boolean"
},
"source": {
"additionalProperties": {
"type": "string"
@ -60434,6 +60541,66 @@
"title": "Limit",
"type": "integer"
}
},
{
"in": "query",
"name": "tag",
"required": false,
"schema": {
"default": "",
"title": "Tag",
"type": "string"
}
},
{
"in": "query",
"name": "security_scan_passed",
"required": false,
"schema": {
"anyOf": [
{
"type": "boolean"
},
{
"type": "null"
}
],
"title": "Security Scan Passed"
}
},
{
"in": "query",
"name": "sort_by",
"required": false,
"schema": {
"default": "name",
"enum": [
"name",
"download_count",
"updated_at",
"created_at",
"created_by",
"published_by",
"approved_by",
"security_scan_passed"
],
"title": "Sort By",
"type": "string"
}
},
{
"in": "query",
"name": "sort_order",
"required": false,
"schema": {
"default": "asc",
"enum": [
"asc",
"desc"
],
"title": "Sort Order",
"type": "string"
}
}
],
"responses": {

View file

@ -2813,6 +2813,14 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
description="How long a blocked source address, or source address and username, stays blocked. Every attempt from a blocked key, right or wrong, is refused with 429 before the password is checked; the block is not extended by refused attempts. Set under `general_settings` in config.yaml. Defaults to 300",
)
allowed_routes: list | None = Field(None, description="Proxy API Endpoints you want users to be able to access")
allowed_client_side_s3_buckets: tuple[str, ...] | None = Field(
None,
description="S3 bucket names a client may send as `s3_bucket_name` or `s3_output_bucket_name` in a request body, e.g. on /v1/batches. Any other bucket is rejected unless client-side credentials are allowed",
)
allowed_client_side_s3_endpoint_urls: tuple[str, ...] | None = Field(
None,
description="Exact `s3_endpoint_url` values a client may send in a request body. Any other value is rejected unless client-side credentials are allowed",
)
reject_clientside_metadata_tags: bool | None = Field(
None,
description="When set to True, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata.",

View file

@ -60,6 +60,7 @@ from litellm.proxy.common_utils.claude_code_marketplace_auth import (
)
from litellm.proxy.common_utils.resource_ownership import is_proxy_admin
from litellm.proxy.deps.repositories import PluginUploadRepositoryDep
from litellm.repositories.plugin_publication import read_plugin_publisher, read_plugin_review
from litellm.repositories.table_repositories import ClaudeCodePluginRepository
from litellm.types.proxy.auth.special_headers import SpecialHeaders
from litellm.types.proxy.claude_code_endpoints import (
@ -666,6 +667,7 @@ async def register_plugin(
response_model=ListPluginsResponse,
)
async def list_plugins(
uploads: PluginUploadRepositoryDep,
enabled_only: bool = False,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
) -> ListPluginsResponse:
@ -678,6 +680,17 @@ async def list_plugins(
Returns:
List of plugins with their metadata.
"""
catalog: Final = await list_visible_plugins(enabled_only, user_api_key_dict)
if uploads is None:
raise _error_response(500, CommonProxyErrors.db_not_connected_error.value)
counts: Final = await uploads.download_counts(tuple(plugin.id for plugin in catalog.plugins))
return ListPluginsResponse(
plugins=[plugin.model_copy(update={"download_count": counts.get(plugin.id, 0)}) for plugin in catalog.plugins],
count=catalog.count,
)
async def list_visible_plugins(enabled_only: bool, user_api_key_dict: UserAPIKeyAuth) -> ListPluginsResponse:
try:
prisma_client: Final = await get_prisma_client()
@ -693,9 +706,14 @@ async def list_plugins(
# Parse manifest to get additional fields
manifest = json.loads(p.manifest_json) if p.manifest_json else {}
review = read_plugin_review(MANIFEST_FIELDS.validate_python(manifest))
plugin_list.append(
PluginListItem(
id=p.id,
created_by=p.created_by,
approved_by=review.approved_by if review else None,
published_by=read_plugin_publisher(MANIFEST_FIELDS.validate_python(manifest)),
security_scan_passed=review.security_scan_passed if review else False,
name=p.name,
version=p.version,
description=p.description,

View file

@ -7,10 +7,10 @@ from functools import lru_cache
from itertools import accumulate, chain, repeat
from logging import Logger
from types import MappingProxyType
from typing import Any, Final, Protocol
from typing import Any, Final, Literal, Protocol, TypeAlias
from fastapi import HTTPException, Request, status
from pydantic import PositiveInt, TypeAdapter, ValidationError
from pydantic import BaseModel, ConfigDict, PositiveInt, TypeAdapter, ValidationError
import litellm
from litellm import Router, constants, provider_list
@ -442,6 +442,9 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = (
# S3; ``sagemaker_base_url`` redirects all SageMaker traffic;
# ``deployment_url`` redirects SAP deployments.
"s3_endpoint_url",
"s3_bucket_name",
"s3_output_bucket_name",
"allow_legacy_cloud_file_ids",
"sagemaker_base_url",
"deployment_url",
# NVIDIA Riva fields consumed by the audio-transcription handler
@ -479,6 +482,48 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = (
)
_ClientSideAllowlistSetting: TypeAlias = Literal[
"allowed_client_side_s3_buckets", "allowed_client_side_s3_endpoint_urls"
]
_PROXY_ALLOWLIST_SETTING_BY_PARAM: Final[Mapping[str, _ClientSideAllowlistSetting]] = MappingProxyType(
{
"s3_bucket_name": "allowed_client_side_s3_buckets",
"s3_output_bucket_name": "allowed_client_side_s3_buckets",
"s3_endpoint_url": "allowed_client_side_s3_endpoint_urls",
}
)
class _ClientSideStorageAllowlist(BaseModel):
model_config = ConfigDict(frozen=True)
allowed_client_side_s3_buckets: tuple[str, ...] | None = None
allowed_client_side_s3_endpoint_urls: tuple[str, ...] | None = None
def admits(self, setting: _ClientSideAllowlistSetting, value: object) -> bool:
match setting:
case "allowed_client_side_s3_buckets":
return value in (self.allowed_client_side_s3_buckets or ())
case "allowed_client_side_s3_endpoint_urls":
return value in (self.allowed_client_side_s3_endpoint_urls or ())
def _banned_param_error(param: str) -> ValueError:
setting: Final = _PROXY_ALLOWLIST_SETTING_BY_PARAM.get(param)
allowlist_hint: Final = (
"" if setting is None else f" To allow specific values only, list them in `general_settings.{setting}`."
)
return ValueError(
f"Rejected Request: {param} is not allowed in request body. "
"Clientside passthrough requires explicit admin opt-in via "
"either `general_settings.allow_client_side_credentials = true` "
"(proxy-wide) or `configurable_clientside_auth_params` on the "
"deployment in your proxy config.yaml. "
f"Relevant Issue: https://huntr.com/bounties/4001e1a2-7b7a-4776-a3ae-e6692ec3d997{allowlist_hint}",
)
def _check_banned_params(
body: Mapping[str, object],
general_settings: dict,
@ -508,6 +553,10 @@ def _check_banned_params(
# Proxy-wide opt-in: every banned param is permitted, exit
# entirely so the rest of the loop doesn't waste work.
return
if (
setting := _PROXY_ALLOWLIST_SETTING_BY_PARAM.get(param)
) is not None and _ClientSideStorageAllowlist.model_validate(general_settings).admits(setting, body[param]):
continue
if (
_allow_model_level_clientside_configurable_parameters(
model=model,
@ -523,14 +572,7 @@ def _check_banned_params(
# ``api_base`` with an unallowed ``langfuse_host`` is still
# rejected for the second field.
continue
raise ValueError(
f"Rejected Request: {param} is not allowed in request body. "
"Clientside passthrough requires explicit admin opt-in via "
"either `general_settings.allow_client_side_credentials = true` "
"(proxy-wide) or `configurable_clientside_auth_params` on the "
"deployment in your proxy config.yaml. "
"Relevant Issue: https://huntr.com/bounties/4001e1a2-7b7a-4776-a3ae-e6692ec3d997",
)
raise _banned_param_error(param)
_FALLBACK_FIELDS: Final[tuple[str, ...]] = (

View file

@ -49,7 +49,6 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( # noqa: F401 #
BATCH_CREATE_HIDDEN_PARAM,
_is_base64_encoded_unified_file_id, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
add_deployment_model_info,
add_internal_model_credentials,
apply_team_provider_credentials,
authorize_model_for_key,
batch_cost_poller_is_active,
@ -763,13 +762,6 @@ async def retrieve_batch(
detail={"error": "LLM Router not initialized. Ensure models added to proxy."},
)
if unified_batch_id:
add_internal_model_credentials(
data=data,
llm_router=llm_router,
model_id=unified_model_id,
)
response = await llm_router.aretrieve_batch(**data)
set_hidden_param(response, "unified_batch_id", unified_batch_id)
if unified_batch_id:

View file

@ -23,7 +23,7 @@ def _fips_mode_setting(general_settings: Mapping[str, object]) -> bool | None:
def openssl_enforces_fips() -> bool:
"""Python's OpenSSL runs with FIPS-only defaults and itself refuses MD5."""
"""Python's OpenSSL runs with FIPS-only defaults and itself refuses scrypt."""
try:
import _hashlib
@ -32,7 +32,7 @@ def openssl_enforces_fips() -> bool:
except (ImportError, AttributeError, ValueError):
return False
try:
_hashlib.new("md5", b"", usedforsecurity=True)
_hashlib.scrypt(b"", salt=b"\0" * 16, n=2, r=1, p=1, dklen=16)
except ValueError:
return True
return False

View file

@ -3,7 +3,8 @@ import hashlib
from collections.abc import Mapping
from dataclasses import dataclass, replace
from datetime import datetime
from typing import Annotated, Final
from itertools import chain
from typing import Annotated, Final, Literal
from urllib.parse import quote
import httpx
@ -33,7 +34,7 @@ from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketp
enable_plugin,
get_plugin_downloads,
header_key_caller,
list_plugins,
list_visible_plugins,
plugin_archive_source_matches,
plugin_archive_url,
plugin_downloader,
@ -65,6 +66,7 @@ from litellm.proxy.discovery_endpoints.skill_import import (
parse_skill_source,
select_imported_skill,
)
from litellm.repositories.plugin_publication import read_plugin_publisher, read_plugin_review
from litellm.types.proxy.claude_code_endpoints import PluginListItem
from litellm.types.proxy.discovery_endpoints.agent_skills_endpoints import AgentSkillsIndexEntry
from litellm.types.proxy.skill_hub import (
@ -78,6 +80,7 @@ from litellm.types.proxy.skill_hub import (
SkillHubSkill,
SkillHubVersion,
SkillHubVersions,
SkillSortField,
)
router: Final = APIRouter(prefix="/skill-hub/skills", tags=["Skills Hub"])
@ -121,6 +124,11 @@ def _skill_response(request: Request, plugin: PluginListItem, download_count: in
namespace=plugin.namespace,
keywords=None if plugin.keywords is None else tuple(plugin.keywords),
updated_at=None if plugin.updated_at is None else datetime.fromisoformat(plugin.updated_at),
created_at=None if plugin.created_at is None else datetime.fromisoformat(plugin.created_at),
created_by=plugin.created_by,
approved_by=plugin.approved_by,
published_by=plugin.published_by,
security_scan_passed=plugin.security_scan_passed,
visibility="public" if plugin.enabled else "private",
source={"source": "archive", "url": _download_url(request, plugin.name)} if local_archive else plugin.source,
download_url=_download_url(request, plugin.name) if local_archive else None,
@ -137,28 +145,74 @@ async def list_skills(
query: str = "",
offset: Annotated[int, Query(ge=0)] = 0,
limit: Annotated[int, Query(ge=1, le=100)] = 50,
tag: str = "",
security_scan_passed: bool | None = None,
sort_by: SkillSortField = "name",
sort_order: Literal["asc", "desc"] = "asc",
) -> SkillHubList:
"""Browse the caller's public and granted private skills."""
catalog: Final = await list_plugins(enabled_only=False, user_api_key_dict=caller)
needle: Final = query.casefold()
catalog: Final = await list_visible_plugins(enabled_only=False, user_api_key_dict=caller)
needle: Final = query.strip().casefold()
selected_tag: Final = tag.strip().casefold()
matched: Final = tuple(
plugin
for plugin in sorted(catalog.plugins, key=lambda item: item.name)
if needle in " ".join((plugin.name, plugin.description or "", *(plugin.keywords or ()))).casefold()
if needle
in " ".join(
(
plugin.name,
plugin.description or "",
plugin.domain or "",
plugin.namespace or "",
plugin.created_by or "",
plugin.approved_by or "",
plugin.published_by or "",
*(plugin.keywords or ()),
)
).casefold()
and (not selected_tag or selected_tag in {word.casefold() for word in plugin.keywords or ()})
and (security_scan_passed is None or plugin.security_scan_passed == security_scan_passed)
)
page: Final = matched[offset : offset + limit]
counts: Final[Mapping[str, int]] = (
await _require_store(uploads).download_counts(tuple(plugin.id for plugin in page))
if is_proxy_admin(caller)
await _require_store(uploads).download_counts(tuple(plugin.id for plugin in matched))
if sort_by == "download_count"
else {}
)
ordered: Final = sorted(
matched, key=lambda plugin: _skill_sort_key(plugin, sort_by, counts), reverse=sort_order == "desc"
)
page: Final = ordered[offset : offset + limit]
page_counts: Final = (
await _require_store(uploads).download_counts(tuple(plugin.id for plugin in page))
if sort_by != "download_count"
else counts
)
return SkillHubList(
skills=tuple(_skill_response(request, plugin, counts.get(plugin.id)) for plugin in page),
skills=tuple(_skill_response(request, plugin, page_counts.get(plugin.id)) for plugin in page),
count=len(matched),
next_offset=offset + limit if offset + limit < len(matched) else None,
tags=tuple(
sorted(set(chain.from_iterable(plugin.keywords or () for plugin in catalog.plugins)), key=str.casefold)
),
)
def _skill_sort_key(plugin: PluginListItem, field: SkillSortField, counts: Mapping[str, int]) -> str | float:
if field == "download_count":
return counts.get(plugin.id, 0)
if field == "security_scan_passed":
return int(plugin.security_scan_passed)
if field in {"created_at", "updated_at"}:
value: Final = plugin.created_at if field == "created_at" else plugin.updated_at
return datetime.fromisoformat(value).timestamp() if value else 0
return {
"name": plugin.name,
"created_by": plugin.created_by or "",
"published_by": plugin.published_by or "",
"approved_by": plugin.approved_by or "",
}[field].casefold()
@router.get("/{name}", response_model=SkillHubSkill)
async def get_skill(
request: Request,
@ -176,6 +230,7 @@ async def get_skill(
def skill_metadata_response(request: Request, metadata: PluginArchiveMetadata) -> SkillHubSkill:
manifest: Final = MANIFEST_FIELDS.validate_json(metadata.manifest_json or "{}")
review: Final = read_plugin_review(manifest)
return _skill_response(
request,
PluginListItem.model_validate(
@ -187,7 +242,11 @@ def skill_metadata_response(request: Request, metadata: PluginArchiveMetadata) -
"source": dict(metadata.source),
"has_agent_skill_metadata": metadata.has_agent_skill_metadata,
"enabled": metadata.enabled,
"created_at": None,
"created_at": metadata.created_at.isoformat() if metadata.created_at is not None else None,
"created_by": metadata.created_by,
"approved_by": review.approved_by if review else None,
"published_by": read_plugin_publisher(manifest),
"security_scan_passed": review.security_scan_passed if review else False,
"updated_at": metadata.updated_at.isoformat() if metadata.updated_at is not None else None,
"domain": manifest.get("domain"),
"namespace": manifest.get("namespace"),
@ -416,7 +475,11 @@ def skill_publication_response(
"version": published.version,
"enabled": published.enabled,
"has_agent_skill_metadata": has_agent_skill_metadata(publication.manifest),
"created_at": None,
"created_at": published.created_at.isoformat() if published.created_at is not None else None,
"created_by": published.created_by,
"approved_by": published.review.approved_by if published.review else None,
"published_by": published.published_by,
"security_scan_passed": published.review.security_scan_passed if published.review else False,
"updated_at": published.updated_at.isoformat(),
"domain": publication.manifest.get("domain"),
"namespace": publication.manifest.get("namespace"),

View file

@ -58,6 +58,8 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import (
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
)
from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import (
split_media_for_tpm,
)
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled

View file

@ -5483,7 +5483,9 @@ async def generate_key_helper_fn(
if query_type == "insert_data":
user_row = cast( # cast-ok: table_name="user" is the insert_data branch returning the user row
"prisma_models.LiteLLM_UserTable | None",
await prisma_client.insert_data(data=user_data, table_name="user"),
await prisma_client.insert_data(
data=user_data, table_name="user", allow_existing_user=request_type != "user"
),
)
if user_row is None:

View file

@ -1294,7 +1294,7 @@ async def _create_user_if_not_exists(user_id: str, created_via: str = "scim_grou
if litellm.default_internal_user_params:
default_role = litellm.default_internal_user_params.get("user_role")
new_user_request: Final = NewUserRequest(
new_user_request: Final = SCIMNewUserRequest(
user_id=user_id,
user_email=user_id, # We don't have email from group membership
user_alias=None,
@ -1873,7 +1873,7 @@ async def create_user(
admin_group: Final = await _get_scim_admin_group()
resolved_role: Final = _resolve_scim_user_role(user.groups or [], admin_group, default_role)
new_user_request: Final = NewUserRequest(
new_user_request: Final = SCIMNewUserRequest(
user_id=user_id,
user_email=user_data["user_email"],
user_alias=user_data["user_alias"],

View file

@ -78,7 +78,6 @@ from litellm.proxy._types import (
LitellmUserRoles,
Member,
NewTeamRequest,
NewUserRequest,
NewUserRequestTeam,
NewUserResponse,
ProxyErrorTypes,
@ -164,6 +163,7 @@ from litellm.types.proxy.management_endpoints.ui_sso import (
MicrosoftGraphAPIUserGroupResponse,
MicrosoftServicePrincipalTeam,
RoleMappings,
SSONewUserRequest,
TeamMappings,
)
from litellm.types.proxy.ui_sso import ParsedOpenIDResult
@ -2915,7 +2915,7 @@ async def insert_sso_user(
if user_defined_values["user_role"] is None:
user_defined_values["user_role"] = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY
new_user_request: Final = NewUserRequest(
new_user_request: Final = SSONewUserRequest(
user_id=user_defined_values["user_id"],
user_email=normalize_email(user_defined_values["user_email"]),
user_alias=_get_sso_user_alias(result_openid),
@ -2924,11 +2924,9 @@ async def insert_sso_user(
budget_duration=user_defined_values["budget_duration"],
sso_user_id=user_defined_values["user_id"],
auto_create_key=False,
metadata={"auth_provider": getattr(result_openid, "provider")} if hasattr(result_openid, "provider") else {},
)
if result_openid and hasattr(result_openid, "provider"):
new_user_request.metadata = {"auth_provider": getattr(result_openid, "provider")}
from litellm.proxy.proxy_server import premium_user
with free_sso_user_creation(new_user_request.user_id, enabled=premium_user is not True):

View file

@ -742,31 +742,6 @@ def apply_team_provider_credentials(
prepare_data_with_credentials(data=data, credentials={**credentials, **explicit_request_credentials(data)})
def add_internal_model_credentials(
data: dict,
llm_router: "Router",
model_id: str | None,
) -> None:
"""
Attach the deployment's immutable server-side credential snapshot to a router-routed
batch call (in-place).
Cost accounting for a completed batch reads the batch's output file, and the Bedrock
file config resolves its bucket only from this snapshot, never from a request param,
because the bucket is what managed file ids are validated against. Without it that
read fails and the batch's cost is never recorded.
"""
if model_id is None:
return
try:
credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
except Exception: # noqa: BLE001 # the snapshot only enables cost accounting; a batch whose deployment no longer resolves must still be retrievable
return
if credentials is None:
return
data["_litellm_internal_model_credentials"] = MappingProxyType(dict(credentials))
def add_deployment_model_info(
data: dict,
llm_router: Optional["Router"],
@ -792,7 +767,6 @@ def prepare_data_with_credentials(
data: dict,
credentials: dict,
file_id: str | None = None,
include_internal_credentials: bool = False,
) -> None:
"""
Update data dictionary with model credentials (in-place).
@ -801,12 +775,8 @@ def prepare_data_with_credentials(
data: Data dictionary to update
credentials: Credentials from router
file_id: Optional original file_id to set (for decoded file IDs)
include_internal_credentials: Preserve an immutable server-side snapshot
for code paths that must distinguish proxy config from request params.
"""
data.update(credentials)
if include_internal_credentials:
data["_litellm_internal_model_credentials"] = MappingProxyType(dict(credentials))
data.pop("custom_llm_provider", None)
if file_id is not None:

View file

@ -43,7 +43,6 @@ class FileContentStreamingHandler:
data=resolved_streaming_data,
credentials=credentials,
file_id=original_file_id,
include_internal_credentials=True,
)
resolved_streaming_data.pop("model", None)
resolved_streaming_provider: Final = cast(str, credentials["custom_llm_provider"])

Some files were not shown because too many files have changed in this diff Show more