mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge remote-tracking branch 'origin/main' into beluga_decrease_anys_20261102
# Conflicts: # litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py # tests/unit/test_utils.py
This commit is contained in:
commit
fdd4b0028a
289 changed files with 10597 additions and 5697 deletions
2
.github/workflows/image-scan.yml
vendored
2
.github/workflows/image-scan.yml
vendored
|
|
@ -53,7 +53,7 @@ jobs:
|
|||
if: >-
|
||||
github.event_name != 'pull_request' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository
|
||||
timeout-minutes: 30
|
||||
timeout-minutes: 60
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
|
|
|
|||
|
|
@ -49,9 +49,9 @@ jobs:
|
|||
id: compliance
|
||||
run: |
|
||||
uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \
|
||||
--interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \
|
||||
--interactions-spec-url=https://aistudio.google.com/api-reference/interactions.openapi.json \
|
||||
--tb=short -vv -p no:rerunfailures -p no:pytest-retry \
|
||||
--allow-hosts=127.0.0.1,::1,localhost,ai.google.dev
|
||||
--allow-hosts=127.0.0.1,::1,localhost,ai.google.dev,aistudio.google.com
|
||||
|
||||
tracking-issue:
|
||||
name: tracking-issue
|
||||
|
|
|
|||
2
.github/workflows/test-linting.yml
vendored
2
.github/workflows/test-linting.yml
vendored
|
|
@ -21,7 +21,7 @@ jobs:
|
|||
pull-requests: read
|
||||
actions: read
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 30
|
||||
services:
|
||||
redis:
|
||||
image: redis@sha256:e2debfb7956fa12c7ddc79d7e645c8cf26b30c99a6e9161ea9bf4171e1668a5f
|
||||
|
|
|
|||
18
.github/workflows/test-unit.yml
vendored
18
.github/workflows/test-unit.yml
vendored
|
|
@ -45,7 +45,7 @@ jobs:
|
|||
id: rust-bridge-key
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
env:
|
||||
RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml') }}
|
||||
RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml', 'packaging/litellm-proxy/pyproject.toml') }}
|
||||
run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up Python
|
||||
|
|
@ -98,6 +98,18 @@ jobs:
|
|||
mkdir -p rust-bridge-dist/litellm/rust_bridge
|
||||
cp litellm/rust_bridge/_native.abi3.so rust-bridge-dist/litellm/rust_bridge/_native.abi3.so
|
||||
|
||||
- name: Build the gateway wheel once for test shards
|
||||
if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true'
|
||||
timeout-minutes: 15
|
||||
working-directory: packaging/litellm-proxy
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
SCCACHE_GHA_VERSION: litellm-ci-v1
|
||||
SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }}
|
||||
CARGO_INCREMENTAL: "0"
|
||||
run: uv run --no-project --with maturin==1.15.0 maturin build --profile dev --strip --out ../../rust-bridge-dist
|
||||
|
||||
- name: Upload Rust bridge artifact
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 10
|
||||
|
|
@ -1024,7 +1036,7 @@ jobs:
|
|||
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
|
||||
else
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
|
||||
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
|
||||
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
|
||||
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
|
||||
|
|
@ -1486,7 +1498,7 @@ jobs:
|
|||
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
|
||||
else
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
|
||||
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
|
||||
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
|
|||
COPY pyproject.toml uv.lock ./
|
||||
COPY enterprise/pyproject.toml enterprise/
|
||||
COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
|
||||
COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/
|
||||
|
||||
# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
|
||||
RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
|
||||
|
|
@ -101,7 +102,8 @@ COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/
|
|||
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
|
||||
|
||||
# Install project and workspace packages (fast - deps already cached)
|
||||
RUN uv sync --frozen --no-default-groups --no-editable \
|
||||
RUN uv build --package litellm-proxy --wheel --out-dir /wheels
|
||||
RUN uv sync --frozen --no-install-package litellm-proxy --no-default-groups --no-editable \
|
||||
--extra proxy \
|
||||
--extra proxy-runtime \
|
||||
--group admin-mcp \
|
||||
|
|
@ -111,6 +113,8 @@ RUN uv sync --frozen --no-default-groups --no-editable \
|
|||
--extra bedrock-realtime \
|
||||
--python python3.13
|
||||
|
||||
RUN uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm_proxy-*.whl
|
||||
|
||||
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
|
||||
npm_config_cache=/root/.npm \
|
||||
prisma generate --schema=./schema.prisma
|
||||
|
|
|
|||
|
|
@ -74,6 +74,7 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
|
|||
COPY pyproject.toml uv.lock ./
|
||||
COPY enterprise/pyproject.toml enterprise/
|
||||
COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
|
||||
COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/
|
||||
|
||||
# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
|
||||
RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
|
||||
|
|
@ -99,7 +100,8 @@ COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/
|
|||
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
|
||||
|
||||
# Install project and workspace packages (fast - deps already cached)
|
||||
RUN uv sync --frozen --no-default-groups --no-editable \
|
||||
RUN uv build --package litellm-proxy --wheel --out-dir /wheels
|
||||
RUN uv sync --frozen --no-install-package litellm-proxy --no-default-groups --no-editable \
|
||||
--extra proxy \
|
||||
--extra proxy-runtime \
|
||||
--group admin-mcp \
|
||||
|
|
@ -109,6 +111,8 @@ RUN uv sync --frozen --no-default-groups --no-editable \
|
|||
--extra bedrock-realtime \
|
||||
--python python3.13
|
||||
|
||||
RUN uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm_proxy-*.whl
|
||||
|
||||
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
|
||||
npm_config_cache=/root/.npm \
|
||||
prisma generate --schema=./schema.prisma
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import re
|
|||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import replace as dataclasses_replace
|
||||
from datetime import datetime, timedelta
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tuple, cast
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
|
|
@ -766,7 +765,6 @@ class CheckBatchCost:
|
|||
credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
|
||||
_file_content = await afile_content(
|
||||
file_id=cast(str, raw_output_file_id),
|
||||
_litellm_internal_model_credentials=MappingProxyType(dict(credentials)),
|
||||
**cast(FileContentCallOptions, credentials),
|
||||
)
|
||||
|
||||
|
|
@ -867,10 +865,7 @@ class CheckBatchCost:
|
|||
error_file_failed_requests: Final = await count_error_file_failed_requests(
|
||||
response,
|
||||
custom_llm_provider=batch_file_provider,
|
||||
litellm_params={
|
||||
**credentials,
|
||||
"_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),
|
||||
},
|
||||
litellm_params=credentials,
|
||||
)
|
||||
batch_result: Final = (
|
||||
output_file_result
|
||||
|
|
@ -1029,11 +1024,7 @@ class CheckBatchCost:
|
|||
custom_llm_provider=deployment_info.litellm_params.custom_llm_provider,
|
||||
)
|
||||
cost_provider: Final = cast(BatchFileProvider, llm_provider)
|
||||
credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
|
||||
file_access: Final = {
|
||||
**credentials,
|
||||
"_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),
|
||||
}
|
||||
file_access: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
|
||||
output_content: Final = (
|
||||
await fetch_batch_managed_file_content(
|
||||
response.output_file_id, custom_llm_provider=cost_provider, litellm_params=file_access
|
||||
|
|
|
|||
|
|
@ -127,7 +127,6 @@ class _ManagedFileRetrieve(Protocol):
|
|||
self,
|
||||
*,
|
||||
file_id: str,
|
||||
_litellm_internal_model_credentials: Mapping[str, object] | None = None,
|
||||
**kwargs: Unpack[FileRetrieveCallOptions],
|
||||
) -> OpenAIFileObject: ...
|
||||
|
||||
|
|
@ -400,7 +399,7 @@ class _RouterFileCallKwargs(TypedDict, total=False):
|
|||
|
||||
_MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str])
|
||||
_FILE_UPLOAD_ONLY_FIELDS: Final = frozenset({"file", "purpose", "expires_after", "extra_body"})
|
||||
_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id", "_litellm_internal_model_credentials"})
|
||||
_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id"})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -459,15 +458,7 @@ async def _delete_provider_file(
|
|||
provider_file_id: str,
|
||||
request_data: Mapping[str, object],
|
||||
) -> None:
|
||||
credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
|
||||
delete_data: Final = {
|
||||
**{k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS},
|
||||
**(
|
||||
{"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
|
||||
if credentials is not None
|
||||
else {}
|
||||
),
|
||||
}
|
||||
delete_data: Final = {k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS}
|
||||
await llm_router.afile_delete(
|
||||
model=model_id,
|
||||
file_id=provider_file_id,
|
||||
|
|
@ -1160,7 +1151,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
*,
|
||||
provider_file_id: str,
|
||||
call_options: Mapping[str, object],
|
||||
internal_model_credentials: Mapping[str, object] | None = None,
|
||||
) -> OpenAIFileObject:
|
||||
"""Retrieve provider file details with transient retries and SDK retries disabled."""
|
||||
retrieve_file: Final = cast(_ManagedFileRetrieve, litellm.afile_retrieve)
|
||||
|
|
@ -1172,14 +1162,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
if attempt > 0:
|
||||
await self._sleep(_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS[attempt - 1])
|
||||
try:
|
||||
if internal_model_credentials is None:
|
||||
return await retrieve_file(
|
||||
file_id=provider_file_id,
|
||||
**retrieve_options,
|
||||
)
|
||||
return await retrieve_file(
|
||||
file_id=provider_file_id,
|
||||
_litellm_internal_model_credentials=MappingProxyType(dict(internal_model_credentials)),
|
||||
**retrieve_options,
|
||||
)
|
||||
except Exception as error:
|
||||
|
|
@ -1221,7 +1205,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
self._afile_retrieve_with_retries(
|
||||
provider_file_id=provider_file_id,
|
||||
call_options=router_credentials,
|
||||
internal_model_credentials=router_credentials,
|
||||
),
|
||||
timeout=BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
|
@ -2866,15 +2849,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
exception_dict = {}
|
||||
for model_id, provider_file_id in specific_model_file_id_mapping.items():
|
||||
try:
|
||||
# Cloud-storage providers (e.g. Bedrock S3) validate file ids
|
||||
# against the deployment's configured bucket, which they only
|
||||
# trust from this immutable server-side snapshot, never from
|
||||
# request params.
|
||||
credentials = llm_router.registry().credentials(model_id, form="with_provider")
|
||||
if credentials is not None:
|
||||
data["_litellm_internal_model_credentials"] = cast(Dict, MappingProxyType(dict(credentials)))
|
||||
else:
|
||||
data.pop("_litellm_internal_model_credentials", None)
|
||||
return cast(
|
||||
HttpxBinaryResponseContent,
|
||||
await llm_router.afile_content(
|
||||
|
|
|
|||
|
|
@ -41,10 +41,14 @@ COPY pyproject.toml README.md rust-toolchain.toml ./
|
|||
COPY .cargo ./.cargo
|
||||
COPY litellm-rust ./litellm-rust
|
||||
COPY model_prices_and_context_window.json ./
|
||||
COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
mkdir -p litellm/rust_bridge /wheels && \
|
||||
touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
|
||||
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
|
||||
python -c "import maturin; maturin.build_wheel('/wheels')" && \
|
||||
cd packaging/litellm-proxy && \
|
||||
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
|
||||
python -c "import maturin; maturin.build_wheel('/wheels')"
|
||||
|
||||
|
|
@ -85,6 +89,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
|||
--mount=type=bind,source=uv.lock,target=uv.lock \
|
||||
--mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \
|
||||
--mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \
|
||||
--mount=type=bind,source=packaging/litellm-proxy/pyproject.toml,target=packaging/litellm-proxy/pyproject.toml \
|
||||
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
|
||||
--extra proxy \
|
||||
--extra proxy-runtime \
|
||||
|
|
@ -98,7 +103,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
|||
COPY . .
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --frozen --no-install-project --no-default-groups --no-editable \
|
||||
uv sync --frozen --no-install-project --no-install-package litellm-proxy --no-default-groups --no-editable \
|
||||
--extra proxy \
|
||||
--extra proxy-runtime \
|
||||
--group admin-mcp \
|
||||
|
|
@ -108,7 +113,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
|||
--python python3.13
|
||||
|
||||
RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
|
||||
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
|
||||
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl /wheels/litellm_proxy-*.whl
|
||||
|
||||
# PYTHONPATH=/app makes the source tree shadow the installed package, so the
|
||||
# compiled Rust extension must live next to the source or it is never imported.
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -4,13 +4,7 @@
|
|||
//! `should_retry_this_error` and `_time_to_sleep_before_retry` in `litellm/router.py`, and
|
||||
//! `run_async_fallback` in `router_utils/fallback_event_handlers.py`.
|
||||
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use std::{borrow::Cow, future::Future, pin::Pin, sync::Arc, time::Duration};
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
|
|
@ -1231,7 +1225,7 @@ impl Core {
|
|||
.unrouted_attempt(snapshot, op, request, target, (retry, max_retries), at)
|
||||
.await;
|
||||
}
|
||||
let pick_started = Instant::now();
|
||||
let pick_started = self.health.now_time();
|
||||
let (deployment, candidates) = match self
|
||||
.picked(snapshot, op, request, target, inputs, retry, at)
|
||||
.await
|
||||
|
|
@ -1248,7 +1242,11 @@ impl Core {
|
|||
self.host.observe(&RouterEvent::Selected {
|
||||
model_group: candidates.model_group.clone(),
|
||||
deployment_id: deployment.id.clone(),
|
||||
pick: pick_started.elapsed(),
|
||||
pick: self
|
||||
.health
|
||||
.now_time()
|
||||
.duration_since(pick_started)
|
||||
.unwrap_or_default(),
|
||||
});
|
||||
if !self.host.supports(op, deployment.provider()) {
|
||||
return Err(Error::Config(ConfigError::Unsupported(Unsupported::Op {
|
||||
|
|
@ -1274,7 +1272,7 @@ impl Core {
|
|||
},
|
||||
None,
|
||||
snapshot.settings.dynamic.default_ref(),
|
||||
jiff::Timestamp::now(),
|
||||
self.health.timestamp(),
|
||||
),
|
||||
Ok(None)
|
||||
);
|
||||
|
|
|
|||
|
|
@ -247,6 +247,10 @@ impl Health {
|
|||
self.clock.now()
|
||||
}
|
||||
|
||||
pub(crate) fn timestamp(&self) -> jiff::Timestamp {
|
||||
jiff::Timestamp::try_from(self.clock.now()).unwrap_or(jiff::Timestamp::UNIX_EPOCH)
|
||||
}
|
||||
|
||||
fn now(&self) -> f64 {
|
||||
unix_seconds(self.clock.now())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -433,6 +433,10 @@ impl<'a> Lookup<'a> {
|
|||
.collect()
|
||||
}
|
||||
|
||||
pub fn has_team_wildcards(&self, team_id: &str) -> bool {
|
||||
self.host.team_patterns.iter().any(|(t, _)| t == team_id)
|
||||
}
|
||||
|
||||
/// The deployments `team_id`'s own wildcard pattern routes `name` to
|
||||
/// (`_core_team_pattern_routers`, keyed by `team_public_model_name`).
|
||||
pub fn team_wildcard(&self, name: &str, team_id: &str) -> Vec<Arc<Deployment>> {
|
||||
|
|
|
|||
|
|
@ -223,14 +223,20 @@ pub(crate) fn wildcard_name<'a>(
|
|||
registry: &'a Registry,
|
||||
name: &'a str,
|
||||
specific_deployment: bool,
|
||||
team_id: Option<&str>,
|
||||
) -> Option<&'a str> {
|
||||
let model = registry.alias_of(name).unwrap_or(name);
|
||||
(registry.has_wildcards()
|
||||
let lookup = crate::lookup::Lookup::new(registry);
|
||||
let team_patterns = team_id.is_some_and(|team| lookup.has_team_wildcards(team));
|
||||
let team_misses = || team_id.is_none_or(|team| lookup.team_wildcard(model, team).is_empty());
|
||||
let global_misses = || registry.has_wildcards() && registry.wildcard_for(model).is_none();
|
||||
((team_patterns || registry.has_wildcards())
|
||||
&& !specific_deployment
|
||||
&& !registry.has_name(model)
|
||||
&& (registry.has_name(name) || registry.deployment(name).is_none())
|
||||
&& registry.routing_group_named(model).is_none()
|
||||
&& registry.wildcard_for(model).is_none())
|
||||
&& team_misses()
|
||||
&& (team_patterns || global_misses()))
|
||||
.then_some(model)
|
||||
}
|
||||
|
||||
|
|
@ -339,15 +345,21 @@ pub fn resolve_request(
|
|||
let kept = reserved(model, deployments, caller, now)?;
|
||||
return resolved(ResolveKind::Group, model, kept);
|
||||
}
|
||||
// `_core_wildcard_route_deployments`: the team's own pattern, else the proxy-wide one.
|
||||
let team_owned = team_id
|
||||
.map(|team| crate::lookup::Lookup::new(registry).team_wildcard(model, team))
|
||||
.filter(|deployments| !deployments.is_empty());
|
||||
if let Some(deployments) = team_owned {
|
||||
// `_core_wildcard_route_deployments`: the team's own pattern, else the proxy-wide one,
|
||||
// each tried as spelled and then under the name's provider (`get_pattern`).
|
||||
let team_owned = team_id.and_then(|team| {
|
||||
let lookup = crate::lookup::Lookup::new(registry);
|
||||
[Some(model), caller.wildcard_request]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|request| (request, lookup.team_wildcard(request, team)))
|
||||
.find(|(_, deployments)| !deployments.is_empty())
|
||||
});
|
||||
if let Some((request, deployments)) = team_owned {
|
||||
let routed = deployments.iter().flat_map(|deployment| {
|
||||
let public = deployment.model_info.team_public_model_name.as_deref();
|
||||
let pattern = public.unwrap_or(model);
|
||||
routed_wildcard(pattern, model, std::slice::from_ref(deployment))
|
||||
let pattern = public.unwrap_or(request);
|
||||
routed_wildcard(pattern, request, std::slice::from_ref(deployment))
|
||||
});
|
||||
let kept = reserved(model, routed.collect(), caller, now)?;
|
||||
return resolved(ResolveKind::Wildcard, model, kept);
|
||||
|
|
@ -428,6 +440,7 @@ pub fn resolve_request(
|
|||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::*;
|
||||
|
|
@ -544,8 +557,11 @@ mod tests {
|
|||
#[test]
|
||||
fn a_name_no_pattern_matches_is_tried_under_its_provider() {
|
||||
let registry = registry(wildcard_group());
|
||||
assert_eq!(wildcard_name(®istry, "gpt-4o", false), Some("gpt-4o"));
|
||||
assert_eq!(wildcard_name(®istry, "openai/gpt-4o", false), None);
|
||||
assert_eq!(
|
||||
wildcard_name(®istry, "gpt-4o", false, None),
|
||||
Some("gpt-4o")
|
||||
);
|
||||
assert_eq!(wildcard_name(®istry, "openai/gpt-4o", false, None), None);
|
||||
let caller = Caller {
|
||||
wildcard_request: Some("openai/gpt-4o"),
|
||||
..Caller::default()
|
||||
|
|
@ -564,6 +580,48 @@ mod tests {
|
|||
assert_eq!(resolution.deployments[0].params.model, "openai/gpt-4o");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_bare_name_is_tried_under_its_provider_against_the_teams_own_wildcard() {
|
||||
let registry = registry(json!([
|
||||
{"model_name": "model_name_tw_1", "litellm_params": {"model": "openai/*"},
|
||||
"model_info": {"id": "tw", "team_id": "TW", "team_public_model_name": "openai/*"}},
|
||||
{"model_name": "g", "litellm_params": {"model": "openai/g"}, "model_info": {"id": "g"}},
|
||||
]));
|
||||
// No proxy-wide wildcard: only the team's patterns call for the provider spelling.
|
||||
assert_eq!(
|
||||
wildcard_name(®istry, "gpt-4o-mini", false, Some("TW")),
|
||||
Some("gpt-4o-mini")
|
||||
);
|
||||
assert_eq!(wildcard_name(®istry, "gpt-4o-mini", false, None), None);
|
||||
assert_eq!(
|
||||
wildcard_name(®istry, "openai/gpt-4o-mini", false, Some("TW")),
|
||||
None
|
||||
);
|
||||
let resolve = |team_id| {
|
||||
let caller = Caller {
|
||||
team_id,
|
||||
wildcard_request: Some("openai/gpt-4o-mini"),
|
||||
..Caller::default()
|
||||
};
|
||||
resolve_request(
|
||||
®istry,
|
||||
"gpt-4o-mini",
|
||||
false,
|
||||
caller,
|
||||
None,
|
||||
None,
|
||||
Timestamp::now(),
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
let resolution = resolve(Some("TW")).unwrap();
|
||||
assert_eq!(resolution.kind, ResolveKind::Wildcard);
|
||||
assert_eq!(resolution.group, "gpt-4o-mini");
|
||||
assert_eq!(ids(Some(resolution.clone())), ["tw"]);
|
||||
assert_eq!(resolution.deployments[0].params.model, "openai/gpt-4o-mini");
|
||||
assert!(resolve(Some("OTHER")).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_deployment_named_by_id_refuses_the_endpoint_like_any_other() {
|
||||
let registry = registry(json!([
|
||||
|
|
|
|||
|
|
@ -281,8 +281,12 @@ impl Core {
|
|||
// No deployment for the name: route to the first default fallback.
|
||||
let default_target = crate::resolve::default_target(&snapshot.settings.dynamic);
|
||||
// `get_pattern`: a name no pattern matches is tried again under its inferred provider.
|
||||
let wildcard_name =
|
||||
crate::resolve::wildcard_name(registry, name, request.specific_deployment);
|
||||
let wildcard_name = crate::resolve::wildcard_name(
|
||||
registry,
|
||||
name,
|
||||
request.specific_deployment,
|
||||
request.team_id.as_deref(),
|
||||
);
|
||||
let wildcard_request = match wildcard_name {
|
||||
Some(model) => {
|
||||
let scope = StrategyScope::Default;
|
||||
|
|
@ -308,7 +312,7 @@ impl Core {
|
|||
caller,
|
||||
default_target,
|
||||
snapshot.settings.dynamic.default_ref(),
|
||||
jiff::Timestamp::now(),
|
||||
self.health.timestamp(),
|
||||
)?
|
||||
.ok_or_else(|| Error::UnknownModel {
|
||||
// The name `_common_checks_available_deployment` looked for, past the alias.
|
||||
|
|
|
|||
|
|
@ -1755,3 +1755,57 @@ async fn a_listed_pass_through_pick_names_the_group_an_alias_resolves_to(config:
|
|||
Error::NoPassThroughDeployments { model } if model == "primary"
|
||||
));
|
||||
}
|
||||
|
||||
struct StoppedClock(u64);
|
||||
|
||||
impl litellm_router_core::health::Clock for StoppedClock {
|
||||
fn now(&self) -> std::time::SystemTime {
|
||||
std::time::UNIX_EPOCH + Duration::from_secs(self.0)
|
||||
}
|
||||
}
|
||||
|
||||
fn utc_hour(hour: u64) -> u64 {
|
||||
1_767_225_600 + hour * 3_600
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::inside_the_window(utc_hour(10), vec!["open"])]
|
||||
#[case::outside_the_window(utc_hour(20), vec!["open", "reserved"])]
|
||||
#[tokio::test]
|
||||
async fn access_windows_follow_the_routers_clock(#[case] now: u64, #[case] expected: Vec<&str>) {
|
||||
let config: RouterConfig = serde_json::from_value(json!({
|
||||
"model_list": [
|
||||
{"model_name": "chat", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "open"}},
|
||||
{"model_name": "chat", "litellm_params": {"model": "openai/b"}, "model_info": {
|
||||
"id": "reserved",
|
||||
"access_windows": [{"start": "09:00", "end": "17:00", "timezone": "UTC", "team_ids": ["batch"]}],
|
||||
}},
|
||||
]
|
||||
}))
|
||||
.unwrap();
|
||||
let router = Router::builder(
|
||||
config,
|
||||
RouterDynamicConfig::default(),
|
||||
Arc::new(ScriptedHost::default()),
|
||||
)
|
||||
.randomness(Arc::new(SeededRandomness::new(7)))
|
||||
.clock(Arc::new(StoppedClock(now)))
|
||||
.build()
|
||||
.unwrap();
|
||||
let mut picked = std::collections::BTreeSet::new();
|
||||
for _ in 0..40 {
|
||||
let deployment = router
|
||||
.select(&Op::from("acompletion"), &request_for("chat"))
|
||||
.await
|
||||
.unwrap();
|
||||
picked.insert(deployment.id.clone());
|
||||
}
|
||||
assert_eq!(picked.into_iter().collect::<Vec<_>>(), expected);
|
||||
}
|
||||
|
||||
fn request_for(model: &str) -> RouterRequest {
|
||||
RouterRequest {
|
||||
model: model.to_owned(),
|
||||
..RouterRequest::default()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -641,9 +641,11 @@ def extract_file_access_credentials(
|
|||
"bucket_name",
|
||||
"s3_endpoint_url",
|
||||
"s3_region_name",
|
||||
"s3_bucket_name",
|
||||
"s3_output_bucket_name",
|
||||
"allow_legacy_cloud_file_ids",
|
||||
"timeout",
|
||||
"max_retries",
|
||||
"_litellm_internal_model_credentials",
|
||||
*AWS_CREDENTIAL_KWARGS_KEYS,
|
||||
# A federated deployment holds no api_key, so without these the fetch that reads a
|
||||
# finished batch's output has nothing to authenticate with and its cost is never billed.
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ from openai.types.batch import BatchRequestCounts
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.get_litellm_params import add_trusted_model_credentials_to_litellm_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.anthropic.batches.handler import AnthropicBatchesHandler
|
||||
from litellm.llms.azure.batches.handler import AzureBatchesAPI
|
||||
|
|
@ -585,7 +584,6 @@ def retrieve_batch(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
**kwargs,
|
||||
)
|
||||
add_trusted_model_credentials_to_litellm_params(litellm_params, kwargs)
|
||||
if litellm_logging_obj is not None:
|
||||
litellm_logging_obj.update_from_kwargs(
|
||||
kwargs=kwargs,
|
||||
|
|
|
|||
|
|
@ -41,12 +41,14 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
|||
)
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, iter_items
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
|
||||
prompt_cache_breakpoint_with_readable_mode,
|
||||
responses_reasoning_items_from_thinking_blocks,
|
||||
whole_prompt_cache_breakpoint_or_none,
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
responses_reasoning_items_from_thinking_blocks,
|
||||
)
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
from litellm.llms.base_llm.bridges.completion_transformation import (
|
||||
CompletionTransformationBridge,
|
||||
|
|
|
|||
|
|
@ -38,7 +38,6 @@ import litellm
|
|||
from litellm import get_secret_str
|
||||
from litellm.files.streaming import FileContentStreamingResponse
|
||||
from litellm.files.types import FileContentProvider, FileContentStreamingResult, FileRetrieveProvider
|
||||
from litellm.litellm_core_utils.get_litellm_params import add_trusted_model_credentials_to_litellm_params
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.azure.common_utils import get_azure_credentials
|
||||
|
|
@ -422,10 +421,6 @@ def file_retrieve(
|
|||
)
|
||||
if provider_config is not None:
|
||||
litellm_params_dict: Final = get_litellm_params(**kwargs)
|
||||
add_trusted_model_credentials_to_litellm_params(
|
||||
litellm_params_dict=litellm_params_dict,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
litellm_params_dict["api_key"] = optional_params.api_key
|
||||
litellm_params_dict["api_base"] = optional_params.api_base
|
||||
|
||||
|
|
@ -544,10 +539,6 @@ def file_delete(
|
|||
pass
|
||||
optional_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
litellm_params_dict: Final = get_litellm_params(**kwargs)
|
||||
add_trusted_model_credentials_to_litellm_params(
|
||||
litellm_params_dict=litellm_params_dict,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
### TIMEOUT LOGIC ###
|
||||
timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600
|
||||
# set timeout for 10 minutes by default
|
||||
|
|
@ -737,10 +728,6 @@ def file_list(
|
|||
)
|
||||
if provider_config is not None:
|
||||
litellm_params_dict: Final = get_litellm_params(**kwargs)
|
||||
add_trusted_model_credentials_to_litellm_params(
|
||||
litellm_params_dict=litellm_params_dict,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
litellm_params_dict["api_key"] = optional_params.api_key
|
||||
litellm_params_dict["api_base"] = optional_params.api_base
|
||||
|
||||
|
|
@ -890,10 +877,6 @@ def file_content(
|
|||
try:
|
||||
optional_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
litellm_params_dict: Final = get_litellm_params(**kwargs)
|
||||
add_trusted_model_credentials_to_litellm_params(
|
||||
litellm_params_dict=litellm_params_dict,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
### TIMEOUT LOGIC ###
|
||||
timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600
|
||||
client: Final = kwargs.get("client")
|
||||
|
|
|
|||
|
|
@ -26,18 +26,18 @@ from litellm.exceptions import UnsupportedParamsError
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.custom_prompt_management import CustomPromptManagement
|
||||
from litellm.integrations.prompt_management_base import PromptManagementClient
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
tool_call_ids_answered_by_client,
|
||||
tool_call_ids_answered_in,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import string_field
|
||||
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
|
||||
PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT,
|
||||
cache_control_as_prompt_cache_breakpoint,
|
||||
prompt_cache_breakpoint_as_cache_control,
|
||||
with_prompt_cache_breakpoint,
|
||||
without_prompt_cache_breakpoints,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
tool_call_ids_answered_by_client,
|
||||
tool_call_ids_answered_in,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import string_field
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
is_claude_code_one_shot_subagent_request,
|
||||
supports_anthropic_cache_control,
|
||||
|
|
|
|||
|
|
@ -898,6 +898,11 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
|
|||
def redacts_messages_itself(self) -> bool:
|
||||
return False
|
||||
|
||||
def exempt_from_message_redaction(self) -> bool:
|
||||
"""Router pre-call checks that key routing off the request and never log it return True, so
|
||||
`turn_off_message_logging` leaves the success event they receive unredacted."""
|
||||
return False
|
||||
|
||||
def redact_standard_logging_payload_from_model_call_details(self, model_call_details: dict) -> dict:
|
||||
"""
|
||||
Redacts or excludes fields from StandardLoggingPayload before callbacks receive it.
|
||||
|
|
|
|||
|
|
@ -71,12 +71,12 @@ if TYPE_CHECKING:
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
_TERMINAL_STATUSES: Final = frozenset(
|
||||
{"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action"}
|
||||
{"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action", "requires_continuation"}
|
||||
)
|
||||
|
||||
_POLLABLE_STATUSES: Final = frozenset({"in_progress", "queued"})
|
||||
|
||||
_STATUSES_THAT_PRODUCED_OUTPUT: Final = frozenset({"completed", "requires_action"})
|
||||
_STATUSES_THAT_PRODUCED_OUTPUT: Final = frozenset({"completed", "requires_action", "requires_continuation"})
|
||||
|
||||
SettlementOutcome: TypeAlias = Literal["billed", "released", "unsettled"]
|
||||
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@ async def acreate(
|
|||
response_mime_type: str | None = None,
|
||||
# Continuation
|
||||
previous_interaction_id: str | None = None,
|
||||
continuation_token: str | None = None,
|
||||
# Extra params
|
||||
extra_headers: dict[str, str] | None = None,
|
||||
extra_body: Mapping[str, object] | None = None,
|
||||
|
|
@ -129,6 +130,7 @@ async def acreate(
|
|||
response_format: JSON schema for response format
|
||||
response_mime_type: MIME type of the response
|
||||
previous_interaction_id: ID of previous interaction for continuation
|
||||
continuation_token: Token from a `requires_continuation` interaction to resume its long decode
|
||||
extra_headers: Additional headers
|
||||
extra_body: Additional body parameters
|
||||
timeout: Request timeout
|
||||
|
|
@ -164,6 +166,7 @@ async def acreate(
|
|||
response_format=response_format,
|
||||
response_mime_type=response_mime_type,
|
||||
previous_interaction_id=previous_interaction_id,
|
||||
continuation_token=continuation_token,
|
||||
extra_headers=extra_headers,
|
||||
extra_body=extra_body,
|
||||
timeout=timeout,
|
||||
|
|
@ -224,6 +227,7 @@ def create(
|
|||
response_mime_type: str | None = None,
|
||||
# Continuation
|
||||
previous_interaction_id: str | None = None,
|
||||
continuation_token: str | None = None,
|
||||
# Extra params
|
||||
extra_headers: dict[str, str] | None = None,
|
||||
extra_body: Mapping[str, object] | None = None,
|
||||
|
|
@ -262,6 +266,7 @@ def create(
|
|||
response_format: JSON schema for response format
|
||||
response_mime_type: MIME type of the response
|
||||
previous_interaction_id: ID of previous interaction for continuation
|
||||
continuation_token: Token from a `requires_continuation` interaction to resume its long decode
|
||||
extra_headers: Additional headers
|
||||
extra_body: Additional body parameters
|
||||
timeout: Request timeout
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ INTERACTIONS_API_OPTIONAL_PARAMS: Final = {
|
|||
"response_format",
|
||||
"response_mime_type",
|
||||
"previous_interaction_id",
|
||||
"continuation_token",
|
||||
"agent_config",
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,12 +2,10 @@ import posixpath
|
|||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from itertools import accumulate, repeat
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from urllib.parse import quote, unquote
|
||||
|
||||
from litellm._uuid import uuid
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
|
||||
VERTEX_AI_MANAGED_GCS_PREFIX: Final = "litellm-vertex-files/"
|
||||
BEDROCK_MANAGED_S3_BATCH_PREFIX: Final = "litellm-bedrock-files-"
|
||||
|
|
@ -19,7 +17,6 @@ BEDROCK_MANAGED_S3_PREFIXES: Final = (
|
|||
BEDROCK_MANAGED_S3_OUTPUT_PREFIX,
|
||||
)
|
||||
MANAGED_CLOUD_STORAGE_SCHEMES: Final = ("s3://", "gs://")
|
||||
_MAPPING_PROXY_TYPE: Final[type] = type(MappingProxyType({}))
|
||||
|
||||
|
||||
MAX_FILE_ID_DECODE_PASSES: Final = 8
|
||||
|
|
@ -138,13 +135,7 @@ def encode_s3_object_key_for_url(object_key: str) -> str:
|
|||
def should_allow_legacy_cloud_file_ids(
|
||||
litellm_params: Mapping[str, object] | None = None,
|
||||
) -> bool:
|
||||
value = None
|
||||
if isinstance(litellm_params, Mapping):
|
||||
trusted_model_credentials: Final = litellm_params.get("_litellm_internal_model_credentials")
|
||||
if isinstance(trusted_model_credentials, _MAPPING_PROXY_TYPE):
|
||||
trusted_mapping: Final = as_str_mapping(trusted_model_credentials)
|
||||
value = trusted_mapping.get("allow_legacy_cloud_file_ids") if trusted_mapping is not None else None
|
||||
|
||||
value: Final = litellm_params.get("allow_legacy_cloud_file_ids") if litellm_params is not None else None
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import reprlib
|
||||
from collections.abc import Mapping, MutableMapping
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, fields
|
||||
from types import MappingProxyType
|
||||
from typing import Final, cast
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
|
@ -61,6 +60,9 @@ OPTIONAL_KWARGS_KEYS: Final = (
|
|||
"client_side_timeout",
|
||||
"gcs_bucket_name",
|
||||
"bucket_name",
|
||||
"s3_bucket_name",
|
||||
"s3_output_bucket_name",
|
||||
"allow_legacy_cloud_file_ids",
|
||||
"s3_endpoint_url",
|
||||
"s3_region_name",
|
||||
"s3_access_key_id",
|
||||
|
|
@ -292,19 +294,3 @@ def get_litellm_params(
|
|||
litellm_params[key] = kwargs[key]
|
||||
|
||||
return litellm_params
|
||||
|
||||
|
||||
def add_trusted_model_credentials_to_litellm_params(
|
||||
litellm_params_dict: MutableMapping[str, object], kwargs: Mapping[str, object]
|
||||
) -> None:
|
||||
"""
|
||||
Carry the immutable server-side credential snapshot into litellm_params.
|
||||
|
||||
get_litellm_params has a fixed signature, so callers that need the snapshot to
|
||||
survive into the logging object and the downstream file read have to re-add it. Only
|
||||
a MappingProxyType is accepted, since providers resolve trusted configuration such
|
||||
as a Bedrock file bucket from it and must not read a request-supplied mapping.
|
||||
"""
|
||||
trusted_model_credentials: Final = kwargs.get("_litellm_internal_model_credentials")
|
||||
if isinstance(trusted_model_credentials, MappingProxyType):
|
||||
litellm_params_dict["_litellm_internal_model_credentials"] = trusted_model_credentials
|
||||
|
|
|
|||
|
|
@ -4440,7 +4440,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if isinstance(callback, CustomLogger): # custom logger class
|
||||
from litellm.integrations.shadow_eval_logger import ShadowEvalLogger
|
||||
|
||||
model_call_details: dict = self.model_call_details
|
||||
model_call_details: dict = (
|
||||
self._model_call_details_with_request_messages()
|
||||
if callback.exempt_from_message_redaction()
|
||||
else self.model_call_details
|
||||
)
|
||||
##################################
|
||||
# call redaction hook for custom logger
|
||||
model_call_details = callback.redact_standard_logging_payload_from_model_call_details(
|
||||
|
|
@ -5004,6 +5008,21 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
) -> None:
|
||||
executor.submit(self.success_handler, result, start_time, end_time, cache_hit)
|
||||
|
||||
def _model_call_details_with_request_messages(self) -> dict[str, object]: # mutable-ok: loggers take a dict
|
||||
"""The success-event payload with the request messages put back where `turn_off_message_logging`
|
||||
redacted them in place, for a callback that routes on the request and never logs it."""
|
||||
request_messages: Final = cast(object, self.messages) # cast-ok: the request messages are only put back as-is
|
||||
standard_logging_object: Final = self.model_call_details.get("standard_logging_object")
|
||||
return {
|
||||
**self.model_call_details,
|
||||
"messages": request_messages,
|
||||
**(
|
||||
{"standard_logging_object": {**standard_logging_object, "messages": request_messages}}
|
||||
if isinstance(standard_logging_object, dict)
|
||||
else {}
|
||||
),
|
||||
}
|
||||
|
||||
def _get_completed_success_callbacks(
|
||||
self, *, asynchronous: bool
|
||||
) -> tuple[str | Callable[..., object] | CustomLogger, ...]:
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ from litellm.constants import (
|
|||
VLLM_CHOICE_FIELDS,
|
||||
)
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
extract_reasoning_content,
|
||||
)
|
||||
from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
|
||||
|
|
@ -1072,9 +1072,10 @@ def convert_to_model_response_object(
|
|||
if isinstance(e, APIError):
|
||||
raise
|
||||
|
||||
hide_reply: Final = response_type == "audio_transcription"
|
||||
received_args: Final = dict(
|
||||
response_object=response_object,
|
||||
model_response_object=model_response_object,
|
||||
response_object="<hidden>" if hide_reply else response_object,
|
||||
model_response_object="<hidden>" if hide_reply else model_response_object,
|
||||
response_type=response_type,
|
||||
stream=stream,
|
||||
start_time=start_time,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -4291,7 +4291,7 @@ def _convert_to_bedrock_tool_call_invoke(
|
|||
- extract name
|
||||
- extract id
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import (
|
||||
split_concatenated_json_objects,
|
||||
)
|
||||
|
||||
|
|
|
|||
635
litellm/litellm_core_utils/prompt_templates/file_id_utils.py
Normal file
635
litellm/litellm_core_utils/prompt_templates/file_id_utils.py
Normal file
|
|
@ -0,0 +1,635 @@
|
|||
import io
|
||||
import mimetypes
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from os import PathLike
|
||||
from pathlib import Path
|
||||
from typing import Final, cast # noqa: TID251 # Existing casts retained unchanged in this pure move
|
||||
|
||||
from litellm import verbose_logger
|
||||
from litellm.litellm_core_utils.batch_utils import InMemoryFile
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionFileObject,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
ExtractedFileData,
|
||||
FileTypes,
|
||||
SpecialEnums,
|
||||
)
|
||||
|
||||
|
||||
def get_format_from_file_id(file_id: str | None) -> str | None:
|
||||
"""
|
||||
Gets format from file id
|
||||
|
||||
unified_file_id = litellm_proxy:{};unified_id,{}
|
||||
If not a unified file id, returns 'file' as default format
|
||||
"""
|
||||
from litellm.proxy.openai_files_endpoints.common_utils import (
|
||||
convert_b64_uid_to_unified_uid,
|
||||
)
|
||||
|
||||
if not file_id:
|
||||
return None
|
||||
try:
|
||||
transformed_file_id: Final = convert_b64_uid_to_unified_uid(file_id)
|
||||
if transformed_file_id.startswith(SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value):
|
||||
match: Final = re.match(
|
||||
f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}:(.*?);unified_id",
|
||||
transformed_file_id,
|
||||
)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def update_messages_with_model_file_ids(
|
||||
messages: list[AllMessageValues],
|
||||
model_id: str | None,
|
||||
model_file_id_mapping: dict[str, dict[str, str]],
|
||||
) -> list[AllMessageValues]:
|
||||
"""
|
||||
Updates messages with model file ids.
|
||||
|
||||
For managed files (unified file IDs), uses model_file_id_mapping if it
|
||||
resolves the id, otherwise decodes the base64-encoded unified file ID
|
||||
and extracts the llm_output_file_id directly. Mirrors the Responses-API
|
||||
sibling `update_responses_input_with_model_file_ids`.
|
||||
|
||||
model_file_id_mapping: Dict[str, Dict[str, str]] = {
|
||||
"litellm_proxy/file_id": {
|
||||
"model_id": "provider_file_id"
|
||||
}
|
||||
}
|
||||
"""
|
||||
from litellm.proxy.openai_files_endpoints.common_utils import (
|
||||
convert_b64_uid_to_unified_uid,
|
||||
get_original_file_id,
|
||||
is_base64_encoded_unified_file_id,
|
||||
is_model_embedded_id,
|
||||
)
|
||||
|
||||
for message in messages:
|
||||
if message.get("role") == "user":
|
||||
content = message.get("content")
|
||||
if content:
|
||||
if isinstance(content, str):
|
||||
continue
|
||||
for c in content:
|
||||
if not isinstance(c, dict):
|
||||
# Content list items aren't always dicts. e.g.
|
||||
# text_completion forwards a token-ids list/list-of-
|
||||
# lists through this path. Skip non-dict items
|
||||
# instead of indexing into them.
|
||||
continue
|
||||
if c.get("type") == "file":
|
||||
file_object = cast(ChatCompletionFileObject, c)
|
||||
file_object_file_field = file_object.get("file")
|
||||
if not isinstance(file_object_file_field, dict):
|
||||
# Content block has `type: "file"` but not the
|
||||
# OpenAI Chat Completions shape (e.g. a LangChain
|
||||
# v1 standardized file block, or a provider-native
|
||||
# shape that also uses `type: "file"`). Nothing to
|
||||
# remap here, so skip instead of crashing.
|
||||
continue
|
||||
file_id = file_object_file_field.get("file_id")
|
||||
format = file_object_file_field.get("format", get_format_from_file_id(file_id))
|
||||
|
||||
if file_id:
|
||||
provider_file_id = (
|
||||
model_file_id_mapping.get(file_id, {}).get(model_id)
|
||||
if model_file_id_mapping and model_id is not None
|
||||
else None
|
||||
)
|
||||
if not provider_file_id and is_base64_encoded_unified_file_id(file_id):
|
||||
unified_file_id = convert_b64_uid_to_unified_uid(file_id)
|
||||
if "llm_output_file_id," in unified_file_id:
|
||||
provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
|
||||
if not provider_file_id and is_model_embedded_id(file_id):
|
||||
provider_file_id = get_original_file_id(file_id)
|
||||
file_object_file_field["file_id"] = provider_file_id or file_id
|
||||
if format:
|
||||
file_object_file_field["format"] = format
|
||||
return messages
|
||||
|
||||
|
||||
def update_responses_input_with_model_file_ids(
|
||||
input: object,
|
||||
model_id: str | None = None,
|
||||
model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None,
|
||||
) -> object:
|
||||
"""
|
||||
Updates responses API input with provider-specific file IDs.
|
||||
File IDs are always inside the content array, not as direct input_file items.
|
||||
|
||||
For managed files (unified file IDs), uses model_file_id_mapping if provided,
|
||||
otherwise decodes the base64-encoded unified file ID and extracts the llm_output_file_id directly.
|
||||
|
||||
Args:
|
||||
input: The responses API input parameter
|
||||
model_id: The model ID to use for looking up provider-specific file IDs
|
||||
model_file_id_mapping: Dictionary mapping litellm file IDs to provider file IDs
|
||||
Format: {"litellm_file_id": {"model_id": "provider_file_id"}}
|
||||
"""
|
||||
from litellm.proxy.openai_files_endpoints.common_utils import (
|
||||
convert_b64_uid_to_unified_uid,
|
||||
get_original_file_id,
|
||||
is_base64_encoded_unified_file_id,
|
||||
is_model_embedded_id,
|
||||
)
|
||||
|
||||
if isinstance(input, str):
|
||||
return input
|
||||
|
||||
if not isinstance(input, list):
|
||||
return input
|
||||
|
||||
updated_input: Final = []
|
||||
for item in input:
|
||||
if not isinstance(item, dict):
|
||||
updated_input.append(item)
|
||||
continue
|
||||
|
||||
updated_item = item.copy()
|
||||
content = item.get("content")
|
||||
if isinstance(content, list):
|
||||
updated_content = []
|
||||
for content_item in content:
|
||||
if isinstance(content_item, dict) and content_item.get("type") == "input_file":
|
||||
file_id = content_item.get("file_id")
|
||||
if isinstance(file_id, str) and file_id:
|
||||
provider_file_id = file_id # Default to original
|
||||
|
||||
# Check if we have a mapping for this file ID
|
||||
if model_file_id_mapping and model_id and file_id in model_file_id_mapping:
|
||||
# Use the model-specific file ID from mapping
|
||||
provider_file_id = model_file_id_mapping.get(file_id, {}).get(model_id) or file_id
|
||||
updated_content_item = content_item.copy()
|
||||
updated_content_item["file_id"] = provider_file_id
|
||||
updated_content.append(updated_content_item)
|
||||
else:
|
||||
# Check if this is a base64-encoded unified file ID without mapping
|
||||
is_unified_file_id = is_base64_encoded_unified_file_id(file_id)
|
||||
if is_unified_file_id:
|
||||
# Fallback: decode unified file ID
|
||||
unified_file_id = convert_b64_uid_to_unified_uid(file_id)
|
||||
if "llm_output_file_id," in unified_file_id:
|
||||
provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
|
||||
|
||||
updated_content_item = content_item.copy()
|
||||
updated_content_item["file_id"] = provider_file_id
|
||||
updated_content.append(updated_content_item)
|
||||
elif is_model_embedded_id(file_id):
|
||||
updated_content_item = content_item.copy()
|
||||
updated_content_item["file_id"] = get_original_file_id(file_id)
|
||||
updated_content.append(updated_content_item)
|
||||
else:
|
||||
# Not a managed file, keep as-is
|
||||
updated_content.append(content_item)
|
||||
else:
|
||||
updated_content.append(content_item)
|
||||
else:
|
||||
updated_content.append(content_item)
|
||||
updated_item["content"] = updated_content
|
||||
|
||||
updated_input.append(updated_item)
|
||||
|
||||
return updated_input
|
||||
|
||||
|
||||
def _decode_vector_store_ids_in_tools(
|
||||
tools: list[dict[str, object]] | None,
|
||||
) -> list[dict[str, object]] | None:
|
||||
"""
|
||||
Decodes unified (LiteLLM-managed) vector_store_ids in file_search tools to
|
||||
provider-native IDs. Non-unified IDs are passed through unchanged.
|
||||
|
||||
This runs unconditionally — no file-ID mapping is required.
|
||||
"""
|
||||
if not tools or not isinstance(tools, list):
|
||||
return tools
|
||||
|
||||
from litellm.llms.base_llm.managed_resources.utils import (
|
||||
is_base64_encoded_unified_id,
|
||||
parse_unified_id,
|
||||
)
|
||||
|
||||
updated_tools: Final = []
|
||||
for tool in tools:
|
||||
if not isinstance(tool, dict) or tool.get("type") != "file_search":
|
||||
updated_tools.append(tool)
|
||||
continue
|
||||
|
||||
vector_store_ids = tool.get("vector_store_ids")
|
||||
if not isinstance(vector_store_ids, list):
|
||||
updated_tools.append(tool)
|
||||
continue
|
||||
|
||||
decoded_ids = []
|
||||
for vs_id in vector_store_ids:
|
||||
if not isinstance(vs_id, str) or not is_base64_encoded_unified_id(vs_id):
|
||||
decoded_ids.append(vs_id)
|
||||
continue
|
||||
|
||||
parsed = parse_unified_id(vs_id)
|
||||
provider_resource_id = parsed.get("provider_resource_id") if parsed else None
|
||||
|
||||
if not provider_resource_id:
|
||||
verbose_logger.warning(
|
||||
"file_search tool contains unified vector_store_id '%s' that could "
|
||||
"not be decoded to a provider resource ID — passing original ID. "
|
||||
"Ensure the vector store was created via LiteLLM.",
|
||||
vs_id,
|
||||
)
|
||||
decoded_ids.append(vs_id)
|
||||
else:
|
||||
decoded_ids.append(provider_resource_id)
|
||||
|
||||
updated_tools.append({**tool, "vector_store_ids": decoded_ids})
|
||||
|
||||
return updated_tools
|
||||
|
||||
|
||||
def update_responses_tools_with_model_file_ids(
|
||||
tools: list[dict[str, object]] | None,
|
||||
model_id: str | None = None,
|
||||
model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None,
|
||||
) -> list[dict[str, object]] | None:
|
||||
"""
|
||||
Updates responses API tools with provider-specific file IDs.
|
||||
|
||||
Pass 1 (always): decode unified vector_store_ids in file_search tools.
|
||||
Pass 2 (needs mapping): map code_interpreter container file_ids to provider IDs.
|
||||
|
||||
Args:
|
||||
tools: The responses API tools parameter
|
||||
model_id: The model ID to use for looking up provider-specific file IDs
|
||||
model_file_id_mapping: Dictionary mapping litellm file IDs to provider file IDs
|
||||
Format: {"litellm_file_id": {"model_id": "provider_file_id"}}
|
||||
"""
|
||||
if not tools or not isinstance(tools, list):
|
||||
return tools
|
||||
|
||||
# Pass 1: decode unified vector_store_ids (no mapping needed)
|
||||
tools = _decode_vector_store_ids_in_tools(tools) or tools
|
||||
|
||||
# Pass 2: map code_interpreter file IDs (requires mapping)
|
||||
if not model_file_id_mapping or not model_id:
|
||||
return tools
|
||||
|
||||
updated_tools: Final = []
|
||||
for tool in tools:
|
||||
if not isinstance(tool, dict):
|
||||
updated_tools.append(tool)
|
||||
continue
|
||||
|
||||
updated_tool = tool.copy()
|
||||
|
||||
# Handle code_interpreter with container file_ids
|
||||
if tool.get("type") == "code_interpreter":
|
||||
container = tool.get("container")
|
||||
if isinstance(container, dict):
|
||||
container_file_ids = container.get("file_ids")
|
||||
if isinstance(container_file_ids, list):
|
||||
updated_file_ids = []
|
||||
for file_id in container_file_ids:
|
||||
if isinstance(file_id, str):
|
||||
# Check if we have a mapping for this file ID
|
||||
if file_id in model_file_id_mapping:
|
||||
# Map to provider-specific file ID
|
||||
provider_file_id = model_file_id_mapping.get(file_id, {}).get(model_id) or file_id
|
||||
updated_file_ids.append(provider_file_id)
|
||||
else:
|
||||
updated_file_ids.append(file_id)
|
||||
else:
|
||||
updated_file_ids.append(file_id)
|
||||
|
||||
# Update the tool with new file IDs
|
||||
updated_container = container.copy()
|
||||
updated_container["file_ids"] = updated_file_ids
|
||||
updated_tool["container"] = updated_container
|
||||
|
||||
updated_tools.append(updated_tool)
|
||||
|
||||
return updated_tools
|
||||
|
||||
|
||||
def extract_file_metadata(file_data: FileTypes) -> tuple[str | None, str | None]:
|
||||
"""
|
||||
Resolve (filename, content_type) without reading the file body.
|
||||
|
||||
Mirrors extract_file_data's metadata resolution but never calls .read(), so
|
||||
it stays O(1) on large uploads. Use this when only metadata is needed (batch
|
||||
detection, GCS object naming) and the body must remain a streamable Path/handle.
|
||||
"""
|
||||
filename: str | None = None
|
||||
content_type: str | None = None
|
||||
file_content: object = None
|
||||
|
||||
if isinstance(file_data, tuple):
|
||||
if len(file_data) == 2:
|
||||
filename, file_content = file_data
|
||||
elif len(file_data) == 3:
|
||||
filename, file_content, content_type = file_data
|
||||
elif len(file_data) == 4:
|
||||
filename, file_content, content_type, _ = file_data
|
||||
elif isinstance(file_data, InMemoryFile):
|
||||
filename = file_data.name
|
||||
content_type = file_data.content_type
|
||||
else:
|
||||
file_content = file_data
|
||||
|
||||
if filename is None:
|
||||
if isinstance(file_content, PathLike):
|
||||
filename = Path(file_content).name
|
||||
elif isinstance(file_content, io.IOBase):
|
||||
name_attr: Final = getattr(file_content, "name", None)
|
||||
if isinstance(name_attr, str):
|
||||
filename = Path(name_attr).name
|
||||
|
||||
if not content_type:
|
||||
guessed: Final = mimetypes.guess_type(filename)[0] if filename else None
|
||||
content_type = guessed or "application/octet-stream"
|
||||
|
||||
return filename, content_type
|
||||
|
||||
|
||||
def extract_file_data(file_data: FileTypes) -> ExtractedFileData:
|
||||
"""
|
||||
Extracts and processes file data from various input formats.
|
||||
|
||||
Args:
|
||||
file_data: Can be a tuple of (filename, content, [content_type], [headers]) or direct file content
|
||||
|
||||
Returns:
|
||||
ExtractedFileData containing:
|
||||
- filename: Name of the file if provided
|
||||
- content: The file content in bytes
|
||||
- content_type: MIME type of the file
|
||||
- headers: Any additional headers
|
||||
"""
|
||||
# Parse the file_data based on its type
|
||||
filename = None
|
||||
file_content = None
|
||||
content_type = None
|
||||
file_headers: Mapping[str, str] = {}
|
||||
|
||||
if isinstance(file_data, tuple):
|
||||
if len(file_data) == 2:
|
||||
filename, file_content = file_data
|
||||
elif len(file_data) == 3:
|
||||
filename, file_content, content_type = file_data
|
||||
elif len(file_data) == 4:
|
||||
filename, file_content, content_type, file_headers = file_data
|
||||
elif isinstance(file_data, InMemoryFile):
|
||||
filename = file_data.name
|
||||
file_content = file_data
|
||||
content_type = file_data.content_type
|
||||
else:
|
||||
file_content = file_data
|
||||
# Convert content to bytes
|
||||
if isinstance(file_content, str):
|
||||
# Bare string inputs are rejected: when this helper runs in a proxy
|
||||
# request handler the string came from an attacker-controlled form
|
||||
# field, and opening it as a path is an arbitrary file read on the
|
||||
# proxy host. SDK callers who want to upload from a path should
|
||||
# either pass a pathlib.Path (a PathLike instance — see the branch
|
||||
# below) or open the file themselves and pass the handle / bytes.
|
||||
raise ValueError(
|
||||
"extract_file_data does not accept bare str inputs. Pass bytes, "
|
||||
"an open file handle, a (filename, content) tuple, or a "
|
||||
"pathlib.Path. To upload a local file from a path, call "
|
||||
"open(path, 'rb') yourself."
|
||||
)
|
||||
if isinstance(file_content, PathLike):
|
||||
# PathLike (pathlib.Path) is a Python-level type that HTTP form
|
||||
# values can't fabricate. Treat as a local file path for SDK
|
||||
# convenience.
|
||||
if filename is None:
|
||||
filename = Path(file_content).name
|
||||
with open(file_content, "rb") as f:
|
||||
content = f.read()
|
||||
elif isinstance(file_content, io.IOBase):
|
||||
# If it's a file-like object
|
||||
# Try to get filename from file handle if not already set
|
||||
if not filename and hasattr(file_content, "name"):
|
||||
filename = Path(file_content.name).name
|
||||
|
||||
content = file_content.read()
|
||||
|
||||
if isinstance(content, str):
|
||||
content = content.encode("utf-8")
|
||||
# Reset file pointer to beginning
|
||||
file_content.seek(0)
|
||||
elif isinstance(file_content, bytes):
|
||||
content = file_content
|
||||
else:
|
||||
raise ValueError(f"Unsupported file content type: {type(file_content)}")
|
||||
|
||||
# Use provided content type or guess based on filename
|
||||
if not content_type:
|
||||
if filename:
|
||||
guessed_type: Final = mimetypes.guess_type(filename)[0]
|
||||
content_type = guessed_type if guessed_type else "application/octet-stream"
|
||||
else:
|
||||
content_type = "application/octet-stream"
|
||||
|
||||
return ExtractedFileData(
|
||||
filename=filename,
|
||||
content=content,
|
||||
content_type=content_type,
|
||||
headers=file_headers,
|
||||
)
|
||||
|
||||
|
||||
def get_image_mime_type_from_url(url: str) -> str | None:
|
||||
"""
|
||||
Get mime type for common image URLs
|
||||
See gemini mime types: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/image-understanding#image-requirements
|
||||
|
||||
Supported by Gemini:
|
||||
application/pdf
|
||||
audio/mpeg
|
||||
audio/mp3
|
||||
audio/wav
|
||||
audio/ogg
|
||||
image/png
|
||||
image/jpeg
|
||||
image/webp
|
||||
text/plain
|
||||
video/mov
|
||||
video/mpeg
|
||||
video/mp4
|
||||
video/mpg
|
||||
video/avi
|
||||
video/wmv
|
||||
video/mpegps
|
||||
video/flv
|
||||
"""
|
||||
from urllib.parse import urlparse
|
||||
|
||||
url = url.lower()
|
||||
|
||||
# Parse URL to extract path without query parameters
|
||||
# This handles URLs like: https://example.com/image.jpg?signature=...
|
||||
parsed: Final = urlparse(url)
|
||||
path: Final = parsed.path
|
||||
|
||||
# Map file extensions to mime types
|
||||
mime_types: Final = {
|
||||
# Images
|
||||
(".jpg", ".jpeg"): "image/jpeg",
|
||||
(".png",): "image/png",
|
||||
(".webp",): "image/webp",
|
||||
# Videos
|
||||
(".mp4",): "video/mp4",
|
||||
(".mov",): "video/mov",
|
||||
(".mpeg", ".mpg"): "video/mpeg",
|
||||
(".avi",): "video/avi",
|
||||
(".wmv",): "video/wmv",
|
||||
(".mpegps",): "video/mpegps",
|
||||
(".flv",): "video/flv",
|
||||
# Audio
|
||||
(".mp3",): "audio/mp3",
|
||||
(".wav",): "audio/wav",
|
||||
(".mpeg",): "audio/mpeg",
|
||||
(".ogg",): "audio/ogg",
|
||||
# Documents
|
||||
(".pdf",): "application/pdf",
|
||||
(".txt",): "text/plain",
|
||||
}
|
||||
|
||||
# Check each extension group against the URL
|
||||
for extensions, mime_type in mime_types.items():
|
||||
if any(path.endswith(ext) for ext in extensions):
|
||||
return mime_type
|
||||
|
||||
return None
|
||||
|
||||
|
||||
_get_image_mime_type_from_url = get_image_mime_type_from_url
|
||||
|
||||
|
||||
def infer_content_type_from_url_and_content(
|
||||
url: str,
|
||||
content: bytes,
|
||||
current_content_type: str | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Infer content type from URL extension and binary content when content-type header is missing or generic.
|
||||
|
||||
This helper implements a fallback strategy for determining MIME types when HTTP headers
|
||||
are missing or provide generic values (like binary/octet-stream). It's commonly used
|
||||
when processing images and documents from various sources (S3, URLs, etc.).
|
||||
|
||||
Fallback Strategy:
|
||||
1. If current_content_type is valid (not None and not generic octet-stream), return it
|
||||
2. Try to infer from URL extension (handles query parameters)
|
||||
3. Try to detect from binary content signature (magic bytes)
|
||||
4. Raise ValueError if all methods fail
|
||||
|
||||
Args:
|
||||
url: The URL of the content (used to extract file extension)
|
||||
content: The binary content (first ~100 bytes are sufficient for detection)
|
||||
current_content_type: The current content-type from headers (may be None or generic)
|
||||
|
||||
Returns:
|
||||
str: The inferred MIME type (e.g., "image/png", "application/pdf")
|
||||
|
||||
Raises:
|
||||
ValueError: If content type cannot be determined by any method
|
||||
|
||||
Example:
|
||||
>>> content_type = infer_content_type_from_url_and_content(
|
||||
... url="https://s3.amazonaws.com/bucket/image.png?AWSAccessKeyId=123",
|
||||
... content=png_binary_data,
|
||||
... current_content_type="binary/octet-stream"
|
||||
... )
|
||||
>>> print(content_type)
|
||||
"image/png"
|
||||
"""
|
||||
from litellm.litellm_core_utils.token_counter import get_image_type
|
||||
|
||||
media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None
|
||||
|
||||
# If we have a valid content type that's not generic, use it
|
||||
if media_type and media_type not in [
|
||||
"binary/octet-stream",
|
||||
"application/octet-stream",
|
||||
]:
|
||||
return media_type
|
||||
|
||||
# Extension to MIME type mapping
|
||||
# Supports images, documents, and other common file types
|
||||
extension_to_mime: Final = {
|
||||
# Image formats
|
||||
"jpg": "image/jpeg",
|
||||
"jpeg": "image/jpeg",
|
||||
"png": "image/png",
|
||||
"gif": "image/gif",
|
||||
"webp": "image/webp",
|
||||
# Document formats
|
||||
"pdf": "application/pdf",
|
||||
"csv": "text/csv",
|
||||
"doc": "application/msword",
|
||||
"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"xls": "application/vnd.ms-excel",
|
||||
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"html": "text/html",
|
||||
"txt": "text/plain",
|
||||
"md": "text/markdown",
|
||||
}
|
||||
|
||||
# Try to infer from URL extension
|
||||
if url:
|
||||
extension: Final = url.split(".")[-1].lower().split("?")[0] # Remove query params
|
||||
inferred_type: Final = extension_to_mime.get(extension)
|
||||
if inferred_type:
|
||||
return inferred_type
|
||||
|
||||
# Try to detect from binary content signature (magic bytes)
|
||||
if content:
|
||||
detected_type: Final = get_image_type(content[:100])
|
||||
if detected_type:
|
||||
type_to_mime: Final = {
|
||||
"png": "image/png",
|
||||
"jpeg": "image/jpeg",
|
||||
"gif": "image/gif",
|
||||
"webp": "image/webp",
|
||||
"heic": "image/heic",
|
||||
}
|
||||
if detected_type in type_to_mime:
|
||||
return type_to_mime[detected_type]
|
||||
|
||||
# If all fallbacks failed, raise error
|
||||
raise ValueError(f"Unable to determine content type from URL: {url}. Response content-type: {current_content_type}")
|
||||
|
||||
|
||||
def get_file_ids_from_messages(messages: list[AllMessageValues]) -> list[str]:
|
||||
"""
|
||||
Gets file ids from messages
|
||||
"""
|
||||
file_ids: Final = []
|
||||
for message in messages:
|
||||
if message.get("role") == "user":
|
||||
content = message.get("content")
|
||||
if content:
|
||||
if isinstance(content, str):
|
||||
continue
|
||||
for c in content:
|
||||
if isinstance(c, dict) and c["type"] == "file":
|
||||
file_object = cast(ChatCompletionFileObject, c)
|
||||
file_object_file_field = file_object.get("file")
|
||||
if not isinstance(file_object_file_field, dict):
|
||||
# Content block has `type: "file"` but not the
|
||||
# OpenAI Chat Completions shape. No file_id to
|
||||
# extract, so skip instead of raising KeyError.
|
||||
continue
|
||||
file_id = file_object_file_field.get("file_id")
|
||||
if file_id:
|
||||
file_ids.append(file_id)
|
||||
return file_ids
|
||||
|
|
@ -21,7 +21,9 @@ import litellm
|
|||
from litellm import verbose_logger
|
||||
from litellm.caching.caching import InMemoryCache
|
||||
from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB, NON_LLM_CONNECTION_TIMEOUT
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import infer_content_type_from_url_and_content
|
||||
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
|
||||
infer_content_type_from_url_and_content,
|
||||
)
|
||||
from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
|
|
|
|||
471
litellm/litellm_core_utils/prompt_templates/json_schema_utils.py
Normal file
471
litellm/litellm_core_utils/prompt_templates/json_schema_utils.py
Normal file
|
|
@ -0,0 +1,471 @@
|
|||
import re
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from graphlib import CycleError, TopologicalSorter
|
||||
from itertools import chain, islice
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
_TOP_LEVEL_SCHEMA_COMBINATORS: Final = ("allOf", "anyOf", "oneOf")
|
||||
_OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS: Final = ("enum", "const", "not")
|
||||
_LOCAL_SCHEMA_REF_PREFIXES: Final = (("#/$defs/", "$defs"), ("#/definitions/", "definitions"))
|
||||
_MAX_SCHEMA_FLATTEN_DEPTH: Final = 32
|
||||
_EMPTY_SCHEMA: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _schema_properties(schema: Mapping[str, object]) -> Mapping[str, object]:
|
||||
properties: Final = schema.get("properties")
|
||||
return properties if isinstance(properties, dict) else _EMPTY_SCHEMA
|
||||
|
||||
|
||||
def _schema_branches(schema: Mapping[str, object], combinator: str) -> tuple[object, ...]:
|
||||
branches: Final = schema.get(combinator)
|
||||
return tuple(branches) if isinstance(branches, list) else ()
|
||||
|
||||
|
||||
def _schema_required_names(schema: Mapping[str, object]) -> frozenset[str]:
|
||||
required: Final = schema.get("required")
|
||||
if not isinstance(required, list):
|
||||
return frozenset()
|
||||
return frozenset(name for name in required if isinstance(name, str))
|
||||
|
||||
|
||||
def _combinator_required_names(combinator: str, branches: tuple[Mapping[str, object], ...]) -> frozenset[str]:
|
||||
branch_names: Final = tuple(_schema_required_names(branch) for branch in branches)
|
||||
if not branch_names:
|
||||
return frozenset()
|
||||
if combinator == "allOf":
|
||||
return branch_names[0].union(*branch_names[1:])
|
||||
return branch_names[0].intersection(*branch_names[1:])
|
||||
|
||||
|
||||
def _resolve_local_schema_ref(root: Mapping[str, object], ref: str) -> Mapping[str, object] | None:
|
||||
matched: Final = next(
|
||||
((prefix, container) for prefix, container in _LOCAL_SCHEMA_REF_PREFIXES if ref.startswith(prefix)),
|
||||
None,
|
||||
)
|
||||
if matched is None:
|
||||
return None
|
||||
prefix, container = matched
|
||||
definitions: Final = root.get(container)
|
||||
if not isinstance(definitions, dict):
|
||||
return None
|
||||
target: Final = definitions.get(ref[len(prefix) :])
|
||||
return target if isinstance(target, dict) else None
|
||||
|
||||
|
||||
def _mergeable_branch(
|
||||
root: Mapping[str, object],
|
||||
branch: object,
|
||||
seen_refs: frozenset[str],
|
||||
depth: int,
|
||||
expanded_refs: dict[str, Mapping[str, object] | None], # mutable-ok: per-call memo bounding repeated $ref work
|
||||
) -> Mapping[str, object] | None:
|
||||
if not isinstance(branch, dict) or depth > _MAX_SCHEMA_FLATTEN_DEPTH:
|
||||
return None
|
||||
ref: Final = branch.get("$ref")
|
||||
if not isinstance(ref, str):
|
||||
flattened: Final = _flatten_schema_against_root(branch, root, seen_refs, depth, expanded_refs)
|
||||
if any(combinator in flattened for combinator in _TOP_LEVEL_SCHEMA_COMBINATORS):
|
||||
return None
|
||||
return flattened
|
||||
if ref in expanded_refs:
|
||||
return expanded_refs[ref]
|
||||
if ref in seen_refs:
|
||||
return None
|
||||
target: Final = _resolve_local_schema_ref(root, ref)
|
||||
expanded: Final = (
|
||||
None
|
||||
if target is None
|
||||
else _mergeable_branch(root, target, seen_refs | frozenset((ref,)), depth + 1, expanded_refs)
|
||||
)
|
||||
expanded_refs[ref] = expanded
|
||||
return expanded
|
||||
|
||||
|
||||
def _is_object_schema(schema: Mapping[str, object]) -> bool:
|
||||
return schema.get("type") == "object" or ("type" not in schema and "properties" in schema)
|
||||
|
||||
|
||||
def _flatten_schema_against_root(
|
||||
schema: Mapping[str, object],
|
||||
root: Mapping[str, object],
|
||||
seen_refs: frozenset[str],
|
||||
depth: int,
|
||||
expanded_refs: dict[str, Mapping[str, object] | None], # mutable-ok: per-call memo bounding repeated $ref work
|
||||
) -> Mapping[str, object]:
|
||||
raw_branch_groups: Final = tuple(
|
||||
(
|
||||
combinator,
|
||||
tuple(
|
||||
_mergeable_branch(root, branch, seen_refs, depth + 1, expanded_refs)
|
||||
for branch in _schema_branches(schema, combinator)
|
||||
),
|
||||
)
|
||||
for combinator in _TOP_LEVEL_SCHEMA_COMBINATORS
|
||||
if isinstance(schema.get(combinator), list)
|
||||
)
|
||||
dropped: Final = (
|
||||
*(combinator for combinator, _ in raw_branch_groups),
|
||||
*(key for key in _OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS if key in schema),
|
||||
)
|
||||
if not dropped:
|
||||
return schema
|
||||
|
||||
if any(branch is None for _, group in raw_branch_groups for branch in group):
|
||||
return schema
|
||||
branch_groups: Final = tuple(
|
||||
(combinator, tuple(branch for branch in group if branch is not None)) for combinator, group in raw_branch_groups
|
||||
)
|
||||
branches: Final = tuple(branch for _, group in branch_groups for branch in group)
|
||||
is_object_schema: Final = _is_object_schema(schema) or (
|
||||
"type" not in schema and branches != () and all(_is_object_schema(branch) for branch in branches)
|
||||
)
|
||||
if not is_object_schema:
|
||||
return schema
|
||||
|
||||
merged_properties: Final = {
|
||||
name: value for source in (*reversed(branches), schema) for name, value in _schema_properties(source).items()
|
||||
}
|
||||
required_names: Final = _schema_required_names(schema).union(
|
||||
*(_combinator_required_names(combinator, group) for combinator, group in branch_groups)
|
||||
)
|
||||
kept: Final = MappingProxyType({key: value for key, value in schema.items() if key not in dropped})
|
||||
required_update: Final = MappingProxyType({"required": sorted(required_names)}) if required_names else _EMPTY_SCHEMA
|
||||
return {
|
||||
**kept,
|
||||
"type": "object",
|
||||
"properties": merged_properties,
|
||||
**required_update,
|
||||
}
|
||||
|
||||
|
||||
def flatten_top_level_schema_combinators(schema: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""Merge top-level ``allOf``/``anyOf``/``oneOf`` branches into an object tool schema.
|
||||
|
||||
OpenAI's function-calling validator rejects tool ``parameters`` carrying
|
||||
'oneOf'/'anyOf'/'allOf'/'enum'/'const'/'not' at the top level (nested uses
|
||||
are accepted), while lenient backends such as the ChatGPT backend Codex
|
||||
talks to natively accept them, so an MCP tool declaring a top-level union
|
||||
400s through LiteLLM. Branch properties merge without clobbering (the
|
||||
top-level schema wins, then earlier branches); ``required`` becomes the
|
||||
top-level list plus the intersection of the branch lists for anyOf/oneOf
|
||||
or their union for allOf. Branches that are local ``$ref``s
|
||||
(``#/$defs/...`` or ``#/definitions/...``) are resolved first, each ref
|
||||
at most once per call, and branches that are themselves combinators are
|
||||
flattened recursively up to a fixed depth; a branch that cannot be fully
|
||||
merged (a boolean schema, an external or cyclic ``$ref``, a non-object
|
||||
union, or nesting past the depth cap) leaves the whole schema untouched so
|
||||
OpenAI's own validation still applies. Non-object schemas pass through
|
||||
unchanged and the input is never mutated.
|
||||
"""
|
||||
return _flatten_schema_against_root(schema, schema, frozenset(), 0, {})
|
||||
|
||||
|
||||
_SUBSCHEMA_KEYWORDS: Final = frozenset(
|
||||
{
|
||||
"additionalItems",
|
||||
"additionalProperties",
|
||||
"contains",
|
||||
"else",
|
||||
"if",
|
||||
"items",
|
||||
"not",
|
||||
"propertyNames",
|
||||
"then",
|
||||
"unevaluatedItems",
|
||||
"unevaluatedProperties",
|
||||
}
|
||||
)
|
||||
_SUBSCHEMA_LIST_KEYWORDS: Final = frozenset({"allOf", "anyOf", "items", "oneOf", "prefixItems"})
|
||||
_SUBSCHEMA_MAP_KEYWORDS: Final = frozenset(
|
||||
{"$defs", "definitions", "dependentSchemas", "patternProperties", "properties"}
|
||||
)
|
||||
|
||||
_MAX_SCHEMA_NESTING: Final = 1024
|
||||
|
||||
|
||||
def drop_non_python_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""Drop every regex in a schema position that Python's ``re`` cannot compile.
|
||||
|
||||
OpenAI validates tool ``parameters`` against the 2020-12 metaschema with
|
||||
``jsonschema``'s format checker, which hands each ``pattern`` value and each
|
||||
``patternProperties`` key to ``re.compile``, so a regex written for an
|
||||
ECMA-262 engine (Unicode property escapes such as ``\\p{Cc}``, as in Claude
|
||||
Code's ``Artifact`` tool) is refused with "'...' is not a 'regex'" by every
|
||||
model family on both the chat and Responses wires. Only schema positions are
|
||||
walked (properties, items, combinators, ``$defs`` and the other applicators),
|
||||
so a ``pattern`` key inside ``default``, ``examples``, ``const`` or vendor
|
||||
extensions is data and stays. Outside strict mode the keyword is only a
|
||||
hint, so dropping it costs the model a constraint and the caller nothing.
|
||||
Compilable regexes and everything else pass through, the input is never
|
||||
mutated, and the same object comes back when nothing was dropped. The walk
|
||||
is level-order rather than recursive, rebuilt deepest level first, and stops
|
||||
at more schema levels than a JSON parser admits, so a cyclic schema built in
|
||||
code cannot spin it.
|
||||
"""
|
||||
return _schema_without_rejected_regex(schema, _is_not_python_regex)
|
||||
|
||||
|
||||
def drop_lookaround_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""Drop every regex in a schema position that uses a lookaround assertion.
|
||||
|
||||
Some Bedrock Converse families compile tool schema regexes with an engine that
|
||||
has no lookahead or lookbehind and refuse the whole request over one. The ``(?=``,
|
||||
``(?!``, ``(?<=`` and ``(?<!`` openers are matched textually, so an escaped literal
|
||||
that spells one is dropped too, trading a hint for a request that goes through.
|
||||
A ``patternProperties`` key dropped from an object closed by ``additionalProperties:
|
||||
false`` leaves its value schema as that object's ``additionalProperties``, so the
|
||||
names it allowed stay allowed; :func:`drop_non_python_regex_patterns` shares the walk.
|
||||
"""
|
||||
return _schema_without_rejected_regex(schema, _uses_regex_lookaround)
|
||||
|
||||
|
||||
def _schema_without_rejected_regex(
|
||||
schema: Mapping[str, object], rejected: Callable[[str], bool]
|
||||
) -> Mapping[str, object]:
|
||||
rebuilt: dict[int, Mapping[str, object]] = {} # mutable-ok: per-call memo of rewritten nodes, deepest level first
|
||||
for level in reversed(tuple(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING))):
|
||||
rebuilt.update(
|
||||
(id(node), rewritten)
|
||||
for node in level
|
||||
if (rewritten := _node_without_rejected_regex(node, rebuilt, rejected)) is not node
|
||||
)
|
||||
return rebuilt.get(id(schema), schema)
|
||||
|
||||
|
||||
def _schema_levels(schema: Mapping[str, object]) -> Iterator[tuple[Mapping[str, object], ...]]:
|
||||
frontier: tuple[Mapping[str, object], ...] = (schema,) # rebind-ok: level-order cursor, one level a round
|
||||
while frontier:
|
||||
yield frontier
|
||||
frontier = tuple(child for node in frontier for child in _subschemas(node))
|
||||
|
||||
|
||||
def schema_fragment_ref(ref: str) -> str:
|
||||
return f"#{ref}" if ref.startswith(("/$defs/", "/definitions/")) else ref
|
||||
|
||||
|
||||
def schema_has_unresolvable_ref(schema: Mapping[str, object]) -> bool:
|
||||
definitions: Final = MappingProxyType(dict(_root_definitions(schema)))
|
||||
if not _refs_in(schema) <= definitions.keys():
|
||||
return True
|
||||
try:
|
||||
tuple(TopologicalSorter({pointer: _refs_in(body) for pointer, body in definitions.items()}).static_order())
|
||||
except CycleError:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def schema_union_property_count(schema: Mapping[str, object]) -> int:
|
||||
return sum(1 for _, node in _property_schemas(schema) if _is_union(node))
|
||||
|
||||
|
||||
def schema_optional_property_count(schema: Mapping[str, object]) -> int:
|
||||
return sum(len(_optional_property_names(node)) for node in _schema_nodes(schema))
|
||||
|
||||
|
||||
def schema_property_count(schema: Mapping[str, object]) -> int:
|
||||
return sum(len(properties) for node in _schema_nodes(schema) if (properties := _schema_at(node, "properties")))
|
||||
|
||||
|
||||
def schema_grammar_branch_count(schema: Mapping[str, object]) -> int:
|
||||
return sum(_grammar_branches(node) for node in _schema_nodes(schema))
|
||||
|
||||
|
||||
def schema_has_open_object(schema: Mapping[str, object]) -> bool:
|
||||
return any(_is_open_object(node) for node in _schema_nodes(schema))
|
||||
|
||||
|
||||
def schema_has_lookaround_pattern(schema: Mapping[str, object]) -> bool:
|
||||
return any(
|
||||
isinstance(pattern := node.get("pattern"), str) and _uses_regex_lookaround(pattern)
|
||||
for node in _schema_nodes(schema)
|
||||
)
|
||||
|
||||
|
||||
def schema_string_formats(schema: Mapping[str, object]) -> frozenset[str]:
|
||||
return frozenset(
|
||||
value for node in _schema_nodes(schema) if _may_be_string(node) and isinstance(value := node.get("format"), str)
|
||||
)
|
||||
|
||||
|
||||
def _may_be_string(node: Mapping[str, object]) -> bool:
|
||||
node_type: Final = node.get("type")
|
||||
return node_type is None or node_type == "string" or "string" in _list_at(node, "type")
|
||||
|
||||
|
||||
def is_object_schema(node: Mapping[str, object]) -> bool:
|
||||
node_type: Final = node.get("type")
|
||||
return node_type == "object" or (isinstance(node_type, list) and "object" in node_type)
|
||||
|
||||
|
||||
def _schema_nodes(schema: Mapping[str, object]) -> Iterator[Mapping[str, object]]:
|
||||
return chain.from_iterable(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING))
|
||||
|
||||
|
||||
_SCHEMA_LIST: Final = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
def _list_at(container: Mapping[str, object], key: str) -> tuple[object, ...]:
|
||||
value: Final = container.get(key)
|
||||
return _SCHEMA_LIST.validate_python(value) if isinstance(value, list) else ()
|
||||
|
||||
|
||||
def _named_schemas(container: Mapping[str, object] | None) -> Iterator[tuple[str, Mapping[str, object]]]:
|
||||
if container is None:
|
||||
return iter(())
|
||||
return ((key, value) for key in container if (value := _schema_at(container, key)) is not None)
|
||||
|
||||
|
||||
def _root_definitions(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]:
|
||||
for defs_key in ("$defs", "definitions"):
|
||||
yield from ((f"#/{defs_key}/{name}", body) for name, body in _named_schemas(_schema_at(schema, defs_key)))
|
||||
|
||||
|
||||
def _refs_in(schema: Mapping[str, object]) -> frozenset[str]:
|
||||
return frozenset(
|
||||
schema_fragment_ref(ref) for node in _schema_nodes(schema) if isinstance(ref := node.get("$ref"), str)
|
||||
)
|
||||
|
||||
|
||||
def _property_schemas(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]:
|
||||
return chain.from_iterable(_named_schemas(_schema_at(node, "properties")) for node in _schema_nodes(schema))
|
||||
|
||||
|
||||
def _is_union(node: Mapping[str, object]) -> bool:
|
||||
return isinstance(node.get("anyOf"), list) or isinstance(node.get("oneOf"), list) or len(_list_at(node, "type")) > 1
|
||||
|
||||
|
||||
def _grammar_branches(node: Mapping[str, object]) -> int:
|
||||
properties: Final = _schema_at(node, "properties")
|
||||
if properties is None:
|
||||
return 0
|
||||
optional: Final = len(_optional_property_names(node))
|
||||
return (1 << optional) * (len(properties) - optional + 1)
|
||||
|
||||
|
||||
def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]:
|
||||
properties: Final = _schema_at(node, "properties")
|
||||
if properties is None:
|
||||
return frozenset()
|
||||
return frozenset(properties) - frozenset(name for name in _list_at(node, "required") if isinstance(name, str))
|
||||
|
||||
|
||||
def _is_open_object(node: Mapping[str, object]) -> bool:
|
||||
additional_properties: Final = node.get("additionalProperties")
|
||||
if additional_properties is True or isinstance(additional_properties, dict) or "patternProperties" in node:
|
||||
return True
|
||||
return is_object_schema(node) and "properties" not in node and additional_properties is not False
|
||||
|
||||
|
||||
def _subschemas(node: Mapping[str, object]) -> Iterator[Mapping[str, object]]:
|
||||
for key, value in node.items():
|
||||
if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict):
|
||||
yield from (sub for sub in value.values() if isinstance(sub, dict))
|
||||
elif key in _SUBSCHEMA_LIST_KEYWORDS and isinstance(value, list):
|
||||
yield from (sub for sub in value if isinstance(sub, dict))
|
||||
elif key in _SUBSCHEMA_KEYWORDS and isinstance(value, dict):
|
||||
yield value
|
||||
|
||||
|
||||
def _node_without_rejected_regex(
|
||||
node: Mapping[str, object],
|
||||
rebuilt: Mapping[int, Mapping[str, object]],
|
||||
rejected: Callable[[str], bool],
|
||||
) -> Mapping[str, object]:
|
||||
kept: Final = {
|
||||
key: _keyword_value_rebuilt(key, value, rebuilt, rejected)
|
||||
for key, value in node.items()
|
||||
if key != "pattern" or not isinstance(value, str) or not rejected(value)
|
||||
}
|
||||
if len(kept) == len(node) and all(kept[key] is node[key] for key in kept):
|
||||
return node
|
||||
dropped_pattern_properties: Final = _dropped_pattern_properties(node, kept, rebuilt)
|
||||
if not dropped_pattern_properties or kept.get("additionalProperties") is not False:
|
||||
return kept
|
||||
return {**kept, "additionalProperties": _any_of(dropped_pattern_properties)}
|
||||
|
||||
|
||||
def _dropped_pattern_properties(
|
||||
node: Mapping[str, object],
|
||||
kept: Mapping[str, object],
|
||||
rebuilt: Mapping[int, Mapping[str, object]],
|
||||
) -> tuple[object, ...]:
|
||||
before: Final = _schema_at(node, "patternProperties")
|
||||
after: Final = _schema_at(kept, "patternProperties")
|
||||
if before is None or after is None:
|
||||
return ()
|
||||
return tuple(rebuilt.get(id(sub), sub) for name, sub in before.items() if name not in after)
|
||||
|
||||
|
||||
def _schema_at(container: Mapping[str, object], key: str) -> Mapping[str, object] | None:
|
||||
value: Final = container.get(key)
|
||||
return value if isinstance(value, dict) else None
|
||||
|
||||
|
||||
def _any_of(schemas: tuple[object, ...]) -> object:
|
||||
return schemas[0] if len(schemas) == 1 else {"anyOf": list(schemas)}
|
||||
|
||||
|
||||
def _keyword_value_rebuilt(
|
||||
key: str,
|
||||
value: object,
|
||||
rebuilt: Mapping[int, Mapping[str, object]],
|
||||
rejected: Callable[[str], bool],
|
||||
) -> object:
|
||||
if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict):
|
||||
kept: Final = {
|
||||
name: rebuilt.get(id(sub), sub)
|
||||
for name, sub in value.items()
|
||||
if key != "patternProperties" or not isinstance(name, str) or not rejected(name)
|
||||
}
|
||||
return value if len(kept) == len(value) and all(kept[name] is value[name] for name in kept) else kept
|
||||
if key in _SUBSCHEMA_LIST_KEYWORDS and isinstance(value, list):
|
||||
items: Final = [rebuilt.get(id(sub), sub) for sub in value]
|
||||
return value if all(new is old for new, old in zip(items, value, strict=True)) else items
|
||||
if key in _SUBSCHEMA_KEYWORDS and isinstance(value, dict):
|
||||
return rebuilt.get(id(value), value)
|
||||
return value
|
||||
|
||||
|
||||
def _is_not_python_regex(pattern: str) -> bool:
|
||||
try:
|
||||
re.compile(pattern)
|
||||
except (re.error, RecursionError):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
_REGEX_LOOKAROUND_RE: Final = re.compile(r"\(\?<?[=!]")
|
||||
|
||||
|
||||
def _uses_regex_lookaround(pattern: str) -> bool:
|
||||
return _REGEX_LOOKAROUND_RE.search(pattern) is not None
|
||||
|
||||
|
||||
def flatten_combinators_and_drop_non_python_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return flatten_top_level_schema_combinators(drop_non_python_regex_patterns(schema))
|
||||
|
||||
|
||||
def tool_with_sanitized_parameters(
|
||||
tool: Mapping[str, object],
|
||||
sanitize: Callable[[Mapping[str, object]], Mapping[str, object]],
|
||||
) -> Mapping[str, object]:
|
||||
"""Run the tool's JSON schema through ``sanitize``: ``function.parameters`` on an
|
||||
OpenAI tool, ``input_schema`` on an Anthropic one. The same object comes back when
|
||||
nothing changed."""
|
||||
function: Final = _schema_at(tool, "function")
|
||||
if function is not None:
|
||||
parameters: Final = _schema_at(function, "parameters")
|
||||
if parameters is None:
|
||||
return tool
|
||||
sanitized_parameters: Final = sanitize(parameters)
|
||||
if sanitized_parameters is parameters:
|
||||
return tool
|
||||
return {**tool, "function": {**function, "parameters": sanitized_parameters}}
|
||||
input_schema: Final = _schema_at(tool, "input_schema")
|
||||
if input_schema is None:
|
||||
return tool
|
||||
sanitized_schema: Final = sanitize(input_schema)
|
||||
return tool if sanitized_schema is input_schema else {**tool, "input_schema": sanitized_schema}
|
||||
|
|
@ -0,0 +1,212 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final, TypeVar, cast # noqa: TID251 # Existing casts retained unchanged in this pure move
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm import verbose_logger
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.types.llms.openai import (
|
||||
ChatCompletionCachedContent,
|
||||
ChatCompletionPromptCacheBreakpoint,
|
||||
PromptCacheBreakpoint,
|
||||
)
|
||||
|
||||
_MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object])
|
||||
|
||||
|
||||
PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT: Final = (
|
||||
"set `drop_params: true` on the request or in litellm_settings (`litellm.drop_params = True` in the SDK)"
|
||||
)
|
||||
|
||||
|
||||
def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
|
||||
"""A breakpoint the client already wrote on the block wins, TTL included."""
|
||||
if marker is None or target.get("prompt_cache_breakpoint") is not None:
|
||||
return target
|
||||
marked: Final = {**target, "prompt_cache_breakpoint": marker}
|
||||
return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key
|
||||
|
||||
|
||||
_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint)
|
||||
|
||||
|
||||
_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(ChatCompletionPromptCacheBreakpoint)
|
||||
|
||||
|
||||
_AcceptedBreakpointT: Final = TypeVar("_AcceptedBreakpointT", bound=Mapping[str, object])
|
||||
|
||||
|
||||
def _accepted_prompt_cache_breakpoint(
|
||||
accepted: TypeAdapter[_AcceptedBreakpointT], marker: object
|
||||
) -> _AcceptedBreakpointT | None:
|
||||
try:
|
||||
return accepted.validate_python(marker)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def prompt_cache_breakpoint_within(
|
||||
accepted: TypeAdapter[_AcceptedBreakpointT], marker: object
|
||||
) -> _AcceptedBreakpointT | None:
|
||||
"""The client's marker cut down to the ``accepted`` shape: an unsupported ttl is dropped and the mode kept,
|
||||
and a marker whose mode the target cannot read is dropped whole."""
|
||||
whole: Final = _accepted_prompt_cache_breakpoint(accepted, marker)
|
||||
if whole is not None:
|
||||
return whole
|
||||
fields: Final = _breakpoint_fields_or_none(marker)
|
||||
without_ttl: Final = (
|
||||
None
|
||||
if fields is None
|
||||
else _accepted_prompt_cache_breakpoint(accepted, {key: item for key, item in fields.items() if key != "ttl"})
|
||||
)
|
||||
verbose_logger.debug(
|
||||
"Dropping malformed prompt_cache_breakpoint %r under drop_params"
|
||||
if without_ttl is None
|
||||
else "Dropping the unsupported ttl of prompt_cache_breakpoint %r under drop_params",
|
||||
marker,
|
||||
)
|
||||
return without_ttl
|
||||
|
||||
|
||||
def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
|
||||
if marker is None or not drop_params:
|
||||
return marker
|
||||
return prompt_cache_breakpoint_within(_PROMPT_CACHE_BREAKPOINT, marker)
|
||||
|
||||
|
||||
def prompt_cache_breakpoint_with_readable_mode(marker: object, drop_params: bool) -> object:
|
||||
"""The client's marker as written, or None under drop_params when no target could read its mode, so the
|
||||
target's own config decides which ttl and keys it keeps."""
|
||||
return None if prompt_cache_breakpoint_for_wire(marker, drop_params) is None else marker
|
||||
|
||||
|
||||
def whole_prompt_cache_breakpoint_or_none(marker: object) -> PromptCacheBreakpoint | None:
|
||||
return _accepted_prompt_cache_breakpoint(_PROMPT_CACHE_BREAKPOINT, marker)
|
||||
|
||||
|
||||
def chat_completion_prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
|
||||
if marker is None or not drop_params:
|
||||
return marker
|
||||
return prompt_cache_breakpoint_within(_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT, marker)
|
||||
|
||||
|
||||
def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
|
||||
"""Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block."""
|
||||
if block.get("cache_control") is None:
|
||||
return block
|
||||
unmarked: Final = {key: value for key, value in block.items() if key != "cache_control"}
|
||||
return cast( # cast-ok: same block shape as the input with the marker key renamed
|
||||
_MarkedT, with_prompt_cache_breakpoint(unmarked, PromptCacheBreakpoint(mode="explicit"))
|
||||
)
|
||||
|
||||
|
||||
def prompt_cache_breakpoint_as_cache_control(block: _MarkedT, drop_params: bool) -> _MarkedT:
|
||||
"""OpenAI's ``prompt_cache_breakpoint`` mark rewritten as Anthropic's ``cache_control`` on the same block.
|
||||
A ``cache_control`` the client already wrote on the block wins, TTL included."""
|
||||
marker: Final = block.get("prompt_cache_breakpoint")
|
||||
if marker is None:
|
||||
return block
|
||||
unmarked: Final = {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"}
|
||||
control: Final = (
|
||||
None
|
||||
if block.get("cache_control") is not None
|
||||
else _cache_control_for_prompt_cache_breakpoint(marker, drop_params)
|
||||
)
|
||||
marked: Final = unmarked if control is None else {**unmarked, "cache_control": control}
|
||||
return cast(_MarkedT, marked) # cast-ok: same block shape as the input with the marker key renamed
|
||||
|
||||
|
||||
def _cache_control_for_prompt_cache_breakpoint(marker: object, drop_params: bool) -> ChatCompletionCachedContent | None:
|
||||
try:
|
||||
breakpoint_mark: Final = _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
|
||||
except ValidationError:
|
||||
if not drop_params:
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"prompt_cache_breakpoint must be {{'mode': 'explicit'}} with an optional ttl of '30m' or '12h', got "
|
||||
f"{marker!r}. Fix the marker, or "
|
||||
f"{PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it."
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
|
||||
return None
|
||||
ttl: Final = breakpoint_mark.get("ttl")
|
||||
if ttl is not None and not drop_params:
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"This model's prompt cache lasts 5m, or 1h where it offers one, so prompt_cache_breakpoint "
|
||||
f"ttl={ttl!r} has no equivalent. Remove the ttl, {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to send "
|
||||
"a 5m breakpoint instead, or send cache_control {'type': 'ephemeral', 'ttl': '1h'} on the block "
|
||||
"where the model offers a 1h cache."
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
return ChatCompletionCachedContent(type="ephemeral")
|
||||
|
||||
|
||||
_BREAKPOINT_BLOCK_LIST_KEYS: Final = frozenset({"content", "tool_calls"})
|
||||
|
||||
|
||||
_BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
_BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object])
|
||||
|
||||
|
||||
def _breakpoint_fields_or_none(value: object) -> Mapping[str, object] | None:
|
||||
try:
|
||||
return _BREAKPOINT_FIELDS.validate_python(value)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _without_prompt_cache_breakpoint(value: object) -> object:
|
||||
if not isinstance(value, Mapping):
|
||||
return value
|
||||
fields: Final = _BREAKPOINT_FIELDS.validate_python(value)
|
||||
return {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"}
|
||||
|
||||
|
||||
def _breakpoints_stripped_from(key: str, value: object) -> object:
|
||||
if key in _BREAKPOINT_BLOCK_LIST_KEYS and isinstance(value, list):
|
||||
return [_without_prompt_cache_breakpoint(block) for block in _BREAKPOINT_BLOCKS.validate_python(value)]
|
||||
return _without_prompt_cache_breakpoint(value) if key == "function" else value
|
||||
|
||||
|
||||
def without_prompt_cache_breakpoints(block: _MarkedT) -> _MarkedT:
|
||||
"""A message or tool definition with every ``prompt_cache_breakpoint`` removed: its own, its content parts',
|
||||
its tool calls' and its ``function``'s."""
|
||||
stripped: Final = {
|
||||
key: _breakpoints_stripped_from(key, value) for key, value in block.items() if key != "prompt_cache_breakpoint"
|
||||
}
|
||||
return cast(_MarkedT, stripped) # cast-ok: same block shape as the input minus the marker keys
|
||||
|
||||
|
||||
def _part_with_chat_completion_prompt_cache_breakpoint(part: object) -> object:
|
||||
if not isinstance(part, Mapping):
|
||||
return part
|
||||
fields: Final = _BREAKPOINT_FIELDS.validate_python(part)
|
||||
marker: Final = fields.get("prompt_cache_breakpoint")
|
||||
if marker is None:
|
||||
return fields
|
||||
unmarked: Final = {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"}
|
||||
return with_prompt_cache_breakpoint(unmarked, chat_completion_prompt_cache_breakpoint_for_wire(marker, True))
|
||||
|
||||
|
||||
def chat_completion_prompt_cache_breakpoints_for_wire(message: _MarkedT, drop_params: bool) -> _MarkedT:
|
||||
content: Final = message.get("content")
|
||||
if not drop_params or not isinstance(content, list):
|
||||
return message
|
||||
parts: Final = [
|
||||
_part_with_chat_completion_prompt_cache_breakpoint(part) for part in _BREAKPOINT_BLOCKS.validate_python(content)
|
||||
]
|
||||
return cast(_MarkedT, {**message, "content": parts}) # cast-ok: same message shape with its parts' markers trimmed
|
||||
|
||||
|
||||
LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS: Final = frozenset(
|
||||
{"thinking_blocks", "reasoning_items", "provider_specific_fields"}
|
||||
)
|
||||
|
||||
|
||||
LITELLM_INTERNAL_MESSAGE_FIELDS: Final = LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS | {"reasoning_content"}
|
||||
274
litellm/litellm_core_utils/prompt_templates/reasoning_utils.py
Normal file
274
litellm/litellm_core_utils/prompt_templates/reasoning_utils.py
Normal file
|
|
@ -0,0 +1,274 @@
|
|||
import base64
|
||||
import binascii
|
||||
import re
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from itertools import groupby
|
||||
from typing import Final, cast # noqa: TID251 # Existing casts retained unchanged in this pure move
|
||||
|
||||
from litellm.types.llms.openai import (
|
||||
ChatCompletionReasoningItem,
|
||||
ChatCompletionReasoningSummaryTextBlock,
|
||||
ChatCompletionRedactedThinkingBlock,
|
||||
ChatCompletionThinkingBlock,
|
||||
)
|
||||
|
||||
|
||||
def _provider_text_or_none(value: object) -> str | None:
|
||||
return cast(str | None, value) # cast-ok: reasoning fields arrive in untyped provider messages
|
||||
|
||||
|
||||
def extract_reasoning_content(message: Mapping[str, object]) -> tuple[str | None, str | None]:
|
||||
"""
|
||||
Extract reasoning content and main content from a message.
|
||||
|
||||
Args:
|
||||
message (dict): The message dictionary that may contain reasoning_content
|
||||
|
||||
Returns:
|
||||
tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content)
|
||||
"""
|
||||
message_content: Final = message.get("content")
|
||||
if "reasoning_content" in message:
|
||||
return _provider_text_or_none(message["reasoning_content"]), _provider_text_or_none(message_content)
|
||||
elif "reasoning" in message:
|
||||
return _provider_text_or_none(message["reasoning"]), _provider_text_or_none(message_content)
|
||||
elif isinstance(message_content, str):
|
||||
return parse_content_for_reasoning(message_content)
|
||||
return None, _provider_text_or_none(message_content)
|
||||
|
||||
|
||||
_extract_reasoning_content = extract_reasoning_content
|
||||
|
||||
|
||||
def _readable_thinking_text(block: Mapping[str, object]) -> str:
|
||||
"""The text a chat model can read back, empty for redacted blocks and malformed ones."""
|
||||
if block.get("type") != "thinking":
|
||||
return ""
|
||||
return str(block.get("thinking") or "")
|
||||
|
||||
|
||||
def reasoning_content_from_thinking_blocks(
|
||||
thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
|
||||
) -> str:
|
||||
"""Flatten Anthropic thinking blocks into the `reasoning_content` string chat models expect.
|
||||
|
||||
Redacted blocks carry no readable text, so they contribute nothing.
|
||||
"""
|
||||
return "\n".join(text for block in thinking_blocks if (text := _readable_thinking_text(block)))
|
||||
|
||||
|
||||
ENCRYPTED_REASONING_SIGNATURE_PREFIX: Final = "litellm_encrypted_reasoning:"
|
||||
|
||||
|
||||
def encrypted_reasoning_signature(encrypted_content: str) -> str:
|
||||
"""The opaque value a Responses API reasoning item's `encrypted_content` travels in.
|
||||
|
||||
Anthropic clients echo a thinking block's `signature` and a redacted block's `data`
|
||||
back verbatim, so either field can carry the encrypted reasoning across turns; the
|
||||
prefix tells the two apart from a signature Anthropic minted.
|
||||
"""
|
||||
return f"{ENCRYPTED_REASONING_SIGNATURE_PREFIX}{encrypted_content}"
|
||||
|
||||
|
||||
def _carries_encrypted_reasoning(signature: object) -> bool:
|
||||
return isinstance(signature, str) and signature.startswith(ENCRYPTED_REASONING_SIGNATURE_PREFIX)
|
||||
|
||||
|
||||
def encrypted_content_from_signature(signature: object) -> str | None:
|
||||
if not isinstance(signature, str) or not _carries_encrypted_reasoning(signature):
|
||||
return None
|
||||
return signature.removeprefix(ENCRYPTED_REASONING_SIGNATURE_PREFIX) or None
|
||||
|
||||
|
||||
def _encrypted_reasoning_field(block: Mapping[str, object]) -> object:
|
||||
match block.get("type"):
|
||||
case "thinking":
|
||||
return block.get("signature")
|
||||
case "redacted_thinking":
|
||||
return block.get("data")
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def encrypted_content_of_block(block: Mapping[str, object]) -> str | None:
|
||||
return encrypted_content_from_signature(_encrypted_reasoning_field(block))
|
||||
|
||||
|
||||
def is_encrypted_reasoning_block(block: object) -> bool:
|
||||
"""A thinking or redacted_thinking block carrying Responses API encrypted reasoning.
|
||||
|
||||
Only the Responses API that minted the content can read it back, so an Anthropic
|
||||
backend has to drop such a block rather than fail signature verification on it.
|
||||
"""
|
||||
if not isinstance(block, Mapping):
|
||||
return False
|
||||
mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance
|
||||
return _carries_encrypted_reasoning(_encrypted_reasoning_field(mapping))
|
||||
|
||||
|
||||
def is_unsignable_thinking_block(block: object) -> bool:
|
||||
"""A thinking block Anthropic cannot accept on input.
|
||||
|
||||
Anthropic verifies the signature cryptographically, so a block with a null,
|
||||
empty, or missing signature (e.g. from an open-source reasoning model) is
|
||||
rejected with a 400, and so is a block whose signature or data carries
|
||||
another provider's encrypted reasoning. A signed block is kept whatever its
|
||||
text, since ``display: omitted`` returns signed blocks with empty ``thinking``.
|
||||
`redacted_thinking` blocks carry no signature and are always kept.
|
||||
"""
|
||||
if is_encrypted_reasoning_block(block):
|
||||
return True
|
||||
if not isinstance(block, Mapping):
|
||||
return False
|
||||
mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance
|
||||
if mapping.get("type") != "thinking":
|
||||
return False
|
||||
signature: Final = mapping.get("signature")
|
||||
return not (isinstance(signature, str) and len(signature) > 0)
|
||||
|
||||
|
||||
def has_blank_redacted_data(block: object) -> bool:
|
||||
match block:
|
||||
case {"type": "redacted_thinking", "data": str() as data}:
|
||||
return not data.strip()
|
||||
case {"type": "redacted_thinking"}:
|
||||
return True
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def native_redacted_thinking_data(data: str) -> str:
|
||||
"""Claude's own `data` string for a redacted_thinking block.
|
||||
|
||||
Bedrock Converse carries it base64-wrapped as `redactedContent`, and LiteLLM passed that
|
||||
wrapper through as `data` until it started decoding Converse replies, so histories saved
|
||||
before then still hold it. Claude's own value never decodes twice, since its first byte
|
||||
decodes to a protobuf tag outside the base64 alphabet.
|
||||
"""
|
||||
return _converse_wrapped_payload(data) or data
|
||||
|
||||
|
||||
def converse_redacted_content(data: str) -> str:
|
||||
"""The `redactedContent` Bedrock Converse expects for a redacted_thinking block's `data`."""
|
||||
return base64.b64encode(native_redacted_thinking_data(data).encode()).decode()
|
||||
|
||||
|
||||
def _converse_wrapped_payload(data: str) -> str | None:
|
||||
try:
|
||||
payload: Final = base64.b64decode(data, validate=True).decode("ascii")
|
||||
_ = base64.b64decode(payload, validate=True)
|
||||
except (binascii.Error, ValueError):
|
||||
return None
|
||||
else:
|
||||
return payload
|
||||
|
||||
|
||||
def strip_encrypted_reasoning_from_messages(
|
||||
messages: object,
|
||||
*,
|
||||
should_strip: Callable[[Mapping[str, object]], bool] | None = None,
|
||||
) -> None:
|
||||
"""Drop the bridge-tagged reasoning blocks a routed deployment cannot decrypt from
|
||||
Anthropic-shaped history.
|
||||
|
||||
The whole block goes, the way #40280 drops undecryptable Responses ``input`` items: a
|
||||
provider that did not mint the block rejects it signed (a foreign signature) and unsigned
|
||||
(a missing signature) alike, so keeping its text as an unsigned thinking block only moves
|
||||
the 400 from the router to the provider.
|
||||
|
||||
Mutates the content lists in place: the router's fallback snapshot shares these
|
||||
message objects, so a rebound list would replay the stripped blocks on the fallback hop.
|
||||
"""
|
||||
if not isinstance(messages, list):
|
||||
return
|
||||
for content in anthropic_content_lists(cast(list[object], messages)): # cast-ok: untyped client json
|
||||
_strip_encrypted_reasoning_from_blocks(content, should_strip=should_strip)
|
||||
|
||||
|
||||
def anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]:
|
||||
return (
|
||||
cast(list[object], content) # cast-ok: narrowed by isinstance
|
||||
for message in messages
|
||||
if isinstance(message, Mapping)
|
||||
for content in (cast(Mapping[str, object], message).get("content"),) # cast-ok: narrowed by isinstance
|
||||
if isinstance(content, list)
|
||||
)
|
||||
|
||||
|
||||
def _strip_encrypted_reasoning_from_blocks(
|
||||
content: object,
|
||||
*,
|
||||
should_strip: Callable[[Mapping[str, object]], bool] | None = None,
|
||||
) -> None:
|
||||
blocks: Final = cast(list[object], content) # cast-ok: narrowed by the caller's isinstance
|
||||
kept: Final = tuple(
|
||||
block
|
||||
for block in blocks
|
||||
if not is_encrypted_reasoning_block(block)
|
||||
or (should_strip is not None and not should_strip(cast(Mapping[str, object], block)))
|
||||
)
|
||||
blocks[:] = kept
|
||||
|
||||
|
||||
def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str:
|
||||
index, block = indexed_block
|
||||
return f"encrypted:{index}" if is_encrypted_reasoning_block(block) else "summary"
|
||||
|
||||
|
||||
def _reasoning_item_from_block_group(group: tuple[Mapping[str, object], ...]) -> ChatCompletionReasoningItem | None:
|
||||
summary: Final[list[ChatCompletionReasoningSummaryTextBlock]] = [ # mutable-ok: API message payload
|
||||
ChatCompletionReasoningSummaryTextBlock(type="summary_text", text=text)
|
||||
for block in group
|
||||
if (text := _readable_thinking_text(block))
|
||||
]
|
||||
encrypted_content: Final = encrypted_content_of_block(group[0])
|
||||
if encrypted_content is not None:
|
||||
return ChatCompletionReasoningItem(type="reasoning", summary=summary, encrypted_content=encrypted_content)
|
||||
if not summary:
|
||||
return None
|
||||
return ChatCompletionReasoningItem(type="reasoning", summary=summary)
|
||||
|
||||
|
||||
def responses_reasoning_items_from_thinking_blocks(
|
||||
thinking_blocks: Iterable[Mapping[str, object]],
|
||||
) -> tuple[ChatCompletionReasoningItem, ...]:
|
||||
"""Build Responses API `reasoning` input items from Anthropic thinking blocks.
|
||||
|
||||
A block carrying encrypted reasoning replays the item it came from byte for byte;
|
||||
a run of plain thinking blocks collapses into one summary-only item. No item carries
|
||||
an `id`: the Responses API 404s on any id it did not mint itself and rejects an empty
|
||||
one, while an item without an id is always accepted.
|
||||
"""
|
||||
return tuple(
|
||||
item
|
||||
for _, group in groupby(enumerate(thinking_blocks), key=_reasoning_replay_group_key)
|
||||
if (item := _reasoning_item_from_block_group(tuple(block for _, block in group))) is not None
|
||||
)
|
||||
|
||||
|
||||
def parse_content_for_reasoning(
|
||||
message_text: str | None,
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""
|
||||
Parse the content for reasoning
|
||||
|
||||
Returns:
|
||||
- reasoning_content: The content of the reasoning
|
||||
- content: The content of the message
|
||||
"""
|
||||
if not message_text:
|
||||
return None, message_text
|
||||
|
||||
reasoning_match: Final = re.match(
|
||||
r"<(?:think|thinking|budget:thinking)>(.*?)</(?:think|thinking|budget:thinking)>(.*)",
|
||||
message_text,
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
if reasoning_match:
|
||||
return reasoning_match.group(1), reasoning_match.group(2)
|
||||
|
||||
return None, message_text
|
||||
|
||||
|
||||
_parse_content_for_reasoning = parse_content_for_reasoning
|
||||
|
|
@ -0,0 +1,252 @@
|
|||
import json
|
||||
from typing import Any, Final # noqa: TID251 # Existing annotations retained unchanged in this pure move
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def _attempt_json_repair(s: str) -> object | None:
|
||||
"""
|
||||
Attempt to repair truncated JSON produced by LLM tool calls.
|
||||
|
||||
Handles the most common truncation patterns where the model generates
|
||||
valid JSON that is cut short (missing closing brackets/braces).
|
||||
|
||||
Returns the parsed value on success, or None if repair fails.
|
||||
"""
|
||||
import json
|
||||
|
||||
stripped: Final = s.rstrip()
|
||||
if not stripped:
|
||||
return None
|
||||
|
||||
# Track the stack of unmatched openers to respect nesting order
|
||||
opener_stack: Final[list] = []
|
||||
in_string = False
|
||||
escape_next = False
|
||||
|
||||
for ch in stripped:
|
||||
if escape_next:
|
||||
escape_next = False
|
||||
continue
|
||||
if ch == "\\":
|
||||
if in_string:
|
||||
escape_next = True
|
||||
continue
|
||||
if ch == '"':
|
||||
in_string = not in_string
|
||||
continue
|
||||
if in_string:
|
||||
continue
|
||||
if ch == "{":
|
||||
opener_stack.append("}")
|
||||
elif ch == "[":
|
||||
opener_stack.append("]")
|
||||
elif ch in ("}", "]"):
|
||||
if opener_stack and opener_stack[-1] == ch:
|
||||
opener_stack.pop()
|
||||
|
||||
if not opener_stack:
|
||||
return None
|
||||
|
||||
# Remove trailing comma before we close brackets
|
||||
candidate = stripped.rstrip(",")
|
||||
|
||||
# Close in reverse order of opening (respects nesting)
|
||||
candidate += "".join(reversed(opener_stack))
|
||||
|
||||
try:
|
||||
return _DECODED_JSON.validate_python(json.loads(candidate))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def parse_tool_call_arguments(
|
||||
arguments: str | None,
|
||||
tool_name: str | None = None,
|
||||
context: str | None = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Parse tool call arguments from a JSON string.
|
||||
|
||||
When the JSON is malformed (e.g. truncated by the model), this function
|
||||
attempts a lightweight repair (closing unmatched brackets/braces) before
|
||||
raising an error. A warning is logged whenever repair succeeds so that
|
||||
callers are aware the arguments were not perfectly formed.
|
||||
|
||||
Args:
|
||||
arguments: The JSON string containing tool arguments, or None.
|
||||
tool_name: Optional name of the tool (for error messages).
|
||||
context: Optional context string (e.g., "Anthropic Messages API").
|
||||
|
||||
Returns:
|
||||
Parsed arguments (usually a dict, but may be any JSON-deserializable
|
||||
type such as list, str, int, float, or None). Returns empty dict if
|
||||
arguments is None or empty.
|
||||
|
||||
Raises:
|
||||
ValueError: If the arguments string is not valid JSON and cannot be repaired.
|
||||
"""
|
||||
import json
|
||||
|
||||
if not arguments or not arguments.strip():
|
||||
return {}
|
||||
|
||||
try:
|
||||
return _DECODED_JSON.validate_python(json.loads(arguments))
|
||||
except json.JSONDecodeError as original_error:
|
||||
repaired: Final = _attempt_json_repair(arguments)
|
||||
if repaired is not None:
|
||||
verbose_logger.warning(
|
||||
"Repaired truncated tool call arguments for tool '%s' (%s). Original (%d chars): %.200s%s",
|
||||
tool_name or "<unknown>",
|
||||
context or "unknown context",
|
||||
len(arguments),
|
||||
arguments,
|
||||
"..." if len(arguments) > 200 else "",
|
||||
)
|
||||
return repaired
|
||||
|
||||
error_parts: Final = ["Failed to parse tool call arguments"]
|
||||
|
||||
if tool_name:
|
||||
error_parts.append(f"for tool '{tool_name}'")
|
||||
if context:
|
||||
error_parts.append(f"({context})")
|
||||
|
||||
error_message: Final = " ".join(error_parts) + f". Error: {original_error}. Arguments: {arguments}"
|
||||
|
||||
raise ValueError(error_message) from original_error
|
||||
|
||||
|
||||
def split_concatenated_json_objects(raw: str) -> list[dict[str, object]]:
|
||||
"""
|
||||
Split a string that contains one or more concatenated JSON objects into
|
||||
a list of parsed dicts.
|
||||
|
||||
LLM providers (notably Bedrock Claude Sonnet 4.5) sometimes return
|
||||
multiple tool-call argument objects concatenated in a single
|
||||
``arguments`` string, e.g.::
|
||||
|
||||
'{"command":["curl",...]}{"command":["curl",...]}{"command":["curl",...]}'
|
||||
|
||||
``json.loads()`` fails on this with ``JSONDecodeError: Extra data``.
|
||||
This helper uses ``json.JSONDecoder.raw_decode()`` to walk the string
|
||||
and extract each JSON object individually.
|
||||
|
||||
The walk degrades gracefully: if the string is malformed or truncated
|
||||
(e.g. a stream that ended mid-tool-call), whatever complete objects were
|
||||
parsed before the bad tail are returned and the remainder is discarded
|
||||
with a warning, rather than raising. The sole caller
|
||||
(``_convert_to_bedrock_tool_call_invoke``) treats an empty result as
|
||||
``input={}`` so the conversation can continue instead of hard-failing.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[dict]
|
||||
A list of parsed dicts – one per JSON object found. If *raw* is
|
||||
empty, whitespace-only, or wholly unparseable, an empty list is
|
||||
returned.
|
||||
"""
|
||||
import json
|
||||
|
||||
raw = raw.strip()
|
||||
if not raw:
|
||||
return []
|
||||
|
||||
decoder: Final = json.JSONDecoder()
|
||||
results: Final[list[dict[str, object]]] = []
|
||||
idx = 0
|
||||
length: Final = len(raw)
|
||||
|
||||
while idx < length:
|
||||
# Skip whitespace between objects
|
||||
while idx < length and raw[idx] in " \t\n\r":
|
||||
idx += 1
|
||||
if idx >= length:
|
||||
break
|
||||
|
||||
try:
|
||||
decoded = decoder.raw_decode(raw, idx)
|
||||
except json.JSONDecodeError as e:
|
||||
verbose_logger.warning(
|
||||
"split_concatenated_json_objects: discarding unparseable tool-call "
|
||||
"arguments tail after %d complete object(s); decode_start=%d error=%s",
|
||||
len(results),
|
||||
idx,
|
||||
e,
|
||||
)
|
||||
break
|
||||
obj = as_str_mapping(_DECODED_JSON.validate_python(decoded[0]))
|
||||
if obj is not None:
|
||||
results.append(dict(obj))
|
||||
else:
|
||||
# Non-dict JSON value – wrap in empty dict (Bedrock requires
|
||||
# toolUse.input to be an object).
|
||||
results.append({})
|
||||
idx = decoded[1]
|
||||
|
||||
return results
|
||||
|
||||
|
||||
MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS: Final = 8
|
||||
|
||||
|
||||
def salvage_concatenated_tool_arguments(raw: str) -> tuple[dict[str, object], ...]:
|
||||
"""Return complete concatenated JSON objects that are safe to expand.
|
||||
|
||||
Identical objects collapse to the first one and are not capped. More than
|
||||
``MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS`` objects that are not all identical
|
||||
returns an empty tuple. Anything that is not a full concatenation of JSON
|
||||
objects returns an empty tuple. Repeated copies of the first object are not
|
||||
retained, and once the cap is passed the rest of the string is only checked.
|
||||
"""
|
||||
stripped: Final = raw.strip()
|
||||
if not stripped:
|
||||
return ()
|
||||
decoder: Final = json.JSONDecoder()
|
||||
length: Final = len(stripped)
|
||||
idx = 0 # rebind-ok: cursor walks the concatenated JSON string
|
||||
count = 0 # rebind-ok: counts complete objects without retaining duplicates
|
||||
kept = () # rebind-ok: holds at most one object past the salvage cap
|
||||
exceeded = False # rebind-ok: cap already passed, the tail is only validated
|
||||
while idx < length:
|
||||
while idx < length and stripped[idx] in " \t\n\r":
|
||||
idx += 1
|
||||
if idx >= length:
|
||||
break
|
||||
try:
|
||||
decoded = decoder.raw_decode(stripped, idx)
|
||||
except json.JSONDecodeError:
|
||||
return ()
|
||||
decoded_object = as_str_mapping(_DECODED_JSON.validate_python(decoded[0]))
|
||||
if decoded_object is None:
|
||||
return ()
|
||||
idx = decoded[1]
|
||||
if exceeded:
|
||||
continue
|
||||
obj = dict(decoded_object)
|
||||
count += 1
|
||||
if not kept:
|
||||
kept = (obj,)
|
||||
continue
|
||||
if obj == kept[0] and len(kept) == 1:
|
||||
continue
|
||||
if len(kept) == 1 and count > 2 and count - 1 > MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS:
|
||||
exceeded = True
|
||||
continue
|
||||
if len(kept) == 1 and count > 2:
|
||||
kept = (kept[0],) * (count - 1)
|
||||
if len(kept) >= MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS:
|
||||
exceeded = True
|
||||
continue
|
||||
kept = (*kept, obj)
|
||||
if exceeded:
|
||||
return ()
|
||||
return kept
|
||||
259
litellm/litellm_core_utils/prompt_templates/tpm_media_utils.py
Normal file
259
litellm/litellm_core_utils/prompt_templates/tpm_media_utils.py
Normal file
|
|
@ -0,0 +1,259 @@
|
|||
import base64
|
||||
import re
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from typing import Final, cast # noqa: TID251 # Existing casts retained unchanged in this pure move
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _TPMMediaProjection:
|
||||
content: object
|
||||
media: tuple[Mapping[str, object], ...] = ()
|
||||
|
||||
|
||||
def _tpm_image_part(url: str, detail: object = "auto") -> Mapping[str, object]:
|
||||
return {"type": "image_url", "image_url": {"url": url, "detail": detail}}
|
||||
|
||||
|
||||
def _tpm_blank_fields(fields: Mapping[str, object], names: tuple[str, ...]) -> Mapping[str, object]:
|
||||
return {key: "" if key in names and isinstance(value, str) else value for key, value in fields.items()}
|
||||
|
||||
|
||||
def _tpm_file_media(part: Mapping[str, object]) -> _TPMMediaProjection:
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type
|
||||
|
||||
nested: Final = part.get("type") == "file"
|
||||
fields: Final = as_str_mapping(part.get("file")) if nested else part
|
||||
if fields is None:
|
||||
return _TPMMediaProjection(part)
|
||||
raw_data: Final = fields.get("file_data")
|
||||
data: Final = raw_data if isinstance(raw_data, str) else ""
|
||||
declared: Final = next(
|
||||
(
|
||||
value
|
||||
for key in ("format", "mime_type", "content_type")
|
||||
if isinstance(value := fields.get(key), str) and value
|
||||
),
|
||||
"",
|
||||
)
|
||||
mime: Final = (declared or parse_mime_type(data[:256]) or "").split(";", 1)[0].strip().lower()
|
||||
if not mime.startswith(("image/", "audio/")):
|
||||
return _TPMMediaProjection(part)
|
||||
raw_url: Final = fields.get("file_url") or fields.get("file_id")
|
||||
url: Final = data or (raw_url if isinstance(raw_url, str) else "")
|
||||
header_end: Final = data.find(",", 0, 256)
|
||||
encoded: Final = (
|
||||
data[header_end + 1 :]
|
||||
if header_end >= 0 and data[:header_end].endswith(";base64")
|
||||
else ""
|
||||
if data.startswith(("data:", "https://", "http://", "gs://"))
|
||||
else data
|
||||
)
|
||||
media: Final = (
|
||||
_tpm_image_part(
|
||||
url
|
||||
if not data or url.startswith(("data:", "https://", "http://", "gs://"))
|
||||
else f"data:{mime};base64,{url}",
|
||||
fields.get("detail", "auto"),
|
||||
)
|
||||
if mime.startswith("image/")
|
||||
else {"type": "input_audio", "input_audio": {"data": encoded, "format": mime.removeprefix("audio/")}}
|
||||
)
|
||||
projected: Final = _tpm_blank_fields(fields, ("file_data", "file_url"))
|
||||
return _TPMMediaProjection({**part, "file": projected} if nested else projected, (media,))
|
||||
|
||||
|
||||
def _tpm_media_leaf(value: object) -> _TPMMediaProjection:
|
||||
part: Final = as_str_mapping(value)
|
||||
if part is None:
|
||||
return _TPMMediaProjection(value)
|
||||
kind: Final = part.get("type")
|
||||
if kind in ("file", "input_file"):
|
||||
return _tpm_file_media(part)
|
||||
if kind == "input_audio":
|
||||
audio: Final = as_str_mapping(part.get("input_audio"))
|
||||
return _TPMMediaProjection(
|
||||
{**part, "input_audio": _tpm_blank_fields(audio, ("data",))} if audio is not None else value,
|
||||
({"type": "input_audio", "input_audio": part.get("input_audio")},),
|
||||
)
|
||||
if kind in ("image_url", "input_image", "image", "computer_screenshot"):
|
||||
image: Final = as_str_mapping(part.get("image_url"))
|
||||
source: Final = as_str_mapping(part.get("source"))
|
||||
canonical: Final = _tpm_image_part(
|
||||
_image_part_url({**part, "type": "input_image"} if kind == "computer_screenshot" else part) or "",
|
||||
(image or part).get("detail", part.get("detail", "auto")),
|
||||
)
|
||||
projected: Final = (
|
||||
{**part, "image_url": _tpm_blank_fields(image, ("url",))}
|
||||
if image is not None and kind in ("image_url", "input_image", "computer_screenshot")
|
||||
else {**part, "source": _tpm_blank_fields(source, ("data", "url"))}
|
||||
if source is not None and kind == "image" and source.get("type") in ("base64", "url")
|
||||
else _tpm_blank_fields(part, ("image_url", "url"))
|
||||
)
|
||||
return _TPMMediaProjection(projected, (canonical,))
|
||||
return _TPMMediaProjection(value)
|
||||
|
||||
|
||||
def _tpm_media_list(value: object, split: Callable[[object], _TPMMediaProjection]) -> _TPMMediaProjection:
|
||||
if not isinstance(value, (list, tuple)):
|
||||
return _TPMMediaProjection(value)
|
||||
parts: Final = cast(Sequence[object], value) # cast-ok: finite JSON content arrays
|
||||
projected: Final = tuple(map(split, parts))
|
||||
media: Final = tuple(chain.from_iterable(part.media for part in projected))
|
||||
content: Final = tuple(part.content for part in projected)
|
||||
return _TPMMediaProjection(parts if not media else content if isinstance(value, tuple) else list(content), media)
|
||||
|
||||
|
||||
def _tpm_replace_field(part: Mapping[str, object], key: str, projection: _TPMMediaProjection) -> _TPMMediaProjection:
|
||||
return _TPMMediaProjection({**part, key: projection.content} if projection.media else part, projection.media)
|
||||
|
||||
|
||||
def _tpm_document_media(value: object) -> _TPMMediaProjection:
|
||||
part: Final = as_str_mapping(value)
|
||||
source: Final = as_str_mapping(part.get("source")) if part is not None else None
|
||||
if part is None or part.get("type") != "document" or source is None or source.get("type") != "content":
|
||||
return _tpm_media_leaf(value)
|
||||
content: Final = _tpm_media_list(source.get("content"), _tpm_media_leaf)
|
||||
return _tpm_replace_field(part, "source", _tpm_replace_field(source, "content", content))
|
||||
|
||||
|
||||
def _tpm_tool_result_media(value: object) -> _TPMMediaProjection:
|
||||
part: Final = as_str_mapping(value)
|
||||
if part is None or part.get("type") != "tool_result":
|
||||
return _tpm_document_media(value)
|
||||
return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_document_media))
|
||||
|
||||
|
||||
def _tpm_code_output_media(value: object) -> _TPMMediaProjection:
|
||||
part: Final = as_str_mapping(value)
|
||||
url: Final = part.get("url") if part is not None else None
|
||||
if part is None or part.get("type") != "image" or not isinstance(url, str) or not url:
|
||||
return _TPMMediaProjection(value)
|
||||
return _TPMMediaProjection(_tpm_blank_fields(part, ("url",)), (_tpm_image_part(url),))
|
||||
|
||||
|
||||
def _tpm_input_media(value: object) -> _TPMMediaProjection:
|
||||
part: Final = as_str_mapping(value)
|
||||
if part is None:
|
||||
return _TPMMediaProjection(value)
|
||||
kind: Final = part.get("type")
|
||||
if kind in ("function_call_output", "custom_tool_call_output"):
|
||||
return _tpm_replace_field(part, "output", _tpm_media_list(part.get("output"), _tpm_tool_result_media))
|
||||
if kind == "computer_call_output":
|
||||
return _tpm_replace_field(part, "output", _tpm_media_leaf(part.get("output")))
|
||||
if kind == "code_interpreter_call":
|
||||
return _tpm_replace_field(part, "outputs", _tpm_media_list(part.get("outputs"), _tpm_code_output_media))
|
||||
result: Final = part.get("result")
|
||||
if kind == "image_generation_call" and isinstance(result, str) and result:
|
||||
return _TPMMediaProjection(
|
||||
_tpm_blank_fields(part, ("result",)),
|
||||
(_tpm_image_part("data:application/octet-stream;base64," + result),),
|
||||
)
|
||||
if kind == "message" or (kind is None and isinstance(part.get("role"), str)):
|
||||
return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_tool_result_media))
|
||||
return _tpm_tool_result_media(value)
|
||||
|
||||
|
||||
def split_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]:
|
||||
projection: Final = _tpm_media_list(value, _tpm_input_media)
|
||||
return projection.content, projection.media
|
||||
|
||||
|
||||
_TPM_EMBEDDING_FILE_MIME_PREFIXES: Final = ("application/pdf", "video/")
|
||||
|
||||
|
||||
def _tpm_embedding_file(data_uri: str) -> tuple[Mapping[str, object], ...]:
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type
|
||||
|
||||
mime: Final = (parse_mime_type(data_uri[:256]) or "").strip().lower()
|
||||
if not mime.startswith(_TPM_EMBEDDING_FILE_MIME_PREFIXES):
|
||||
return ()
|
||||
return ({"type": "file", "file": {"file_data": data_uri, "format": mime}},)
|
||||
|
||||
|
||||
_BASE64_PAYLOAD: Final = re.compile(r"[A-Za-z0-9+/]*={0,2}")
|
||||
|
||||
|
||||
def _is_base64_data_uri(value: str) -> bool:
|
||||
header_end: Final = value.find(",", 0, 256)
|
||||
if header_end < 0 or not value[:header_end].endswith(";base64"):
|
||||
return False
|
||||
payload_start: Final = header_end + 1
|
||||
if (len(value) - payload_start) % 4 or _BASE64_PAYLOAD.fullmatch(value, payload_start) is None:
|
||||
return False
|
||||
last_quantum: Final = value[max(payload_start, len(value) - 4) :]
|
||||
return base64.b64encode(base64.b64decode(last_quantum)).decode("ascii") == last_quantum
|
||||
|
||||
|
||||
def _tpm_embedding_file_block(part: Mapping[str, object]) -> _TPMMediaProjection:
|
||||
projection: Final = _tpm_file_media(part)
|
||||
fields: Final = as_str_mapping(part.get("file"))
|
||||
data: Final = fields.get("file_data") if fields is not None else None
|
||||
if projection.media or fields is None or not isinstance(data, str) or not _is_base64_data_uri(data):
|
||||
return projection
|
||||
media: Final = _tpm_embedding_file(data)
|
||||
return _TPMMediaProjection({**part, "file": _tpm_blank_fields(fields, ("file_data",))} if media else part, media)
|
||||
|
||||
|
||||
def _tpm_embedding_leaf(value: object) -> _TPMMediaProjection:
|
||||
part: Final = as_str_mapping(value)
|
||||
if part is not None and part.get("type") == "file":
|
||||
return _tpm_embedding_file_block(part)
|
||||
if not isinstance(value, str) or not value.startswith("data:"):
|
||||
return _tpm_media_leaf(value)
|
||||
if not _is_base64_data_uri(value):
|
||||
return _TPMMediaProjection(value)
|
||||
media: Final = _tpm_file_media({"file_data": value}).media or _tpm_embedding_file(value)
|
||||
return _TPMMediaProjection("" if media else value, media)
|
||||
|
||||
|
||||
def _is_json_array(value: object) -> bool:
|
||||
return isinstance(value, (list, tuple))
|
||||
|
||||
|
||||
def _tpm_embedding_item(value: object) -> _TPMMediaProjection:
|
||||
return _tpm_media_list(value, _tpm_embedding_leaf) if _is_json_array(value) else _tpm_embedding_leaf(value)
|
||||
|
||||
|
||||
def split_embedding_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]:
|
||||
projection: Final = (
|
||||
_tpm_media_list(value, _tpm_embedding_item) if _is_json_array(value) else _tpm_embedding_leaf(value)
|
||||
)
|
||||
return projection.content, projection.media
|
||||
|
||||
|
||||
def embedding_input_is_all_media(value: object) -> bool:
|
||||
items: Final = cast(Sequence[object], value) if _is_json_array(value) else (value,) # cast-ok: JSON input array
|
||||
return all(_tpm_embedding_leaf(item).media for item in items)
|
||||
|
||||
|
||||
def anthropic_image_source_to_openai_url(image_source: Mapping[str, object]) -> str | None:
|
||||
"""Data or remote URL for an Anthropic ``source`` block, in the form chat completions expects."""
|
||||
source_type: Final = image_source.get("type")
|
||||
if source_type == "base64":
|
||||
media_type: Final = image_source.get("media_type") or "image/jpeg"
|
||||
image_data: Final = image_source.get("data") or ""
|
||||
return f"data:{media_type};base64,{image_data}" if image_data else None
|
||||
if source_type == "url":
|
||||
url: Final = image_source.get("url")
|
||||
return url if isinstance(url, str) else ""
|
||||
return None
|
||||
|
||||
|
||||
def _image_part_url(part: Mapping[str, object]) -> str | None:
|
||||
"""The image URL carried by one content part, whichever of the three dialects wrote it."""
|
||||
part_type: Final = part.get("type")
|
||||
if part_type in ("image_url", "input_image"):
|
||||
image_url: Final = part.get("image_url") or part.get("url")
|
||||
if isinstance(image_url, str):
|
||||
return image_url
|
||||
image_fields: Final = as_str_mapping(image_url)
|
||||
url: Final = image_fields.get("url") if image_fields is not None else None
|
||||
return url if isinstance(url, str) else None
|
||||
if part_type == "image":
|
||||
source: Final = part.get("source")
|
||||
return anthropic_image_source_to_openai_url(source) if isinstance(source, Mapping) else None
|
||||
return None
|
||||
|
|
@ -409,7 +409,7 @@ def deployments_the_caller_may_use(
|
|||
return ()
|
||||
|
||||
|
||||
_WEB_SEARCH_TOOL_TYPES: Final = frozenset({"web_search", "web_search_preview"})
|
||||
_WEB_SEARCH_TOOL_KEYS: Final = frozenset({"googleSearch", "google_search"})
|
||||
|
||||
|
||||
def _deployment_supports_web_search(deployment: dict) -> bool:
|
||||
|
|
@ -433,11 +433,18 @@ def _deployment_supports_web_search(deployment: dict) -> bool:
|
|||
|
||||
|
||||
def is_web_search_request(request_kwargs: Mapping[str, object]) -> bool:
|
||||
"""OpenAI ``web_search`` / ``web_search_preview``, Anthropic ``web_search_<version>``, Gemini
|
||||
``googleSearch`` / ``google_search``."""
|
||||
tools: Final[object] = request_kwargs.get("tools")
|
||||
return isinstance(tools, (list, tuple)) and any(
|
||||
isinstance(tool, dict)
|
||||
and isinstance(tool_type := tool.get("type"), str)
|
||||
and tool_type in _WEB_SEARCH_TOOL_TYPES
|
||||
and (
|
||||
any(key in tool for key in _WEB_SEARCH_TOOL_KEYS)
|
||||
or (
|
||||
isinstance(tool_type := tool.get("type"), str)
|
||||
and (tool_type == "web_search" or tool_type.startswith("web_search_"))
|
||||
)
|
||||
)
|
||||
for tool in tools
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -706,7 +706,7 @@ def _data_uri_media_scope(model: str, media: Sequence[Mapping[str, object]]) ->
|
|||
def split_embedding_input_tokens(
|
||||
value: _EmbeddingInputT, media_models: Callable[[], Sequence[str]], fallback_model: str | None = None
|
||||
) -> tuple[_EmbeddingInputT, int]:
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import (
|
||||
embedding_input_is_all_media,
|
||||
split_embedding_media_for_tpm,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -37,18 +37,9 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
DEFS_MAX_INLINED_BYTES,
|
||||
invalid_type_message,
|
||||
is_object_schema,
|
||||
is_present_non_string,
|
||||
is_unhashable_value_behind,
|
||||
sanitize_input_schema_for_anthropic,
|
||||
schema_fragment_ref,
|
||||
schema_grammar_branch_count,
|
||||
schema_has_lookaround_pattern,
|
||||
schema_has_open_object,
|
||||
schema_has_unresolvable_ref,
|
||||
schema_optional_property_count,
|
||||
schema_string_formats,
|
||||
schema_union_property_count,
|
||||
unpack_defs,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
||||
|
|
@ -59,6 +50,17 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
|||
inline_remote_image_urls,
|
||||
is_unformatted_pdf_link_image,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.json_schema_utils import (
|
||||
is_object_schema,
|
||||
schema_fragment_ref,
|
||||
schema_grammar_branch_count,
|
||||
schema_has_lookaround_pattern,
|
||||
schema_has_open_object,
|
||||
schema_has_unresolvable_ref,
|
||||
schema_optional_property_count,
|
||||
schema_string_formats,
|
||||
schema_union_property_count,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
|
||||
message_field,
|
||||
parts_of,
|
||||
|
|
|
|||
|
|
@ -29,17 +29,19 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, lax_bool
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_file_ids_from_messages,
|
||||
is_encrypted_reasoning_block,
|
||||
native_redacted_thinking_data,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import THOUGHT_SIGNATURE_SEPARATOR
|
||||
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
|
||||
get_file_ids_from_messages,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
|
||||
find_anthropic_server_tool_result,
|
||||
message_field,
|
||||
parts_of,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
is_encrypted_reasoning_block,
|
||||
native_redacted_thinking_data,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import DroppedThinkingBudget, non_numeric_thinking_budget_error
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.wif import (
|
||||
|
|
|
|||
|
|
@ -21,7 +21,9 @@ import httpx
|
|||
from openai.types.file_deleted import FileDeleted
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
|
||||
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
|
||||
extract_file_data,
|
||||
)
|
||||
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.files.transformation import (
|
||||
|
|
|
|||
|
|
@ -123,15 +123,23 @@ def create_tool_name_mapping(
|
|||
from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingChoice
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
anthropic_image_source_to_openai_url,
|
||||
anthropic_inline_document,
|
||||
parse_tool_call_arguments,
|
||||
reasoning_content_from_thinking_blocks,
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
THOUGHT_SIGNATURE_SEPARATOR,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
reasoning_content_from_thinking_blocks,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import (
|
||||
parse_tool_call_arguments,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import (
|
||||
anthropic_image_source_to_openai_url,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import (
|
||||
reasoning_effort_from_thinking,
|
||||
reasoning_effort_from_thinking_silently,
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from litellm.litellm_core_utils.asyncify import (
|
|||
run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
|
||||
cache_control_as_prompt_cache_breakpoint,
|
||||
prompt_cache_breakpoint_with_readable_mode,
|
||||
with_prompt_cache_breakpoint,
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from litellm._logging import redact_internal_details_from_client_message
|
|||
from litellm._uuid import uuid
|
||||
from litellm.exceptions import APIError, MidStreamFallbackError
|
||||
from litellm.litellm_core_utils.core_helpers import has_attributes, iter_items
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
encrypted_reasoning_signature,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
|
|
|
|||
|
|
@ -16,9 +16,13 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
TOOL_RESULT_IMAGE_BOUNDARY,
|
||||
TOOL_RESULT_IMAGE_PLACEHOLDER,
|
||||
anthropic_inline_document,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
encrypted_reasoning_signature,
|
||||
responses_reasoning_items_from_thinking_blocks,
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import (
|
||||
reasoning_effort_from_thinking,
|
||||
|
|
|
|||
|
|
@ -14,17 +14,21 @@ from litellm.integrations.anthropic_cache_control_hook import deployment_prompt_
|
|||
from litellm.litellm_core_utils.allowed_tools_tool_choice import is_allowed_tools_tool_choice
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
chat_completion_prompt_cache_breakpoints_for_wire,
|
||||
drop_tool_reference_parts_from_tool_messages,
|
||||
flatten_combinators_and_drop_non_python_regex_patterns,
|
||||
hoist_images_from_tool_messages,
|
||||
system_messages_first,
|
||||
tool_with_sanitized_parameters,
|
||||
without_prompt_cache_breakpoints,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
convert_to_azure_openai_messages,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.json_schema_utils import (
|
||||
flatten_combinators_and_drop_non_python_regex_patterns,
|
||||
tool_with_sanitized_parameters,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
|
||||
chat_completion_prompt_cache_breakpoints_for_wire,
|
||||
without_prompt_cache_breakpoints,
|
||||
)
|
||||
from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.openai.chat.gpt_reasoning_transformation import GPT_REASONING_SERIES_MARKERS
|
||||
|
|
|
|||
|
|
@ -11,7 +11,9 @@ from pydantic import TypeAdapter
|
|||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import prompt_cache_breakpoint_within
|
||||
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
|
||||
prompt_cache_breakpoint_within,
|
||||
)
|
||||
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
|
||||
from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
drop_tool_reference_parts_from_tool_messages,
|
||||
filter_value_from_dict,
|
||||
is_pdf_file_part,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
|
||||
prompt_cache_breakpoint_for_wire,
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -38,25 +38,27 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
safe_deep_copy,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
drop_lookaround_regex_patterns,
|
||||
is_object_schema,
|
||||
native_redacted_thinking_data,
|
||||
parse_content_for_reasoning,
|
||||
tool_with_sanitized_parameters,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
BedrockConverseMessagesProcessor,
|
||||
_bedrock_converse_messages_pt,
|
||||
_bedrock_tools_pt,
|
||||
make_valid_bedrock_tool_name,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.json_schema_utils import (
|
||||
drop_lookaround_regex_patterns,
|
||||
is_object_schema,
|
||||
tool_with_sanitized_parameters,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
|
||||
CONVERTED_SYSTEM_NOTE,
|
||||
is_system_message,
|
||||
message_field,
|
||||
parts_of,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
native_redacted_thinking_data,
|
||||
parse_content_for_reasoning,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
|
||||
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
|
||||
from litellm.llms.anthropic.chat.transformation import (
|
||||
|
|
@ -77,6 +79,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
bedrock_model_supports_regex_lookaround,
|
||||
bedrock_native_output_format,
|
||||
bedrock_native_output_is_opt_in,
|
||||
bedrock_native_output_retry_requested,
|
||||
bedrock_supports_tool_search,
|
||||
deployment_model_info,
|
||||
requests_output_config_format,
|
||||
|
|
@ -1425,6 +1428,16 @@ class AmazonConverseConfig(BaseConfig):
|
|||
deployment_model_info(litellm_params),
|
||||
)
|
||||
|
||||
def uses_native_output(
|
||||
self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
|
||||
) -> bool:
|
||||
return bedrock_native_output_retry_requested(
|
||||
bedrock_capability_model(model, litellm_params.get("base_model")),
|
||||
optional_params,
|
||||
self.custom_llm_provider or "bedrock_converse",
|
||||
deployment_model_info(litellm_params),
|
||||
)
|
||||
|
||||
def update_optional_params_with_thinking_tokens(self, non_default_params: dict, optional_params: dict):
|
||||
"""
|
||||
Handles scenario where max tokens is not specified. For anthropic models (anthropic api/bedrock/vertex ai), this requires having the max tokens being set and being greater than the thinking token budget.
|
||||
|
|
|
|||
|
|
@ -16,7 +16,9 @@ from litellm.litellm_core_utils.litellm_logging import Logging
|
|||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers
|
||||
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import native_redacted_thinking_data
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
native_redacted_thinking_data,
|
||||
)
|
||||
from litellm.llms.anthropic.chat.handler import (
|
||||
ModelResponseIterator as AnthropicModelResponseIterator,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Final, cast
|
|||
from httpx import Response
|
||||
|
||||
from litellm import verbose_logger
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
parse_content_for_reasoning,
|
||||
)
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
bedrock_capability_model,
|
||||
bedrock_chat_uses_native_structured_output,
|
||||
bedrock_cleaned_output_schema,
|
||||
bedrock_native_output_retry_requested,
|
||||
bedrock_supports_tool_search,
|
||||
deployment_model_info,
|
||||
get_anthropic_beta_from_headers,
|
||||
|
|
@ -161,6 +162,13 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
) -> bool:
|
||||
return requests_opt_in_native_output(model, optional_params, deployment_model_info(litellm_params))
|
||||
|
||||
def uses_native_output(
|
||||
self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
|
||||
) -> bool:
|
||||
return bedrock_native_output_retry_requested(
|
||||
model, optional_params, model_info=deployment_model_info(litellm_params)
|
||||
)
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from collections.abc import Callable, Coroutine
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
|
|
@ -6,6 +7,7 @@ from litellm.llms.anthropic.common_utils import is_claude_strict_tools_rejection
|
|||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
NativeOutputFallback,
|
||||
bedrock_strict_tools_fallback,
|
||||
is_bedrock_native_output_rejection,
|
||||
warn_bedrock_native_output_rejected,
|
||||
warn_bedrock_strict_tools_dropped,
|
||||
|
|
@ -17,7 +19,15 @@ NativeOutputResponse: TypeAlias = ModelResponse | CustomStreamWrapper
|
|||
NativeOutputAttemptResult: TypeAlias = NativeOutputResponse | Coroutine[object, object, NativeOutputResponse]
|
||||
NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeOutputAttemptResult]
|
||||
NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None]
|
||||
_RetryFor: TypeAlias = Callable[[BaseLLMException], NativeOutputFallback | None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SelectedFallback:
|
||||
fallback: NativeOutputFallback
|
||||
native_output_retry: bool
|
||||
|
||||
|
||||
_RetryFor: TypeAlias = Callable[[BaseLLMException], _SelectedFallback | None]
|
||||
|
||||
|
||||
def _native_output_retry(
|
||||
|
|
@ -47,17 +57,42 @@ def _fallback_for(
|
|||
fallback: NativeOutputFallbackFactory | None,
|
||||
strict_tools_fallback: NativeOutputFallbackFactory | None,
|
||||
model: str,
|
||||
) -> NativeOutputFallback | None:
|
||||
) -> _SelectedFallback | None:
|
||||
native_output_retry: Final = _native_output_retry(error, fallback, model)
|
||||
if native_output_retry is not None:
|
||||
return native_output_retry
|
||||
return _strict_tools_retry(error, strict_tools_fallback, model)
|
||||
return _SelectedFallback(native_output_retry, native_output_retry=True)
|
||||
strict_tools_retry: Final = _strict_tools_retry(error, strict_tools_fallback, model)
|
||||
if strict_tools_retry is None:
|
||||
return None
|
||||
return _SelectedFallback(strict_tools_retry, native_output_retry=False)
|
||||
|
||||
|
||||
def _strict_tools_retry_after_native_output_retry(
|
||||
error: BaseLLMException,
|
||||
retry: _SelectedFallback,
|
||||
strict_tools_fallback: NativeOutputFallbackFactory | None,
|
||||
model: str,
|
||||
) -> NativeOutputFallback | None:
|
||||
if (
|
||||
not retry.native_output_retry
|
||||
or strict_tools_fallback is None
|
||||
or not is_claude_strict_tools_rejection(error.status_code, error.message)
|
||||
):
|
||||
return None
|
||||
composed_retry: Final = bedrock_strict_tools_fallback(
|
||||
model, retry.fallback.optional_params, retry.fallback.litellm_params
|
||||
)
|
||||
if composed_retry is not None:
|
||||
warn_bedrock_strict_tools_dropped(model)
|
||||
return composed_retry
|
||||
|
||||
|
||||
async def _await_with_native_output_fallback(
|
||||
first_attempt: Coroutine[object, object, NativeOutputResponse],
|
||||
attempt: NativeOutputAttempt,
|
||||
retry_for: _RetryFor,
|
||||
strict_tools_fallback: NativeOutputFallbackFactory | None,
|
||||
model: str,
|
||||
) -> NativeOutputResponse:
|
||||
try:
|
||||
return await first_attempt
|
||||
|
|
@ -65,8 +100,17 @@ async def _await_with_native_output_fallback(
|
|||
retry: Final = retry_for(error)
|
||||
if retry is None:
|
||||
raise
|
||||
second_attempt: Final = attempt(retry)
|
||||
return await second_attempt if isinstance(second_attempt, Coroutine) else second_attempt
|
||||
try:
|
||||
second_attempt: Final = attempt(retry.fallback)
|
||||
return await second_attempt if isinstance(second_attempt, Coroutine) else second_attempt
|
||||
except BaseLLMException as error:
|
||||
composed_retry: Final = _strict_tools_retry_after_native_output_retry(
|
||||
error, retry, strict_tools_fallback, model
|
||||
)
|
||||
if composed_retry is None:
|
||||
raise
|
||||
third_attempt: Final = attempt(composed_retry)
|
||||
return await third_attempt if isinstance(third_attempt, Coroutine) else third_attempt
|
||||
|
||||
|
||||
def call_with_native_output_fallback(
|
||||
|
|
@ -86,7 +130,15 @@ def call_with_native_output_fallback(
|
|||
retry: Final = retry_for(error)
|
||||
if retry is None:
|
||||
raise
|
||||
return attempt(retry)
|
||||
try:
|
||||
return attempt(retry.fallback)
|
||||
except BaseLLMException as retry_error:
|
||||
composed_retry: Final = _strict_tools_retry_after_native_output_retry(
|
||||
retry_error, retry, strict_tools_fallback, model
|
||||
)
|
||||
if composed_retry is None:
|
||||
raise
|
||||
return attempt(composed_retry)
|
||||
if isinstance(first_attempt, Coroutine):
|
||||
return _await_with_native_output_fallback(first_attempt, attempt, retry_for)
|
||||
return _await_with_native_output_fallback(first_attempt, attempt, retry_for, strict_tools_fallback, model)
|
||||
return first_attempt
|
||||
|
|
|
|||
|
|
@ -36,7 +36,9 @@ from litellm import verbose_logger
|
|||
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
|
||||
from litellm.litellm_core_utils.core_helpers import decoded_json_object, lax_bool, require_attributes
|
||||
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count
|
||||
from litellm.litellm_core_utils.prompt_templates.json_schema_utils import (
|
||||
schema_property_count,
|
||||
)
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import (
|
||||
|
|
@ -293,6 +295,15 @@ def _bedrock_model_supports(model: str, key: str, custom_llm_provider: str = "be
|
|||
return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key=key)
|
||||
|
||||
|
||||
def _bedrock_native_output_enabled(
|
||||
model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None
|
||||
) -> bool:
|
||||
deployment_flag: Final = (model_info or {}).get("supports_native_structured_output")
|
||||
if isinstance(deployment_flag, bool):
|
||||
return deployment_flag
|
||||
return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider)
|
||||
|
||||
|
||||
_DEPLOYMENT_MODEL_INFO: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
|
|
@ -313,9 +324,9 @@ def _bedrock_native_output_opt_in(
|
|||
def bedrock_supports_native_structured_output(
|
||||
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
|
||||
) -> bool:
|
||||
return _bedrock_model_supports(
|
||||
model, "supports_native_structured_output", custom_llm_provider
|
||||
) or _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
|
||||
return _bedrock_native_output_enabled(model, custom_llm_provider, model_info) or _bedrock_native_output_opt_in(
|
||||
model, custom_llm_provider, model_info
|
||||
)
|
||||
|
||||
|
||||
def _bedrock_map_native_output_flag(model: str) -> object:
|
||||
|
|
@ -350,7 +361,7 @@ def bedrock_chat_opts_into_native_structured_output(
|
|||
custom_llm_provider: str = "bedrock",
|
||||
model_info: Mapping[str, object] | None = None,
|
||||
) -> bool:
|
||||
return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) or (
|
||||
return _bedrock_native_output_enabled(model, custom_llm_provider, model_info) or (
|
||||
thinking_requested and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
|
||||
)
|
||||
|
||||
|
|
@ -371,15 +382,15 @@ def bedrock_chat_uses_native_structured_output(
|
|||
def bedrock_native_output_is_deployment_opt_in(
|
||||
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
|
||||
) -> bool:
|
||||
return not _bedrock_model_supports(
|
||||
model, "supports_native_structured_output", custom_llm_provider
|
||||
) and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
|
||||
return not _bedrock_native_output_enabled(model, custom_llm_provider, model_info) and _bedrock_native_output_opt_in(
|
||||
model, custom_llm_provider, model_info
|
||||
)
|
||||
|
||||
|
||||
def bedrock_native_output_is_opt_in(
|
||||
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
|
||||
) -> bool:
|
||||
return not _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) and (
|
||||
return not _bedrock_native_output_enabled(model, custom_llm_provider, model_info) and (
|
||||
_bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
|
||||
or _bedrock_native_output_replaces_unforceable_tool(model, model_info)
|
||||
)
|
||||
|
|
@ -388,7 +399,21 @@ def bedrock_native_output_is_opt_in(
|
|||
def without_bedrock_native_output_opt_in(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {
|
||||
**litellm_params,
|
||||
"model_info": {**deployment_model_info(litellm_params), "supports_bedrock_native_structured_output": False},
|
||||
"model_info": {
|
||||
**deployment_model_info(litellm_params),
|
||||
"supports_bedrock_native_structured_output": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def for_bedrock_json_tool_retry(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
opt_in_disabled: Final = without_bedrock_native_output_opt_in(litellm_params)
|
||||
return {
|
||||
**opt_in_disabled,
|
||||
"model_info": {
|
||||
**deployment_model_info(opt_in_disabled),
|
||||
"supports_native_structured_output": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -396,6 +421,7 @@ _BEDROCK_NATIVE_OUTPUT_UNSUPPORTED: Final = "output_config.format: Extra inputs
|
|||
_BEDROCK_NATIVE_OUTPUT_REJECTIONS: Final = (
|
||||
_BEDROCK_NATIVE_OUTPUT_UNSUPPORTED,
|
||||
"compilation exceeded limits: Schema is too complex",
|
||||
"The compiled grammar is too large",
|
||||
"output_config.format.schema:",
|
||||
)
|
||||
|
||||
|
|
@ -476,8 +502,27 @@ def requests_output_config_format(optional_params: Mapping[str, object]) -> bool
|
|||
|
||||
|
||||
def requests_native_output(anthropic_optional_params: Mapping[str, object]) -> bool:
|
||||
return isinstance(anthropic_optional_params.get("output_format"), Mapping) or requests_output_config_format(
|
||||
anthropic_optional_params
|
||||
output_config: Final = json_object_or_none(anthropic_optional_params.get("outputConfig"))
|
||||
return (
|
||||
isinstance(anthropic_optional_params.get("output_format"), Mapping)
|
||||
or requests_output_config_format(anthropic_optional_params)
|
||||
or (
|
||||
anthropic_optional_params.get("json_mode") is True
|
||||
and output_config is not None
|
||||
and isinstance(output_config.get("textFormat"), Mapping)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def bedrock_native_output_retry_requested(
|
||||
model: str,
|
||||
optional_params: Mapping[str, object],
|
||||
custom_llm_provider: str = "bedrock",
|
||||
model_info: Mapping[str, object] | None = None,
|
||||
) -> bool:
|
||||
return requests_native_output(optional_params) and (
|
||||
bedrock_native_output_is_opt_in(model, custom_llm_provider, model_info)
|
||||
or _bedrock_native_output_enabled(model, custom_llm_provider, model_info)
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -519,7 +564,7 @@ def bedrock_native_output_schema(
|
|||
model_info: Mapping[str, object] | None = None,
|
||||
) -> Mapping[str, object] | None:
|
||||
native_schema: Final = bedrock_cleaned_output_schema(model, schema, custom_llm_provider)
|
||||
if _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider):
|
||||
if _bedrock_native_output_enabled(model, custom_llm_provider, model_info):
|
||||
return schema if native_schema is None else native_schema
|
||||
if _bedrock_native_output_opt_in(model, custom_llm_provider, model_info):
|
||||
return native_schema
|
||||
|
|
|
|||
|
|
@ -41,9 +41,11 @@ from litellm.litellm_core_utils.cloud_storage_security import (
|
|||
)
|
||||
from litellm.litellm_core_utils.core_helpers import require_attributes
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
extract_file_data,
|
||||
text_completion_prompt_to_messages,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
|
||||
extract_file_data,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError
|
||||
from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
|
@ -237,15 +239,6 @@ class _S3RequestTarget:
|
|||
request_params: _BedrockS3RequestParams
|
||||
|
||||
|
||||
class _TrustedS3ModelCredentials(LiteLLMBaseModel):
|
||||
"""The S3 buckets the server trusts file ids against, from the deployment snapshot."""
|
||||
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
s3_bucket_name: str | None = None
|
||||
s3_output_bucket_name: str | None = None
|
||||
|
||||
|
||||
def extract_s3_uri_from_file_id(file_id: str) -> str:
|
||||
"""
|
||||
Resolve a Bedrock file id to its S3 URI.
|
||||
|
|
@ -269,27 +262,31 @@ def extract_s3_uri_from_file_id(file_id: str) -> str:
|
|||
raise ValueError("file_id must be a managed LiteLLM S3 file id")
|
||||
|
||||
|
||||
_S3_BUCKET_REQUIRED_ERROR: Final = "S3 bucket_name is required. Set 's3_bucket_name' in proxy config or AWS_S3_BUCKET_NAME for Bedrock file content retrieval."
|
||||
_S3_BUCKET_REQUIRED_ERROR: Final = (
|
||||
"S3 bucket_name is required. Pass 's3_bucket_name', set it in proxy config, "
|
||||
"or set AWS_S3_BUCKET_NAME for Bedrock file content retrieval."
|
||||
)
|
||||
|
||||
|
||||
def _trusted_s3_model_credentials(litellm_params: Mapping[str, object]) -> _TrustedS3ModelCredentials:
|
||||
trusted_model_credentials: Final = litellm_params.get("_litellm_internal_model_credentials")
|
||||
if not isinstance(trusted_model_credentials, MappingProxyType):
|
||||
return _TrustedS3ModelCredentials()
|
||||
snapshot: Final[dict[str, object]] = {}
|
||||
snapshot.update(trusted_model_credentials) # any-ok: untyped snapshot
|
||||
return _TrustedS3ModelCredentials.model_validate(snapshot)
|
||||
def _s3_param(litellm_params: Mapping[str, object], key: str, env_var: str) -> str | None:
|
||||
value: Final = litellm_params.get(key)
|
||||
return value if isinstance(value, str) and value else os.getenv(env_var)
|
||||
|
||||
|
||||
def _input_bucket_name(litellm_params: Mapping[str, object]) -> str | None:
|
||||
return _s3_param(litellm_params, "s3_bucket_name", "AWS_S3_BUCKET_NAME")
|
||||
|
||||
|
||||
def _output_bucket_name(litellm_params: Mapping[str, object]) -> str | None:
|
||||
return _s3_param(litellm_params, "s3_output_bucket_name", "AWS_S3_OUTPUT_BUCKET_NAME")
|
||||
|
||||
|
||||
def get_configured_s3_bucket_name(litellm_params: Mapping[str, object]) -> str:
|
||||
"""
|
||||
Resolve the server-configured S3 bucket for Bedrock file operations.
|
||||
|
||||
Only trusts the immutable server-side credential snapshot or the
|
||||
environment; never a request-supplied param, since the bucket is what
|
||||
`validate_managed_cloud_file_id` checks file ids against.
|
||||
Resolve the S3 bucket for Bedrock file operations: the call's ``s3_bucket_name``,
|
||||
else ``AWS_S3_BUCKET_NAME``. File ids are validated against this bucket.
|
||||
"""
|
||||
bucket_name: Final = _trusted_s3_model_credentials(litellm_params).s3_bucket_name or os.getenv("AWS_S3_BUCKET_NAME")
|
||||
bucket_name: Final = _input_bucket_name(litellm_params)
|
||||
if not bucket_name:
|
||||
raise ValueError(_S3_BUCKET_REQUIRED_ERROR)
|
||||
return bucket_name
|
||||
|
|
@ -297,16 +294,13 @@ def get_configured_s3_bucket_name(litellm_params: Mapping[str, object]) -> str:
|
|||
|
||||
def get_configured_s3_bucket_names(litellm_params: Mapping[str, object]) -> tuple[str, ...]:
|
||||
"""
|
||||
Resolve the server-configured S3 buckets a Bedrock file id may live in.
|
||||
Resolve the S3 buckets a Bedrock file id may live in.
|
||||
|
||||
Bedrock batch outputs land in ``s3_output_bucket_name`` when it differs from
|
||||
the input bucket, so retrieval validates against both. Same trust rules as
|
||||
``get_configured_s3_bucket_name``: only the immutable credential snapshot or
|
||||
the environment, never a request param.
|
||||
the input bucket, so retrieval validates against both.
|
||||
"""
|
||||
trusted: Final = _trusted_s3_model_credentials(litellm_params)
|
||||
input_bucket: Final = trusted.s3_bucket_name or os.getenv("AWS_S3_BUCKET_NAME")
|
||||
output_bucket: Final = trusted.s3_output_bucket_name or os.getenv("AWS_S3_OUTPUT_BUCKET_NAME")
|
||||
input_bucket: Final = _input_bucket_name(litellm_params)
|
||||
output_bucket: Final = _output_bucket_name(litellm_params)
|
||||
buckets: Final = tuple(dict.fromkeys(bucket for bucket in (input_bucket, output_bucket) if bucket))
|
||||
if not buckets:
|
||||
raise ValueError(_S3_BUCKET_REQUIRED_ERROR)
|
||||
|
|
@ -522,12 +516,7 @@ def _output_location_still_unlisted(litellm_params: Mapping[str, object]) -> boo
|
|||
def _listing_bucket_name(litellm_params: Mapping[str, object], purpose: str | None) -> str:
|
||||
if purpose != "batch_output":
|
||||
return get_configured_s3_bucket_name(litellm_params)
|
||||
trusted: Final = _trusted_s3_model_credentials(litellm_params)
|
||||
return (
|
||||
trusted.s3_output_bucket_name
|
||||
or os.getenv("AWS_S3_OUTPUT_BUCKET_NAME")
|
||||
or get_configured_s3_bucket_name(litellm_params)
|
||||
)
|
||||
return _output_bucket_name(litellm_params) or get_configured_s3_bucket_name(litellm_params)
|
||||
|
||||
|
||||
def _listed_object_created_at(entry: ET.Element) -> int:
|
||||
|
|
|
|||
|
|
@ -40,15 +40,17 @@ from litellm.llms.bedrock.common_utils import (
|
|||
BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS,
|
||||
BedrockError,
|
||||
apply_bedrock_invoke_structured_output,
|
||||
bedrock_native_output_is_deployment_opt_in,
|
||||
bedrock_native_output_format,
|
||||
bedrock_native_output_retry_requested,
|
||||
bedrock_supports_tool_search,
|
||||
ensure_bedrock_anthropic_messages_tool_names,
|
||||
for_bedrock_json_tool_retry,
|
||||
get_anthropic_beta_from_headers,
|
||||
is_bedrock_native_output_rejection,
|
||||
json_object_or_none,
|
||||
normalize_bedrock_opus_output_config_effort,
|
||||
normalize_custom_field_on_tools,
|
||||
normalize_tool_input_schema_types_for_bedrock_invoke,
|
||||
requests_native_output,
|
||||
requests_output_config_format,
|
||||
strip_unsupported_bedrock_invoke_output_config_keys,
|
||||
supports_bedrock_prompt_cache_ttl,
|
||||
|
|
@ -133,13 +135,24 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
) -> GenericLiteLLMParams | None:
|
||||
model_info: Final = _MODEL_INFO.validate_python(litellm_params.model_info)
|
||||
if not (
|
||||
requests_native_output(anthropic_messages_optional_request_params)
|
||||
and bedrock_native_output_is_deployment_opt_in(model, model_info=model_info)
|
||||
bedrock_native_output_retry_requested(
|
||||
model, anthropic_messages_optional_request_params, model_info=model_info
|
||||
)
|
||||
):
|
||||
return None
|
||||
return litellm_params.model_copy(
|
||||
update={"model_info": {**(model_info or {}), "supports_bedrock_native_structured_output": False}}
|
||||
legacy_output_format: Final = json_object_or_none(
|
||||
anthropic_messages_optional_request_params.get("output_format")
|
||||
)
|
||||
output_config: Final = json_object_or_none(anthropic_messages_optional_request_params.get("output_config"))
|
||||
output_config_format: Final = (
|
||||
json_object_or_none(output_config.get("format")) if output_config is not None else None
|
||||
)
|
||||
output_format: Final = legacy_output_format if legacy_output_format is not None else output_config_format
|
||||
if output_format is None or (bedrock_native_output_format(model, output_format, model_info=model_info) is None):
|
||||
return None
|
||||
retry_params: Final = for_bedrock_json_tool_retry({"model_info": model_info})
|
||||
retry_model_info: Final = _MODEL_INFO.validate_python(retry_params.get("model_info"))
|
||||
return litellm_params.model_copy(update={"model_info": retry_model_info})
|
||||
|
||||
def should_retry_anthropic_messages_without_native_output(
|
||||
self, e: httpx.HTTPStatusError, request_body: Mapping[str, object], model: str
|
||||
|
|
|
|||
|
|
@ -24,7 +24,9 @@ import httpx
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import chat_completion_prompt_cache_breakpoint_for_wire
|
||||
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
|
||||
chat_completion_prompt_cache_breakpoint_for_wire,
|
||||
)
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items
|
||||
|
|
|
|||
|
|
@ -210,7 +210,7 @@ def _prepare_multipart_file_upload(
|
|||
Returns:
|
||||
Tuple of (files_dict, headers_without_content_type)
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
|
||||
extract_file_data,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -19,12 +19,14 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
|
|||
should_convert_tool_call_to_json_mode,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
extract_reasoning_content,
|
||||
merge_consecutive_system_messages,
|
||||
strip_litellm_internal_message_fields,
|
||||
strip_name_from_message,
|
||||
strip_prefill_prefix_field,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
extract_reasoning_content,
|
||||
)
|
||||
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
return response
|
||||
|
||||
except Exception as e:
|
||||
raise ValueError(f"Error transforming Deepgram response: {e}\nResponse: {raw_response.text}")
|
||||
raise ValueError(f"Error transforming Deepgram response: {e}") from None
|
||||
|
||||
def _reconstruct_diarized_transcript(self, words: list) -> str:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -160,7 +160,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
return response
|
||||
|
||||
except Exception as e:
|
||||
raise ValueError(f"Error transforming ElevenLabs response: {e}\nResponse: {raw_response.text}")
|
||||
raise ValueError(f"Error transforming ElevenLabs response: {e}") from None
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -138,13 +138,13 @@ class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
except ValueError:
|
||||
raise GeminiError(
|
||||
status_code=raw_response.status_code,
|
||||
message=f"Received non-JSON response from Gemini Interactions API: {raw_response.text}",
|
||||
message="Received non-JSON response from Gemini Interactions API",
|
||||
)
|
||||
parsed: Final = GeminiTranscriptionInteractionResponse.model_validate(response_json)
|
||||
if parsed.status != "completed":
|
||||
raise GeminiError(
|
||||
status_code=raw_response.status_code,
|
||||
message=f"Gemini transcription interaction did not complete (status={parsed.status}): {raw_response.text}",
|
||||
message=f"Gemini transcription interaction did not complete (status={parsed.status})",
|
||||
)
|
||||
text_contents: Final = tuple(
|
||||
content
|
||||
|
|
|
|||
|
|
@ -14,7 +14,9 @@ from openai.types.file_deleted import FileDeleted
|
|||
from typing_extensions import ReadOnly, Required
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
|
||||
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
|
||||
extract_file_data,
|
||||
)
|
||||
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
|
||||
from litellm.llms.base_llm.files.transformation import (
|
||||
BaseFilesConfig,
|
||||
|
|
|
|||
|
|
@ -139,6 +139,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
|
|||
"response_format",
|
||||
"response_mime_type",
|
||||
"previous_interaction_id",
|
||||
"continuation_token",
|
||||
]
|
||||
|
||||
def validate_environment(
|
||||
|
|
@ -219,6 +220,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
|
|||
"environment",
|
||||
"response_modalities",
|
||||
"previous_interaction_id",
|
||||
"continuation_token",
|
||||
]
|
||||
dropped_keys: Final = (
|
||||
frozenset({"agent_config"})
|
||||
|
|
@ -314,7 +316,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
|
|||
) -> InteractionsAPIStreamingResponse:
|
||||
"""Parse streaming chunk."""
|
||||
verbose_logger.debug("Google AI Interactions streaming chunk: %s", parsed_chunk)
|
||||
return InteractionsAPIStreamingResponse(**parsed_chunk)
|
||||
return InteractionsAPIStreamingResponse.model_validate(parsed_chunk)
|
||||
|
||||
# GET / DELETE / CANCEL - just build URLs, responses match spec directly
|
||||
|
||||
|
|
|
|||
|
|
@ -7,10 +7,10 @@ import json
|
|||
from collections.abc import Coroutine
|
||||
from typing import Final, Literal, cast, overload
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type
|
||||
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
|
||||
get_image_mime_type_from_url,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import (
|
||||
reasoning_effort_from_thinking,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -20,7 +20,9 @@ from pydantic import ConfigDict, TypeAdapter
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
|
||||
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
|
||||
extract_file_data,
|
||||
)
|
||||
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.files.transformation import (
|
||||
|
|
|
|||
|
|
@ -133,7 +133,7 @@ class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
response_json: Final = raw_response.json()
|
||||
except Exception:
|
||||
raise MistralAudioTranscriptionException(
|
||||
message=raw_response.text,
|
||||
message="Received non-JSON response from Mistral audio transcription",
|
||||
status_code=raw_response.status_code,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -17,7 +17,9 @@ from openai.types.file_deleted import FileDeleted
|
|||
from pydantic import ConfigDict
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
|
||||
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
|
||||
extract_file_data,
|
||||
)
|
||||
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.files.transformation import BaseFilesConfig, LiteLLMLoggingObj
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
|||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
convert_content_list_to_str,
|
||||
extract_images_from_message,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
extract_reasoning_content,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
|
||||
|
|
@ -353,7 +355,7 @@ class OllamaChatConfig(BaseConfig):
|
|||
del response_json_message["thinking"]
|
||||
elif response_json_message.get("content") is not None:
|
||||
# parse reasoning content from content
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
parse_content_for_reasoning,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ class _OllamaGenerateReasoning(LiteLLMBaseModel):
|
|||
def split(self) -> tuple[str | None, str | None]:
|
||||
"""Reasoning reaches `/api/generate` either in the top-level `thinking` field or
|
||||
inline in `<think>` tags, never both. The field wins, matching `ollama_chat`."""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
|
||||
parse_content_for_reasoning,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -21,20 +21,24 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
|
|||
should_convert_tool_call_to_json_mode,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
chat_completion_prompt_cache_breakpoints_for_wire,
|
||||
drop_non_python_regex_patterns,
|
||||
drop_tool_reference_parts_from_tool_messages,
|
||||
flatten_combinators_and_drop_non_python_regex_patterns,
|
||||
get_tool_call_names,
|
||||
hoist_images_from_tool_messages,
|
||||
system_messages_first,
|
||||
tool_with_sanitized_parameters,
|
||||
without_prompt_cache_breakpoints,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
||||
async_convert_url_to_base64,
|
||||
convert_url_to_base64,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.json_schema_utils import (
|
||||
drop_non_python_regex_patterns,
|
||||
flatten_combinators_and_drop_non_python_regex_patterns,
|
||||
tool_with_sanitized_parameters,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
|
||||
chat_completion_prompt_cache_breakpoints_for_wire,
|
||||
without_prompt_cache_breakpoints,
|
||||
)
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
from litellm.llms.base_llm.base_utils import (
|
||||
|
|
|
|||
|
|
@ -21,10 +21,12 @@ from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
|||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
safe_convert_created_field,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
cache_control_as_prompt_cache_breakpoint,
|
||||
from litellm.litellm_core_utils.prompt_templates.json_schema_utils import (
|
||||
drop_non_python_regex_patterns,
|
||||
flatten_combinators_and_drop_non_python_regex_patterns,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
|
||||
cache_control_as_prompt_cache_breakpoint,
|
||||
prompt_cache_breakpoint_for_wire,
|
||||
whole_prompt_cache_breakpoint_or_none,
|
||||
with_prompt_cache_breakpoint,
|
||||
|
|
|
|||
|
|
@ -141,7 +141,4 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
if any(key in raw_response_json for key in TranscriptionResponse.model_fields):
|
||||
return TranscriptionResponse(**raw_response_json)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Invalid response format. Received response does not match the expected format. Got: ",
|
||||
raw_response_json,
|
||||
)
|
||||
raise ValueError("Invalid response format. Received response does not match the expected format.")
|
||||
|
|
|
|||
|
|
@ -144,7 +144,7 @@ class OVHCloudAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
response_json: Final = raw_response.json()
|
||||
except Exception:
|
||||
raise OVHCloudException(
|
||||
message=raw_response.text,
|
||||
message="Received non-JSON response from OVHCloud audio transcription",
|
||||
status_code=raw_response.status_code,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ class ScalewayAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
response_json: Final = raw_response.json()
|
||||
except Exception:
|
||||
raise ScalewayAudioTranscriptionException(
|
||||
message=raw_response.text,
|
||||
message="Received non-JSON response from Scaleway audio transcription",
|
||||
status_code=raw_response.status_code,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -183,7 +183,7 @@ class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexB
|
|||
except ValueError:
|
||||
raise VertexAIError(
|
||||
status_code=raw_response.status_code,
|
||||
message=f"Received non-JSON response from Vertex AI Gemini transcription: {raw_response.text}",
|
||||
message="Received non-JSON response from Vertex AI Gemini transcription",
|
||||
)
|
||||
parsed: Final = VertexGeminiTranscriptionResponse.model_validate(response_json)
|
||||
texts: Final = tuple(
|
||||
|
|
|
|||
|
|
@ -172,7 +172,7 @@ class VertexAIAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase)
|
|||
except ValueError:
|
||||
raise VertexAIError(
|
||||
status_code=raw_response.status_code,
|
||||
message=f"Received non-JSON response from Google Speech-to-Text: {raw_response.text}",
|
||||
message="Received non-JSON response from Google Speech-to-Text",
|
||||
)
|
||||
parsed: Final = VertexSpeechToTextRecognizeResponse.model_validate(response_json)
|
||||
transcripts: Final = tuple(
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ from litellm.litellm_core_utils.cloud_storage_security import (
|
|||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, iter_items
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
|
||||
extract_file_data,
|
||||
extract_file_metadata,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -20,9 +20,6 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, has_attributes
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_image_mime_type_from_url,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
GeminiMediaSource,
|
||||
convert_to_anthropic_image_obj,
|
||||
|
|
@ -33,6 +30,9 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
|
|||
get_thought_signature_from_tool,
|
||||
response_schema_prompt,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
|
||||
get_image_mime_type_from_url,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia, async_inline_remote_media
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran
|
|||
try:
|
||||
raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json())
|
||||
except Exception as e:
|
||||
raise ValueError(f"Error transforming response to json: {e}\nResponse: {raw_response.text}")
|
||||
raise ValueError(f"Error transforming response to json: {e}")
|
||||
|
||||
# Extract only valid fields for TranscriptionResponse.__init__()
|
||||
# TranscriptionResponse only accepts 'text' and 'usage' in __init__()
|
||||
|
|
@ -191,10 +191,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran
|
|||
response_kwargs["usage"] = usage
|
||||
|
||||
if not response_kwargs:
|
||||
raise ValueError(
|
||||
"Invalid response format. Received response does not match the expected format. Got: ",
|
||||
response_object,
|
||||
)
|
||||
raise ValueError("Invalid response format. Received response does not match the expected format.")
|
||||
|
||||
response: Final = TranscriptionResponse(**response_kwargs)
|
||||
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ class _XAISttWord(LiteLLMBaseModel):
|
|||
|
||||
|
||||
class _XAISttResponse(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
model_config = ConfigDict(extra="allow", hide_input_in_errors=True)
|
||||
text: str = ""
|
||||
language: str = "unknown"
|
||||
duration: float | None = None
|
||||
|
|
|
|||
|
|
@ -13,7 +13,9 @@ from openai.types.file_deleted import FileDeleted
|
|||
from pydantic import ConfigDict
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
|
||||
from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
|
||||
extract_file_data,
|
||||
)
|
||||
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.files.transformation import BaseFilesConfig, LiteLLMLoggingObj
|
||||
|
|
|
|||
|
|
@ -144,7 +144,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
bedrock_route_for_request,
|
||||
bedrock_strict_tools_fallback,
|
||||
deployment_model_info,
|
||||
without_bedrock_native_output_opt_in,
|
||||
for_bedrock_json_tool_retry,
|
||||
without_bedrock_route_prefix,
|
||||
)
|
||||
from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config
|
||||
|
|
@ -4374,7 +4374,7 @@ def _native_output_fallback(
|
|||
first_attempt_params: Mapping[str, object],
|
||||
first_attempt_litellm_params: Mapping[str, object],
|
||||
) -> NativeOutputFallback:
|
||||
litellm_params: Final = without_bedrock_native_output_opt_in(first_attempt_litellm_params)
|
||||
litellm_params: Final = for_bedrock_json_tool_retry(first_attempt_litellm_params)
|
||||
remapped_params: Final = ctx.optional_params_for_model_info(deployment_model_info(litellm_params))
|
||||
return NativeOutputFallback(
|
||||
optional_params={
|
||||
|
|
@ -4505,7 +4505,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu
|
|||
converse_attempt,
|
||||
(
|
||||
partial(_converse_native_output_fallback, ctx, converse_params, dict(litellm_params))
|
||||
if AmazonConverseConfig().uses_opt_in_native_output(converse_model, converse_params, litellm_params)
|
||||
if AmazonConverseConfig().uses_native_output(converse_model, converse_params, litellm_params)
|
||||
else None
|
||||
),
|
||||
converse_model,
|
||||
|
|
@ -4561,7 +4561,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu
|
|||
(
|
||||
partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params))
|
||||
if isinstance(provider_config, AmazonAnthropicClaudeConfig)
|
||||
and provider_config.uses_opt_in_native_output(model, invoke_params, litellm_params)
|
||||
and provider_config.uses_native_output(model, invoke_params, litellm_params)
|
||||
else None
|
||||
),
|
||||
model,
|
||||
|
|
|
|||
|
|
@ -2242,7 +2242,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -2331,7 +2331,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -2420,7 +2420,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -2504,7 +2504,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -2593,7 +2593,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -2715,7 +2715,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -2759,7 +2759,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -2803,7 +2803,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -2847,7 +2847,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -2891,7 +2891,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -4046,6 +4046,8 @@
|
|||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
"deprecation_date": "2027-09-01",
|
||||
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
|
|
@ -4068,7 +4070,7 @@
|
|||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_max_reasoning_effort": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models"
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement"
|
||||
},
|
||||
"azure_ai/claude-opus-5": {
|
||||
"deprecation_date": "2027-07-08",
|
||||
|
|
@ -13458,13 +13460,16 @@
|
|||
"supports_reasoning": true
|
||||
},
|
||||
"azure_ai/mistral-document-ai-2512": {
|
||||
"deprecation_date": "2027-02-15",
|
||||
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
|
||||
"litellm_provider": "azure_ai",
|
||||
"ocr_cost_per_page": 0.003,
|
||||
"mode": "ocr",
|
||||
"supported_endpoints": [
|
||||
"/v1/ocr"
|
||||
],
|
||||
"source": "https://ai.azure.com/catalog/models/mistral-document-ai-2512"
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
|
||||
"successor_model": "azure_ai/mistral-ocr-4-0"
|
||||
},
|
||||
"azure_ai/mistral-ocr-4-0": {
|
||||
"litellm_provider": "azure_ai",
|
||||
|
|
@ -75499,7 +75504,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -75539,7 +75544,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -76502,6 +76507,33 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"us-gov.xai.grok-4.7": {
|
||||
"supports_regex_lookaround": false,
|
||||
"supported_endpoints": ["/v1/chat/completions"],
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"cache_read_input_token_cost": 6.6e-07,
|
||||
"cache_read_input_token_cost_flex": 3.3e-07,
|
||||
"cache_read_input_token_cost_priority": 1.155e-06,
|
||||
"input_cost_per_token": 2.64e-06,
|
||||
"input_cost_per_token_batches": 1.32e-06,
|
||||
"input_cost_per_token_flex": 1.32e-06,
|
||||
"input_cost_per_token_priority": 4.62e-06,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 524288,
|
||||
"max_output_tokens": 524288,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 7.92e-06,
|
||||
"output_cost_per_token_batches": 3.96e-06,
|
||||
"output_cost_per_token_flex": 3.96e-06,
|
||||
"output_cost_per_token_priority": 1.386e-05,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"baseten/deepseek-ai/DeepSeek-V4.1-Flash-Fast": {
|
||||
"cache_read_input_token_cost": 1.4e-08,
|
||||
"input_cost_per_token": 6e-07,
|
||||
|
|
@ -78470,7 +78502,7 @@
|
|||
"cache_creation_input_token_cost": 1e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 1.6e-05,
|
||||
"cache_read_input_token_cost": 4e-07,
|
||||
"deprecation_date": "2027-10-01",
|
||||
"deprecation_date": "2027-01-15",
|
||||
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
|
||||
"input_cost_per_token": 8e-06,
|
||||
"litellm_provider": "azure_ai",
|
||||
|
|
@ -78480,7 +78512,7 @@
|
|||
"output_cost_per_token": 4e-05,
|
||||
"prompt_cache_min_tokens": 512.0,
|
||||
"search_context_cost_per_query": {"search_context_size_high":0.02,"search_context_size_low":0.02,"search_context_size_medium":0.02},
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models",
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
|
|
@ -86471,5 +86503,32 @@
|
|||
"supports_system_messages": false,
|
||||
"supports_video_input": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"databricks/databricks-kimi-k3-1-pro": {
|
||||
"cache_creation_input_token_cost": 1.99997e-06,
|
||||
"cache_read_input_token_cost": 1.9999e-07,
|
||||
"input_cost_per_token": 1.99997e-06,
|
||||
"input_dbu_cost_per_token": 2.8571e-05,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"metadata": {"notes":"Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference."},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 9.99999e-06,
|
||||
"output_dbu_cost_per_token": 0.000142857,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"token_count_framing": 100.0
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,6 +41,14 @@ class PluginArchiveMetadata:
|
|||
description: str | None = None
|
||||
has_agent_skill_metadata: bool = False
|
||||
updated_at: datetime | None = None
|
||||
created_at: datetime | None = None
|
||||
created_by: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PluginReview:
|
||||
approved_by: str
|
||||
security_scan_passed: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -51,6 +59,7 @@ class PluginPublication:
|
|||
manifest: Mapping[str, object]
|
||||
created_by: str | None
|
||||
content: bytes | None = None
|
||||
review: PluginReview | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -66,6 +75,10 @@ class PublishedPlugin:
|
|||
version: str
|
||||
enabled: bool
|
||||
updated_at: datetime
|
||||
created_at: datetime | None = None
|
||||
created_by: str | None = None
|
||||
review: PluginReview | None = None
|
||||
published_by: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
|
|||
|
|
@ -7683,6 +7683,17 @@
|
|||
"PluginListItem": {
|
||||
"description": "Plugin item in list responses.",
|
||||
"properties": {
|
||||
"approved_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Approved By"
|
||||
},
|
||||
"author": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
|
@ -7715,6 +7726,17 @@
|
|||
],
|
||||
"title": "Created At"
|
||||
},
|
||||
"created_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Created By"
|
||||
},
|
||||
"description": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
|
@ -7737,6 +7759,17 @@
|
|||
],
|
||||
"title": "Domain"
|
||||
},
|
||||
"download_count": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Download Count"
|
||||
},
|
||||
"enabled": {
|
||||
"title": "Enabled",
|
||||
"type": "boolean"
|
||||
|
|
@ -7801,6 +7834,22 @@
|
|||
],
|
||||
"title": "Namespace"
|
||||
},
|
||||
"published_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Published By"
|
||||
},
|
||||
"security_scan_passed": {
|
||||
"default": false,
|
||||
"title": "Security Scan Passed",
|
||||
"type": "boolean"
|
||||
},
|
||||
"source": {
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
|
|
@ -59415,6 +59464,14 @@
|
|||
},
|
||||
"title": "Skills",
|
||||
"type": "array"
|
||||
},
|
||||
"tags": {
|
||||
"default": [],
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": "Tags",
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
|
@ -59690,6 +59747,17 @@
|
|||
},
|
||||
"SkillHubSkill": {
|
||||
"properties": {
|
||||
"approved_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Approved By"
|
||||
},
|
||||
"content_url": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
|
@ -59701,6 +59769,29 @@
|
|||
],
|
||||
"title": "Content Url"
|
||||
},
|
||||
"created_at": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "date-time",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Created At"
|
||||
},
|
||||
"created_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Created By"
|
||||
},
|
||||
"description": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
|
@ -59732,7 +59823,7 @@
|
|||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Gateway ZIP downloads, included in proxy-admin catalog listings",
|
||||
"description": "Gateway ZIP downloads, including repeat downloads",
|
||||
"title": "Download Count"
|
||||
},
|
||||
"download_url": {
|
||||
|
|
@ -59779,6 +59870,22 @@
|
|||
],
|
||||
"title": "Namespace"
|
||||
},
|
||||
"published_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Published By"
|
||||
},
|
||||
"security_scan_passed": {
|
||||
"default": false,
|
||||
"title": "Security Scan Passed",
|
||||
"type": "boolean"
|
||||
},
|
||||
"source": {
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
|
|
@ -60434,6 +60541,66 @@
|
|||
"title": "Limit",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
{
|
||||
"in": "query",
|
||||
"name": "tag",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"default": "",
|
||||
"title": "Tag",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"in": "query",
|
||||
"name": "security_scan_passed",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Security Scan Passed"
|
||||
}
|
||||
},
|
||||
{
|
||||
"in": "query",
|
||||
"name": "sort_by",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"default": "name",
|
||||
"enum": [
|
||||
"name",
|
||||
"download_count",
|
||||
"updated_at",
|
||||
"created_at",
|
||||
"created_by",
|
||||
"published_by",
|
||||
"approved_by",
|
||||
"security_scan_passed"
|
||||
],
|
||||
"title": "Sort By",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"in": "query",
|
||||
"name": "sort_order",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"default": "asc",
|
||||
"enum": [
|
||||
"asc",
|
||||
"desc"
|
||||
],
|
||||
"title": "Sort Order",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
|
|
|
|||
|
|
@ -2813,6 +2813,14 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
|
|||
description="How long a blocked source address, or source address and username, stays blocked. Every attempt from a blocked key, right or wrong, is refused with 429 before the password is checked; the block is not extended by refused attempts. Set under `general_settings` in config.yaml. Defaults to 300",
|
||||
)
|
||||
allowed_routes: list | None = Field(None, description="Proxy API Endpoints you want users to be able to access")
|
||||
allowed_client_side_s3_buckets: tuple[str, ...] | None = Field(
|
||||
None,
|
||||
description="S3 bucket names a client may send as `s3_bucket_name` or `s3_output_bucket_name` in a request body, e.g. on /v1/batches. Any other bucket is rejected unless client-side credentials are allowed",
|
||||
)
|
||||
allowed_client_side_s3_endpoint_urls: tuple[str, ...] | None = Field(
|
||||
None,
|
||||
description="Exact `s3_endpoint_url` values a client may send in a request body. Any other value is rejected unless client-side credentials are allowed",
|
||||
)
|
||||
reject_clientside_metadata_tags: bool | None = Field(
|
||||
None,
|
||||
description="When set to True, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata.",
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ from litellm.proxy.common_utils.claude_code_marketplace_auth import (
|
|||
)
|
||||
from litellm.proxy.common_utils.resource_ownership import is_proxy_admin
|
||||
from litellm.proxy.deps.repositories import PluginUploadRepositoryDep
|
||||
from litellm.repositories.plugin_publication import read_plugin_publisher, read_plugin_review
|
||||
from litellm.repositories.table_repositories import ClaudeCodePluginRepository
|
||||
from litellm.types.proxy.auth.special_headers import SpecialHeaders
|
||||
from litellm.types.proxy.claude_code_endpoints import (
|
||||
|
|
@ -666,6 +667,7 @@ async def register_plugin(
|
|||
response_model=ListPluginsResponse,
|
||||
)
|
||||
async def list_plugins(
|
||||
uploads: PluginUploadRepositoryDep,
|
||||
enabled_only: bool = False,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
) -> ListPluginsResponse:
|
||||
|
|
@ -678,6 +680,17 @@ async def list_plugins(
|
|||
Returns:
|
||||
List of plugins with their metadata.
|
||||
"""
|
||||
catalog: Final = await list_visible_plugins(enabled_only, user_api_key_dict)
|
||||
if uploads is None:
|
||||
raise _error_response(500, CommonProxyErrors.db_not_connected_error.value)
|
||||
counts: Final = await uploads.download_counts(tuple(plugin.id for plugin in catalog.plugins))
|
||||
return ListPluginsResponse(
|
||||
plugins=[plugin.model_copy(update={"download_count": counts.get(plugin.id, 0)}) for plugin in catalog.plugins],
|
||||
count=catalog.count,
|
||||
)
|
||||
|
||||
|
||||
async def list_visible_plugins(enabled_only: bool, user_api_key_dict: UserAPIKeyAuth) -> ListPluginsResponse:
|
||||
try:
|
||||
prisma_client: Final = await get_prisma_client()
|
||||
|
||||
|
|
@ -693,9 +706,14 @@ async def list_plugins(
|
|||
# Parse manifest to get additional fields
|
||||
manifest = json.loads(p.manifest_json) if p.manifest_json else {}
|
||||
|
||||
review = read_plugin_review(MANIFEST_FIELDS.validate_python(manifest))
|
||||
plugin_list.append(
|
||||
PluginListItem(
|
||||
id=p.id,
|
||||
created_by=p.created_by,
|
||||
approved_by=review.approved_by if review else None,
|
||||
published_by=read_plugin_publisher(MANIFEST_FIELDS.validate_python(manifest)),
|
||||
security_scan_passed=review.security_scan_passed if review else False,
|
||||
name=p.name,
|
||||
version=p.version,
|
||||
description=p.description,
|
||||
|
|
|
|||
|
|
@ -7,10 +7,10 @@ from functools import lru_cache
|
|||
from itertools import accumulate, chain, repeat
|
||||
from logging import Logger
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, Protocol
|
||||
from typing import Any, Final, Literal, Protocol, TypeAlias
|
||||
|
||||
from fastapi import HTTPException, Request, status
|
||||
from pydantic import PositiveInt, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, ConfigDict, PositiveInt, TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm import Router, constants, provider_list
|
||||
|
|
@ -442,6 +442,9 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = (
|
|||
# S3; ``sagemaker_base_url`` redirects all SageMaker traffic;
|
||||
# ``deployment_url`` redirects SAP deployments.
|
||||
"s3_endpoint_url",
|
||||
"s3_bucket_name",
|
||||
"s3_output_bucket_name",
|
||||
"allow_legacy_cloud_file_ids",
|
||||
"sagemaker_base_url",
|
||||
"deployment_url",
|
||||
# NVIDIA Riva fields consumed by the audio-transcription handler
|
||||
|
|
@ -479,6 +482,48 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = (
|
|||
)
|
||||
|
||||
|
||||
_ClientSideAllowlistSetting: TypeAlias = Literal[
|
||||
"allowed_client_side_s3_buckets", "allowed_client_side_s3_endpoint_urls"
|
||||
]
|
||||
|
||||
_PROXY_ALLOWLIST_SETTING_BY_PARAM: Final[Mapping[str, _ClientSideAllowlistSetting]] = MappingProxyType(
|
||||
{
|
||||
"s3_bucket_name": "allowed_client_side_s3_buckets",
|
||||
"s3_output_bucket_name": "allowed_client_side_s3_buckets",
|
||||
"s3_endpoint_url": "allowed_client_side_s3_endpoint_urls",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class _ClientSideStorageAllowlist(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
allowed_client_side_s3_buckets: tuple[str, ...] | None = None
|
||||
allowed_client_side_s3_endpoint_urls: tuple[str, ...] | None = None
|
||||
|
||||
def admits(self, setting: _ClientSideAllowlistSetting, value: object) -> bool:
|
||||
match setting:
|
||||
case "allowed_client_side_s3_buckets":
|
||||
return value in (self.allowed_client_side_s3_buckets or ())
|
||||
case "allowed_client_side_s3_endpoint_urls":
|
||||
return value in (self.allowed_client_side_s3_endpoint_urls or ())
|
||||
|
||||
|
||||
def _banned_param_error(param: str) -> ValueError:
|
||||
setting: Final = _PROXY_ALLOWLIST_SETTING_BY_PARAM.get(param)
|
||||
allowlist_hint: Final = (
|
||||
"" if setting is None else f" To allow specific values only, list them in `general_settings.{setting}`."
|
||||
)
|
||||
return ValueError(
|
||||
f"Rejected Request: {param} is not allowed in request body. "
|
||||
"Clientside passthrough requires explicit admin opt-in via "
|
||||
"either `general_settings.allow_client_side_credentials = true` "
|
||||
"(proxy-wide) or `configurable_clientside_auth_params` on the "
|
||||
"deployment in your proxy config.yaml. "
|
||||
f"Relevant Issue: https://huntr.com/bounties/4001e1a2-7b7a-4776-a3ae-e6692ec3d997{allowlist_hint}",
|
||||
)
|
||||
|
||||
|
||||
def _check_banned_params(
|
||||
body: Mapping[str, object],
|
||||
general_settings: dict,
|
||||
|
|
@ -508,6 +553,10 @@ def _check_banned_params(
|
|||
# Proxy-wide opt-in: every banned param is permitted, exit
|
||||
# entirely so the rest of the loop doesn't waste work.
|
||||
return
|
||||
if (
|
||||
setting := _PROXY_ALLOWLIST_SETTING_BY_PARAM.get(param)
|
||||
) is not None and _ClientSideStorageAllowlist.model_validate(general_settings).admits(setting, body[param]):
|
||||
continue
|
||||
if (
|
||||
_allow_model_level_clientside_configurable_parameters(
|
||||
model=model,
|
||||
|
|
@ -523,14 +572,7 @@ def _check_banned_params(
|
|||
# ``api_base`` with an unallowed ``langfuse_host`` is still
|
||||
# rejected for the second field.
|
||||
continue
|
||||
raise ValueError(
|
||||
f"Rejected Request: {param} is not allowed in request body. "
|
||||
"Clientside passthrough requires explicit admin opt-in via "
|
||||
"either `general_settings.allow_client_side_credentials = true` "
|
||||
"(proxy-wide) or `configurable_clientside_auth_params` on the "
|
||||
"deployment in your proxy config.yaml. "
|
||||
"Relevant Issue: https://huntr.com/bounties/4001e1a2-7b7a-4776-a3ae-e6692ec3d997",
|
||||
)
|
||||
raise _banned_param_error(param)
|
||||
|
||||
|
||||
_FALLBACK_FIELDS: Final[tuple[str, ...]] = (
|
||||
|
|
|
|||
|
|
@ -49,7 +49,6 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( # noqa: F401 #
|
|||
BATCH_CREATE_HIDDEN_PARAM,
|
||||
_is_base64_encoded_unified_file_id, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
add_deployment_model_info,
|
||||
add_internal_model_credentials,
|
||||
apply_team_provider_credentials,
|
||||
authorize_model_for_key,
|
||||
batch_cost_poller_is_active,
|
||||
|
|
@ -763,13 +762,6 @@ async def retrieve_batch(
|
|||
detail={"error": "LLM Router not initialized. Ensure models added to proxy."},
|
||||
)
|
||||
|
||||
if unified_batch_id:
|
||||
add_internal_model_credentials(
|
||||
data=data,
|
||||
llm_router=llm_router,
|
||||
model_id=unified_model_id,
|
||||
)
|
||||
|
||||
response = await llm_router.aretrieve_batch(**data)
|
||||
set_hidden_param(response, "unified_batch_id", unified_batch_id)
|
||||
if unified_batch_id:
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ def _fips_mode_setting(general_settings: Mapping[str, object]) -> bool | None:
|
|||
|
||||
|
||||
def openssl_enforces_fips() -> bool:
|
||||
"""Python's OpenSSL runs with FIPS-only defaults and itself refuses MD5."""
|
||||
"""Python's OpenSSL runs with FIPS-only defaults and itself refuses scrypt."""
|
||||
try:
|
||||
import _hashlib
|
||||
|
||||
|
|
@ -32,7 +32,7 @@ def openssl_enforces_fips() -> bool:
|
|||
except (ImportError, AttributeError, ValueError):
|
||||
return False
|
||||
try:
|
||||
_hashlib.new("md5", b"", usedforsecurity=True)
|
||||
_hashlib.scrypt(b"", salt=b"\0" * 16, n=2, r=1, p=1, dklen=16)
|
||||
except ValueError:
|
||||
return True
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@ import hashlib
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Final
|
||||
from itertools import chain
|
||||
from typing import Annotated, Final, Literal
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
|
|
@ -33,7 +34,7 @@ from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketp
|
|||
enable_plugin,
|
||||
get_plugin_downloads,
|
||||
header_key_caller,
|
||||
list_plugins,
|
||||
list_visible_plugins,
|
||||
plugin_archive_source_matches,
|
||||
plugin_archive_url,
|
||||
plugin_downloader,
|
||||
|
|
@ -65,6 +66,7 @@ from litellm.proxy.discovery_endpoints.skill_import import (
|
|||
parse_skill_source,
|
||||
select_imported_skill,
|
||||
)
|
||||
from litellm.repositories.plugin_publication import read_plugin_publisher, read_plugin_review
|
||||
from litellm.types.proxy.claude_code_endpoints import PluginListItem
|
||||
from litellm.types.proxy.discovery_endpoints.agent_skills_endpoints import AgentSkillsIndexEntry
|
||||
from litellm.types.proxy.skill_hub import (
|
||||
|
|
@ -78,6 +80,7 @@ from litellm.types.proxy.skill_hub import (
|
|||
SkillHubSkill,
|
||||
SkillHubVersion,
|
||||
SkillHubVersions,
|
||||
SkillSortField,
|
||||
)
|
||||
|
||||
router: Final = APIRouter(prefix="/skill-hub/skills", tags=["Skills Hub"])
|
||||
|
|
@ -121,6 +124,11 @@ def _skill_response(request: Request, plugin: PluginListItem, download_count: in
|
|||
namespace=plugin.namespace,
|
||||
keywords=None if plugin.keywords is None else tuple(plugin.keywords),
|
||||
updated_at=None if plugin.updated_at is None else datetime.fromisoformat(plugin.updated_at),
|
||||
created_at=None if plugin.created_at is None else datetime.fromisoformat(plugin.created_at),
|
||||
created_by=plugin.created_by,
|
||||
approved_by=plugin.approved_by,
|
||||
published_by=plugin.published_by,
|
||||
security_scan_passed=plugin.security_scan_passed,
|
||||
visibility="public" if plugin.enabled else "private",
|
||||
source={"source": "archive", "url": _download_url(request, plugin.name)} if local_archive else plugin.source,
|
||||
download_url=_download_url(request, plugin.name) if local_archive else None,
|
||||
|
|
@ -137,28 +145,74 @@ async def list_skills(
|
|||
query: str = "",
|
||||
offset: Annotated[int, Query(ge=0)] = 0,
|
||||
limit: Annotated[int, Query(ge=1, le=100)] = 50,
|
||||
tag: str = "",
|
||||
security_scan_passed: bool | None = None,
|
||||
sort_by: SkillSortField = "name",
|
||||
sort_order: Literal["asc", "desc"] = "asc",
|
||||
) -> SkillHubList:
|
||||
"""Browse the caller's public and granted private skills."""
|
||||
catalog: Final = await list_plugins(enabled_only=False, user_api_key_dict=caller)
|
||||
needle: Final = query.casefold()
|
||||
catalog: Final = await list_visible_plugins(enabled_only=False, user_api_key_dict=caller)
|
||||
needle: Final = query.strip().casefold()
|
||||
selected_tag: Final = tag.strip().casefold()
|
||||
matched: Final = tuple(
|
||||
plugin
|
||||
for plugin in sorted(catalog.plugins, key=lambda item: item.name)
|
||||
if needle in " ".join((plugin.name, plugin.description or "", *(plugin.keywords or ()))).casefold()
|
||||
if needle
|
||||
in " ".join(
|
||||
(
|
||||
plugin.name,
|
||||
plugin.description or "",
|
||||
plugin.domain or "",
|
||||
plugin.namespace or "",
|
||||
plugin.created_by or "",
|
||||
plugin.approved_by or "",
|
||||
plugin.published_by or "",
|
||||
*(plugin.keywords or ()),
|
||||
)
|
||||
).casefold()
|
||||
and (not selected_tag or selected_tag in {word.casefold() for word in plugin.keywords or ()})
|
||||
and (security_scan_passed is None or plugin.security_scan_passed == security_scan_passed)
|
||||
)
|
||||
page: Final = matched[offset : offset + limit]
|
||||
counts: Final[Mapping[str, int]] = (
|
||||
await _require_store(uploads).download_counts(tuple(plugin.id for plugin in page))
|
||||
if is_proxy_admin(caller)
|
||||
await _require_store(uploads).download_counts(tuple(plugin.id for plugin in matched))
|
||||
if sort_by == "download_count"
|
||||
else {}
|
||||
)
|
||||
ordered: Final = sorted(
|
||||
matched, key=lambda plugin: _skill_sort_key(plugin, sort_by, counts), reverse=sort_order == "desc"
|
||||
)
|
||||
page: Final = ordered[offset : offset + limit]
|
||||
page_counts: Final = (
|
||||
await _require_store(uploads).download_counts(tuple(plugin.id for plugin in page))
|
||||
if sort_by != "download_count"
|
||||
else counts
|
||||
)
|
||||
return SkillHubList(
|
||||
skills=tuple(_skill_response(request, plugin, counts.get(plugin.id)) for plugin in page),
|
||||
skills=tuple(_skill_response(request, plugin, page_counts.get(plugin.id)) for plugin in page),
|
||||
count=len(matched),
|
||||
next_offset=offset + limit if offset + limit < len(matched) else None,
|
||||
tags=tuple(
|
||||
sorted(set(chain.from_iterable(plugin.keywords or () for plugin in catalog.plugins)), key=str.casefold)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _skill_sort_key(plugin: PluginListItem, field: SkillSortField, counts: Mapping[str, int]) -> str | float:
|
||||
if field == "download_count":
|
||||
return counts.get(plugin.id, 0)
|
||||
if field == "security_scan_passed":
|
||||
return int(plugin.security_scan_passed)
|
||||
if field in {"created_at", "updated_at"}:
|
||||
value: Final = plugin.created_at if field == "created_at" else plugin.updated_at
|
||||
return datetime.fromisoformat(value).timestamp() if value else 0
|
||||
return {
|
||||
"name": plugin.name,
|
||||
"created_by": plugin.created_by or "",
|
||||
"published_by": plugin.published_by or "",
|
||||
"approved_by": plugin.approved_by or "",
|
||||
}[field].casefold()
|
||||
|
||||
|
||||
@router.get("/{name}", response_model=SkillHubSkill)
|
||||
async def get_skill(
|
||||
request: Request,
|
||||
|
|
@ -176,6 +230,7 @@ async def get_skill(
|
|||
|
||||
def skill_metadata_response(request: Request, metadata: PluginArchiveMetadata) -> SkillHubSkill:
|
||||
manifest: Final = MANIFEST_FIELDS.validate_json(metadata.manifest_json or "{}")
|
||||
review: Final = read_plugin_review(manifest)
|
||||
return _skill_response(
|
||||
request,
|
||||
PluginListItem.model_validate(
|
||||
|
|
@ -187,7 +242,11 @@ def skill_metadata_response(request: Request, metadata: PluginArchiveMetadata) -
|
|||
"source": dict(metadata.source),
|
||||
"has_agent_skill_metadata": metadata.has_agent_skill_metadata,
|
||||
"enabled": metadata.enabled,
|
||||
"created_at": None,
|
||||
"created_at": metadata.created_at.isoformat() if metadata.created_at is not None else None,
|
||||
"created_by": metadata.created_by,
|
||||
"approved_by": review.approved_by if review else None,
|
||||
"published_by": read_plugin_publisher(manifest),
|
||||
"security_scan_passed": review.security_scan_passed if review else False,
|
||||
"updated_at": metadata.updated_at.isoformat() if metadata.updated_at is not None else None,
|
||||
"domain": manifest.get("domain"),
|
||||
"namespace": manifest.get("namespace"),
|
||||
|
|
@ -416,7 +475,11 @@ def skill_publication_response(
|
|||
"version": published.version,
|
||||
"enabled": published.enabled,
|
||||
"has_agent_skill_metadata": has_agent_skill_metadata(publication.manifest),
|
||||
"created_at": None,
|
||||
"created_at": published.created_at.isoformat() if published.created_at is not None else None,
|
||||
"created_by": published.created_by,
|
||||
"approved_by": published.review.approved_by if published.review else None,
|
||||
"published_by": published.published_by,
|
||||
"security_scan_passed": published.review.security_scan_passed if published.review else False,
|
||||
"updated_at": published.updated_at.isoformat(),
|
||||
"domain": publication.manifest.get("domain"),
|
||||
"namespace": publication.manifest.get("namespace"),
|
||||
|
|
|
|||
|
|
@ -58,6 +58,8 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_str_from_messages,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import (
|
||||
split_media_for_tpm,
|
||||
)
|
||||
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
|
||||
|
|
|
|||
|
|
@ -5483,7 +5483,9 @@ async def generate_key_helper_fn(
|
|||
if query_type == "insert_data":
|
||||
user_row = cast( # cast-ok: table_name="user" is the insert_data branch returning the user row
|
||||
"prisma_models.LiteLLM_UserTable | None",
|
||||
await prisma_client.insert_data(data=user_data, table_name="user"),
|
||||
await prisma_client.insert_data(
|
||||
data=user_data, table_name="user", allow_existing_user=request_type != "user"
|
||||
),
|
||||
)
|
||||
|
||||
if user_row is None:
|
||||
|
|
|
|||
|
|
@ -1294,7 +1294,7 @@ async def _create_user_if_not_exists(user_id: str, created_via: str = "scim_grou
|
|||
if litellm.default_internal_user_params:
|
||||
default_role = litellm.default_internal_user_params.get("user_role")
|
||||
|
||||
new_user_request: Final = NewUserRequest(
|
||||
new_user_request: Final = SCIMNewUserRequest(
|
||||
user_id=user_id,
|
||||
user_email=user_id, # We don't have email from group membership
|
||||
user_alias=None,
|
||||
|
|
@ -1873,7 +1873,7 @@ async def create_user(
|
|||
admin_group: Final = await _get_scim_admin_group()
|
||||
resolved_role: Final = _resolve_scim_user_role(user.groups or [], admin_group, default_role)
|
||||
|
||||
new_user_request: Final = NewUserRequest(
|
||||
new_user_request: Final = SCIMNewUserRequest(
|
||||
user_id=user_id,
|
||||
user_email=user_data["user_email"],
|
||||
user_alias=user_data["user_alias"],
|
||||
|
|
|
|||
|
|
@ -78,7 +78,6 @@ from litellm.proxy._types import (
|
|||
LitellmUserRoles,
|
||||
Member,
|
||||
NewTeamRequest,
|
||||
NewUserRequest,
|
||||
NewUserRequestTeam,
|
||||
NewUserResponse,
|
||||
ProxyErrorTypes,
|
||||
|
|
@ -164,6 +163,7 @@ from litellm.types.proxy.management_endpoints.ui_sso import (
|
|||
MicrosoftGraphAPIUserGroupResponse,
|
||||
MicrosoftServicePrincipalTeam,
|
||||
RoleMappings,
|
||||
SSONewUserRequest,
|
||||
TeamMappings,
|
||||
)
|
||||
from litellm.types.proxy.ui_sso import ParsedOpenIDResult
|
||||
|
|
@ -2915,7 +2915,7 @@ async def insert_sso_user(
|
|||
if user_defined_values["user_role"] is None:
|
||||
user_defined_values["user_role"] = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY
|
||||
|
||||
new_user_request: Final = NewUserRequest(
|
||||
new_user_request: Final = SSONewUserRequest(
|
||||
user_id=user_defined_values["user_id"],
|
||||
user_email=normalize_email(user_defined_values["user_email"]),
|
||||
user_alias=_get_sso_user_alias(result_openid),
|
||||
|
|
@ -2924,11 +2924,9 @@ async def insert_sso_user(
|
|||
budget_duration=user_defined_values["budget_duration"],
|
||||
sso_user_id=user_defined_values["user_id"],
|
||||
auto_create_key=False,
|
||||
metadata={"auth_provider": getattr(result_openid, "provider")} if hasattr(result_openid, "provider") else {},
|
||||
)
|
||||
|
||||
if result_openid and hasattr(result_openid, "provider"):
|
||||
new_user_request.metadata = {"auth_provider": getattr(result_openid, "provider")}
|
||||
|
||||
from litellm.proxy.proxy_server import premium_user
|
||||
|
||||
with free_sso_user_creation(new_user_request.user_id, enabled=premium_user is not True):
|
||||
|
|
|
|||
|
|
@ -742,31 +742,6 @@ def apply_team_provider_credentials(
|
|||
prepare_data_with_credentials(data=data, credentials={**credentials, **explicit_request_credentials(data)})
|
||||
|
||||
|
||||
def add_internal_model_credentials(
|
||||
data: dict,
|
||||
llm_router: "Router",
|
||||
model_id: str | None,
|
||||
) -> None:
|
||||
"""
|
||||
Attach the deployment's immutable server-side credential snapshot to a router-routed
|
||||
batch call (in-place).
|
||||
|
||||
Cost accounting for a completed batch reads the batch's output file, and the Bedrock
|
||||
file config resolves its bucket only from this snapshot, never from a request param,
|
||||
because the bucket is what managed file ids are validated against. Without it that
|
||||
read fails and the batch's cost is never recorded.
|
||||
"""
|
||||
if model_id is None:
|
||||
return
|
||||
try:
|
||||
credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
|
||||
except Exception: # noqa: BLE001 # the snapshot only enables cost accounting; a batch whose deployment no longer resolves must still be retrievable
|
||||
return
|
||||
if credentials is None:
|
||||
return
|
||||
data["_litellm_internal_model_credentials"] = MappingProxyType(dict(credentials))
|
||||
|
||||
|
||||
def add_deployment_model_info(
|
||||
data: dict,
|
||||
llm_router: Optional["Router"],
|
||||
|
|
@ -792,7 +767,6 @@ def prepare_data_with_credentials(
|
|||
data: dict,
|
||||
credentials: dict,
|
||||
file_id: str | None = None,
|
||||
include_internal_credentials: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Update data dictionary with model credentials (in-place).
|
||||
|
|
@ -801,12 +775,8 @@ def prepare_data_with_credentials(
|
|||
data: Data dictionary to update
|
||||
credentials: Credentials from router
|
||||
file_id: Optional original file_id to set (for decoded file IDs)
|
||||
include_internal_credentials: Preserve an immutable server-side snapshot
|
||||
for code paths that must distinguish proxy config from request params.
|
||||
"""
|
||||
data.update(credentials)
|
||||
if include_internal_credentials:
|
||||
data["_litellm_internal_model_credentials"] = MappingProxyType(dict(credentials))
|
||||
data.pop("custom_llm_provider", None)
|
||||
|
||||
if file_id is not None:
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ class FileContentStreamingHandler:
|
|||
data=resolved_streaming_data,
|
||||
credentials=credentials,
|
||||
file_id=original_file_id,
|
||||
include_internal_credentials=True,
|
||||
)
|
||||
resolved_streaming_data.pop("model", None)
|
||||
resolved_streaming_provider: Final = cast(str, credentials["custom_llm_provider"])
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue