mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge remote-tracking branch 'origin/main' into litellm_bedrock_gpt_tools_reasoning_responses_bridge
Keeps main's OpenAI reasoning_effort dict handling on the Responses bridge next to this branch's Bedrock bridge params, and validates additional_drop_params as a list only on the Bedrock path, so a non-list value on other providers is ignored as main expects.
This commit is contained in:
commit
ed30e41edb
240 changed files with 15453 additions and 1689 deletions
|
|
@ -19,7 +19,7 @@ jobs:
|
|||
name: interactions-spec-canary
|
||||
if: github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
|
|
@ -39,8 +39,11 @@ jobs:
|
|||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Set up the Rust bridge
|
||||
uses: ./.github/actions/rust-bridge
|
||||
|
||||
- name: Install dependencies
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy --no-install-project
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
|
||||
|
||||
- name: Run the compliance tests against Google's live spec
|
||||
id: compliance
|
||||
|
|
|
|||
7
.github/workflows/test-merge-smoke.yml
vendored
7
.github/workflows/test-merge-smoke.yml
vendored
|
|
@ -132,7 +132,7 @@ jobs:
|
|||
windows-sdk:
|
||||
name: windows-sdk
|
||||
runs-on: windows-2025
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
LITELLM_LOCAL_MODEL_COST_MAP: "True"
|
||||
defaults:
|
||||
|
|
@ -154,8 +154,11 @@ jobs:
|
|||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Set up the Rust bridge
|
||||
uses: ./.github/actions/rust-bridge
|
||||
|
||||
- name: Install dependencies
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --no-install-workspace --group dev --python 3.11
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --group dev --python 3.11
|
||||
|
||||
- name: Run the Windows tests
|
||||
run: uv run --no-sync python -m pytest --tb=short -v tests/windows_tests
|
||||
|
|
|
|||
10
.github/workflows/test-unit.yml
vendored
10
.github/workflows/test-unit.yml
vendored
|
|
@ -1306,6 +1306,8 @@ jobs:
|
|||
done
|
||||
openai-sdk-compat:
|
||||
name: OpenAI ${{ matrix.openai-version }} compatibility
|
||||
needs: rust-bridge
|
||||
if: ${{ !cancelled() }}
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
|
@ -1342,10 +1344,18 @@ jobs:
|
|||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Set up the Rust bridge
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 5
|
||||
uses: ./.github/actions/rust-bridge
|
||||
with:
|
||||
artifact: ${{ needs.rust-bridge.outputs.artifact }}
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
run: |
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group dev --group proxy-dev --extra proxy --no-install-workspace
|
||||
cp rust-bridge-dist/litellm/rust_bridge/_native.* litellm/rust_bridge/
|
||||
uv pip install "openai==${OPENAI_VERSION:?}" --exclude-newer-package openai=2026-10-07T00:00:00Z
|
||||
uv run --no-sync python -c 'import os, openai; assert openai.__version__ == os.environ["OPENAI_VERSION"], openai.__version__'
|
||||
|
||||
|
|
|
|||
|
|
@ -427,6 +427,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
|
|||
| [Watsonx Text (`watsonx_text`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [xAI (`xai`)](https://docs.litellm.ai/docs/providers/xai) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Xinference (`xinference`)](https://docs.litellm.ai/docs/providers/xinference) | | | | ✅ | | | | | | |
|
||||
| [ZeroGPU (`zerogpu`)](https://docs.litellm.ai/docs/providers/zerogpu) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
|
||||
[**Read the Docs**](https://docs.litellm.ai/docs/)
|
||||
|
||||
|
|
|
|||
|
|
@ -5,9 +5,9 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t
|
|||
import asyncio
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import replace as dataclasses_replace
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from datetime import datetime, timedelta
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tuple, cast
|
||||
|
||||
|
|
@ -28,6 +28,7 @@ from litellm.constants import (
|
|||
from litellm.repositories.table_repositories import ManagedObjectRepository
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.repositories.verification_token_repository import VerificationTokenRepository
|
||||
from litellm.utils import get_utc_datetime
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prisma import models as prisma_models
|
||||
|
|
@ -42,11 +43,14 @@ if TYPE_CHECKING:
|
|||
|
||||
|
||||
def _error_names_resource(error: Exception, resource_id: str) -> bool:
|
||||
return resource_id in re.findall(r"""[^\s"'`]+""", str(error))
|
||||
tokens: Final[list[str]] = re.findall(r"""[^\s"'`<>(]+""", str(error))
|
||||
return any(token.rstrip(".,:;)") == resource_id for token in tokens)
|
||||
|
||||
|
||||
CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost"
|
||||
RECENT_BATCH_POLL_WINDOW: Final = timedelta(hours=48)
|
||||
GIVE_UP_POLLING_AFTER: Final = timedelta(days=2 * MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
|
||||
|
||||
|
||||
PROVIDER_TERMINAL_BATCH_STATUSES: Final[Tuple[str, ...]] = (
|
||||
"completed",
|
||||
|
|
@ -91,6 +95,10 @@ def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.Li
|
|||
return TeamRepository(prisma_client).table
|
||||
|
||||
|
||||
def _unique_jobs(jobs: Sequence[ManagedObjectRow]) -> tuple[ManagedObjectRow, ...]:
|
||||
return tuple({job.id: job for job in jobs}.values())
|
||||
|
||||
|
||||
class CheckBatchCost:
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -98,6 +106,7 @@ class CheckBatchCost:
|
|||
prisma_client: "PrismaClient",
|
||||
llm_router: "Router",
|
||||
track_unmanaged_batch_cost: bool = False,
|
||||
clock: Callable[[], datetime] = get_utc_datetime,
|
||||
):
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.router import Router
|
||||
|
|
@ -106,13 +115,14 @@ class CheckBatchCost:
|
|||
self.prisma_client: PrismaClient = prisma_client
|
||||
self.llm_router: Router = llm_router
|
||||
self._track_unmanaged_batch_cost = track_unmanaged_batch_cost
|
||||
self._clock: Final = clock
|
||||
self._creator_attribution: Final = ManagedObjectCreatorAttribution(prisma_client, "CheckBatchCost")
|
||||
# Cached after the first poll cycle. Once we know the column is absent we skip
|
||||
# the guaranteed-failing primary query on every subsequent cycle.
|
||||
self._has_batch_processed_column: bool = True
|
||||
self.batch_processed_support_confirmed: bool = False
|
||||
self._poll_cursor: tuple[datetime, str] | None = None
|
||||
self._poll_before: datetime = datetime.now(timezone.utc)
|
||||
self._poll_before: datetime = clock()
|
||||
self._arrival_cursor: tuple[datetime, str] | None = None
|
||||
self._arrival_before: datetime = self._poll_before
|
||||
self._poll_arrivals: bool = False
|
||||
|
|
@ -147,27 +157,6 @@ class CheckBatchCost:
|
|||
return
|
||||
self.batch_processed_support_confirmed = True
|
||||
|
||||
async def _cleanup_stale_managed_objects(self) -> None:
|
||||
"""
|
||||
Mark managed objects older than MANAGED_OBJECT_STALENESS_CUTOFF_DAYS days
|
||||
in non-terminal states as 'stale_expired'. These will never complete and
|
||||
should not be polled.
|
||||
"""
|
||||
cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
|
||||
result: Final = await _managed_object_table(self.prisma_client).update_many(
|
||||
where={
|
||||
"file_purpose": "batch",
|
||||
"status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)},
|
||||
"created_at": {"lt": cutoff},
|
||||
},
|
||||
data={"status": "stale_expired"},
|
||||
)
|
||||
if result > 0:
|
||||
verbose_proxy_logger.warning(
|
||||
f"CheckBatchCost: marked {result} stale managed objects "
|
||||
f"(older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days) as stale_expired"
|
||||
)
|
||||
|
||||
async def _fallback_find_jobs(self) -> "Sequence[ManagedObjectRow]":
|
||||
"""Query batch jobs without the batch_processed filter (for older schemas)."""
|
||||
return await self._find_jobs(
|
||||
|
|
@ -187,20 +176,43 @@ class CheckBatchCost:
|
|||
)
|
||||
|
||||
async def _find_jobs(self, where: Mapping[str, object]) -> Sequence[ManagedObjectRow]:
|
||||
arrivals: Final = self._poll_arrivals
|
||||
page: Final = await self._find_job_page(where, arrivals=arrivals)
|
||||
self._poll_arrivals = not arrivals if page else arrivals
|
||||
if page:
|
||||
return page
|
||||
return await self._find_job_page(where, arrivals=not arrivals)
|
||||
lead_arrivals: Final = self._poll_arrivals
|
||||
self._poll_arrivals = not lead_arrivals
|
||||
lead_share: Final = MAX_OBJECTS_PER_POLL_CYCLE - MAX_OBJECTS_PER_POLL_CYCLE // 2
|
||||
lead: Final = await self._find_job_page(where, arrivals=lead_arrivals, take=lead_share)
|
||||
trail: Final = await self._find_job_page(
|
||||
where, arrivals=not lead_arrivals, take=MAX_OBJECTS_PER_POLL_CYCLE - len(lead)
|
||||
)
|
||||
spill: Final = (
|
||||
await self._find_job_page(
|
||||
where, arrivals=lead_arrivals, take=MAX_OBJECTS_PER_POLL_CYCLE - len(lead) - len(trail)
|
||||
)
|
||||
if len(lead) == lead_share
|
||||
else ()
|
||||
)
|
||||
return _unique_jobs((*lead, *trail, *spill))
|
||||
|
||||
async def _find_job_page(self, where: Mapping[str, object], *, arrivals: bool) -> Sequence[ManagedObjectRow]:
|
||||
async def _find_job_page(
|
||||
self, where: Mapping[str, object], *, arrivals: bool, take: int
|
||||
) -> Sequence[ManagedObjectRow]:
|
||||
if take == 0:
|
||||
return ()
|
||||
resumed_mid_lane: Final = (self._arrival_cursor if arrivals else self._poll_cursor) is not None
|
||||
page: Final = await self._find_job_page_from_cursor(where, arrivals=arrivals, take=take)
|
||||
if len(page) == take or not resumed_mid_lane:
|
||||
return page
|
||||
wrapped: Final = await self._find_job_page_from_cursor(where, arrivals=arrivals, take=take - len(page))
|
||||
return _unique_jobs((*page, *wrapped))
|
||||
|
||||
async def _find_job_page_from_cursor(
|
||||
self, where: Mapping[str, object], *, arrivals: bool, take: int
|
||||
) -> Sequence[ManagedObjectRow]:
|
||||
cursor: Final = self._arrival_cursor if arrivals else self._poll_cursor
|
||||
if cursor is None:
|
||||
if arrivals:
|
||||
self._arrival_before = datetime.now(timezone.utc)
|
||||
self._arrival_before = self._clock()
|
||||
else:
|
||||
self._poll_before = datetime.now(timezone.utc) - RECENT_BATCH_POLL_WINDOW
|
||||
self._poll_before = self._clock() - RECENT_BATCH_POLL_WINDOW
|
||||
before: Final = self._arrival_before if arrivals else self._poll_before
|
||||
page: Final = await _managed_object_table(self.prisma_client).find_many(
|
||||
where={
|
||||
|
|
@ -220,10 +232,10 @@ class CheckBatchCost:
|
|||
else {}
|
||||
),
|
||||
},
|
||||
take=MAX_OBJECTS_PER_POLL_CYCLE,
|
||||
take=take,
|
||||
order=[{"created_at": "asc"}, {"id": "asc"}],
|
||||
)
|
||||
next_cursor: Final = (page[-1].created_at, page[-1].id) if len(page) == MAX_OBJECTS_PER_POLL_CYCLE else None
|
||||
next_cursor: Final = (page[-1].created_at, page[-1].id) if len(page) == take else None
|
||||
if arrivals:
|
||||
self._arrival_cursor = next_cursor
|
||||
else:
|
||||
|
|
@ -231,11 +243,47 @@ class CheckBatchCost:
|
|||
return page
|
||||
|
||||
def _unprocessed_snapshot(self, job: ManagedObjectRow) -> Mapping[str, object]:
|
||||
return {
|
||||
"id": job.id,
|
||||
"updated_at": job.updated_at,
|
||||
**({"batch_processed": False} if self._has_batch_processed_column else {}),
|
||||
}
|
||||
if self._has_batch_processed_column:
|
||||
return {"id": job.id, "batch_processed": False}
|
||||
return {"id": job.id, "updated_at": job.updated_at}
|
||||
|
||||
def _is_past_give_up_age(self, job: ManagedObjectRow) -> bool:
|
||||
return job.created_at < self._clock() - GIVE_UP_POLLING_AFTER
|
||||
|
||||
async def _expire_if_old_and_unfinished(self, job: ManagedObjectRow) -> None:
|
||||
if job.status in TERMINAL_MANAGED_OBJECT_STATUSES or not self._is_past_give_up_age(job):
|
||||
return
|
||||
await self._expire_unfinished(job)
|
||||
|
||||
@staticmethod
|
||||
def _is_completed_with_output_id_pending(response: "LiteLLMBatch") -> bool:
|
||||
from litellm.proxy.openai_files_endpoints.common_utils import (
|
||||
_completed_batch_safe_to_retire,
|
||||
)
|
||||
|
||||
return (
|
||||
response.status in ("completed", "complete")
|
||||
and response.output_file_id is None
|
||||
and not _completed_batch_safe_to_retire(response)
|
||||
)
|
||||
|
||||
async def _expire_unfinished(self, job: ManagedObjectRow) -> None:
|
||||
try:
|
||||
expired: Final = await _managed_object_table(self.prisma_client).update_many(
|
||||
where={
|
||||
**self._unprocessed_snapshot(job),
|
||||
"status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)},
|
||||
},
|
||||
data={"status": "stale_expired"},
|
||||
)
|
||||
except Exception as db_err:
|
||||
verbose_proxy_logger.error(f"CheckBatchCost: failed to mark stale job {job.id} as stale_expired: {db_err}")
|
||||
return
|
||||
if expired > 0:
|
||||
verbose_proxy_logger.warning(
|
||||
f"CheckBatchCost: job {job.id} is older than {GIVE_UP_POLLING_AFTER.days} days "
|
||||
"and still not finished, so it is marked stale_expired and no longer polled"
|
||||
)
|
||||
|
||||
async def _retire_job(self, job: "ManagedObjectRow", reason: str) -> None:
|
||||
"""
|
||||
|
|
@ -356,15 +404,21 @@ class CheckBatchCost:
|
|||
def _is_output_file_gone_at_provider(error: Exception, output_file_id: Optional[str]) -> bool:
|
||||
"""A 404 naming the output file means there is nothing to fetch on this or any
|
||||
later poll: providers like Vertex AI advertise an output path for every batch,
|
||||
including terminal ones that never wrote it. Any other failure may be
|
||||
transient, so it keeps retrying until the staleness sweep bounds it."""
|
||||
including terminal ones that never wrote it. GCS names a gs:// object without its scheme,
|
||||
and S3 names an s3:// object by its key alone. Any other failure keeps retrying."""
|
||||
import openai
|
||||
|
||||
from litellm.exceptions import NotFoundError
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
if not output_file_id:
|
||||
return False
|
||||
return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, output_file_id)
|
||||
is_not_found: Final = isinstance(error, (NotFoundError, openai.NotFoundError)) or (
|
||||
isinstance(error, BaseLLMException) and error.status_code == 404
|
||||
)
|
||||
object_path: Final = output_file_id.split("://", 1)[-1]
|
||||
names: Final = (output_file_id, object_path, object_path.split("/", 1)[-1])
|
||||
return is_not_found and any(_error_names_resource(error, name) for name in names)
|
||||
|
||||
async def _finalize_unbilled_terminal_job(
|
||||
self, job: "ManagedObjectRow", response: "LiteLLMBatch"
|
||||
|
|
@ -1094,13 +1148,6 @@ class CheckBatchCost:
|
|||
|
||||
processed_models: List[Tuple[Optional[str], Optional[str]]] = []
|
||||
|
||||
try:
|
||||
await self._cleanup_stale_managed_objects()
|
||||
except Exception as cleanup_err:
|
||||
verbose_proxy_logger.warning(
|
||||
f"CheckBatchCost: stale cleanup failed (poll will continue): {cleanup_err}"
|
||||
)
|
||||
|
||||
# Look for all batches that have not yet been processed by CheckBatchCost.
|
||||
# self._has_batch_processed_column is cached after the first probe so that
|
||||
# older schemas don't pay a guaranteed-failing primary query + warning on
|
||||
|
|
@ -1142,6 +1189,8 @@ class CheckBatchCost:
|
|||
if routing is None:
|
||||
if self._has_unified_id_without_model(job):
|
||||
await self._retire_job(job, "unified object id has no model id")
|
||||
else:
|
||||
await self._expire_if_old_and_unfinished(job)
|
||||
continue
|
||||
model_id, batch_id = routing
|
||||
|
||||
|
|
@ -1166,6 +1215,14 @@ class CheckBatchCost:
|
|||
prom_logger.record_check_batch_cost_error("provider_retrieval_error")
|
||||
if self._is_batch_gone_at_provider(e, batch_id) and self._batch_deployment_exists(model_id):
|
||||
await self._retire_job(job, f"batch {batch_id} no longer exists at the provider")
|
||||
else:
|
||||
await self._expire_if_old_and_unfinished(job)
|
||||
continue
|
||||
if self._is_completed_with_output_id_pending(response):
|
||||
verbose_proxy_logger.info(
|
||||
f"CheckBatchCost: batch {batch_id} is completed but its output file id "
|
||||
f"has not appeared yet; leaving job {job.id} for the next poll cycle"
|
||||
)
|
||||
continue
|
||||
|
||||
## RETRIEVE THE BATCH JOB OUTPUT FILE
|
||||
|
|
@ -1217,17 +1274,9 @@ class CheckBatchCost:
|
|||
self._store_batch_line_items_in_background(job, response, model_id, batch_id, billed=True)
|
||||
|
||||
elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES:
|
||||
from litellm.proxy.openai_files_endpoints.common_utils import (
|
||||
_completed_batch_safe_to_retire,
|
||||
)
|
||||
|
||||
if response.status in ("completed", "complete") and not _completed_batch_safe_to_retire(response):
|
||||
verbose_proxy_logger.info(
|
||||
f"CheckBatchCost: batch {batch_id} is completed but its output file id "
|
||||
f"has not appeared yet; leaving job {job.id} for the next poll cycle"
|
||||
)
|
||||
continue
|
||||
await self._retire_unbilled_terminal_job(job, response, model_id, batch_id)
|
||||
else:
|
||||
await self._expire_if_old_and_unfinished(job)
|
||||
|
||||
# Record polling run metrics (always, even if nothing was processed)
|
||||
if prom_logger:
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import base64
|
|||
import json
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from datetime import datetime, timedelta
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
|
|
@ -39,6 +40,7 @@ from litellm.constants import (
|
|||
BATCH_OUTPUT_FILE_FALLBACK_RETRY_AFTER_SECONDS,
|
||||
BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS,
|
||||
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
|
||||
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
|
||||
MAX_FILE_LIST_LIMIT,
|
||||
PENDING_RESPONSE_STATUSES,
|
||||
)
|
||||
|
|
@ -113,6 +115,7 @@ from litellm.types.utils import (
|
|||
LLMResponseTypes,
|
||||
SpecialEnums,
|
||||
)
|
||||
from litellm.utils import get_utc_datetime
|
||||
|
||||
|
||||
class _ManagedFileRetrieve(Protocol):
|
||||
|
|
@ -507,10 +510,12 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
prisma_client: PrismaClient,
|
||||
*,
|
||||
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
||||
clock: Callable[[], datetime] = get_utc_datetime,
|
||||
):
|
||||
self.internal_usage_cache = internal_usage_cache
|
||||
self.prisma_client = prisma_client
|
||||
self._sleep = sleep
|
||||
self._clock = clock
|
||||
|
||||
@staticmethod
|
||||
def _get_prometheus_logger():
|
||||
|
|
@ -2387,12 +2392,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
where={
|
||||
"file_purpose": "batch",
|
||||
"batch_processed": False,
|
||||
"status": {"not_in": ["failed", "expired", "cancelled"]},
|
||||
"status": {"not_in": ["failed", "expired", "cancelled", "stale_expired"]},
|
||||
},
|
||||
take=MAX_MATCHES_TO_RETURN,
|
||||
order={"created_at": "desc"},
|
||||
)
|
||||
|
||||
input_file_cutoff: Final = self._clock() - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
|
||||
referencing_batches: Final[list[dict[str, object]]] = []
|
||||
for batch in batches:
|
||||
try:
|
||||
|
|
@ -2405,7 +2411,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
# Extract file IDs from batch
|
||||
# Batches typically reference the unified file ID in input_file_id
|
||||
# Output and error files are generated by the provider
|
||||
input_file_id = batch_data.get("input_file_id")
|
||||
input_file_id = batch_data.get("input_file_id") if batch.created_at >= input_file_cutoff else None
|
||||
output_file_id = batch_data.get("output_file_id")
|
||||
error_file_id = batch_data.get("error_file_id")
|
||||
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
|
|||
find /app/.venv -type d -path "*/tornado/test" -delete && \
|
||||
chmod -R a+rX /opt/prisma && \
|
||||
python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \
|
||||
python -c "import litellm; from litellm.rust_bridge.loader import native_bridge_available; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__; assert native_bridge_available()"
|
||||
python -c "import litellm, litellm.rust_bridge._native; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__"
|
||||
|
||||
USER nonroot
|
||||
|
||||
|
|
|
|||
|
|
@ -113,6 +113,7 @@ const OPENAI_COMPATIBLE_PROVIDERS: &[&str] = &[
|
|||
"meta",
|
||||
"cognition",
|
||||
"scx-ai",
|
||||
"zerogpu",
|
||||
];
|
||||
|
||||
impl ExceptionFamily {
|
||||
|
|
|
|||
|
|
@ -187,6 +187,7 @@ fn native_events_reach_python_with_levels_context_reentry_and_http_deduplication
|
|||
locals
|
||||
.set_item("http_warning", wrap_pyfunction!(http_warning, py).unwrap())
|
||||
.unwrap();
|
||||
locals.set_item("native", crate::native_module(py)).unwrap();
|
||||
let importable = py
|
||||
.eval(
|
||||
c"__import__('importlib.util', fromlist=['util']).find_spec('dotenv') is not None",
|
||||
|
|
@ -205,6 +206,7 @@ import asyncio
|
|||
import logging
|
||||
import sys
|
||||
sys.path.insert(0, repo_root)
|
||||
sys.modules['litellm.rust_bridge._native'] = native
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger, session_id_var, trace_id_var
|
||||
|
||||
|
|
|
|||
|
|
@ -681,6 +681,7 @@ moonshot_models: Set = set()
|
|||
publicai_models: Set = set()
|
||||
darkbloom_models: Set = set()
|
||||
openinfer_models: Final[Set[str]] = set()
|
||||
zerogpu_models: Final[Set[str]] = set()
|
||||
v0_models: Set = set()
|
||||
morph_models: Set = set()
|
||||
opencode_go_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads
|
||||
|
|
@ -969,6 +970,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
darkbloom_models.add(key)
|
||||
elif value.get("litellm_provider") == "openinfer":
|
||||
openinfer_models.add(key)
|
||||
elif value.get("litellm_provider") == "zerogpu":
|
||||
zerogpu_models.add(key)
|
||||
elif value.get("litellm_provider") == "v0":
|
||||
v0_models.add(key)
|
||||
elif value.get("litellm_provider") == "morph":
|
||||
|
|
@ -1142,6 +1145,7 @@ model_list = list(
|
|||
| publicai_models
|
||||
| darkbloom_models
|
||||
| openinfer_models
|
||||
| zerogpu_models
|
||||
| v0_models
|
||||
| morph_models
|
||||
| opencode_go_models
|
||||
|
|
@ -1261,6 +1265,7 @@ def _build_models_by_provider() -> dict:
|
|||
"publicai": publicai_models,
|
||||
"darkbloom": darkbloom_models,
|
||||
"openinfer": openinfer_models,
|
||||
"zerogpu": zerogpu_models,
|
||||
"v0": v0_models,
|
||||
"morph": morph_models,
|
||||
"opencode_go": opencode_go_models,
|
||||
|
|
@ -1449,6 +1454,7 @@ from .exceptions import (
|
|||
Timeout,
|
||||
APIConnectionError,
|
||||
UnsupportedParamsError,
|
||||
UnsupportedEndpointError as UnsupportedEndpointError,
|
||||
APIResponseValidationError,
|
||||
UnprocessableEntityError,
|
||||
InternalServerError,
|
||||
|
|
@ -1738,6 +1744,9 @@ if TYPE_CHECKING:
|
|||
from .llms.lm_studio.decisions.transformation import (
|
||||
LMStudioDecisionsConfig as LMStudioDecisionsConfig,
|
||||
)
|
||||
from .llms.ollama.decisions.transformation import (
|
||||
OllamaDecisionsConfig as OllamaDecisionsConfig,
|
||||
)
|
||||
from .llms.laya.decisions.transformation import (
|
||||
LayaDecisionsConfig as LayaDecisionsConfig,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -171,6 +171,7 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"DatabricksDecisionsConfig",
|
||||
"HostedVLLMDecisionsConfig",
|
||||
"LMStudioDecisionsConfig",
|
||||
"OllamaDecisionsConfig",
|
||||
"LayaDecisionsConfig",
|
||||
"BespokeDecisionsConfig",
|
||||
"OpenAIDecisionsConfig",
|
||||
|
|
@ -749,6 +750,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
"HostedVLLMDecisionsConfig",
|
||||
),
|
||||
"LMStudioDecisionsConfig": (".llms.lm_studio.decisions.transformation", "LMStudioDecisionsConfig"),
|
||||
"OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"),
|
||||
"LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
|
||||
"BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
|
||||
"OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ Handler for transforming /chat/completions api requests to litellm.responses req
|
|||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import accumulate, chain
|
||||
|
|
@ -44,8 +43,8 @@ from litellm.responses.sse_output_recovery import (
|
|||
)
|
||||
from litellm.responses.utils import (
|
||||
ResponsesAPIRequestUtils,
|
||||
auto_reasoning_summary,
|
||||
normalize_responses_api_stream_options,
|
||||
opt_in_auto_reasoning_summary,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
ChatCompletionAnnotation,
|
||||
|
|
@ -1840,19 +1839,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
**cast(Reasoning, reasoning_effort) # cast-ok: dict is forwarded verbatim to the provider
|
||||
)
|
||||
|
||||
# Check if auto-summary is enabled via flag or environment variable
|
||||
# Priority: litellm.reasoning_auto_summary flag > LITELLM_REASONING_AUTO_SUMMARY env var
|
||||
auto_summary_enabled: Final = (
|
||||
litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"
|
||||
)
|
||||
|
||||
summary: Final = opt_in_auto_reasoning_summary(model, custom_llm_provider)
|
||||
return (
|
||||
Reasoning(
|
||||
effort=reasoning_effort,
|
||||
summary="detailed" if model is None else auto_reasoning_summary(model, custom_llm_provider),
|
||||
)
|
||||
if auto_summary_enabled
|
||||
else Reasoning(effort=reasoning_effort)
|
||||
Reasoning(effort=reasoning_effort)
|
||||
if summary is None
|
||||
else Reasoning(effort=reasoning_effort, summary=summary)
|
||||
)
|
||||
|
||||
_map_reasoning_effort = map_reasoning_effort
|
||||
|
|
|
|||
|
|
@ -1096,6 +1096,7 @@ openai_compatible_providers: Final[list[str]] = [
|
|||
"llamafile",
|
||||
"lm_studio",
|
||||
"galadriel",
|
||||
"zerogpu",
|
||||
"github_copilot", # GitHub Copilot Chat API
|
||||
"chatgpt", # ChatGPT subscription API
|
||||
"novita",
|
||||
|
|
@ -1737,6 +1738,7 @@ PASS_THROUGH_HEADER_PREFIX: Final = "x-pass-"
|
|||
TYPESAFE_DANGER_ZONE_HEADER_PREFIX: Final = "typesafe-danger-zone-"
|
||||
INTERNAL_KWARG_PREFIX: Final = "_litellm_"
|
||||
CONTROL_OPTIONS_KEY: Final = f"{INTERNAL_KWARG_PREFIX}control"
|
||||
SESSION_ID_FROM_TRACE_ID_PARAM_KEY: Final = f"{INTERNAL_KWARG_PREFIX}session_id_from_trace_id"
|
||||
|
||||
AZURE_SPEECH_CUSTOM_LLM_PROVIDER: Final = "azure_speech"
|
||||
AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX: Final = "/azure_speech"
|
||||
|
|
|
|||
|
|
@ -232,7 +232,7 @@ class BadRequestError(openai.BadRequestError):
|
|||
litellm_debug_info: str | None = None,
|
||||
max_retries: int | None = None,
|
||||
num_retries: int | None = None,
|
||||
body: dict | None = None,
|
||||
body: Mapping[str, object] | None = None,
|
||||
):
|
||||
self.status_code = 400
|
||||
self.message = f"litellm.BadRequestError: {message}"
|
||||
|
|
@ -276,7 +276,7 @@ class ImageFetchError(BadRequestError):
|
|||
litellm_debug_info: str | None = None,
|
||||
max_retries: int | None = None,
|
||||
num_retries: int | None = None,
|
||||
body: dict | None = None,
|
||||
body: Mapping[str, object] | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
message=message,
|
||||
|
|
@ -317,6 +317,10 @@ class MalformedRequestFieldError(BadRequestError):
|
|||
pass
|
||||
|
||||
|
||||
class UnsupportedEndpointError(BadRequestError):
|
||||
pass
|
||||
|
||||
|
||||
class UnprocessableEntityError(openai.UnprocessableEntityError):
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -556,6 +560,7 @@ class ContextWindowExceededError(BadRequestError):
|
|||
llm_provider,
|
||||
response: httpx.Response | None = None,
|
||||
litellm_debug_info: str | None = None,
|
||||
body: Mapping[str, object] | None = None,
|
||||
):
|
||||
self.status_code = 400
|
||||
self.model = model
|
||||
|
|
@ -567,6 +572,7 @@ class ContextWindowExceededError(BadRequestError):
|
|||
llm_provider=self.llm_provider,
|
||||
response=response,
|
||||
litellm_debug_info=self.litellm_debug_info,
|
||||
body=body,
|
||||
) # Call the base class constructor with the parameters it needs
|
||||
|
||||
# set after, to make it clear the raised error is a context window exceeded error
|
||||
|
|
@ -685,7 +691,7 @@ class ContentPolicyViolationError(BadRequestError):
|
|||
response: httpx.Response | None = None,
|
||||
litellm_debug_info: str | None = None,
|
||||
provider_specific_fields: dict | None = None,
|
||||
body: dict | None = None,
|
||||
body: Mapping[str, object] | None = None,
|
||||
):
|
||||
self.status_code = 400
|
||||
self.message = f"litellm.ContentPolicyViolationError: {message}"
|
||||
|
|
@ -1054,6 +1060,7 @@ LITELLM_EXCEPTION_TYPES: Final = [
|
|||
BadRequestError,
|
||||
UnprocessableEntityError,
|
||||
UnsupportedParamsError,
|
||||
UnsupportedEndpointError,
|
||||
Timeout,
|
||||
PermissionDeniedError,
|
||||
RateLimitError,
|
||||
|
|
|
|||
|
|
@ -284,6 +284,12 @@ def image_generation(
|
|||
extra_headers: Final = kwargs.get("extra_headers", None)
|
||||
headers: Final[dict] = kwargs.get("headers", None) or {}
|
||||
base_model: Final = kwargs.get("base_model", None)
|
||||
call_kwargs: Final = cast(Mapping[str, object], kwargs) # cast-ok: image_generation's **kwargs is untyped
|
||||
max_retries: Final = call_kwargs.get("max_retries")
|
||||
is_router_call: Final = any(
|
||||
isinstance(call_metadata := call_kwargs.get(k), Mapping) and "model_group" in call_metadata
|
||||
for k in ("metadata", "litellm_metadata")
|
||||
)
|
||||
if extra_headers is not None:
|
||||
headers.update(extra_headers)
|
||||
model_response: ImageResponse = litellm.utils.ImageResponse()
|
||||
|
|
@ -550,6 +556,7 @@ def image_generation(
|
|||
optional_params["extra_headers"] = extra_headers
|
||||
# Forward OpenAI organization if present (set by proxy pre-call utils)
|
||||
organization: Final[str | None] = kwargs.get("organization", None)
|
||||
sdk_max_retries: Final = 0 if is_router_call else max_retries
|
||||
model_response = openai_chat_completions.image_generation(
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
|
|
@ -557,7 +564,9 @@ def image_generation(
|
|||
api_key=api_key or dynamic_api_key,
|
||||
api_base=api_base,
|
||||
logging_obj=litellm_logging_obj,
|
||||
optional_params=optional_params,
|
||||
optional_params=(
|
||||
optional_params if sdk_max_retries is None else {**optional_params, "max_retries": sdk_max_retries}
|
||||
),
|
||||
model_response=model_response,
|
||||
organization=organization,
|
||||
aimg_generation=aimg_generation,
|
||||
|
|
@ -1053,7 +1062,7 @@ def image_edit(
|
|||
|
||||
@client
|
||||
async def aimage_edit(
|
||||
image: FileTypes | list[FileTypes],
|
||||
image: FileTypes | list[FileTypes] | None,
|
||||
model: str,
|
||||
prompt: str,
|
||||
mask: str | None = None,
|
||||
|
|
@ -1093,7 +1102,7 @@ async def aimage_edit(
|
|||
model=model, api_base=local_vars.get("base_url", None)
|
||||
)
|
||||
|
||||
images: Final = image if isinstance(image, list) else [image]
|
||||
images: Final = image if isinstance(image, list) else ([image] if image is not None else [])
|
||||
|
||||
func: Final = partial(
|
||||
image_edit,
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ from enum import Enum
|
|||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
from litellm.integrations.otel.mappers.utils import drop_none
|
||||
from litellm.integrations.otel.model.semconv import PASSTHROUGH_OPERATION_KEY, GenAIOperation
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -148,8 +149,11 @@ class OTELGenAISemconvMixin:
|
|||
"""Map a LiteLLM ``call_type`` to spec ``gen_ai.operation.name``.
|
||||
|
||||
Substring match (e.g. ``aembedding`` -> ``embeddings``); defaults to
|
||||
``chat``.
|
||||
``chat``. A pass-through call's operation, stored under ``PASSTHROUGH_OPERATION_KEY``, wins.
|
||||
"""
|
||||
passthrough_operation: Final = kwargs.get(PASSTHROUGH_OPERATION_KEY) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # kwargs is an untyped dict
|
||||
if isinstance(passthrough_operation, GenAIOperation):
|
||||
return passthrough_operation.value
|
||||
call_type: Final = kwargs.get("call_type", "") or ""
|
||||
match call_type:
|
||||
case s if "embedding" in s:
|
||||
|
|
|
|||
|
|
@ -600,6 +600,7 @@ class OpenTelemetryV2(CustomLogger):
|
|||
trace=call.trace,
|
||||
session_id=call.session_id,
|
||||
metadata_keys=tuple(self.config.baggage_metadata_keys),
|
||||
operation=call.operation,
|
||||
)
|
||||
end_time_ns: Final = to_ns(end_time)
|
||||
if carrier is not None and carrier.span is not None:
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ from litellm.constants import (
|
|||
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL,
|
||||
SESSION_ID_GENERATED_METADATA_KEY,
|
||||
)
|
||||
from litellm.integrations.otel.model.semconv import resolve_operation
|
||||
from litellm.integrations.otel.model.semconv import PASSTHROUGH_OPERATION_KEY, GenAIOperation, resolve_operation
|
||||
from litellm.integrations.otel.model.trace_controls import TraceControls, caller_trace_controls
|
||||
from litellm.integrations.otel.model.utils import as_str, as_str_mapping, to_seconds
|
||||
from litellm.types.utils import InternalCallOrigin
|
||||
|
|
@ -262,6 +262,7 @@ class LLMCallEvent:
|
|||
# never hands off to a provider, so ``upstream_started`` stays False even
|
||||
# though the call completed and is worth a span.
|
||||
cache_hit: bool
|
||||
operation: GenAIOperation
|
||||
# A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The
|
||||
# span is renamed from the typed payload at close (``finish_span``); this only
|
||||
# needs to be reasonable for a span that never gets closed (a leak).
|
||||
|
|
@ -275,7 +276,7 @@ class LLMCallEvent:
|
|||
def from_dict(cls, kwargs: Mapping[str, object]) -> LLMCallEvent:
|
||||
raw_payload: Final = kwargs.get("standard_logging_object")
|
||||
payload: Final = cast("StandardLoggingPayload", raw_payload) if raw_payload else None
|
||||
operation: Final = resolve_operation(as_str(kwargs.get("call_type")))
|
||||
operation: Final = resolve_operation(as_str(kwargs.get("call_type")), kwargs.get(PASSTHROUGH_OPERATION_KEY))
|
||||
model: Final = as_str(kwargs.get("model")) or ""
|
||||
trace: Final = caller_trace_controls(kwargs)
|
||||
return cls(
|
||||
|
|
@ -288,6 +289,7 @@ class LLMCallEvent:
|
|||
upstream_start_seconds=_epoch_seconds(kwargs.get("api_call_start_time")),
|
||||
purpose=internal_call_origin(payload, kwargs),
|
||||
cache_hit=bool(payload.get("cache_hit")) if payload else False, # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown
|
||||
operation=operation,
|
||||
provisional_span_name=f"{operation.value} {model}".strip(),
|
||||
time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs),
|
||||
completion_start_seconds=_epoch_seconds(kwargs.get("completion_start_time")),
|
||||
|
|
|
|||
|
|
@ -464,6 +464,7 @@ class LLMCallSpanData:
|
|||
session_id: str | None = None,
|
||||
*,
|
||||
metadata_keys: tuple[str, ...] = (),
|
||||
operation: GenAIOperation | None = None,
|
||||
) -> LLMCallSpanData:
|
||||
params: Final = cast(Mapping[str, object], payload.get("model_parameters") or {})
|
||||
cache_hit: Final = as_bool(payload.get("cache_hit")) is True # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown
|
||||
|
|
@ -486,13 +487,13 @@ class LLMCallSpanData:
|
|||
# no prompt/response text.
|
||||
finish_reasons: Final = _finish_reasons(choices_out)
|
||||
call_type: Final = as_str(payload.get("call_type"))
|
||||
operation: Final = resolve_operation(call_type)
|
||||
resolved_operation: Final = operation or resolve_operation(call_type)
|
||||
embedding_output: Final = (
|
||||
EmbeddingOutput.from_response(response) if operation is GenAIOperation.EMBEDDINGS else None
|
||||
EmbeddingOutput.from_response(response) if resolved_operation is GenAIOperation.EMBEDDINGS else None
|
||||
)
|
||||
cost_lookup_failed: Final = payload.get("response_cost_failure_debug_info") is not None and not cache_hit
|
||||
return cls(
|
||||
operation=operation,
|
||||
operation=resolved_operation,
|
||||
provider=resolve_provider(as_str(payload.get("custom_llm_provider"))),
|
||||
request_model=context.request_model,
|
||||
response_model=context.response_model,
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ class GenAIOperation(str, Enum):
|
|||
LITELLM_VECTOR_STORE_FILE_MANAGEMENT = "litellm.vector_store_file_management"
|
||||
LITELLM_RESPONSES_MANAGEMENT = "litellm.responses_management"
|
||||
LITELLM_MODERATION = "litellm.moderation"
|
||||
LITELLM_PASSTHROUGH = "litellm.passthrough"
|
||||
|
||||
|
||||
class GenAIOutputType(str, Enum):
|
||||
|
|
@ -482,14 +483,21 @@ def resolve_provider(custom_llm_provider: str | None) -> str:
|
|||
return mapped.value if mapped is not None else custom_llm_provider
|
||||
|
||||
|
||||
def resolve_operation(call_type: str | None) -> GenAIOperation:
|
||||
PASSTHROUGH_OPERATION_KEY: Final = "passthrough_gen_ai_operation"
|
||||
|
||||
|
||||
def resolve_operation(call_type: str | None, passthrough_operation: object = None) -> GenAIOperation:
|
||||
"""Map a litellm ``call_type`` to a ``gen_ai.operation.name`` value.
|
||||
|
||||
An unmapped call type still falls back to ``chat`` so every series keeps an
|
||||
operation label, but it logs at debug rather than falling through silently:
|
||||
a new call type mislabelled as ``chat`` mixes its latency and cost into
|
||||
everyone's chat charts, which is invisible until someone reads the numbers.
|
||||
|
||||
A pass-through call's operation, stored under ``PASSTHROUGH_OPERATION_KEY``, wins over its call type.
|
||||
"""
|
||||
if isinstance(passthrough_operation, GenAIOperation):
|
||||
return passthrough_operation
|
||||
if not call_type:
|
||||
return GenAIOperation.CHAT
|
||||
mapped: Final = _OPERATION_BY_CALL_TYPE.get(call_type.lower())
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
|||
if TYPE_CHECKING:
|
||||
from fastapi import FastAPI
|
||||
from opentelemetry.trace import Span, TracerProvider
|
||||
from starlette.types import ASGIApp, Receive, Scope, Send
|
||||
|
||||
# Routes excluded from server-span tracing by default: high-frequency pollers and
|
||||
# static UI/docs assets, none of which are LLM traffic. Entries are substring-matched
|
||||
|
|
@ -44,6 +45,33 @@ _DEFAULT_EXCLUDED_ROUTES: Final = (
|
|||
)
|
||||
_DEFAULT_EXCLUDED_URLS: Final = ",".join(_DEFAULT_EXCLUDED_ROUTES)
|
||||
|
||||
|
||||
class _RequestTraceExclusionMiddleware:
|
||||
def __init__(self, app: "ASGIApp", excluded_urls: str) -> None:
|
||||
from opentelemetry.util.http import parse_excluded_urls
|
||||
|
||||
self.app = app
|
||||
self.excluded_urls = parse_excluded_urls(excluded_urls)
|
||||
|
||||
async def __call__(self, scope: "Scope", receive: "Receive", send: "Send") -> None:
|
||||
if scope["type"] not in ("http", "websocket"):
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
from opentelemetry.instrumentation.asgi import ( # pyright: ignore[reportMissingTypeStubs] # upstream ships no py.typed marker
|
||||
get_host_port_url_tuple, # pyright: ignore[reportUnknownVariableType] # upstream URL helper has no annotations
|
||||
)
|
||||
|
||||
from litellm.integrations.otel.plumbing.context import (
|
||||
request_trace_scope,
|
||||
request_traces_excluded,
|
||||
)
|
||||
|
||||
url: Final = str(get_host_port_url_tuple(scope)[2]) # pyright: ignore[reportUnknownArgumentType] # upstream constructs a str URL but leaves its return type unannotated
|
||||
excluded: Final = request_traces_excluded() or self.excluded_urls.url_disabled(url)
|
||||
with request_trace_scope(excluded):
|
||||
await self.app(scope, receive, send)
|
||||
|
||||
|
||||
# Passthrough routes are catch-alls (e.g. "/openai/{endpoint:path}"), so the
|
||||
# default OTel server-span name "{method} {route}" collapses every upstream
|
||||
# endpoint into "POST /openai/{endpoint:path}". The hook below renames those spans
|
||||
|
|
@ -129,6 +157,8 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
|
|||
if app.middleware_stack is not None:
|
||||
verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: the app has already started")
|
||||
return
|
||||
if getattr(app, "_is_instrumented_by_opentelemetry", False):
|
||||
return
|
||||
|
||||
# Lazy: only the V2-enabled path needs the optional
|
||||
# ``opentelemetry-instrumentation-fastapi`` package, which is not part of the
|
||||
|
|
@ -151,11 +181,7 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
|
|||
claim_server_span(span)
|
||||
_passthrough_span_name_hook(span, scope)
|
||||
|
||||
excluded_urls: Final = (
|
||||
os.environ.get("OTEL_PYTHON_FASTAPI_EXCLUDED_URLS")
|
||||
if "OTEL_PYTHON_FASTAPI_EXCLUDED_URLS" in os.environ
|
||||
else _DEFAULT_EXCLUDED_URLS
|
||||
)
|
||||
excluded_urls: Final = os.environ.get("OTEL_PYTHON_FASTAPI_EXCLUDED_URLS", _DEFAULT_EXCLUDED_URLS)
|
||||
FastAPIInstrumentor.instrument_app(
|
||||
app,
|
||||
tracer_provider=tracer_provider,
|
||||
|
|
@ -166,6 +192,12 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
|
|||
# template in their name, which can't be rewritten from a hook.
|
||||
exclude_spans=["receive", "send"],
|
||||
)
|
||||
instrumented_stack: Final = app.build_middleware_stack
|
||||
|
||||
def build_middleware_stack() -> "ASGIApp":
|
||||
return _RequestTraceExclusionMiddleware(instrumented_stack(), excluded_urls)
|
||||
|
||||
app.build_middleware_stack = build_middleware_stack # rebind-ok: wrap the complete ASGI stack
|
||||
if get_global_response_propagator() is None:
|
||||
set_global_response_propagator(UpstreamFirstTraceResponsePropagator())
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -30,6 +30,21 @@ if TYPE_CHECKING:
|
|||
|
||||
_PROPAGATOR: Final = TraceContextTextMapPropagator()
|
||||
_W3C_TRACE_HEADERS: Final = frozenset(("traceparent", "tracestate"))
|
||||
_request_traces_excluded: Final[ContextVar[bool]] = ContextVar("litellm_otel_request_traces_excluded", default=False)
|
||||
|
||||
|
||||
def request_traces_excluded() -> bool:
|
||||
return _request_traces_excluded.get()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def request_trace_scope(excluded: bool) -> Generator[None]:
|
||||
token: Final = _request_traces_excluded.set(excluded)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_request_traces_excluded.reset(token)
|
||||
|
||||
|
||||
# The request's root span — the FastAPI-owned SERVER span — captured ONCE when the
|
||||
# proxy first resolves it, so request-level spans (the LLM call, guardrails) can
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from litellm.integrations.opentelemetry import (
|
|||
)
|
||||
from litellm.integrations.otel.model.metadata import time_to_first_chunk_seconds
|
||||
from litellm.integrations.otel.model.semconv import (
|
||||
PASSTHROUGH_OPERATION_KEY,
|
||||
Error,
|
||||
GenAI,
|
||||
Metric,
|
||||
|
|
@ -280,7 +281,9 @@ class GenAIMetricRecorder:
|
|||
def _common_attributes(self, kwargs: Mapping[str, Any]) -> dict:
|
||||
params: Final = kwargs.get("litellm_params") or {}
|
||||
common_attrs: Final[dict] = {
|
||||
GenAI.OPERATION_NAME: resolve_operation(kwargs.get("call_type")).value,
|
||||
GenAI.OPERATION_NAME: resolve_operation(
|
||||
kwargs.get("call_type"), kwargs.get(PASSTHROUGH_OPERATION_KEY)
|
||||
).value,
|
||||
**_provider_attributes(params.get("custom_llm_provider")),
|
||||
GenAI.REQUEST_MODEL: kwargs.get("model"),
|
||||
"gen_ai.framework": "litellm",
|
||||
|
|
|
|||
|
|
@ -35,7 +35,8 @@ from opentelemetry.sdk.trace.export import (
|
|||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
|
||||
InMemorySpanExporter,
|
||||
)
|
||||
from opentelemetry.trace import Span, SpanContext, SpanKind, Status, StatusCode, Tracer
|
||||
from opentelemetry.sdk.trace.sampling import Decision, Sampler, SamplingResult
|
||||
from opentelemetry.trace import Link, Span, SpanContext, SpanKind, Status, StatusCode, Tracer, TraceState
|
||||
from opentelemetry.util.re import parse_env_headers
|
||||
from opentelemetry.util.types import Attributes, AttributeValue
|
||||
|
||||
|
|
@ -68,6 +69,7 @@ from litellm.integrations.otel.model.semconv import (
|
|||
from litellm.integrations.otel.model.spans import LiteLLMSpanKind
|
||||
from litellm.integrations.otel.plumbing.context import (
|
||||
request_destinations,
|
||||
request_traces_excluded,
|
||||
suppressed_backends,
|
||||
)
|
||||
from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls
|
||||
|
|
@ -1499,6 +1501,28 @@ def _spec_processor(
|
|||
return _MessageContentFilter(scoped) if strip_message_content else scoped
|
||||
|
||||
|
||||
class _RequestExclusionSampler(Sampler):
|
||||
def __init__(self, delegate: Sampler) -> None:
|
||||
self.delegate = delegate
|
||||
|
||||
def should_sample(
|
||||
self,
|
||||
parent_context: Context | None,
|
||||
trace_id: int,
|
||||
name: str,
|
||||
kind: SpanKind | None = None,
|
||||
attributes: Attributes = None,
|
||||
links: Sequence[Link] | None = None,
|
||||
trace_state: TraceState | None = None,
|
||||
) -> SamplingResult:
|
||||
if request_traces_excluded():
|
||||
return SamplingResult(Decision.DROP)
|
||||
return self.delegate.should_sample(parent_context, trace_id, name, kind, attributes, links, trace_state)
|
||||
|
||||
def get_description(self) -> str:
|
||||
return f"RequestExclusion({self.delegate.get_description()})"
|
||||
|
||||
|
||||
def build_tracer_provider(
|
||||
config: OpenTelemetryV2Config,
|
||||
exporter: SpanExporter | None = None,
|
||||
|
|
@ -1531,6 +1555,7 @@ def build_tracer_provider(
|
|||
destination asked for.
|
||||
"""
|
||||
provider: Final = TracerProvider(resource=build_resource(config))
|
||||
provider.sampler = _RequestExclusionSampler(provider.sampler)
|
||||
if baggage_processor is None:
|
||||
baggage_processor = LiteLLMBaggageSpanProcessor(allowed_keys=config.baggage_promoted_keys)
|
||||
provider.add_span_processor(baggage_processor)
|
||||
|
|
|
|||
|
|
@ -32,11 +32,11 @@ Usage:
|
|||
|
||||
# Create a managed agent on the provider side
|
||||
result = litellm.interactions.agents.create(
|
||||
name="waverunner",
|
||||
id="waverunner",
|
||||
custom_llm_provider="gemini",
|
||||
api_key="...",
|
||||
base_agent="gemini-2.5-flash",
|
||||
instructions="You are a helpful assistant.",
|
||||
base_agent="antigravity-preview-09-2026",
|
||||
system_instruction="You are a helpful assistant.",
|
||||
)
|
||||
|
||||
Methods:
|
||||
|
|
|
|||
|
|
@ -3,11 +3,11 @@ litellm.interactions.agents
|
|||
|
||||
Full CRUD SDK for provider-side managed agents (e.g. Gemini v1beta/agents).
|
||||
|
||||
litellm.interactions.agents.create(name=..., ...)
|
||||
litellm.interactions.agents.create(id=..., ...)
|
||||
litellm.interactions.agents.list(api_key=...)
|
||||
litellm.interactions.agents.get(name=..., ...)
|
||||
litellm.interactions.agents.delete(name=..., ...)
|
||||
litellm.interactions.agents.list_versions(name=..., ...)
|
||||
litellm.interactions.agents.get(id=..., ...)
|
||||
litellm.interactions.agents.delete(id=..., ...)
|
||||
litellm.interactions.agents.list_versions(id=..., ...)
|
||||
|
||||
Async counterparts: acreate, alist, aget, adelete, alist_versions
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
|||
from litellm.llms.base_llm.agents.transformation import BaseAgentsAPIConfig
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.types.agents import (
|
||||
AgentCreateRequest,
|
||||
AgentCreateResponse,
|
||||
AgentDeleteResult,
|
||||
AgentListResponse,
|
||||
|
|
@ -35,7 +36,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
|
|||
def create_agent(
|
||||
self,
|
||||
agents_api_config: BaseAgentsAPIConfig,
|
||||
name: str,
|
||||
create_request: AgentCreateRequest,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
extra_headers: dict[str, str] | None = None,
|
||||
|
|
@ -47,7 +48,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
|
|||
if _is_async:
|
||||
return self.async_create_agent(
|
||||
agents_api_config=agents_api_config,
|
||||
name=name,
|
||||
create_request=create_request,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=logging_obj,
|
||||
extra_headers=extra_headers,
|
||||
|
|
@ -63,12 +64,13 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
|
|||
api_base=litellm_params.get("api_base"),
|
||||
litellm_params=dict(litellm_params),
|
||||
)
|
||||
data: Final = agents_api_config.transform_create_request(name=name, litellm_params=dict(litellm_params))
|
||||
if extra_body:
|
||||
data.update(extra_body)
|
||||
data: Final = {
|
||||
**agents_api_config.transform_create_request(create_request),
|
||||
**(extra_body or {}),
|
||||
}
|
||||
|
||||
logging_obj.pre_call(
|
||||
input=name,
|
||||
input=create_request.id,
|
||||
api_key="",
|
||||
additional_args={
|
||||
"complete_input_dict": data,
|
||||
|
|
@ -85,12 +87,12 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
|
|||
original_response=response.text,
|
||||
additional_args={"complete_input_dict": data},
|
||||
)
|
||||
return agents_api_config.transform_create_response(raw_response=response, name=name)
|
||||
return agents_api_config.transform_create_response(raw_response=response, name=create_request.id)
|
||||
|
||||
async def async_create_agent(
|
||||
self,
|
||||
agents_api_config: BaseAgentsAPIConfig,
|
||||
name: str,
|
||||
create_request: AgentCreateRequest,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
extra_headers: dict[str, str] | None = None,
|
||||
|
|
@ -106,12 +108,13 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
|
|||
api_base=litellm_params.get("api_base"),
|
||||
litellm_params=dict(litellm_params),
|
||||
)
|
||||
data: Final = agents_api_config.transform_create_request(name=name, litellm_params=dict(litellm_params))
|
||||
if extra_body:
|
||||
data.update(extra_body)
|
||||
data: Final = {
|
||||
**agents_api_config.transform_create_request(create_request),
|
||||
**(extra_body or {}),
|
||||
}
|
||||
|
||||
logging_obj.pre_call(
|
||||
input=name,
|
||||
input=create_request.id,
|
||||
api_key="",
|
||||
additional_args={
|
||||
"complete_input_dict": data,
|
||||
|
|
@ -130,7 +133,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
|
|||
original_response=response.text,
|
||||
additional_args={"complete_input_dict": data},
|
||||
)
|
||||
return agents_api_config.transform_create_response(raw_response=response, name=name)
|
||||
return agents_api_config.transform_create_response(raw_response=response, name=create_request.id)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# LIST #
|
||||
|
|
|
|||
|
|
@ -6,41 +6,47 @@ Usage:
|
|||
|
||||
# Create
|
||||
response = litellm.interactions.agents.create(
|
||||
name="waverunner",
|
||||
id="waverunner",
|
||||
custom_llm_provider="gemini",
|
||||
api_key="...",
|
||||
base_agent="gemini-2.5-flash",
|
||||
instructions="You are a helpful assistant.",
|
||||
base_agent="antigravity-preview-09-2026",
|
||||
system_instruction="You are a helpful assistant.",
|
||||
agent_config={"type": "antigravity", "model": "gemini-3.8-flash"},
|
||||
)
|
||||
|
||||
# List
|
||||
response = litellm.interactions.agents.list(api_key="...", custom_llm_provider="gemini")
|
||||
|
||||
# Get
|
||||
response = litellm.interactions.agents.get(name="waverunner", api_key="...")
|
||||
response = litellm.interactions.agents.get(id="waverunner", api_key="...")
|
||||
|
||||
# Delete
|
||||
result = litellm.interactions.agents.delete(name="waverunner", api_key="...")
|
||||
result = litellm.interactions.agents.delete(id="waverunner", api_key="...")
|
||||
|
||||
# List versions
|
||||
result = litellm.interactions.agents.list_versions(name="waverunner", api_key="...")
|
||||
result = litellm.interactions.agents.list_versions(id="waverunner", api_key="...")
|
||||
|
||||
# Async versions: acreate, alist, aget, adelete, alist_versions
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import contextvars
|
||||
from collections.abc import Coroutine, Mapping
|
||||
from collections.abc import Coroutine, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.interactions.agents.http_handler import agents_http_handler
|
||||
from litellm.interactions.agents.utils import get_provider_agents_api_config
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.agents import (
|
||||
AgentCreateRequest,
|
||||
AgentCreateResponse,
|
||||
AgentDeleteResult,
|
||||
AgentListResponse,
|
||||
|
|
@ -89,14 +95,120 @@ def _make_logging_obj(
|
|||
return litellm_logging_obj
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _InvalidAgentRequest:
|
||||
message: str
|
||||
|
||||
|
||||
def _alias_conflict(field: str, value: str | None, alias: str, alias_value: str | None) -> str | None:
|
||||
if value is None or alias_value is None or value == alias_value:
|
||||
return None
|
||||
return f"'{alias}' is an alias of '{field}', but the request gives them different values. Pass only '{field}'."
|
||||
|
||||
|
||||
def _agent_id_or_error(id: str | None, name: str | None) -> str | _InvalidAgentRequest:
|
||||
conflict: Final = _alias_conflict("id", id, "name", name)
|
||||
if conflict is not None:
|
||||
return _InvalidAgentRequest(conflict)
|
||||
agent_id: Final = id if id is not None else name
|
||||
if agent_id is None:
|
||||
return _InvalidAgentRequest("Missing required parameter: 'id' (or its alias 'name').")
|
||||
return agent_id
|
||||
|
||||
|
||||
def _required_agent_id(id: str | None, name: str | None, llm_provider: str) -> str:
|
||||
agent_id: Final = _agent_id_or_error(id, name)
|
||||
if isinstance(agent_id, _InvalidAgentRequest):
|
||||
raise litellm.BadRequestError(message=agent_id.message, model=id or name or "", llm_provider=llm_provider)
|
||||
return agent_id
|
||||
|
||||
|
||||
# ================================================================== #
|
||||
# CREATE #
|
||||
# ================================================================== #
|
||||
|
||||
|
||||
_CREATE_FIELD_SHAPES: Final = MappingProxyType(
|
||||
{
|
||||
"id": "a string",
|
||||
"base_agent": "a string",
|
||||
"agent_config": "an object",
|
||||
"system_instruction": "a string",
|
||||
"base_environment": "an environment id string or an object",
|
||||
"description": "a string",
|
||||
"tools": "a list of objects",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _invalid_create_fields_message(invalid_fields: frozenset[str], caller_names: Mapping[str, str]) -> str:
|
||||
problems: Final = "; ".join(
|
||||
f"'{caller_names.get(field, field)}' must be {_CREATE_FIELD_SHAPES[field]}" for field in sorted(invalid_fields)
|
||||
)
|
||||
drop_hint: Final = "" if "id" in invalid_fields else " Set drop_params to drop invalid optional fields."
|
||||
return f"Invalid agent create request: {problems}.{drop_hint}"
|
||||
|
||||
|
||||
def _validated_create_request(
|
||||
fields: Mapping[str, object], drop_invalid_fields: bool, caller_names: Mapping[str, str]
|
||||
) -> AgentCreateRequest | _InvalidAgentRequest:
|
||||
try:
|
||||
return AgentCreateRequest.model_validate(fields)
|
||||
except ValidationError as e:
|
||||
invalid_fields: Final = frozenset(str(error["loc"][0]) for error in e.errors())
|
||||
if drop_invalid_fields and "id" not in invalid_fields:
|
||||
return AgentCreateRequest.model_validate(
|
||||
{field: value for field, value in fields.items() if field not in invalid_fields}
|
||||
)
|
||||
return _InvalidAgentRequest(_invalid_create_fields_message(invalid_fields, caller_names))
|
||||
|
||||
|
||||
def _create_request_or_error(
|
||||
id: str | None,
|
||||
name: str | None,
|
||||
base_agent: str | None,
|
||||
agent_config: Mapping[str, object] | None,
|
||||
system_instruction: str | None,
|
||||
instructions: str | None,
|
||||
base_environment: InteractionEnvironment | None,
|
||||
description: str | None,
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
drop_invalid_fields: bool,
|
||||
) -> AgentCreateRequest | _InvalidAgentRequest:
|
||||
agent_id: Final = _agent_id_or_error(id, name)
|
||||
if isinstance(agent_id, _InvalidAgentRequest):
|
||||
return agent_id
|
||||
instruction_conflict: Final = _alias_conflict(
|
||||
"system_instruction", system_instruction, "instructions", instructions
|
||||
)
|
||||
if instruction_conflict is not None and not drop_invalid_fields:
|
||||
return _InvalidAgentRequest(instruction_conflict)
|
||||
sent_only_instructions: Final = system_instruction is None and instructions is not None
|
||||
return _validated_create_request(
|
||||
MappingProxyType(
|
||||
{
|
||||
"id": agent_id,
|
||||
"base_agent": base_agent,
|
||||
"agent_config": agent_config,
|
||||
"system_instruction": instructions if sent_only_instructions else system_instruction,
|
||||
"base_environment": base_environment,
|
||||
"description": description,
|
||||
"tools": tools,
|
||||
}
|
||||
),
|
||||
drop_invalid_fields,
|
||||
MappingProxyType(
|
||||
{
|
||||
"id": "name" if id is None else "id",
|
||||
"system_instruction": "instructions" if sent_only_instructions else "system_instruction",
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@client
|
||||
async def acreate(
|
||||
name: str,
|
||||
id: str | None = None,
|
||||
base_agent: str | None = None,
|
||||
instructions: str | None = None,
|
||||
base_environment: InteractionEnvironment | None = None,
|
||||
|
|
@ -104,6 +216,12 @@ async def acreate(
|
|||
extra_headers: dict[str, str] | None = None,
|
||||
extra_body: Mapping[str, object] | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
*,
|
||||
name: str | None = None,
|
||||
agent_config: Mapping[str, object] | None = None,
|
||||
system_instruction: str | None = None,
|
||||
description: str | None = None,
|
||||
tools: Sequence[Mapping[str, object]] | None = None,
|
||||
**kwargs,
|
||||
) -> AgentCreateResponse:
|
||||
"""Async: Create a managed agent on the provider side."""
|
||||
|
|
@ -113,7 +231,7 @@ async def acreate(
|
|||
kwargs["acreate_agent"] = True
|
||||
func: Final = partial(
|
||||
create,
|
||||
name=name,
|
||||
id=id,
|
||||
base_agent=base_agent,
|
||||
instructions=instructions,
|
||||
base_environment=base_environment,
|
||||
|
|
@ -121,6 +239,11 @@ async def acreate(
|
|||
extra_headers=extra_headers,
|
||||
extra_body=extra_body,
|
||||
timeout=timeout,
|
||||
name=name,
|
||||
agent_config=agent_config,
|
||||
system_instruction=system_instruction,
|
||||
description=description,
|
||||
tools=tools,
|
||||
**kwargs,
|
||||
)
|
||||
ctx: Final = contextvars.copy_context()
|
||||
|
|
@ -130,7 +253,7 @@ async def acreate(
|
|||
return init_response
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=name,
|
||||
model=id or name or "",
|
||||
custom_llm_provider=custom_llm_provider or "gemini",
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
|
|
@ -140,7 +263,7 @@ async def acreate(
|
|||
|
||||
@client
|
||||
def create(
|
||||
name: str,
|
||||
id: str | None = None,
|
||||
base_agent: str | None = None,
|
||||
instructions: str | None = None,
|
||||
base_environment: InteractionEnvironment | None = None,
|
||||
|
|
@ -148,40 +271,61 @@ def create(
|
|||
extra_headers: dict[str, str] | None = None,
|
||||
extra_body: Mapping[str, object] | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
*,
|
||||
name: str | None = None,
|
||||
agent_config: Mapping[str, object] | None = None,
|
||||
system_instruction: str | None = None,
|
||||
description: str | None = None,
|
||||
tools: Sequence[Mapping[str, object]] | None = None,
|
||||
**kwargs,
|
||||
) -> AgentCreateResponse | Coroutine[object, object, AgentCreateResponse]:
|
||||
"""
|
||||
Sync: Create a managed agent on the provider side.
|
||||
|
||||
Args:
|
||||
name: Name for the agent (required).
|
||||
base_agent: Base agent to derive from (e.g. "waverunner").
|
||||
instructions: System instructions for the agent.
|
||||
id: Identifier for the agent. Exactly one of ``id`` or its alias ``name`` is required.
|
||||
base_agent: Base agent to derive from (e.g. "antigravity-preview-09-2026").
|
||||
instructions: Alias of ``system_instruction``.
|
||||
base_environment: Environment to fork from — an env_id string or a
|
||||
dict like ``{"type": "remote", "sources": [...]}``.
|
||||
custom_llm_provider: Provider to use, e.g. "gemini".
|
||||
extra_headers: Additional HTTP headers.
|
||||
extra_body: Additional request body fields.
|
||||
timeout: Request timeout.
|
||||
name: Alias of ``id``.
|
||||
agent_config: Agent runtime config, e.g. ``{"type": "antigravity", "model": "gemini-3.8-flash"}``.
|
||||
system_instruction: System instruction for the agent.
|
||||
description: Description of the agent for developers.
|
||||
tools: Tools available to the agent, e.g. ``[{"type": "code_execution"}]``.
|
||||
**kwargs: Forwarded to GenericLiteLLMParams (api_key, api_base, etc.).
|
||||
"""
|
||||
local_vars: Final = locals()
|
||||
custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
|
||||
custom_llm_provider = custom_llm_provider or "gemini"
|
||||
try:
|
||||
_is_async: Final = kwargs.pop("acreate_agent", False) is True
|
||||
if base_agent is not None:
|
||||
kwargs["base_agent"] = base_agent
|
||||
if instructions is not None:
|
||||
kwargs["instructions"] = instructions
|
||||
if base_environment is not None:
|
||||
kwargs["base_environment"] = base_environment
|
||||
create_request: Final = _create_request_or_error(
|
||||
id=id,
|
||||
name=name,
|
||||
base_agent=base_agent,
|
||||
agent_config=agent_config,
|
||||
system_instruction=system_instruction,
|
||||
instructions=instructions,
|
||||
base_environment=base_environment,
|
||||
description=description,
|
||||
tools=tools,
|
||||
drop_invalid_fields=drops_unsupported_params(kwargs.get("drop_params")),
|
||||
)
|
||||
if isinstance(create_request, _InvalidAgentRequest):
|
||||
raise litellm.BadRequestError(
|
||||
message=create_request.message, model=id or name or "", llm_provider=custom_llm_provider
|
||||
)
|
||||
kwargs.setdefault("custom_llm_provider", custom_llm_provider)
|
||||
litellm_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
logging_obj: Final = _make_logging_obj(kwargs, name, custom_llm_provider, "create_agent", {})
|
||||
logging_obj: Final = _make_logging_obj(kwargs, create_request.id, custom_llm_provider, "create_agent", {})
|
||||
config: Final = _get_agents_api_config(custom_llm_provider)
|
||||
return agents_http_handler.create_agent(
|
||||
agents_api_config=config,
|
||||
name=name,
|
||||
create_request=create_request,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=logging_obj,
|
||||
extra_headers=extra_headers,
|
||||
|
|
@ -191,7 +335,7 @@ def create(
|
|||
)
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=name,
|
||||
model=id or name or "",
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
|
|
@ -247,7 +391,7 @@ def list(
|
|||
) -> AgentListResponse | Coroutine[object, object, AgentListResponse]:
|
||||
"""Sync: List all agents on the provider side."""
|
||||
local_vars: Final = locals()
|
||||
custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
|
||||
custom_llm_provider = custom_llm_provider or "gemini"
|
||||
try:
|
||||
_is_async: Final = kwargs.pop("alist_agents", False) is True
|
||||
kwargs.setdefault("custom_llm_provider", custom_llm_provider)
|
||||
|
|
@ -279,19 +423,22 @@ def list(
|
|||
|
||||
@client
|
||||
async def aget(
|
||||
name: str,
|
||||
id: str | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
extra_headers: dict[str, str] | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
*,
|
||||
name: str | None = None,
|
||||
**kwargs,
|
||||
) -> AgentCreateResponse:
|
||||
"""Async: Get a specific agent by name."""
|
||||
"""Async: Get a specific agent by id (``name`` is an alias of ``id``)."""
|
||||
local_vars: Final = locals()
|
||||
try:
|
||||
loop: Final = asyncio.get_event_loop()
|
||||
kwargs["aget_agent"] = True
|
||||
func: Final = partial(
|
||||
get,
|
||||
id=id,
|
||||
name=name,
|
||||
custom_llm_provider=custom_llm_provider or "gemini",
|
||||
extra_headers=extra_headers,
|
||||
|
|
@ -305,7 +452,7 @@ async def aget(
|
|||
return init_response
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=name,
|
||||
model=id or name or "",
|
||||
custom_llm_provider=custom_llm_provider or "gemini",
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
|
|
@ -315,24 +462,27 @@ async def aget(
|
|||
|
||||
@client
|
||||
def get(
|
||||
name: str,
|
||||
id: str | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
extra_headers: dict[str, str] | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
*,
|
||||
name: str | None = None,
|
||||
**kwargs,
|
||||
) -> AgentCreateResponse | Coroutine[object, object, AgentCreateResponse]:
|
||||
"""Sync: Get a specific agent by name."""
|
||||
"""Sync: Get a specific agent by id (``name`` is an alias of ``id``)."""
|
||||
local_vars: Final = locals()
|
||||
custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
|
||||
custom_llm_provider = custom_llm_provider or "gemini"
|
||||
try:
|
||||
_is_async: Final = kwargs.pop("aget_agent", False) is True
|
||||
agent_id: Final = _required_agent_id(id, name, custom_llm_provider)
|
||||
kwargs.setdefault("custom_llm_provider", custom_llm_provider)
|
||||
litellm_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
logging_obj: Final = _make_logging_obj(kwargs, name, custom_llm_provider, "get_agent", {"name": name})
|
||||
logging_obj: Final = _make_logging_obj(kwargs, agent_id, custom_llm_provider, "get_agent", {"name": agent_id})
|
||||
config: Final = _get_agents_api_config(custom_llm_provider)
|
||||
return agents_http_handler.get_agent(
|
||||
agents_api_config=config,
|
||||
name=name,
|
||||
name=agent_id,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=logging_obj,
|
||||
extra_headers=extra_headers,
|
||||
|
|
@ -341,7 +491,7 @@ def get(
|
|||
)
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=name,
|
||||
model=id or name or "",
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
|
|
@ -356,19 +506,22 @@ def get(
|
|||
|
||||
@client
|
||||
async def adelete(
|
||||
name: str,
|
||||
id: str | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
extra_headers: dict[str, str] | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
*,
|
||||
name: str | None = None,
|
||||
**kwargs,
|
||||
) -> AgentDeleteResult:
|
||||
"""Async: Delete a specific agent by name."""
|
||||
"""Async: Delete a specific agent by id (``name`` is an alias of ``id``)."""
|
||||
local_vars: Final = locals()
|
||||
try:
|
||||
loop: Final = asyncio.get_event_loop()
|
||||
kwargs["adelete_agent"] = True
|
||||
func: Final = partial(
|
||||
delete,
|
||||
id=id,
|
||||
name=name,
|
||||
custom_llm_provider=custom_llm_provider or "gemini",
|
||||
extra_headers=extra_headers,
|
||||
|
|
@ -382,7 +535,7 @@ async def adelete(
|
|||
return init_response
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=name,
|
||||
model=id or name or "",
|
||||
custom_llm_provider=custom_llm_provider or "gemini",
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
|
|
@ -392,24 +545,29 @@ async def adelete(
|
|||
|
||||
@client
|
||||
def delete(
|
||||
name: str,
|
||||
id: str | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
extra_headers: dict[str, str] | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
*,
|
||||
name: str | None = None,
|
||||
**kwargs,
|
||||
) -> AgentDeleteResult | Coroutine[object, object, AgentDeleteResult]:
|
||||
"""Sync: Delete a specific agent by name."""
|
||||
"""Sync: Delete a specific agent by id (``name`` is an alias of ``id``)."""
|
||||
local_vars: Final = locals()
|
||||
custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
|
||||
custom_llm_provider = custom_llm_provider or "gemini"
|
||||
try:
|
||||
_is_async: Final = kwargs.pop("adelete_agent", False) is True
|
||||
agent_id: Final = _required_agent_id(id, name, custom_llm_provider)
|
||||
kwargs.setdefault("custom_llm_provider", custom_llm_provider)
|
||||
litellm_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
logging_obj: Final = _make_logging_obj(kwargs, name, custom_llm_provider, "delete_agent", {"name": name})
|
||||
logging_obj: Final = _make_logging_obj(
|
||||
kwargs, agent_id, custom_llm_provider, "delete_agent", {"name": agent_id}
|
||||
)
|
||||
config: Final = _get_agents_api_config(custom_llm_provider)
|
||||
return agents_http_handler.delete_agent(
|
||||
agents_api_config=config,
|
||||
name=name,
|
||||
name=agent_id,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=logging_obj,
|
||||
extra_headers=extra_headers,
|
||||
|
|
@ -418,7 +576,7 @@ def delete(
|
|||
)
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=name,
|
||||
model=id or name or "",
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
|
|
@ -433,19 +591,22 @@ def delete(
|
|||
|
||||
@client
|
||||
async def alist_versions(
|
||||
name: str,
|
||||
id: str | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
extra_headers: dict[str, str] | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
*,
|
||||
name: str | None = None,
|
||||
**kwargs,
|
||||
) -> AgentVersionsResponse:
|
||||
"""Async: List versions of a specific agent."""
|
||||
"""Async: List versions of a specific agent by id (``name`` is an alias of ``id``)."""
|
||||
local_vars: Final = locals()
|
||||
try:
|
||||
loop: Final = asyncio.get_event_loop()
|
||||
kwargs["alist_agent_versions"] = True
|
||||
func: Final = partial(
|
||||
list_versions,
|
||||
id=id,
|
||||
name=name,
|
||||
custom_llm_provider=custom_llm_provider or "gemini",
|
||||
extra_headers=extra_headers,
|
||||
|
|
@ -459,7 +620,7 @@ async def alist_versions(
|
|||
return init_response
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=name,
|
||||
model=id or name or "",
|
||||
custom_llm_provider=custom_llm_provider or "gemini",
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
|
|
@ -469,24 +630,29 @@ async def alist_versions(
|
|||
|
||||
@client
|
||||
def list_versions(
|
||||
name: str,
|
||||
id: str | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
extra_headers: dict[str, str] | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
*,
|
||||
name: str | None = None,
|
||||
**kwargs,
|
||||
) -> AgentVersionsResponse | Coroutine[object, object, AgentVersionsResponse]:
|
||||
"""Sync: List versions of a specific agent."""
|
||||
"""Sync: List versions of a specific agent by id (``name`` is an alias of ``id``)."""
|
||||
local_vars: Final = locals()
|
||||
custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
|
||||
custom_llm_provider = custom_llm_provider or "gemini"
|
||||
try:
|
||||
_is_async: Final = kwargs.pop("alist_agent_versions", False) is True
|
||||
agent_id: Final = _required_agent_id(id, name, custom_llm_provider)
|
||||
kwargs.setdefault("custom_llm_provider", custom_llm_provider)
|
||||
litellm_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
logging_obj: Final = _make_logging_obj(kwargs, name, custom_llm_provider, "list_agent_versions", {"name": name})
|
||||
logging_obj: Final = _make_logging_obj(
|
||||
kwargs, agent_id, custom_llm_provider, "list_agent_versions", {"name": agent_id}
|
||||
)
|
||||
config: Final = _get_agents_api_config(custom_llm_provider)
|
||||
return agents_http_handler.list_agent_versions(
|
||||
agents_api_config=config,
|
||||
name=name,
|
||||
name=agent_id,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=logging_obj,
|
||||
extra_headers=extra_headers,
|
||||
|
|
@ -495,7 +661,7 @@ def list_versions(
|
|||
)
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=name,
|
||||
model=id or name or "",
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ from litellm.interactions.utils import (
|
|||
InteractionsAPIRequestUtils,
|
||||
get_provider_interactions_api_config,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.interactions import (
|
||||
CancelInteractionResult,
|
||||
|
|
@ -100,6 +101,8 @@ async def acreate(
|
|||
timeout: float | httpx.Timeout | None = None,
|
||||
# LiteLLM params
|
||||
custom_llm_provider: str | None = None,
|
||||
*,
|
||||
agent_config: Mapping[str, object] | None = None,
|
||||
**kwargs,
|
||||
) -> InteractionsAPIResponse | AsyncIterator[InteractionsAPIStreamingResponse]:
|
||||
"""
|
||||
|
|
@ -121,6 +124,7 @@ async def acreate(
|
|||
string, or a config object such as
|
||||
``{"type": "remote", "sources": [...]}`` /
|
||||
``{"type": "remote", "network": {...}}``
|
||||
agent_config: Agent runtime config, e.g. ``{"type": "antigravity", "model": "gemini-3.8-flash"}``
|
||||
response_modalities: Requested response modalities (TEXT, IMAGE, AUDIO)
|
||||
response_format: JSON schema for response format
|
||||
response_mime_type: MIME type of the response
|
||||
|
|
@ -155,6 +159,7 @@ async def acreate(
|
|||
store=store,
|
||||
background=background,
|
||||
environment=environment,
|
||||
agent_config=agent_config,
|
||||
response_modalities=response_modalities,
|
||||
response_format=response_format,
|
||||
response_mime_type=response_mime_type,
|
||||
|
|
@ -225,6 +230,8 @@ def create(
|
|||
timeout: float | httpx.Timeout | None = None,
|
||||
# LiteLLM params
|
||||
custom_llm_provider: str | None = None,
|
||||
*,
|
||||
agent_config: Mapping[str, object] | None = None,
|
||||
**kwargs,
|
||||
) -> (
|
||||
InteractionsAPIResponse
|
||||
|
|
@ -250,6 +257,7 @@ def create(
|
|||
string, or a config object such as
|
||||
``{"type": "remote", "sources": [...]}`` /
|
||||
``{"type": "remote", "network": {...}}``
|
||||
agent_config: Agent runtime config, e.g. ``{"type": "antigravity", "model": "gemini-3.8-flash"}``
|
||||
response_modalities: Requested response modalities (TEXT, IMAGE, AUDIO)
|
||||
response_format: JSON schema for response format
|
||||
response_mime_type: MIME type of the response
|
||||
|
|
@ -300,6 +308,15 @@ def create(
|
|||
# Check if this is a bridge provider (litellm_responses) - similar to responses API
|
||||
# Either provider is explicitly "litellm_responses" or no config found (bridge to responses)
|
||||
if custom_llm_provider == "litellm_responses" or interactions_api_config is None:
|
||||
if optional_params.get("agent_config") is not None and not drops_unsupported_params(
|
||||
kwargs.get("drop_params")
|
||||
):
|
||||
raise litellm.UnsupportedParamsError(
|
||||
message=f"'agent_config' is not supported for {custom_llm_provider} interactions. "
|
||||
"Set drop_params to drop it.",
|
||||
llm_provider=custom_llm_provider,
|
||||
model=model,
|
||||
)
|
||||
# Bridge to litellm.responses() for non-native providers
|
||||
from litellm.interactions.litellm_responses_transformation.handler import (
|
||||
LiteLLMResponsesInteractionsHandler,
|
||||
|
|
|
|||
|
|
@ -11,4 +11,4 @@ Core files:
|
|||
- `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s"
|
||||
- `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion])
|
||||
|
||||
Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Missing native bindings or unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer.
|
||||
Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer.
|
||||
|
|
|
|||
|
|
@ -2,8 +2,12 @@
|
|||
Utils used for litellm.transcription() and litellm.atranscription()
|
||||
"""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import math
|
||||
import os
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -331,6 +335,45 @@ def calculate_request_duration(file: FileTypes) -> float | None:
|
|||
return None
|
||||
|
||||
|
||||
DEFAULT_AUDIO_TOKEN_ESTIMATE: Final = 300
|
||||
AUDIO_TOKENS_PER_SECOND: Final = 10
|
||||
AUDIO_BYTES_PER_TOKEN: Final = 1600
|
||||
_MAX_AUDIO_ESTIMATE_BYTES: Final = 16 * 1024 * 1024
|
||||
|
||||
|
||||
def _audio_token_rate(model: str) -> int | None:
|
||||
if model.startswith(
|
||||
("gemini-", "gemini/gemini-", "vertex_ai/gemini-", "vertex_ai_beta/gemini-", "openrouter/google/gemini-")
|
||||
):
|
||||
family: Final = model.rsplit("/", 1)[-1]
|
||||
return 25 if family.startswith(("gemini-3-", "gemini-3.")) else 32
|
||||
if model.startswith(("gpt-", "openai/gpt-", "azure/gpt-", "openrouter/openai/gpt-")):
|
||||
return AUDIO_TOKENS_PER_SECOND
|
||||
return None
|
||||
|
||||
|
||||
def estimate_audio_tokens(data: str | None, models: Sequence[str] = (), fallback_model: str | None = None) -> int:
|
||||
"""Estimate audio tokens from container duration within a bounded decoding budget."""
|
||||
if not data:
|
||||
return DEFAULT_AUDIO_TOKEN_ESTIMATE
|
||||
fallback_estimate: Final = max(len(data) * 3 // 4 // AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE)
|
||||
padding: Final = 2 if data.endswith("==") else 1 if data.endswith("=") else 0
|
||||
if len(data) * 3 // 4 - padding > _MAX_AUDIO_ESTIMATE_BYTES:
|
||||
return fallback_estimate
|
||||
try:
|
||||
duration: Final = calculate_request_duration(base64.b64decode(data, validate=True))
|
||||
except (ValueError, binascii.Error):
|
||||
return fallback_estimate
|
||||
if duration is None or not math.isfinite(duration) or duration <= 0:
|
||||
return fallback_estimate
|
||||
fallback_rate: Final = _audio_token_rate(fallback_model) if fallback_model is not None else None
|
||||
rate: Final = max(
|
||||
(_audio_token_rate(model) or fallback_rate or AUDIO_TOKENS_PER_SECOND for model in models),
|
||||
default=fallback_rate or AUDIO_TOKENS_PER_SECOND,
|
||||
)
|
||||
return max(1, math.ceil(duration * rate))
|
||||
|
||||
|
||||
DEFAULT_SPEECH_MEDIA_TYPE: Final = "audio/mpeg"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -353,9 +353,9 @@ def is_cli_token_fresh(
|
|||
buffer_hours: float = CLI_TOKEN_FRESHNESS_BUFFER_SECONDS / 3600,
|
||||
) -> bool:
|
||||
"""Check whether a cached CLI token is still within its expiration window.
|
||||
Used by `lite auth print-token` to fail fast, without a network round trip,
|
||||
once the cached token is past `LITELLM_CLI_JWT_EXPIRATION_HOURS`. A `--pkce`
|
||||
credential carries its own `expires_at`, which is authoritative when present."""
|
||||
Used by `lite auth print-token` to skip asking the proxy while the cached token
|
||||
is younger than `LITELLM_CLI_JWT_EXPIRATION_HOURS`. A `--pkce` credential
|
||||
carries its own `expires_at`, which is authoritative when present."""
|
||||
from litellm.constants import CLI_JWT_EXPIRATION_HOURS
|
||||
|
||||
expires_at: Final = (
|
||||
|
|
|
|||
|
|
@ -2036,14 +2036,6 @@ def _map_azure_exception(
|
|||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
)
|
||||
elif "This model's maximum context length is" in error_str:
|
||||
raise ContextWindowExceededError(
|
||||
message=f"AzureException ContextWindowExceededError - {message}",
|
||||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
)
|
||||
elif "DeploymentNotFound" in error_str:
|
||||
raise NotFoundError(
|
||||
message=f"AzureException NotFoundError - {message}",
|
||||
|
|
@ -2085,6 +2077,18 @@ def _map_azure_exception(
|
|||
response=getattr(original_exception, "response", None),
|
||||
body=getattr(original_exception, "body", None),
|
||||
)
|
||||
elif getattr(original_exception, "status_code", None) in (None, 400) and (
|
||||
azure_error_code == "context_length_exceeded"
|
||||
or ExceptionCheckers.is_error_str_context_window_exceeded(error_str)
|
||||
):
|
||||
raise ContextWindowExceededError(
|
||||
message=f"AzureException ContextWindowExceededError - {message}",
|
||||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
body=getattr(original_exception, "body", None),
|
||||
)
|
||||
elif "invalid_request_error" in error_str and getattr(original_exception, "status_code", None) in (None, 400):
|
||||
raise BadRequestError(
|
||||
message=f"AzureException BadRequestError - {message}",
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from typing import Final, cast
|
|||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.constants import CONTROL_OPTIONS_KEY
|
||||
from litellm.constants import CONTROL_OPTIONS_KEY, SESSION_ID_FROM_TRACE_ID_PARAM_KEY
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
||||
from litellm.llms.openai.data_residency import infer_openai_data_residency
|
||||
from litellm.types.litellm_params import MAX_CONTROL_INT_DIGITS, ControlOptions
|
||||
|
|
@ -212,6 +212,9 @@ def get_litellm_params(
|
|||
|
||||
# Derive litellm_session_id / litellm_trace_id from metadata when not provided (call chaining)
|
||||
_meta: Final = resolved_metadata or {}
|
||||
session_id_from_trace_id: Final = (
|
||||
litellm_session_id is None and not _meta.get("session_id") and bool(_meta.get("trace_id"))
|
||||
)
|
||||
if litellm_session_id is None:
|
||||
litellm_session_id = _meta.get("session_id") or _meta.get("trace_id")
|
||||
if litellm_trace_id is None:
|
||||
|
|
@ -254,6 +257,7 @@ def get_litellm_params(
|
|||
or (get_base_model_from_litellm_call_metadata(metadata=metadata) if metadata else None),
|
||||
"litellm_trace_id": litellm_trace_id,
|
||||
"litellm_session_id": litellm_session_id,
|
||||
**({SESSION_ID_FROM_TRACE_ID_PARAM_KEY: True} if session_id_from_trace_id else {}),
|
||||
"hf_model_name": hf_model_name,
|
||||
"custom_prompt_dict": custom_prompt_dict,
|
||||
"litellm_metadata": litellm_metadata,
|
||||
|
|
|
|||
|
|
@ -842,6 +842,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self._async_success_enqueued: bool = False
|
||||
self._sync_success_scheduled: bool = False
|
||||
self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
|
||||
self._stream_failure_claimed: bool = False
|
||||
self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
|
||||
|
||||
def copy_for_new_call(self, litellm_call_id: str, start_time: datetime.datetime) -> "Logging":
|
||||
|
|
@ -2561,6 +2562,12 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
},
|
||||
)
|
||||
|
||||
def claim_stream_failure_report(self) -> bool:
|
||||
if self._stream_failure_claimed:
|
||||
return False
|
||||
self._stream_failure_claimed = True
|
||||
return True
|
||||
|
||||
def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
|
||||
"""Bill a fully streamed response on the failure log when a post-call hook rejects it."""
|
||||
usage: Final = getattr(assembled, "usage", None)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import json
|
|||
import mimetypes
|
||||
import re
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from graphlib import CycleError, TopologicalSorter
|
||||
from itertools import chain, groupby, islice
|
||||
from os import PathLike
|
||||
|
|
@ -236,6 +237,160 @@ _IMAGE_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_image", "image
|
|||
_IMAGE_SCAN_MAX_DEPTH: Final = 4
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _TPMMediaProjection:
|
||||
content: object
|
||||
media: tuple[Mapping[str, object], ...] = ()
|
||||
|
||||
|
||||
def _tpm_image_part(url: str, detail: object = "auto") -> Mapping[str, object]:
|
||||
return {"type": "image_url", "image_url": {"url": url, "detail": detail}}
|
||||
|
||||
|
||||
def _tpm_blank_fields(fields: Mapping[str, object], names: tuple[str, ...]) -> Mapping[str, object]:
|
||||
return {key: "" if key in names and isinstance(value, str) else value for key, value in fields.items()}
|
||||
|
||||
|
||||
def _tpm_file_media(part: Mapping[str, object]) -> _TPMMediaProjection:
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type
|
||||
|
||||
nested: Final = part.get("type") == "file"
|
||||
fields: Final = as_str_mapping(part.get("file")) if nested else part
|
||||
if fields is None:
|
||||
return _TPMMediaProjection(part)
|
||||
raw_data: Final = fields.get("file_data")
|
||||
data: Final = raw_data if isinstance(raw_data, str) else ""
|
||||
declared: Final = next(
|
||||
(
|
||||
value
|
||||
for key in ("format", "mime_type", "content_type")
|
||||
if isinstance(value := fields.get(key), str) and value
|
||||
),
|
||||
"",
|
||||
)
|
||||
mime: Final = (declared or parse_mime_type(data[:256]) or "").split(";", 1)[0].strip().lower()
|
||||
if not mime.startswith(("image/", "audio/")):
|
||||
return _TPMMediaProjection(part)
|
||||
raw_url: Final = fields.get("file_url") or fields.get("file_id")
|
||||
url: Final = data or (raw_url if isinstance(raw_url, str) else "")
|
||||
header_end: Final = data.find(",", 0, 256)
|
||||
encoded: Final = (
|
||||
data[header_end + 1 :]
|
||||
if header_end >= 0 and data[:header_end].endswith(";base64")
|
||||
else ""
|
||||
if data.startswith(("data:", "https://", "http://", "gs://"))
|
||||
else data
|
||||
)
|
||||
media: Final = (
|
||||
_tpm_image_part(
|
||||
url
|
||||
if not data or url.startswith(("data:", "https://", "http://", "gs://"))
|
||||
else f"data:{mime};base64,{url}",
|
||||
fields.get("detail", "auto"),
|
||||
)
|
||||
if mime.startswith("image/")
|
||||
else {"type": "input_audio", "input_audio": {"data": encoded, "format": mime.removeprefix("audio/")}}
|
||||
)
|
||||
projected: Final = _tpm_blank_fields(fields, ("file_data", "file_url"))
|
||||
return _TPMMediaProjection({**part, "file": projected} if nested else projected, (media,))
|
||||
|
||||
|
||||
def _tpm_media_leaf(value: object) -> _TPMMediaProjection:
|
||||
part: Final = as_str_mapping(value)
|
||||
if part is None:
|
||||
return _TPMMediaProjection(value)
|
||||
kind: Final = part.get("type")
|
||||
if kind in ("file", "input_file"):
|
||||
return _tpm_file_media(part)
|
||||
if kind == "input_audio":
|
||||
audio: Final = as_str_mapping(part.get("input_audio"))
|
||||
return _TPMMediaProjection(
|
||||
{**part, "input_audio": _tpm_blank_fields(audio, ("data",))} if audio is not None else value,
|
||||
({"type": "input_audio", "input_audio": part.get("input_audio")},),
|
||||
)
|
||||
if kind in ("image_url", "input_image", "image", "computer_screenshot"):
|
||||
image: Final = as_str_mapping(part.get("image_url"))
|
||||
source: Final = as_str_mapping(part.get("source"))
|
||||
canonical: Final = _tpm_image_part(
|
||||
_image_part_url({**part, "type": "input_image"} if kind == "computer_screenshot" else part) or "",
|
||||
(image or part).get("detail", part.get("detail", "auto")),
|
||||
)
|
||||
projected: Final = (
|
||||
{**part, "image_url": _tpm_blank_fields(image, ("url",))}
|
||||
if image is not None and kind in ("image_url", "input_image", "computer_screenshot")
|
||||
else {**part, "source": _tpm_blank_fields(source, ("data", "url"))}
|
||||
if source is not None and kind == "image" and source.get("type") in ("base64", "url")
|
||||
else _tpm_blank_fields(part, ("image_url", "url"))
|
||||
)
|
||||
return _TPMMediaProjection(projected, (canonical,))
|
||||
return _TPMMediaProjection(value)
|
||||
|
||||
|
||||
def _tpm_media_list(value: object, split: Callable[[object], _TPMMediaProjection]) -> _TPMMediaProjection:
|
||||
if not isinstance(value, (list, tuple)):
|
||||
return _TPMMediaProjection(value)
|
||||
parts: Final = cast(Sequence[object], value) # cast-ok: finite JSON content arrays
|
||||
projected: Final = tuple(map(split, parts))
|
||||
media: Final = tuple(chain.from_iterable(part.media for part in projected))
|
||||
content: Final = tuple(part.content for part in projected)
|
||||
return _TPMMediaProjection(parts if not media else content if isinstance(value, tuple) else list(content), media)
|
||||
|
||||
|
||||
def _tpm_replace_field(part: Mapping[str, object], key: str, projection: _TPMMediaProjection) -> _TPMMediaProjection:
|
||||
return _TPMMediaProjection({**part, key: projection.content} if projection.media else part, projection.media)
|
||||
|
||||
|
||||
def _tpm_document_media(value: object) -> _TPMMediaProjection:
|
||||
part: Final = as_str_mapping(value)
|
||||
source: Final = as_str_mapping(part.get("source")) if part is not None else None
|
||||
if part is None or part.get("type") != "document" or source is None or source.get("type") != "content":
|
||||
return _tpm_media_leaf(value)
|
||||
content: Final = _tpm_media_list(source.get("content"), _tpm_media_leaf)
|
||||
return _tpm_replace_field(part, "source", _tpm_replace_field(source, "content", content))
|
||||
|
||||
|
||||
def _tpm_tool_result_media(value: object) -> _TPMMediaProjection:
|
||||
part: Final = as_str_mapping(value)
|
||||
if part is None or part.get("type") != "tool_result":
|
||||
return _tpm_document_media(value)
|
||||
return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_document_media))
|
||||
|
||||
|
||||
def _tpm_code_output_media(value: object) -> _TPMMediaProjection:
|
||||
part: Final = as_str_mapping(value)
|
||||
url: Final = part.get("url") if part is not None else None
|
||||
if part is None or part.get("type") != "image" or not isinstance(url, str) or not url:
|
||||
return _TPMMediaProjection(value)
|
||||
return _TPMMediaProjection(_tpm_blank_fields(part, ("url",)), (_tpm_image_part(url),))
|
||||
|
||||
|
||||
def _tpm_input_media(value: object) -> _TPMMediaProjection:
|
||||
part: Final = as_str_mapping(value)
|
||||
if part is None:
|
||||
return _TPMMediaProjection(value)
|
||||
kind: Final = part.get("type")
|
||||
if kind in ("function_call_output", "custom_tool_call_output"):
|
||||
return _tpm_replace_field(part, "output", _tpm_media_list(part.get("output"), _tpm_tool_result_media))
|
||||
if kind == "computer_call_output":
|
||||
return _tpm_replace_field(part, "output", _tpm_media_leaf(part.get("output")))
|
||||
if kind == "code_interpreter_call":
|
||||
return _tpm_replace_field(part, "outputs", _tpm_media_list(part.get("outputs"), _tpm_code_output_media))
|
||||
result: Final = part.get("result")
|
||||
if kind == "image_generation_call" and isinstance(result, str) and result:
|
||||
return _TPMMediaProjection(
|
||||
_tpm_blank_fields(part, ("result",)),
|
||||
(_tpm_image_part("data:application/octet-stream;base64," + result),),
|
||||
)
|
||||
if kind == "message" or (kind is None and isinstance(part.get("role"), str)):
|
||||
return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_tool_result_media))
|
||||
return _tpm_tool_result_media(value)
|
||||
|
||||
|
||||
def split_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]:
|
||||
projection: Final = _tpm_media_list(value, _tpm_input_media)
|
||||
return projection.content, projection.media
|
||||
|
||||
|
||||
def _content_parts_contain_image(parts: Sequence[object]) -> bool:
|
||||
"""Depth-bounded frontier walk over nested content lists, iterative because the repo bans
|
||||
recursion; an Anthropic tool_result nests its image parts exactly one level down."""
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import re
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY
|
||||
from litellm.constants import SESSION_ID_FROM_TRACE_ID_PARAM_KEY, SESSION_ID_GENERATED_METADATA_KEY
|
||||
|
||||
_HTTP_HEADER_NAME_PATTERN: Final = re.compile(r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$")
|
||||
_FORBIDDEN_AFFINITY_HEADERS: Final = frozenset(
|
||||
|
|
@ -53,36 +53,51 @@ def _get_provider_affinity_header_name(litellm_params: object | None, default_he
|
|||
return validate_provider_affinity_header_name(header)
|
||||
|
||||
|
||||
def get_stable_session_id(litellm_params: object | None) -> str | None:
|
||||
if litellm_params is None:
|
||||
return None
|
||||
|
||||
direct_session_id: Final = _get_value(litellm_params, "session_id")
|
||||
if direct_session_id:
|
||||
return str(direct_session_id)
|
||||
|
||||
metadata_values: Final[tuple[object, ...]] = tuple(
|
||||
def _metadata_values(litellm_params: object) -> tuple[object, ...]:
|
||||
return tuple(
|
||||
value for key in ("metadata", "litellm_metadata") if (value := _get_value(litellm_params, key)) is not None
|
||||
)
|
||||
|
||||
|
||||
def _non_generated_litellm_session_id(litellm_params: object) -> str | None:
|
||||
has_generated_session_id: Final = any(
|
||||
isinstance(metadata, Mapping) and metadata.get(SESSION_ID_GENERATED_METADATA_KEY)
|
||||
for metadata in metadata_values
|
||||
_get_value(metadata, SESSION_ID_GENERATED_METADATA_KEY) for metadata in _metadata_values(litellm_params)
|
||||
)
|
||||
|
||||
litellm_session_id: Final = _get_value(litellm_params, "litellm_session_id")
|
||||
if litellm_session_id and not has_generated_session_id:
|
||||
return str(litellm_session_id)
|
||||
return str(litellm_session_id) if litellm_session_id and not has_generated_session_id else None
|
||||
|
||||
for metadata in metadata_values:
|
||||
if (
|
||||
isinstance(metadata, Mapping)
|
||||
and not metadata.get(SESSION_ID_GENERATED_METADATA_KEY)
|
||||
and (value := metadata.get("session_id"))
|
||||
|
||||
def _metadata_session_id(litellm_params: object) -> str | None:
|
||||
for metadata in _metadata_values(litellm_params):
|
||||
if not _get_value(metadata, SESSION_ID_GENERATED_METADATA_KEY) and (
|
||||
value := _get_value(metadata, "session_id")
|
||||
):
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
|
||||
def get_caller_litellm_session_id(litellm_params: object | None) -> str | None:
|
||||
if litellm_params is None or _get_value(litellm_params, SESSION_ID_FROM_TRACE_ID_PARAM_KEY):
|
||||
return None
|
||||
return _non_generated_litellm_session_id(litellm_params)
|
||||
|
||||
|
||||
def get_requested_session_id(litellm_params: object | None) -> str | None:
|
||||
if litellm_params is None:
|
||||
return None
|
||||
direct_session_id: Final = _get_value(litellm_params, "session_id")
|
||||
return str(direct_session_id) if direct_session_id else _metadata_session_id(litellm_params)
|
||||
|
||||
|
||||
def get_stable_session_id(litellm_params: object | None) -> str | None:
|
||||
if litellm_params is None:
|
||||
return None
|
||||
direct_session_id: Final = _get_value(litellm_params, "session_id")
|
||||
if direct_session_id:
|
||||
return str(direct_session_id)
|
||||
return _non_generated_litellm_session_id(litellm_params) or _metadata_session_id(litellm_params)
|
||||
|
||||
|
||||
def add_provider_affinity_header(
|
||||
headers: Mapping[str, object], litellm_params: object | None, default_header: str | None = None
|
||||
) -> dict[str, object]: # mutable-ok: downstream handlers add auth and signing headers
|
||||
|
|
|
|||
92
litellm/litellm_core_utils/single_flight.py
Normal file
92
litellm/litellm_core_utils/single_flight.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
import threading
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import Future
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
from typing import Final, Generic, ParamSpec, TypeVar
|
||||
|
||||
_P: Final = ParamSpec("_P")
|
||||
_R: Final = TypeVar("_R")
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SingleFlight(Generic[_R]):
|
||||
lock: threading.Lock = field(default_factory=threading.Lock)
|
||||
in_flight: Future[_R] | None = None
|
||||
flights_started: int = 0
|
||||
last_flight_failed: bool = False
|
||||
|
||||
def run(self, fetch: Callable[[], _R], while_busy: _R | None = None) -> _R:
|
||||
leader: Final[Future[_R]] = Future()
|
||||
with self.lock:
|
||||
in_flight: Final = self.in_flight
|
||||
joined_flight: Final = self.flights_started
|
||||
retry_if_it_fails: Final = not self.last_flight_failed
|
||||
if in_flight is None:
|
||||
self._take_off(leader)
|
||||
if in_flight is None:
|
||||
return self._lead(leader, fetch)
|
||||
if while_busy is not None:
|
||||
return while_busy
|
||||
try:
|
||||
return in_flight.result()
|
||||
except Exception:
|
||||
retry: Final = self._claim_retry(joined_flight) if retry_if_it_fails else None
|
||||
if retry is None:
|
||||
raise
|
||||
retry_flight, leads_retry = retry
|
||||
return self._lead(retry_flight, fetch) if leads_retry else retry_flight.result()
|
||||
|
||||
def _take_off(self, leader: Future[_R]) -> None:
|
||||
self.in_flight = leader
|
||||
self.flights_started += 1
|
||||
|
||||
def _claim_retry(self, failed_flight: int) -> tuple[Future[_R], bool] | None:
|
||||
leader: Final[Future[_R]] = Future()
|
||||
with self.lock:
|
||||
in_flight: Final = self.in_flight
|
||||
if in_flight is not None:
|
||||
return in_flight, False
|
||||
if self.flights_started > failed_flight and self.last_flight_failed:
|
||||
return None
|
||||
self._take_off(leader)
|
||||
return leader, True
|
||||
|
||||
def _lead(self, flight: Future[_R], fetch: Callable[[], _R]) -> _R:
|
||||
try:
|
||||
result: Final = fetch()
|
||||
except BaseException as error:
|
||||
self._land(failed=True)
|
||||
flight.set_exception(error)
|
||||
raise
|
||||
self._land(failed=False)
|
||||
flight.set_result(result)
|
||||
return result
|
||||
|
||||
def _land(self, failed: bool) -> None:
|
||||
with self.lock:
|
||||
self.in_flight = None
|
||||
self.last_flight_failed = failed
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SingleFlightCache(Generic[_P, _R]):
|
||||
cached: Callable[..., _R]
|
||||
clear: Callable[[], None]
|
||||
lock: threading.Lock = field(default_factory=threading.Lock)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
*args: _P.args,
|
||||
**kwargs: _P.kwargs, # kwargs-ok: forwarded verbatim to the cached builder, typed by its own signature
|
||||
) -> _R:
|
||||
with self.lock:
|
||||
return self.cached(*args, **kwargs)
|
||||
|
||||
def cache_clear(self) -> None:
|
||||
self.clear()
|
||||
|
||||
|
||||
def single_flight_lru_cache(build: Callable[_P, _R]) -> SingleFlightCache[_P, _R]:
|
||||
cached: Final = lru_cache(maxsize=128)(build)
|
||||
return SingleFlightCache(cached, cached.cache_clear)
|
||||
|
|
@ -1557,6 +1557,12 @@ class CustomStreamWrapper:
|
|||
else: # openai / azure chat model
|
||||
if isinstance(chunk, ChatCompletionChunk):
|
||||
self._openai_handler_receives_chat_chunks = True
|
||||
if any(str(choice.finish_reason) == "abort" for choice in chunk.choices or ()):
|
||||
raise litellm.InternalServerError(
|
||||
message="Upstream generation aborted (finish_reason=abort)",
|
||||
model=chunk.model,
|
||||
llm_provider=self.custom_llm_provider,
|
||||
)
|
||||
if self.custom_llm_provider in [
|
||||
LlmProviders.AZURE.value,
|
||||
LlmProviders.AZURE_AI.value,
|
||||
|
|
|
|||
|
|
@ -3,9 +3,12 @@
|
|||
import base64
|
||||
import io
|
||||
import math
|
||||
import re
|
||||
import struct
|
||||
from bisect import bisect_right
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
|
||||
from itertools import accumulate
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast
|
||||
|
||||
import anyio
|
||||
|
|
@ -35,6 +38,7 @@ from litellm.constants import (
|
|||
TOKEN_COUNTER_MAX_EXACT_CHARS,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.chat_template_tools import (
|
||||
ChatTemplateToolSection,
|
||||
chat_template_tool_section,
|
||||
|
|
@ -188,22 +192,23 @@ def resize_image_high_res(
|
|||
|
||||
# Test the function with the given example
|
||||
def calculate_tiles_needed(
|
||||
resized_width,
|
||||
resized_height,
|
||||
tile_width=MAX_TILE_WIDTH,
|
||||
tile_height=MAX_TILE_HEIGHT,
|
||||
):
|
||||
resized_width: int,
|
||||
resized_height: int,
|
||||
tile_width: int = MAX_TILE_WIDTH,
|
||||
tile_height: int = MAX_TILE_HEIGHT,
|
||||
) -> int:
|
||||
tiles_across: Final = (resized_width + tile_width - 1) // tile_width
|
||||
tiles_down: Final = (resized_height + tile_height - 1) // tile_height
|
||||
total_tiles: Final = tiles_across * tiles_down
|
||||
return total_tiles
|
||||
|
||||
|
||||
def high_detail_image_token_upper_bound(base_tokens: int = 85) -> int:
|
||||
def high_detail_image_token_upper_bound(base_tokens: int = 85, tile_tokens: int | None = None) -> int:
|
||||
largest_tile_count: Final = calculate_tiles_needed(
|
||||
MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES, MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES
|
||||
)
|
||||
return base_tokens + (base_tokens * 2) * largest_tile_count
|
||||
per_tile: Final = base_tokens * 2 if tile_tokens is None else tile_tokens
|
||||
return base_tokens + per_tile * largest_tile_count
|
||||
|
||||
|
||||
def _unpack_ints(fmt: str, buffer: bytes) -> tuple[int, ...]:
|
||||
|
|
@ -381,6 +386,219 @@ def calculate_img_tokens(
|
|||
return total_tokens
|
||||
|
||||
|
||||
_IMAGE_HEADER_BASE64_CHARS: Final = 65_536
|
||||
_MAX_IMAGE_PATCHES: Final = 30_000
|
||||
_ANTHROPIC_IMAGE_LIMITS: Final = MappingProxyType(
|
||||
{"claude-standard": (ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX, 1568), "claude-high": (2576, 4784)}
|
||||
)
|
||||
_TILE_IMAGE_TOKEN_COSTS: Final = MappingProxyType(
|
||||
{
|
||||
"gpt-4o": (85, 170),
|
||||
"gpt-4.1": (85, 170),
|
||||
"gpt-4o-mini": (2833, 5667),
|
||||
"gpt-5": (70, 140),
|
||||
"gpt-5.1": (70, 140),
|
||||
"o1": (75, 150),
|
||||
"o1-pro": (75, 150),
|
||||
"o3": (75, 150),
|
||||
}
|
||||
)
|
||||
_PATCH_IMAGE_MULTIPLIERS: Final = MappingProxyType(
|
||||
{
|
||||
"gpt-6-astra": 1.2,
|
||||
"gpt-5.6-sol": 1.2,
|
||||
"gpt-5.6-terra": 1.2,
|
||||
"gpt-5.6-luna": 1.2,
|
||||
"gpt-5.5": 1.2,
|
||||
"gpt-5.4": 1.2,
|
||||
"gpt-5.4-mini": 1.2,
|
||||
"gpt-5.4-nano": 1.2,
|
||||
"gpt-5.2": 1.2,
|
||||
"gpt-4.1-mini": 1.62,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _inline_image_dimensions(data: str) -> tuple[int, int] | None:
|
||||
separator: Final = data.find(",", 0, 256)
|
||||
if not data.startswith("data:") or separator < 0 or not data[:separator].endswith(";base64"):
|
||||
return None
|
||||
encoded_header: Final = data[separator + 1 : separator + 1 + _IMAGE_HEADER_BASE64_CHARS]
|
||||
try:
|
||||
header: Final = base64.b64decode(encoded_header, validate=True)
|
||||
image_type: Final = get_image_type(header)
|
||||
if image_type not in ("png", "gif", "jpeg", "webp"):
|
||||
return None
|
||||
if image_type == "webp" and header[12:16] not in (b"VP8X", b"VP8 ", b"VP8L"):
|
||||
return None
|
||||
dimensions: Final = _image_dimensions_from_bytes(header)
|
||||
except (ValueError, TypeError, struct.error):
|
||||
return None
|
||||
width, height = dimensions
|
||||
return (width, height) if width > 0 and height > 0 else None
|
||||
|
||||
|
||||
def _patch_image_limits(model: str, detail: str) -> tuple[int, int | None] | None:
|
||||
if model in ("gpt-5.2", "gpt-4.1-mini"):
|
||||
return None if detail == "original" else (2048, 6144)
|
||||
original_default: Final = model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.5")
|
||||
effective_detail: Final = ("original" if original_default else "high") if detail == "auto" else detail
|
||||
if effective_detail == "low":
|
||||
return (2048, 6144) if model in ("gpt-5.4", "gpt-5.4-mini", "gpt-5.4-nano") else (512, None)
|
||||
if effective_detail == "high":
|
||||
return (65_535 if model == "gpt-6-astra" else 2048, 2500)
|
||||
return (
|
||||
(65_535, None) if model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna") else (6000, 10_000)
|
||||
)
|
||||
|
||||
|
||||
def _image_patch_count(width: int, height: int, max_dimension: int, budget: int | None) -> int:
|
||||
dimension_scale: Final = min(1.0, max_dimension / max(width, height))
|
||||
fitted_width: Final = max(1, round(width * dimension_scale))
|
||||
fitted_height: Final = max(1, round(height * dimension_scale))
|
||||
patches: Final = math.ceil(fitted_width / 32) * math.ceil(fitted_height / 32)
|
||||
if budget is None or patches <= budget:
|
||||
return patches
|
||||
shrink: Final = math.sqrt(32**2 * budget / (fitted_width * fitted_height))
|
||||
width_patches: Final = fitted_width * shrink / 32
|
||||
height_patches: Final = fitted_height * shrink / 32
|
||||
aligned_shrink: Final = shrink * min(
|
||||
math.floor(width_patches) / width_patches, math.floor(height_patches) / height_patches
|
||||
)
|
||||
resized_width: Final = max(1, math.floor(fitted_width * aligned_shrink))
|
||||
resized_height: Final = max(1, math.floor(fitted_height * aligned_shrink))
|
||||
return math.ceil(resized_width / 32) * math.ceil(resized_height / 32)
|
||||
|
||||
|
||||
def _image_policy_model_name(model: str) -> str | None:
|
||||
provider, separator, provider_model = model.partition("/")
|
||||
if not separator:
|
||||
return model
|
||||
if provider == "openai":
|
||||
return provider_model
|
||||
if provider == "openrouter":
|
||||
return provider_model.removeprefix("openai/") if provider_model.startswith("openai/") else None
|
||||
if provider not in ("azure", "azure_ai"):
|
||||
return None
|
||||
catalog_entry: Final = as_str_mapping(
|
||||
cast( # cast-ok: read-only view of the legacy string-keyed registry; as_str_mapping checks the entry
|
||||
Mapping[str, object], litellm.model_cost
|
||||
).get(model)
|
||||
)
|
||||
if catalog_entry is not None and catalog_entry.get("litellm_provider") in ("azure", "azure_ai"):
|
||||
return provider_model.rsplit("/", 1)[-1]
|
||||
return provider_model.removeprefix("responses/").removeprefix("o_series/").removeprefix("openai/")
|
||||
|
||||
|
||||
def _anthropic_image_family(model: str) -> str | None:
|
||||
from litellm.utils import get_potential_model_names
|
||||
|
||||
provider, separator, _ = model.partition("/")
|
||||
if separator and provider not in (
|
||||
"anthropic",
|
||||
"bedrock",
|
||||
"bedrock_converse",
|
||||
"bedrock_mantle",
|
||||
"vertex_ai",
|
||||
"vertex_ai_beta",
|
||||
"azure_ai",
|
||||
"openrouter",
|
||||
):
|
||||
return None
|
||||
candidates: Final = get_potential_model_names(model, provider if separator else "anthropic")
|
||||
catalog: Final = cast(Mapping[str, object], litellm.model_cost) # cast-ok: read-only view of the legacy registry
|
||||
for candidate in dict.fromkeys(candidates.values()):
|
||||
if not isinstance(candidate, str):
|
||||
continue
|
||||
if (entry := as_str_mapping(catalog.get(candidate))) is None or entry.get("supports_vision") is not True:
|
||||
continue
|
||||
if (
|
||||
version := re.search(
|
||||
r"(?:^|[./])claude-(?:(?:haiku|sonnet|opus)-)?(\d+)(?:[-.](\d{1,2})(?!\d))?", candidate
|
||||
)
|
||||
) is None:
|
||||
continue
|
||||
major, minor = int(version[1]), int(version[2] or 0)
|
||||
return "claude-high" if (major, minor) >= (4, 7) else "claude-standard"
|
||||
return None
|
||||
|
||||
|
||||
def _anthropic_image_tokens(dimensions: tuple[int, int] | None, limits: tuple[int, int]) -> int:
|
||||
max_edge, token_budget = limits
|
||||
if dimensions is None:
|
||||
return token_budget
|
||||
long_edge, short_edge = max(dimensions), min(dimensions)
|
||||
aspect_ratio: Final = long_edge / short_edge
|
||||
|
||||
def tokens_at_edge(edge: int) -> int:
|
||||
return math.ceil(edge / 28) * math.ceil(max(round(edge / aspect_ratio), 1) / 28)
|
||||
|
||||
fitted_edge: Final = bisect_right(range(1, min(long_edge, max_edge) + 1), token_budget, key=tokens_at_edge)
|
||||
return tokens_at_edge(max(fitted_edge, 1))
|
||||
|
||||
|
||||
def _image_model_family(model: str) -> str | None:
|
||||
provider_model: Final = _image_policy_model_name(model)
|
||||
if provider_model is None:
|
||||
return _anthropic_image_family(model)
|
||||
family: Final = re.sub(r"-\d{4}-\d{2}-\d{2}$", "", provider_model)
|
||||
canonical: Final = litellm.azure_llms.get(f"azure/{family}", family) if model.startswith("azure/") else family
|
||||
return (
|
||||
canonical
|
||||
if canonical in _TILE_IMAGE_TOKEN_COSTS or canonical in _PATCH_IMAGE_MULTIPLIERS
|
||||
else _anthropic_image_family(model)
|
||||
)
|
||||
|
||||
|
||||
def _image_policy_tokens(family: str | None, dimensions: tuple[int, int] | None, detail: str) -> int:
|
||||
if family is None:
|
||||
return DEFAULT_IMAGE_TOKEN_COUNT
|
||||
anthropic_limits: Final = _ANTHROPIC_IMAGE_LIMITS.get(family)
|
||||
if anthropic_limits is not None:
|
||||
return _anthropic_image_tokens(dimensions, anthropic_limits)
|
||||
tile_costs: Final = _TILE_IMAGE_TOKEN_COSTS.get(family)
|
||||
multiplier: Final = _PATCH_IMAGE_MULTIPLIERS.get(family)
|
||||
if tile_costs is not None:
|
||||
base_tokens, tile_tokens = tile_costs
|
||||
if detail == "original":
|
||||
return DEFAULT_IMAGE_TOKEN_COUNT
|
||||
if detail == "low":
|
||||
return base_tokens
|
||||
if dimensions is None:
|
||||
return high_detail_image_token_upper_bound(base_tokens, tile_tokens)
|
||||
width, height = dimensions
|
||||
scale: Final = min(1.0, 2048 / max(width, height), 768 / min(width, height))
|
||||
tiles: Final = calculate_tiles_needed(max(1, math.floor(width * scale)), max(1, math.floor(height * scale)))
|
||||
return base_tokens + tiles * tile_tokens
|
||||
limits: Final = _patch_image_limits(family, detail)
|
||||
if limits is None or multiplier is None:
|
||||
return DEFAULT_IMAGE_TOKEN_COUNT
|
||||
max_dimension, patch_budget = limits
|
||||
patches: Final = (
|
||||
min(math.ceil(max_dimension / 32) ** 2, patch_budget if patch_budget is not None else _MAX_IMAGE_PATCHES)
|
||||
if dimensions is None
|
||||
else _image_patch_count(*dimensions, max_dimension, patch_budget)
|
||||
)
|
||||
return math.ceil(patches * multiplier)
|
||||
|
||||
|
||||
def estimate_image_tokens(
|
||||
model: str | Sequence[str], data: str, detail: str = "auto", *, fallback_model: str | None = None
|
||||
) -> int:
|
||||
"""Estimate image policies from at most 48 KiB of inline headers, without fetching media."""
|
||||
if detail not in ("low", "high", "auto", "original"):
|
||||
return DEFAULT_IMAGE_TOKEN_COUNT
|
||||
models: Final = (model,) if isinstance(model, str) else model
|
||||
fallback_family: Final = _image_model_family(fallback_model) if fallback_model is not None else None
|
||||
families: Final = tuple(
|
||||
dict.fromkeys(_image_model_family(candidate) or fallback_family for candidate in models or ("",))
|
||||
)
|
||||
if all(family is None for family in families):
|
||||
return DEFAULT_IMAGE_TOKEN_COUNT
|
||||
dimensions: Final = _inline_image_dimensions(data)
|
||||
return max(_image_policy_tokens(family, dimensions, detail) for family in families)
|
||||
|
||||
|
||||
TokenCounterFunction = Callable[[str], int]
|
||||
"""
|
||||
Type for a function that counts tokens in a string.
|
||||
|
|
|
|||
|
|
@ -82,4 +82,6 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig):
|
|||
model_response.model = _json_response.get("model")
|
||||
model_response.object = _json_response.get("object")
|
||||
model_response.system_fingerprint = _json_response.get("system_fingerprint")
|
||||
if "service_tier" in _json_response:
|
||||
return model_response.model_copy(update={"service_tier": _json_response.get("service_tier")})
|
||||
return model_response
|
||||
|
|
|
|||
|
|
@ -161,11 +161,58 @@ def _has_non_chat_user_block(message: Mapping[str, object]) -> bool:
|
|||
return any(isinstance(block, dict) and block.get("type") not in ValidUserMessageContentTypes for block in blocks)
|
||||
|
||||
|
||||
_OPENAI_RESPONSES_BUILT_IN_TOOL_TYPES: Final = frozenset(
|
||||
(
|
||||
"apply_patch",
|
||||
"code_interpreter",
|
||||
"computer",
|
||||
"computer_use_preview",
|
||||
"file_search",
|
||||
"image_generation",
|
||||
"local_shell",
|
||||
"mcp",
|
||||
"shell",
|
||||
"web_search",
|
||||
"web_search_2025_08_26",
|
||||
"web_search_preview",
|
||||
"web_search_preview_2025_03_11",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
_CHAT_INFERENCE_TRANSLATED_TOOL_TYPES: Final = frozenset(("namespace", "tool_search"))
|
||||
|
||||
|
||||
def _is_responses_built_in_tool(tool: Mapping[str, object]) -> bool:
|
||||
tool_type: Final = tool.get("type")
|
||||
return isinstance(tool_type, str) and tool_type in _OPENAI_RESPONSES_BUILT_IN_TOOL_TYPES
|
||||
|
||||
|
||||
def _is_chat_inference_translated_tool(tool: Mapping[str, object]) -> bool:
|
||||
tool_type: Final = tool.get("type")
|
||||
return isinstance(tool_type, str) and tool_type in _CHAT_INFERENCE_TRANSLATED_TOOL_TYPES
|
||||
|
||||
|
||||
def _needs_tool_translation(tool: Mapping[str, object]) -> bool:
|
||||
return _is_chat_tool(tool) or _is_chat_inference_translated_tool(tool) or _is_responses_built_in_tool(tool)
|
||||
|
||||
|
||||
def _gets_placeholder_name(tool: Mapping[str, object]) -> bool:
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import is_provider_native_tool_dict
|
||||
|
||||
name: Final = tool.get("name")
|
||||
return not (_is_chat_tool(tool) or is_provider_native_tool_dict(tool) or (isinstance(name, str) and name.strip()))
|
||||
return not (
|
||||
_is_chat_tool(tool)
|
||||
or _is_chat_inference_translated_tool(tool)
|
||||
or is_provider_native_tool_dict(tool)
|
||||
or (isinstance(name, str) and name.strip())
|
||||
)
|
||||
|
||||
|
||||
def _chat_tools_for(tool: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import chat_completion_tools_for
|
||||
|
||||
return (tool,) if _is_chat_inference_translated_tool(tool) else chat_completion_tools_for(tool)
|
||||
|
||||
|
||||
def _untranslatable_input(
|
||||
|
|
@ -182,8 +229,8 @@ def _untranslatable_input(
|
|||
unnamed_tool: Final = next(filter(_gets_placeholder_name, tools), None)
|
||||
if unnamed_tool is not None:
|
||||
return ChatCountTranslationError(
|
||||
message=f"Invalid request for token counting: tool type {unnamed_tool.get('type')!r} "
|
||||
"has no Chat Completions equivalent"
|
||||
message=f"Invalid request for token counting: LiteLLM cannot count a tool of type "
|
||||
f"{unnamed_tool.get('type')!r} for this model"
|
||||
)
|
||||
return None
|
||||
|
||||
|
|
@ -223,9 +270,7 @@ def _translated_chat_request(
|
|||
chat_params: Mapping[str, object],
|
||||
translator: _ChatRequestTranslator,
|
||||
) -> _ChatParams:
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import chat_completion_tools_for
|
||||
|
||||
chat_tools: Final = [dict(chat_tool) for chat_tool in chain.from_iterable(map(chat_completion_tools_for, tools))]
|
||||
chat_tools: Final = [dict(chat_tool) for chat_tool in chain.from_iterable(map(_chat_tools_for, tools))]
|
||||
openai_params: Final = {name: value for name, value in chat_params.items() if name != "drop_params"}
|
||||
mapped_params: Final = {**openai_params, **({"tools": chat_tools} if chat_tools else {})}
|
||||
chat_messages: Final = cast( # cast-ok: chat-shaped request messages, read the way chat inference reads them
|
||||
|
|
@ -255,7 +300,7 @@ def anthropic_fields_for_chat_tool_use(
|
|||
counted_tools: Final = tools or ()
|
||||
if (
|
||||
not chat_params
|
||||
and not any(map(_is_chat_tool, counted_tools))
|
||||
and not any(map(_needs_tool_translation, counted_tools))
|
||||
and not any(map(_needs_chat_translation, messages))
|
||||
):
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -2,15 +2,16 @@
|
|||
## Translates OpenAI call to Anthropic `/v1/messages` format
|
||||
import asyncio
|
||||
from collections import deque
|
||||
from collections.abc import AsyncGenerator, Iterator, Mapping
|
||||
from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import anyio
|
||||
from pydantic import ConfigDict, field_validator
|
||||
|
||||
from litellm import verbose_logger
|
||||
from litellm._logging import redact_internal_details_from_client_message
|
||||
from litellm._uuid import uuid
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
from litellm.exceptions import APIError, MidStreamFallbackError
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
encrypted_reasoning_signature,
|
||||
)
|
||||
|
|
@ -22,6 +23,7 @@ from litellm.llms.anthropic.messages.native.utils import (
|
|||
from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
|
||||
from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
|
||||
from litellm.responses.streaming_iterator import (
|
||||
map_stream_error_to_exception,
|
||||
partial_responses_incomplete_event_on_disconnect,
|
||||
stream_error_status_and_message,
|
||||
)
|
||||
|
|
@ -130,10 +132,12 @@ class AnthropicResponsesStreamWrapper:
|
|||
) -> None:
|
||||
self.responses_stream = responses_stream
|
||||
self.model = model
|
||||
self.litellm_logging_obj = litellm_logging_obj
|
||||
self.safeguards_evaluator: Final = safeguards_evaluator
|
||||
self._message_id: str = f"msg_{uuid.uuid4()}"
|
||||
if litellm_logging_obj is not None:
|
||||
litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id)
|
||||
self._unreported_failure: Exception | None = None
|
||||
self._current_block_index: int = -1
|
||||
# Map item_id -> content_block_index so we can stop the right block later
|
||||
self._item_id_to_block_index: dict[str, int] = {}
|
||||
|
|
@ -377,16 +381,24 @@ class AnthropicResponsesStreamWrapper:
|
|||
|
||||
if event_type == "response.failed":
|
||||
failed: Final = _FailedResponseEvent.model_validate(event)
|
||||
status_code, message = stream_error_status_and_message(
|
||||
failed.response.error if failed.response is not None else None
|
||||
)
|
||||
upstream_error: Final = failed.response.error if failed.response is not None else None
|
||||
status_code, message = stream_error_status_and_message(upstream_error)
|
||||
verbose_logger.error(
|
||||
"AnthropicResponsesStreamWrapper: upstream Responses stream for %s failed (%s): %s",
|
||||
self.model,
|
||||
status_code,
|
||||
message,
|
||||
)
|
||||
self._fail_stream(status_code, message)
|
||||
self._fail_stream(
|
||||
status_code,
|
||||
message,
|
||||
failure=map_stream_error_to_exception(
|
||||
upstream_error,
|
||||
self.model,
|
||||
getattr(self.responses_stream, "custom_llm_provider", None) or "",
|
||||
logging_obj=self.litellm_logging_obj,
|
||||
),
|
||||
)
|
||||
return
|
||||
|
||||
# ---- response completed -> message_delta + message_stop ----
|
||||
|
|
@ -439,10 +451,36 @@ class AnthropicResponsesStreamWrapper:
|
|||
self._sent_message_stop = True
|
||||
return
|
||||
|
||||
def _fail_stream(self, status_code: int, message: str) -> None:
|
||||
def _fail_stream(self, status_code: int, message: str, failure: Exception | None = None) -> None:
|
||||
self._stream_failed = True
|
||||
self._unreported_failure = failure or APIError(
|
||||
status_code=status_code,
|
||||
message=message,
|
||||
llm_provider=getattr(self.responses_stream, "custom_llm_provider", None),
|
||||
model=self.model,
|
||||
)
|
||||
self._chunk_queue.append(_anthropic_error_chunk(status_code, message))
|
||||
|
||||
async def _report_unreported_failure(self) -> None:
|
||||
failure: Final = self._unreported_failure
|
||||
if failure is None:
|
||||
return
|
||||
self._unreported_failure = None
|
||||
on_failure: Final[Callable[[Exception], Awaitable[None]] | None] = getattr(
|
||||
self.litellm_logging_obj, "_on_detached_stream_failure", None
|
||||
)
|
||||
if on_failure is None:
|
||||
return
|
||||
try:
|
||||
with anyio.CancelScope(shield=True):
|
||||
await on_failure(failure)
|
||||
except Exception as hook_failure: # noqa: BLE001 # a failing proxy hook must not hide the error event from the client
|
||||
verbose_logger.warning(
|
||||
"AnthropicResponsesStreamWrapper: stream failure hook raised: %s(%s)",
|
||||
type(hook_failure).__name__,
|
||||
hook_failure,
|
||||
)
|
||||
|
||||
def __aiter__(self) -> "AnthropicResponsesStreamWrapper":
|
||||
return self
|
||||
|
||||
|
|
@ -450,6 +488,7 @@ class AnthropicResponsesStreamWrapper:
|
|||
if self._chunk_queue:
|
||||
return self._chunk_queue.popleft()
|
||||
if self._stream_failed:
|
||||
await self._report_unreported_failure()
|
||||
raise StopAsyncIteration
|
||||
|
||||
if not self._sent_message_start:
|
||||
|
|
@ -478,7 +517,7 @@ class AnthropicResponsesStreamWrapper:
|
|||
verbose_logger.exception(
|
||||
"AnthropicResponsesStreamWrapper: upstream Responses stream for %s failed", self.model
|
||||
)
|
||||
self._fail_stream(*_failure_status_and_message(e))
|
||||
self._fail_stream(*_failure_status_and_message(e), failure=_original_failure(e))
|
||||
|
||||
if not self._chunk_queue and not self._sent_message_stop and not self._stream_failed:
|
||||
verbose_logger.error(
|
||||
|
|
@ -498,8 +537,14 @@ class AnthropicResponsesStreamWrapper:
|
|||
|
||||
async def _sse_frames(self) -> AsyncGenerator[bytes, None]:
|
||||
safeguard_results: Final = StreamedSafeguardResults(self.safeguards_evaluator)
|
||||
async for chunk in self:
|
||||
yield anthropic_sse_frame(await safeguard_results.observe(chunk))
|
||||
try:
|
||||
async for chunk in self:
|
||||
yield anthropic_sse_frame(await safeguard_results.observe(chunk))
|
||||
finally:
|
||||
from litellm.llms.anthropic.messages.native.streaming_iterator import closing_superseded_attempt
|
||||
|
||||
if not closing_superseded_attempt():
|
||||
await self._report_unreported_failure()
|
||||
|
||||
def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
|
||||
return partial_responses_incomplete_event_on_disconnect(self.responses_stream)
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import os
|
|||
import re
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from contextlib import closing
|
||||
from functools import lru_cache
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, NamedTuple, cast
|
||||
|
||||
|
|
@ -18,15 +17,18 @@ import litellm
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.constants import DEFAULT_MAX_RETRIES
|
||||
from litellm.litellm_core_utils.single_flight import SingleFlight, single_flight_lru_cache
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIAsyncHTTPClient, OpenAIHTTPClient
|
||||
from litellm.secret_managers.get_azure_ad_token_provider import (
|
||||
get_azure_ad_token_provider,
|
||||
single_flight_bearer_token_provider,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.secret_managers.get_azure_ad_token_provider import AzureCredentialType
|
||||
from litellm.utils import add_path_to_api_base
|
||||
|
||||
azure_ad_cache: Final = DualCache()
|
||||
|
|
@ -94,7 +96,7 @@ def process_azure_headers(headers: httpx.Headers | dict) -> dict:
|
|||
return {**llm_response_headers, **openai_headers}
|
||||
|
||||
|
||||
@lru_cache(maxsize=128)
|
||||
@single_flight_lru_cache
|
||||
def _cached_entra_id_token_provider(
|
||||
tenant_id: str,
|
||||
client_id: str,
|
||||
|
|
@ -107,33 +109,38 @@ def _cached_entra_id_token_provider(
|
|||
to expiry, so reusing the provider keeps one AAD round trip per token lifetime instead of one
|
||||
per request.
|
||||
"""
|
||||
from azure.identity import ClientSecretCredential, get_bearer_token_provider
|
||||
from azure.identity import ClientSecretCredential
|
||||
|
||||
return get_bearer_token_provider(ClientSecretCredential(tenant_id, client_id, client_secret), scope)
|
||||
return single_flight_bearer_token_provider(ClientSecretCredential(tenant_id, client_id, client_secret), scope)
|
||||
|
||||
|
||||
@lru_cache(maxsize=128)
|
||||
@single_flight_lru_cache
|
||||
def _cached_username_password_token_provider(
|
||||
client_id: str,
|
||||
azure_username: str,
|
||||
azure_password: str,
|
||||
scope: str,
|
||||
) -> Callable[[], str]:
|
||||
from azure.identity import UsernamePasswordCredential, get_bearer_token_provider
|
||||
from azure.identity import UsernamePasswordCredential
|
||||
|
||||
credential: Final = UsernamePasswordCredential(
|
||||
client_id=client_id,
|
||||
username=azure_username,
|
||||
password=azure_password,
|
||||
)
|
||||
return get_bearer_token_provider(credential, scope)
|
||||
return single_flight_bearer_token_provider(credential, scope)
|
||||
|
||||
|
||||
@lru_cache(maxsize=128)
|
||||
@single_flight_lru_cache
|
||||
def _cached_azure_ad_token_refresh_provider(scope: str) -> Callable[[], str]:
|
||||
return get_azure_ad_token_provider(azure_scope=scope)
|
||||
|
||||
|
||||
@single_flight_lru_cache
|
||||
def _cached_default_azure_credential_provider(scope: str) -> Callable[[], str]:
|
||||
return get_azure_ad_token_provider(azure_scope=scope, azure_credential=AzureCredentialType.DefaultAzureCredential)
|
||||
|
||||
|
||||
def cached_azure_ad_token_refresh_provider(scope: str) -> Callable[[], str]:
|
||||
return _cached_azure_ad_token_refresh_provider(scope)
|
||||
|
||||
|
|
@ -240,6 +247,58 @@ def get_azure_ad_token_from_username_password(
|
|||
return token_provider
|
||||
|
||||
|
||||
@single_flight_lru_cache
|
||||
def _oidc_token_exchange(cache_key: str) -> SingleFlight[str]:
|
||||
return SingleFlight()
|
||||
|
||||
|
||||
def _cached_oidc_token(cache_key: str) -> str | None:
|
||||
cached: Final = azure_ad_cache.get_cache(cache_key)
|
||||
return cached if isinstance(cached, str) else None
|
||||
|
||||
|
||||
def _exchange_oidc_token(cache_key: str, token_url: str, client_id: str, scope: str, oidc_token: str) -> str:
|
||||
cached_access_token: Final = _cached_oidc_token(cache_key)
|
||||
if cached_access_token is not None:
|
||||
return cached_access_token
|
||||
|
||||
with closing(HTTPHandler(timeout=litellm.request_timeout)) as client:
|
||||
req_token: Final = client.post(
|
||||
token_url,
|
||||
data={
|
||||
"client_id": client_id,
|
||||
"grant_type": "client_credentials",
|
||||
"scope": scope,
|
||||
"client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer",
|
||||
"client_assertion": oidc_token,
|
||||
},
|
||||
)
|
||||
|
||||
if req_token.status_code != 200:
|
||||
raise AzureOpenAIError(
|
||||
status_code=req_token.status_code,
|
||||
message=req_token.text,
|
||||
)
|
||||
|
||||
azure_ad_token_json: Final[_AzureAdTokenJson] = req_token.json()
|
||||
azure_ad_token_access_token: Final = azure_ad_token_json.get("access_token", None)
|
||||
azure_ad_token_expires_in: Final = azure_ad_token_json.get("expires_in", None)
|
||||
|
||||
if azure_ad_token_access_token is None:
|
||||
raise AzureOpenAIError(status_code=422, message="Azure AD Token access_token not returned")
|
||||
|
||||
if azure_ad_token_expires_in is None:
|
||||
raise AzureOpenAIError(status_code=422, message="Azure AD Token expires_in not returned")
|
||||
|
||||
azure_ad_cache.set_cache(
|
||||
key=cache_key,
|
||||
value=azure_ad_token_access_token,
|
||||
ttl=azure_ad_token_expires_in,
|
||||
)
|
||||
|
||||
return azure_ad_token_access_token
|
||||
|
||||
|
||||
def get_azure_ad_token_from_oidc(
|
||||
azure_ad_token: str,
|
||||
azure_client_id: str | None = None,
|
||||
|
|
@ -282,50 +341,25 @@ def get_azure_ad_token_from_oidc(
|
|||
"azure_client_id": azure_client_id,
|
||||
"azure_tenant_id": azure_tenant_id,
|
||||
"azure_authority_host": azure_authority_host,
|
||||
"scope": scope,
|
||||
"oidc_token": oidc_token,
|
||||
}
|
||||
)
|
||||
|
||||
azure_ad_token_access_token = azure_ad_cache.get_cache(azure_ad_token_cache_key)
|
||||
if azure_ad_token_access_token is not None:
|
||||
return azure_ad_token_access_token
|
||||
cached_access_token: Final = _cached_oidc_token(azure_ad_token_cache_key)
|
||||
if cached_access_token is not None:
|
||||
return cached_access_token
|
||||
|
||||
with closing(HTTPHandler(timeout=litellm.request_timeout)) as client:
|
||||
req_token: Final = client.post(
|
||||
f"{azure_authority_host}/{azure_tenant_id}/oauth2/v2.0/token",
|
||||
data={
|
||||
"client_id": azure_client_id,
|
||||
"grant_type": "client_credentials",
|
||||
"scope": scope,
|
||||
"client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer",
|
||||
"client_assertion": oidc_token,
|
||||
},
|
||||
return _oidc_token_exchange(azure_ad_token_cache_key).run(
|
||||
lambda: _exchange_oidc_token(
|
||||
azure_ad_token_cache_key,
|
||||
token_url=f"{azure_authority_host}/{azure_tenant_id}/oauth2/v2.0/token",
|
||||
client_id=azure_client_id,
|
||||
scope=scope,
|
||||
oidc_token=oidc_token,
|
||||
)
|
||||
|
||||
if req_token.status_code != 200:
|
||||
raise AzureOpenAIError(
|
||||
status_code=req_token.status_code,
|
||||
message=req_token.text,
|
||||
)
|
||||
|
||||
azure_ad_token_json: Final[_AzureAdTokenJson] = req_token.json()
|
||||
azure_ad_token_access_token = azure_ad_token_json.get("access_token", None)
|
||||
azure_ad_token_expires_in: Final = azure_ad_token_json.get("expires_in", None)
|
||||
|
||||
if azure_ad_token_access_token is None:
|
||||
raise AzureOpenAIError(status_code=422, message="Azure AD Token access_token not returned")
|
||||
|
||||
if azure_ad_token_expires_in is None:
|
||||
raise AzureOpenAIError(status_code=422, message="Azure AD Token expires_in not returned")
|
||||
|
||||
azure_ad_cache.set_cache(
|
||||
key=azure_ad_token_cache_key,
|
||||
value=azure_ad_token_access_token,
|
||||
ttl=azure_ad_token_expires_in,
|
||||
)
|
||||
|
||||
return azure_ad_token_access_token
|
||||
|
||||
|
||||
def select_azure_base_url_or_endpoint(azure_client_params: dict):
|
||||
azure_endpoint: Final = azure_client_params.get("azure_endpoint", None)
|
||||
|
|
@ -503,17 +537,10 @@ class BaseAzureLLM(BaseOpenAILLM):
|
|||
Returns:
|
||||
Token provider callable if DefaultAzureCredential is enabled and available, None otherwise
|
||||
"""
|
||||
from litellm.types.secret_managers.get_azure_ad_token_provider import (
|
||||
AzureCredentialType,
|
||||
)
|
||||
|
||||
verbose_logger.debug("Attempting to use DefaultAzureCredential for Azure Auth")
|
||||
|
||||
try:
|
||||
azure_ad_token_provider: Final = get_azure_ad_token_provider(
|
||||
azure_scope=scope,
|
||||
azure_credential=AzureCredentialType.DefaultAzureCredential,
|
||||
)
|
||||
azure_ad_token_provider: Final = _cached_default_azure_credential_provider(scope)
|
||||
verbose_logger.debug("Successfully obtained Azure AD token provider using DefaultAzureCredential")
|
||||
return azure_ad_token_provider
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from collections.abc import Mapping
|
|||
import httpx
|
||||
|
||||
from litellm.types.agents import (
|
||||
AgentCreateRequest,
|
||||
AgentCreateResponse,
|
||||
AgentDeleteResult,
|
||||
AgentListResponse,
|
||||
|
|
@ -50,10 +51,9 @@ class BaseAgentsAPIConfig(ABC):
|
|||
@abstractmethod
|
||||
def transform_create_request(
|
||||
self,
|
||||
name: str,
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
"""Map name + litellm_params to the provider's create-agent body."""
|
||||
create_request: AgentCreateRequest,
|
||||
) -> Mapping[str, object]:
|
||||
"""Map the create request to the provider's create-agent body."""
|
||||
|
||||
@abstractmethod
|
||||
def transform_create_response(
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from typing import Any, Final
|
|||
from pydantic import JsonValue
|
||||
from typing_extensions import assert_never
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.anthropic.count_tokens.transformation import ChatCountTranslationError, anthropic_count_fields
|
||||
|
|
@ -53,6 +54,18 @@ class CountTokensFailure:
|
|||
CountTokensOutcome = CountedTokens | CountTokensFailure
|
||||
|
||||
|
||||
def _claude_count_translator(litellm_params: Mapping[str, object]) -> AnthropicConfig:
|
||||
deployment_model: Final = litellm_params.get("model")
|
||||
if not isinstance(deployment_model, str):
|
||||
return AnthropicConfig()
|
||||
provider_hint: Final = litellm_params.get("custom_llm_provider")
|
||||
model, provider, _, _ = litellm.get_llm_provider(
|
||||
model=deployment_model, custom_llm_provider=provider_hint if isinstance(provider_hint, str) else None
|
||||
)
|
||||
chat_config: Final = litellm.ProviderConfigManager.get_provider_chat_config(model, LlmProviders(provider))
|
||||
return chat_config if isinstance(chat_config, AnthropicConfig) else AnthropicConfig()
|
||||
|
||||
|
||||
def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model: str) -> bool:
|
||||
return (
|
||||
isinstance(outcome, CountTokensFailure)
|
||||
|
|
@ -138,16 +151,22 @@ class BedrockTokenCounter(BaseTokenCounter):
|
|||
counted_model: Final = model_to_use.removeprefix("bedrock_mantle/")
|
||||
resolved_model: Final = get_bedrock_base_model(counted_model)
|
||||
primary_tokenizer: Final = MANTLE_TOKENIZER_TYPE if use_mantle else RUNTIME_TOKENIZER_TYPE
|
||||
litellm_params: Final = (deployment or {}).get("litellm_params", {})
|
||||
translated: Final = (
|
||||
await anthropic_count_fields(
|
||||
counted_model, messages, system, tools, messages_format, AnthropicConfig(), chat_params
|
||||
counted_model,
|
||||
messages,
|
||||
system,
|
||||
tools,
|
||||
messages_format,
|
||||
_claude_count_translator(litellm_params),
|
||||
chat_params,
|
||||
)
|
||||
if is_mantle_claude_model(resolved_model)
|
||||
else None
|
||||
)
|
||||
if isinstance(translated, ChatCountTranslationError):
|
||||
return translated.response(request_model, model_to_use, primary_tokenizer)
|
||||
litellm_params: Final = (deployment or {}).get("litellm_params", {})
|
||||
runtime_request: Final = (
|
||||
_request_data(model_to_use, messages, tools, system)
|
||||
if translated is None
|
||||
|
|
|
|||
|
|
@ -206,6 +206,10 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
encoded_body: Final = base64.b64encode(json.dumps(body_data).encode()).decode()
|
||||
return {"input": {"invokeModel": {"body": encoded_body}}}
|
||||
|
||||
@staticmethod
|
||||
def count_tokens_model_id(model: str) -> str:
|
||||
return get_bedrock_base_model(model).removeprefix("bedrock/")
|
||||
|
||||
def get_bedrock_count_tokens_endpoint(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -225,12 +229,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
Returns:
|
||||
Complete endpoint URL for CountTokens API
|
||||
"""
|
||||
# Use existing LiteLLM function to get the base model ID (removes region prefix)
|
||||
model_id = get_bedrock_base_model(model)
|
||||
|
||||
# Remove bedrock/ prefix if present
|
||||
model_id = model_id.removeprefix("bedrock/") # Remove "bedrock/" prefix
|
||||
encoded_model_id: Final = self.encode_model_id(model_id=model_id)
|
||||
encoded_model_id: Final = self.encode_model_id(model_id=self.count_tokens_model_id(model))
|
||||
|
||||
base_url, _ = self.get_runtime_endpoint(
|
||||
api_base=api_base,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Constants and helpers for ChatGPT subscription OAuth.
|
|||
"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
|
|
@ -17,6 +18,7 @@ import httpx
|
|||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
|
||||
from litellm.litellm_core_utils.provider_affinity import get_caller_litellm_session_id, get_requested_session_id
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.utils import ProviderSpecificHeader
|
||||
|
||||
|
|
@ -395,22 +397,31 @@ def _normalize_litellm_params(litellm_params: object) -> Mapping[str, object]:
|
|||
return MappingProxyType({})
|
||||
|
||||
|
||||
def _prompt_cache_session_id(prompt_cache_key: object) -> str | None:
|
||||
if not isinstance(prompt_cache_key, str) or not prompt_cache_key:
|
||||
return None
|
||||
return (
|
||||
prompt_cache_key
|
||||
if _safe_header_value(prompt_cache_key.strip()) == prompt_cache_key
|
||||
else hashlib.sha256(prompt_cache_key.encode("utf-8", "surrogatepass")).hexdigest()
|
||||
)
|
||||
|
||||
|
||||
def get_chatgpt_session_id(litellm_params: object) -> str | None:
|
||||
params: Final = _normalize_litellm_params(litellm_params)
|
||||
for key in ("litellm_session_id", "session_id"):
|
||||
value = params.get(key)
|
||||
if value:
|
||||
return str(value)
|
||||
metadata: Final = _normalize_litellm_params(params.get("metadata"))
|
||||
if metadata:
|
||||
value = metadata.get("session_id")
|
||||
if value:
|
||||
return str(value)
|
||||
for key in ("litellm_trace_id", "litellm_call_id"):
|
||||
value = params.get(key)
|
||||
if value:
|
||||
return str(value)
|
||||
return None
|
||||
return (
|
||||
get_caller_litellm_session_id(params)
|
||||
or get_requested_session_id(params)
|
||||
or _prompt_cache_session_id(params.get("prompt_cache_key"))
|
||||
or next(
|
||||
(
|
||||
str(value)
|
||||
for key in ("litellm_session_id", "litellm_trace_id", "litellm_call_id")
|
||||
if (value := params.get(key))
|
||||
),
|
||||
None,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def ensure_chatgpt_session_id(litellm_params: object) -> str:
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import ssl
|
|||
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence
|
||||
from contextlib import asynccontextmanager
|
||||
from functools import lru_cache, partial, reduce
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
|
|
@ -62,7 +63,7 @@ from litellm.litellm_core_utils.realtime_errors import (
|
|||
)
|
||||
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
|
||||
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import (
|
||||
BaseAnthropicMessagesConfig,
|
||||
)
|
||||
|
|
@ -572,6 +573,22 @@ def _unwrap_provider_error(error: Exception) -> Exception:
|
|||
return error
|
||||
|
||||
|
||||
def _merge_messages_header_sources(*sources: Mapping[str, object] | None) -> Mapping[str, object]:
|
||||
"""Later sources win per header, except ``anthropic-beta``, which unions every source so client betas
|
||||
never drop a deployment's configured ones."""
|
||||
entries: Final = tuple(chain.from_iterable((source or {}).items() for source in sources))
|
||||
beta_values: Final = tuple(
|
||||
cast("str | Sequence[str]", value) # cast-ok: the merge helper skips non-str list entries
|
||||
for name, value in entries
|
||||
if name.lower() == "anthropic-beta" and isinstance(value, str | Sequence)
|
||||
)
|
||||
merged_beta: Final = reduce(merge_anthropic_beta_headers, beta_values, "")
|
||||
return MappingProxyType(
|
||||
{name: value for name, value in entries if name.lower() != "anthropic-beta"}
|
||||
| ({"anthropic-beta": merged_beta} if merged_beta else {})
|
||||
)
|
||||
|
||||
|
||||
class BaseLLMHTTPHandler:
|
||||
async def _make_common_async_call(
|
||||
self,
|
||||
|
|
@ -2481,13 +2498,8 @@ class BaseLLMHTTPHandler:
|
|||
forwarded_headers: Final = kwargs.get("headers", None)
|
||||
# Also check for extra_headers in kwargs (from config or direct calls)
|
||||
extra_headers_from_kwargs: Final = kwargs.get("extra_headers", None)
|
||||
# Merge all header sources: forwarded < extra_headers < provider_specific
|
||||
merged_headers: Final[Mapping[str, object]] = MappingProxyType(
|
||||
{
|
||||
**(forwarded_headers or {}),
|
||||
**(extra_headers_from_kwargs or {}),
|
||||
**(provider_specific_headers or {}),
|
||||
}
|
||||
merged_headers: Final = _merge_messages_header_sources(
|
||||
forwarded_headers, extra_headers_from_kwargs, provider_specific_headers
|
||||
)
|
||||
request_params: Final = reduce(
|
||||
delete_nested_value,
|
||||
|
|
|
|||
|
|
@ -20,16 +20,13 @@ from litellm.litellm_core_utils.url_utils import join_api_base
|
|||
from litellm.llms.base_llm.agents.transformation import BaseAgentsAPIConfig
|
||||
from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo
|
||||
from litellm.types.agents import (
|
||||
AgentCreateRequest,
|
||||
AgentCreateResponse,
|
||||
AgentDeleteResult,
|
||||
AgentListResponse,
|
||||
AgentVersionsResponse,
|
||||
)
|
||||
|
||||
# Keys inside litellm_params that should be forwarded to the Gemini
|
||||
# create-agent body verbatim.
|
||||
_GEMINI_AGENT_BODY_KEYS: Final = ("base_agent", "instructions", "base_environment")
|
||||
|
||||
|
||||
class _JsonResponse(Protocol):
|
||||
def json(self) -> object: ...
|
||||
|
|
@ -39,30 +36,6 @@ def _json_body(response: _JsonResponse) -> object:
|
|||
return response.json()
|
||||
|
||||
|
||||
# LiteLLM-internal keys that must never be forwarded to Gemini.
|
||||
_LITELLM_INTERNAL_KEYS: Final = frozenset(
|
||||
{
|
||||
"custom_llm_provider",
|
||||
"api_key",
|
||||
"api_base",
|
||||
"make_public",
|
||||
"cost_per_query",
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
"require_trace_id_on_calls_to_agent",
|
||||
"require_trace_id_on_calls_by_agent",
|
||||
"max_iterations",
|
||||
"max_budget_per_session",
|
||||
"guardrails",
|
||||
"is_public",
|
||||
"agent_name",
|
||||
"agent_id",
|
||||
"agent_card_params",
|
||||
"provider_agent_response",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class GeminiAgentsConfig(BaseAgentsAPIConfig):
|
||||
"""
|
||||
Configuration for the Google AI Studio (Gemini) native Agents API.
|
||||
|
|
@ -143,14 +116,9 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig):
|
|||
|
||||
def transform_create_request(
|
||||
self,
|
||||
name: str,
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
body: Final[dict[str, object]] = {"name": name}
|
||||
for key in _GEMINI_AGENT_BODY_KEYS:
|
||||
value = litellm_params.get(key)
|
||||
if value is not None:
|
||||
body[key] = value
|
||||
create_request: AgentCreateRequest,
|
||||
) -> Mapping[str, object]:
|
||||
body: Final[Mapping[str, object]] = create_request.model_dump(mode="json", exclude_none=True)
|
||||
verbose_logger.debug("GeminiAgentsConfig create body: %s", body)
|
||||
return body
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,25 @@ from litellm.secret_managers.main import get_secret_str
|
|||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
# 2.5 Flash has no configurable imageSize and supports ten aspect ratios, excluding these four
|
||||
# https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/2-5-flash-image
|
||||
_GEMINI_2_5_FLASH_MODEL_FAMILY: Final = "2.5-flash"
|
||||
_GEMINI_2_5_FLASH_UNSUPPORTED_ASPECT_RATIOS: Final = frozenset(("1:4", "4:1", "1:8", "8:1"))
|
||||
|
||||
# 3.1 Flash supports the 512 tier, while other Gemini image models start at 1K
|
||||
# https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/3-1-flash-image
|
||||
_GEMINI_512_IMAGE_SIZE_MODEL: Final = "gemini-3.1-flash-image"
|
||||
|
||||
# Flash Lite supports only 1K, even when the requested dimensions are larger
|
||||
# https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/3-1-flash-lite-image
|
||||
_GEMINI_1K_ONLY_IMAGE_SIZE_MODEL: Final = "gemini-3.1-flash-lite-image"
|
||||
|
||||
# LiteLLM chooses the nearest resolution tier using the side length of a square with the same total number of pixels
|
||||
# as the requested image
|
||||
_GEMINI_IMAGE_SIZE_1K_THRESHOLD: Final = (512 + 1024) // 2
|
||||
_GEMINI_IMAGE_SIZE_2K_THRESHOLD: Final = (1024 + 2048) // 2
|
||||
_GEMINI_IMAGE_SIZE_4K_THRESHOLD: Final = (2048 + 4096) // 2
|
||||
|
||||
GEMINI_IMAGE_ASPECT_RATIOS: Final[dict[str, float]] = {
|
||||
"1:1": 1 / 1,
|
||||
"1:4": 1 / 4,
|
||||
|
|
@ -105,8 +124,8 @@ def map_openai_size_to_gemini_image_config(size: str, model: str) -> dict[str, s
|
|||
return None
|
||||
|
||||
width, height = dimensions
|
||||
image_config: Final = {"aspectRatio": _map_dimensions_to_gemini_aspect_ratio(width, height)}
|
||||
image_size: Final = _map_dimensions_to_gemini_image_size(width, height)
|
||||
image_config: Final = {"aspectRatio": _map_dimensions_to_gemini_aspect_ratio(width, height, model)}
|
||||
image_size: Final = _map_dimensions_to_gemini_image_size(width, height, model)
|
||||
if is_gemini_image_model(model):
|
||||
if supports_gemini_image_size(model):
|
||||
image_config["imageSize"] = image_size
|
||||
|
|
@ -123,7 +142,7 @@ def supports_gemini_image_size(model: str) -> bool:
|
|||
return bool(value)
|
||||
except Exception:
|
||||
pass
|
||||
return "2.5-flash" not in model
|
||||
return _GEMINI_2_5_FLASH_MODEL_FAMILY not in model
|
||||
|
||||
|
||||
def is_gemini_image_model(model: str) -> bool:
|
||||
|
|
@ -333,24 +352,36 @@ def _parse_openai_image_size(size: str) -> tuple[int, int] | None:
|
|||
return width, height
|
||||
|
||||
|
||||
def _map_dimensions_to_gemini_aspect_ratio(width: int, height: int) -> str:
|
||||
if (width, height) in GEMINI_IMAGE_SIZE_TO_ASPECT_RATIO:
|
||||
return GEMINI_IMAGE_SIZE_TO_ASPECT_RATIO[(width, height)]
|
||||
def _map_dimensions_to_gemini_aspect_ratio(width: int, height: int, model: str) -> str:
|
||||
aspect_ratios: Final = tuple(
|
||||
ratio
|
||||
for ratio in GEMINI_IMAGE_ASPECT_RATIOS
|
||||
if _GEMINI_2_5_FLASH_MODEL_FAMILY not in model or ratio not in _GEMINI_2_5_FLASH_UNSUPPORTED_ASPECT_RATIOS
|
||||
)
|
||||
known_ratio: Final = GEMINI_IMAGE_SIZE_TO_ASPECT_RATIO.get((width, height))
|
||||
if known_ratio is not None and known_ratio in aspect_ratios:
|
||||
return known_ratio
|
||||
|
||||
requested_ratio: Final = width / height
|
||||
return min(
|
||||
GEMINI_IMAGE_ASPECT_RATIOS,
|
||||
aspect_ratios,
|
||||
key=lambda aspect_ratio: abs(math.log(GEMINI_IMAGE_ASPECT_RATIOS[aspect_ratio] / requested_ratio)),
|
||||
)
|
||||
|
||||
|
||||
def _map_dimensions_to_gemini_image_size(width: int, height: int) -> str:
|
||||
effective_square_side: Final = math.sqrt(width * height)
|
||||
if effective_square_side < 768:
|
||||
return "512"
|
||||
if effective_square_side < 1536:
|
||||
def _map_dimensions_to_gemini_image_size(width: int, height: int, model: str) -> str:
|
||||
if _GEMINI_1K_ONLY_IMAGE_SIZE_MODEL in model:
|
||||
return "1K"
|
||||
if effective_square_side < 3072:
|
||||
|
||||
effective_square_side: Final = math.sqrt(width * height)
|
||||
if effective_square_side < _GEMINI_IMAGE_SIZE_1K_THRESHOLD:
|
||||
if is_gemini_image_model(model) and _GEMINI_512_IMAGE_SIZE_MODEL not in model:
|
||||
return "1K"
|
||||
else:
|
||||
return "512"
|
||||
if effective_square_side < _GEMINI_IMAGE_SIZE_2K_THRESHOLD:
|
||||
return "1K"
|
||||
if effective_square_side < _GEMINI_IMAGE_SIZE_4K_THRESHOLD:
|
||||
return "2K"
|
||||
return "4K"
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import httpx
|
|||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import process_response_headers
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, process_response_headers
|
||||
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
|
||||
from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig
|
||||
from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo
|
||||
|
|
@ -41,6 +41,14 @@ else:
|
|||
_JsonObject: TypeAlias = dict[str, object]
|
||||
|
||||
|
||||
def _has_agent_config_type(agent_config: object) -> bool:
|
||||
match agent_config:
|
||||
case {"type": str()}:
|
||||
return True
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
class _InteractionPayload(TypedDict, total=False):
|
||||
"""JSON body of an Interactions API interaction, keyed as ``InteractionsAPIResponse`` fields."""
|
||||
|
||||
|
|
@ -105,6 +113,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
|
|||
return [
|
||||
"model",
|
||||
"agent",
|
||||
"agent_config",
|
||||
"input",
|
||||
"tools",
|
||||
"system_instruction",
|
||||
|
|
@ -188,6 +197,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
|
|||
request_body["input"] = input
|
||||
|
||||
optional_keys: Final = [
|
||||
"agent_config",
|
||||
"tools",
|
||||
"system_instruction",
|
||||
"stream",
|
||||
|
|
@ -197,8 +207,14 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
|
|||
"response_modalities",
|
||||
"previous_interaction_id",
|
||||
]
|
||||
dropped_keys: Final = (
|
||||
frozenset({"agent_config"})
|
||||
if drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
and (model or not _has_agent_config_type(optional_params.get("agent_config")))
|
||||
else frozenset()
|
||||
)
|
||||
for key in optional_keys:
|
||||
if optional_params.get(key) is not None:
|
||||
if key not in dropped_keys and optional_params.get(key) is not None:
|
||||
request_body[key] = optional_params[key]
|
||||
|
||||
# response_mime_type is removed — fold it into response_format.
|
||||
|
|
|
|||
|
|
@ -6,16 +6,17 @@ Why separate file? Make it easy to see how transformation works
|
|||
Docs - https://docs.mistral.ai/api/
|
||||
"""
|
||||
|
||||
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping
|
||||
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
|
||||
from itertools import accumulate, count
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload
|
||||
|
||||
import httpx
|
||||
from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
handle_messages_with_content_list_to_str_conversion,
|
||||
convert_content_list_to_str,
|
||||
strip_litellm_internal_message_fields,
|
||||
strip_none_values_from_message,
|
||||
)
|
||||
|
|
@ -29,8 +30,8 @@ from litellm.router_utils.reasoning_effort_capability import (
|
|||
nearest_declared_reasoning_effort,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.mistral import MistralThinkingBlock, MistralToolCallMessage
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.llms.mistral import MistralTextBlock, MistralThinkingBlock, MistralToolCallMessage
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionTextObject
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream
|
||||
from litellm.utils import convert_to_model_response_object, supports_reasoning
|
||||
|
||||
|
|
@ -58,6 +59,10 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider:
|
|||
_NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset())
|
||||
|
||||
|
||||
def _is_json_array(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a decoded JSON array is a list
|
||||
return isinstance(value, list)
|
||||
|
||||
|
||||
class MistralConfig(OpenAIGPTConfig):
|
||||
"""
|
||||
Reference: https://docs.mistral.ai/api/
|
||||
|
|
@ -118,11 +123,14 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
def custom_llm_provider(self) -> str:
|
||||
return "mistral"
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
is_magistral: Final = "magistral" in model.lower()
|
||||
accepts_reasoning_effort: Final = is_magistral or supports_reasoning(
|
||||
def _accepts_reasoning(self, model: str) -> bool:
|
||||
return "magistral" in model.lower() or supports_reasoning(
|
||||
model=model, custom_llm_provider=self.custom_llm_provider
|
||||
)
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
is_magistral: Final = "magistral" in model.lower()
|
||||
accepts_reasoning_effort: Final = self._accepts_reasoning(model)
|
||||
return [
|
||||
"stream",
|
||||
"temperature",
|
||||
|
|
@ -260,25 +268,32 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
The above statement is not valid now. Need to plan to remove all the #1,2,3
|
||||
Mistral API supports content as a list.
|
||||
"""
|
||||
messages = [self._strip_output_only_fields(m) for m in messages]
|
||||
keeps_reasoning: Final = self._accepts_reasoning(model)
|
||||
final_position: Final = len(messages) - 1
|
||||
replayed_messages: Final = [
|
||||
MistralConfig._with_replayed_reasoning(m, keeps_reasoning, position == final_position)
|
||||
for position, m in enumerate(messages)
|
||||
]
|
||||
|
||||
## 1. If 'image_url' or 'file' in content, then transform with base class and mistral-specific handling
|
||||
for m in messages:
|
||||
for m in replayed_messages:
|
||||
_content_block = m.get("content")
|
||||
if _content_block and isinstance(_content_block, list):
|
||||
if any(c.get("type") in ["image_url", "file"] for c in _content_block):
|
||||
if is_async:
|
||||
return self._transform_messages_async(messages, model)
|
||||
return self._transform_messages_async(replayed_messages, model)
|
||||
else:
|
||||
messages = self._transform_messages_sync(messages, model)
|
||||
return messages
|
||||
return self._transform_messages_sync(replayed_messages, model)
|
||||
|
||||
## 2. If content is list, then convert to string
|
||||
messages = handle_messages_with_content_list_to_str_conversion(messages)
|
||||
flattened_messages: Final = tuple(
|
||||
MistralConfig._with_flattened_content(m, keeps_reasoning) for m in replayed_messages
|
||||
)
|
||||
|
||||
## 3. Handle name in message
|
||||
prepared_messages: Final = tuple(
|
||||
MistralConfig._handle_tool_call_message(MistralConfig._handle_name_in_message(m)) for m in messages
|
||||
MistralConfig._handle_tool_call_message(MistralConfig._handle_name_in_message(m))
|
||||
for m in flattened_messages
|
||||
)
|
||||
empty_flags: Final = tuple(map(MistralConfig._is_empty_assistant_message, prepared_messages))
|
||||
tool_call_turns: Final = accumulate(
|
||||
|
|
@ -442,17 +457,163 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
|
||||
return cleaned_tools
|
||||
|
||||
@classmethod
|
||||
def _strip_output_only_fields(cls, message: AllMessageValues) -> AllMessageValues:
|
||||
"""
|
||||
LiteLLM attaches output-only fields (``reasoning_content``, ``thinking_blocks``,
|
||||
``provider_specific_fields``) to assistant responses. Mistral's input schema forbids
|
||||
unknown fields, so replaying them verbatim in a follow-up turn triggers a
|
||||
422 ``extra_forbidden``. Drop them before the request is sent.
|
||||
"""
|
||||
@staticmethod
|
||||
def _is_mistral_think_chunk(part: object) -> bool:
|
||||
match part:
|
||||
case {"type": "thinking", "thinking": list()}:
|
||||
return True
|
||||
case _:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _text_item_text(item: object) -> str | None:
|
||||
match item:
|
||||
case {"type": "text", "text": str(text)}:
|
||||
return text
|
||||
case _:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _sendable_think_chunk(
|
||||
chunk: object, items: Sequence[object], may_stay_open: bool
|
||||
) -> MistralThinkingBlock | None:
|
||||
text_items: Final = [
|
||||
MistralTextBlock(type="text", text=text)
|
||||
for text in map(MistralConfig._text_item_text, items)
|
||||
if text is not None
|
||||
]
|
||||
if not text_items:
|
||||
return None
|
||||
match chunk:
|
||||
case {"closed": False} if not may_stay_open:
|
||||
return MistralThinkingBlock(type="thinking", thinking=text_items)
|
||||
case {"closed": bool(closed)}:
|
||||
return MistralThinkingBlock(type="thinking", thinking=text_items, closed=closed)
|
||||
case _:
|
||||
return MistralThinkingBlock(type="thinking", thinking=text_items)
|
||||
|
||||
@staticmethod
|
||||
def _sendable_part(part: object, may_stay_open: bool) -> object | None:
|
||||
match part:
|
||||
case {"type": "thinking", "thinking": object() as items} if _is_json_array(items):
|
||||
return MistralConfig._sendable_think_chunk(part, items, may_stay_open)
|
||||
case _:
|
||||
return part
|
||||
|
||||
@staticmethod
|
||||
def _with_sendable_think_chunks(
|
||||
message: AllMessageValues, content_parts: tuple[object, ...], may_end_open: bool
|
||||
) -> AllMessageValues:
|
||||
last_position: Final = len(content_parts) - 1
|
||||
sendable_parts: Final = [
|
||||
sendable
|
||||
for position, part in enumerate(content_parts)
|
||||
if (sendable := MistralConfig._sendable_part(part, may_end_open and position == last_position)) is not None
|
||||
]
|
||||
return cast( # cast-ok: Mistral's ThinkChunk is outside the OpenAI assistant content union
|
||||
AllMessageValues, {**message, "content": sendable_parts or None}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _thinking_block_text(block: object) -> str:
|
||||
match block:
|
||||
case {"type": "thinking", "thinking": str(thinking)}:
|
||||
return thinking
|
||||
case _:
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _replayed_reasoning_text(message: ChatCompletionAssistantMessage) -> str | None:
|
||||
reasoning_content: Final = message.get("reasoning_content")
|
||||
if isinstance(reasoning_content, str) and reasoning_content.strip():
|
||||
return reasoning_content
|
||||
thinking_blocks: Final = message.get("thinking_blocks")
|
||||
thinking_text: Final = (
|
||||
"".join(map(MistralConfig._thinking_block_text, thinking_blocks))
|
||||
if isinstance(thinking_blocks, list)
|
||||
else ""
|
||||
)
|
||||
return thinking_text if thinking_text.strip() else None
|
||||
|
||||
@staticmethod
|
||||
def _assistant_content_parts(message: ChatCompletionAssistantMessage) -> tuple[object, ...] | None:
|
||||
match message.get("content"):
|
||||
case None | "":
|
||||
return ()
|
||||
case str(text):
|
||||
return (ChatCompletionTextObject(type="text", text=text),)
|
||||
case list(parts) | tuple(parts):
|
||||
return tuple(parts)
|
||||
case _:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _is_prefix_turn(message: object) -> bool:
|
||||
match message:
|
||||
case {"prefix": True}:
|
||||
return True
|
||||
case _:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _with_replayed_reasoning(
|
||||
message: AllMessageValues, keeps_reasoning: bool, is_final_turn: bool
|
||||
) -> AllMessageValues:
|
||||
if message["role"] != "assistant":
|
||||
return message
|
||||
return strip_litellm_internal_message_fields(message)
|
||||
stripped_message: Final = strip_litellm_internal_message_fields(message)
|
||||
content_parts: Final = MistralConfig._assistant_content_parts(message)
|
||||
if not keeps_reasoning:
|
||||
return MistralConfig._without_think_chunks(stripped_message, content_parts)
|
||||
if content_parts is not None and any(map(MistralConfig._is_mistral_think_chunk, content_parts)):
|
||||
return MistralConfig._with_sendable_think_chunks(
|
||||
stripped_message, content_parts, is_final_turn and MistralConfig._is_prefix_turn(message)
|
||||
)
|
||||
reasoning_text: Final = MistralConfig._replayed_reasoning_text(message)
|
||||
has_turn_output: Final = bool(content_parts) or bool(message.get("tool_calls"))
|
||||
if reasoning_text is None or content_parts is None or not has_turn_output:
|
||||
return stripped_message
|
||||
think_chunk: Final = MistralThinkingBlock(
|
||||
type="thinking", thinking=[MistralTextBlock(type="text", text=reasoning_text)], closed=True
|
||||
)
|
||||
return cast( # cast-ok: Mistral's ThinkChunk is outside the OpenAI assistant content union
|
||||
AllMessageValues, {**stripped_message, "content": [think_chunk, *content_parts]}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _without_think_chunks(message: AllMessageValues, content_parts: tuple[object, ...] | None) -> AllMessageValues:
|
||||
if content_parts is None or not any(map(MistralConfig._is_mistral_think_chunk, content_parts)):
|
||||
return message
|
||||
kept_parts: Final = [part for part in content_parts if not MistralConfig._is_mistral_think_chunk(part)]
|
||||
return cast( # cast-ok: an assistant turn takes a content list or no content
|
||||
AllMessageValues, {**message, "content": kept_parts or None}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _with_flattened_content(message: AllMessageValues, keeps_reasoning: bool) -> AllMessageValues:
|
||||
flattened_text: Final = convert_content_list_to_str(message)
|
||||
match message:
|
||||
case {"role": "assistant", "content": list(content)} if keeps_reasoning and any(
|
||||
map(MistralConfig._is_mistral_think_chunk, content)
|
||||
):
|
||||
text_parts: Final = (
|
||||
(ChatCompletionTextObject(type="text", text=flattened_text),) if flattened_text else ()
|
||||
)
|
||||
think_chunks: Final = tuple(filter(MistralConfig._is_mistral_think_chunk, content))
|
||||
sent_chunks: Final = tuple(
|
||||
sendable
|
||||
for chunk in think_chunks
|
||||
if (sendable := MistralConfig._sendable_part(chunk, not text_parts)) is not None
|
||||
)
|
||||
return cast( # cast-ok: Mistral's ThinkChunk is outside the OpenAI assistant content union
|
||||
AllMessageValues, {**message, "content": [*sent_chunks, *text_parts]}
|
||||
)
|
||||
case _ if flattened_text:
|
||||
return cast( # cast-ok: every message role takes str content
|
||||
AllMessageValues, {**message, "content": flattened_text}
|
||||
)
|
||||
case _:
|
||||
return message
|
||||
|
||||
@classmethod
|
||||
def _handle_name_in_message(cls, message: AllMessageValues) -> AllMessageValues:
|
||||
|
|
|
|||
17
litellm/llms/ollama/decisions/transformation.py
Normal file
17
litellm/llms/ollama/decisions/transformation.py
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.constants import PLACEHOLDER_API_KEY
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
|
||||
|
||||
class OllamaDecisionsConfig(BaseDecisionsConfig):
|
||||
api_key_env = ("OLLAMA_API_KEY",)
|
||||
api_base_env = ("OLLAMA_API_BASE",)
|
||||
api_key_required = False
|
||||
|
||||
def get_default_api_base(self) -> str | None:
|
||||
return "http://localhost:11434"
|
||||
|
||||
def resolve_api_key(self, api_key: str | None) -> str | None:
|
||||
resolved: Final = super().resolve_api_key(api_key)
|
||||
return None if resolved == PLACEHOLDER_API_KEY else resolved
|
||||
|
|
@ -76,6 +76,7 @@ else:
|
|||
|
||||
_NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_OBJECT_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
_CHOICE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
|
|
@ -958,8 +959,13 @@ class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator):
|
|||
status_code=error_status_code,
|
||||
message=error_message,
|
||||
)
|
||||
choices = chunk.get("choices", [])
|
||||
choices = self._map_reasoning_to_reasoning_content(choices)
|
||||
raw_choices: Final = chunk.get("choices") or []
|
||||
if any(choice.get("finish_reason") == "abort" for choice in _CHOICE_MAPPINGS.validate_python(raw_choices)):
|
||||
raise OpenAIError(
|
||||
status_code=500,
|
||||
message="Upstream generation aborted (finish_reason=abort)",
|
||||
)
|
||||
choices: Final = self._map_reasoning_to_reasoning_content(raw_choices)
|
||||
fingerprint: Final = _OBJECT_MAPPING.validate_python(chunk).get("system_fingerprint")
|
||||
|
||||
kwargs: Final[dict[str, Any]] = {
|
||||
|
|
|
|||
|
|
@ -499,8 +499,13 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
)
|
||||
if sanitized_tools is not None:
|
||||
response_api_optional_request_params["tools"] = sanitized_tools
|
||||
final_request_params: Final = dict(
|
||||
ResponsesAPIRequestParams(model=model, input=replay_safe_input, **response_api_optional_request_params)
|
||||
final_request_params: Final = cast( # cast-ok: typed dict spread is inferred as Unknown by basedpyright
|
||||
dict[str, object],
|
||||
{
|
||||
"model": model,
|
||||
"input": replay_safe_input,
|
||||
**response_api_optional_request_params,
|
||||
},
|
||||
)
|
||||
|
||||
return final_request_params
|
||||
|
|
@ -1209,8 +1214,13 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
)
|
||||
if sanitized_tools is not None:
|
||||
response_api_optional_request_params["tools"] = sanitized_tools
|
||||
data: Final = dict(
|
||||
ResponsesAPIRequestParams(model=model, input=replay_safe_input, **response_api_optional_request_params)
|
||||
data: Final = cast( # cast-ok: typed dict spread is inferred as Unknown by basedpyright
|
||||
dict[str, object],
|
||||
{
|
||||
"model": model,
|
||||
"input": replay_safe_input,
|
||||
**response_api_optional_request_params,
|
||||
},
|
||||
)
|
||||
|
||||
return url, data
|
||||
|
|
|
|||
|
|
@ -58,6 +58,12 @@
|
|||
"max_completion_tokens": "max_tokens"
|
||||
}
|
||||
},
|
||||
"zerogpu": {
|
||||
"base_url": "https://api.zerogpu.ai/v1",
|
||||
"api_key_env": "ZEROGPU_API_KEY",
|
||||
"api_base_env": "ZEROGPU_API_BASE",
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
|
||||
},
|
||||
"chutes": {
|
||||
"base_url": "https://llm.chutes.ai/v1/",
|
||||
"api_key_env": "CHUTES_API_KEY",
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import math
|
||||
import re
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from copy import deepcopy
|
||||
|
|
@ -18,6 +19,11 @@ from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
|||
from litellm.litellm_core_utils.prompt_templates.common_utils import unpack_defs
|
||||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.gemini.common_utils import (
|
||||
GEMINI_IMAGE_ASPECT_RATIOS,
|
||||
GEMINI_IMAGE_SIZE_TO_ASPECT_RATIO,
|
||||
map_openai_size_to_gemini_image_config,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.llms.vertex_ai import (
|
||||
VERTEX_AI_PROVIDER_METADATA_FIELDS,
|
||||
|
|
@ -35,6 +41,31 @@ GEMINI_VIDEO_METADATA_KEYS: Final = MappingProxyType(
|
|||
GEMINI_FILES_API_URI_PREFIX: Final = "https://generativelanguage.googleapis.com/v1beta/files/"
|
||||
|
||||
|
||||
# https://cloud.google.com/vertex-ai/generative-ai/docs/image/configure-aspect-ratio
|
||||
_IMAGEN_ASPECT_RATIOS: Final = MappingProxyType(
|
||||
{"1:1": 1.0, "3:4": 3 / 4, "4:3": 4 / 3, "9:16": 9 / 16, "16:9": 16 / 9}
|
||||
)
|
||||
|
||||
|
||||
def map_vertex_image_size(size: object, model: str, *, imagen: bool = False) -> Mapping[str, str] | None:
|
||||
if not isinstance(size, str) or (image_config := map_openai_size_to_gemini_image_config(size, model)) is None:
|
||||
return None
|
||||
|
||||
width, height = map(int, size.lower().split("x"))
|
||||
requested_ratio: Final = width / height
|
||||
aspect_ratio: Final = (
|
||||
min(_IMAGEN_ASPECT_RATIOS, key=lambda ratio: abs(math.log(_IMAGEN_ASPECT_RATIOS[ratio] / requested_ratio)))
|
||||
if imagen
|
||||
else image_config["aspectRatio"]
|
||||
)
|
||||
if (
|
||||
not math.isclose(GEMINI_IMAGE_ASPECT_RATIOS[aspect_ratio], requested_ratio)
|
||||
and GEMINI_IMAGE_SIZE_TO_ASPECT_RATIO.get((width, height)) != aspect_ratio
|
||||
):
|
||||
verbose_logger.warning("Vertex AI size %s is approximated with aspectRatio %s", size, aspect_ratio)
|
||||
return MappingProxyType({"aspectRatio": aspect_ratio} if imagen else image_config)
|
||||
|
||||
|
||||
def gemini_video_metadata_from_openai(video_metadata: Mapping[str, object]) -> dict[str, object]:
|
||||
return {
|
||||
gemini_key: video_metadata[openai_key]
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from httpx._types import RequestFiles
|
|||
import litellm
|
||||
from litellm.images.utils import ImageEditRequestUtils
|
||||
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
|
||||
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
|
||||
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url, map_vertex_image_size
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.images.main import ImageEditOptionalRequestParams
|
||||
|
|
@ -67,7 +67,7 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM):
|
|||
return {}
|
||||
|
||||
size: Final = image_edit_optional_params.get("size")
|
||||
return {"aspectRatio": self._map_size_to_aspect_ratio(size or "")}
|
||||
return dict(map_vertex_image_size(size, model) or {"aspectRatio": "1:1"})
|
||||
|
||||
def _resolve_vertex_project(self) -> str | None:
|
||||
return (
|
||||
|
|
@ -176,11 +176,11 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM):
|
|||
contents: Final[dict[str, object]] = {"role": "USER", "parts": parts}
|
||||
|
||||
# Add image-specific configuration
|
||||
image_config: Final = (
|
||||
{"aspect_ratio": image_edit_optional_request_params["aspectRatio"]}
|
||||
if "aspectRatio" in image_edit_optional_request_params
|
||||
else None
|
||||
)
|
||||
image_config: Final = {
|
||||
field: image_edit_optional_request_params[param]
|
||||
for param, field in (("aspectRatio", "aspect_ratio"), ("imageSize", "image_size"))
|
||||
if param in image_edit_optional_request_params
|
||||
}
|
||||
|
||||
generation_config: Final[dict[str, object]] = {
|
||||
key: value for key, value in (("response_modalities", ["IMAGE"]), ("image_config", image_config)) if value
|
||||
|
|
@ -222,17 +222,6 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM):
|
|||
model_response.data = cast(list[OpenAIImage], data_list)
|
||||
return model_response
|
||||
|
||||
def _map_size_to_aspect_ratio(self, size: str) -> str:
|
||||
"""Map OpenAI size format to Gemini aspect ratio format"""
|
||||
aspect_ratio_map: Final = {
|
||||
"1024x1024": "1:1",
|
||||
"1792x1024": "16:9",
|
||||
"1024x1792": "9:16",
|
||||
"1280x896": "4:3",
|
||||
"896x1280": "3:4",
|
||||
}
|
||||
return aspect_ratio_map.get(size, "1:1")
|
||||
|
||||
def _prepare_inline_image_parts(self, image: FileTypes | list[FileTypes]) -> list[HttpxPartType]:
|
||||
images: Final[list[FileTypes]] = image if isinstance(image, list) else [image]
|
||||
return [
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import litellm
|
|||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
|
||||
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
|
||||
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url, map_vertex_image_size
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.images.main import ImageEditOptionalRequestParams
|
||||
|
|
@ -70,13 +70,11 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM):
|
|||
if "n" in filtered_params:
|
||||
mapped_params["sampleCount"] = filtered_params["n"]
|
||||
|
||||
if "size" in filtered_params:
|
||||
mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio(filtered_params["size"])
|
||||
|
||||
if "mask" in filtered_params:
|
||||
mapped_params["mask"] = filtered_params["mask"]
|
||||
|
||||
return mapped_params
|
||||
size_params: Final = map_vertex_image_size(filtered_params.get("size"), model, imagen=True) or {}
|
||||
return {**mapped_params, **size_params}
|
||||
|
||||
def _resolve_vertex_project(self) -> str | None:
|
||||
return (
|
||||
|
|
@ -189,6 +187,11 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM):
|
|||
"sampleCount": sample_count,
|
||||
"editMode": edit_mode,
|
||||
"editConfig": {"baseSteps": base_steps},
|
||||
**(
|
||||
{"aspectRatio": image_edit_optional_request_params["aspectRatio"]}
|
||||
if "aspectRatio" in image_edit_optional_request_params
|
||||
else {}
|
||||
),
|
||||
}
|
||||
|
||||
# Set default values for Vertex AI-specific parameters (not configurable by users via OpenAI API)
|
||||
|
|
@ -239,17 +242,6 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM):
|
|||
model_response.data = cast(list[OpenAIImage], data_list)
|
||||
return model_response
|
||||
|
||||
def _map_size_to_aspect_ratio(self, size: str) -> str:
|
||||
"""Map OpenAI size format to Imagen aspect ratio format"""
|
||||
aspect_ratio_map: Final = {
|
||||
"1024x1024": "1:1",
|
||||
"1792x1024": "16:9",
|
||||
"1024x1792": "9:16",
|
||||
"1280x896": "4:3",
|
||||
"896x1280": "3:4",
|
||||
}
|
||||
return aspect_ratio_map.get(size, "1:1")
|
||||
|
||||
def _prepare_reference_images(
|
||||
self,
|
||||
image: FileTypes | list[FileTypes],
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from litellm.llms.gemini.common_utils import (
|
|||
get_gemini_image_web_search_requests,
|
||||
map_gemini_image_tools_params,
|
||||
)
|
||||
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
|
||||
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url, map_vertex_image_size
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
|
@ -89,8 +89,7 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM):
|
|||
if k == "n":
|
||||
mapped_params["candidate_count"] = v
|
||||
elif k == "size":
|
||||
# Map OpenAI size format to Gemini aspectRatio
|
||||
mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio(v)
|
||||
continue
|
||||
elif k in ("aspectRatio", "aspect_ratio"):
|
||||
mapped_params["aspectRatio"] = v
|
||||
elif k in ("imageSize", "image_size"):
|
||||
|
|
@ -103,22 +102,23 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM):
|
|||
elif k not in ("tools", "web_search_options", "imageConfig"):
|
||||
mapped_params[k] = v
|
||||
|
||||
if "size" in non_default_params and "size" not in optional_params:
|
||||
image_params: Final = _JSON_OBJECT.validate_python(mapped_params)
|
||||
image_config: Final = _JSON_OBJECT.validate_python(image_params.get("imageConfig", {}))
|
||||
size_params: Final = {
|
||||
key: value
|
||||
for key, value in (
|
||||
map_vertex_image_size(non_default_params["size"], model) or {"aspectRatio": "1:1"}
|
||||
).items()
|
||||
if key not in image_config
|
||||
}
|
||||
return map_gemini_image_tools_params(
|
||||
_JSON_OBJECT.validate_python(non_default_params), {**size_params, **image_params}
|
||||
)
|
||||
|
||||
mapped_params = map_gemini_image_tools_params(non_default_params, mapped_params)
|
||||
return mapped_params
|
||||
|
||||
def _map_size_to_aspect_ratio(self, size: str) -> str:
|
||||
"""
|
||||
Map OpenAI size format to Gemini aspect ratio format
|
||||
"""
|
||||
aspect_ratio_map: Final = {
|
||||
"1024x1024": "1:1",
|
||||
"1792x1024": "16:9",
|
||||
"1024x1792": "9:16",
|
||||
"1280x896": "4:3",
|
||||
"896x1280": "3:4",
|
||||
}
|
||||
return aspect_ratio_map.get(size, "1:1")
|
||||
|
||||
def _resolve_vertex_project(self) -> str | None:
|
||||
return (
|
||||
getattr(self, "_vertex_project", None)
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from litellm.llms.base_llm.image_generation.transformation import (
|
|||
)
|
||||
from litellm.llms.vertex_ai.common_utils import (
|
||||
get_vertex_base_url,
|
||||
map_vertex_image_size,
|
||||
pop_vertex_request_labels,
|
||||
)
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
|
||||
|
|
@ -72,25 +73,14 @@ class VertexAIImagenImageGenerationConfig(BaseImageGenerationConfig, VertexLLM):
|
|||
mapped_params["sampleCount"] = v
|
||||
elif k == "size":
|
||||
# Map OpenAI size format to Imagen aspectRatio
|
||||
mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio(v)
|
||||
mapped_params["aspectRatio"] = (
|
||||
map_vertex_image_size(v, model, imagen=True) or {"aspectRatio": "1:1"}
|
||||
)["aspectRatio"]
|
||||
else:
|
||||
mapped_params[k] = v
|
||||
|
||||
return mapped_params
|
||||
|
||||
def _map_size_to_aspect_ratio(self, size: str) -> str:
|
||||
"""
|
||||
Map OpenAI size format to Imagen aspect ratio format
|
||||
"""
|
||||
aspect_ratio_map: Final = {
|
||||
"1024x1024": "1:1",
|
||||
"1792x1024": "16:9",
|
||||
"1024x1792": "9:16",
|
||||
"1280x896": "4:3",
|
||||
"896x1280": "3:4",
|
||||
}
|
||||
return aspect_ratio_map.get(size, "1:1")
|
||||
|
||||
def _resolve_vertex_project(self) -> str | None:
|
||||
return (
|
||||
getattr(self, "_vertex_project", None)
|
||||
|
|
|
|||
|
|
@ -6133,7 +6133,7 @@ def completion(
|
|||
}
|
||||
)
|
||||
)
|
||||
bedrock_bridges_to_responses: Final = bedrock_request_bridges_to_responses(
|
||||
bedrock_bridges_to_responses: Final = custom_llm_provider == "bedrock" and bedrock_request_bridges_to_responses(
|
||||
custom_llm_provider,
|
||||
model,
|
||||
request_params,
|
||||
|
|
@ -6148,6 +6148,7 @@ def completion(
|
|||
responses_api_model_info.get("mode") == "responses" or bedrock_bridges_to_responses
|
||||
) and not skip_responses_api_bridge:
|
||||
from litellm.completion_extras import responses_api_bridge
|
||||
from litellm.responses.utils import openai_bridge_reasoning
|
||||
|
||||
bridge_optional_params, bridge_litellm_params = (
|
||||
bedrock_responses_bridge_call_params(optional_params, litellm_params, api_base)
|
||||
|
|
@ -6157,6 +6158,21 @@ def completion(
|
|||
stripped_optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(
|
||||
bridge_optional_params
|
||||
)
|
||||
bridged_optional_params: Final = (
|
||||
{
|
||||
**stripped_optional_params,
|
||||
"reasoning_effort": openai_bridge_reasoning(
|
||||
requested=resolved_reasoning_effort,
|
||||
effort_kept="reasoning_effort" in stripped_optional_params,
|
||||
summary_alias=rs_val,
|
||||
model=model,
|
||||
drop_params=drop_unsupported_reasoning,
|
||||
additional_drop_params=routing_kwargs.get("additional_drop_params"),
|
||||
),
|
||||
}
|
||||
if custom_llm_provider == "openai" and isinstance(resolved_reasoning_effort, dict)
|
||||
else with_reasoning_summary_in_effort(stripped_optional_params, resolved_reasoning_effort, rs_val)
|
||||
)
|
||||
|
||||
return responses_api_bridge.completion( # pyright: ignore[reportReturnType] # bridge returns a coroutine on the acompletion path; awaited by the async caller
|
||||
model=model,
|
||||
|
|
@ -6167,9 +6183,7 @@ def completion(
|
|||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=with_reasoning_summary_in_effort(
|
||||
stripped_optional_params, resolved_reasoning_effort, rs_val
|
||||
),
|
||||
optional_params=bridged_optional_params,
|
||||
litellm_params=bridge_litellm_params,
|
||||
timeout=timeout,
|
||||
client=client, # pass AsyncOpenAI, OpenAI client
|
||||
|
|
@ -8410,7 +8424,7 @@ def transcription(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
response: TranscriptionResponse | Coroutine[object, object, TranscriptionResponse] | None = None
|
||||
response: TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]
|
||||
|
||||
provider_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
|
||||
model=model,
|
||||
|
|
@ -8572,10 +8586,16 @@ def transcription(
|
|||
provider_config=provider_config,
|
||||
shared_session=shared_session,
|
||||
)
|
||||
else:
|
||||
raise litellm.UnsupportedEndpointError(
|
||||
message=f"{custom_llm_provider}/{model} does not support audio transcription.",
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
# Store duration in _hidden_params for cost calculation without
|
||||
# exposing it in the response body (see sync path comment above).
|
||||
if response is not None and not isinstance(response, Coroutine):
|
||||
if not isinstance(response, Coroutine):
|
||||
existing_duration: Final = getattr(response, "duration", None)
|
||||
if existing_duration is None:
|
||||
calculated_duration: Final = calculate_request_duration(file)
|
||||
|
|
@ -8585,8 +8605,6 @@ def transcription(
|
|||
)
|
||||
response_hidden_params["audio_transcription_duration"] = calculated_duration
|
||||
|
||||
if response is None:
|
||||
raise ValueError("Unmapped provider passed in. Unable to get the response.")
|
||||
return response
|
||||
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1225,7 +1225,7 @@
|
|||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": true,
|
||||
"image_generations": false,
|
||||
"image_generations": true,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
|
|
@ -4437,6 +4437,32 @@
|
|||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"zerogpu": {
|
||||
"display_name": "ZeroGPU (`zerogpu`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/zerogpu",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"ragflow": {
|
||||
"display_name": "RAGFlow (`ragflow`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/ragflow",
|
||||
|
|
|
|||
|
|
@ -15015,7 +15015,7 @@
|
|||
]
|
||||
},
|
||||
"post": {
|
||||
"description": "Create a named custom agent on the Gemini side.\n\nExample:\n```bash\ncurl -X POST \"http://localhost:4000/v1beta/agents\" \\\n -H \"Authorization: Bearer sk-...\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"name\": \"my-custom-slides-agent\",\n \"base_agent\": \"waverunner\",\n \"instructions\": \"You are a helpful assistant that creates slides.\",\n \"base_environment\": {\n \"type\": \"remote\",\n \"sources\": [\n {\"type\": \"gcs\", \"source\": \"gs://eap-templates/slides-skill\",\n \"target\": \"/.agents/skills/slides-skill\"}\n ]\n }\n }'\n```",
|
||||
"description": "Create a named custom agent on the Gemini side.\n\nExample:\n```bash\ncurl -X POST \"http://localhost:4000/v1beta/agents\" \\\n -H \"Authorization: Bearer sk-...\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"id\": \"my-custom-slides-agent\",\n \"base_agent\": \"antigravity-preview-09-2026\",\n \"system_instruction\": \"You are a helpful assistant that creates slides.\",\n \"agent_config\": {\"type\": \"antigravity\", \"model\": \"gemini-3.8-flash\"},\n \"base_environment\": {\n \"type\": \"remote\",\n \"sources\": [\n {\"type\": \"gcs\", \"source\": \"gs://eap-templates/slides-skill\",\n \"target\": \"/.agents/skills/slides-skill\"}\n ]\n }\n }'\n```",
|
||||
"operationId": "create_gemini_agent_v1beta_agents_post",
|
||||
"responses": {
|
||||
"200": {
|
||||
|
|
|
|||
|
|
@ -3461,12 +3461,12 @@ class LiteLLM_SpendLogs_ResponseObject(LiteLLMPydanticObjectBase):
|
|||
class TokenCountRequest(LiteLLMPydanticObjectBase):
|
||||
model: str
|
||||
prompt: str | None = None
|
||||
messages: list[dict] | None = None
|
||||
messages: list[dict[Any, Any]] | None = None
|
||||
"""
|
||||
Anthropic token counting endpoint uses /messages
|
||||
"""
|
||||
|
||||
contents: list[dict] | None = None
|
||||
contents: list[dict[Any, Any]] | None = None
|
||||
"""
|
||||
Google /countTokens endpoint expects contents to be a list of dicts with the following structure:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ from litellm.anthropic_interface.exceptions import (
|
|||
AnthropicExceptionMapping,
|
||||
)
|
||||
from litellm.integrations.custom_guardrail import ModifyResponseException
|
||||
from litellm.litellm_core_utils.token_counter import UnsupportedContentTypeError
|
||||
from litellm.litellm_core_utils.tool_fields import first_validation_error
|
||||
from litellm.llms.anthropic.messages.context_management import (
|
||||
AnthropicContextManagementError,
|
||||
|
|
@ -49,6 +48,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
openai_error_type,
|
||||
with_litellm_call_id,
|
||||
)
|
||||
from litellm.proxy.common_utils.token_count_log import rejected_token_count
|
||||
from litellm.proxy.common_utils.token_count_settings import count_source_fields
|
||||
from litellm.proxy.litellm_pre_call_utils import (
|
||||
forwarded_anthropic_oauth_token,
|
||||
|
|
@ -394,12 +394,17 @@ async def count_tokens(
|
|||
# Extract required fields
|
||||
model_name: Final = data.get("model")
|
||||
messages: Final = data.get("messages", [])
|
||||
request_model: Final = model_name if isinstance(model_name, str) else None
|
||||
|
||||
if not model_name:
|
||||
raise HTTPException(status_code=400, detail="model parameter is required")
|
||||
raise rejected_token_count(
|
||||
HTTPException(status_code=400, detail="model parameter is required"), litellm_call_id, request_model
|
||||
)
|
||||
|
||||
if not messages:
|
||||
raise HTTPException(status_code=400, detail="messages parameter is required")
|
||||
raise rejected_token_count(
|
||||
HTTPException(status_code=400, detail="messages parameter is required"), litellm_call_id, request_model
|
||||
)
|
||||
|
||||
auto_drop_params: Final = should_auto_drop_params_for_agentic_cli(
|
||||
request.headers.get("user-agent", ""), "drop_params" in data, proxy_config
|
||||
|
|
@ -409,19 +414,18 @@ async def count_tokens(
|
|||
token_fields if isinstance(token_fields, ProxyException) else _token_count_request(token_fields)
|
||||
)
|
||||
if isinstance(token_request, ProxyException):
|
||||
return _anthropic_error_json_response(token_request, request)
|
||||
return rejected_token_count(
|
||||
_anthropic_error_json_response(token_request, request), litellm_call_id, request_model
|
||||
)
|
||||
|
||||
token_response: Final = await count_request_tokens(
|
||||
request=token_request,
|
||||
call_endpoint=True,
|
||||
messages_format="anthropic",
|
||||
litellm_call_id=litellm_call_id,
|
||||
client_oauth_token=forwarded_anthropic_oauth_token(request.headers),
|
||||
**_count_tokens_body_options(data, auto_drop_params),
|
||||
)
|
||||
except UnsupportedContentTypeError as e:
|
||||
return _anthropic_error_json_response(
|
||||
ProxyException(message=str(e), type="invalid_request_error", param="messages", code=400), request
|
||||
)
|
||||
except HTTPException as e:
|
||||
return _anthropic_error_json_response(proxy_exception_from_http_exception(e, {}), request)
|
||||
except ProxyException as e:
|
||||
|
|
|
|||
|
|
@ -342,7 +342,7 @@ The CLI provides these authentication commands:
|
|||
- **`lite login --pkce`** - Sign in through the system browser with OAuth authorization code + PKCE; the key renews itself with a refresh token
|
||||
- **`lite logout`** - Clear stored authentication token (and revoke a `--pkce` refresh token on the proxy)
|
||||
- **`lite whoami`** - Show current authentication status
|
||||
- **`lite auth print-token`** - Print the cached token (used as Claude Code's `apiKeyHelper`); renews a `--pkce` key first and fails once a classic token has expired
|
||||
- **`lite auth print-token`** - Print the cached token (used as Claude Code's `apiKeyHelper`); renews a `--pkce` key first, and fails once a classic token is past 24 hours and the proxy rejects it
|
||||
|
||||
### Authentication Flow Steps
|
||||
|
||||
|
|
@ -382,7 +382,7 @@ Keychain storage needs the `keyring` package, which ships with `pip install 'lit
|
|||
|
||||
`lite logout` clears both stores. If the keychain is locked at that moment it says so, and re-running it once the keychain is unlocked finishes the job.
|
||||
|
||||
The stored credential is a short-lived, per-session agent token, not a managed virtual key. It is scoped to the user and team you logged in as and inherits their models and budgets; spend is tracked against the shared team and user budgets rather than a separate per-session cap, so multiple logins or several concurrent agents all draw down the same allowance. It is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); re-run `lite login` to refresh it and pick up your latest team and user settings. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it while fresh and fails once it expires -- there is no silent renewal. It is accepted on a default deployment without `EXPERIMENTAL_UI_LOGIN`, does not appear in the Keys UI, and cannot be rotated or revoked mid-session. A credential from `lite login --pkce` is the exception: it carries a refresh token, so the CLI renews the key shortly before it expires and `lite logout` revokes the refresh token on the proxy (see [Browser sign-in with PKCE](https://docs.litellm.ai/docs/proxy/cli_sso#browser-sign-in-with-pkce)). Only the holder can end a `--pkce` session early, with `lite logout`; an admin has no button for it, but every renewal re-reads the user on the proxy, so deactivating the user or removing them from the team makes the next renewal fail and the key runs out within `LITELLM_CLI_JWT_EXPIRATION_HOURS`. On a proxy with more than one worker or replica, configure Redis (`litellm_settings.cache` with Redis `cache_params`, or `general_settings.coordination_redis`) so a refresh token stays single-use and `lite logout` holds on every worker; without Redis each worker keeps its own record. For a long-lived, rotatable, Keys-UI-visible credential, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY`.
|
||||
The stored credential is a short-lived, per-session agent token, not a managed virtual key. It is scoped to the user and team you logged in as and inherits their models and budgets; spend is tracked against the shared team and user budgets rather than a separate per-session cap, so multiple logins or several concurrent agents all draw down the same allowance. It is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); re-run `lite login` to refresh it and pick up your latest team and user settings. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it, and once it is past 24 hours (or the client's `LITELLM_CLI_JWT_EXPIRATION_HOURS`) asks the proxy first and fails if the proxy rejects it. There is no silent renewal. It is accepted on a default deployment without `EXPERIMENTAL_UI_LOGIN`, does not appear in the Keys UI, and cannot be rotated or revoked mid-session. A credential from `lite login --pkce` is the exception: it carries a refresh token, so the CLI renews the key shortly before it expires and `lite logout` revokes the refresh token on the proxy (see [Browser sign-in with PKCE](https://docs.litellm.ai/docs/proxy/cli_sso#browser-sign-in-with-pkce)). Only the holder can end a `--pkce` session early, with `lite logout`; an admin has no button for it, but every renewal re-reads the user on the proxy, so deactivating the user or removing them from the team makes the next renewal fail and the key runs out within `LITELLM_CLI_JWT_EXPIRATION_HOURS`. On a proxy with more than one worker or replica, configure Redis (`litellm_settings.cache` with Redis `cache_params`, or `general_settings.coordination_redis`) so a refresh token stays single-use and `lite logout` holds on every worker; without Redis each worker keeps its own record. For a long-lived, rotatable, Keys-UI-visible credential, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY`.
|
||||
|
||||
### Usage
|
||||
|
||||
|
|
|
|||
|
|
@ -506,7 +506,7 @@ To pin the model, pass the agent's own model flag (for example `lite claude --mo
|
|||
|
||||
The token minted by `lite login` is a short-lived, per-session agent credential, not a managed virtual key. It is scoped to the user and team you authenticated as, inherits that user's and team's models and budgets, and is enforced on the proxy exactly like a virtual key on the same team (guardrails, routing, logging, spend). Spend is tracked against the shared team and user budgets, so running several agents (or logging in more than once) does not hand each session its own separate budget; they all draw down the same team/user allowance. There is no separate per-session cap, so sustained agent use is not capped at a small chat-session limit.
|
||||
|
||||
The credential is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); run `lite login` again to refresh it, which also re-reads your latest team and user settings. It does not appear in the Keys UI and cannot be rotated or revoked mid-session. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it while it's still fresh and fails once it expires -- there is no silent renewal, so a long-running session needs a fresh `lite login` once a day. `lite claude`, `lite codex`, `lite opencode`, and `lite pi` work with it on a default deployment; `EXPERIMENTAL_UI_LOGIN` is not required. `lite login --pkce` is the exception to the daily re-login: it signs in through your system browser with OAuth authorization code and PKCE and stores a refresh token next to the key, so every `lite` command and `lite auth print-token` renew the key on their own shortly before it expires, `lite whoami` shows when the current key expires, and `lite logout` revokes the refresh token on the proxy (it needs a proxy that serves `/.well-known/litellm-cli-auth`; see [Browser sign-in with PKCE](https://docs.litellm.ai/docs/proxy/cli_sso#browser-sign-in-with-pkce)). When a renewal is refused, for example after a `lite logout` run from another copy of the credential, the command prints why on stderr and, once the key has run out, tells you to run `lite login --pkce` again. Only the holder can end a `--pkce` session early, with `lite logout`; an admin has no button for it, but every renewal re-reads the user on the proxy, so deactivating the user or removing them from the team makes the next renewal fail and the key runs out within `LITELLM_CLI_JWT_EXPIRATION_HOURS`. On a proxy with more than one worker or replica, configure Redis (`litellm_settings.cache` with Redis `cache_params`, or `general_settings.coordination_redis`) so a refresh token stays single-use and `lite logout` holds on every worker; without Redis each worker keeps its own record. If you need a long-lived, rotatable key that shows up in the Keys UI, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY` instead.
|
||||
The credential is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); run `lite login` again to refresh it, which also re-reads your latest team and user settings. It does not appear in the Keys UI and cannot be rotated or revoked mid-session. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it without asking the proxy for its first 24 hours, or `LITELLM_CLI_JWT_EXPIRATION_HOURS` if you set it on the client. After that, `print-token` and `lite whoami` ask the proxy whether it still accepts the token, and `print-token` fails once the proxy rejects it. There is no silent renewal, so a long-running session needs a fresh `lite login` each time the proxy's token lifetime runs out. If the proxy's tokens last less than 24 hours, set the same `LITELLM_CLI_JWT_EXPIRATION_HOURS` on the client so `print-token` stops printing the token once it runs out. `lite up` still starts a new login once the token is older than that same client window. It copies the token into Claude Code's settings as a static key that it never renews, so a fresh login gives Claude Code the proxy's full token lifetime instead of whatever an old token has left. `lite claude`, `lite codex`, `lite opencode`, and `lite pi` work with it on a default deployment; `EXPERIMENTAL_UI_LOGIN` is not required. `lite login --pkce` is the exception to the re-login: it signs in through your system browser with OAuth authorization code and PKCE and stores a refresh token next to the key, so every `lite` command and `lite auth print-token` renew the key on their own shortly before it expires, `lite whoami` shows when the current key expires, and `lite logout` revokes the refresh token on the proxy (it needs a proxy that serves `/.well-known/litellm-cli-auth`; see [Browser sign-in with PKCE](https://docs.litellm.ai/docs/proxy/cli_sso#browser-sign-in-with-pkce)). When a renewal is refused, for example after a `lite logout` run from another copy of the credential, the command prints why on stderr and, once the key has run out, tells you to run `lite login --pkce` again. Only the holder can end a `--pkce` session early, with `lite logout`; an admin has no button for it, but every renewal re-reads the user on the proxy, so deactivating the user or removing them from the team makes the next renewal fail and the key runs out within `LITELLM_CLI_JWT_EXPIRATION_HOURS`. On a proxy with more than one worker or replica, configure Redis (`litellm_settings.cache` with Redis `cache_params`, or `general_settings.coordination_redis`) so a refresh token stays single-use and `lite logout` holds on every worker; without Redis each worker keeps its own record. If you need a long-lived, rotatable key that shows up in the Keys UI, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY` instead.
|
||||
|
||||
### Route Every Claude Code Session Through the Proxy
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ from rich.console import Console
|
|||
from rich.table import Table
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict, assert_never
|
||||
|
||||
from litellm.constants import CLI_JWT_EXPIRATION_HOURS
|
||||
from litellm.litellm_core_utils.cli_keyring import (
|
||||
DISABLE_KEYRING_ENV_VAR,
|
||||
SYSTEM_KEYRING,
|
||||
|
|
@ -143,6 +142,9 @@ _TeamMapping: Final = TypeVar("_TeamMapping", bound=Mapping[str, object])
|
|||
|
||||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
|
||||
_PROXY_KEY_CHECK_TIMEOUT_SECONDS: Final = 3
|
||||
_PROXY_REJECTED_KEY_STATUSES: Final = frozenset({401, 403})
|
||||
|
||||
KEYRING_INSTALL_HINT: Final = "pip install 'litellm[cli]'"
|
||||
|
||||
KEYRING_ENABLE_HINT: Final = "keyring --enable (or unset PYTHON_KEYRING_BACKEND)"
|
||||
|
|
@ -1033,10 +1035,12 @@ def print_token(ctx: click.Context):
|
|||
|
||||
Designed to be used as Claude Code's `apiKeyHelper`
|
||||
(https://docs.claude.com/en/docs/claude-code/settings): stdout must
|
||||
contain only the token, so all diagnostics go to stderr. The token
|
||||
expires after `LITELLM_CLI_JWT_EXPIRATION_HOURS` (default 24h); a
|
||||
`lite login --pkce` token renews itself here first, and once a token
|
||||
has expired for good, run the same `lite login` command again.
|
||||
contain only the token, so all diagnostics go to stderr. A token younger
|
||||
than `LITELLM_CLI_JWT_EXPIRATION_HOURS` (default 24h) is printed without
|
||||
asking the proxy. An older one is checked against the proxy and refused
|
||||
only if the proxy rejects it. A `lite login --pkce`
|
||||
token renews itself here first, and once a token has expired for good,
|
||||
run the same `lite login` command again.
|
||||
"""
|
||||
vault: Final = context_secret_vault(ctx)
|
||||
token_data: Final = load_token(vault=vault)
|
||||
|
|
@ -1055,7 +1059,7 @@ def print_token(ctx: click.Context):
|
|||
sys.exit(1)
|
||||
|
||||
renews: Final = "refresh_token" in token_data
|
||||
if not is_cli_token_fresh(token_data) and not renews:
|
||||
if not renews and token_expired(token_data):
|
||||
click.echo("Token expired. Run 'lite login' again.", err=True)
|
||||
sys.exit(1)
|
||||
|
||||
|
|
@ -1111,8 +1115,32 @@ def whoami(ctx: click.Context):
|
|||
expires_at: Final = token_data.get("expires_at")
|
||||
if isinstance(expires_at, (int, float)):
|
||||
click.echo(_key_expiry_line(expires_at, renews="refresh_token" in token_data))
|
||||
elif age_hours > CLI_JWT_EXPIRATION_HOURS:
|
||||
click.echo(f"Warning: Token is more than {CLI_JWT_EXPIRATION_HOURS} hours old and may have expired.")
|
||||
elif token_expired(token_data):
|
||||
click.echo("Token expired. Run 'lite login' again.")
|
||||
|
||||
|
||||
def token_expired(token_data: Mapping[str, object]) -> bool:
|
||||
if is_cli_token_fresh(token_data):
|
||||
return False
|
||||
if isinstance(token_data.get("expires_at"), (int, float)):
|
||||
return True
|
||||
return _proxy_rejects_key(token_data)
|
||||
|
||||
|
||||
def _proxy_rejects_key(token_data: Mapping[str, object]) -> bool:
|
||||
key: Final = token_data.get("key")
|
||||
base_url: Final = token_data.get("base_url")
|
||||
if not isinstance(key, str) or not isinstance(base_url, str) or not base_url:
|
||||
return True
|
||||
try:
|
||||
response: Final = requests.get(
|
||||
f"{base_url}/v1/models",
|
||||
headers={"Authorization": f"Bearer {key}"},
|
||||
timeout=_PROXY_KEY_CHECK_TIMEOUT_SECONDS,
|
||||
)
|
||||
except requests.RequestException:
|
||||
return False
|
||||
return response.status_code in _PROXY_REJECTED_KEY_STATUSES
|
||||
|
||||
|
||||
def _key_expiry_line(expires_at: float, renews: bool) -> str:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import json
|
|||
import logging
|
||||
import math
|
||||
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from functools import lru_cache
|
||||
from itertools import chain
|
||||
|
|
@ -136,7 +137,7 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports
|
|||
_check_and_merge_model_level_guardrails, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
check_and_merge_model_level_guardrails,
|
||||
)
|
||||
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect
|
||||
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect, spawn_logging_task
|
||||
from litellm.router import Router
|
||||
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
|
||||
|
|
@ -2006,6 +2007,49 @@ def _content_text(fields: Mapping[str, object]) -> str:
|
|||
return str(fields.get("content", ""))
|
||||
|
||||
|
||||
async def _no_stream_failure_to_release() -> None:
|
||||
return None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _ClientStreamFailureReport:
|
||||
logging_obj: LiteLLMLoggingObj
|
||||
report: Callable[[Exception], Awaitable[None]]
|
||||
held: Exception | None = None
|
||||
|
||||
@classmethod
|
||||
def hold_while_streaming(cls, logging_obj: LiteLLMLoggingObj | None) -> Callable[[], Awaitable[None]]:
|
||||
if logging_obj is None or logging_obj.on_detached_stream_failure is None:
|
||||
return _no_stream_failure_to_release
|
||||
return cls(logging_obj, logging_obj.on_detached_stream_failure)._start_holding()
|
||||
|
||||
def _start_holding(self) -> Callable[[], Awaitable[None]]:
|
||||
self.logging_obj.on_detached_stream_failure = self.hold
|
||||
return self.release
|
||||
|
||||
async def hold(self, failure: Exception) -> None:
|
||||
if self.held is None:
|
||||
self.held = failure
|
||||
|
||||
async def release(self) -> None:
|
||||
self.logging_obj.on_detached_stream_failure = self.report
|
||||
if self.held is None:
|
||||
return
|
||||
await asyncio.shield(
|
||||
spawn_logging_task(
|
||||
asyncio.get_running_loop(), self._report_held(self.held), task_name="stream failure report"
|
||||
)
|
||||
)
|
||||
|
||||
async def _report_held(self, failure: Exception) -> None:
|
||||
try:
|
||||
await self.report(failure)
|
||||
except Exception as hook_failure: # noqa: BLE001 # a failing failure hook must not skip the stream cleanup
|
||||
verbose_proxy_logger.warning(
|
||||
"Stream failure hook raised: %s(%s)", type(hook_failure).__name__, sanitize_for_log(hook_failure)
|
||||
)
|
||||
|
||||
|
||||
class ProxyBaseLLMRequestProcessing:
|
||||
def __init__(self, data: dict, served_model: str | None = None):
|
||||
self.data = data
|
||||
|
|
@ -3498,6 +3542,8 @@ class ProxyBaseLLMRequestProcessing:
|
|||
request_data: Final = self.data
|
||||
|
||||
async def _on_detached_stream_failure(exc: Exception) -> None:
|
||||
if not logging_obj.claim_stream_failure_report():
|
||||
return
|
||||
try:
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
@ -4448,11 +4494,15 @@ class ProxyBaseLLMRequestProcessing:
|
|||
client_disconnected = False
|
||||
delivered_chunk = False
|
||||
recent_tail = SSE_STREAM_START_TAIL # rebind-ok: rolling window over the yielded bytes
|
||||
stream_logging_obj: Final[object] = request_data.get("litellm_logging_obj") # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # legacy request data is narrowed to Logging below
|
||||
guarded_stream: Final[AsyncGenerator[object, None]] = proxy_logging_obj.async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
response=response,
|
||||
request_data=request_data,
|
||||
)
|
||||
release_stream_failure: Final = _ClientStreamFailureReport.hold_while_streaming(
|
||||
stream_logging_obj if isinstance(stream_logging_obj, LiteLLMLoggingObj) else None
|
||||
)
|
||||
try:
|
||||
str_so_far = ""
|
||||
async for chunk in guarded_stream:
|
||||
|
|
@ -4523,10 +4573,18 @@ class ProxyBaseLLMRequestProcessing:
|
|||
verbose_proxy_logger.exception(
|
||||
"litellm.proxy.proxy_server.async_data_generator(): Exception occured - %s", e
|
||||
)
|
||||
transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=e,
|
||||
request_data=request_data,
|
||||
already_reported: Final = (
|
||||
isinstance(stream_logging_obj, LiteLLMLoggingObj)
|
||||
and not stream_logging_obj.claim_stream_failure_report()
|
||||
)
|
||||
transformed_exception: Final = (
|
||||
None
|
||||
if already_reported
|
||||
else await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=e,
|
||||
request_data=request_data,
|
||||
)
|
||||
)
|
||||
if transformed_exception is not None:
|
||||
e = transformed_exception
|
||||
|
|
@ -4549,15 +4607,18 @@ class ProxyBaseLLMRequestProcessing:
|
|||
seal: Final = "" if seal_open_frame is None else seal_open_frame(recent_tail)
|
||||
yield seal + error_frame if seal else error_frame
|
||||
finally:
|
||||
await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup(
|
||||
request=request,
|
||||
request_data=request_data,
|
||||
response=response,
|
||||
stream_completed=stream_completed,
|
||||
client_disconnected=client_disconnected,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
try:
|
||||
await release_stream_failure()
|
||||
finally:
|
||||
await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup(
|
||||
request=request,
|
||||
request_data=request_data,
|
||||
response=response,
|
||||
stream_completed=stream_completed,
|
||||
client_disconnected=client_disconnected,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def async_sse_data_generator(
|
||||
|
|
|
|||
133
litellm/proxy/common_utils/token_count_log.py
Normal file
133
litellm/proxy/common_utils/token_count_log.py
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
import json
|
||||
import re
|
||||
from asyncio import CancelledError
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeVar
|
||||
|
||||
from fastapi import status
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy.common_utils.openai_error_payload import error_status_code
|
||||
from litellm.types.token_count import TokenCountSource
|
||||
from litellm.types.utils import TokenCountResponse
|
||||
|
||||
_BARE_LOGFMT_VALUE: Final = re.compile(r"[\w.:/@+-]+", re.ASCII)
|
||||
_JSON_ESCAPED_BRACES: Final = str.maketrans({"{": "\\u007b", "}": "\\u007d"})
|
||||
|
||||
|
||||
def _logfmt_value(value: str | None) -> str:
|
||||
if value is None:
|
||||
return "None"
|
||||
if _BARE_LOGFMT_VALUE.fullmatch(value):
|
||||
return value
|
||||
return json.dumps(value).translate(_JSON_ESCAPED_BRACES)
|
||||
|
||||
|
||||
_Counted: Final = TypeVar("_Counted")
|
||||
_Rejection: Final = TypeVar("_Rejection")
|
||||
_ProviderCount: Final = TypeVar("_ProviderCount", bound=Mapping[str, object])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _CountFields:
|
||||
model_used: str | None
|
||||
input_tokens: int | None
|
||||
count_source: TokenCountSource | None
|
||||
|
||||
|
||||
_FAILED_COUNT: Final = _CountFields(model_used=None, input_tokens=None, count_source=None)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _CountRequest:
|
||||
litellm_call_id: str
|
||||
request_model: str | None
|
||||
model_id: str | None
|
||||
|
||||
|
||||
def deployment_model_id(deployment: Mapping[str, object] | None) -> str | None:
|
||||
match deployment:
|
||||
case {"model_info": {"id": str() as model_id}}:
|
||||
return model_id
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _log_token_count(request: _CountRequest, fields: _CountFields, status_code: int) -> None:
|
||||
verbose_proxy_logger.info(
|
||||
"token count - litellm_call_id=%s request_model=%s model_id=%s model_used=%s input_tokens=%s "
|
||||
"count_source=%s status_code=%s",
|
||||
_logfmt_value(request.litellm_call_id),
|
||||
_logfmt_value(request.request_model),
|
||||
_logfmt_value(request.model_id),
|
||||
_logfmt_value(fields.model_used),
|
||||
fields.input_tokens,
|
||||
fields.count_source,
|
||||
status_code,
|
||||
extra=MappingProxyType(
|
||||
{
|
||||
"litellm_call_id": request.litellm_call_id,
|
||||
"request_model": request.request_model,
|
||||
"model_id": request.model_id,
|
||||
"model_used": fields.model_used,
|
||||
"input_tokens": fields.input_tokens,
|
||||
"count_source": fields.count_source,
|
||||
"status_code": status_code,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def rejected_token_count(rejection: _Rejection, litellm_call_id: str, request_model: str | None) -> _Rejection:
|
||||
_log_token_count(
|
||||
_CountRequest(litellm_call_id, request_model, model_id=None),
|
||||
_FAILED_COUNT,
|
||||
error_status_code(rejection, status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
return rejection
|
||||
|
||||
|
||||
async def _logged(
|
||||
count: Awaitable[_Counted], request: _CountRequest, fields_of: Callable[[_Counted], _CountFields]
|
||||
) -> _Counted:
|
||||
try:
|
||||
counted: Final = await count
|
||||
except CancelledError:
|
||||
raise
|
||||
except BaseException as e:
|
||||
_log_token_count(request, _FAILED_COUNT, error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR))
|
||||
raise
|
||||
_log_token_count(request, fields_of(counted), status.HTTP_200_OK)
|
||||
return counted
|
||||
|
||||
|
||||
def _token_count_response_fields(counted: TokenCountResponse) -> _CountFields:
|
||||
return _CountFields(
|
||||
model_used=counted.model_used, input_tokens=counted.total_tokens, count_source=counted.count_source
|
||||
)
|
||||
|
||||
|
||||
async def logged_token_count(
|
||||
count: Awaitable[TokenCountResponse], litellm_call_id: str, request_model: str, model_id: str | None
|
||||
) -> TokenCountResponse:
|
||||
return await _logged(count, _CountRequest(litellm_call_id, request_model, model_id), _token_count_response_fields)
|
||||
|
||||
|
||||
async def logged_provider_token_count(
|
||||
count: Awaitable[_ProviderCount],
|
||||
litellm_call_id: str,
|
||||
request_model: str,
|
||||
model_id: str | None,
|
||||
model_used: Callable[[], str],
|
||||
) -> _ProviderCount:
|
||||
def provider_fields(counted: _ProviderCount) -> _CountFields:
|
||||
input_tokens: Final = counted.get("input_tokens")
|
||||
return _CountFields(
|
||||
model_used=model_used(),
|
||||
input_tokens=input_tokens if isinstance(input_tokens, int) and not isinstance(input_tokens, bool) else None,
|
||||
count_source="provider",
|
||||
)
|
||||
|
||||
return await _logged(count, _CountRequest(litellm_call_id, request_model, model_id), provider_fields)
|
||||
|
|
@ -13,6 +13,7 @@ from litellm.types.token_count import TokenCountProvider, TokenCountSourceFields
|
|||
from litellm.types.utils import TokenCountResponse
|
||||
|
||||
CANNOT_COUNT_MODEL_STATUSES: Final = frozenset({400, 404})
|
||||
REGION_DENIED_STATUSES: Final = frozenset({401, 403})
|
||||
TOKENIZER_LOAD_WAIT_SECONDS: Final = 5.0
|
||||
TOKENIZER_LOAD_REUSE_SECONDS: Final = 60.0
|
||||
TOKENIZER_LOAD_MAX_RUNNING: Final = 4
|
||||
|
|
@ -47,6 +48,7 @@ class TokenCountModel:
|
|||
model: str
|
||||
region: str | None
|
||||
litellm_provider: TokenCountProvider | None = None
|
||||
configured: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -66,7 +68,13 @@ class ProviderCountFailure:
|
|||
def cannot_count_model(self) -> bool:
|
||||
return not self.rejected_before_provider_call and self.status_code in CANNOT_COUNT_MODEL_STATUSES
|
||||
|
||||
@property
|
||||
def denied_in_region(self) -> bool:
|
||||
return not self.rejected_before_provider_call and self.status_code in REGION_DENIED_STATUSES
|
||||
|
||||
def with_hint(self, hint: str) -> "ProviderCountFailure":
|
||||
if self.rejected_before_provider_call:
|
||||
return self
|
||||
return replace(self, message=_with_hint(self.message, hint))
|
||||
|
||||
def to_proxy_exception(self) -> ProxyException:
|
||||
|
|
@ -111,7 +119,7 @@ def _count_provider(value: object) -> TokenCountProvider | None:
|
|||
return None
|
||||
|
||||
|
||||
def _count_model(value: Mapping[str, object]) -> TokenCountModel | None:
|
||||
def _count_model(value: Mapping[str, object], configured: bool) -> TokenCountModel | None:
|
||||
model: Final = _non_blank(value.get("model"))
|
||||
return (
|
||||
None
|
||||
|
|
@ -120,6 +128,7 @@ def _count_model(value: Mapping[str, object]) -> TokenCountModel | None:
|
|||
model=model,
|
||||
region=_non_blank(value.get("region")),
|
||||
litellm_provider=_count_provider(value.get("litellm_provider")),
|
||||
configured=configured,
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -146,12 +155,12 @@ def _resolved_count_model(
|
|||
) -> TokenCountModel | None:
|
||||
configured_mapping: Final = as_str_mapping(configured)
|
||||
if configured_mapping is not None:
|
||||
return _count_model(configured_mapping)
|
||||
return _count_model(configured_mapping, configured=True)
|
||||
default_mapping: Final = as_str_mapping(default)
|
||||
return (
|
||||
None
|
||||
if default_mapping is None
|
||||
else _within_deployment_geography(_count_model(default_mapping), deployment_region)
|
||||
else _within_deployment_geography(_count_model(default_mapping, configured=False), deployment_region)
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -160,9 +160,10 @@ async def create_gemini_agent(
|
|||
-H "Authorization: Bearer sk-..." \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-d '{
|
||||
"name": "my-custom-slides-agent",
|
||||
"base_agent": "waverunner",
|
||||
"instructions": "You are a helpful assistant that creates slides.",
|
||||
"id": "my-custom-slides-agent",
|
||||
"base_agent": "antigravity-preview-09-2026",
|
||||
"system_instruction": "You are a helpful assistant that creates slides.",
|
||||
"agent_config": {"type": "antigravity", "model": "gemini-3.8-flash"},
|
||||
"base_environment": {
|
||||
"type": "remote",
|
||||
"sources": [
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg
|
|||
_read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
read_request_body,
|
||||
)
|
||||
from litellm.proxy.common_utils.openai_error_payload import LITELLM_CALL_ID_HEADER
|
||||
from litellm.proxy.common_utils.token_count_log import rejected_token_count
|
||||
from litellm.proxy.common_utils.token_count_settings import count_source_fields
|
||||
from litellm.types.llms.vertex_ai import TokenCountDetailsResponse
|
||||
|
||||
|
|
@ -286,25 +288,35 @@ async def google_count_tokens(request: Request, model_name: str):
|
|||
from litellm.proxy.common_utils.http_parsing_utils import read_request_body
|
||||
from litellm.proxy.proxy_server import token_counter as internal_token_counter
|
||||
|
||||
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get(LITELLM_CALL_ID_HEADER))
|
||||
data: Final = await read_request_body(request=request)
|
||||
try:
|
||||
token_request: Final = _google_count_tokens_request(model_name, data)
|
||||
except ValidationError:
|
||||
return _google_error_response(
|
||||
request,
|
||||
ProxyException(
|
||||
message="Invalid Google countTokens request body", type="invalid_request_error", param=None, code=400
|
||||
return rejected_token_count(
|
||||
_google_error_response(
|
||||
request,
|
||||
ProxyException(
|
||||
message="Invalid Google countTokens request body",
|
||||
type="invalid_request_error",
|
||||
param=None,
|
||||
code=400,
|
||||
),
|
||||
),
|
||||
litellm_call_id,
|
||||
model_name,
|
||||
)
|
||||
|
||||
try:
|
||||
token_response: Final = await internal_token_counter(request=token_request, call_endpoint=True)
|
||||
token_response: Final = await internal_token_counter(
|
||||
request=token_request, call_endpoint=True, litellm_call_id=litellm_call_id
|
||||
)
|
||||
except ProxyException as error:
|
||||
return _google_error_response(request, error)
|
||||
except HTTPException as error:
|
||||
return _google_error_response(request, proxy_exception_from_http_exception(error, {}))
|
||||
except Exception as error:
|
||||
log_llm_api_exception(error, resolve_litellm_call_id(request.headers.get("x-litellm-call-id")))
|
||||
log_llm_api_exception(error, litellm_call_id)
|
||||
return _google_error_response(
|
||||
request,
|
||||
ProxyException(message="Internal server error", type="internal_server_error", param=None, code=500),
|
||||
|
|
|
|||
|
|
@ -45,11 +45,14 @@ from litellm.caching.redis_cache import log_redis_failure
|
|||
from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY
|
||||
from litellm.cost_calculator import get_usage_object
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_str_from_messages,
|
||||
split_media_for_tpm,
|
||||
)
|
||||
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
|
||||
from litellm.litellm_core_utils.token_counter import offload_token_count
|
||||
from litellm.litellm_core_utils.token_counter import estimate_image_tokens, offload_token_count
|
||||
from litellm.llms.azure.ptu_capacity import normalized_tokens
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_utils import (
|
||||
|
|
@ -98,6 +101,7 @@ if TYPE_CHECKING:
|
|||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
|
||||
from litellm.router import Router
|
||||
from litellm.types.agents import AgentResponse
|
||||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
|
||||
|
|
@ -135,6 +139,18 @@ class RateLimitedModel:
|
|||
return requested_limit if requested_limit is not None else limits.get(self.group)
|
||||
|
||||
|
||||
def _resolve_media_models(router: "Router | None", model: str, team_id: str | None) -> tuple[str, ...]:
|
||||
if router is None:
|
||||
return (model,)
|
||||
return tuple(dict.fromkeys(router.resolved_litellm_models(model, team_id, use_base_model=True))) or (model,)
|
||||
|
||||
|
||||
def _resolve_media_models_via_proxy_router(model: str, team_id: str | None) -> tuple[str, ...]:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
return _resolve_media_models(llm_router, model, team_id)
|
||||
|
||||
|
||||
def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
|
|
@ -862,6 +878,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
time_provider: Callable[[], datetime] | None = None,
|
||||
tag_rate_limit_resolver: TagRateLimitResolver = resolve_tag_rate_limits_from_db,
|
||||
model_group_resolver: Callable[[str], str | None] = _resolve_model_group_alias_via_proxy_router,
|
||||
media_model_resolver: Callable[[str, str | None], tuple[str, ...]] = _resolve_media_models_via_proxy_router,
|
||||
ptu_team_ceiling_resolver: Callable[
|
||||
[str, str], PTUTeamCeiling | None
|
||||
] = _resolve_ptu_team_ceiling_via_proxy_router,
|
||||
|
|
@ -872,6 +889,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
self._time_provider = time_provider or datetime.now
|
||||
self._tag_rate_limit_resolver = tag_rate_limit_resolver
|
||||
self._model_group_resolver = model_group_resolver
|
||||
self._media_model_resolver = media_model_resolver
|
||||
self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver
|
||||
self._fail_closed_resolver = fail_closed_resolver
|
||||
self._force_hash_tag_grouping_resolver = force_hash_tag_grouping_resolver
|
||||
|
|
@ -1133,11 +1151,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
|
||||
def _estimate_tokens_for_request(
|
||||
self,
|
||||
data: dict,
|
||||
data: object,
|
||||
model: str | None = None,
|
||||
min_configured_tpm_limit: int | None = None,
|
||||
call_type: str | None = None,
|
||||
configured_output_tokens: int | None = None,
|
||||
team_id: str | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
Estimate total tokens this request will consume so we can reserve them
|
||||
|
|
@ -1164,6 +1183,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
min_configured_tpm_limit=min_configured_tpm_limit,
|
||||
call_type=call_type,
|
||||
configured_output_tokens=configured_output_tokens,
|
||||
team_id=team_id,
|
||||
media_model_resolver=self._media_model_resolver,
|
||||
)
|
||||
total_estimated: Final = estimated_input_tokens + max_tokens_estimate
|
||||
|
||||
|
|
@ -1184,6 +1205,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
call_type: str | None,
|
||||
configured_output_tokens: int | None,
|
||||
raw_estimate: int,
|
||||
team_id: str | None = None,
|
||||
) -> int:
|
||||
"""The team PTU ceiling counts Azure normalized tokens, so its reservation weighs the
|
||||
output budget the way the ceiling does instead of the raw sum the other scopes reserve."""
|
||||
|
|
@ -1194,6 +1216,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
min_configured_tpm_limit=min_configured_tpm_limit,
|
||||
call_type=call_type,
|
||||
configured_output_tokens=configured_output_tokens,
|
||||
team_id=team_id,
|
||||
media_model_resolver=self._media_model_resolver,
|
||||
)
|
||||
normalized: Final = normalized_tokens(
|
||||
ceiling, prompt_tokens=estimated_input_tokens, completion_tokens=max_tokens_estimate
|
||||
|
|
@ -1206,6 +1230,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
min_configured_tpm_limit: int | None = None,
|
||||
call_type: str | None = None,
|
||||
configured_output_tokens: int | None = None,
|
||||
include_media: bool = True,
|
||||
team_id: str | None = None,
|
||||
media_model_resolver: Callable[[str, str | None], tuple[str, ...]] | None = None,
|
||||
) -> tuple[int, int]:
|
||||
"""
|
||||
Estimate input tokens and output (max_tokens) budget separately, so
|
||||
|
|
@ -1229,10 +1256,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
floor entirely, so the reservation reflects what this tenant's model
|
||||
actually emits rather than one constant shared by every tenant.
|
||||
"""
|
||||
if not isinstance(data, dict):
|
||||
request_data: Final = as_str_mapping(data)
|
||||
if not isinstance(data, dict) or request_data is None:
|
||||
return 0, 0
|
||||
request_model: Final = request_data.get("model")
|
||||
translated_data: Final = self._translate_google_genai_native_request(data, call_type)
|
||||
estimable_data: Final = translated_data if translated_data is not None else data
|
||||
estimable_data: Final = translated_data if translated_data is not None else request_data
|
||||
selected_fields: Final[tuple[object | None, object | None, object | None]] = (
|
||||
(None, None, estimable_data.get("input"))
|
||||
if call_type in RESPONSES_API_CALL_TYPES or call_type in EMBEDDING_API_CALL_TYPES
|
||||
|
|
@ -1249,10 +1278,27 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
)
|
||||
messages, prompt, raw_input = selected_fields
|
||||
input_text: Final = (
|
||||
original_input: Final = (
|
||||
billable_input_since_newest_compaction(raw_input) if call_type in RESPONSES_API_CALL_TYPES else raw_input
|
||||
)
|
||||
|
||||
media_source: Final = messages if messages is not None else original_input
|
||||
projected, media = (
|
||||
split_media_for_tpm(media_source)
|
||||
if include_media and (isinstance(messages, list) or call_type in RESPONSES_API_CALL_TYPES)
|
||||
else (original_input, ())
|
||||
)
|
||||
input_text: Final = projected if call_type in RESPONSES_API_CALL_TYPES else original_input
|
||||
fallback_model: Final = request_model if isinstance(request_model, str) else None
|
||||
resolved_models: Final = (
|
||||
media_model_resolver(fallback_model or "", team_id)
|
||||
if media and media_model_resolver is not None
|
||||
else (fallback_model or "",)
|
||||
)
|
||||
media_tokens: Final = sum(
|
||||
self._estimate_media_block_tokens(block, resolved_models, fallback_model) for block in media
|
||||
)
|
||||
|
||||
total_chars: Final = (
|
||||
len(get_str_from_messages(messages))
|
||||
if isinstance(messages, list) and messages
|
||||
|
|
@ -1267,7 +1313,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
else 0
|
||||
)
|
||||
|
||||
estimated_input_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0
|
||||
estimated_input_tokens: Final = (
|
||||
max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0
|
||||
) + media_tokens
|
||||
|
||||
explicit_max_tokens: Final = self._get_explicit_output_cap(data, call_type)
|
||||
generates_no_output: Final = (
|
||||
|
|
@ -1283,7 +1331,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
max_tokens_estimate: Final = (
|
||||
0
|
||||
if generates_no_output
|
||||
or (explicit_max_tokens is None and total_chars == 0 and configured_output_tokens is None)
|
||||
or (explicit_max_tokens is None and estimated_input_tokens == 0 and configured_output_tokens is None)
|
||||
else explicit_max_tokens
|
||||
if explicit_max_tokens is not None
|
||||
else configured_output_tokens
|
||||
|
|
@ -1293,6 +1341,24 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
|
||||
return estimated_input_tokens, max_tokens_estimate * self.get_output_candidate_count(data, call_type)
|
||||
|
||||
@staticmethod
|
||||
def _estimate_media_block_tokens(
|
||||
block: Mapping[str, object], models: tuple[str, ...], fallback_model: str | None
|
||||
) -> int:
|
||||
if block.get("type") == "input_audio":
|
||||
audio: Final = _callback_kwargs_mapping(block.get("input_audio"))
|
||||
data: Final = audio.get("data")
|
||||
return estimate_audio_tokens(data if isinstance(data, str) else None, models, fallback_model=fallback_model)
|
||||
image: Final = _callback_kwargs_mapping(block.get("image_url"))
|
||||
url: Final = image.get("url")
|
||||
detail: Final = image.get("detail")
|
||||
return estimate_image_tokens(
|
||||
models,
|
||||
url if isinstance(url, str) else "",
|
||||
detail if isinstance(detail, str) else "auto",
|
||||
fallback_model=fallback_model,
|
||||
)
|
||||
|
||||
def _is_redis_cluster(self) -> bool:
|
||||
"""
|
||||
Check if the dual cache is using Redis cluster.
|
||||
|
|
@ -3958,7 +4024,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate
|
||||
if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str):
|
||||
return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN)
|
||||
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(data=data, call_type=call_type)
|
||||
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(
|
||||
data=data, call_type=call_type, include_media=False
|
||||
)
|
||||
return estimated_input_tokens + audio_token_estimate
|
||||
|
||||
async def _reserve_project_io_tokens_or_raise(
|
||||
|
|
@ -4003,6 +4071,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
data=data,
|
||||
min_configured_tpm_limit=min_configured_otpm_limit,
|
||||
call_type=call_type,
|
||||
include_media=False,
|
||||
)
|
||||
raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)(
|
||||
data=data, model=requested_model, call_type=call_type
|
||||
|
|
@ -4360,12 +4429,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
# Post-call reconciliation refunds the over-reservation
|
||||
# delta when actual usage comes in below the floor.
|
||||
estimated_tokens: Final = max(
|
||||
self._estimate_tokens_for_request(
|
||||
await offload_token_count(self._estimate_tokens_for_request)(
|
||||
data=data,
|
||||
model=requested_model,
|
||||
min_configured_tpm_limit=min_configured_tpm_limit,
|
||||
call_type=call_type,
|
||||
configured_output_tokens=configured_output_tokens,
|
||||
team_id=user_api_key_dict.team_id,
|
||||
),
|
||||
1,
|
||||
)
|
||||
|
|
@ -4383,13 +4453,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
|
||||
capped_request: Final = _REQUEST_RATE_LIMIT_DATA.validate_python(data)
|
||||
ptu_estimated_tokens: Final = self._estimate_ptu_tokens_for_request(
|
||||
ptu_estimated_tokens: Final = await offload_token_count(self._estimate_ptu_tokens_for_request)(
|
||||
ceiling=stash.ptu_ceiling,
|
||||
data=capped_request,
|
||||
min_configured_tpm_limit=min_configured_tpm_limit,
|
||||
call_type=call_type,
|
||||
configured_output_tokens=configured_output_tokens,
|
||||
raw_estimate=estimated_tokens,
|
||||
team_id=user_api_key_dict.team_id,
|
||||
)
|
||||
if stash.ptu_ceiling is not None and ptu_estimated_tokens > stash.ptu_ceiling.tpm_limit:
|
||||
if not reuses_request_slot:
|
||||
|
|
|
|||
|
|
@ -49,6 +49,8 @@ from litellm.proxy.management_endpoints.common_daily_activity import daily_activ
|
|||
from litellm.proxy.management_helpers.auto_router_permissions import (
|
||||
authorize_member_auto_router_dependencies,
|
||||
authorize_member_auto_router_team,
|
||||
auto_router_environment_reference_error,
|
||||
raise_auto_router_environment_reference_error,
|
||||
validate_member_auto_router_config,
|
||||
)
|
||||
from litellm.repositories.autorouter_session_repository import AutoRouterSessionRepository
|
||||
|
|
@ -402,6 +404,10 @@ async def validate_complexity_router_config(
|
|||
"""
|
||||
member_team: Final = await _authorize_router_dry_run(user_api_key_dict=user_api_key_dict, team_id=data.team_id)
|
||||
|
||||
raise_auto_router_environment_reference_error(
|
||||
auto_router_environment_reference_error(data.complexity_router_config, user_api_key_dict)
|
||||
)
|
||||
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
validate_complexity_router_config_write,
|
||||
)
|
||||
|
|
@ -560,6 +566,12 @@ async def preview_auto_router_routing(
|
|||
detail={"error": CommonProxyErrors.no_llm_router.value},
|
||||
)
|
||||
resolved: Final = await _resolve_saved_routing_test(data, user_api_key_dict, llm_router)
|
||||
if data.saved_model_id is None:
|
||||
raise_auto_router_environment_reference_error(
|
||||
auto_router_environment_reference_error(
|
||||
data.complexity_router_config.model_dump(exclude_none=True), user_api_key_dict
|
||||
)
|
||||
)
|
||||
actor: Final = (
|
||||
await _authorize_member_dry_run_config(
|
||||
config=resolved.complexity_router_config.model_dump(exclude_none=True),
|
||||
|
|
|
|||
|
|
@ -102,6 +102,8 @@ from litellm.proxy.management_helpers.auto_router_permissions import (
|
|||
authorize_member_auto_router_dependencies,
|
||||
authorize_member_auto_router_team,
|
||||
authorize_member_auto_router_write,
|
||||
auto_router_environment_reference_error,
|
||||
raise_auto_router_environment_reference_error,
|
||||
)
|
||||
from litellm.proxy.management_helpers.model_allowlist_rename_sync import sync_model_allowlists_for_renamed_model
|
||||
from litellm.proxy.management_helpers.model_quota_policy import (
|
||||
|
|
@ -2627,6 +2629,14 @@ class ModelManagementAuthChecks:
|
|||
ModelManagementAuthChecks._reject_team_inline_api_key_secret_reference(
|
||||
model_params=model_params, incoming_params=incoming_params, incoming_model_params=incoming_model_params
|
||||
)
|
||||
raise_auto_router_environment_reference_error(
|
||||
auto_router_environment_reference_error(
|
||||
config=_OPTIONAL_JSON_OBJECT.validate_python(
|
||||
incoming_params.get("complexity_router_config") if incoming_params is not None else None
|
||||
),
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
)
|
||||
# Federation fields choose which server-side secret is read and where the org-scoped token
|
||||
# it buys is sent, so only a proxy admin may point a federated deployment somewhere else.
|
||||
# Evaluated on the RESULTING deployment: a patch attaching a credential by name inherits
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ from types import MappingProxyType
|
|||
from typing import TYPE_CHECKING, Final, Literal
|
||||
|
||||
from fastapi import HTTPException
|
||||
from pydantic import ConfigDict, Field, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
from pydantic import ConfigDict, Field, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict, assert_never
|
||||
|
||||
from litellm.models.organization import LiteLLM_OrganizationTable
|
||||
from litellm.models.project import LiteLLM_ProjectTable
|
||||
|
|
@ -36,6 +36,7 @@ from litellm.repositories.table_repositories import TeamMembershipRepository
|
|||
from litellm.router import Router
|
||||
from litellm.router_strategy.complexity_router.config import (
|
||||
ComplexityRouterConfigWrite,
|
||||
normalize_classifier_config_aliases,
|
||||
resolve_complexity_router_config_write,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.request_models import RequestComplexityRouterConfig
|
||||
|
|
@ -47,6 +48,36 @@ if TYPE_CHECKING:
|
|||
from prisma import types as prisma_types
|
||||
|
||||
|
||||
def auto_router_environment_reference_error(
|
||||
config: Mapping[str, object] | None, user_api_key_dict: UserAPIKeyAuth
|
||||
) -> Literal["environment_reference_denied"] | None:
|
||||
if config is None or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
|
||||
return None
|
||||
classifier: Final = normalize_classifier_config_aliases(config).get("opensource_classifier_config")
|
||||
if not isinstance(classifier, Mapping):
|
||||
return None
|
||||
fields: Final = TypeAdapter(Mapping[str, object]).validate_python(classifier)
|
||||
if any(
|
||||
isinstance(value, str) and value.startswith("os.environ/")
|
||||
for value in (fields.get("api_key"), fields.get("api_base"))
|
||||
):
|
||||
return "environment_reference_denied"
|
||||
return None
|
||||
|
||||
|
||||
def raise_auto_router_environment_reference_error(error: Literal["environment_reference_denied"] | None) -> None:
|
||||
match error:
|
||||
case None:
|
||||
return
|
||||
case "environment_reference_denied":
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Only proxy admins can submit classifier connection environment references",
|
||||
)
|
||||
case _:
|
||||
assert_never(error)
|
||||
|
||||
|
||||
class _MemberRouterThinking(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ from litellm.proxy.common_request_processing import (
|
|||
ProxyBaseLLMRequestProcessing,
|
||||
model_deployments_support_stream_options,
|
||||
open_sse_before_first_byte,
|
||||
resolve_litellm_call_id,
|
||||
)
|
||||
from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports
|
||||
_read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
|
|
@ -99,10 +100,16 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg
|
|||
safe_get_request_headers,
|
||||
safe_set_request_parsed_body,
|
||||
)
|
||||
from litellm.proxy.common_utils.openai_error_payload import LITELLM_CALL_ID_HEADER
|
||||
from litellm.proxy.common_utils.resource_ownership import is_proxy_admin
|
||||
from litellm.proxy.common_utils.sse_keepalive import (
|
||||
wrap_passthrough_sse_bytes_with_keepalive_pings,
|
||||
)
|
||||
from litellm.proxy.common_utils.token_count_log import (
|
||||
deployment_model_id,
|
||||
logged_provider_token_count,
|
||||
rejected_token_count,
|
||||
)
|
||||
from litellm.proxy.pass_through_endpoints.common_utils import get_litellm_virtual_key
|
||||
from litellm.proxy.pass_through_endpoints.injected_stream_usage import (
|
||||
stream_usage_relay,
|
||||
|
|
@ -1259,6 +1266,7 @@ async def handle_bedrock_count_tokens(
|
|||
from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get(LITELLM_CALL_ID_HEADER))
|
||||
try:
|
||||
# Initialize the handler
|
||||
handler: Final = BedrockCountTokensHandler()
|
||||
|
|
@ -1266,40 +1274,45 @@ async def handle_bedrock_count_tokens(
|
|||
# Extract model from request body
|
||||
model: Final = _optional_str(request_body.get("model"))
|
||||
if not model:
|
||||
raise HTTPException(status_code=400, detail={"error": "Model is required in request body"})
|
||||
raise rejected_token_count(
|
||||
HTTPException(status_code=400, detail={"error": "Model is required in request body"}),
|
||||
litellm_call_id,
|
||||
None,
|
||||
)
|
||||
|
||||
# Get model parameters from router
|
||||
litellm_params: Final = {"user_api_key_dict": user_api_key_dict}
|
||||
resolved_model = model # Default fallback
|
||||
|
||||
if llm_router:
|
||||
deployments: Final = llm_router.get_model_list(model_name=model)
|
||||
if deployments and len(deployments) > 0:
|
||||
# Get the first matching deployment
|
||||
deployment: Final = deployments[0]
|
||||
model_litellm_params: Final = deployment.get("litellm_params", {})
|
||||
deployments: Final = llm_router.get_model_list(model_name=model) if llm_router else None
|
||||
deployment: Final = deployments[0] if deployments else None
|
||||
if deployment is not None:
|
||||
model_litellm_params: Final = deployment.get("litellm_params", {})
|
||||
|
||||
# Get the resolved model ID from the configuration
|
||||
if "model" in model_litellm_params:
|
||||
resolved_model = model_litellm_params["model"]
|
||||
# Get the resolved model ID from the configuration
|
||||
if "model" in model_litellm_params:
|
||||
resolved_model = model_litellm_params["model"]
|
||||
|
||||
# Copy all litellm_params - BaseAWSLLM will handle AWS credential discovery
|
||||
for key, value in model_litellm_params.items():
|
||||
if key != "user_api_key_dict": # Don't overwrite user_api_key_dict
|
||||
litellm_params[key] = value
|
||||
# Copy all litellm_params - BaseAWSLLM will handle AWS credential discovery
|
||||
for key, value in model_litellm_params.items():
|
||||
if key != "user_api_key_dict": # Don't overwrite user_api_key_dict
|
||||
litellm_params[key] = value
|
||||
|
||||
verbose_proxy_logger.debug("Count tokens litellm_params: %s", litellm_params)
|
||||
verbose_proxy_logger.debug("Resolved model: %s", resolved_model)
|
||||
|
||||
# Handle the count tokens request
|
||||
result: Final = await handler.handle_count_tokens_request(
|
||||
request_data=request_body,
|
||||
litellm_params=litellm_params,
|
||||
resolved_model=resolved_model,
|
||||
return await logged_provider_token_count(
|
||||
handler.handle_count_tokens_request(
|
||||
request_data=request_body,
|
||||
litellm_params=litellm_params,
|
||||
resolved_model=resolved_model,
|
||||
),
|
||||
litellm_call_id=litellm_call_id,
|
||||
request_model=model,
|
||||
model_id=deployment_model_id(deployment),
|
||||
model_used=partial(handler.count_tokens_model_id, resolved_model),
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
except BedrockError as e:
|
||||
# Convert BedrockError to HTTPException for FastAPI
|
||||
verbose_proxy_logger.error("BedrockError in handle_bedrock_count_tokens: %s", e)
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ from litellm.integrations.custom_guardrail import (
|
|||
without_server_streaming_classification,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.otel.model.semconv import PASSTHROUGH_OPERATION_KEY, GenAIOperation
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
bind_budget_reservation_to_callbacks,
|
||||
get_metadata_variable_name_from_kwargs,
|
||||
|
|
@ -151,6 +152,7 @@ from .success_handler import PassThroughEndpointLogging
|
|||
from .upstream_usage_headers import (
|
||||
UpstreamReportedUsage,
|
||||
apply_upstream_reported_model,
|
||||
apply_upstream_reported_operation,
|
||||
apply_upstream_reported_usage,
|
||||
)
|
||||
|
||||
|
|
@ -1134,6 +1136,7 @@ async def pass_through_request(
|
|||
guardrails_config: dict | None = None,
|
||||
timeout: float | None = None,
|
||||
upstream_headers_filter: Callable[[Mapping[str, str]], Mapping[str, str]] = _unfiltered_headers,
|
||||
operation: GenAIOperation | None = None,
|
||||
):
|
||||
"""
|
||||
Pass through endpoint handler, makes the httpx request for pass-through endpoints and ensures logging hooks are called
|
||||
|
|
@ -1155,6 +1158,7 @@ async def pass_through_request(
|
|||
timeout: Optional per-endpoint timeout in seconds. Falls back to
|
||||
general_settings.pass_through_request_timeout, then 600s.
|
||||
upstream_headers_filter: Optional last rewrite of the headers sent upstream, after x-pass- expansion
|
||||
operation: Optional GenAI operation the route reports, unless the upstream names its own
|
||||
"""
|
||||
from litellm.exceptions import ModifyResponseException
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
|
@ -1353,6 +1357,8 @@ async def pass_through_request(
|
|||
call_type="pass_through_endpoint",
|
||||
)
|
||||
logging_obj.model_call_details["litellm_call_id"] = litellm_call_id
|
||||
if operation is not None:
|
||||
logging_obj.model_call_details[PASSTHROUGH_OPERATION_KEY] = operation
|
||||
|
||||
## PASSTHROUGH MANAGED ID RESOLUTION (INPUT) ##
|
||||
# Resolve managed IDs in path, query params, and body back to raw
|
||||
|
|
@ -1553,6 +1559,7 @@ async def pass_through_request(
|
|||
logging_obj=logging_obj,
|
||||
headers=response.headers,
|
||||
)
|
||||
apply_upstream_reported_operation(logging_obj, response.headers)
|
||||
|
||||
relay_response: Final = await _log_passthrough_upstream_failure(
|
||||
response=response,
|
||||
|
|
@ -1646,6 +1653,7 @@ async def pass_through_request(
|
|||
logging_obj=logging_obj,
|
||||
headers=response.headers,
|
||||
)
|
||||
apply_upstream_reported_operation(logging_obj, response.headers)
|
||||
|
||||
if _is_streaming_response(response) is True:
|
||||
logging_obj.stream = True
|
||||
|
|
@ -2161,6 +2169,7 @@ def create_pass_through_route(
|
|||
config_file_path: str | None = None,
|
||||
timeout: float | None = None,
|
||||
upstream_headers_filter: Callable[[Mapping[str, str]], Mapping[str, str]] = _unfiltered_headers,
|
||||
operation: GenAIOperation | None = None,
|
||||
):
|
||||
# check if target is an adapter.py or a url
|
||||
from litellm._uuid import uuid
|
||||
|
|
@ -2301,6 +2310,7 @@ def create_pass_through_route(
|
|||
guardrails_config=cast(dict | None, param_guardrails),
|
||||
timeout=cast(float | None, param_timeout),
|
||||
upstream_headers_filter=upstream_headers_filter,
|
||||
operation=operation,
|
||||
)
|
||||
finally:
|
||||
if hasattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY):
|
||||
|
|
@ -3232,6 +3242,7 @@ class InitPassThroughEndpointHelpers:
|
|||
guardrails=guardrails,
|
||||
config_file_path=config_file_path,
|
||||
timeout=timeout,
|
||||
operation=GenAIOperation.LITELLM_PASSTHROUGH,
|
||||
),
|
||||
methods=methods,
|
||||
dependencies=dependencies,
|
||||
|
|
@ -3316,6 +3327,7 @@ class InitPassThroughEndpointHelpers:
|
|||
guardrails=guardrails,
|
||||
config_file_path=config_file_path,
|
||||
timeout=timeout,
|
||||
operation=GenAIOperation.LITELLM_PASSTHROUGH,
|
||||
),
|
||||
methods=methods,
|
||||
dependencies=dependencies,
|
||||
|
|
|
|||
|
|
@ -5,18 +5,21 @@ price the request from the response body. Those targets report the totals for
|
|||
the whole HTTP request in ``x-litellm-*`` response headers instead; LiteLLM
|
||||
records the reported values without recomputing them. A target may also name
|
||||
the request in ``x-litellm-upstream-model`` so its spend can be told apart from
|
||||
other requests that carry no ``model`` in the body.
|
||||
other requests that carry no ``model`` in the body, and name the GenAI
|
||||
operation it performed, which labels the request's trace.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import MAX_SPEND_LOG_MODEL_NAME_LENGTH
|
||||
from litellm.integrations.otel.model.semconv import PASSTHROUGH_OPERATION_KEY, GenAIOperation
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
|
@ -25,6 +28,8 @@ UPSTREAM_TOTAL_TOKENS_HEADER: Final = "x-litellm-total-tokens"
|
|||
# Deliberately not x-litellm-model-name/-id/-group: the gateway already sends
|
||||
# those to clients, and pass-through relays upstream headers back to the client.
|
||||
UPSTREAM_MODEL_HEADER: Final = "x-litellm-upstream-model"
|
||||
UPSTREAM_OPERATION_NAME_HEADER: Final = "x-litellm-operation-name"
|
||||
_OPERATIONS_BY_NAME: Final = MappingProxyType({operation.value: operation for operation in GenAIOperation})
|
||||
|
||||
# model_call_details key holding what the upstream reported, so later stages of
|
||||
# the success path can tell an upstream-reported cost apart from one LiteLLM
|
||||
|
|
@ -175,3 +180,19 @@ def has_upstream_reported_usage(logging_obj: LiteLLMLoggingObj) -> bool:
|
|||
means zero, never a fallback to someone else's estimate.
|
||||
"""
|
||||
return isinstance(logging_obj.model_call_details.get(UPSTREAM_REPORTED_USAGE_KEY), UpstreamReportedUsage)
|
||||
|
||||
|
||||
def apply_upstream_reported_operation(logging_obj: LiteLLMLoggingObj, headers: Mapping[str, str]) -> None:
|
||||
"""Record the GenAI operation the upstream named, overriding the route's label. An unknown name is ignored."""
|
||||
raw_operation: Final = headers.get(UPSTREAM_OPERATION_NAME_HEADER)
|
||||
if raw_operation is None:
|
||||
return
|
||||
operation: Final = _OPERATIONS_BY_NAME.get(raw_operation)
|
||||
if operation is None:
|
||||
verbose_proxy_logger.warning(
|
||||
"pass_through_endpoint: upstream sent unknown %s=%r; ignoring it",
|
||||
UPSTREAM_OPERATION_NAME_HEADER,
|
||||
raw_operation,
|
||||
)
|
||||
return
|
||||
logging_obj.model_call_details[PASSTHROUGH_OPERATION_KEY] = operation # rebind-ok: annotates the request's record
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ from litellm.litellm_core_utils.litellm_logging import (
|
|||
)
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.litellm_core_utils.token_counter import offload_token_count
|
||||
from litellm.litellm_core_utils.token_counter import UnsupportedContentTypeError, offload_token_count
|
||||
from litellm.llms.bedrock.count_tokens.kimi_k3 import (
|
||||
HUGGING_FACE_SOURCE,
|
||||
KimiK3Encodings,
|
||||
|
|
@ -552,6 +552,11 @@ from litellm.proxy.common_utils.timezone_utils import (
|
|||
get_budget_reset_settings,
|
||||
get_budget_reset_time,
|
||||
)
|
||||
from litellm.proxy.common_utils.token_count_log import (
|
||||
deployment_model_id,
|
||||
logged_token_count,
|
||||
rejected_token_count,
|
||||
)
|
||||
from litellm.proxy.common_utils.user_api_key_cache import (
|
||||
UserApiKeyCache,
|
||||
end_user_cache_key,
|
||||
|
|
@ -14848,6 +14853,7 @@ async def _try_native_responses_count(
|
|||
async def _provider_token_count(
|
||||
count_with: Callable[[str, str | None], Awaitable[Union["TokenCountResponse", ProviderCountFailure, None]]],
|
||||
model_to_use: str,
|
||||
request_model: str,
|
||||
settings: TokenCountSettings,
|
||||
local_estimate_allowed: bool,
|
||||
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
|
||||
|
|
@ -14870,17 +14876,28 @@ async def _provider_token_count(
|
|||
if count_model is None:
|
||||
return counted.model_copy(update={"count_source": "provider"})
|
||||
return counted.model_copy(update={"count_source": "token_count_model", "model_used": count_model.model})
|
||||
failure: Final = (
|
||||
counted
|
||||
if count_model is None or count_model.region is None
|
||||
else counted.with_hint(
|
||||
f"This model counts tokens with {count_model.model} in {count_model.region}. "
|
||||
"If that region is not enabled for the AWS account, enable it or set token_count_model in model_info"
|
||||
)
|
||||
pinned_count_model: Final = None if count_model is None or count_model.region is None else count_model
|
||||
region_hint: Final = (
|
||||
None
|
||||
if pinned_count_model is None
|
||||
else f"This model counts tokens with {pinned_count_model.model} in {pinned_count_model.region}. "
|
||||
"If that region is not enabled for the AWS account, enable it or set token_count_model in model_info"
|
||||
)
|
||||
if count_model is None and failure.cannot_count_model and settings.framing is not None and local_estimate_allowed:
|
||||
return failure
|
||||
raise failure.to_proxy_exception()
|
||||
failure: Final = counted if region_hint is None else counted.with_hint(region_hint)
|
||||
framing_can_stand_in: Final = (count_model is None or not count_model.configured) and (
|
||||
failure.cannot_count_model or (pinned_count_model is not None and failure.denied_in_region)
|
||||
)
|
||||
if not (framing_can_stand_in and settings.framing is not None and local_estimate_allowed):
|
||||
raise failure.to_proxy_exception()
|
||||
if pinned_count_model is not None:
|
||||
verbose_proxy_logger.warning(
|
||||
"Token count for %s answered %s from %s in %s, so it falls back to the framed local estimate",
|
||||
request_model,
|
||||
failure.status_code,
|
||||
pinned_count_model.model,
|
||||
pinned_count_model.region,
|
||||
)
|
||||
return failure
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -15253,16 +15270,21 @@ def _system_message(system: object) -> ChatCompletionSystemMessage | None:
|
|||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=TokenCountResponse,
|
||||
)
|
||||
async def token_counter(request: TokenCountRequest, call_endpoint: bool = False):
|
||||
async def token_counter(
|
||||
request: TokenCountRequest,
|
||||
call_endpoint: bool = False,
|
||||
litellm_call_id: Annotated[str | None, Header(alias=LITELLM_CALL_ID_HEADER)] = None,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
request: TokenCountRequest
|
||||
call_endpoint: bool - When set to "True" it will call the token counting endpoint - e.g Anthropic or Google AI Studio Token Counting APIs.
|
||||
litellm_call_id: the x-litellm-call-id header, logged with the count so it joins the inference call's logs
|
||||
|
||||
Returns:
|
||||
TokenCountResponse
|
||||
"""
|
||||
return await count_request_tokens(request=request, call_endpoint=call_endpoint)
|
||||
return await count_request_tokens(request=request, call_endpoint=call_endpoint, litellm_call_id=litellm_call_id)
|
||||
|
||||
|
||||
async def count_request_tokens(
|
||||
|
|
@ -15273,7 +15295,69 @@ async def count_request_tokens(
|
|||
context_management: JsonValue = None,
|
||||
request_litellm_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
responses_request: Mapping[str, object] | None = None,
|
||||
litellm_call_id: str | None = None,
|
||||
client_oauth_token: str | None = None,
|
||||
) -> TokenCountResponse:
|
||||
call_id: Final = resolve_litellm_call_id(litellm_call_id)
|
||||
if request.prompt is None and request.messages is None and request.contents is None:
|
||||
raise rejected_token_count(
|
||||
HTTPException(status_code=400, detail="prompt or messages or contents must be provided"),
|
||||
call_id,
|
||||
request.model,
|
||||
)
|
||||
routed: Final = await _routed_count_deployment(request.model, responses_request)
|
||||
count: Final = _count_request_tokens(
|
||||
request=request,
|
||||
routed=routed,
|
||||
call_endpoint=call_endpoint,
|
||||
messages_format=messages_format,
|
||||
output_format=output_format,
|
||||
context_management=context_management,
|
||||
request_litellm_params=request_litellm_params,
|
||||
responses_request=responses_request,
|
||||
client_oauth_token=client_oauth_token,
|
||||
)
|
||||
return await logged_token_count(
|
||||
_with_anthropic_content_errors(count) if messages_format == "anthropic" else count,
|
||||
litellm_call_id=call_id,
|
||||
request_model=request.model,
|
||||
model_id=deployment_model_id(routed),
|
||||
)
|
||||
|
||||
|
||||
async def _with_anthropic_content_errors(count: Awaitable[TokenCountResponse]) -> TokenCountResponse:
|
||||
try:
|
||||
return await count
|
||||
except UnsupportedContentTypeError as e:
|
||||
raise ProxyException(message=str(e), type="invalid_request_error", param="messages", code=400) from e
|
||||
|
||||
|
||||
async def _routed_count_deployment(
|
||||
model: str, responses_request: Mapping[str, object] | None
|
||||
) -> Mapping[str, object] | None:
|
||||
if llm_router is None:
|
||||
return None
|
||||
try:
|
||||
return await llm_router.async_get_available_deployment(
|
||||
model=model, request_kwargs=dict(_responses_routing_kwargs(responses_request))
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.exception(
|
||||
"litellm.proxy.proxy_server.token_counter(): Exception occured while getting deployment"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
async def _count_request_tokens(
|
||||
request: TokenCountRequest,
|
||||
routed: Mapping[str, object] | None,
|
||||
call_endpoint: bool,
|
||||
messages_format: TokenCountMessagesFormat,
|
||||
output_format: JsonValue,
|
||||
context_management: JsonValue,
|
||||
request_litellm_params: Mapping[str, JsonValue],
|
||||
responses_request: Mapping[str, object] | None,
|
||||
client_oauth_token: str | None,
|
||||
) -> TokenCountResponse:
|
||||
prompt: Final = request.prompt
|
||||
messages: Final = request.messages
|
||||
|
|
@ -15281,26 +15365,9 @@ async def count_request_tokens(
|
|||
tools: Final = request.tools
|
||||
system: Final = request.system
|
||||
|
||||
#########################################################
|
||||
# Validate request
|
||||
#########################################################
|
||||
if prompt is None and messages is None and contents is None:
|
||||
raise HTTPException(status_code=400, detail="prompt or messages or contents must be provided")
|
||||
|
||||
deployment: dict[str, Any] | None = None
|
||||
deployment: dict[str, Any] | None = None if routed is None else dict(routed)
|
||||
litellm_model_name = None
|
||||
model_info: ModelMapInfo | None = None
|
||||
if llm_router is not None:
|
||||
# get 1 deployment corresponding to the model
|
||||
try:
|
||||
deployment = await llm_router.async_get_available_deployment(
|
||||
model=request.model,
|
||||
request_kwargs=dict(_responses_routing_kwargs(responses_request)),
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.exception(
|
||||
"litellm.proxy.proxy_server.token_counter(): Exception occured while getting deployment"
|
||||
)
|
||||
if deployment is not None:
|
||||
litellm_model_name = deployment.get("litellm_params", {}).get("model")
|
||||
model_info = deployment.get("model_info", {})
|
||||
|
|
@ -15400,6 +15467,7 @@ async def count_request_tokens(
|
|||
chat_params=MappingProxyType({}) if chat_count is None else chat_count.params,
|
||||
),
|
||||
model_to_use=model_to_use,
|
||||
request_model=request.model,
|
||||
settings=settings,
|
||||
local_estimate_allowed=litellm.disable_token_counter is not True and contents_estimable,
|
||||
)
|
||||
|
|
@ -19899,12 +19967,14 @@ async def update_config_general_settings(
|
|||
)
|
||||
|
||||
try:
|
||||
ConfigGeneralSettings.model_validate({data.field_name: requested_value})
|
||||
validated: Final = ConfigGeneralSettings.model_validate({data.field_name: requested_value})
|
||||
except Exception:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"error": f"Invalid type of field value={type(requested_value)} passed in."},
|
||||
)
|
||||
parsed_value: Final[object] = _UNTYPED_VALUE.validate_python(getattr(validated, data.field_name))
|
||||
stored_value: Final = parsed_value if isinstance(parsed_value, bool) else requested_value
|
||||
|
||||
if data.field_name == "alerting_args":
|
||||
try:
|
||||
|
|
@ -19933,10 +20003,10 @@ async def update_config_general_settings(
|
|||
|
||||
proxy_config.reject_config_owned_writes(
|
||||
section_name="general_settings",
|
||||
changed_keys={data.field_name: cast(JsonValue, requested_value)}, # cast-ok: validated above
|
||||
changed_keys={data.field_name: cast(JsonValue, stored_value)}, # cast-ok: validated above
|
||||
)
|
||||
|
||||
field_value = requested_value
|
||||
field_value = stored_value
|
||||
if data.field_name == "plugins":
|
||||
field_value = _preserve_redacted_plugin_keys(field_value, general_settings.get("plugins"))
|
||||
|
||||
|
|
|
|||
|
|
@ -4145,6 +4145,34 @@
|
|||
],
|
||||
"default_model_placeholder": "gpt-3.5-turbo"
|
||||
},
|
||||
{
|
||||
"provider": "ZeroGPU",
|
||||
"provider_display_name": "ZeroGPU",
|
||||
"litellm_provider": "zerogpu",
|
||||
"credential_fields": [
|
||||
{
|
||||
"key": "api_base",
|
||||
"label": "API Base",
|
||||
"placeholder": "https://api.zerogpu.ai/v1",
|
||||
"tooltip": null,
|
||||
"required": false,
|
||||
"field_type": "text",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
},
|
||||
{
|
||||
"key": "api_key",
|
||||
"label": "API Key",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": true,
|
||||
"field_type": "password",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
}
|
||||
],
|
||||
"default_model_placeholder": "zerogpu/gpt-oss-120b"
|
||||
},
|
||||
{
|
||||
"provider": "CURSOR",
|
||||
"provider_display_name": "Cursor",
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ from litellm.proxy.common_request_processing import (
|
|||
cost_headers,
|
||||
create_response,
|
||||
resolve_counted_tool_fields,
|
||||
resolve_litellm_call_id,
|
||||
)
|
||||
from litellm.proxy.common_utils.error_body_call_id import error_body_call_id, with_call_id
|
||||
from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports
|
||||
|
|
@ -69,6 +70,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
error_status_code,
|
||||
openai_error_type_for_status,
|
||||
)
|
||||
from litellm.proxy.common_utils.token_count_log import rejected_token_count
|
||||
from litellm.proxy.common_utils.token_count_settings import count_source_fields
|
||||
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
|
||||
from litellm.proxy.hooks.responses_id_security import ResponsesIDSecurity, decrypted_response_id
|
||||
|
|
@ -1407,7 +1409,12 @@ class _TokenCountPayload(TypedDict):
|
|||
|
||||
class _TokenCounter(Protocol):
|
||||
def __call__(
|
||||
self, *, request: TokenCountRequest, call_endpoint: bool, responses_request: Mapping[str, object] | None
|
||||
self,
|
||||
*,
|
||||
request: TokenCountRequest,
|
||||
call_endpoint: bool,
|
||||
responses_request: Mapping[str, object] | None,
|
||||
litellm_call_id: str | None,
|
||||
) -> Awaitable[TokenCountResponse]: ...
|
||||
|
||||
|
||||
|
|
@ -1648,18 +1655,23 @@ async def responses_input_tokens(
|
|||
|
||||
Returns: `{"object": "response.input_tokens", "input_tokens": <count>}`
|
||||
"""
|
||||
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get(LITELLM_CALL_ID_HEADER))
|
||||
data: Final = await read_request_body(request=request)
|
||||
model_name: Final = data.get("model")
|
||||
input_value: Final = data.get("input")
|
||||
if not isinstance(model_name, str) or not model_name:
|
||||
return _missing_responses_param_response("model")
|
||||
return rejected_token_count(_missing_responses_param_response("model"), litellm_call_id, None)
|
||||
if input_value is None:
|
||||
return _missing_responses_param_response("input")
|
||||
return rejected_token_count(_missing_responses_param_response("input"), litellm_call_id, model_name)
|
||||
if isinstance(input_value, (str, list)) and not input_value:
|
||||
return _responses_invalid_request_response(
|
||||
message="""One of "input" or "previous_response_id" or 'prompt' or 'conversation' must be provided.""",
|
||||
param=None,
|
||||
code="missing_required_parameter",
|
||||
return rejected_token_count(
|
||||
_responses_invalid_request_response(
|
||||
message="""One of "input" or "previous_response_id" or 'prompt' or 'conversation' must be provided.""",
|
||||
param=None,
|
||||
code="missing_required_parameter",
|
||||
),
|
||||
litellm_call_id,
|
||||
model_name,
|
||||
)
|
||||
|
||||
counted_fields: Final = {
|
||||
|
|
@ -1667,7 +1679,11 @@ async def responses_input_tokens(
|
|||
}
|
||||
token_fields: Final = resolve_counted_tool_fields(counted_fields, llm_router, user_api_key_dict.team_id)
|
||||
if isinstance(token_fields, ProxyException):
|
||||
return _responses_invalid_request_response(message=token_fields.message, param=token_fields.param, code=None)
|
||||
return rejected_token_count(
|
||||
_responses_invalid_request_response(message=token_fields.message, param=token_fields.param, code=None),
|
||||
litellm_call_id,
|
||||
model_name,
|
||||
)
|
||||
|
||||
try:
|
||||
payload: Final[_TokenCountPayload] = {
|
||||
|
|
@ -1680,27 +1696,44 @@ async def responses_input_tokens(
|
|||
}
|
||||
token_request: Final = TokenCountRequest.model_validate(payload)
|
||||
except ValidationError as e:
|
||||
return _responses_invalid_request_response(
|
||||
message=first_validation_error(e), param=str(e.errors(include_url=False)[0]["loc"][0]), code=None
|
||||
return rejected_token_count(
|
||||
_responses_invalid_request_response(
|
||||
message=first_validation_error(e), param=str(e.errors(include_url=False)[0]["loc"][0]), code=None
|
||||
),
|
||||
litellm_call_id,
|
||||
model_name,
|
||||
)
|
||||
except Exception as e:
|
||||
return _responses_invalid_request_response(
|
||||
message=f"Invalid request for token counting: {e}", param=None, code=None
|
||||
return rejected_token_count(
|
||||
_responses_invalid_request_response(
|
||||
message=f"Invalid request for token counting: {e}", param=None, code=None
|
||||
),
|
||||
litellm_call_id,
|
||||
model_name,
|
||||
)
|
||||
|
||||
request_fields: Final = _PROXY_SERVER_REQUEST_ADAPTER.validate_python(data)
|
||||
responses_request: Final = await _with_managed_file_id_mapping(
|
||||
_with_authorized_previous_response(
|
||||
{**{name: value for name, value in request_fields.items() if name not in counted_fields}, **token_fields},
|
||||
response_id_security,
|
||||
try:
|
||||
responses_request: Final = await _with_managed_file_id_mapping(
|
||||
_with_authorized_previous_response(
|
||||
{
|
||||
**{name: value for name, value in request_fields.items() if name not in counted_fields},
|
||||
**token_fields,
|
||||
},
|
||||
response_id_security,
|
||||
user_api_key_dict,
|
||||
),
|
||||
managed_files,
|
||||
user_api_key_dict,
|
||||
),
|
||||
managed_files,
|
||||
user_api_key_dict,
|
||||
)
|
||||
)
|
||||
except ProxyException as e:
|
||||
raise rejected_token_count(e, litellm_call_id, model_name)
|
||||
try:
|
||||
token_response: Final = await token_counter(
|
||||
request=token_request, call_endpoint=True, responses_request=responses_request
|
||||
request=token_request,
|
||||
call_endpoint=True,
|
||||
responses_request=responses_request,
|
||||
litellm_call_id=litellm_call_id,
|
||||
)
|
||||
except ProxyException as e:
|
||||
if e.type != _PROVIDER_COUNT_ERROR_TYPE:
|
||||
|
|
|
|||
|
|
@ -256,7 +256,6 @@ REQUIRED_PRESENT_BODY_PARAMS_BY_ROUTE: Final[Mapping[str, tuple[str, ...]]] = Ma
|
|||
"avideo_create_character": ("name", "video"),
|
||||
"acreate_container": ("name",),
|
||||
"aupload_container_file": ("file",),
|
||||
"acreate_agent": ("name",),
|
||||
"acreate_interaction": ("input",),
|
||||
"acreate_eval": ("data_source_config", "testing_criteria"),
|
||||
"acreate_run": ("data_source",),
|
||||
|
|
@ -264,7 +263,7 @@ REQUIRED_PRESENT_BODY_PARAMS_BY_ROUTE: Final[Mapping[str, tuple[str, ...]]] = Ma
|
|||
)
|
||||
|
||||
REQUIRED_ONE_OF_BODY_PARAMS_BY_ROUTE: Final[Mapping[str, tuple[str, str]]] = MappingProxyType(
|
||||
{"acreate_interaction": ("model", "agent")}
|
||||
{"acreate_interaction": ("model", "agent"), "acreate_agent": ("id", "name")}
|
||||
)
|
||||
|
||||
JSON_OBJECT_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
|
||||
|
|
|
|||
|
|
@ -2341,6 +2341,7 @@ async def acancel_responses(
|
|||
timeout: float | httpx.Timeout | None = None,
|
||||
# LiteLLM specific params,
|
||||
custom_llm_provider: str | None = None,
|
||||
litellm_metadata: Mapping[str, object] | None = None,
|
||||
**kwargs,
|
||||
) -> ResponsesAPIResponse:
|
||||
"""
|
||||
|
|
@ -2365,6 +2366,7 @@ async def acancel_responses(
|
|||
cancel_responses,
|
||||
response_id=response_id,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_metadata=litellm_metadata,
|
||||
extra_headers=extra_headers,
|
||||
extra_query=extra_query,
|
||||
extra_body=extra_body,
|
||||
|
|
@ -2380,7 +2382,12 @@ async def acancel_responses(
|
|||
response = await init_response
|
||||
else:
|
||||
response = init_response
|
||||
return response
|
||||
|
||||
return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id(
|
||||
responses_api_response=response,
|
||||
litellm_metadata=litellm_metadata,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=None,
|
||||
|
|
@ -2402,6 +2409,7 @@ def cancel_responses(
|
|||
timeout: float | httpx.Timeout | None = None,
|
||||
# LiteLLM specific params,
|
||||
custom_llm_provider: str | None = None,
|
||||
litellm_metadata: Mapping[str, object] | None = None,
|
||||
**kwargs,
|
||||
) -> ResponsesAPIResponse | Coroutine[object, object, ResponsesAPIResponse]:
|
||||
"""
|
||||
|
|
@ -2471,7 +2479,13 @@ def cancel_responses(
|
|||
shared_session=kwargs.get("shared_session"),
|
||||
)
|
||||
|
||||
return response
|
||||
if not isinstance(response, ResponsesAPIResponse):
|
||||
return response
|
||||
return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id(
|
||||
responses_api_response=response,
|
||||
litellm_metadata=litellm_metadata,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=None,
|
||||
|
|
|
|||
|
|
@ -41,7 +41,11 @@ from litellm.exceptions import MidStreamFallbackError, RateLimitError
|
|||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.core_helpers import process_response_headers
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base
|
||||
from litellm.litellm_core_utils.internal_call_metadata import without_budget_reservation
|
||||
from litellm.litellm_core_utils.internal_call_metadata import (
|
||||
is_background_response,
|
||||
requests_background,
|
||||
without_budget_reservation,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
|
||||
|
|
@ -65,7 +69,7 @@ from litellm.types.llms.openai import (
|
|||
ResponsesAPIStreamingResponse,
|
||||
)
|
||||
from litellm.types.responses.main import GenericResponseOutputItem, OutputText
|
||||
from litellm.types.utils import CallTypes
|
||||
from litellm.types.utils import CallTypes, ModelResponse
|
||||
from litellm.utils import async_post_call_success_deployment_hook
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -288,7 +292,7 @@ def stream_error_status_and_message(error_obj: object) -> tuple[int, str]:
|
|||
return _status_code_for_error_fields(error_type, error_code), message
|
||||
|
||||
|
||||
def _map_stream_error_to_exception(
|
||||
def map_stream_error_to_exception(
|
||||
error_obj: object,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
|
|
@ -578,6 +582,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
self.request_data.get("input"),
|
||||
self.request_data,
|
||||
self._generated_content + self._generated_tool_arguments,
|
||||
use_default_image_token_count=False,
|
||||
)
|
||||
if _estimate_wanted
|
||||
else None
|
||||
|
|
@ -731,7 +736,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
self._earlier_rounds_spend = earlier
|
||||
|
||||
def _map_error_event_exception(self, error_obj: object) -> Exception:
|
||||
return _map_stream_error_to_exception(
|
||||
return map_stream_error_to_exception(
|
||||
error_obj,
|
||||
self.model or "",
|
||||
self.custom_llm_provider or "",
|
||||
|
|
@ -981,6 +986,39 @@ class BaseResponsesAPIStreamingIterator:
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
def _record_partial_usage_for_failure(self) -> None:
|
||||
generated_text: Final = self._generated_content + self._generated_tool_arguments
|
||||
if (
|
||||
not generated_text
|
||||
or requests_background(self.request_data)
|
||||
or is_background_response(self._created_response)
|
||||
or self._completed_response_logged
|
||||
or self.logging_obj.model_call_details.get("combined_usage_object") is not None
|
||||
):
|
||||
return
|
||||
estimate: Final = _estimate_usage_safely(
|
||||
self.model or "",
|
||||
self.request_data.get("input"),
|
||||
self.request_data,
|
||||
generated_text,
|
||||
use_default_image_token_count=True,
|
||||
)
|
||||
if estimate is None:
|
||||
return
|
||||
usage: Final = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( # pyright: ignore[reportPrivateUsage] # same shared transform the success path above uses
|
||||
estimate
|
||||
)
|
||||
served_tier: Final[object] = getattr(self._created_response, "service_tier", None)
|
||||
assembled: Final = ModelResponse(
|
||||
model=self.model, usage=usage, service_tier=served_tier if isinstance(served_tier, str) else None
|
||||
)
|
||||
if self._earlier_rounds_spend is None:
|
||||
self.logging_obj.record_assembled_response_for_failure(assembled)
|
||||
return
|
||||
self.logging_obj.record_failure_after_earlier_rounds(
|
||||
self._earlier_rounds_spend, usage, self.logging_obj.response_cost_calculator(result=assembled) or 0.0
|
||||
)
|
||||
|
||||
def _handle_failure(self, exception: Exception):
|
||||
"""
|
||||
Trigger failure handlers before bubbling the exception.
|
||||
|
|
@ -990,6 +1028,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
if self._failure_handled:
|
||||
return
|
||||
self._failure_handled = True
|
||||
self._record_partial_usage_for_failure()
|
||||
|
||||
traceback_exception: Final = traceback.format_exc()
|
||||
end_time: Final = datetime.now()
|
||||
|
|
@ -1060,15 +1099,20 @@ class BaseResponsesAPIStreamingIterator:
|
|||
the same shape the success handler assembles for a natural end of stream.
|
||||
Its response is the response.created payload (it carries the served
|
||||
service_tier) plus the text and tool arguments relayed so far, with counted
|
||||
usage since the provider never sent any. None when nothing was relayed yet
|
||||
or the stream already reached its terminal event and billed itself.
|
||||
usage since the provider never sent any. None when nothing was relayed yet,
|
||||
the stream already reached its terminal event and billed itself, or its failure
|
||||
log already billed the relayed output.
|
||||
"""
|
||||
created: Final = self._created_response
|
||||
generated: Final = self._generated_content + self._generated_tool_arguments
|
||||
if self.completed_response is not None or created is None or not generated:
|
||||
if created is None or not generated or self._billed_without_disconnect():
|
||||
return None
|
||||
usage: Final = _estimate_usage_safely(
|
||||
self.model or "", self.request_data.get("input"), self.request_data, generated
|
||||
self.model or "",
|
||||
self.request_data.get("input"),
|
||||
self.request_data,
|
||||
generated,
|
||||
use_default_image_token_count=False,
|
||||
)
|
||||
if usage is None:
|
||||
return None
|
||||
|
|
@ -1082,6 +1126,9 @@ class BaseResponsesAPIStreamingIterator:
|
|||
stamp_responses_usage_cost(partial, self.logging_obj)
|
||||
return ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=partial)
|
||||
|
||||
def _billed_without_disconnect(self) -> bool:
|
||||
return self.completed_response is not None or self._failure_handled
|
||||
|
||||
def _note_yielded_event(self, event: ResponsesAPIStreamingResponse) -> None:
|
||||
self._yielded_first_chunk = True
|
||||
if event.type not in PRE_OUTPUT_LIFECYCLE_EVENT_TYPES:
|
||||
|
|
@ -1218,6 +1265,9 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
raise StopAsyncIteration from e
|
||||
self._raise_for_transport_error(e)
|
||||
except httpx.TimeoutException as e:
|
||||
if self.completed_response is not None:
|
||||
self.finished = True
|
||||
raise StopAsyncIteration from e
|
||||
self._raise_for_transport_timeout(e)
|
||||
except httpx.HTTPError as e:
|
||||
# Handle HTTP errors
|
||||
|
|
@ -1303,6 +1353,9 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
raise StopIteration from e
|
||||
self._raise_for_transport_error(e)
|
||||
except httpx.TimeoutException as e:
|
||||
if self.completed_response is not None:
|
||||
self.finished = True
|
||||
raise StopIteration from e
|
||||
self._raise_for_transport_timeout(e)
|
||||
except httpx.HTTPError as e:
|
||||
# Handle HTTP errors
|
||||
|
|
@ -1727,6 +1780,8 @@ def _estimate_usage_from_text(
|
|||
request_input: object,
|
||||
responses_api_request: Mapping[str, object],
|
||||
generated_text: str,
|
||||
*,
|
||||
use_default_image_token_count: bool = False,
|
||||
) -> ResponseAPIUsage:
|
||||
messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # the transformer's signature is partially untyped
|
||||
input=request_input, # pyright: ignore[reportArgumentType] # the raw Responses API input is a str or ResponseInputParam list, matching the helper's declared union
|
||||
|
|
@ -1734,7 +1789,7 @@ def _estimate_usage_from_text(
|
|||
only_since_compaction=True,
|
||||
)
|
||||
input_tokens: Final = litellm.token_counter( # pyright: ignore[reportUnknownMemberType] # token_counter's public signature is untyped
|
||||
model=model, messages=messages
|
||||
model=model, messages=messages, use_default_image_token_count=use_default_image_token_count
|
||||
)
|
||||
output_tokens: Final = litellm.token_counter( # pyright: ignore[reportUnknownMemberType] # token_counter's public signature is untyped
|
||||
model=model, text=generated_text, count_response_tokens=True
|
||||
|
|
@ -1751,6 +1806,8 @@ def _estimate_usage_safely(
|
|||
request_input: object,
|
||||
responses_api_request: Mapping[str, object],
|
||||
generated_text: str,
|
||||
*,
|
||||
use_default_image_token_count: bool,
|
||||
) -> ResponseAPIUsage | None:
|
||||
try:
|
||||
return _estimate_usage_from_text(
|
||||
|
|
@ -1758,6 +1815,7 @@ def _estimate_usage_safely(
|
|||
request_input=request_input,
|
||||
responses_api_request=responses_api_request,
|
||||
generated_text=generated_text,
|
||||
use_default_image_token_count=use_default_image_token_count,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.debug("Could not estimate usage from stream text, billing $0: %s", e)
|
||||
|
|
@ -2276,7 +2334,7 @@ class _ResponsesWsTurn:
|
|||
)
|
||||
if failed_event is None:
|
||||
return None
|
||||
return _map_stream_error_to_exception(
|
||||
return map_stream_error_to_exception(
|
||||
_ws_event_error(failed_event), model, custom_llm_provider, logging_obj=connection_logging_obj
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
import base64
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Callable, Collection, Iterable, Mapping, Sequence
|
||||
from functools import reduce
|
||||
from typing import Any, Final, Optional, TypeVar, Union, cast, get_type_hints, overload
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, Optional, TypeVar, Union, cast, get_args, get_type_hints, overload
|
||||
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from typing_extensions import TypedDict as ExtraItemsTypedDict
|
||||
|
|
@ -14,6 +16,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or
|
|||
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path
|
||||
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
InputTokensDetails,
|
||||
|
|
@ -29,6 +32,7 @@ from litellm.types.responses.main import DecodedResponseId
|
|||
from litellm.types.utils import (
|
||||
CacheCreationTokenDetails,
|
||||
CompletionTokensDetailsWrapper,
|
||||
ModelInfo,
|
||||
PromptTokensDetailsWrapper,
|
||||
ProviderSpecificHeader,
|
||||
SpecialEnums,
|
||||
|
|
@ -399,7 +403,7 @@ class ResponsesAPIRequestUtils:
|
|||
def update_responses_api_response_id_with_model_id(
|
||||
responses_api_response: ResponsesAPIResponse,
|
||||
custom_llm_provider: str | None,
|
||||
litellm_metadata: dict[str, object] | None = None,
|
||||
litellm_metadata: Mapping[str, object] | None = None,
|
||||
) -> ResponsesAPIResponse:
|
||||
...
|
||||
|
||||
|
|
@ -408,7 +412,7 @@ class ResponsesAPIRequestUtils:
|
|||
def update_responses_api_response_id_with_model_id(
|
||||
responses_api_response: dict[str, object],
|
||||
custom_llm_provider: str | None,
|
||||
litellm_metadata: dict[str, object] | None = None,
|
||||
litellm_metadata: Mapping[str, object] | None = None,
|
||||
) -> dict[str, object]:
|
||||
...
|
||||
|
||||
|
|
@ -418,7 +422,7 @@ class ResponsesAPIRequestUtils:
|
|||
def update_responses_api_response_id_with_model_id(
|
||||
responses_api_response: ResponsesAPIResponse | dict[str, Any],
|
||||
custom_llm_provider: str | None,
|
||||
litellm_metadata: dict[str, Any] | None = None,
|
||||
litellm_metadata: Mapping[str, Any] | None = None,
|
||||
) -> ResponsesAPIResponse | dict[str, object]:
|
||||
"""Update the responses_api_response_id with model_id and custom_llm_provider.
|
||||
|
||||
|
|
@ -1110,7 +1114,7 @@ class ResponsesAPIRequestUtils:
|
|||
def _update_container_ids_in_response(
|
||||
responses_api_response: ResponsesAPIResponse | dict[str, object],
|
||||
custom_llm_provider: str | None,
|
||||
litellm_metadata: dict[str, Any] | None = None,
|
||||
litellm_metadata: Mapping[str, Any] | None = None,
|
||||
) -> ResponsesAPIResponse | dict[str, object]:
|
||||
"""Encode container IDs in the response output with provider/model info.
|
||||
|
||||
|
|
@ -1382,3 +1386,98 @@ def auto_reasoning_summary(model: str, custom_llm_provider: object) -> Reasoning
|
|||
provider, model.removeprefix(f"{provider}/")
|
||||
)
|
||||
return "detailed" if responses_config is None else responses_config.auto_reasoning_summary()
|
||||
|
||||
|
||||
def opt_in_auto_reasoning_summary(model: str | None, custom_llm_provider: object) -> ReasoningSummaryLevel | None:
|
||||
if not (litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"):
|
||||
return None
|
||||
return "detailed" if model is None else auto_reasoning_summary(model, custom_llm_provider)
|
||||
|
||||
|
||||
def reasoning_with_summary_default(
|
||||
reasoning: Mapping[str, object], summary_alias: object, model: str, custom_llm_provider: object
|
||||
) -> Mapping[str, object]:
|
||||
if "summary" in reasoning:
|
||||
return {**reasoning}
|
||||
summary: Final = (
|
||||
summary_alias if summary_alias is not None else opt_in_auto_reasoning_summary(model, custom_llm_provider)
|
||||
)
|
||||
return {**reasoning} if summary is None else {**reasoning, "summary": summary}
|
||||
|
||||
|
||||
_SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None))
|
||||
|
||||
|
||||
def _openai_model_info(model: str) -> ModelInfo | None:
|
||||
from litellm.utils import ModelInfoLookupMiss, lookup_model_info_for_registration
|
||||
|
||||
model_info: Final = lookup_model_info_for_registration(model.removeprefix("responses/"), "openai")
|
||||
return None if isinstance(model_info, ModelInfoLookupMiss) else model_info
|
||||
|
||||
|
||||
def _declared_values(values: Sequence[str] | None) -> frozenset[str | None] | None:
|
||||
return None if values is None else frozenset(values)
|
||||
|
||||
|
||||
def _openai_reasoning_field_values(model: str) -> Mapping[str, frozenset[str | None] | None]:
|
||||
model_info: Final = _openai_model_info(model)
|
||||
if model_info is None:
|
||||
return MappingProxyType({"summary": _SUMMARY_VALUES, "mode": None, "context": None})
|
||||
if not model_info.get("supports_reasoning"):
|
||||
return MappingProxyType({})
|
||||
return MappingProxyType(
|
||||
{
|
||||
"summary": _SUMMARY_VALUES,
|
||||
"mode": _declared_values(model_info.get("reasoning_modes")),
|
||||
"context": _declared_values(model_info.get("reasoning_contexts")),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _openai_accepts_reasoning_field(
|
||||
key: str, value: object, values_by_field: Mapping[str, frozenset[str | None] | None]
|
||||
) -> bool:
|
||||
if key == "effort":
|
||||
return True
|
||||
if key not in values_by_field:
|
||||
return False
|
||||
allowed: Final = values_by_field[key]
|
||||
return allowed is None or (isinstance(value, str | None) and value in allowed)
|
||||
|
||||
|
||||
def _openai_accepted_reasoning(reasoning: Mapping[str, object], model: str) -> Mapping[str, object]:
|
||||
values_by_field: Final = _openai_reasoning_field_values(model)
|
||||
return {
|
||||
key: value for key, value in reasoning.items() if _openai_accepts_reasoning_field(key, value, values_by_field)
|
||||
}
|
||||
|
||||
|
||||
def _additional_drop_paths(additional_drop_params: object) -> tuple[str, ...]:
|
||||
if additional_drop_params.__class__ is not list:
|
||||
return ()
|
||||
return tuple(path for path in iter_items(additional_drop_params) if isinstance(path, str))
|
||||
|
||||
|
||||
def openai_bridge_reasoning(
|
||||
*,
|
||||
requested: Mapping[str, object],
|
||||
effort_kept: bool,
|
||||
summary_alias: object,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
additional_drop_params: object,
|
||||
) -> Mapping[str, object] | None:
|
||||
drop_paths: Final = _additional_drop_paths(additional_drop_params)
|
||||
if "reasoning_effort" in drop_paths:
|
||||
return None
|
||||
kept: Final = requested if effort_kept else {key: value for key, value in requested.items() if key != "effort"}
|
||||
with_summary: Final = reasoning_with_summary_default(kept, summary_alias, model, "openai")
|
||||
accepted: Final = (
|
||||
_openai_accepted_reasoning(with_summary, model)
|
||||
if drop_params and OpenAIGPTConfig.is_openai_catalog_model(model)
|
||||
else {**with_summary}
|
||||
)
|
||||
remaining: Final = as_str_mapping(
|
||||
_apply_nested_drop_params({"reasoning_effort": accepted}, list(drop_paths)).get("reasoning_effort")
|
||||
)
|
||||
return remaining or None
|
||||
|
|
|
|||
|
|
@ -216,6 +216,8 @@ from litellm.router_utils.common_utils import (
|
|||
deployments_the_caller_may_use,
|
||||
filter_team_based_models,
|
||||
filter_web_search_deployments,
|
||||
format_content_policy_fallback_hint,
|
||||
format_content_policy_no_fallback_note,
|
||||
format_fallback_outcome_message,
|
||||
format_no_fallback_group_message,
|
||||
get_request_team_id,
|
||||
|
|
@ -631,6 +633,7 @@ _DEPLOYMENT_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | GenericLite
|
|||
_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object)
|
||||
_DEPLOYMENT_SELECTED_EVENT: Final = "litellm.request.deployment_selected"
|
||||
_CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
|
||||
_DEPLOYMENTS_ADAPTER: Final = TypeAdapter(Sequence[Mapping[str, object]])
|
||||
|
||||
|
||||
def _extend_trailing_text_block(blocks: Sequence[Mapping[str, object]], text: str) -> Sequence[Mapping[str, object]]:
|
||||
|
|
@ -692,6 +695,12 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj
|
|||
return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {})
|
||||
|
||||
|
||||
def _every_deployment_refused(
|
||||
deployments: Sequence[Mapping[str, object]], refused_deployment_ids: tuple[str, ...]
|
||||
) -> bool:
|
||||
return all(_configured_model_info(deployment).get("id") in refused_deployment_ids for deployment in deployments)
|
||||
|
||||
|
||||
def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]:
|
||||
if silent_model is None:
|
||||
return ()
|
||||
|
|
@ -832,6 +841,10 @@ def _request_model_group(kwargs: Mapping[str, object]) -> str | None:
|
|||
return model_group if isinstance(model_group, str) else None
|
||||
|
||||
|
||||
def _mid_stream_error_before_output(error: "MidStreamFallbackError") -> bool:
|
||||
return not error.generated_content or error.is_pre_first_chunk
|
||||
|
||||
|
||||
def _mid_stream_retry_trigger(error: "MidStreamFallbackError") -> Exception:
|
||||
"""The provider's own error, which is what the retry policy and should_retry_this_error classify."""
|
||||
return error.original_exception if error.original_exception is not None else error
|
||||
|
|
@ -962,6 +975,10 @@ def _responses_stream_holds_event(item: object, held_event_count: int) -> bool:
|
|||
return getattr(item, "type", None) in PRE_OUTPUT_LIFECYCLE_EVENT_TYPES
|
||||
|
||||
|
||||
def _responses_event_announces_background_run(item: object) -> bool:
|
||||
return getattr(getattr(item, "response", None), "background", None) is True
|
||||
|
||||
|
||||
class FallbackAwareAnthropicMessagesStream:
|
||||
"""
|
||||
Bare async generators can't carry the `_hidden_params` attribute the
|
||||
|
|
@ -4127,11 +4144,16 @@ class Router:
|
|||
|
||||
async def stream_with_fallbacks():
|
||||
held_lifecycle_events: tuple[object, ...] = () # rebind-ok: flushed at first output, dropped on fallback
|
||||
response_id_exposed = False # rebind-ok: set once, before an early-released lifecycle event is yielded
|
||||
try:
|
||||
async for item in cast( # cast-ok: response iterators are async streams
|
||||
AsyncIterator[object], source_iterator
|
||||
):
|
||||
if _responses_stream_holds_event(item, len(held_lifecycle_events)):
|
||||
if not response_id_exposed and self._aresponses_background_stream_can_release(
|
||||
item, source_iterator, initial_kwargs
|
||||
):
|
||||
response_id_exposed = True
|
||||
if not response_id_exposed and _responses_stream_holds_event(item, len(held_lifecycle_events)):
|
||||
held_lifecycle_events = (*held_lifecycle_events, item)
|
||||
continue
|
||||
for held_event in held_lifecycle_events:
|
||||
|
|
@ -4141,6 +4163,8 @@ class Router:
|
|||
for held_event in held_lifecycle_events:
|
||||
yield held_event
|
||||
except MidStreamFallbackError as e:
|
||||
if response_id_exposed:
|
||||
self._raise_released_stream_error(e, initial_kwargs)
|
||||
async with contextlib.aclosing(
|
||||
self._aresponses_fallback_attempt(
|
||||
e, source_iterator, initial_kwargs, wrapper.adopt_fallback_headers, held_lifecycle_events
|
||||
|
|
@ -4166,6 +4190,54 @@ class Router:
|
|||
wrapper: Final = FallbackResponsesStreamWrapper(stream_with_fallbacks())
|
||||
return wrapper
|
||||
|
||||
def _aresponses_background_stream_can_release(
|
||||
self,
|
||||
item: object,
|
||||
source_iterator: "BaseResponsesAPIStreamingIterator",
|
||||
kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
|
||||
|
||||
if (
|
||||
type(source_iterator) is not ResponsesAPIStreamingIterator
|
||||
or source_iterator.custom_llm_provider != "openai"
|
||||
):
|
||||
return False
|
||||
if not _responses_event_announces_background_run(item):
|
||||
return False
|
||||
model_group: Final = kwargs.get("model")
|
||||
return isinstance(model_group, str) and not self._stream_can_fall_back(model_group, kwargs)
|
||||
|
||||
def _raise_released_stream_error(
|
||||
self, e: "MidStreamFallbackError", initial_kwargs: Mapping[str, object]
|
||||
) -> NoReturn:
|
||||
original: Final = e.original_exception
|
||||
model_group: Final = initial_kwargs.get("model")
|
||||
if (
|
||||
litellm.expose_router_debug_in_errors
|
||||
and isinstance(original, litellm.ContentPolicyViolationError)
|
||||
and isinstance(model_group, str)
|
||||
and not fallbacks_disabled_for_request(initial_kwargs)
|
||||
and _mid_stream_error_before_output(e)
|
||||
):
|
||||
hop_depth: Final = initial_kwargs.get("fallback_depth")
|
||||
original.message += format_content_policy_no_fallback_note(
|
||||
model_group,
|
||||
fallback_lookup_groups(initial_kwargs, model_group),
|
||||
_request_fallback_list(
|
||||
initial_kwargs,
|
||||
"content_policy_fallbacks",
|
||||
cast("list[object] | None", self.content_policy_fallbacks), # cast-ok: untyped router attribute
|
||||
),
|
||||
_request_fallback_list(
|
||||
initial_kwargs,
|
||||
"fallbacks",
|
||||
cast("list[object] | None", self.fallbacks), # cast-ok: untyped router attribute
|
||||
),
|
||||
isinstance(hop_depth, int) and hop_depth > 0,
|
||||
)
|
||||
Router._raise_original_mid_stream_error(e)
|
||||
|
||||
def _responses_continuation(
|
||||
self,
|
||||
e: "MidStreamFallbackError",
|
||||
|
|
@ -4175,7 +4247,7 @@ class Router:
|
|||
"""What a fallback continues from once a Responses stream broke after text, raising the primary's error
|
||||
when the request constrains its output or the stream began a tool call, since a continued JSON document or
|
||||
tool call would not parse, and when a content filter cut it, so no other model finishes filtered text."""
|
||||
if not e.generated_content or e.is_pre_first_chunk:
|
||||
if _mid_stream_error_before_output(e):
|
||||
return None
|
||||
if source_iterator.started_tool_call or isinstance(e.original_exception, litellm.ContentPolicyViolationError):
|
||||
Router._raise_original_mid_stream_error(e)
|
||||
|
|
@ -4706,7 +4778,13 @@ class Router:
|
|||
@staticmethod
|
||||
def _stamp_retry_skip_deployment_id(exception: Exception, kwargs: Mapping[str, object]) -> None:
|
||||
effective_model_info: Final = kwargs.get("model_info")
|
||||
deployment_id: Final = effective_model_info.get("id") if isinstance(effective_model_info, Mapping) else None
|
||||
if not isinstance(effective_model_info, Mapping):
|
||||
return
|
||||
model_info: Final = _MODEL_INFO_ADAPTER.validate_python(effective_model_info)
|
||||
configured_deployment_id: Final = (
|
||||
model_info.get("original_model_id") if isinstance(exception, litellm.UnsupportedEndpointError) else None
|
||||
)
|
||||
deployment_id: Final = configured_deployment_id or model_info.get("id")
|
||||
if isinstance(deployment_id, str) and deployment_id:
|
||||
exception.retry_skip_deployment_id = deployment_id # pyright: ignore[reportAttributeAccessIssue] # dynamic stamp, read by _deployment_ids_to_skip_on_retry
|
||||
|
||||
|
|
@ -6378,6 +6456,9 @@ class Router:
|
|||
hop_kwargs: Final = mid_stream_fallback_hop_kwargs(
|
||||
model=model, original_generic_function=original_generic_function, controls=controls, kwargs=kwargs
|
||||
)
|
||||
shared_logging: Final = kwargs.get("litellm_logging_obj")
|
||||
if isinstance(shared_logging, LiteLLMLogging):
|
||||
shared_logging.carry_failed_attempt_usage()
|
||||
response: Final = await self._ageneric_api_call_with_fallbacks_helper(
|
||||
model=model, original_generic_function=original_generic_function, **kwargs
|
||||
)
|
||||
|
|
@ -6487,7 +6568,7 @@ class Router:
|
|||
model: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group
|
||||
has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither a retry nor a fallback can take over
|
||||
self._anthropic_messages_stream_can_retry(initial_kwargs)
|
||||
or self._anthropic_messages_stream_can_fall_back(model, initial_kwargs)
|
||||
or self._stream_can_fall_back(model, initial_kwargs)
|
||||
)
|
||||
buffered_lifecycle_chunks: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline
|
||||
try:
|
||||
|
|
@ -6710,7 +6791,7 @@ class Router:
|
|||
budget, _ = self._stream_retry_budget(frame_error, kwargs)
|
||||
if budget > attempted_retries_for_request(kwargs):
|
||||
return frame_error
|
||||
if self._anthropic_messages_stream_can_fall_back(model_group, kwargs):
|
||||
if self._stream_can_fall_back(model_group, kwargs):
|
||||
return frame_error
|
||||
return None
|
||||
|
||||
|
|
@ -8725,7 +8806,9 @@ class Router:
|
|||
)
|
||||
return response
|
||||
else:
|
||||
error_message = f"model={model_group}. content_policy_fallback={mask_sensitive_structure(content_policy_fallbacks)}. fallbacks={mask_sensitive_structure(fallbacks)}.\n\nSet 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbacks"
|
||||
error_message = format_content_policy_fallback_hint(
|
||||
model_group, content_policy_fallbacks, fallbacks
|
||||
)
|
||||
verbose_router_logger.info(
|
||||
msg=f"Got 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting \
|
||||
to fallbacks, if available.{error_message}"
|
||||
|
|
@ -9048,6 +9131,10 @@ class Router:
|
|||
)
|
||||
if first_skipped_ids:
|
||||
kwargs["_retry_skipped_deployment_ids"] = first_skipped_ids # rebind-ok: the next attempt reads it
|
||||
if isinstance(original_exception, litellm.UnsupportedEndpointError) and _every_deployment_refused(
|
||||
_DEPLOYMENTS_ADAPTER.validate_python(_healthy_deployments), first_skipped_ids
|
||||
):
|
||||
raise
|
||||
else:
|
||||
raise
|
||||
|
||||
|
|
@ -9127,6 +9214,10 @@ class Router:
|
|||
)
|
||||
if skipped_ids:
|
||||
kwargs["_retry_skipped_deployment_ids"] = skipped_ids # rebind-ok: the next attempt reads it
|
||||
if isinstance(e, litellm.UnsupportedEndpointError) and _every_deployment_refused(
|
||||
_DEPLOYMENTS_ADAPTER.validate_python(_healthy_deployments), skipped_ids
|
||||
):
|
||||
raise
|
||||
_timeout = self._time_to_sleep_before_retry(
|
||||
e=e,
|
||||
remaining_retries=remaining_retries,
|
||||
|
|
@ -9223,7 +9314,7 @@ class Router:
|
|||
status_code: Final = getattr(error, "status_code", None)
|
||||
if status_code is not None and not litellm.should_retry(status_code):
|
||||
# 401/403 are special cases - allow retry if multiple deployments exist (handled below)
|
||||
if status_code not in (401, 403):
|
||||
if status_code not in (401, 403) and not isinstance(error, litellm.UnsupportedEndpointError):
|
||||
raise error
|
||||
|
||||
if isinstance(error, litellm.NotFoundError):
|
||||
|
|
@ -9873,7 +9964,7 @@ class Router:
|
|||
)
|
||||
return has_unattempted_fallback_target(resolved, kwargs)
|
||||
|
||||
def _anthropic_messages_order_levels(self, model_group: str, kwargs: Mapping[str, Any]) -> tuple[int, ...]:
|
||||
def _stream_order_levels(self, model_group: str, kwargs: Mapping[str, Any]) -> tuple[int, ...]:
|
||||
"""
|
||||
The distinct deployment order levels the fallback dispatcher would see for this request,
|
||||
computed the same way: the tier a pre-routing hook selected wins over the requested group.
|
||||
|
|
@ -9891,7 +9982,7 @@ class Router:
|
|||
)
|
||||
)
|
||||
|
||||
def _anthropic_messages_stream_can_fall_back(self, model_group: str, kwargs: Mapping[str, Any]) -> bool:
|
||||
def _stream_can_fall_back(self, model_group: str, kwargs: Mapping[str, Any]) -> bool:
|
||||
"""
|
||||
Whether async_function_with_fallbacks_common_utils could still route a
|
||||
MidStreamFallbackError somewhere for this request (order levels, weighted
|
||||
|
|
@ -9903,7 +9994,7 @@ class Router:
|
|||
return False
|
||||
if self.enable_weighted_failover:
|
||||
return True
|
||||
order_levels: Final = self._anthropic_messages_order_levels(model_group, kwargs)
|
||||
order_levels: Final = self._stream_order_levels(model_group, kwargs)
|
||||
if len(order_levels) > 1:
|
||||
current_target: Final = kwargs.get("_target_order")
|
||||
skip_up_to: Final = current_target if current_target is not None else order_levels[0]
|
||||
|
|
@ -10276,28 +10367,38 @@ class Router:
|
|||
model_info[field] = backend_value
|
||||
|
||||
@staticmethod
|
||||
def _cost_map_backend_model(deployment: Deployment) -> str:
|
||||
model_info_base_model: Final = deployment.model_info.base_model
|
||||
def _cost_map_backend_model(deployment: Deployment | DeploymentTypedDict) -> str:
|
||||
model_info_base_model: Final = (
|
||||
deployment.model_info.base_model
|
||||
if isinstance(deployment, Deployment)
|
||||
else _configured_model_info(deployment).get("base_model")
|
||||
)
|
||||
if isinstance(model_info_base_model, str) and model_info_base_model:
|
||||
return model_info_base_model
|
||||
params_base_model: Final = deployment.litellm_params.get("base_model")
|
||||
params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(
|
||||
deployment.litellm_params.model_dump(exclude_none=True)
|
||||
if isinstance(deployment, Deployment)
|
||||
else deployment["litellm_params"]
|
||||
)
|
||||
params_base_model: Final = params.get("base_model")
|
||||
if isinstance(params_base_model, str) and params_base_model:
|
||||
return params_base_model
|
||||
params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(deployment.litellm_params.model_dump(exclude_none=True))
|
||||
model: Final = (
|
||||
deployment.litellm_params.model
|
||||
if isinstance(deployment, Deployment)
|
||||
else deployment["litellm_params"].get("model", "")
|
||||
)
|
||||
invoked_model: Final = params.get("model_id")
|
||||
if (
|
||||
isinstance(invoked_model, str)
|
||||
and invoked_model
|
||||
and (
|
||||
deployment.litellm_params.custom_llm_provider == "bedrock"
|
||||
or deployment.litellm_params.model.startswith("bedrock/")
|
||||
)
|
||||
and (params.get("custom_llm_provider") == "bedrock" or model.startswith("bedrock/"))
|
||||
):
|
||||
profile_model: Final = invoked_model if invoked_model.startswith("bedrock/") else f"bedrock/{invoked_model}"
|
||||
profile_info: Final = lookup_model_info_for_registration(model=profile_model, custom_llm_provider="bedrock")
|
||||
if not isinstance(profile_info, ModelInfoLookupMiss):
|
||||
return profile_model
|
||||
return deployment.litellm_params.model
|
||||
return model
|
||||
|
||||
@staticmethod
|
||||
def _inherit_builtin_service_tier_pricing(
|
||||
|
|
@ -13857,7 +13958,9 @@ class Router:
|
|||
]
|
||||
return named or self._get_wildcard_deployments(model_name=model_group)
|
||||
|
||||
def resolved_litellm_models(self, model_name: str, team_id: str | None = None) -> tuple[str, ...]:
|
||||
def resolved_litellm_models(
|
||||
self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False
|
||||
) -> tuple[str, ...]:
|
||||
"""The provider model strings `model_name` can actually be served by on this proxy.
|
||||
|
||||
`get_model_list` composes every channel the request path itself uses (exact name,
|
||||
|
|
@ -13870,6 +13973,12 @@ class Router:
|
|||
deployment behind it - so the fallback for an empty result is the caller's policy,
|
||||
never this function's.
|
||||
"""
|
||||
if use_base_model:
|
||||
return tuple(
|
||||
backend_model
|
||||
for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or ()
|
||||
if (backend_model := self._cost_map_backend_model(deployment))
|
||||
)
|
||||
return tuple(
|
||||
litellm_model
|
||||
for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or ()
|
||||
|
|
@ -14439,7 +14548,6 @@ class Router:
|
|||
Filter out model in model group, if:
|
||||
|
||||
- model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
|
||||
- filter models above rpm limits
|
||||
- if region given, filter out models not in that region / unknown region
|
||||
- [TODO] function call and model doesn't support function calling
|
||||
"""
|
||||
|
|
@ -14454,8 +14562,6 @@ class Router:
|
|||
|
||||
_context_window_error = False
|
||||
_potential_error_str = ""
|
||||
_rate_limit_error = False
|
||||
parent_otel_span: Final = get_parent_otel_span_from_kwargs(request_kwargs)
|
||||
|
||||
has_countable_input: Final = (messages is not None or input is not None) and not compaction_pending(
|
||||
request_kwargs
|
||||
|
|
@ -14479,13 +14585,6 @@ class Router:
|
|||
else frozenset()
|
||||
)
|
||||
|
||||
## get model group RPM ##
|
||||
dt: Final = get_utc_datetime()
|
||||
current_minute: Final = dt.strftime("%H-%M")
|
||||
rpm_key: Final = f"{model}:rpm:{current_minute}"
|
||||
model_group_cache: Final = (
|
||||
self.cache.get_cache(key=rpm_key, local_only=True, parent_otel_span=parent_otel_span) or {}
|
||||
) # check the in-memory cache used by lowest_latency and usage-based routing. Only check the local cache.
|
||||
for idx, deployment in enumerate(_returned_deployments):
|
||||
# Cache nested dict access to avoid repeated temporary dict allocations
|
||||
_litellm_params = deployment.get("litellm_params", {})
|
||||
|
|
@ -14508,24 +14607,6 @@ class Router:
|
|||
)
|
||||
continue
|
||||
|
||||
model_id = _model_info.get("id", "")
|
||||
## RPM CHECK ##
|
||||
### get local router cache ###
|
||||
current_request_cache_local = (
|
||||
self.cache.get_cache(key=model_id, local_only=True, parent_otel_span=parent_otel_span) or 0
|
||||
)
|
||||
### get usage based cache ###
|
||||
if isinstance(model_group_cache, dict) and self.routing_strategy != "usage-based-routing-v2":
|
||||
model_group_cache[model_id] = model_group_cache.get(model_id, 0)
|
||||
|
||||
current_request = max(current_request_cache_local, model_group_cache[model_id])
|
||||
|
||||
if isinstance(_litellm_params, dict) and _litellm_params.get("rpm", None) is not None:
|
||||
if isinstance(_litellm_params["rpm"], int) and _litellm_params["rpm"] <= current_request:
|
||||
invalid_model_indices.add(idx)
|
||||
_rate_limit_error = True
|
||||
continue
|
||||
|
||||
## REGION CHECK ##
|
||||
if request_kwargs is not None and request_kwargs.get("allowed_model_region") is not None:
|
||||
allowed_model_region = request_kwargs.get("allowed_model_region")
|
||||
|
|
@ -14562,12 +14643,6 @@ class Router:
|
|||
)
|
||||
|
||||
if len(excluded_indices) == len(_returned_deployments):
|
||||
"""
|
||||
- no healthy deployments available b/c context window checks or rate limit error
|
||||
|
||||
- First check for rate limit errors (if this is true, it means the model passed the context window check but failed the rate limit check)
|
||||
"""
|
||||
|
||||
group_deployments: Final = (
|
||||
tuple(request_deployments)
|
||||
if request_deployments is not None
|
||||
|
|
@ -14589,12 +14664,7 @@ class Router:
|
|||
),
|
||||
model=model,
|
||||
)
|
||||
if _rate_limit_error is True: # allow generic fallback logic to take place
|
||||
raise RouterRateLimitErrorBasic(
|
||||
model=model,
|
||||
)
|
||||
|
||||
elif _context_window_error is True:
|
||||
if _context_window_error is True:
|
||||
raise litellm.ContextWindowExceededError(
|
||||
message=f"litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.\n{_potential_error_str}",
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import random
|
|||
import re
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from itertools import accumulate, chain, islice, takewhile
|
||||
from threading import Lock
|
||||
|
|
@ -1390,6 +1391,11 @@ def _session_affinity_cache_value(model: str, tier: ComplexityTier | str | None)
|
|||
return {"model": model, "tier": tier_value}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _JevConnectionError:
|
||||
message: str
|
||||
|
||||
|
||||
class ComplexityRouter(CustomLogger):
|
||||
"""
|
||||
Complexity router that classifies requests and routes to appropriate models.
|
||||
|
|
@ -1405,7 +1411,7 @@ class ComplexityRouter(CustomLogger):
|
|||
- Question complexity (multiple questions)
|
||||
"""
|
||||
|
||||
def _build_jev_client(self, config: OpenSourceClassifierConfig) -> JevClassifierClient:
|
||||
def _build_jev_client(self, config: OpenSourceClassifierConfig) -> JevClassifierClient | _JevConnectionError:
|
||||
if config.deployment_name is not None:
|
||||
return RouterJevClassifierClient(self.litellm_router_instance, config.deployment_name)
|
||||
if config.provider in ("laya", "bespoke"):
|
||||
|
|
@ -1426,12 +1432,22 @@ class ComplexityRouter(CustomLogger):
|
|||
http_client=get_async_httpx_client(httpxSpecialProvider.PassThroughEndpoint),
|
||||
provider="databricks",
|
||||
)
|
||||
api_key: Final = config.api_key or get_secret_str("TYPESAFE_API_KEY")
|
||||
if not api_key:
|
||||
raise ValueError(
|
||||
api_key: Final = (
|
||||
get_secret_str(config.api_key)
|
||||
if config.api_key is not None and config.api_key.startswith("os.environ/")
|
||||
else config.api_key or get_secret_str("TYPESAFE_API_KEY")
|
||||
)
|
||||
if not api_key or not api_key.strip():
|
||||
return _JevConnectionError(
|
||||
"opensource_classifier_config.api_key or TYPESAFE_API_KEY is required for classifier_type 'oss_classifier'"
|
||||
)
|
||||
api_base: Final = config.api_base or get_secret_str("TYPESAFE_API_BASE") or "https://api.typesafe.ai"
|
||||
api_base: Final = (
|
||||
get_secret_str(config.api_base)
|
||||
if config.api_base is not None and config.api_base.startswith("os.environ/")
|
||||
else config.api_base or get_secret_str("TYPESAFE_API_BASE") or "https://api.typesafe.ai"
|
||||
)
|
||||
if not api_base or not api_base.strip():
|
||||
return _JevConnectionError("opensource_classifier_config.api_base must resolve to a non-empty value")
|
||||
return HttpJevClassifierClient(
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
|
|
@ -1474,13 +1490,18 @@ class ComplexityRouter(CustomLogger):
|
|||
self.config.default_model = default_model
|
||||
|
||||
jev_config: Final = self.config.opensource_classifier_config
|
||||
self._jev_client: JevClassifierClient | None = (
|
||||
jev_client_result: Final = (
|
||||
jev_client
|
||||
if jev_client is not None
|
||||
else self._build_jev_client(jev_config)
|
||||
if self.config.classifier_type == "oss_classifier" and jev_config is not None
|
||||
else None
|
||||
)
|
||||
match jev_client_result:
|
||||
case _JevConnectionError(message=message):
|
||||
raise ValueError(message)
|
||||
case _:
|
||||
self._jev_client: JevClassifierClient | None = jev_client_result
|
||||
|
||||
self._tier_affinity_config = hashlib.sha256(
|
||||
self.config.model_dump_json(include=MappingProxyType({"tiers": True, "tier_model_configs": True})).encode()
|
||||
|
|
|
|||
|
|
@ -39,6 +39,9 @@ _DEPLOYMENT_ROWS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] =
|
|||
tuple[Mapping[str, object], ...]
|
||||
)
|
||||
_DEPLOYMENT_IDS_ADAPTER: Final = TypeAdapter(tuple[str, ...])
|
||||
_FALLBACK_ENTRIES_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter(
|
||||
tuple[Mapping[str, object], ...]
|
||||
)
|
||||
|
||||
|
||||
def with_admin_api_key_secret_reference(deployment: Deployment) -> Deployment:
|
||||
|
|
@ -256,6 +259,40 @@ def format_fallback_outcome_message(
|
|||
return f"{lead} Fallback model group(s) configured: {targets}."
|
||||
|
||||
|
||||
def format_content_policy_fallback_hint(
|
||||
model_group: str | None, content_policy_fallbacks: object, fallbacks: object
|
||||
) -> str:
|
||||
return (
|
||||
f"model={model_group}. content_policy_fallback={mask_sensitive_structure(content_policy_fallbacks)}. "
|
||||
f"fallbacks={mask_sensitive_structure(fallbacks)}.\n\nSet 'content_policy_fallback' - "
|
||||
"https://docs.litellm.ai/docs/routing#fallbacks"
|
||||
)
|
||||
|
||||
|
||||
def _as_fallback_entries(fallbacks: object) -> tuple[Mapping[str, object], ...] | None:
|
||||
try:
|
||||
return _FALLBACK_ENTRIES_ADAPTER.validate_python(fallbacks)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def format_content_policy_no_fallback_note(
|
||||
model_group: str,
|
||||
lookup_groups: tuple[str, ...],
|
||||
content_policy_fallbacks: object,
|
||||
fallbacks: object,
|
||||
nested_fallback_hop: bool,
|
||||
) -> str:
|
||||
outcome: Final = "" if nested_fallback_hop else format_fallback_outcome_message(model_group, None, "")
|
||||
if content_policy_fallbacks is not None:
|
||||
return outcome
|
||||
hint: Final = f"\n{format_content_policy_fallback_hint(model_group, content_policy_fallbacks, fallbacks)}"
|
||||
fallback_entries: Final = _as_fallback_entries(fallbacks) if fallbacks is not None else None
|
||||
if fallback_entries is None or not lookup_groups or nested_fallback_hop:
|
||||
return hint + outcome
|
||||
return hint + format_no_fallback_group_message(lookup_groups, fallback_entries)
|
||||
|
||||
|
||||
def get_litellm_params_sensitive_credential_hash(litellm_params: dict) -> str:
|
||||
"""
|
||||
Hash of the credential params, used for mapping the file id to the right model
|
||||
|
|
|
|||
|
|
@ -356,7 +356,7 @@ def _should_run_cooldown_logic(
|
|||
)
|
||||
return False
|
||||
|
||||
if isinstance(original_exception, litellm.MalformedRequestFieldError):
|
||||
if isinstance(original_exception, (litellm.MalformedRequestFieldError, litellm.UnsupportedEndpointError)):
|
||||
verbose_router_logger.debug(
|
||||
"Should Not Run Cooldown Logic: litellm rejected the request before calling the deployment"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -261,7 +261,10 @@ def is_bad_request_without_fallback(exception: Exception, fallback_on_unsupporte
|
|||
return False
|
||||
if isinstance(exception, litellm.UnsupportedParamsError):
|
||||
return True
|
||||
if isinstance(exception, (litellm.ContextWindowExceededError, litellm.ContentPolicyViolationError)):
|
||||
if isinstance(
|
||||
exception,
|
||||
(litellm.ContextWindowExceededError, litellm.ContentPolicyViolationError, litellm.UnsupportedEndpointError),
|
||||
):
|
||||
return False
|
||||
return exception.status_code == 400 and bool(getattr(exception, "llm_provider", ""))
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from litellm.exceptions import (
|
|||
RateLimitError,
|
||||
ServiceUnavailableError,
|
||||
Timeout,
|
||||
UnsupportedEndpointError,
|
||||
)
|
||||
from litellm.types.router import RetryPolicy
|
||||
|
||||
|
|
@ -59,6 +60,8 @@ def get_num_retries_from_retry_policy(
|
|||
policy: Final = resolve_retry_policy(retry_policy, model_group, model_group_retry_policy)
|
||||
if policy is None:
|
||||
return None
|
||||
if isinstance(exception, UnsupportedEndpointError):
|
||||
return policy.DefaultRetries
|
||||
by_class: Final = (
|
||||
_RETRIES_BY_EXCEPTION_TYPE[cls](policy) for cls in type(exception).__mro__ if cls in _RETRIES_BY_EXCEPTION_TYPE
|
||||
)
|
||||
|
|
|
|||
|
|
@ -46,6 +46,12 @@
|
|||
- A failure after execution starts is terminal and never triggers fallback
|
||||
- Creating an unstarted native coroutine acquires no clients or credentials and captures no execution context
|
||||
|
||||
# Native extension
|
||||
|
||||
- `_native` ships in every wheel and the sdist builds it, so it is always importable
|
||||
- Import it with `from litellm.rust_bridge import _native` and use the attributes `_native.pyi` declares
|
||||
- Never add a `None` branch, `getattr` default or stand-in class for a missing extension
|
||||
|
||||
# Verification
|
||||
|
||||
- Check cross-language behavior against a freshly installed extension, covering unstarted-call release, setup ordering, exception identity, sync and async streams, cancellation and GC
|
||||
|
|
|
|||
|
|
@ -1,10 +1,5 @@
|
|||
"""LiteLLM Rust bridge package."""
|
||||
|
||||
from litellm.rust_bridge.configuration import rust
|
||||
from litellm.rust_bridge.loader import (
|
||||
get_native_bridge,
|
||||
native_bridge_available,
|
||||
reset_native_bridge_cache,
|
||||
)
|
||||
|
||||
__all__ = ["get_native_bridge", "native_bridge_available", "reset_native_bridge_cache", "rust"]
|
||||
__all__ = ["rust"]
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from __future__ import annotations
|
|||
from collections.abc import Callable
|
||||
from typing import Final, Generic, TypeVar
|
||||
|
||||
from litellm.rust_bridge.loader import get_native_bridge
|
||||
from litellm.rust_bridge import _native
|
||||
|
||||
BindingT = TypeVar("BindingT")
|
||||
|
||||
|
|
@ -18,32 +18,24 @@ _UNSET: Final = _Unset()
|
|||
class NativeBinding(Generic[BindingT]):
|
||||
"""Resolve one native attribute with an explicit, resettable test override."""
|
||||
|
||||
def __init__(self, attribute: str, *, validate: Callable[[object], BindingT | None]) -> None:
|
||||
def __init__(
|
||||
self, attribute: str, *, validate: Callable[[object], BindingT | None], source: object = _native
|
||||
) -> None:
|
||||
self._attribute: Final = attribute
|
||||
self._validate: Final = validate
|
||||
self._source: Final = source
|
||||
self._override: BindingT | None | _Unset = _UNSET
|
||||
|
||||
def load(self) -> BindingT | None:
|
||||
if not isinstance(self._override, _Unset):
|
||||
return self._override
|
||||
native: Final = get_native_bridge()
|
||||
if native is None:
|
||||
return None
|
||||
return self._validate(getattr(native, self._attribute, None))
|
||||
return self._validate(getattr(self._source, self._attribute, None))
|
||||
|
||||
def with_source(self, source: object) -> NativeBinding[BindingT]:
|
||||
return NativeBinding(self._attribute, validate=self._validate, source=source)
|
||||
|
||||
def override(self, value: BindingT | None) -> None:
|
||||
self._override = value
|
||||
|
||||
def reset(self) -> None:
|
||||
self._override = _UNSET
|
||||
|
||||
|
||||
def native_exception_types() -> tuple[type[BaseException], type[BaseException]] | None:
|
||||
native: Final = get_native_bridge()
|
||||
if native is None:
|
||||
return None
|
||||
declined: Final = getattr(native, "RustBridgeDeclined", None)
|
||||
upstream: Final = getattr(native, "RustUpstreamError", None)
|
||||
if not isinstance(declined, type) or not isinstance(upstream, type):
|
||||
return None
|
||||
return declined, upstream
|
||||
|
|
|
|||
|
|
@ -1,12 +1,10 @@
|
|||
import os
|
||||
from collections.abc import Awaitable, Mapping, Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Protocol, runtime_checkable
|
||||
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
from typing import Final
|
||||
|
||||
from litellm.constants import DEFAULT_AGENT_TRACING_RETENTION_DAYS, DEFAULT_CLICKHOUSE_DATABASE
|
||||
from litellm.rust_bridge.loader import get_native_bridge
|
||||
from litellm.rust_bridge import _native
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, repr=False)
|
||||
|
|
@ -16,27 +14,6 @@ class SpendStorageConfig:
|
|||
retention_days: int = DEFAULT_AGENT_TRACING_RETENTION_DAYS
|
||||
|
||||
|
||||
class NativeSpendConfig(Protocol):
|
||||
def __init__(self, database: str, url: str, retention_days: int) -> None: ...
|
||||
|
||||
|
||||
class NativeSpendStorage(Protocol):
|
||||
def __init__(self, config: NativeSpendConfig) -> None: ...
|
||||
def ensure_schema(self) -> Awaitable[None]: ...
|
||||
def insert_rows(self, rows: Sequence[Mapping[str, object]]) -> Awaitable[None]: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class NativeSpendBridge(Protocol):
|
||||
NativeClickHouseSpendConfig: type[NativeSpendConfig]
|
||||
NativeClickHouseSpendStorage: type[NativeSpendStorage]
|
||||
|
||||
|
||||
_NATIVE: Final[TypeAdapter[NativeSpendBridge]] = TypeAdapter(
|
||||
NativeSpendBridge, config=ConfigDict(arbitrary_types_allowed=True)
|
||||
)
|
||||
|
||||
|
||||
def spend_storage_config(environ: Mapping[str, str] = os.environ) -> SpendStorageConfig:
|
||||
url: Final = environ.get("CLICKHOUSE_URL")
|
||||
if not url:
|
||||
|
|
@ -53,12 +30,8 @@ def spend_storage_config(environ: Mapping[str, str] = os.environ) -> SpendStorag
|
|||
|
||||
class ClickHouseSpendStorage:
|
||||
def __init__(self, config: SpendStorageConfig) -> None:
|
||||
bridge: Final = get_native_bridge()
|
||||
if bridge is None:
|
||||
raise RuntimeError("ClickHouse spend logging requires the Rust extension")
|
||||
native: Final = _NATIVE.validate_python(bridge)
|
||||
self._native: Final = native.NativeClickHouseSpendStorage(
|
||||
native.NativeClickHouseSpendConfig(config.database, config.url, config.retention_days)
|
||||
self._native: Final = _native.NativeClickHouseSpendStorage(
|
||||
_native.NativeClickHouseSpendConfig(config.database, config.url, config.retention_days)
|
||||
)
|
||||
|
||||
async def ensure_schema(self) -> None:
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue