From 02a6a96decd5b1c3bec3bab07ae68dc3fcde55c3 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Fri, 30 Oct 2026 21:08:43 -0700 Subject: [PATCH 01/15] feat(accounting): add bounded transactional usage persistence (#49190) * feat(accounting): add bounded transactional usage persistence * fix(accounting): preserve usage flush and shutdown on Python 3.10 * test(accounting): cover batch rejection and partial shutdown recovery * fix(accounting): batch durable usage on a five-second interval * fix(accounting): drain queued full batches without timer delays * fix(accounting): isolate invalid producer submissions within usage batches * fix(accounting): await commit prerequisites within bounded usage admission --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/proxy/db/accounting_ledger.py | 83 ++++ litellm/proxy/db/exception_handler.py | 18 + litellm/proxy/spend_tracking/usage_meter.py | 268 ++++++++++++ .../database/test_accounting_ledger.py | 73 +++- tests/unit/proxy/db/test_accounting_ledger.py | 17 +- tests/unit/proxy/db/test_exception_handler.py | 30 ++ .../proxy/spend_tracking/test_usage_meter.py | 389 ++++++++++++++++++ 7 files changed, 876 insertions(+), 2 deletions(-) create mode 100644 litellm/proxy/spend_tracking/usage_meter.py create mode 100644 tests/unit/proxy/spend_tracking/test_usage_meter.py diff --git a/litellm/proxy/db/accounting_ledger.py b/litellm/proxy/db/accounting_ledger.py index 91f8307283e..e37ce8d14d4 100644 --- a/litellm/proxy/db/accounting_ledger.py +++ b/litellm/proxy/db/accounting_ledger.py @@ -31,6 +31,13 @@ class AccountingConflictError(ValueError): pass +@dataclass(frozen=True, slots=True) +class UsageWrite: + event: InboundUsageEvent | OutboundUsageEvent + envelope: str + targets: tuple[DeliveryTarget, ...] = () + + _Transaction: TypeAlias = AbstractAsyncContextManager["Prisma"] @@ -107,6 +114,35 @@ SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE kind = 'cost' AND source_ev _LOCK_EVENT: Final = """ SELECT payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 FOR UPDATE """ +_INSERT_USAGE_BATCH: Final = """ +INSERT INTO "LiteLLM_AccountingEvent" + (event_id, kind, schema_version, payload_hash, occurred_at, payload) +SELECT event_id, kind::"AccountingEventKind", 1, payload_hash, occurred_at::timestamp, payload +FROM jsonb_to_recordset($1::jsonb) AS records( + event_id text, kind text, payload_hash text, occurred_at text, payload jsonb) +ORDER BY event_id +ON CONFLICT (event_id) DO NOTHING RETURNING event_id +""" +_LOCK_USAGE_BATCH: Final = """ +SELECT event_id, payload_hash FROM "LiteLLM_AccountingEvent" +WHERE event_id IN (SELECT value FROM jsonb_array_elements_text($1::jsonb)) +ORDER BY event_id FOR UPDATE +""" +_LOAD_USAGE_BATCH: Final = """ +SELECT event.payload, ARRAY( + SELECT target::text FROM "LiteLLM_AccountingDelivery" delivery + WHERE delivery.event_id = event.event_id ORDER BY target::text +) AS targets +FROM "LiteLLM_AccountingEvent" event +WHERE event.event_id IN (SELECT value FROM jsonb_array_elements_text($1::jsonb)) + AND event.kind = 'outbound_usage' AND event.schema_version = 1 +""" +_INSERT_USAGE_DELIVERIES: Final = """ +INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version) +SELECT records.event_id, destination.value::"AccountingDeliveryTarget", 1 +FROM jsonb_to_recordset($1::jsonb) AS records(event_id text, targets jsonb) +CROSS JOIN LATERAL jsonb_array_elements_text(records.targets) AS destination +""" _COMPLETED_FAMILY: Final = """ root.source_event_id IS NULL AND root.kind IN ('inbound_usage', 'outbound_usage') AND root.recorded_at < $1::timestamp @@ -199,6 +235,53 @@ class AccountingLedger: return cls(transaction) + async def insert_usage_batch(self, writes: tuple[UsageWrite, ...]) -> None: + records: Final = tuple(_prepare(write.event, write.targets, write.envelope, 1, 1) for write in writes) + if not records: + return + unique: Final = {record.event_id: record for record in records} + if any(record.payload_hash != unique[record.event_id].payload_hash for record in records): + raise AccountingConflictError("Batch contains conflicting copies of a usage identity") + batch: Final = _json( + tuple( + { + "event_id": record.event_id, + "kind": record.kind.value, + "payload_hash": record.payload_hash, + "occurred_at": record.occurred_at.astimezone(timezone.utc).isoformat(), + "payload": _JSON.validate_json(record.payload), + } + for record in unique.values() + ) + ) + async with self._transaction() as db: + inserted: Final = _STRINGS.validate_python(await db.query_raw(_INSERT_USAGE_BATCH, batch)) + accepted: Final = _STRINGS.validate_python(await db.query_raw(_LOCK_USAGE_BATCH, _json(tuple(unique)))) + if len(accepted) != len(unique) or any( + row["payload_hash"] != unique[row["event_id"]].payload_hash for row in accepted + ): + raise AccountingConflictError("Usage identity conflicts with accepted content or delivery set") + deliveries: Final = _json( + tuple({"event_id": row["event_id"], "targets": unique[row["event_id"]].targets} for row in inserted) + ) + await db.execute_raw(_INSERT_USAGE_DELIVERIES, deliveries) + + async def load_usage_batch(self, event_ids: tuple[str, ...]) -> tuple[UsageWrite, ...]: + if not event_ids: + return () + async with self._transaction() as db: + rows: Final = TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python( + await db.query_raw(_LOAD_USAGE_BATCH, _json(event_ids)) + ) + return tuple( + UsageWrite( + OutboundUsageEvent.model_validate(_JSON.validate_python(row["payload"])["event"]), + _json(row["payload"]), + TypeAdapter(tuple[DeliveryTarget, ...]).validate_python(row["targets"]), + ) + for row in rows + ) + async def insert_event( self, event: AccountingEvent, diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index baf53604ea5..13a14e95e56 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -72,6 +72,22 @@ def _exception_types(*candidates: object) -> tuple[type[BaseException], ...]: return tuple(c for c in candidates if isinstance(c, type) and issubclass(c, BaseException)) +def _prisma_error_code(e: Exception) -> str | None: + import prisma.engine.errors + + if isinstance(e, _exception_types(prisma.errors.DataError)): + code: Final[object] = getattr(e, "code", None) + return code if isinstance(code, str) else None + if not isinstance(e, _exception_types(prisma.engine.errors.EngineRequestError)): + return None + try: + body: Final = _DATABASE_ERROR_META.validate_json(str(e).partition(": ")[2]) + except ValidationError: + return None + engine_code: Final = body.get("error_code") + return engine_code if isinstance(engine_code, str) else None + + class PrismaDBExceptionHandler: """ Class to handle DB Exceptions or Connection Errors @@ -206,6 +222,8 @@ class PrismaDBExceptionHandler: return True if PrismaDBExceptionHandler.is_database_capacity_error(e): return False + if _prisma_error_code(e) == "P1017": + return True if isinstance(e, _exception_types(prisma.errors.PrismaError)): error_message: Final = str(e).lower() connection_keywords: Final = ( diff --git a/litellm/proxy/spend_tracking/usage_meter.py b/litellm/proxy/spend_tracking/usage_meter.py new file mode 100644 index 00000000000..3bc7572b476 --- /dev/null +++ b/litellm/proxy/spend_tracking/usage_meter.py @@ -0,0 +1,268 @@ +from __future__ import annotations + +import asyncio +from collections import deque +from collections.abc import Awaitable, Callable +from concurrent.futures import Future +from dataclasses import dataclass +from itertools import accumulate, chain, count, takewhile +from threading import Lock +from typing import Final + +from pydantic import BaseModel, ConfigDict, Field + +from litellm._logging import verbose_proxy_logger +from litellm.accounting import ( + DeliveryTarget, + Dimension, + DimensionName, + InboundUsageEvent, + OutboundUsageEvent, +) +from litellm.proxy.db.accounting_ledger import AccountingConflictError, UsageWrite + +_DAILY_TARGETS: Final = ( + (DimensionName.KEY, DeliveryTarget.DAILY_USER_SPEND), + (DimensionName.USER, DeliveryTarget.DAILY_USER_SPEND), + (DimensionName.TEAM, DeliveryTarget.DAILY_TEAM_SPEND), + (DimensionName.ORGANIZATION, DeliveryTarget.DAILY_ORGANIZATION_SPEND), + (DimensionName.END_USER, DeliveryTarget.DAILY_END_USER_SPEND), + (DimensionName.AGENT, DeliveryTarget.DAILY_AGENT_SPEND), + (DimensionName.TAG, DeliveryTarget.DAILY_TAG_SPEND), + (DimensionName.MODEL, DeliveryTarget.DAILY_MODEL_USAGE), + (DimensionName.TOOL, DeliveryTarget.DAILY_TOOL_SPEND), +) + + +def usage_delivery_targets(dimensions: tuple[Dimension, ...]) -> tuple[DeliveryTarget, ...]: + names: Final = frozenset(dimension.name for dimension in dimensions) + return tuple(dict.fromkeys(target for dimension, target in _DAILY_TARGETS if dimension in names)) + + +class UsageCaptureSettings(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + batch_size: int = Field(default=1000, ge=2, le=1000) + flush_interval_seconds: float = Field(default=5.0, gt=0, le=10) + max_pending_bytes: int = Field(default=16 * 1024 * 1024, ge=1024) + max_pending_records: int = Field(default=10000, ge=2) + + +class UsageCaptureError(RuntimeError): + pass + + +@dataclass(frozen=True, slots=True) +class _Pending: + writes: tuple[UsageWrite, ...] + size: int + committed: Future[None] + finished: asyncio.Event + prerequisites: tuple[asyncio.Future[None], ...] + + +async def _wait_for_batch(wakeup: asyncio.Event, timeout: float) -> None: + try: + await asyncio.wait_for(wakeup.wait(), timeout) + except asyncio.TimeoutError: + return + + +def _report_failure(receipt: Future[None]) -> None: + if not receipt.cancelled() and receipt.exception() is not None: + verbose_proxy_logger.error("Usage capture did not acknowledge commit", exc_info=receipt.exception()) + + +def _retryable_persistence_failure(error: Exception) -> bool: + from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain + + outage: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(error) + return (outage is not None and not PrismaDBExceptionHandler.is_permanent_database_fault(outage)) or any( + isinstance(cause, Exception) + and ( + PrismaDBExceptionHandler.is_deadlock_error(cause) + or PrismaDBExceptionHandler.is_lock_timeout_error(cause) + or PrismaDBExceptionHandler.postgres_sqlstate(cause) == "40001" + ) + for cause in exception_chain(error) + ) + + +def _isolatable_persistence_failure(error: Exception) -> bool: + from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain + + return isinstance(error, AccountingConflictError) or any( + isinstance(cause, Exception) and (PrismaDBExceptionHandler.postgres_sqlstate(cause) or "").startswith("22") + for cause in exception_chain(error) + ) + + +class DurableUsageMeter: + def __init__( + self, + write_batch: Callable[[tuple[UsageWrite, ...]], Awaitable[None]], + settings: UsageCaptureSettings, + *, + wait_for_batch: Callable[[asyncio.Event, float], Awaitable[None]] = _wait_for_batch, + retry_delay: Callable[[float], Awaitable[None]] = asyncio.sleep, + ) -> None: + self._write_batch: Final = write_batch + self.settings: Final = settings + self._wait_for_batch: Final = wait_for_batch + self._retry_delay: Final = retry_delay + self._loop: Final = asyncio.get_running_loop() + self._admission: Final = Lock() + self._queue: Final[deque[_Pending]] = deque() # mutable-ok: one bounded producer/consumer queue + self._wakeup: Final = asyncio.Event() + self._task: asyncio.Task[None] | None = None + self._closed = False + self.pending_bytes = 0 + self.pending_records = 0 + + def submit( + self, writes: tuple[UsageWrite, ...], *, prerequisites: tuple[asyncio.Future[None], ...] = () + ) -> asyncio.Future[None]: + size: Final = sum(len(write.envelope.encode("utf-8")) for write in writes) + with self._admission: + if ( + self._closed + or not writes + or len(writes) > self.settings.batch_size + or self.pending_bytes + size > self.settings.max_pending_bytes + or self.pending_records + len(writes) > self.settings.max_pending_records + ): + raise UsageCaptureError("Usage capture admission rejected: closed, empty or capacity exhausted") + receipt: Final = Future[None]() + receipt.set_running_or_notify_cancel() + receipt.add_done_callback(_report_failure) + self._queue.append(_Pending(writes, size, receipt, asyncio.Event(), prerequisites)) + self.pending_bytes += size + self.pending_records += len(writes) + self._loop.call_soon_threadsafe(self._ensure_worker) + wrapped: Final = asyncio.wrap_future(receipt, loop=self._loop) + wrapped.add_done_callback(lambda done: None if done.cancelled() else done.exception()) + return wrapped + + def _ensure_worker(self) -> None: + if self.pending_records >= self.settings.batch_size or self._queue and self._queue[0].prerequisites: + self._wakeup.set() + if self._task is None or self._task.done(): + self._task = asyncio.create_task(self._run(), name="accounting usage capture") + + async def record(self, event: InboundUsageEvent | OutboundUsageEvent) -> None: + envelope: Final = '{"event":' + event.model_dump_json() + "}" + await asyncio.shield(self.submit((UsageWrite(event, envelope, usage_delivery_targets(event.dimensions)),))) + + async def flush(self) -> None: + self._ensure_worker() + self._wakeup.set() + if self._task is not None: + await asyncio.shield(self._task) + + async def close(self) -> None: + with self._admission: + self._closed = True + await self.flush() + + async def stop(self, timeout: float) -> None: + with self._admission: + self._closed = True + try: + await asyncio.wait_for(self.close(), timeout) + except asyncio.TimeoutError: + if self._task is not None: + self._task.cancel() + await asyncio.gather(self._task, return_exceptions=True) + while self._queue: + self._finish( + self._take_batch(), UsageCaptureError("Usage capture stopped before commit acknowledgement") + ) + verbose_proxy_logger.error("Usage capture shutdown deadline exhausted; commit acknowledgement incomplete") + + def _take_batch(self) -> tuple[_Pending, ...]: + with self._admission: + if self._queue[0].prerequisites: + return (self._queue.popleft(),) + independent: Final = takewhile(lambda item: not item.prerequisites, self._queue) + sizes: Final = accumulate(len(item.writes) for item in independent) + count: Final = sum(1 for _ in takewhile(lambda size: size <= self.settings.batch_size, sizes)) + return tuple(self._queue.popleft() for _ in range(count)) + + async def _persist(self, writes: tuple[UsageWrite, ...]) -> None: + for attempt in count(): + try: + await self._write_batch(writes) + except Exception as error: # noqa: BLE001 # retry only classified database failures, preserving frozen writes + if not _retryable_persistence_failure(error): + raise + verbose_proxy_logger.warning("Usage persistence unavailable; retaining admitted batch for retry") + await self._retry_delay(min(10.0, 0.2 * 2 ** min(attempt, 6))) + else: + return + + async def _commit(self, batch: tuple[_Pending, ...]) -> None: + writes: Final = tuple(chain.from_iterable(item.writes for item in batch)) + try: + if batch[0].prerequisites: + outcomes: Final = await asyncio.gather( + *(asyncio.shield(receipt) for receipt in batch[0].prerequisites), return_exceptions=True + ) + for outcome in outcomes: + if isinstance(outcome, BaseException): + raise UsageCaptureError("Usage dependency did not acknowledge commit") from outcome + await self._persist(writes) + except asyncio.CancelledError: + self._finish(batch, UsageCaptureError("Usage commit acknowledgement interrupted")) + raise + except Exception as error: # noqa: BLE001 # isolate data faults, surface systemic failures to all producers + if len(batch) > 1 and _isolatable_persistence_failure(error): + try: + for item in batch: + await self._commit((item,)) + except asyncio.CancelledError: + self._finish(batch, UsageCaptureError("Usage commit acknowledgement interrupted")) + raise + return + self._finish( + batch, + UsageCaptureError("Usage identity conflicts with accepted content") + if isinstance(error, AccountingConflictError) + else error, + ) + else: + self._finish(batch, None) + + def _finish(self, batch: tuple[_Pending, ...], error: Exception | None) -> None: + for item in batch: + if item.finished.is_set(): + continue + item.finished.set() + with self._admission: + self.pending_bytes -= item.size + self.pending_records -= len(item.writes) + if item.committed.done(): + continue + if error is None: + item.committed.set_result(None) + else: + item.committed.set_exception(error) + + async def _run(self) -> None: + try: + while self._queue: + await self._wait_for_batch(self._wakeup, self.settings.flush_interval_seconds) + self._wakeup.clear() + await self._commit(self._take_batch()) + if ( + self._closed + or self.pending_records >= self.settings.batch_size + or self._queue + and self._queue[0].prerequisites + ): + self._wakeup.set() + except (Exception, asyncio.CancelledError): # noqa: BLE001 # reject queued receipts if the writer exits abnormally + while self._queue: + self._finish( + self._take_batch(), UsageCaptureError("Usage capture stopped before commit acknowledgement") + ) + raise diff --git a/tests/integration/database/test_accounting_ledger.py b/tests/integration/database/test_accounting_ledger.py index 0c2ceb62eaf..86c2b325226 100644 --- a/tests/integration/database/test_accounting_ledger.py +++ b/tests/integration/database/test_accounting_ledger.py @@ -32,7 +32,7 @@ from litellm.accounting import ( PricingSnapshot, ) from litellm.proxy.db import accounting_ledger -from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingEvent, AccountingLedger +from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingEvent, AccountingLedger, UsageWrite from litellm.types.utils import CallTypes from tests.integration._support.database import scratch_database from tests.integration._support.prisma_clients import connected_proxy_prisma @@ -532,3 +532,74 @@ def test_family_cleanup_lookups_use_the_kind_source_index(ledger_database: str, name == "LiteLLM_AccountingEvent_kind_source_event_id_idx" and "source_event_id" in str(condition) for name, condition in conditions ), conditions + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["before_commit", "after_commit", "conflict"]) +async def test_usage_batch_is_atomic_and_replay_preserves_receipts( + writer: _Writer, ledger_database: str, failure: str +) -> None: + client, store = writer + events: Final = (_usage("caller", inbound=True), _usage("attempt")) + writes: Final = tuple(UsageWrite(event, _envelope(event), _TARGETS) for event in events) + + @asynccontextmanager + async def interrupted() -> AsyncGenerator[Prisma]: + async with client.writer_db.tx() as tx: + yield tx + if failure == "before_commit": + raise RuntimeError("injected pre-commit failure") + if failure == "after_commit": + raise RuntimeError("injected lost acknowledgement") + + if failure == "conflict": + await store.insert_usage_batch((writes[1],)) + _complete_and_age(ledger_database) + rejected: Final = (writes[0], UsageWrite(events[1], _envelope(events[1], context={"changed": True}), _TARGETS)) + with pytest.raises(AccountingConflictError): + await store.insert_usage_batch(rejected) + assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent"') == [("attempt",)] + else: + with pytest.raises(RuntimeError, match="injected"): + await AccountingLedger(interrupted).insert_usage_batch(writes) + count: Final = 2 if failure == "after_commit" else 0 + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(count,)] + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(count * len(_TARGETS),)] + await store.insert_usage_batch(writes) + recovered: Final = await store.load_usage_batch(tuple(event.event_id for event in events)) + assert len(recovered) == 1 + assert recovered[0].event == events[1] + assert json.loads(recovered[0].envelope) == json.loads(writes[1].envelope) + assert set(recovered[0].targets) == set(_TARGETS) + _complete_and_age(ledger_database) + before: Final = _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') + await store.insert_usage_batch((*writes, *writes)) + assert _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') == before + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(2,)] + + +@pytest.mark.asyncio +async def test_invalid_jsonb_submission_cannot_reject_another_producers_commit( + writer: _Writer, ledger_database: str +) -> None: + from prisma.errors import DataError + + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + + _, store = writer + invalid: Final = _usage("invalid") + valid: Final = _usage("valid") + meter: Final = DurableUsageMeter(store.insert_usage_batch, UsageCaptureSettings(batch_size=2)) + rejected: Final = meter.submit( + (UsageWrite(invalid, _envelope(invalid, context={"parent": "bad\x00text"}), _TARGETS),) + ) + accepted: Final = meter.submit((UsageWrite(valid, _envelope(valid), _TARGETS),)) + try: + with pytest.raises(DataError): + await rejected + await accepted + finally: + await meter.close() + assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent"') == [("valid",)] + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(len(_TARGETS),)] + assert (meter.pending_bytes, meter.pending_records) == (0, 0) diff --git a/tests/unit/proxy/db/test_accounting_ledger.py b/tests/unit/proxy/db/test_accounting_ledger.py index 3916fe869e6..6ca719ed849 100644 --- a/tests/unit/proxy/db/test_accounting_ledger.py +++ b/tests/unit/proxy/db/test_accounting_ledger.py @@ -6,7 +6,7 @@ from typing import Final, Literal, NoReturn import pytest from litellm.accounting import InboundUsageEvent, Metric, MetricName, Outcome -from litellm.proxy.db.accounting_ledger import AccountingLedger +from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingLedger, UsageWrite from litellm.types.utils import CallTypes @@ -101,3 +101,18 @@ async def test_unknown_delivery_target_is_rejected_before_database_access(event: ("unknown",), # pyright: ignore[reportArgumentType] # Exercise an unknown runtime target envelope=envelope, ) + + +@pytest.mark.asyncio +async def test_empty_usage_batches_do_not_open_a_transaction() -> None: + ledger: Final = AccountingLedger(_no_transaction) + await ledger.insert_usage_batch(()) + assert await ledger.load_usage_batch(()) == () + + +@pytest.mark.asyncio +async def test_conflicting_usage_within_one_batch_is_rejected_before_database_access(event: InboundUsageEvent) -> None: + original: Final = UsageWrite(event, json.dumps({"event": event.model_dump(mode="json")})) + changed: Final = UsageWrite(event, json.dumps({"event": event.model_dump(mode="json"), "pricing": {"rate": 2}})) + with pytest.raises(AccountingConflictError, match="conflicting copies"): + await AccountingLedger(_no_transaction).insert_usage_batch((original, changed)) diff --git a/tests/unit/proxy/db/test_exception_handler.py b/tests/unit/proxy/db/test_exception_handler.py index 75346ca08a6..d70d4403962 100644 --- a/tests/unit/proxy/db/test_exception_handler.py +++ b/tests/unit/proxy/db/test_exception_handler.py @@ -951,3 +951,33 @@ def test_is_lock_timeout_error_recognises_a_statement_cancelled_under_lock_timeo ) def test_is_lock_timeout_error_excludes_other_failures(error: Exception) -> None: assert PrismaDBExceptionHandler.is_lock_timeout_error(error) is False + + +@pytest.mark.parametrize( + ("body", "retryable"), + [ + ('{"error_code":"P1017","message":"Server has closed the connection."}', True), + ('{"error_code":"P1000","message":"Authentication failed."}', False), + ('{"error_code":"P2021","message":"Table does not exist."}', False), + ('{"error_code":"P9999","message":"Unknown engine failure."}', False), + ("invalid engine response", False), + ], +) +def test_engine_connection_closure_is_recoverable_without_allowing_unverified_auth( + body: str, + retryable: bool, +) -> None: + from prisma.http import Response + + error: Final = EngineRequestError(Response(httpx.Response(500)), body) + assert PrismaDBExceptionHandler.is_database_transport_error(error) is retryable + assert PrismaDBExceptionHandler.is_permanent_database_fault(error) is not retryable + assert PrismaDBExceptionHandler.is_database_connection_error(error) is False + + +def test_connection_closure_code_survives_prisma_data_error_wrapping() -> None: + error: Final = DataError( + data={"user_facing_error": {"error_code": "P1017", "message": "Server has closed the connection."}} + ) + assert PrismaDBExceptionHandler.is_database_transport_error(error) is True + assert PrismaDBExceptionHandler.is_permanent_database_fault(error) is False diff --git a/tests/unit/proxy/spend_tracking/test_usage_meter.py b/tests/unit/proxy/spend_tracking/test_usage_meter.py new file mode 100644 index 00000000000..de7d7c6591d --- /dev/null +++ b/tests/unit/proxy/spend_tracking/test_usage_meter.py @@ -0,0 +1,389 @@ +import asyncio +import json +from datetime import datetime, timezone +from decimal import Decimal +from typing import Final + +import pytest + +from litellm.accounting import InboundUsageEvent, Metric, MetricName, Outcome +from litellm.proxy.db.accounting_ledger import AccountingConflictError, UsageWrite +from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureError, UsageCaptureSettings +from litellm.types.utils import CallTypes + + +def _write(identity: str) -> UsageWrite: + event: Final = InboundUsageEvent( + event_id=identity, + request_id=identity, + call_type=CallTypes.acompletion, + started_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + ended_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + outcome=Outcome.SUCCEEDED, + dimensions=(), + metrics=(Metric(name=MetricName.REQUESTS, value=Decimal(1)),), + ) + return UsageWrite(event, json.dumps({"event": event.model_dump(mode="json")})) + + +class CommitBarrier: + def __init__(self, *, conflict: str | None = None, failure: Exception | None = None) -> None: + self.entered: Final = asyncio.Event() + self.release: Final = asyncio.Event() + self.committed: tuple[str, ...] = () + self.transactions = 0 + self.conflict: Final = conflict + self.failure: Final = failure + + async def write(self, writes: tuple[UsageWrite, ...]) -> None: + self.transactions += 1 + self.entered.set() + await self.release.wait() + if self.failure is not None: + raise self.failure + identities: Final = tuple(write.event.event_id for write in writes) + if self.conflict in identities: + raise AccountingConflictError("Conflicting test event") + self.committed += identities + + +async def _manual_flush(wakeup: asyncio.Event, timeout: float) -> None: + await wakeup.wait() + + +@pytest.mark.asyncio +async def test_receipts_wait_for_one_batched_commit_and_keep_inflight_capacity_reserved() -> None: + database: Final = CommitBarrier() + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2, max_pending_records=2)) + first: Final = meter.submit((_write("one"),)) + second: Final = meter.submit((_write("two"),)) + await database.entered.wait() + assert not first.done() and not second.done() + assert meter.pending_records == 2 + with pytest.raises(UsageCaptureError, match="capacity"): + meter.submit((_write("three"),)) + database.release.set() + await asyncio.gather(first, second) + await meter.close() + assert database.committed == ("one", "two") + assert database.transactions == 1 + assert (meter.pending_bytes, meter.pending_records) == (0, 0) + + +@pytest.mark.asyncio +async def test_byte_limit_rejects_new_work_without_evicting_admitted_work() -> None: + database: Final = CommitBarrier() + write: Final = _write("large" * 300) + size: Final = len(write.envelope.encode()) + meter: Final = DurableUsageMeter( + database.write, UsageCaptureSettings(max_pending_bytes=size), wait_for_batch=_manual_flush + ) + receipt: Final = meter.submit((write,)) + with pytest.raises(UsageCaptureError, match="capacity"): + meter.submit((_write("extra"),)) + assert meter.pending_bytes == size + database.release.set() + await meter.close() + await receipt + assert database.committed == (write.event.event_id,) + with pytest.raises(UsageCaptureError, match="closed"): + meter.submit((_write("after-close"),)) + + +@pytest.mark.asyncio +async def test_conflicting_group_rolls_back_and_valid_neighbor_still_commits() -> None: + database: Final = CommitBarrier(conflict="conflict") + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=3)) + rejected: Final = meter.submit((_write("caller"), _write("conflict"))) + accepted: Final = meter.submit((_write("valid"),)) + await database.entered.wait() + database.release.set() + with pytest.raises(UsageCaptureError, match="identity"): + await rejected + await accepted + await meter.close() + assert database.committed == ("valid",) + assert meter.pending_records == 0 + + +@pytest.mark.asyncio +async def test_permanent_persistence_failure_reaches_each_receipt_and_releases_capacity() -> None: + failure: Final = ValueError("invalid envelope") + database: Final = CommitBarrier(failure=failure) + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2)) + first: Final = meter.submit((_write("one"),)) + second: Final = meter.submit((_write("two"),)) + await database.entered.wait() + database.release.set() + results: Final = await asyncio.gather(first, second, return_exceptions=True) + await meter.close() + assert results == [failure, failure] + assert meter.pending_bytes == 0 + + +@pytest.mark.asyncio +async def test_cancelling_record_waiter_does_not_cancel_persistence() -> None: + database: Final = CommitBarrier() + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2)) + waiting: Final = asyncio.create_task(meter.record(_write("caller").event)) + await database.entered.wait() + waiting.cancel() + with pytest.raises(asyncio.CancelledError): + await waiting + database.release.set() + await meter.close() + assert database.committed == ("caller",) + assert meter.pending_records == 0 + + +@pytest.mark.asyncio +async def test_shutdown_deadline_rejects_unacknowledged_work_and_releases_capacity() -> None: + database: Final = CommitBarrier() + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2)) + first: Final = meter.submit((_write("in-flight"), _write("in-flight-two"))) + await database.entered.wait() + queued: Final = meter.submit((_write("queued"),)) + await meter.stop(0) + results: Final = await asyncio.gather(first, queued, return_exceptions=True) + assert all(isinstance(result, UsageCaptureError) for result in results) + assert meter.pending_records == 0 + assert meter.pending_bytes == 0 + assert database.committed == () + + +@pytest.mark.asyncio +async def test_shutdown_during_database_backoff_releases_reserved_capacity() -> None: + retrying: Final = asyncio.Event() + blocked: Final = asyncio.Event() + + async def unavailable(writes: tuple[UsageWrite, ...]) -> None: + raise ConnectionError("database unavailable") + + async def backoff(delay: float) -> None: + retrying.set() + await blocked.wait() + + meter: Final = DurableUsageMeter(unavailable, UsageCaptureSettings(batch_size=2), retry_delay=backoff) + receipt: Final = meter.submit((_write("one"), _write("two"))) + await retrying.wait() + await meter.stop(0) + with pytest.raises(UsageCaptureError, match="interrupted"): + await receipt + assert (meter.pending_records, meter.pending_bytes) == (0, 0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("wrapped", [False, True]) +async def test_closed_pooled_connection_retains_batch_until_automatic_recovery(wrapped: bool) -> None: + import httpx + from prisma.engine.errors import EngineRequestError + from prisma.http import Response + + database: Final = CommitBarrier() + attempts: Final = asyncio.Queue[tuple[UsageWrite, ...]]() + retrying: Final = asyncio.Event() + recovered: Final = asyncio.Event() + closure: Final = EngineRequestError( + Response(httpx.Response(500)), '{"error_code":"P1017","message":"Server has closed the connection."}' + ) + failure: Final = RuntimeError("ledger transaction failed") if wrapped else closure + if wrapped: + failure.__cause__ = closure + + async def persist(writes: tuple[UsageWrite, ...]) -> None: + attempts.put_nowait(writes) + if attempts.qsize() == 1: + raise failure + await database.write(writes) + + async def backoff(delay: float) -> None: + retrying.set() + await recovered.wait() + + meter: Final = DurableUsageMeter( + persist, UsageCaptureSettings(batch_size=2, max_pending_records=2), retry_delay=backoff + ) + writes: Final = (_write("caller"), _write("attempt")) + receipt: Final = meter.submit(writes) + retry_signal: Final = asyncio.create_task(retrying.wait()) + reserved: Final = meter.pending_bytes + try: + await asyncio.wait((receipt, retry_signal), return_when=asyncio.FIRST_COMPLETED) + assert not receipt.done(), "A dropped pooled connection must retain admitted usage for retry" + assert (meter.pending_records, meter.pending_bytes) == (2, reserved) + with pytest.raises(UsageCaptureError, match="capacity"): + meter.submit((_write("overflow"),)) + database.release.set() + recovered.set() + await receipt + assert attempts.get_nowait() == attempts.get_nowait() == writes + assert database.committed == ("caller", "attempt") + assert (meter.pending_records, meter.pending_bytes) == (0, 0) + finally: + retry_signal.cancel() + await asyncio.gather(retry_signal, return_exceptions=True) + await meter.stop(0) + await asyncio.gather(receipt, return_exceptions=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("first_commits", [False, True]) +async def test_shutdown_during_conflict_isolation_completes_every_receipt(first_commits: bool) -> None: + isolating: Final = asyncio.Event() + blocked: Final = asyncio.Event() + + async def persist(writes: tuple[UsageWrite, ...]) -> None: + if len(writes) > 1: + raise AccountingConflictError("one producer conflicts") + if first_commits and writes[0].event.event_id == "one": + return + isolating.set() + await blocked.wait() + + meter: Final = DurableUsageMeter(persist, UsageCaptureSettings(batch_size=2)) + first: Final = meter.submit((_write("one"),)) + second: Final = meter.submit((_write("two"),)) + await isolating.wait() + await meter.stop(0) + results: Final = await asyncio.gather(first, second, return_exceptions=True) + assert results[0] is None if first_commits else isinstance(results[0], UsageCaptureError) + assert isinstance(results[1], UsageCaptureError) + assert (meter.pending_records, meter.pending_bytes) == (0, 0) + + +@pytest.mark.asyncio +async def test_elapsed_flush_deadline_commits_a_partial_batch() -> None: + from litellm.proxy.spend_tracking.usage_meter import _wait_for_batch + + database: Final = CommitBarrier() + database.release.set() + + async def expired_flush(wakeup: asyncio.Event, timeout: float) -> None: + assert timeout == 5.0 + await _wait_for_batch(wakeup, 0) + + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=10), wait_for_batch=expired_flush) + receipt: Final = meter.submit((_write("partial-batch"),)) + try: + await receipt + assert database.committed == ("partial-batch",) + assert database.transactions == 1 + assert (meter.pending_records, meter.pending_bytes) == (0, 0) + finally: + await meter.stop(0) + + +@pytest.mark.asyncio +async def test_burst_drains_full_batches_without_waiting_for_partial_batch_deadline() -> None: + database: Final = CommitBarrier() + database.release.set() + waits: Final = asyncio.Queue[bool]() + + async def observe_wait(wakeup: asyncio.Event, timeout: float) -> None: + waits.put_nowait(wakeup.is_set()) + await wakeup.wait() + + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2), wait_for_batch=observe_wait) + receipts: Final = tuple(meter.submit((_write(str(index)),)) for index in range(5)) + try: + assert await waits.get() is True + await asyncio.gather(*receipts[:2]) + assert await waits.get() is True, "A full queued batch must not wait for the flush deadline" + await asyncio.gather(*receipts[2:4]) + assert await waits.get() is False + assert database.committed == ("0", "1", "2", "3") + assert meter.pending_records == 1 + assert not receipts[-1].done() + await meter.close() + await receipts[-1] + assert database.committed == ("0", "1", "2", "3", "4") + assert database.transactions == 3 + assert (meter.pending_records, meter.pending_bytes) == (0, 0) + finally: + await meter.stop(0) + await asyncio.gather(*receipts, return_exceptions=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sqlstate", ["22P05", "22003", "42P01"]) +async def test_invalid_submission_isolated_without_retrying_systemic_faults(sqlstate: str) -> None: + from prisma.errors import DataError + + database: Final = CommitBarrier() + database.release.set() + attempts: Final = asyncio.Queue[tuple[str, ...]]() + + async def persist(writes: tuple[UsageWrite, ...]) -> None: + identities: Final = tuple(write.event.event_id for write in writes) + attempts.put_nowait(identities) + if "invalid" in identities or sqlstate == "42P01": + raise DataError( + data={"user_facing_error": {"message": "database rejected input", "meta": {"code": sqlstate}}} + ) + await database.write(writes) + + meter: Final = DurableUsageMeter(persist, UsageCaptureSettings(batch_size=3)) + rejected: Final = meter.submit((_write("caller"), _write("invalid"))) + neighbor: Final = meter.submit((_write("neighbor"),)) + try: + with pytest.raises(DataError): + await rejected + if sqlstate == "42P01": + with pytest.raises(DataError): + await neighbor + assert attempts.qsize() == 1 + else: + await neighbor + assert database.committed == ("neighbor",) + assert attempts.qsize() == 3 + finally: + await meter.close() + assert (meter.pending_records, meter.pending_bytes) == (0, 0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("dependency_outcome", ["commit", "failure", "cancel"]) +async def test_commit_dependencies_preserve_capacity_and_isolate_failed_dependents(dependency_outcome: str) -> None: + database: Final = CommitBarrier() + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=3, max_pending_records=4)) + prerequisite: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future() + dependent: Final = meter.submit((_write("parent"), _write("final")), prerequisites=(prerequisite,)) + neighbor: Final = meter.submit((_write("neighbor"),)) + assert meter.pending_records == 3 + assert not dependent.done() + with pytest.raises(UsageCaptureError, match="capacity"): + meter.submit((_write("excess-one"), _write("excess-two"))) + if dependency_outcome == "commit": + prerequisite.set_result(None) + elif dependency_outcome == "failure": + prerequisite.set_exception(ValueError("dependency failed")) + else: + prerequisite.cancel() + database.release.set() + await meter.close() + await neighbor + if dependency_outcome == "commit": + await dependent + assert database.committed == ("parent", "final", "neighbor") + else: + with pytest.raises(UsageCaptureError, match="dependency"): + await dependent + assert database.committed == ("neighbor",) + assert meter.pending_records == 0 + assert meter.pending_bytes == 0 + + +@pytest.mark.asyncio +async def test_shutdown_does_not_cancel_dependency_receipt_or_leak_reserved_capacity() -> None: + database: Final = CommitBarrier() + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2)) + prerequisite: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future() + dependent: Final = meter.submit((_write("parent"), _write("final")), prerequisites=(prerequisite,)) + await meter.stop(0) + with pytest.raises(UsageCaptureError, match="acknowledgement"): + await dependent + assert not prerequisite.done() + assert database.committed == () + assert meter.pending_records == 0 + assert meter.pending_bytes == 0 + prerequisite.set_result(None) From 61efbd071e81a1cc59a147aed29569a9ac74e420 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 21:18:02 -0700 Subject: [PATCH 02/15] fix(aiohttp): bill the usage a stream sent before its connection dropped, and deflake the CyberArk override test (#47838) * fix(aiohttp): hand out bytes buffered before a dropped connection so a cut stream bills its partial usage test(integration): stop the CyberArk override test failing when a short config value lands inside ciphertext Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(aiohttp): cover bytes already read before a dropped connection Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/custom_httpx/aiohttp_transport.py | 15 +++++++ .../security/test_config_overrides.py | 5 ++- .../custom_httpx/test_aiohttp_transport.py | 43 +++++++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/litellm/llms/custom_httpx/aiohttp_transport.py b/litellm/llms/custom_httpx/aiohttp_transport.py index 9297e76624d..c32e75bafa8 100644 --- a/litellm/llms/custom_httpx/aiohttp_transport.py +++ b/litellm/llms/custom_httpx/aiohttp_transport.py @@ -110,6 +110,15 @@ def map_aiohttp_exceptions() -> Generator[None, None, None]: raise mapped_exc(message) from exc +def _unread_payload(content: object) -> bytes: + """Bytes the upstream sent before the connection dropped that aiohttp buffered but never handed out, since its + reads raise the stored payload error ahead of the buffer.""" + if not isinstance(content, aiohttp.StreamReader): + return b"" + buffered: Final = b"".join(content._buffer) # pyright: ignore[reportPrivateUsage] # no public read drains it + return buffered[content._buffer_offset :] # pyright: ignore[reportPrivateUsage] # see above + + def _release_transport_anchor(_transport: object) -> None: """Release a transport anchored to a response stream. See ``_anchor_transport_to``.""" @@ -144,6 +153,12 @@ class AiohttpResponseStream(httpx.AsyncByteStream): raise httpx.ReadError(str(e)) from e except aiohttp.http_exceptions.TransferEncodingError as e: raise httpx.ReadError(str(e)) from e + except aiohttp.ClientPayloadError: + unread: Final = _unread_payload(self._aiohttp_response.content) + if unread: + yield unread + with map_aiohttp_exceptions(): + raise except Exception: # For other exceptions, use the normal mapping with map_aiohttp_exceptions(): diff --git a/tests/integration/security/test_config_overrides.py b/tests/integration/security/test_config_overrides.py index 70f9226ea79..6b8e8791b93 100644 --- a/tests/integration/security/test_config_overrides.py +++ b/tests/integration/security/test_config_overrides.py @@ -715,8 +715,11 @@ def test_cyberark_incremental_update_encrypts_reloads_on_restart_and_delete_clea }, saved.text stored_full: Final = _stored_override(database_url, "cyberark") assert set(stored_full) == set(full_config), stored_full + assert all(stored_full[key] != value for key, value in full_config.items()), stored_full serialized_full: Final = json.dumps(stored_full) - assert all(value not in serialized_full for value in full_config.values()), serialized_full + assert all(value not in serialized_full for value in (cyberark_wire.url, account, username, api_key)), ( + serialized_full + ) model_one: Final = "integration-cyberark-" + uuid.uuid4().hex _new_model(first, model_one, provider_wire.url, secret_names[0]) diff --git a/tests/unit/llms/custom_httpx/test_aiohttp_transport.py b/tests/unit/llms/custom_httpx/test_aiohttp_transport.py index aed808223bd..4e4d5df9fb0 100644 --- a/tests/unit/llms/custom_httpx/test_aiohttp_transport.py +++ b/tests/unit/llms/custom_httpx/test_aiohttp_transport.py @@ -144,6 +144,49 @@ async def test_client_payload_error_mid_stream_raises_read_error(): assert mock_response.closed is True +@pytest.mark.asyncio +async def test_bytes_buffered_before_a_dropped_connection_are_yielded_before_the_read_error(): + loop: Final = asyncio.get_running_loop() + content: Final = aiohttp.StreamReader(aiohttp.base_protocol.BaseProtocol(loop), 2**16, loop=loop) + content.feed_data(b"event: message_start\n\n") + content.set_exception(aiohttp.ClientPayloadError("Response payload is not completed")) + mock_response: Final = MockAiohttpResponse() + mock_response.content = content + received: Final[list[bytes]] = [] + + async def _drain() -> None: + async for chunk in AiohttpResponseStream(mock_response): # pyright: ignore[reportArgumentType] # mock response + received.append(chunk) + + with pytest.raises(httpx.ReadError): + await _drain() + + assert received == [b"event: message_start\n\n"] + assert mock_response.closed is True + + +@pytest.mark.asyncio +async def test_bytes_already_read_before_a_dropped_connection_are_not_yielded_again(): + loop: Final = asyncio.get_running_loop() + content: Final = aiohttp.StreamReader(aiohttp.base_protocol.BaseProtocol(loop), 2**16, loop=loop) + content.feed_data(b"event: message_start\n\n") + content.feed_data(b"event: ping\n\n") + assert await content.read(7) == b"event: " + content.set_exception(aiohttp.ClientPayloadError("Response payload is not completed")) + mock_response: Final = MockAiohttpResponse() + mock_response.content = content + received: Final[list[bytes]] = [] + + async def _drain() -> None: + async for chunk in AiohttpResponseStream(mock_response): # pyright: ignore[reportArgumentType] # mock response + received.append(chunk) + + with pytest.raises(httpx.ReadError): + await _drain() + + assert received == [b"message_start\n\nevent: ping\n\n"] + + @pytest.mark.asyncio async def test_client_payload_error_before_first_chunk_raises_read_error(): """A connection reset before any body byte must surface, not yield an empty 200 body""" From 5950af8fb53c243e8fe647a05a971a70a90ad8b9 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 30 Oct 2026 21:41:20 -0700 Subject: [PATCH 03/15] fix(responses): preserve text verbosity when bridging to chat completions (#49049) * test(responses): cover text verbosity on the chat completions bridge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): send text.verbosity as chat verbosity on the chat completions bridge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(responses): type the bridged text verbosity lookup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): cover text verbosity on the client tool search follow-up bridge turn Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): cover text verbosity validation on the codex tool search follow-up turn Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): send text.verbosity on the client tool search follow-up bridge turn Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): cover text.verbosity on previous_response_id client tool_search follow-up Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../handler.py | 26 +- .../test_responses_bridge_verbosity_wire.py | 413 ++++++++++++++++++ .../test_anthropic_responses_bridge.py | 54 +++ .../test_responses_api_bridge_flag.py | 279 +++++++++++- 4 files changed, 763 insertions(+), 9 deletions(-) create mode 100644 tests/integration/providers/test_responses_bridge_verbosity_wire.py diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 1f56ddf863e..e13151eff1b 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -33,6 +33,14 @@ from litellm.types.utils import ModelResponse _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object]) +def _chat_verbosity(responses_api_request: Mapping[str, object]) -> Mapping[str, object]: + match responses_api_request.get("text"): + case {"verbosity": str() as verbosity}: + return {"verbosity": verbosity} + case _: + return {} + + @dataclass(frozen=True, slots=True) class _BridgedRequest: input: str | ResponseInputParam @@ -95,8 +103,8 @@ class LiteLLMCompletionTransformationHandler: bridged: Final = _bridged_request(hoisted, responses_api_request, ()) bridged_input: Final = bridged.input bridged_request: Final = bridged.responses_api_request - litellm_completion_request: Final[dict] = ( - LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + litellm_completion_request: Final[dict] = { + **LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model=model, input=bridged_input, responses_api_request=bridged_request, @@ -104,8 +112,9 @@ class LiteLLMCompletionTransformationHandler: stream=stream, extra_headers=extra_headers, **kwargs, - ) - ) + ), + **_chat_verbosity(bridged_request), + } if _is_async: return self.async_response_api_handler( @@ -169,8 +178,8 @@ class LiteLLMCompletionTransformationHandler: return await self.async_response_api_handler( previous_response_client_ids, stored_turns, - litellm_completion_request=( - LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + litellm_completion_request={ + **LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model=model, input=bridged.input, responses_api_request=bridged.responses_api_request, @@ -178,8 +187,9 @@ class LiteLLMCompletionTransformationHandler: stream=stream, extra_headers=extra_headers, **request_kwargs, - ) - ), + ), + **_chat_verbosity(bridged.responses_api_request), + }, request_input=bridged.input, responses_api_request=bridged.responses_api_request, **request_kwargs, diff --git a/tests/integration/providers/test_responses_bridge_verbosity_wire.py b/tests/integration/providers/test_responses_bridge_verbosity_wire.py new file mode 100644 index 00000000000..037450ba820 --- /dev/null +++ b/tests/integration/providers/test_responses_bridge_verbosity_wire.py @@ -0,0 +1,413 @@ +import uuid +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal + +import httpx +import pytest +import yaml +from integration._support import responses_vendor as rv +from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment +from integration._support.openai_wire import answering_model_discovery, chat_reply, responses_reply +from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.timeout(120) + +_GPT: Final = "gpt-5.6" +_DATABRICKS: Final = "databricks-meta-llama-4-maverick" +_PROVIDER_KEY: Final = "synthetic-provider-key" +_NO_CACHE: Final[dict[str, JsonValue]] = {"no-cache": True} +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_WEATHER_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"city": {"type": "string"}, "celsius": {"type": "number"}}, + "required": ["city", "celsius"], + "additionalProperties": False, +} +_TEXT_FORMAT: Final[dict[str, JsonValue]] = { + "type": "json_schema", + "name": "weather", + "schema": _WEATHER_SCHEMA, + "strict": True, +} +_RESPONSE_FORMAT: Final[dict[str, JsonValue]] = { + "type": "json_schema", + "json_schema": {"name": "weather", "schema": _WEATHER_SCHEMA, "strict": True}, +} +_STREAM_OPTIONS: Final[dict[str, JsonValue]] = {"stream_options": {"include_usage": True}} + +Verbosity = Literal["low", "high"] + + +def _prompt(marker: str) -> str: + return f"verbosity probe marker-{marker}" + + +def _answer(marker: str) -> str: + return f"scripted answer marker-{marker}" + + +def _marker_of(request: Request) -> str: + body: Final = request.body.decode() + start: Final = body.rfind("marker-") + return body[start + len("marker-") : start + len("marker-") + 32] if start >= 0 else "health" + + +def _vendor(request: Request) -> Reply: + body: Final = _JSON_OBJECT.validate_json(request.body or b"{}") + marker: Final = _marker_of(request) + stream: Final = body.get("stream") is True + if request.target.endswith("/responses"): + return responses_reply(f"resp_{marker}", _GPT, _answer(marker), stream=stream) + return chat_reply(f"chatcmpl-{marker}", str(body.get("model")), _answer(marker), stream=stream) + + +@contextmanager +def _upstream() -> Iterator[Wire]: + with wire_server(answering_model_discovery(_vendor)) as wire: + yield wire + + +def _bridged_gpt(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str: + return scenario.model( + model=f"openai/{_GPT}", + api_base=f"{wire.url}/v1", + api_key=_PROVIDER_KEY, + use_chat_completions_api=True, + **parameters, + ) + + +def _native_gpt(scenario: Scenario, wire: Wire) -> str: + return scenario.model(model=f"openai/{_GPT}", api_base=f"{wire.url}/v1", api_key=_PROVIDER_KEY) + + +def _databricks(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str: + return scenario.model( + model=f"databricks/{_DATABRICKS}", + api_base=f"{wire.url}/serving-endpoints", + api_key=_PROVIDER_KEY, + **parameters, + ) + + +def _responses(gateway: Gateway, model: str, marker: str, **fields: JsonValue) -> httpx.Response: + return gateway.request( + "POST", "/v1/responses", {"model": model, "input": _prompt(marker), "cache": _NO_CACHE, **fields} + ) + + +def _streamed_events(gateway: Gateway, model: str, marker: str, **fields: JsonValue) -> list[dict[str, JsonValue]]: + with gateway.client.stream( + "POST", + "/v1/responses", + json={"model": model, "input": _prompt(marker), "cache": _NO_CACHE, "stream": True, **fields}, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) as response: + assert response.status_code == 200, response.read() + return [ + _JSON_OBJECT.validate_json(line.removeprefix("data: ")) + for line in response.iter_lines() + if line.startswith("data: {") + ] + + +def _output_text(response: Mapping[str, JsonValue]) -> str: + output: Final = response["output"] + assert isinstance(output, list), response + return "".join( + str(part["text"]) + for item in output + if isinstance(item, dict) and item.get("type") == "message" + for part in item["content"] + if isinstance(part, dict) and part.get("type") == "output_text" + ) + + +def _answered(response: httpx.Response, marker: str) -> None: + assert response.status_code == 200, response.text + body: Final = _JSON_OBJECT.validate_json(response.text) + assert body["status"] == "completed" and _output_text(body) == _answer(marker), body + + +def _streamed_answer(events: list[dict[str, JsonValue]], marker: str) -> None: + completed: Final = [event for event in events if event["type"] == "response.completed"] + assert len(completed) == 1, [event["type"] for event in events] + final: Final = completed[0]["response"] + assert isinstance(final, dict) and _output_text(final) == _answer(marker), final + deltas: Final = "".join(str(event["delta"]) for event in events if event["type"] == "response.output_text.delta") + assert deltas == _answer(marker), events + + +def _posted(wire: Wire) -> tuple[Request, ...]: + return tuple(request for request in wire.drain() if request.method == "POST") + + +def _only_post(wire: Wire, target: str) -> dict[str, JsonValue]: + posted: Final = _posted(wire) + assert [request.target for request in posted] == [target], posted + return _JSON_OBJECT.validate_json(posted[0].body) + + +def _chat_messages(marker: str) -> list[JsonValue]: + return [{"role": "user", "content": _prompt(marker)}] + + +def _unsupported_param_error(response: httpx.Response) -> None: + assert response.status_code == 400, response.text + error: Final = _JSON_OBJECT.validate_json(response.text)["error"] + assert isinstance(error, dict) and "verbosity" in str(error["message"]), response.text + + +@pytest.mark.parametrize("verbosity", ["low", "high"]) +def test_regression_explicit_bridge_sends_text_verbosity_as_top_level_verbosity( + gateway: Gateway, verbosity: Verbosity +) -> None: + marker: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _bridged_gpt(scenario, wire) + _answered(_responses(gateway, model, marker, text={"verbosity": verbosity}), marker) + sent: Final = _only_post(wire, "/v1/chat/completions") + assert sent == {"model": _GPT, "messages": _chat_messages(marker), "verbosity": verbosity}, sent + + +@pytest.mark.parametrize("verbosity", ["low", "high"]) +def test_regression_explicit_bridge_stream_sends_text_verbosity_as_top_level_verbosity( + gateway: Gateway, verbosity: Verbosity +) -> None: + marker: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _bridged_gpt(scenario, wire) + _streamed_answer(_streamed_events(gateway, model, marker, text={"verbosity": verbosity}), marker) + sent: Final = _only_post(wire, "/v1/chat/completions") + assert sent == { + "model": _GPT, + "messages": _chat_messages(marker), + "stream": True, + **_STREAM_OPTIONS, + "verbosity": verbosity, + }, sent + + +def test_regression_explicit_bridge_sends_verbosity_beside_the_json_schema_response_format(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _bridged_gpt(scenario, wire) + _answered(_responses(gateway, model, marker, text={"format": _TEXT_FORMAT, "verbosity": "low"}), marker) + sent: Final = _only_post(wire, "/v1/chat/completions") + assert sent == { + "model": _GPT, + "messages": _chat_messages(marker), + "response_format": _RESPONSE_FORMAT, + "verbosity": "low", + }, sent + + +def test_regression_automatic_bridge_refuses_unsupported_verbosity_before_the_wire(gateway: Gateway) -> None: + refused: Final = uuid.uuid4().hex + control: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _databricks(scenario, wire, drop_params=False) + _unsupported_param_error(_responses(gateway, model, refused, text={"verbosity": "low"})) + _answered(_responses(gateway, model, control), control) + posted: Final = _posted(wire) + assert [(request.target, _marker_of(request)) for request in posted] == [ + ("/serving-endpoints/chat/completions", control) + ], posted + + +def test_regression_automatic_bridge_stream_refuses_unsupported_verbosity_before_the_wire(gateway: Gateway) -> None: + refused: Final = uuid.uuid4().hex + control: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _databricks(scenario, wire, drop_params=False) + _unsupported_param_error(_responses(gateway, model, refused, text={"verbosity": "high"}, stream=True)) + _answered(_responses(gateway, model, control), control) + posted: Final = _posted(wire) + assert [(request.target, _marker_of(request)) for request in posted] == [ + ("/serving-endpoints/chat/completions", control) + ], posted + + +@pytest.mark.parametrize("source", ["deployment", "request"]) +def test_regression_allowed_openai_params_forwards_verbosity_through_the_automatic_bridge( + gateway: Gateway, source: Literal["deployment", "request"] +) -> None: + marker: Final = uuid.uuid4().hex + allowed: Final[dict[str, JsonValue]] = {"allowed_openai_params": ["verbosity"]} + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _databricks(scenario, wire, drop_params=False, **(allowed if source == "deployment" else {})) + extra: Final = allowed if source == "request" else {} + _answered(_responses(gateway, model, marker, text={"verbosity": "low"}, **extra), marker) + sent: Final = _only_post(wire, "/serving-endpoints/chat/completions") + assert sent == {"model": _DATABRICKS, "messages": _chat_messages(marker), "verbosity": "low"}, sent + + +@pytest.mark.parametrize("source", ["deployment", "request"]) +def test_guard_drop_params_drops_unsupported_verbosity_and_keeps_the_json_schema_format( + gateway: Gateway, source: Literal["deployment", "request"] +) -> None: + marker: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _databricks(scenario, wire, drop_params=source == "deployment") + extra: Final[dict[str, JsonValue]] = {"drop_params": True} if source == "request" else {} + _answered( + _responses(gateway, model, marker, text={"format": _TEXT_FORMAT, "verbosity": "low"}, **extra), marker + ) + sent: Final = _only_post(wire, "/serving-endpoints/chat/completions") + assert sent == {"model": _DATABRICKS, "messages": _chat_messages(marker), "response_format": _RESPONSE_FORMAT}, sent + + +def test_guard_drop_params_stream_drops_unsupported_verbosity(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _databricks(scenario, wire, drop_params=True) + _streamed_answer(_streamed_events(gateway, model, marker, text={"verbosity": "high"}), marker) + sent: Final = _only_post(wire, "/serving-endpoints/chat/completions") + assert sent == {"model": _DATABRICKS, "messages": _chat_messages(marker), "stream": True}, sent + + +def test_guard_explicit_bridge_without_verbosity_sends_only_the_json_schema_format(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _bridged_gpt(scenario, wire) + _answered(_responses(gateway, model, marker, text={"format": _TEXT_FORMAT}), marker) + sent: Final = _only_post(wire, "/v1/chat/completions") + assert sent == {"model": _GPT, "messages": _chat_messages(marker), "response_format": _RESPONSE_FORMAT}, sent + + +def test_guard_explicit_bridge_without_text_sends_no_verbosity(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _bridged_gpt(scenario, wire) + _answered(_responses(gateway, model, marker), marker) + sent: Final = _only_post(wire, "/v1/chat/completions") + assert sent == {"model": _GPT, "messages": _chat_messages(marker)}, sent + + +@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"]) +def test_guard_native_responses_route_forwards_text_verbosity_unchanged(gateway: Gateway, stream: bool) -> None: + marker: Final = uuid.uuid4().hex + text: Final[dict[str, JsonValue]] = {"format": _TEXT_FORMAT, "verbosity": "high"} + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _native_gpt(scenario, wire) + if stream: + _streamed_answer(_streamed_events(gateway, model, marker, text=text), marker) + else: + _answered(_responses(gateway, model, marker, text=text), marker) + sent: Final = _only_post(wire, "/v1/responses") + assert sent == {"model": _GPT, "input": _prompt(marker), "text": text, **({"stream": True} if stream else {})}, sent + + +_CLIENT_TOOL_SEARCH: Final[dict[str, JsonValue]] = {"type": "tool_search", "execution": "client"} +_CLIENT_TOOL_SEARCH_ON_THE_WIRE: Final[dict[str, JsonValue]] = { + "type": "function", + "function": { + "name": "tool_search", + "description": ( + "Search the deferred tools by keyword. The matching tools are loaded and become callable on the next turn." + ), + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query for deferred tools."}, + "limit": {"type": "integer", "description": "Maximum number of tools to return. Defaults to 8."}, + }, + "required": ["query"], + "additionalProperties": False, + }, + }, +} + + +@dataclass(frozen=True, slots=True) +class _FollowUpWire: + sent: JsonValue + expected: dict[str, JsonValue] + + +def _stored_prompts_config(directory: Path) -> Path: + stock: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + general: Final = _JSON_OBJECT.validate_python(stock["general_settings"]) + path: Final = directory / "verbosity-stored-turns.yaml" + path.write_text(yaml.safe_dump({**stock, "general_settings": {**general, "store_prompts_in_spend_logs": True}})) + return path + + +@pytest.fixture(scope="module") +def stored_turns_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[OwnedProxy, Wire]]: + directory: Final = tmp_path_factory.mktemp("verbosity-stored-turns") + with _upstream() as wire, gateway_from_environment() as gateway: + with owned_proxy_process(gateway, directory, {}, config=_stored_prompts_config(directory)) as owned: + yield owned, wire + + +def _stored_first_turn(proxy: Gateway, model: str, marker: str) -> str: + first: Final = _responses(proxy, model, marker, tools=[_CLIENT_TOOL_SEARCH]) + _answered(first, marker) + response_id: Final = str(_JSON_OBJECT.validate_json(first.text)["id"]) + eventually( + lambda: proxy.rows('SELECT request_id FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)), + lambda rows: any(rv.same_response(str(row["request_id"]), response_id) for row in rows), + seconds=70, + ) + return response_id + + +def _client_tool_search_follow_up( + stored_turns_proxy: tuple[OwnedProxy, Wire], stream: bool, **fields: JsonValue +) -> _FollowUpWire: + owned, wire = stored_turns_proxy + proxy: Final = owned.gateway + first: Final = uuid.uuid4().hex + second: Final = uuid.uuid4().hex + with proxy.scenario() as scenario: + model: Final = _bridged_gpt(scenario, wire) + previous: Final = _stored_first_turn(proxy, model, first) + wire.drain() + follow_up: Final[dict[str, JsonValue]] = { + "previous_response_id": previous, + "tools": [_CLIENT_TOOL_SEARCH], + **fields, + } + if stream: + _streamed_answer(_streamed_events(proxy, model, second, **follow_up), second) + else: + _answered(_responses(proxy, model, second, **follow_up), second) + sent: Final = _only_post(wire, "/v1/chat/completions") + return _FollowUpWire( + sent, + { + "model": _GPT, + "messages": [ + *_chat_messages(first), + {"role": "assistant", "content": _answer(first)}, + *_chat_messages(second), + ], + "tools": [_CLIENT_TOOL_SEARCH_ON_THE_WIRE], + **({"stream": True, **_STREAM_OPTIONS} if stream else {}), + }, + ) + + +@pytest.mark.timeout(240) +@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"]) +@pytest.mark.parametrize("verbosity", ["low", "high"]) +def test_regression_client_tool_search_follow_up_sends_text_verbosity_as_top_level_verbosity( + stored_turns_proxy: tuple[OwnedProxy, Wire], stream: bool, verbosity: Verbosity +) -> None: + observed: Final = _client_tool_search_follow_up(stored_turns_proxy, stream, text={"verbosity": verbosity}) + assert observed.sent == {**observed.expected, "verbosity": verbosity}, observed.sent + + +@pytest.mark.timeout(240) +@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"]) +def test_guard_client_tool_search_follow_up_without_verbosity_sends_no_verbosity( + stored_turns_proxy: tuple[OwnedProxy, Wire], stream: bool +) -> None: + observed: Final = _client_tool_search_follow_up(stored_turns_proxy, stream) + assert observed.sent == observed.expected, observed.sent diff --git a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py index 0341511496c..5627167432d 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py @@ -1595,3 +1595,57 @@ async def test_previous_response_id_keeps_the_tools_an_earlier_codex_tool_search [("tool_result", "toolu_standup"), ("text", None)], ] assert prisma_client.db.queries == 1 + + +async def _codex_follow_up_body(**kwargs: object) -> Mapping[str, object]: + anthropic: Final = _RecordingRawAnthropicMessages() + with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_SEARCH_TURN, _CALENDAR_TURN])): + await litellm.aresponses( + model=_MODEL, + previous_response_id="chatcmpl-calendar-turn", + input=[{"type": "function_call_output", "call_id": "toolu_standup", "output": "Created Standup."}], + tools=[_CODEX_TOOL_SEARCH, _SHELL_TOOL], + api_key="sk-ant-fake", + client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)), + **kwargs, + ) + return json.loads(anthropic.body) + + +@pytest.mark.asyncio +async def test_codex_tool_search_follow_up_rejects_text_verbosity_anthropic_does_not_support( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + anthropic: Final = _RecordingRawAnthropicMessages() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_SEARCH_TURN, _CALENDAR_TURN])), + pytest.raises(litellm.UnsupportedParamsError, match="verbosity"), + ): + await litellm.aresponses( + model=_MODEL, + previous_response_id="chatcmpl-calendar-turn", + input=[{"type": "function_call_output", "call_id": "toolu_standup", "output": "Created Standup."}], + tools=[_CODEX_TOOL_SEARCH, _SHELL_TOOL], + text={"verbosity": "low"}, + api_key="sk-ant-fake", + num_retries=0, + client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)), + ) + + assert anthropic.body == "" + + +@pytest.mark.asyncio +async def test_codex_tool_search_follow_up_text_verbosity_follows_provider_param_validation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + without_text: Final = await _codex_follow_up_body() + + dropped: Final = await _codex_follow_up_body(text={"verbosity": "low"}, drop_params=True) + allowed: Final = await _codex_follow_up_body(text={"verbosity": "low"}, allowed_openai_params=["verbosity"]) + + assert dropped == without_text + assert allowed == {**without_text, "verbosity": "low"} diff --git a/tests/unit/responses/test_responses_api_bridge_flag.py b/tests/unit/responses/test_responses_api_bridge_flag.py index de96e995dd4..210488f0888 100644 --- a/tests/unit/responses/test_responses_api_bridge_flag.py +++ b/tests/unit/responses/test_responses_api_bridge_flag.py @@ -554,7 +554,6 @@ class TestUseResponsesApiBridgeFlag: truncation="auto", metadata={"k": "v"}, background=False, - text={"verbosity": "low"}, api_base="https://example.databricks.test/serving-endpoints", api_key="fake-databricks-api-key", num_retries=0, @@ -1103,3 +1102,281 @@ async def test_chatgpt_completion_routes_to_responses_without_a_provider_catalog def test_missing_catalog_entry_does_not_force_other_providers_to_responses(provider: str) -> None: info, model = responses_api_bridge_check(model="test-chatgpt-routing-control", custom_llm_provider=provider) assert (info.get("mode"), model) == (None, "test-chatgpt-routing-control") + + +_GPT_5_6_CHUNKS: Final = ( + { + "id": "chatcmpl-123", + "object": "chat.completion.chunk", + "created": 1677652288, + "model": "gpt-5.6", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Answer"}, "finish_reason": None}], + }, + { + "id": "chatcmpl-123", + "object": "chat.completion.chunk", + "created": 1677652288, + "model": "gpt-5.6", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, +) + + +def _openai_chat_upstream(respx_mock: respx.MockRouter, stream: bool) -> respx.Route: + route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions") + if stream: + return route.respond( + 200, + headers={"content-type": "text/event-stream"}, + text="".join(f"data: {json.dumps(chunk)}\n\n" for chunk in _GPT_5_6_CHUNKS) + "data: [DONE]\n\n", + ) + return route.respond( + 200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + + +async def _bridged_responses_call(asynchronous: bool, stream: bool, **kwargs: object) -> None: + if not asynchronous: + response = litellm.responses( + input="Hello", stream=stream, api_key="fake-provider-api-key", num_retries=0, **kwargs + ) + _ = list(response) if stream else response + return + response = await litellm.aresponses( + input="Hello", stream=stream, api_key="fake-provider-api-key", num_retries=0, **kwargs + ) + _ = [event async for event in response] if stream else response + + +@pytest.mark.parametrize("verbosity", ["low", "high"]) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.asyncio +async def test_bridge_sends_text_verbosity_as_chat_verbosity( + verbosity: str, asynchronous: bool, stream: bool, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + upstream: Final = _openai_chat_upstream(respx_mock, stream) + + await _bridged_responses_call( + asynchronous, stream, model="openai/gpt-5.6", use_chat_completions_api=True, text={"verbosity": verbosity} + ) + + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.read()) == { + "model": "gpt-5.6", + "messages": [{"role": "user", "content": "Hello"}], + "verbosity": verbosity, + **({"stream": True, "stream_options": {"include_usage": True}} if stream else {}), + } + + +_JSON_SCHEMA_TEXT_FORMAT: Final = { + "type": "json_schema", + "name": "answer", + "schema": {"type": "object", "properties": {"answer": {"type": "string"}}}, + "strict": True, +} +_JSON_SCHEMA_RESPONSE_FORMAT: Final = { + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": {"type": "object", "properties": {"answer": {"type": "string"}}}, + "strict": True, + }, +} + + +@pytest.mark.parametrize( + ("text", "expected_fields"), + [ + pytest.param( + {"verbosity": "low", "format": _JSON_SCHEMA_TEXT_FORMAT}, + {"response_format": _JSON_SCHEMA_RESPONSE_FORMAT, "verbosity": "low"}, + id="verbosity-and-format", + ), + pytest.param( + {"format": _JSON_SCHEMA_TEXT_FORMAT}, {"response_format": _JSON_SCHEMA_RESPONSE_FORMAT}, id="format-only" + ), + pytest.param(None, {}, id="no-text"), + ], +) +def test_bridge_maps_text_format_alongside_verbosity( + text: dict[str, object] | None, + expected_fields: dict[str, object], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + upstream: Final = _openai_chat_upstream(respx_mock, stream=False) + + litellm.responses( + model="openai/gpt-5.6", + input="Hello", + use_chat_completions_api=True, + api_key="fake-provider-api-key", + num_retries=0, + **({"text": text} if text is not None else {}), + ) + + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.read()) == { + "model": "gpt-5.6", + "messages": [{"role": "user", "content": "Hello"}], + **expected_fields, + } + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.asyncio +async def test_bridge_rejects_text_verbosity_the_provider_does_not_support_when_drop_params_is_off( + asynchronous: bool, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + upstream: Final = _databricks_chat_upstream(respx_mock) + + with pytest.raises(litellm.UnsupportedParamsError) as raised: + await _bridged_responses_call( + asynchronous, + False, + model="databricks/my-custom-model", + text={"verbosity": "low"}, + api_base="https://example.databricks.test/serving-endpoints", + ) + + assert upstream.call_count == 0, "the provider must not be called with a verbosity it does not support" + assert raised.value.status_code == 400 + assert "verbosity" in str(raised.value) + + +@pytest.mark.parametrize( + ("call_kwargs", "global_drop_params", "expected_fields"), + [ + pytest.param({"drop_params": True}, False, {}, id="request-drop-params"), + pytest.param({}, True, {}, id="global-drop-params"), + pytest.param({"allowed_openai_params": ["verbosity"]}, False, {"verbosity": "low"}, id="allowed-openai-params"), + ], +) +def test_bridge_text_verbosity_follows_provider_param_validation( + call_kwargs: dict[str, object], + global_drop_params: bool, + expected_fields: dict[str, object], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "drop_params", global_drop_params) + upstream: Final = _databricks_chat_upstream(respx_mock) + + response: Final = litellm.responses( + model="databricks/my-custom-model", + input="Hello", + text={"verbosity": "low"}, + api_base="https://example.databricks.test/serving-endpoints", + api_key="fake-databricks-api-key", + num_retries=0, + **call_kwargs, + ) + + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.read()) == { + "model": "my-custom-model", + "messages": [{"role": "user", "content": "Hello"}], + **expected_fields, + } + assert response.output[0].content[0].text == "Answer" + + +def test_native_responses_route_keeps_text_verbosity_nested( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + 200, + json={ + "id": "resp_1", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + ) + + litellm.responses( + model="openai/gpt-5.6", + input="Hello", + text={"verbosity": "low"}, + api_key="fake-provider-api-key", + num_retries=0, + ) + + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.read()) == { + "model": "gpt-5.6", + "input": "Hello", + "text": {"verbosity": "low"}, + } + + +_CLIENT_TOOL_SEARCH: Final = { + "type": "tool_search", + "execution": "client", + "description": "Search the deferred app tools.", + "parameters": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}, +} + + +@pytest.mark.parametrize("verbosity", ["low", "high"]) +@pytest.mark.asyncio +async def test_bridge_sends_text_verbosity_on_a_client_tool_search_follow_up_turn( + verbosity: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + upstream: Final = _openai_chat_upstream(respx_mock, stream=False) + + await _bridged_responses_call( + True, + False, + model="openai/gpt-5.6", + use_chat_completions_api=True, + previous_response_id="chatcmpl-earlier-turn", + tools=[_CLIENT_TOOL_SEARCH], + text={"verbosity": verbosity}, + ) + + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.read()) == { + "model": "gpt-5.6", + "messages": [{"role": "user", "content": "Hello"}], + "tools": [ + { + "type": "function", + "function": { + "name": "tool_search", + "description": _CLIENT_TOOL_SEARCH["description"], + "parameters": _CLIENT_TOOL_SEARCH["parameters"], + }, + } + ], + "verbosity": verbosity, + } From 8801a17f0df7b71e112224c9987ad0191f7a24e2 Mon Sep 17 00:00:00 2001 From: quietjules <349048734+quietjules@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:07:32 -0700 Subject: [PATCH 04/15] fix(proxy)!: bind admin-owned team keys to their team's MCP servers (#49161) * test(proxy): pin admin-owned team keys to their team's MCP servers and toolsets A virtual key on a real team owned by a proxy_admin or proxy_admin_viewer, with no MCP list of its own, currently reaches every MCP server in the registry and any toolset, ignoring its team. These tests expect it to resolve only what its team grants, like the same key owned by an internal user, while the admin dashboard session keeps the whole registry. * fix(proxy)!: bind admin-owned team keys to their team's MCP servers A virtual key on a real team now resolves MCP servers and toolsets through its team even when its owner is a proxy_admin or proxy_admin_viewer, the way team model access already binds it. The unscoped admin view stays for the master key, admin keys with no team, the admin dashboard session, and admitted user subjects. The toolset grant checks on /toolset/{name}/mcp and on the Responses litellm_proxy MCP tool use the same rule. BREAKING CHANGE: a team key owned by a proxy admin or admin viewer no longer reaches MCP servers or toolsets its team does not grant. Grant them to the team or use a key with no team. * fix(proxy): answer 403 for a toolset the key cannot use on LLM routes A litellm_proxy MCP tool on /v1/responses, /v1/chat/completions or /v1/messages that named a toolset the key's team was never granted was skipped, which left the server filter empty and handed the model every tool the key could reach. The denied name now stays in the filter, so the request gets the same 403 an unknown server name gets. Admin credentials on a real team stay bound to it whatever carries them, a lite login session included, the way the team already bounds their model access. The tests pin that and drop an unused patch. * fix(proxy): keep a denied toolset out of the scoped key beside a granted one * test(responses): cover toolset name fallbacks and align MCP admin docstrings * test(mcp): cover no-team admins' toolset access on LLM routes and drop a dead admin-check clause * fix(responses): keep a whole-gateway MCP entry serving the key's tools beside a denied toolset * test(responses): check the tools a whole-gateway request lists beside a denied toolset * test(responses): keep the toolset lookup real in the whole-gateway listing test * test(responses): run the toolset gateway tests on a real MCP server manager * fix(responses): keep denied toolset names out of the MCP server filter A denied toolset named like a server the key holds was matched to that server by the loose server-name lookup. Denied names now never reach the filter, and a request naming only denied toolsets answers 403 directly, each name once * fix(responses): let a held MCP access group outrank a same-named ungranted toolset and give denied toolsets the generic 403 * test(responses): pin access group and granted toolset precedence for LLM-route MCP names --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/_lazy_openapi_snapshot.json | 14 +- .../mcp_server/auth/user_api_key_auth_mcp.py | 28 +- .../proxy/mcp_server/mcp_server_manager.py | 2 +- litellm/proxy/mcp_server/operations.py | 9 +- litellm/proxy/mcp_server/server.py | 8 +- litellm/proxy/proxy_server.py | 6 +- .../mcp/litellm_proxy_mcp_handler.py | 108 ++++-- litellm/responses/mcp/tool_precheck.py | 7 + .../auth/test_user_api_key_auth_mcp.py | 56 +++ .../mcp_server/test_mcp_toolset_scope.py | 19 +- .../mcp/test_litellm_proxy_mcp_handler.py | 334 +++++++++++++++--- ui/litellm-dashboard/src/lib/http/schema.d.ts | 42 +-- 12 files changed, 512 insertions(+), 121 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index dc8b9415020..41ab7fcaf88 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -58015,7 +58015,7 @@ "paths": { "/toolset/{toolset_name}/mcp": { "delete": { - "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.", "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_delete", "parameters": [ { @@ -58054,7 +58054,7 @@ ] }, "get": { - "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.", "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_get", "parameters": [ { @@ -58093,7 +58093,7 @@ ] }, "head": { - "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.", "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_head", "parameters": [ { @@ -58132,7 +58132,7 @@ ] }, "options": { - "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.", "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_options", "parameters": [ { @@ -58171,7 +58171,7 @@ ] }, "patch": { - "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.", "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_patch", "parameters": [ { @@ -58210,7 +58210,7 @@ ] }, "post": { - "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.", "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_post", "parameters": [ { @@ -58249,7 +58249,7 @@ ] }, "put": { - "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.", "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", "parameters": [ { diff --git a/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py index db640b6d689..be24b1630b0 100644 --- a/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py @@ -438,6 +438,14 @@ def is_mcp_admitted_user_subject(user_api_key_auth: UserAPIKeyAuth | None) -> bo _is_mcp_admitted_user_subject: Final = is_mcp_admitted_user_subject +def has_team_unbound_mcp_admin_view(user_api_key_auth: UserAPIKeyAuth | None) -> bool: + return ( + user_api_key_auth is not None + and user_api_key_has_admin_view(user_api_key_auth) + and (not user_api_key_auth.team_id or user_api_key_auth.team_id == UI_TEAM_ID) + ) + + def _gateway_dcr_challenge_target( route: str, mcp_servers: list[str] | None, @@ -4290,19 +4298,19 @@ class MCPRequestHandler: """Whether this principal's admin-view role grants the unscoped MCP resolution, whatever credential carries it (admin key, dashboard session, or OAuth-admitted session subject). - Two bounds disqualify, one per ownership of the row. A CREDENTIAL's explicit - ``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An - admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is - [] by DB default, so for that shape the row binds through the entitlement ceiling instead - (any non-empty entitlement, or an unresolved one, disqualifies), exactly as - ``operator_open_server_ids`` reads the same row. The one owner of this predicate: the - server-axis registry resolution in ``get_allowed_mcp_servers`` and the tools-axis open - channel in ``resolve_admitted_subject_tools`` both consult it, so the two axes cannot - disagree.""" + A credential on a real team never qualifies, a virtual key or a ``lite login`` session alike: its team + bounds its MCP access whoever owns it, as it bounds the credential's model access. Two more bounds disqualify, one per ownership of the row. A CREDENTIAL's + explicit ``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An + admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is [] by DB + default, so for that shape the row binds through the entitlement ceiling instead (any non-empty + entitlement, or an unresolved one, disqualifies), exactly as ``operator_open_server_ids`` reads the same + row. The one owner of this predicate: the server-axis registry resolution in ``get_allowed_mcp_servers`` + and the tools-axis open channel in ``resolve_admitted_subject_tools`` both consult it, so the two axes + cannot disagree.""" if ( user_api_key_auth is None or user_api_key_auth.mcp_explicit_grants_only - or not user_api_key_has_admin_view(user_api_key_auth) + or not has_team_unbound_mcp_admin_view(user_api_key_auth) ): return False object_permission: Final = user_api_key_auth.object_permission diff --git a/litellm/proxy/mcp_server/mcp_server_manager.py b/litellm/proxy/mcp_server/mcp_server_manager.py index c772e2e9d09..eacba25257e 100644 --- a/litellm/proxy/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/mcp_server/mcp_server_manager.py @@ -3705,7 +3705,7 @@ class MCPServerManager: Priority: 1. If object_permission.mcp_servers is explicitly set, use it (even for admins) - 2. If admin and no object_permission, return all servers + 2. If admin, not bound to a real team, and no object_permission, return all servers 3. Otherwise, use standard permission checks """ if managed_agent_policy(user_api_key_auth) is not None: diff --git a/litellm/proxy/mcp_server/operations.py b/litellm/proxy/mcp_server/operations.py index 2526522f97f..66e46c73f98 100644 --- a/litellm/proxy/mcp_server/operations.py +++ b/litellm/proxy/mcp_server/operations.py @@ -223,6 +223,7 @@ __all__ = ( "mcp_get_prompt", "mcp_read_resource", "raise_denied_scoped_mcp_access", + "raise_generic_mcp_access_denial", ) @@ -763,8 +764,14 @@ async def raise_denied_scoped_mcp_access( ) } raise HTTPException(status_code=403, detail=group_denial) + raise_generic_mcp_access_denial(requested_names) + + +def raise_generic_mcp_access_denial(requested_names: Sequence[str]) -> NoReturn: generic_denial: Final[_McpDeniedDetail] = { - "error": f"The key is not allowed to access the requested MCP servers: {', '.join(requested_names)}" + "error": ( + f"The key is not allowed to access the requested MCP servers: {', '.join(dict.fromkeys(requested_names))}" + ) } raise HTTPException(status_code=403, detail=generic_denial) diff --git a/litellm/proxy/mcp_server/server.py b/litellm/proxy/mcp_server/server.py index 8e0826b105e..aeb640a7c8b 100644 --- a/litellm/proxy/mcp_server/server.py +++ b/litellm/proxy/mcp_server/server.py @@ -520,6 +520,7 @@ if MCP_AVAILABLE: "mcp_get_prompt", "mcp_read_resource", "raise_denied_scoped_mcp_access", + "raise_generic_mcp_access_denial", "redact_mcp_resource_url", ) from mcp.server import Server @@ -1197,6 +1198,7 @@ if MCP_AVAILABLE: apply_display_name_overrides, filter_tools_by_allowed_tools, raise_denied_scoped_mcp_access, + raise_generic_mcp_access_denial, ) @contextlib.asynccontextmanager @@ -1824,10 +1826,10 @@ if MCP_AVAILABLE: row capping it. Raises HTTPException(403) unless the principal holds toolset_id through one - of its grant sources. Admins always pass. + of its grant sources. Admins pass unless the key is bound to a real team. """ from litellm.proxy._types import LiteLLM_ObjectPermissionTable - from litellm.proxy.management_endpoints.common_utils import user_api_key_has_admin_view + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import has_team_unbound_mcp_admin_view # A key scoped to no MCP servers opts out of every MCP path. Enforce it # here too, since toolset scoping replaces mcp_servers and would otherwise @@ -1841,7 +1843,7 @@ if MCP_AVAILABLE: ) acting: Final = await acting_user(user_api_key_auth) - is_admin: Final = user_api_key_has_admin_view(acting) + is_admin: Final = has_team_unbound_mcp_admin_view(acting) if not is_admin and toolset_id not in await granted(acting): raise HTTPException( status_code=403, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b9b15d332e9..a767f5423ed 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -21985,9 +21985,9 @@ async def toolset_mcp_route(toolset_name: str, request: Request): Namespace a toolset as its own MCP endpoint. Connecting to /toolset//mcp exposes exactly the tools defined in - the toolset. Access is enforced: non-admin API keys must have the toolset - listed in their object_permission.mcp_toolsets grant list, or the request - will be rejected with a 403. + the toolset. Access is enforced: API keys other than an admin's key with no + team must hold the toolset through one of their grant sources, or the + request will be rejected with a 403. """ try: from litellm.proxy.mcp_server.mcp_server_manager import ( diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index c88a8c1cc28..1104aee353e 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -23,6 +23,7 @@ from litellm.responses.mcp.tool_precheck import ( LITELLM_PROXY_MCP_SERVER_URL_PREFIX, may_name_litellm_mcp_gateway, names_gateway_explicitly, + names_whole_gateway, proxy_path_mcp_name, ) from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator @@ -182,8 +183,8 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod async def _apply_toolset_permissions( - resolved_toolset_ids: list[str], - resolved_mcp_servers: list[str], + resolved_toolset_ids: Sequence[str], + resolved_mcp_servers: Sequence[str], user_api_key_auth: "UserAPIKeyAuth", ) -> "UserAPIKeyAuth": """Apply resolved toolset permissions to user_api_key_auth and return updated auth.""" @@ -195,7 +196,7 @@ class LiteLLM_Proxy_MCP_Handler: ) tool_permissions: Final = await global_mcp_server_manager.resolve_toolset_tool_permissions( - toolset_ids=resolved_toolset_ids + toolset_ids=list(resolved_toolset_ids) ) all_server_ids: Final = list(set(tool_permissions.keys()) | set(resolved_mcp_servers)) existing_op: Final = user_api_key_auth.object_permission @@ -224,6 +225,62 @@ class LiteLLM_Proxy_MCP_Handler: verbose_logger.debug("Could not apply toolset permissions: %s", _e) return user_api_key_auth + @staticmethod + async def _resolve_proxy_mcp_name( + name: str, + user_api_key_auth: "UserAPIKeyAuth | None", + granted_toolsets: "GrantedToolsetIds | None", + ) -> tuple[Literal["server", "toolset", "denied_toolset"], str]: + from litellm.proxy.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + if global_mcp_server_manager.get_mcp_server_by_name(name): + return ("server", name) + try: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + return ("server", name) + toolset: Final = await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name) + if toolset is None: + return ("server", name) + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( + has_team_unbound_mcp_admin_view, + ) + from litellm.proxy.mcp_server.ui_session_utils import ( + granted_toolset_ids, + ) + + denied: Final = ( + user_api_key_auth is not None + and not has_team_unbound_mcp_admin_view(user_api_key_auth) + and toolset.toolset_id not in await (granted_toolsets or granted_toolset_ids)(user_api_key_auth) + ) + except Exception as _e: + verbose_logger.debug("Could not resolve '%s' as toolset: %s", name, _e) + return ("server", name) + if not denied: + return ("toolset", toolset.toolset_id) + if await LiteLLM_Proxy_MCP_Handler._holds_access_group_named(name, user_api_key_auth): + return ("server", name) + verbose_logger.debug("Key does not have access to toolset '%s', denying it.", name) + return ("denied_toolset", name) + + @staticmethod + async def _holds_access_group_named(name: str, user_api_key_auth: "UserAPIKeyAuth | None") -> bool: + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( + MCPRequestHandler, + ) + from litellm.proxy.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + group_server_ids: Final = await MCPRequestHandler.get_mcp_servers_from_access_groups([name]) + return bool(group_server_ids) and not frozenset(group_server_ids).isdisjoint( + await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth) + ) + @staticmethod @catalog_operation(global_manager) async def _get_mcp_tools_from_manager( @@ -256,6 +313,7 @@ class LiteLLM_Proxy_MCP_Handler: from litellm.proxy.mcp_server.server import ( _get_allowed_mcp_servers_from_mcp_server_names, _get_tools_from_mcp_servers, + raise_generic_mcp_access_denial, ) mcp_servers: Final = [ @@ -268,36 +326,15 @@ class LiteLLM_Proxy_MCP_Handler: # Resolve toolset names: collect all toolset IDs first, then apply their # combined permissions in a single pass so multiple toolsets are unioned # rather than the last one overwriting the others. - resolved_mcp_servers: Final[list[str]] = [] - resolved_toolset_ids: Final[list[str]] = [] - for name in mcp_servers: - if not global_mcp_server_manager.get_mcp_server_by_name(name): - try: - from litellm.proxy.proxy_server import prisma_client - - if prisma_client is not None: - toolset = await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name) - if toolset is not None: - if user_api_key_auth is not None: - from litellm.proxy.management_endpoints.common_utils import ( - user_api_key_has_admin_view, - ) - from litellm.proxy.mcp_server.ui_session_utils import ( - granted_toolset_ids, - ) - - if not user_api_key_has_admin_view(user_api_key_auth) and toolset.toolset_id not in ( - await (granted_toolsets or granted_toolset_ids)(user_api_key_auth) - ): - verbose_logger.debug("Key does not have access to toolset '%s', skipping.", name) - continue - resolved_toolset_ids.append(toolset.toolset_id) - # Don't add to resolved_mcp_servers — toolset scope - # restricts via object_permission, not server name filter. - continue - except Exception as _e: - verbose_logger.debug("Could not resolve '%s' as toolset: %s", name, _e) - resolved_mcp_servers.append(name) + resolutions: Final = tuple( + [ + await LiteLLM_Proxy_MCP_Handler._resolve_proxy_mcp_name(name, user_api_key_auth, granted_toolsets) + for name in mcp_servers + ] + ) + resolved_mcp_servers: Final = tuple(value for kind, value in resolutions if kind == "server") + resolved_toolset_ids: Final = tuple(value for kind, value in resolutions if kind == "toolset") + denied_toolset_names: Final = tuple(value for kind, value in resolutions if kind == "denied_toolset") # Apply all resolved toolsets at once (union), avoiding permission overwrite. if resolved_toolset_ids and user_api_key_auth is not None: @@ -311,7 +348,10 @@ class LiteLLM_Proxy_MCP_Handler: # full union (toolset server IDs + direct server names). Passing a name-based # filter here would exclude those toolset server IDs (which are UUIDs, not # names), so use None and let the auth object's mcp_servers do the filtering. - effective_server_filter: Final = None if resolved_toolset_ids else (resolved_mcp_servers or None) + asks_whole_gateway: Final = any(map(names_whole_gateway, mcp_tools_with_litellm_proxy or ())) + if denied_toolset_names and not (asks_whole_gateway or resolved_mcp_servers or resolved_toolset_ids): + raise_generic_mcp_access_denial(denied_toolset_names) + effective_server_filter: Final = None if resolved_toolset_ids else (list(resolved_mcp_servers) or None) listing: Final = await _get_tools_from_mcp_servers( user_api_key_auth=user_api_key_auth, diff --git a/litellm/responses/mcp/tool_precheck.py b/litellm/responses/mcp/tool_precheck.py index a29e6895f9a..a67d81e70ae 100644 --- a/litellm/responses/mcp/tool_precheck.py +++ b/litellm/responses/mcp/tool_precheck.py @@ -22,6 +22,13 @@ def names_gateway_explicitly(tool: object) -> bool: return (_mcp_server_url(tool) or "").startswith(LITELLM_PROXY_MCP_SERVER_URL) +def names_whole_gateway(tool: object) -> bool: + server_url: Final = _mcp_server_url(tool) or "" + return server_url.startswith(LITELLM_PROXY_MCP_SERVER_URL) and not server_url.startswith( + LITELLM_PROXY_MCP_SERVER_URL_PREFIX + ) + + def proxy_path_mcp_name(tool: object) -> str | None: server_url: Final = _mcp_server_url(tool) match: Final = None if server_url is None else _PROXY_MCP_PATH_RE.match(server_url) diff --git a/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py index ee3ef3f5323..1b95cbf3c94 100644 --- a/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -21,6 +21,7 @@ from litellm.proxy._types import ( LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, LiteLLM_UserTable, + LitellmUserRoles, ProxyException, SpecialHeaders, SpecialMCPServerNames, @@ -13196,3 +13197,58 @@ async def test_personal_key_mcp_access_is_denied_when_owner_teams_cannot_be_read assert await MCPRequestHandler.get_allowed_mcp_servers(personal_key_owner_teams) == [] assert await MCPRequestHandler.get_allowed_tools_for_server("srv-a", personal_key_owner_teams) == [] assert await MCPRequestHandler.get_allowed_tools_for_server("srv-open", personal_key_owner_teams) == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +@pytest.mark.parametrize( + "owner_role", + [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY], + ids=["proxy_admin", "proxy_admin_viewer"], +) +@pytest.mark.parametrize( + ("key_team_id", "expected_servers"), + [("team-a", {"srv-a", "srv-open"}), ("team-without-mcp", {"srv-open"})], + ids=["team_grants_one_server", "team_grants_no_server"], +) +@pytest.mark.parametrize( + "credential_shape", + [{"is_session_token": False}, {"is_session_token": True}, {"api_key": None, "jwt_claims": {"sub": "admin"}}], + ids=["virtual_key", "cli_session", "jwt"], +) +async def test_admin_owned_team_key_reaches_only_its_team_mcp_servers( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, + owner_role: LitellmUserRoles, + key_team_id: str, + expected_servers: set[str], + credential_shape: dict[str, object], +) -> None: + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return _personal_key_team(team_id) if team_id in _PERSONAL_KEY_TEAMS else LiteLLM_TeamTable(team_id=team_id) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + team_key: Final = personal_key_owner_teams.model_copy( + update={"team_id": key_team_id, "user_role": owner_role, **credential_shape} + ) + + assert set(await global_mcp_server_manager.get_allowed_mcp_servers(team_key)) == expected_servers + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_admin_dashboard_session_keeps_the_whole_mcp_registry( + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager + + dashboard_session: Final = personal_key_owner_teams.model_copy( + update={"team_id": UI_TEAM_ID, "user_role": LitellmUserRoles.PROXY_ADMIN} + ) + + assert set(await global_mcp_server_manager.get_allowed_mcp_servers(dashboard_session)) == set( + global_mcp_server_manager.get_registry() + ) diff --git a/tests/unit/proxy/mcp_server/test_mcp_toolset_scope.py b/tests/unit/proxy/mcp_server/test_mcp_toolset_scope.py index f3cc38eacf9..d1337138cbc 100644 --- a/tests/unit/proxy/mcp_server/test_mcp_toolset_scope.py +++ b/tests/unit/proxy/mcp_server/test_mcp_toolset_scope.py @@ -2,7 +2,7 @@ import asyncio from collections.abc import Awaitable, Callable -from typing import Dict, List, Optional +from typing import Dict, Final, List, Optional from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -112,6 +112,23 @@ class TestApplyToolsetScope: assert op.mcp_servers == ["server-a"] assert op.mcp_tool_permissions == toolset_perms + @pytest.mark.asyncio + @pytest.mark.parametrize( + "owner_role", + [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY], + ids=["proxy_admin", "proxy_admin_viewer"], + ) + async def test_admin_owned_team_key_without_the_toolset_grant_is_denied(self, owner_role: LitellmUserRoles): + from litellm.proxy.mcp_server.server import apply_toolset_scope + + team_key: Final = UserAPIKeyAuth( + api_key="sk-test", team_id="team-a", user_role=owner_role, object_permission=None + ) + with pytest.raises(HTTPException) as exc_info: + await apply_toolset_scope(team_key, "toolset-123", granted=_granted_through_team("toolset-other")) + + assert exc_info.value.status_code == 403 + @pytest.mark.asyncio async def test_team_granted_toolset_is_served_to_a_key_without_its_own_grant(self): """A team key whose own row carries no toolset grant is admitted to the toolset its team diff --git a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py index 2f27622d77c..a16661c0174 100644 --- a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -4,6 +4,8 @@ import subprocess import sys import textwrap import types +from types import MappingProxyType +from collections.abc import Mapping, Sequence from contextlib import nullcontext from typing import Any, Final, Literal, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -18,10 +20,14 @@ import litellm from litellm.caching.caching import DualCache from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.mcp_server import operations as mcp_operations +from litellm.proxy.common_utils.user_api_key_cache import object_permission_cache_key from litellm.proxy.mcp_server.faults.list_outcomes import AggregateToolListing from litellm.proxy.mcp_server.mcp_server_manager import ListedToolsCaller, MCPServerManager +from litellm.proxy._types import UI_TEAM_ID, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTableCachedObj +from litellm.proxy.mcp_server.ui_session_utils import GrantedToolsetIds, granted_toolset_ids +from litellm.types.mcp_server.mcp_toolset import MCPToolset from litellm.proxy.utils import ProxyLogging from litellm.responses import main as responses_main from litellm.responses.mcp import litellm_proxy_mcp_handler as mcp_handler_module @@ -1575,61 +1581,309 @@ async def test_concurrent_bridge_calls_use_their_own_served_metadata(monkeypatch ProxyLogging._callback_capabilities_cache.clear() -def _toolset_gateway_manager(toolset_id: str, server_id: str) -> types.SimpleNamespace: - return types.SimpleNamespace( - catalog=types.SimpleNamespace(operation=nullcontext), - get_registry=MagicMock(return_value={}), - get_allowed_mcp_servers=AsyncMock(return_value=[]), - get_mcp_servers_from_ids=MagicMock(return_value=[]), - get_mcp_server_by_name=MagicMock(return_value=None), - get_toolset_by_name_cached=AsyncMock(return_value=types.SimpleNamespace(toolset_id=toolset_id)), - resolve_toolset_tool_permissions=AsyncMock(return_value={server_id: ["add"]}), - ) +TOOLSET_ROWS: Final = ( + MCPToolset( + toolset_id="ts-granted", toolset_name="team-toolset", tools=[{"server_id": "srv-1", "tool_name": "add"}] + ), + MCPToolset( + toolset_id="ts-other", toolset_name="other-toolset", tools=[{"server_id": "srv-2", "tool_name": "lookup"}] + ), + MCPToolset(toolset_id="ts-shouty", toolset_name="CALC", tools=[{"server_id": "srv-2", "tool_name": "lookup"}]), + MCPToolset(toolset_id="ts-group", toolset_name="reference", tools=[{"server_id": "srv-1", "tool_name": "add"}]), +) +GATEWAY_SERVERS: Final = ( + MCPServer(server_id="srv-1", name="calc", alias="calc", transport=MCPTransport.http, access_groups=["wiki"]), + MCPServer(server_id="srv-2", name="docs", alias="docs", transport=MCPTransport.http, access_groups=["reference"]), +) +UPSTREAM_TOOLS: Final = MappingProxyType( + { + "calc": ( + MCPTool(name="add", inputSchema={"type": "object"}), + MCPTool(name="subtract", inputSchema={"type": "object"}), + ), + "docs": (MCPTool(name="lookup", inputSchema={"type": "object"}),), + } +) -async def _tools_listing_kwargs_for_toolset_url(monkeypatch, team_toolset_id: str) -> dict[str, object]: - from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LitellmUserRoles, UserAPIKeyAuth - from litellm.proxy.mcp_server.ui_session_utils import granted_toolset_ids +def _toolset_table(lookup_error: Exception | None = None) -> types.SimpleNamespace: + async def find_first(*, where: Mapping[str, str]) -> MCPToolset | None: + if lookup_error is not None: + raise lookup_error + return next((row for row in TOOLSET_ROWS if row.toolset_name == where["toolset_name"]), None) - mock_get_tools = AsyncMock(return_value=AggregateToolListing(tools=[], outcomes={})) - monkeypatch.setattr("litellm.proxy.mcp_server.server._get_tools_from_mcp_servers", mock_get_tools) - monkeypatch.setattr( - "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", - _toolset_gateway_manager("ts-granted", "srv-1"), - ) - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MagicMock()) + async def find_many(*, where: Mapping[str, Mapping[str, Sequence[str]]]) -> list[MCPToolset]: + return [row for row in TOOLSET_ROWS if row.toolset_id in where["toolset_id"]["in"]] + return types.SimpleNamespace(find_first=find_first, find_many=find_many) + + +def _team_grants(toolset_id: str) -> GrantedToolsetIds: async def team_permission(context: UserAPIKeyAuth) -> LiteLLM_ObjectPermissionTable: - return LiteLLM_ObjectPermissionTable(object_permission_id="op-team", mcp_toolsets=[team_toolset_id]) + return LiteLLM_ObjectPermissionTable(object_permission_id="op-team", mcp_toolsets=[toolset_id]) async def granted_through_team(context: UserAPIKeyAuth) -> frozenset[str]: return await granted_toolset_ids(context, team_object_permission=team_permission, require_key_access=False) - team_key: Final = UserAPIKeyAuth(api_key="sk-team", team_id="team-1", user_role=LitellmUserRoles.INTERNAL_USER) - await LiteLLM_Proxy_MCP_Handler._get_mcp_tools_from_manager( - user_api_key_auth=team_key, - mcp_tools_with_litellm_proxy=[{"type": "mcp", "server_url": "litellm_proxy/mcp/team-toolset"}], - granted_toolsets=granted_through_team, + return granted_through_team + + +async def _no_toolset_grants(context: UserAPIKeyAuth) -> frozenset[str]: + return frozenset() + + +async def _tools_listed_through_the_gateway( + monkeypatch: pytest.MonkeyPatch, + user_api_key_auth: UserAPIKeyAuth, + server_urls: tuple[str, ...], + granted_toolsets: GrantedToolsetIds, + toolset_table: types.SimpleNamespace | None = None, + agent_permission: LiteLLM_ObjectPermissionTable | None = None, +) -> list[str]: + async def upstream_listing(_client: object, server_name: str) -> list[MCPTool]: + return list(UPSTREAM_TOOLS[server_name]) + + manager: Final = MCPServerManager() + manager.registry = {server.server_id: server for server in GATEWAY_SERVERS} + manager.config_mcp_servers = dict(manager.registry) + manager.create_mcp_client = AsyncMock(return_value=object()) + manager._fetch_tools_with_timeout = AsyncMock(side_effect=upstream_listing) + tables: Final = types.SimpleNamespace(litellm_mcptoolsettable=toolset_table or _toolset_table()) + monkeypatch.setattr("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", manager) + monkeypatch.setattr(mcp_operations, "global_mcp_server_manager", manager) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", types.SimpleNamespace(db=tables, writer_db=tables)) + cache: Final = DualCache() + await cache.async_set_cache(key="team_id:team-1", value=LiteLLM_TeamTableCachedObj(team_id="team-1")) + if agent_permission is not None: + await cache.async_set_cache( + key=f"agent_object_permission_id:{user_api_key_auth.agent_id}", + value=agent_permission.object_permission_id, + ) + await cache.async_set_cache( + key=object_permission_cache_key(agent_permission.object_permission_id), value=agent_permission + ) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", cache) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"supported_db_objects": []}) + try: + tools, _server_names = await LiteLLM_Proxy_MCP_Handler._get_mcp_tools_from_manager( + user_api_key_auth=user_api_key_auth, + mcp_tools_with_litellm_proxy=[{"type": "mcp", "server_url": url} for url in server_urls], + granted_toolsets=granted_toolsets, + ) + finally: + for server in GATEWAY_SERVERS: + manager._drop_listed_tools(server.server_id) + return sorted(tool.name for tool in tools) + + +def _key_holding(*server_ids: str, role: LitellmUserRoles = LitellmUserRoles.INTERNAL_USER) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="sk-team", + team_id="team-1", + user_role=role, + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-key", mcp_servers=list(server_ids)), ) - assert mock_get_tools.await_args is not None - return mock_get_tools.await_args.kwargs @pytest.mark.asyncio -async def test_toolset_gateway_url_scopes_a_team_granted_toolset_for_a_key_without_its_own_grant(monkeypatch): - kwargs: Final = await _tools_listing_kwargs_for_toolset_url(monkeypatch, team_toolset_id="ts-granted") - scoped = kwargs["user_api_key_auth"].object_permission - assert scoped is not None - assert scoped.mcp_servers == ["srv-1"] - assert scoped.mcp_tool_permissions == {"srv-1": ["add"]} - assert kwargs["mcp_servers"] is None +async def test_toolset_gateway_url_scopes_a_team_granted_toolset_for_a_key_without_its_own_grant( + monkeypatch: pytest.MonkeyPatch, +): + listed: Final = await _tools_listed_through_the_gateway( + monkeypatch, + UserAPIKeyAuth(api_key="sk-team", team_id="team-1", user_role=LitellmUserRoles.INTERNAL_USER), + ("litellm_proxy/mcp/team-toolset",), + _team_grants("ts-granted"), + ) + assert listed == ["calc-add"] @pytest.mark.asyncio -async def test_toolset_gateway_url_skips_a_toolset_the_team_does_not_grant(monkeypatch): - kwargs: Final = await _tools_listing_kwargs_for_toolset_url(monkeypatch, team_toolset_id="ts-other") - assert kwargs["user_api_key_auth"].object_permission is None - assert kwargs["mcp_servers"] is None +@pytest.mark.parametrize( + "owner_role", + [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY], + ids=["internal_user", "proxy_admin", "proxy_admin_viewer"], +) +async def test_toolset_gateway_url_answers_403_for_a_toolset_the_team_does_not_grant( + monkeypatch: pytest.MonkeyPatch, owner_role: LitellmUserRoles +): + with pytest.raises(HTTPException) as denial: + await _tools_listed_through_the_gateway( + monkeypatch, + UserAPIKeyAuth(api_key="sk-team", team_id="team-1", user_role=owner_role), + ("litellm_proxy/mcp/team-toolset",), + _team_grants("ts-other"), + ) + assert denial.value.status_code == 403 + assert "team-toolset" in str(denial.value.detail) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "requested_toolsets", + [("team-toolset", "other-toolset"), ("other-toolset", "team-toolset")], + ids=["granted_first", "denied_first"], +) +async def test_a_denied_toolset_beside_a_granted_one_adds_nothing_to_the_scoped_key( + monkeypatch: pytest.MonkeyPatch, requested_toolsets: tuple[str, ...] +): + listed: Final = await _tools_listed_through_the_gateway( + monkeypatch, + UserAPIKeyAuth(api_key="sk-team", team_id="team-1", user_role=LitellmUserRoles.PROXY_ADMIN), + tuple(f"litellm_proxy/mcp/{name}" for name in requested_toolsets), + _team_grants("ts-granted"), + ) + assert listed == ["calc-add"] + + +@pytest.mark.asyncio +async def test_a_whole_gateway_entry_beside_an_ungranted_toolset_still_lists_the_keys_own_servers( + monkeypatch: pytest.MonkeyPatch, +): + listed: Final = await _tools_listed_through_the_gateway( + monkeypatch, + _key_holding("srv-1", role=LitellmUserRoles.PROXY_ADMIN), + ("litellm_proxy", "litellm_proxy/mcp/other-toolset"), + _team_grants("ts-granted"), + ) + assert listed == ["calc-add", "calc-subtract"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("team_id", [None, UI_TEAM_ID], ids=["no_team", "dashboard_session"]) +async def test_an_admin_credential_without_a_team_keeps_a_toolset_no_grant_names( + monkeypatch: pytest.MonkeyPatch, team_id: str | None +): + listed: Final = await _tools_listed_through_the_gateway( + monkeypatch, + UserAPIKeyAuth(api_key="sk-admin", team_id=team_id, user_role=LitellmUserRoles.PROXY_ADMIN), + ("litellm_proxy/mcp/team-toolset",), + _team_grants("ts-other"), + ) + assert listed == ["calc-add"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "lookup_error", [None, ConnectionError("database unreachable")], ids=["no_such_toolset", "toolset_lookup_fails"] +) +async def test_a_name_that_does_not_resolve_to_a_toolset_is_listed_as_a_server_name( + monkeypatch: pytest.MonkeyPatch, lookup_error: Exception | None +): + listed: Final = await _tools_listed_through_the_gateway( + monkeypatch, + _key_holding("srv-1", "srv-2"), + ("litellm_proxy", "litellm_proxy/mcp/wiki"), + _no_toolset_grants, + _toolset_table(lookup_error), + ) + assert listed == ["calc-add", "calc-subtract"] + + +@pytest.mark.asyncio +async def test_an_ungranted_toolset_named_like_a_server_the_key_holds_answers_403_once( + monkeypatch: pytest.MonkeyPatch, +): + with pytest.raises(HTTPException) as denial: + await _tools_listed_through_the_gateway( + monkeypatch, + _key_holding("srv-1", "srv-2"), + ("litellm_proxy/mcp/CALC", "litellm_proxy/mcp/CALC"), + _no_toolset_grants, + ) + assert denial.value.status_code == 403 + assert str(denial.value.detail).count("CALC") == 1 + + +@pytest.mark.asyncio +async def test_an_ungranted_toolset_named_like_a_server_adds_no_server_beside_a_named_one( + monkeypatch: pytest.MonkeyPatch, +): + listed: Final = await _tools_listed_through_the_gateway( + monkeypatch, + _key_holding("srv-1", "srv-2"), + ("litellm_proxy/mcp/docs", "litellm_proxy/mcp/CALC"), + _no_toolset_grants, + ) + assert listed == ["docs-lookup"] + + +@pytest.mark.asyncio +async def test_ungranted_toolsets_are_named_in_request_order_in_the_403(monkeypatch: pytest.MonkeyPatch): + with pytest.raises(HTTPException) as denial: + await _tools_listed_through_the_gateway( + monkeypatch, + _key_holding("srv-1", "srv-2"), + ("litellm_proxy/mcp/other-toolset", "litellm_proxy/mcp/CALC"), + _no_toolset_grants, + ) + assert denial.value.status_code == 403 + assert str(denial.value.detail).endswith("other-toolset, CALC'}") + + +@pytest.mark.asyncio +async def test_an_agent_bound_key_naming_an_ungranted_toolset_gets_the_generic_403(monkeypatch: pytest.MonkeyPatch): + with pytest.raises(HTTPException) as denial: + await _tools_listed_through_the_gateway( + monkeypatch, + _key_holding("srv-1", "srv-2").model_copy(update={"agent_id": "agent-1"}), + ("litellm_proxy/mcp/CALC",), + _no_toolset_grants, + ) + assert denial.value.status_code == 403 + assert denial.value.detail == {"error": "The key is not allowed to access the requested MCP servers: CALC"} + + +@pytest.mark.asyncio +async def test_an_access_group_the_key_holds_outranks_an_ungranted_toolset_of_the_same_name( + monkeypatch: pytest.MonkeyPatch, +): + listed: Final = await _tools_listed_through_the_gateway( + monkeypatch, + _key_holding("srv-1", "srv-2"), + ("litellm_proxy/mcp/reference",), + _no_toolset_grants, + ) + assert listed == ["docs-lookup"] + + +@pytest.mark.asyncio +async def test_an_access_group_the_keys_agent_does_not_grant_leaves_the_same_named_toolset_denied( + monkeypatch: pytest.MonkeyPatch, +): + key_holding_the_group: Final = UserAPIKeyAuth( + api_key="sk-team", + team_id="team-1", + user_role=LitellmUserRoles.INTERNAL_USER, + agent_id="agent-1", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="op-key", mcp_servers=["srv-1"], mcp_access_groups=["reference"] + ), + ) + with pytest.raises(HTTPException) as denial: + await _tools_listed_through_the_gateway( + monkeypatch, + key_holding_the_group, + ("litellm_proxy/mcp/reference",), + _no_toolset_grants, + agent_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-agent", mcp_servers=["srv-1"]), + ) + assert denial.value.status_code == 403 + assert denial.value.detail == {"error": "The key is not allowed to access the requested MCP servers: reference"} + + +@pytest.mark.asyncio +async def test_a_granted_toolset_outranks_an_access_group_of_the_same_name(monkeypatch: pytest.MonkeyPatch): + async def granted_reference_toolset(context: UserAPIKeyAuth) -> frozenset[str]: + return frozenset({"ts-group"}) + + listed: Final = await _tools_listed_through_the_gateway( + monkeypatch, + _key_holding("srv-1", "srv-2"), + ("litellm_proxy/mcp/reference",), + granted_reference_toolset, + ) + assert listed == ["calc-add"] @pytest.mark.asyncio diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 30d61f181cb..02b8a80e128 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -18082,9 +18082,9 @@ export interface paths { * @description Namespace a toolset as its own MCP endpoint. * * Connecting to /toolset//mcp exposes exactly the tools defined in - * the toolset. Access is enforced: non-admin API keys must have the toolset - * listed in their object_permission.mcp_toolsets grant list, or the request - * will be rejected with a 403. + * the toolset. Access is enforced: API keys other than an admin's key with no + * team must hold the toolset through one of their grant sources, or the + * request will be rejected with a 403. */ get: operations["toolset_mcp_route_toolset__toolset_name__mcp_get"]; /** @@ -18092,9 +18092,9 @@ export interface paths { * @description Namespace a toolset as its own MCP endpoint. * * Connecting to /toolset//mcp exposes exactly the tools defined in - * the toolset. Access is enforced: non-admin API keys must have the toolset - * listed in their object_permission.mcp_toolsets grant list, or the request - * will be rejected with a 403. + * the toolset. Access is enforced: API keys other than an admin's key with no + * team must hold the toolset through one of their grant sources, or the + * request will be rejected with a 403. */ put: operations["toolset_mcp_route_toolset__toolset_name__mcp_put"]; /** @@ -18102,9 +18102,9 @@ export interface paths { * @description Namespace a toolset as its own MCP endpoint. * * Connecting to /toolset//mcp exposes exactly the tools defined in - * the toolset. Access is enforced: non-admin API keys must have the toolset - * listed in their object_permission.mcp_toolsets grant list, or the request - * will be rejected with a 403. + * the toolset. Access is enforced: API keys other than an admin's key with no + * team must hold the toolset through one of their grant sources, or the + * request will be rejected with a 403. */ post: operations["toolset_mcp_route_toolset__toolset_name__mcp_post"]; /** @@ -18112,9 +18112,9 @@ export interface paths { * @description Namespace a toolset as its own MCP endpoint. * * Connecting to /toolset//mcp exposes exactly the tools defined in - * the toolset. Access is enforced: non-admin API keys must have the toolset - * listed in their object_permission.mcp_toolsets grant list, or the request - * will be rejected with a 403. + * the toolset. Access is enforced: API keys other than an admin's key with no + * team must hold the toolset through one of their grant sources, or the + * request will be rejected with a 403. */ delete: operations["toolset_mcp_route_toolset__toolset_name__mcp_delete"]; /** @@ -18122,9 +18122,9 @@ export interface paths { * @description Namespace a toolset as its own MCP endpoint. * * Connecting to /toolset//mcp exposes exactly the tools defined in - * the toolset. Access is enforced: non-admin API keys must have the toolset - * listed in their object_permission.mcp_toolsets grant list, or the request - * will be rejected with a 403. + * the toolset. Access is enforced: API keys other than an admin's key with no + * team must hold the toolset through one of their grant sources, or the + * request will be rejected with a 403. */ options: operations["toolset_mcp_route_toolset__toolset_name__mcp_options"]; /** @@ -18132,9 +18132,9 @@ export interface paths { * @description Namespace a toolset as its own MCP endpoint. * * Connecting to /toolset//mcp exposes exactly the tools defined in - * the toolset. Access is enforced: non-admin API keys must have the toolset - * listed in their object_permission.mcp_toolsets grant list, or the request - * will be rejected with a 403. + * the toolset. Access is enforced: API keys other than an admin's key with no + * team must hold the toolset through one of their grant sources, or the + * request will be rejected with a 403. */ head: operations["toolset_mcp_route_toolset__toolset_name__mcp_head"]; /** @@ -18142,9 +18142,9 @@ export interface paths { * @description Namespace a toolset as its own MCP endpoint. * * Connecting to /toolset//mcp exposes exactly the tools defined in - * the toolset. Access is enforced: non-admin API keys must have the toolset - * listed in their object_permission.mcp_toolsets grant list, or the request - * will be rejected with a 403. + * the toolset. Access is enforced: API keys other than an admin's key with no + * team must hold the toolset through one of their grant sources, or the + * request will be rejected with a 403. */ patch: operations["toolset_mcp_route_toolset__toolset_name__mcp_patch"]; trace?: never; From 84617874b89e8c1601725ba0d418dec5c4086a09 Mon Sep 17 00:00:00 2001 From: tessellark <349031000+tessellark@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:09:33 -0700 Subject: [PATCH 05/15] fix(ui): resolve auto router templates through ids a * catch-all lists over a partial wildcard (#49331) * fix(ui): resolve auto router templates through ids a * catch-all lists over a partial wildcard * fix(ui): resolve wildcard-listed ids by their routed model in the Simple view and named aliases A catch-all's fixed model or base_model no longer keys every listed id as that one model, a named alias like claude/* over anthropic/claude-* keys its ids by the model the router sends them to, and the Simple view suggests a pair from those ids too * fix(ui): credit a wildcard-listed id only through the pattern and provider the router uses When several named wildcards match a listed id, only the one the router's specificity sort picks is credited. A captured route counts only when its provider is one /model_group/info reports for the group, so an alias that points elsewhere keeps its own name. Paused catch-alls credit nothing, and a name with its own text before the star captures like the proxy lists it. * fix(ui): resolve model group aliases before wildcards in auto router templates An alias that shadows an id the catch-all lists now counts as the model the router sends it to, and paused deployments credit no template model * fix(ui): wait for aliases, read them as admin only, and skip paused wildcard routes in auto router templates * fix(ui): let slashless wildcard model names take part in route precedence The router puts every model_name holding a star into its pattern list, with or without a slash, so a listed id like opus-5-5 under a "*" catch-all goes to a more specific opus-* deployment first. The dashboard skipped slashless names when it picked the route, so a paused opus-* still credited opus-5-5 through the catch-all and a template could fill a tier that answers 403. The old "derives no pattern from a slashless wildcard model_name" test described /models listing, not routing, and is replaced by the routed case. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../add_auto_router_tab.integration.test.tsx | 101 ++++++- .../add_model/add_auto_router_tab.tsx | 22 +- .../components/add_model/auto_setup.test.ts | 60 ++++ .../src/components/add_model/auto_setup.ts | 34 ++- .../src/lib/autorouter_presets.test.ts | 278 +++++++++++++++++- .../src/lib/autorouter_presets.ts | 79 ++--- .../src/lib/wildcard_routes.ts | 116 ++++++++ 7 files changed, 635 insertions(+), 55 deletions(-) create mode 100644 ui/litellm-dashboard/src/lib/wildcard_routes.ts diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx index 78782e755ac..d4c74785107 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx @@ -113,8 +113,9 @@ const { mockFetchAvailableModels, mockFetchAllModelDeployments } = vi.hoisted(() mockFetchAllModelDeployments: vi.fn(), })); -const { validateAutoRouterConfig } = vi.hoisted(() => ({ +const { validateAutoRouterConfig, getRouterSettingsCall } = vi.hoisted(() => ({ validateAutoRouterConfig: vi.fn().mockResolvedValue({ valid: true }), + getRouterSettingsCall: vi.fn(), })); vi.mock("../networking", () => ({ @@ -129,6 +130,7 @@ vi.mock("../networking", () => ({ modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }), testAutoRouterRouting: vi.fn(), validateAutoRouterConfig, + getRouterSettingsCall, })); vi.mock("@/components/llm_calls/fetch_models", () => ({ @@ -196,6 +198,7 @@ describe("AddAutoRouterTab", () => { testQueryClient.clear(); mockFetchAvailableModels.mockResolvedValue([]); mockFetchAllModelDeployments.mockResolvedValue([]); + getRouterSettingsCall.mockResolvedValue({ current_values: {} }); vi.mocked(modelCostMap).mockResolvedValue({}); }); @@ -2194,6 +2197,102 @@ describe("AddAutoRouterTab", () => { expect(optionByLabel("Anthropic Family")!).toHaveTextContent(/Matches your deployments/); }); + it("enables a preset whose models a bare-star catch-all lists over a partial wildcard", async () => { + mockFetchAvailableModels.mockResolvedValue( + [...getRequiredModelsInPreset(ANTHROPIC_PRESET)].map((model) => ({ + model_group: model.replace(/^claude-/, ""), + mode: "chat", + providers: ["anthropic"], + })), + ); + mockFetchAllModelDeployments.mockResolvedValue([ + { model_name: "*", litellm_params: { model: "anthropic/claude-*" } }, + ]); + + renderAdvanced(); + openTemplateDropdown(); + + await waitFor(() => { + expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); + }); + }); + + it("names a preset model missing when an alias over its listed id routes elsewhere", async () => { + mockFetchAvailableModels.mockResolvedValue( + [...getRequiredModelsInPreset(ANTHROPIC_PRESET)].map((model) => ({ + model_group: model.replace(/^claude-/, ""), + mode: "chat", + providers: ["anthropic"], + })), + ); + mockFetchAllModelDeployments.mockResolvedValue([ + { model_name: "*", litellm_params: { model: "anthropic/claude-*" } }, + { model_name: "corp-opus", litellm_params: { model: "anthropic/claude-opus-5-5" } }, + ]); + getRouterSettingsCall.mockResolvedValue({ current_values: { model_group_alias: { "haiku-4-5": "corp-opus" } } }); + + renderAdvanced(); + openTemplateDropdown(); + + await waitFor(() => { + expect(optionByLabel("Anthropic Family")!).toHaveTextContent(/Missing: claude-haiku-4-5$/); + }); + expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(true); + }); + + it("withholds templates until the alias over a listed id has loaded", async () => { + let completeSettings: ((settings: unknown) => void) | undefined; + getRouterSettingsCall.mockReturnValue( + new Promise((resolve) => { + completeSettings = resolve; + }), + ); + mockFetchAvailableModels.mockResolvedValue( + [...getRequiredModelsInPreset(ANTHROPIC_PRESET)].map((model) => ({ + model_group: model.replace(/^claude-/, ""), + mode: "chat", + providers: ["anthropic"], + })), + ); + mockFetchAllModelDeployments.mockResolvedValue([ + { model_name: "*", litellm_params: { model: "anthropic/claude-*" } }, + { model_name: "corp-opus", litellm_params: { model: "anthropic/claude-opus-5-5" } }, + ]); + + renderAdvanced(); + openTemplateDropdown(); + await waitFor(() => expect(screen.getByLabelText("Auto Router Name")).toHaveValue("auto-router")); + + expect(optionByLabel("Anthropic Family")!).toHaveTextContent("Checking model availability..."); + expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(true); + + completeSettings?.({ current_values: { model_group_alias: { "haiku-4-5": "corp-opus" } } }); + await waitFor(() => { + expect(optionByLabel("Anthropic Family")!).toHaveTextContent(/Missing: claude-haiku-4-5$/); + }); + }); + + it("credits a catch-all's listed ids for a non-admin without reading admin-only router settings", async () => { + mockFetchAvailableModels.mockResolvedValue( + [...getRequiredModelsInPreset(ANTHROPIC_PRESET)].map((model) => ({ + model_group: model.replace(/^claude-/, ""), + mode: "chat", + providers: ["anthropic"], + })), + ); + mockFetchAllModelDeployments.mockResolvedValue([ + { model_name: "*", litellm_params: { model: "anthropic/claude-*" } }, + ]); + + renderAdvanced(); + openTemplateDropdown(); + + await waitFor(() => { + expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); + }); + expect(getRouterSettingsCall).not.toHaveBeenCalled(); + }); + it("prefills the expanded group names and submits them", async () => { const user = userEvent.setup(); mockFetchAvailableModels.mockResolvedValue(EXPANDED_HUB_GROUPS); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 8c3f402df1f..ceca03a924e 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -18,9 +18,9 @@ import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/comp import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; import { useZodForm } from "@/lib/forms/useZodForm"; import AccessGroupTagsCombobox from "./AccessGroupTagsCombobox"; -import { modelAvailableCall, validateAutoRouterConfig, type Team } from "../networking"; +import { getRouterSettingsCall, modelAvailableCall, validateAutoRouterConfig, type Team } from "../networking"; import { labelWithHint } from "@/components/shared/form/LabelWithHint"; -import { all_admin_roles } from "@/utils/roles"; +import { all_admin_roles, isProxyAdminRole } from "@/utils/roles"; import { canCreateAutoRouterForTeam, canModifyModel, type ModelWriteScope } from "@/utils/modelPermissions"; import TeamDropdown from "../common_components/team_dropdown"; import { type AddAutoRouterValues, handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit"; @@ -73,6 +73,7 @@ import { buildPresetPrefill, buildModelAvailability, deploymentRefsFromModelInfo, + modelGroupAliasTargets, ModelAvailability, PresetPrefill, AutoRouterPreset, @@ -105,6 +106,7 @@ interface AddAutoRouterTabProps { } const NO_PRESETS: AutoRouterPreset[] = []; +const NO_ALIASES: ReadonlyMap = new Map(); // Why the submit is unavailable, or null when it is available. The button reads this to disable // itself and to say what is missing, so the two can never give different answers. Checks the @@ -256,7 +258,13 @@ const AddAutoRouterTab: React.FC = ({ queryFn: () => fetchAllModelDeployments(accessToken, userId ?? "", userRole), enabled: Boolean(accessToken), }); - const modelsLoading = groupsLoading || deploymentsLoading; + const { data: aliasTargets = NO_ALIASES, isLoading: aliasesLoading } = useQuery({ + queryKey: ["routerSettings", "modelGroupAlias", accessToken], + queryFn: async () => + modelGroupAliasTargets((await getRouterSettingsCall(accessToken))?.current_values?.model_group_alias), + enabled: Boolean(accessToken) && isProxyAdminRole(userRole), + }); + const modelsLoading = groupsLoading || deploymentsLoading || aliasesLoading; useEffect(() => { if (!deployments || form.getFieldState("auto_router_name").isDirty) return; form.setValue( @@ -287,8 +295,10 @@ const AddAutoRouterTab: React.FC = ({ buildModelAvailability( modelInfo.map((m) => m.model_group), deploymentRefsFromModelInfo(deployments ?? []), + new Map(modelInfo.map((m) => [m.model_group, m.providers ?? []])), + aliasTargets, ), - [modelInfo, deployments], + [modelInfo, deployments, aliasTargets], ); const groupsOnlyAvailability = React.useMemo( () => @@ -299,8 +309,8 @@ const AddAutoRouterTab: React.FC = ({ [modelInfo], ); const solverModels = React.useMemo( - () => deployedRouterModels(modelInfo, deployments ?? []), - [modelInfo, deployments], + () => deployedRouterModels(modelInfo, deployments ?? [], aliasTargets), + [modelInfo, deployments, aliasTargets], ); const recommended = React.useMemo( () => recommendSolverPair(solverModels, goal, family), diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts index c0d4931873f..6b20cab8b1f 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts @@ -160,6 +160,66 @@ describe("deployedRouterModels", () => { expect(models[0]).toMatchObject({ name: "group", version: [5, 5], provider: "Anthropic, Bedrock" }); }); + it.each([ + ["a bare-star catch-all", "*", ""], + ["a named alias", "claude/*", "claude/"], + ])("suggests the Claude ids %s lists over a partial wildcard", (_label, name, prefix) => { + const groups = ["haiku-4-5", "opus-5-5"].map((id) => ({ + model_group: `${prefix}${id}`, + mode: "chat", + providers: ["anthropic"], + })); + const suggest = (row: AutoRouterDeployment) => + names(recommendSolverPair(deployedRouterModels(groups, [row]), "balanced", "auto")); + expect(suggest(deployment(name, "anthropic/claude-*"))).toEqual([`${prefix}haiku-4-5`, `${prefix}opus-5-5`]); + expect(suggest(deployment(name, "anthropic/claude-*", { model_info: { blocked: true } }))).toBeNull(); + }); + + it.each(["team/", ""])( + "suggests no id whose most specific wildcard %j is paused, even when a broader one is live", + (prefix) => { + const groups = ["haiku-4-5", "opus-5-5"].map((id) => ({ + model_group: `${prefix}${id}`, + mode: "chat", + providers: ["anthropic"], + })); + const rows = [ + deployment(`${prefix}*`, "anthropic/claude-*"), + deployment(`${prefix}opus-*`, "anthropic/claude-opus-*", { model_info: { blocked: true } }), + ]; + expect(names(recommendSolverPair(deployedRouterModels(groups, rows), "balanced", "auto"))).toBeNull(); + expect(names(recommendSolverPair(deployedRouterModels(groups, rows.slice(0, 1)), "balanced", "auto"))).toEqual([ + `${prefix}haiku-4-5`, + `${prefix}opus-5-5`, + ]); + }, + ); + + it("takes no Claude identity for an alias the router sends to another provider", () => { + const [model] = deployedRouterModels( + [{ model_group: "sonnet-5", mode: "chat", providers: ["openai"] }], + [deployment("*", "anthropic/claude-*")], + ); + expect(model).toEqual({ name: "sonnet-5", model: "sonnet-5", provider: "OpenAI" }); + }); + + it("takes the identity of the model an alias over a listed id routes to", () => { + const [model] = deployedRouterModels( + [{ model_group: "haiku-4-5", mode: "chat", providers: ["anthropic"] }], + [deployment("*", "anthropic/claude-*"), deployment("corp-opus", "anthropic/claude-opus-5-5")], + new Map([["haiku-4-5", "corp-opus"]]), + ); + expect(model).toMatchObject({ name: "haiku-4-5", line: "opus", version: [5, 5] }); + }); + + it("keeps a canonical alias's own identity when the catch-all's listed id does not parse", () => { + const [model] = deployedRouterModels( + [{ model_group: "claude-opus-5-5", mode: "chat", providers: ["anthropic"] }], + [deployment("*", "anthropic/claude-*")], + ); + expect(model).toMatchObject({ name: "claude-opus-5-5", line: "opus", version: [5, 5] }); + }); + it("keeps arbitrary canonical names available without pretending an unversioned alias is latest", () => { expect( deployedRouterModels([{ model_group: "claude-opus-8-12" }, { model_group: "claude-opus-latest" }], []).map( diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts index eb21e17c265..dc5774d813a 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts @@ -1,6 +1,7 @@ import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; import { isAutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; import type { ModelGroup } from "@/components/llm_calls/fetch_models"; +import { type WildcardDeployment, type WildcardRoute, wildcardRoutes } from "@/lib/wildcard_routes"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; export const ROUTER_FAMILIES = { @@ -105,6 +106,19 @@ const identityOf = (deployment: AutoRouterDeployment): ModelIdentity | null => parseModel(deployment.litellm_params?.model ?? "") ?? parseModel(deployment.model_info?.base_model ?? deployment.litellm_params?.base_model ?? ""); +const wildcardDeployments = (deployments: readonly AutoRouterDeployment[]): WildcardDeployment[] => + deployments.flatMap((row) => { + const model = row.litellm_params?.model; + return row.model_name?.includes("*") && model + ? [{ modelGroup: row.model_name, underlyingModels: [model], blocked: Boolean(row.model_info?.blocked) }] + : []; + }); + +const routedIdentities = (routes: readonly WildcardRoute[], group: ModelGroup): (ModelIdentity | null)[] => { + const routed = routes.filter(({ paused }) => !paused).map(({ model }) => parseModel(model)); + return routed.some((identity) => identity !== null) ? routed : [parseModel(group.model_group)]; +}; + const providerOf = (deployment: AutoRouterDeployment, group: ModelGroup, family?: RouterFamily): string => { const raw = deployment.litellm_params?.model ?? ""; const provider = @@ -152,16 +166,21 @@ export const routingModelGroups = (groups: readonly ModelGroup[]): ModelGroup[] export const deployedRouterModels = ( groups: readonly ModelGroup[], deployments: readonly AutoRouterDeployment[], -): SolverModel[] => - routingModelGroups(groups) + aliasTargets: ReadonlyMap = new Map(), +): SolverModel[] => { + const wildcards = wildcardDeployments(deployments); + return routingModelGroups(groups) .map((group) => { - const rows = deployments.filter((row) => row.model_name === group.model_group); + const routed = { ...group, model_group: aliasTargets.get(group.model_group) ?? group.model_group }; + const rows = deployments.filter((row) => row.model_name === routed.model_group); const active = rows.filter((row) => !row.model_info?.blocked); - const isRouter = rows.some(isAutoRouterDeployment) || group.model_group.startsWith("auto_router/"); + const isRouter = rows.some(isAutoRouterDeployment) || routed.model_group.startsWith("auto_router/"); const hasWildcard = - group.model_group.includes("*") || active.some((row) => row.litellm_params?.model?.includes("*")); - const manualOnly = isRouter || hasWildcard || (rows.length > 0 && active.length === 0); - const identities = active.length > 0 ? active.map(identityOf) : [parseModel(group.model_group)]; + routed.model_group.includes("*") || active.some((row) => row.litellm_params?.model?.includes("*")); + const routes = rows.length > 0 ? [] : wildcardRoutes(routed.model_group, wildcards, routed.providers ?? []); + const paused = rows.length > 0 ? active.length === 0 : routes.length > 0 && routes.every((route) => route.paused); + const manualOnly = isRouter || hasWildcard || paused; + const identities = active.length > 0 ? active.map(identityOf) : routedIdentities(routes, routed); const first = identities[0]; const homogeneous = first && @@ -183,6 +202,7 @@ export const deployedRouterModels = ( }; }) .toSorted((a, b) => a.name.localeCompare(b.name)); +}; const latestLines = (models: readonly RouterModel[]): RouterModel[] => models.filter( diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index f4e45e473df..f5f19c745c4 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -12,6 +12,7 @@ import { buildPresetPrefill, buildModelAvailability, deploymentRefsFromModelInfo, + modelGroupAliasTargets, normalizeModelName, resolveAvailableModel, resolveAvailableModels, @@ -646,11 +647,284 @@ describe("autorouter_presets", () => { expect(availability.underlyingIndex.size).toBe(0); }); - it("derives no pattern from a slashless wildcard model_name", () => { - const availability = buildModelAvailability(["gpt-5.4"], [wildcardDeployment("gpt*")]); + const servedBy = (provider: string, ...groups: string[]) => new Map(groups.map((group) => [group, [provider]])); + + it("resolves a model through the id a bare-star catch-all lists over a partial wildcard", () => { + const availability = buildModelAvailability( + ["claude-haiku-4-5-20251001-v1:0"], + [{ modelGroup: "*", underlyingModels: ["bedrock/us.anthropic.*"] }], + servedBy("bedrock", "claude-haiku-4-5-20251001-v1:0"), + ); + const config = simpleTierConfig("claude-haiku-4-5"); + expect(getMissingModels(config, availability)).toEqual([]); + expect(buildPresetPrefill(config, availability).complexityRouterConfig.tiers.SIMPLE).toEqual([ + "claude-haiku-4-5-20251001-v1:0", + ]); + }); + + it("keys a catch-all's listed id by the model it routes to, not by its own name", () => { + const availability = buildModelAvailability( + ["opus-5-5"], + [{ modelGroup: "*", underlyingModels: ["anthropic/claude-*"] }], + servedBy("anthropic", "opus-5-5"), + ); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBe("opus-5-5"); + }); + + it("credits no listed id to an alias the router sends to another provider", () => { + const availability = buildModelAvailability( + ["sonnet-5"], + [{ modelGroup: "*", underlyingModels: ["anthropic/claude-*"] }], + servedBy("openai", "sonnet-5"), + ); + expect(resolveAvailableModel("claude-sonnet-5", availability)).toBeUndefined(); + }); + + it("credits no listed id to a paused catch-all", () => { + const availability = buildModelAvailability( + ["opus-5-5"], + deploymentRefsFromModelInfo([ + { model_name: "*", litellm_params: { model: "anthropic/claude-*" }, model_info: { blocked: true } }, + ]), + servedBy("anthropic", "opus-5-5"), + ); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBeUndefined(); + }); + + it("keys an alias that shadows a listed id by the model its target serves", () => { + const availability = buildModelAvailability( + ["corp-opus", "haiku-4-5"], + [ + { modelGroup: "*", underlyingModels: ["anthropic/claude-*"] }, + { modelGroup: "corp-opus", underlyingModels: ["anthropic/claude-opus-5-5"] }, + ], + servedBy("anthropic", "corp-opus", "haiku-4-5"), + new Map([["haiku-4-5", "corp-opus"]]), + ); + expect(resolveAvailableModel("claude-haiku-4-5", availability)).toBeUndefined(); + expect(resolveAvailableModels("claude-opus-5-5", availability)).toEqual(["corp-opus", "haiku-4-5"]); + }); + + it("keys an alias to a listed id by the model the catch-all routes that id to", () => { + const availability = buildModelAvailability( + ["fast"], + [{ modelGroup: "*", underlyingModels: ["anthropic/claude-*"] }], + servedBy("anthropic", "fast"), + new Map([["fast", "haiku-4-5"]]), + ); + expect(resolveAvailableModel("claude-haiku-4-5", availability)).toBe("fast"); + }); + + it("keys an alias to a deployed group by that group's model, not by the id the catch-all lists", () => { + const availability = buildModelAvailability( + ["latest"], + [ + { modelGroup: "*", underlyingModels: ["anthropic/claude-*"] }, + { modelGroup: "sonnet-5", underlyingModels: ["anthropic/claude-opus-5-5"] }, + ], + servedBy("anthropic", "latest"), + new Map([["latest", "sonnet-5"]]), + ); + expect(resolveAvailableModel("claude-sonnet-5", availability)).toBeUndefined(); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBe("latest"); + }); + + it("credits no model to a group whose only deployment is paused", () => { + const availability = buildModelAvailability( + ["corp-opus"], + deploymentRefsFromModelInfo([ + { + model_name: "corp-opus", + litellm_params: { model: "anthropic/claude-opus-5-5" }, + model_info: { blocked: true }, + }, + ]), + servedBy("anthropic", "corp-opus"), + ); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBeUndefined(); + }); + + it("credits no model to a paused group named like the model", () => { + const availability = buildModelAvailability( + ["claude-opus-5-5"], + deploymentRefsFromModelInfo([ + { + model_name: "claude-opus-5-5", + litellm_params: { model: "anthropic/claude-opus-5-5" }, + model_info: { blocked: true }, + }, + ]), + servedBy("anthropic", "claude-opus-5-5"), + ); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBeUndefined(); + }); + + it("reads both alias forms the router accepts and skips malformed entries", () => { + const aliases = { + fast: "haiku-4-5", + big: { model: "corp-opus", hidden: true }, + unnamed: { hidden: true }, + blank: "", + numeric: 3, + }; + expect(modelGroupAliasTargets(aliases)).toEqual( + new Map([ + ["fast", "haiku-4-5"], + ["big", "corp-opus"], + ]), + ); + expect(modelGroupAliasTargets(undefined).size).toBe(0); + }); + + it("leaves a group another wildcard lists to that wildcard instead of the catch-all", () => { + const availability = buildModelAvailability( + ["a-cloud", "openai/gpt-6-sol", "sonnet-5"], + [ + { modelGroup: "a-cloud", underlyingModels: ["azure/gpt-6-sol"] }, + { modelGroup: "openai/*", underlyingModels: ["openai/*"] }, + { modelGroup: "*", underlyingModels: ["anthropic/claude-*"] }, + ], + servedBy("anthropic", "sonnet-5"), + ); + expect(resolveAvailableModel("gpt-6-sol", availability)).toBe("openai/gpt-6-sol"); + expect(resolveAvailableModel("claude-sonnet-5", availability)).toBe("sonnet-5"); + }); + + it.each([ + ["the text before its star ends mid-word", "anthropic/claude*"], + ["it names no provider", "claude-*"], + ])("lists nothing for a bare-star catch-all whose partial wildcard %s", (_label, underlying) => { + const availability = buildModelAvailability(["opus-5-5"], [{ modelGroup: "*", underlyingModels: [underlying] }]); expect(availability.underlyingIndex.size).toBe(0); }); + it("keeps a catch-all's declared base model from claiming every id the catch-all lists", () => { + const availability = buildModelAvailability( + ["fable-5", "sonnet-5"], + [{ modelGroup: "*", underlyingModels: ["anthropic/claude-*", "anthropic/claude-sonnet-5"] }], + servedBy("anthropic", "fable-5", "sonnet-5"), + ); + expect(resolveAvailableModel("claude-sonnet-5", availability)).toBe("sonnet-5"); + expect(resolveAvailableModel("claude-fable-5", availability)).toBe("fable-5"); + }); + + it("credits no listed id to a catch-all over a single model", () => { + const availability = buildModelAvailability( + ["team-alias"], + [{ modelGroup: "*", underlyingModels: ["anthropic/claude-opus-5-5"] }], + ); + expect(availability.underlyingIndex.size).toBe(0); + }); + + it("keys a named alias over a partial wildcard by the model it routes to", () => { + const availability = buildModelAvailability( + ["claude/haiku-4-5"], + [{ modelGroup: "claude/*", underlyingModels: ["anthropic/claude-*"] }], + servedBy("anthropic", "claude/haiku-4-5"), + ); + expect(resolveAvailableModel("claude-haiku-4-5", availability)).toBe("claude/haiku-4-5"); + }); + + it("keys a named wildcard whose prefix runs into the routed id's own words", () => { + const availability = buildModelAvailability( + ["team/gpt-4o-prod-mini"], + [{ modelGroup: "team/gpt-4o-prod*", underlyingModels: ["openai/gpt-4o*"] }], + servedBy("openai", "team/gpt-4o-prod-mini"), + ); + expect(resolveAvailableModel("gpt-4o-mini", availability)).toBe("team/gpt-4o-prod-mini"); + }); + + const anthropicTeam = { modelGroup: "team/*", underlyingModels: ["anthropic/claude-*"] }; + const bedrockOpusTeam = { modelGroup: "team/opus-*", underlyingModels: ["bedrock/anthropic.claude-sonnet-*-v1:0"] }; + const opuPrefixTeam = { modelGroup: "team/opu*", underlyingModels: ["anthropic/claude-opus-4-1"] }; + const dashFiveTeam = { modelGroup: "team/*-5", underlyingModels: ["anthropic/claude-*-5"] }; + + it.each([ + [ + "a longer pattern listed after a shorter one", + "team/opus-5-5", + [anthropicTeam, bedrockOpusTeam], + "claude-sonnet-5-5", + ], + [ + "a longer pattern listed before a shorter one", + "team/opus-5-5", + [bedrockOpusTeam, anthropicTeam], + "claude-sonnet-5-5", + ], + [ + "more regex characters breaking a length tie", + "team/opus-4-5", + [opuPrefixTeam, dashFiveTeam], + "claude-opus-4-5", + ], + ])("keys a listed id only by the wildcard the router picks, %s", (_label, group, deployments, routedModel) => { + const availability = buildModelAvailability([group], deployments, new Map([[group, ["anthropic", "bedrock"]]])); + expect(resolveAvailableModel(routedModel, availability)).toBe(group); + }); + + it("does not credit a listed id to a wildcard the router never reaches", () => { + const availability = buildModelAvailability( + ["team/opus-5-5"], + [bedrockOpusTeam, anthropicTeam], + servedBy("bedrock", "team/opus-5-5"), + ); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBeUndefined(); + }); + + it.each(["team/", ""])("credits nothing to a listed id whose most specific wildcard %j is paused", (prefix) => { + const pausedOpus = { + modelGroup: `${prefix}opus-*`, + underlyingModels: ["anthropic/claude-opus-*"], + blocked: true, + }; + const availability = buildModelAvailability( + [`${prefix}opus-5-5`, `${prefix}haiku-4-5`], + [{ ...anthropicTeam, modelGroup: `${prefix}*` }, pausedOpus], + servedBy("anthropic", `${prefix}opus-5-5`, `${prefix}haiku-4-5`), + ); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBeUndefined(); + expect(resolveAvailableModel("claude-haiku-4-5", availability)).toBe(`${prefix}haiku-4-5`); + }); + + it("keeps replacement patterns in a listed id literal", () => { + const availability = buildModelAvailability( + ["haiku-4-5$&"], + [{ modelGroup: "*", underlyingModels: ["anthropic/claude-*"] }], + servedBy("anthropic", "haiku-4-5$&"), + ); + expect(resolveAvailableModel("claude-haiku-4-5$&", availability)).toBe("haiku-4-5$&"); + }); + + it("prefers an id a native catch-all lists over a cloud group serving the same model", () => { + const availability = buildModelAvailability( + ["a-cloud", "opus-5-5"], + [ + { modelGroup: "a-cloud", underlyingModels: ["bedrock/anthropic.claude-opus-5-5"] }, + { modelGroup: "*", underlyingModels: ["anthropic/claude-*"] }, + ], + servedBy("anthropic", "opus-5-5"), + ); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBe("opus-5-5"); + }); + + it.each([ + ["its head and tail would overlap", "team/a*a", "team/a"], + ["an earlier middle part is missing", "team/*x*y*", "team/y"], + ])("does not match a named wildcard when %s", (_label, pattern, group) => { + const availability = buildModelAvailability([group], [wildcardDeployment(pattern)]); + expect(availability.underlyingIndex.size).toBe(0); + }); + + it("credits a listed id through the slashless wildcard the router sends it to", () => { + const availability = buildModelAvailability( + ["opus-5-5"], + [{ modelGroup: "opus-*", underlyingModels: ["anthropic/claude-opus-*"] }], + servedBy("anthropic", "opus-5-5"), + ); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBe("opus-5-5"); + }); + it("does not trust a group's name when no wildcard deployment covers it", () => { const availability = buildModelAvailability( ["team-a/claude-opus-5", "openai/*"], diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index fd400a8163a..827361839a0 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -29,6 +29,7 @@ import { } from "@/components/add_model/complexity_router_tiers"; import { DEFAULT_ESCALATION_KEYWORDS } from "@/components/add_model/EscalationKeywords"; import { DEFAULT_MATCH_THRESHOLD } from "@/components/add_model/SemanticKeywordMatching"; +import { wildcardRoutes } from "@/lib/wildcard_routes"; // `key` is the stable JSON object key (e.g. "anthropic_family"); `label` is display text and // never an identity. @@ -68,6 +69,7 @@ export interface DeploymentModelRef { modelGroup: string; underlyingModels: readonly string[]; provider?: string; + blocked?: boolean; } export interface ModelAvailability { @@ -110,29 +112,14 @@ const normalizeUnderlyingModel = (model: string): string | null => { return stripped.toLowerCase() || null; }; -// A linear glob scan rather than a RegExp: patterns are admin-controlled model_name values, and a -// backtracking regex built from one ("a*a*a*...") can freeze another admin's dashboard. -const matchesWildcard = (pattern: string, name: string): boolean => { - const parts = pattern.split("*"); - if (parts.length === 1) return pattern === name; - const head = parts[0]; - const tail = parts[parts.length - 1]; - if (!name.startsWith(head) || !name.endsWith(tail)) return false; - if (name.length < head.length + tail.length) return false; - const scanEnd = name.length - tail.length; - const scanResult = parts.slice(1, -1).reduce((searchFrom: number, part: string) => { - if (searchFrom < 0) return -1; - const found = name.indexOf(part, searchFrom); - return found === -1 || found + part.length > scanEnd ? -1 : found + part.length; - }, head.length); - return scanResult >= 0; -}; - export const buildModelAvailability = ( modelGroups: Iterable, deployments: readonly DeploymentModelRef[], + groupProviders: ReadonlyMap = new Map(), + aliasTargets: ReadonlyMap = new Map(), ): ModelAvailability => { const groups = new Set(modelGroups); + const routedGroup = (group: string): string => aliasTargets.get(group) ?? group; const deploymentGroups = new Set(deployments.map((deployment) => deployment.modelGroup)); const deploymentProviders = new Map>(); for (const deployment of deployments) { @@ -140,27 +127,33 @@ export const buildModelAvailability = ( providers.add(deploymentProvider(deployment)); deploymentProviders.set(deployment.modelGroup, providers); } - const literalEntries = deployments - .filter((deployment) => groups.has(deployment.modelGroup)) - .flatMap((deployment) => + const routableDeployments = deployments.filter((deployment) => !deployment.blocked); + const routableByGroup = new Map(); + for (const deployment of routableDeployments) { + routableByGroup.set(deployment.modelGroup, [...(routableByGroup.get(deployment.modelGroup) ?? []), deployment]); + } + const literalEntries = Array.from(groups).flatMap((group) => + (routableByGroup.get(routedGroup(group)) ?? []).flatMap((deployment) => deployment.underlyingModels .map(normalizeUnderlyingModel) - .map((key) => ({ key, modelGroup: deployment.modelGroup, sourceGroup: deployment.modelGroup })), - ); - // Mirrors get_known_models_from_wildcard: a bare "*" model_name expands via its underlying - // wildcard (or not at all), and a wildcard without a "/" expands to nothing. - const wildcardPatterns = deployments.flatMap((deployment) => - (deployment.modelGroup === "*" ? deployment.underlyingModels : [deployment.modelGroup]) - .filter((pattern) => pattern !== "*" && pattern.includes("*") && pattern.includes("/")) - .map((pattern) => ({ pattern, sourceGroup: deployment.modelGroup })), + .map((key) => ({ key, modelGroup: group, sourceGroup: deployment.modelGroup })), + ), + ); + const pausedGroups = new Set(Array.from(deploymentGroups).filter((group) => !routableByGroup.has(group))); + const servedGroups = new Set(Array.from(groups).filter((group) => !pausedGroups.has(routedGroup(group)))); + const unlistedGroups = Array.from(groups).filter( + (group) => !group.includes("*") && !deploymentGroups.has(routedGroup(group)), + ); + const patternDeployments = deployments.filter(({ modelGroup }) => modelGroup.includes("*")); + const wildcardEntries = unlistedGroups.flatMap((group) => + wildcardRoutes(routedGroup(group), patternDeployments, groupProviders.get(group) ?? []) + .filter(({ paused }) => !paused) + .map(({ model, sourceGroup }) => ({ + key: normalizeUnderlyingModel(model), + modelGroup: group, + sourceGroup, + })), ); - const wildcardEntries = Array.from(groups) - .filter((group) => !group.includes("*") && !deploymentGroups.has(group)) - .flatMap((group) => - wildcardPatterns - .filter(({ pattern }) => matchesWildcard(pattern, group)) - .map(({ sourceGroup }) => ({ key: normalizeUnderlyingModel(group), modelGroup: group, sourceGroup })), - ); const entries = [...literalEntries, ...wildcardEntries]; const grouped = new Map>(); const providersByGroup = new Map>(); @@ -186,14 +179,14 @@ export const buildModelAvailability = ( }), ]), ); - return { modelGroups: groups, underlyingIndex, nativeUnderlyingIndex }; + return { modelGroups: servedGroups, underlyingIndex, nativeUnderlyingIndex }; }; export const deploymentRefsFromModelInfo = ( rows: readonly { model_name?: string | null; litellm_params?: { model?: string | null; base_model?: string | null; custom_llm_provider?: string | null } | null; - model_info?: { base_model?: string | null } | null; + model_info?: { base_model?: string | null; blocked?: boolean | null } | null; }[], ): DeploymentModelRef[] => rows.flatMap((row) => { @@ -204,10 +197,18 @@ export const deploymentRefsFromModelInfo = ( ].filter((model): model is string => Boolean(model)); const provider = row.litellm_params?.custom_llm_provider || routingProvider(row.litellm_params?.model ?? ""); return row.model_name && underlyingModels.length > 0 - ? [{ modelGroup: row.model_name, underlyingModels, provider }] + ? [{ modelGroup: row.model_name, underlyingModels, provider, blocked: row.model_info?.blocked || undefined }] : []; }); +export const modelGroupAliasTargets = (aliases: unknown): ReadonlyMap => + new Map( + Object.entries(typeof aliases === "object" && aliases !== null ? aliases : {}).flatMap(([alias, entry]) => { + const target: unknown = typeof entry === "object" && entry !== null && "model" in entry ? entry.model : entry; + return typeof target === "string" && target ? [[alias, target] as const] : []; + }), + ); + export const resolveAvailableModels = (requiredModel: string, availability: ModelAvailability): readonly string[] => { const { modelGroups, underlyingIndex } = availability; if (modelGroups.has(requiredModel)) return [requiredModel]; diff --git a/ui/litellm-dashboard/src/lib/wildcard_routes.ts b/ui/litellm-dashboard/src/lib/wildcard_routes.ts new file mode 100644 index 00000000000..607e5d106a3 --- /dev/null +++ b/ui/litellm-dashboard/src/lib/wildcard_routes.ts @@ -0,0 +1,116 @@ +export type WildcardDeployment = { + readonly modelGroup: string; + readonly underlyingModels: readonly string[]; + readonly blocked?: boolean; +}; +export type WildcardRoute = { readonly model: string; readonly sourceGroup: string; readonly paused: boolean }; + +// A linear glob scan rather than a RegExp: patterns are admin-controlled model_name values, and a +// backtracking regex built from one ("a*a*a*...") can freeze another admin's dashboard. +const matchesWildcard = (pattern: string, name: string): boolean => { + const parts = pattern.split("*"); + const head = parts[0]; + const tail = parts[parts.length - 1]; + if (!name.startsWith(head) || !name.endsWith(tail)) return false; + if (name.length < head.length + tail.length) return false; + const scanEnd = name.length - tail.length; + const scanResult = parts.slice(1, -1).reduce((searchFrom: number, part: string) => { + if (searchFrom < 0) return -1; + const found = name.indexOf(part, searchFrom); + return found === -1 || found + part.length > scanEnd ? -1 : found + part.length; + }, head.length); + return scanResult >= 0; +}; + +const isProviderWide = (pattern: string): boolean => pattern.slice(pattern.indexOf("/") + 1) === "*"; + +const partialWordBeforeStar = (pattern: string): string => + pattern.slice(0, pattern.indexOf("*")).split("/").at(-1) ?? ""; + +const routesCapturedIds = (namePattern: string, deploymentPattern: string): boolean => { + const singleStar = namePattern.split("*").length === 2 && deploymentPattern.split("*").length === 2; + const listsCatalogIds = isProviderWide(namePattern) && isProviderWide(deploymentPattern); + if (!deploymentPattern.includes("/") || listsCatalogIds) return false; + const listsNameFragments = + /[a-z0-9]$/i.test(partialWordBeforeStar(deploymentPattern)) && partialWordBeforeStar(namePattern) === ""; + return singleStar && !listsNameFragments; +}; + +const substituteCapture = (namePattern: string, deploymentPattern: string, group: string): string => { + const [head, tail] = namePattern.split("*"); + return deploymentPattern.replace("*", () => group.slice(head.length, group.length - tail.length)); +}; + +const routingPatterns = (deployment: WildcardDeployment): readonly string[] => + deployment.modelGroup === "*" + ? deployment.underlyingModels.filter((model) => model !== "*" && isProviderWide(model)) + : [deployment.modelGroup].filter((pattern) => pattern.includes("*")); + +const routesThroughPattern = ( + deployment: WildcardDeployment, + pattern: string, + group: string, +): readonly WildcardRoute[] => { + const capturing = + deployment.modelGroup === "*" + ? [] + : deployment.underlyingModels.filter((model) => routesCapturedIds(pattern, model)); + const models = capturing.length > 0 ? capturing.map((model) => substituteCapture(pattern, model, group)) : [group]; + return models.map((model) => ({ model, sourceGroup: deployment.modelGroup, paused: Boolean(deployment.blocked) })); +}; + +const PYTHON_ESCAPED_CHARS = new Set(Array.from("()[]{}?*+-|^$\\.&~# \t\n\r\v\f")); +const ROUTER_COMPLEXITY_CHARS = new Set(Array.from("*+?\\^$|()")); + +const routerRegexPart = (char: string): string => { + if (char === "*") return "(.*)"; + return PYTHON_ESCAPED_CHARS.has(char) ? `\\${char}` : char; +}; + +const routerSpecificity = (namePattern: string): readonly [number, number] => { + const regex = Array.from(namePattern, routerRegexPart).join(""); + return [regex.length, Array.from(regex).filter((char) => ROUTER_COMPLEXITY_CHARS.has(char)).length]; +}; + +const isMoreSpecific = (namePattern: string, than: string): boolean => { + const [length, complexity] = routerSpecificity(namePattern); + const [otherLength, otherComplexity] = routerSpecificity(than); + return length > otherLength || (length === otherLength && complexity > otherComplexity); +}; + +const catchAllRoutes = (group: string, deployments: readonly WildcardDeployment[]): readonly WildcardRoute[] => + deployments + .filter((deployment) => deployment.modelGroup === "*") + .flatMap((deployment) => + deployment.underlyingModels + .filter((model) => routesCapturedIds("*", model)) + .map((model) => ({ + model: substituteCapture("*", model, group), + sourceGroup: "*", + paused: Boolean(deployment.blocked), + })), + ); + +const routesByPrecedence = (group: string, deployments: readonly WildcardDeployment[]): readonly WildcardRoute[] => { + const matches = deployments.flatMap((deployment) => + routingPatterns(deployment) + .filter((pattern) => matchesWildcard(pattern, group)) + .map((pattern) => ({ deployment, pattern })), + ); + if (matches.length === 0) return catchAllRoutes(group, deployments); + const routedName = matches + .map(({ deployment }) => deployment.modelGroup) + .reduce((best, name) => (isMoreSpecific(name, best) ? name : best)); + return matches + .filter(({ deployment }) => deployment.modelGroup === routedName) + .flatMap(({ deployment, pattern }) => routesThroughPattern(deployment, pattern, group)); +}; + +export const wildcardRoutes = ( + group: string, + deployments: readonly WildcardDeployment[], + groupProviders: readonly string[], +): readonly WildcardRoute[] => + routesByPrecedence(group, deployments).filter( + ({ model }) => model === group || groupProviders.includes(model.split("/")[0]), + ); From afec145edca8b8b75a7386ff4480049b120bde0e Mon Sep 17 00:00:00 2001 From: shrey Date: Fri, 30 Oct 2026 22:52:25 -0700 Subject: [PATCH 06/15] feat(passthrough): let a WebSocket relay bill a session from its client text (#49281) --- .../pass_through_endpoints.py | 17 ++++ .../pass_through_endpoints.py | 16 ++- .../test_pass_through_endpoints.py | 97 +++++++++++++++++++ 3 files changed, 129 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index cfac28db1a9..647e145aac1 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -148,6 +148,8 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, EndpointType, PassthroughStandardLoggingPayload, + WebSocketSessionMeter, + WebSocketSessionUsage, ) from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, CallTypes, Usage @@ -2595,6 +2597,13 @@ def _with_trace_context(headers: Mapping[str, str], parent_span: object) -> dict return inject_trace_context(headers, parent_span=parent_span) +def _log_session_usage(logging_obj: LiteLLMLoggingObj, usage: WebSocketSessionUsage) -> None: + logging_obj.model = usage.model # rebind-ok: the spend logger reads model and cost off the shared logging object + logging_obj.model_call_details["model"] = usage.model # rebind-ok: same shared logging object + logging_obj.model_call_details["custom_llm_provider"] = usage.custom_llm_provider # rebind-ok: same + logging_obj.model_call_details["response_cost"] = usage.response_cost # rebind-ok: same + + async def websocket_passthrough_request( websocket: WebSocket, target: str, @@ -2607,6 +2616,7 @@ async def websocket_passthrough_request( setup_model_rewriter: Callable[[str], str] | None = None, builtin_provider_route: bool = False, model: str | None = None, + meter: WebSocketSessionMeter | None = None, ): """ WebSocket passthrough request handler. @@ -2622,6 +2632,7 @@ async def websocket_passthrough_request( setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream builtin_provider_route: Whether a built-in provider route serves the connection, so its URL names the provider model: The model the caller named before connecting, logged on a failure before the setup frame arrives + meter: Prices the session from the client text frames delivered upstream, for an upstream that reports no usage """ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj @@ -2632,6 +2643,7 @@ async def websocket_passthrough_request( # Initialize tracking variables start_time: Final = datetime.now() websocket_messages: Final[list[dict[str, object]]] = [] + metered_total = 0 # rebind-ok: forward_client_to_upstream adds the measure of each client text frame it delivers litellm_call_id: Final = str(uuid.uuid4()) verbose_proxy_logger.info("WebSocket passthrough (%s): Starting WebSocket connection to %s", endpoint, target) @@ -2750,6 +2762,7 @@ async def websocket_passthrough_request( async def forward_client_to_upstream() -> None: """Forward messages from client to upstream WebSocket""" + nonlocal metered_total try: while True: message = await websocket.receive() @@ -2819,6 +2832,8 @@ async def websocket_passthrough_request( # Not a JSON message or doesn't contain setup data await upstream_ws.send(_rewrite_vertex_live_setup_model(text_data, setup_model_rewriter)) + if meter is not None: + metered_total += meter.measure(text_data) elif bytes_data is not None: await upstream_ws.send(bytes_data) except asyncio.CancelledError: @@ -2945,6 +2960,8 @@ async def websocket_passthrough_request( self.url = target_url mock_response: Final = MockWebSocketResponse(target) + if meter is not None: + _log_session_usage(logging_obj, meter.price(metered_total)) # Use the same success handler as HTTP passthrough endpoints GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( diff --git a/litellm/types/passthrough_endpoints/pass_through_endpoints.py b/litellm/types/passthrough_endpoints/pass_through_endpoints.py index 0d99cd54a6b..2f269dc87d9 100644 --- a/litellm/types/passthrough_endpoints/pass_through_endpoints.py +++ b/litellm/types/passthrough_endpoints/pass_through_endpoints.py @@ -1,5 +1,6 @@ +from dataclasses import dataclass from enum import Enum -from typing import Final +from typing import Final, Protocol from pydantic import Field from typing_extensions import TypedDict @@ -75,6 +76,19 @@ class PassthroughStandardLoggingPayload(TypedDict, total=False): """ +@dataclass(frozen=True, slots=True) +class WebSocketSessionUsage: + model: str + custom_llm_provider: str + response_cost: float | None + + +class WebSocketSessionMeter(Protocol): + def measure(self, client_text: str) -> int: ... + + def price(self, total: int) -> WebSocketSessionUsage: ... + + class PassThroughGuardrailSettings(LiteLLMPydanticObjectBase): """ Settings for a specific guardrail on a passthrough endpoint. diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index a9098ac3d59..b7e76ce0068 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -77,6 +77,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, EndpointType, PassthroughStandardLoggingPayload, + WebSocketSessionUsage, ) from tests._master_key import MASTER_KEY from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -7736,6 +7737,102 @@ async def test_websocket_passthrough_sends_deepgram_audio_bytes_and_control_text upstream_ws.close.assert_awaited_once() +@dataclass(frozen=True, slots=True) +class _LengthMeter: + rate: float + + def measure(self, client_text: str) -> int: + return len(client_text) + + def price(self, total: int) -> WebSocketSessionUsage: + return WebSocketSessionUsage(model="voice-1", custom_llm_provider="tts-provider", response_cost=total * self.rate) + + +@pytest.mark.asyncio +async def test_websocket_passthrough_logs_the_meters_price_for_the_client_text_it_delivered(): + """Text frames reach the upstream unchanged and are measured, binary frames are not, and the session is logged at + the model, provider and cost the meter prices for the total.""" + frames = ("hello", b"\x00\x01binary", "world!") + upstream_ws = RecordingUpstreamWebSocket() + websocket = _client_websocket( + AsyncMock( + side_effect=[ + *( + {"type": "websocket.receive", "bytes": frame} + if isinstance(frame, bytes) + else {"type": "websocket.receive", "text": frame} + for frame in frames + ), + {"type": "websocket.disconnect"}, + ] + ) + ) + + with ( + _patched_websocket_passthrough_environment(upstream_ws), + patch( # test-quality-ok: pass_through_endpoint_logging is a module global read inside websocket_passthrough_request; there is no injection seam + "litellm.proxy.pass_through_endpoints.pass_through_endpoints." + "pass_through_endpoint_logging.pass_through_async_success_handler", + new=AsyncMock(), + ) as success_handler, + ): + await websocket_passthrough_request( + websocket=websocket, + target="wss://upstream.example/v1/stream", + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(), + forward_headers=False, + endpoint="/provider/v1/stream", + accept_websocket=False, + meter=_LengthMeter(rate=0.5), + ) + + assert [call.args[0] for call in upstream_ws.send.await_args_list] == list(frames) + logging_obj = success_handler.call_args.kwargs["logging_obj"] + assert logging_obj.model == "voice-1" + assert logging_obj.model_call_details["model"] == "voice-1" + assert logging_obj.model_call_details["custom_llm_provider"] == "tts-provider" + assert logging_obj.model_call_details["response_cost"] == (len("hello") + len("world!")) * 0.5 + + +@pytest.mark.asyncio +async def test_websocket_passthrough_meters_only_client_text_the_upstream_accepted(): + upstream_ws = RecordingUpstreamWebSocket() + upstream_ws.send = AsyncMock(side_effect=[None, ConnectionError("upstream went away")]) + websocket = _client_websocket( + AsyncMock( + side_effect=[ + {"type": "websocket.receive", "text": "hello"}, + {"type": "websocket.receive", "text": "never delivered"}, + {"type": "websocket.disconnect"}, + ] + ) + ) + + with ( + _patched_websocket_passthrough_environment(upstream_ws), + patch( # test-quality-ok: pass_through_endpoint_logging is a module global read inside websocket_passthrough_request; there is no injection seam + "litellm.proxy.pass_through_endpoints.pass_through_endpoints." + "pass_through_endpoint_logging.pass_through_async_success_handler", + new=AsyncMock(), + ) as success_handler, + ): + await websocket_passthrough_request( + websocket=websocket, + target="wss://upstream.example/v1/stream", + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(), + forward_headers=False, + endpoint="/provider/v1/stream", + accept_websocket=False, + meter=_LengthMeter(rate=0.5), + ) + + logging_obj = success_handler.call_args.kwargs["logging_obj"] + assert [call.args[0] for call in upstream_ws.send.await_args_list] == ["hello", "never delivered"] + assert logging_obj.model_call_details["response_cost"] == len("hello") * 0.5 + + @pytest.mark.asyncio async def test_websocket_passthrough_vertex_live_setup_ack_names_the_model_but_is_not_billed_as_usage(): """Vertex Live keeps its special first frame: the setup acknowledgement is forwarded verbatim, read for the From d1d6d550631d0aaa3a2dc3eeaf48e7d3379cbcec Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:11:34 -0700 Subject: [PATCH 07/15] fix(hosted_vllm): accept the vLLM production-stack router's file and batch objects (#49068) * fix(files): default a missing file object type to "file" so OpenAI-compatible servers that omit it stop 500ing * fix(batches): accept the vLLM production-stack router's batch object The router leaves out the batch object's "object" field and reports "pending" and "running" statuses, so batch create and retrieve through a hosted_vllm deployment failed validation with a 500. A missing or null object now reads as "batch", pending maps to validating and running to in_progress, and any other unknown value is still rejected * fix(batches): return OpenAI batch objects from batch list behind the vLLM router * fix(batches): type the batch list conversion without a TypeVar and refuse an async client on the sync path * fix(batches): keep batch list items as OpenAI's Batch so an odd item never fails a list * refactor(types): reuse the shared JSON value alias on the batch type * fix: return the provider page for an async client in sync batch list and pass through non-string list statuses * test: inject the SDK's own transport in the sync batch list test so it passes on OpenAI SDK 3.x --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/llms/openai/openai.py | 22 ++- litellm/types/llms/openai.py | 7 +- litellm/types/utils.py | 30 ++- tests/unit/batches/test_main.py | 186 +++++++++++++++++- tests/unit/files/test_main.py | 56 ++++++ tests/unit/llms/openai/test_openai.py | 41 ++++ .../unit/types/llms/test_types_llms_openai.py | 17 ++ 7 files changed, 349 insertions(+), 10 deletions(-) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index bdd9f285823..6e22ef9112e 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -16,6 +16,7 @@ from openai._base_client import make_request_options from openai._constants import RAW_RESPONSE_HEADER from openai._legacy_response import LegacyAPIResponse from openai._types import RequestOptions +from openai.pagination import SyncCursorPage from openai.types import CreateEmbeddingResponse from openai.types.beta.assistant_deleted import AssistantDeleted from openai.types.chat import ChatCompletion, ChatCompletionChunk @@ -40,6 +41,7 @@ from litellm.types.utils import ( LiteLLMBatch, ModelResponse, ModelResponseStream, + openai_batch_status, ) from litellm.utils import ( CustomStreamWrapper, @@ -2182,6 +2184,18 @@ class OpenAIFilesAPI(BaseLLM): return response +def _with_openai_batch_fields(batches: Sequence[Batch]) -> Sequence[Batch]: + return [ + batch.model_copy( + update={ + "object": batch.object or "batch", + "status": openai_batch_status(batch.status), + } + ) + for batch in batches + ] + + class OpenAIBatchesAPI(BaseLLM): """ OpenAI methods to support for batches @@ -2372,7 +2386,7 @@ class OpenAIBatchesAPI(BaseLLM): ): verbose_logger.debug("listing batches, after= %s, limit= %s", after, limit) response: Final = await openai_client.batches.list(after=after, limit=limit) - return response + return response.model_copy(update={"data": _with_openai_batch_fields(response.data)}) def list_batches( self, @@ -2384,7 +2398,7 @@ class OpenAIBatchesAPI(BaseLLM): organization: str | None, after: str | None = None, limit: int | None = None, - client: OpenAI | None = None, + client: OpenAI | AsyncOpenAI | None = None, litellm_params: Mapping[str, object] | None = None, ): openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client( @@ -2409,7 +2423,9 @@ class OpenAIBatchesAPI(BaseLLM): ) return self.alist_batches(openai_client=openai_client, after=after, limit=limit) response: Final = openai_client.batches.list(after=after, limit=limit) - return response + if not isinstance(response, SyncCursorPage): + return response + return response.model_copy(update={"data": _with_openai_batch_fields(response.data)}) class OpenAIAssistantsAPI(BaseLLM): diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index a54d66c4beb..ee86ec52327 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -402,7 +402,7 @@ class OpenAIFileObject(LiteLLMBaseModel): filename: str """The name of the file.""" - object: Literal["file"] + object: Literal["file"] = "file" """The object type, which is always `file`.""" purpose: OpenAIFilesPurpose @@ -440,6 +440,11 @@ class OpenAIFileObject(LiteLLMBaseModel): _hidden_params: dict = PrivateAttr(default={"response_cost": 0.0}) # no cost for writing a file + @field_validator("object", mode="before") + @classmethod + def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue: + return "file" if value is None else value + @property def hidden_params(self) -> dict[str, builtins.object]: # mutable-ok: API requires mutation return self._hidden_params diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 0c3440e88f7..2f6918d0744 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -106,6 +106,7 @@ from .llms.openai import ( OpenAIRealtimeStreamList, ResponsesAPIResponse, WebSearchOptions, + _JsonValue, ) from .rerank import RerankResponse as RerankResponse @@ -4963,29 +4964,50 @@ def _batch_usage_from_chat_usage(usage: Usage) -> BatchUsage: ) +OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType( + {"pending": "validating", "running": "in_progress"} +) + + +def openai_batch_status(status: _JsonValue) -> _JsonValue: + return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(status, status) if isinstance(status, str) else status + + class LiteLLMBatch(Batch): _hidden_params: dict = {} + object: Literal["batch"] = "batch" + + @field_validator("object", mode="before") + @classmethod + def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue: + return "batch" if value is None else value + + @field_validator("status", mode="before") + @classmethod + def _map_vllm_router_status(cls, value: _JsonValue) -> _JsonValue: + return openai_batch_status(value) + @property - def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation + def hidden_params(self) -> dict[str, _JsonValue]: # mutable-ok: API requires mutation return self._hidden_params @hidden_params.setter - def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation + def hidden_params(self, hidden_params: dict[str, _JsonValue]) -> None: # mutable-ok: API requires mutation self._hidden_params = hidden_params usage: Usage | None = None @field_validator("usage", mode="before") @classmethod - def _usage_from_batch_token_counts(cls, value: object) -> object: + def _usage_from_batch_token_counts(cls, value: _JsonValue) -> _JsonValue: counts: Final = _parse_batch_token_counts(value) return value if counts is None else _usage_with_batch_token_counts(Usage.model_validate(value), counts) @field_serializer("usage") def _serialize_usage_with_batch_fields( self, usage: Usage | None, info: FieldSerializationInfo - ) -> Mapping[str, object] | None: + ) -> Mapping[str, _JsonValue] | None: if not isinstance(usage, Usage): return usage chat_fields: Final = usage.model_dump( diff --git a/tests/unit/batches/test_main.py b/tests/unit/batches/test_main.py index fb97e0e4919..8b22395f462 100644 --- a/tests/unit/batches/test_main.py +++ b/tests/unit/batches/test_main.py @@ -37,13 +37,15 @@ import litellm.batches.main as bm import asyncio import datetime import json -from collections.abc import Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from typing import Final import httpx import respx -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm.integrations.custom_logger import CustomLogger +from openai.types.batch import Batch +from litellm.types.utils import LiteLLMBatch # --------------------------------------------------------------------------- # @@ -1204,3 +1206,183 @@ async def test_delete_batch_output_file(respx_mock: respx.MockRouter, monkeypatc assert delete_route.call_count == 1 assert delete_response.id == "file-output123" assert delete_response.deleted is True + + +VLLM_ROUTER_API_BASE: Final = "http://vllm-router.test/v1" +VLLM_ROUTER_BATCH: Final = MappingProxyType( + { + "id": "batch_5f8dac", + "status": "pending", + "input_file_id": "file-b73f4c", + "created_at": 1793239207, + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "output_file_id": None, + "total_requests": None, + "completed_requests": 0, + "failed_requests": 0, + "metadata": {}, + } +) + + +async def _create_on_vllm_router() -> LiteLLMBatch: + return await litellm.acreate_batch( + completion_window="24h", + endpoint="/v1/chat/completions", + input_file_id="file-b73f4c", + custom_llm_provider="hosted_vllm", + api_key="sk-test", + api_base=VLLM_ROUTER_API_BASE, + ) + + +async def _retrieve_from_vllm_router() -> LiteLLMBatch: + return await litellm.aretrieve_batch( + batch_id="batch_5f8dac", custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE + ) + + +VLLM_ROUTER_BATCH_CALLS: Final = ( + pytest.param("POST", "/batches", _create_on_vllm_router, id="acreate_batch"), + pytest.param("GET", "/batches/batch_5f8dac", _retrieve_from_vllm_router, id="aretrieve_batch"), +) + + +@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_BATCH_CALLS) +@pytest.mark.parametrize( + "router_status, openai_status", + ( + ("pending", "validating"), + ("running", "in_progress"), + ("completed", "completed"), + ("failed", "failed"), + ("cancelled", "cancelled"), + ), +) +async def test_hosted_vllm_router_batch_comes_back_as_an_openai_batch( + respx_mock: respx.MockRouter, + method: str, + path: str, + call_router: Callable[[], Awaitable[LiteLLMBatch]], + router_status: str, + openai_status: str, +): + respx_mock.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond( + 200, json={**VLLM_ROUTER_BATCH, "status": router_status} + ) + + batch: Final = await call_router() + + assert (batch.id, batch.object, batch.status) == ("batch_5f8dac", "batch", openai_status) + + +@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_BATCH_CALLS) +@pytest.mark.parametrize("field, value", (("status", "queued"), ("object", "file"))) +async def test_hosted_vllm_router_batch_with_an_unknown_value_is_rejected( + respx_mock: respx.MockRouter, + method: str, + path: str, + call_router: Callable[[], Awaitable[LiteLLMBatch]], + field: str, + value: str, +): + respx_mock.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond( + 200, json={**VLLM_ROUTER_BATCH, field: value} + ) + + with pytest.raises(ValidationError) as exc_info: + await call_router() + + assert [error["loc"] for error in exc_info.value.errors()] == [(field,)] + + +async def _list_on_vllm_router() -> Sequence[Batch]: + page: Final = await litellm.alist_batches( + custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE + ) + return page.data + + +async def _list_on_vllm_router_sync() -> Sequence[Batch]: + return litellm.list_batches(custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE).data + + +@pytest.mark.parametrize("list_router", (_list_on_vllm_router, _list_on_vllm_router_sync)) +async def test_hosted_vllm_router_batch_list_comes_back_as_openai_batches( + respx_mock: respx.MockRouter, list_router: Callable[[], Awaitable[Sequence[Batch]]] +): + respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond( + 200, + json={ + "object": "list", + "data": [ + {**VLLM_ROUTER_BATCH, "id": "batch_5f8dac", "status": "pending"}, + {**VLLM_ROUTER_BATCH, "id": "batch_c74802", "status": "running"}, + ], + "first_id": "batch_5f8dac", + "last_id": "batch_c74802", + "has_more": False, + }, + ) + + batches: Final = await list_router() + + assert [(batch.id, batch.object, batch.status) for batch in batches] == [ + ("batch_5f8dac", "batch", "validating"), + ("batch_c74802", "batch", "in_progress"), + ] + + +async def test_hosted_vllm_router_batch_list_passes_through_a_status_that_is_not_a_string( + respx_mock: respx.MockRouter, +): + respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond( + 200, + json={ + "object": "list", + "data": [{**VLLM_ROUTER_BATCH, "status": ["pending"]}], + "first_id": "batch_5f8dac", + "last_id": "batch_5f8dac", + "has_more": False, + }, + ) + + batches: Final = await _list_on_vllm_router() + + assert [(batch.id, batch.object, batch.status) for batch in batches] == [("batch_5f8dac", "batch", ["pending"])] + + +async def test_openai_batch_list_items_come_back_as_openai_sent_them(respx_mock: respx.MockRouter): + respx_mock.get("http://openai.test/v1/batches").respond( + 200, + json={ + "object": "list", + "data": [ + { + **_OPENAI_BATCH_JSON, + "status": "completed", + "usage": { + "input_tokens": 10, + "input_tokens_details": {"cached_tokens": 4}, + "output_tokens": 5, + "output_tokens_details": {"reasoning_tokens": 2}, + "total_tokens": 15, + }, + }, + {**_OPENAI_BATCH_JSON, "id": "batch_def456", "status": "queued"}, + ], + "first_id": "batch_abc123", + "last_id": "batch_def456", + "has_more": False, + }, + ) + + page: Final = await litellm.alist_batches( + custom_llm_provider="openai", api_key="sk-test", api_base="http://openai.test/v1" + ) + + first, second = page.data + assert first.usage is not None + assert (first.usage.input_tokens_details.cached_tokens, first.usage.output_tokens_details.reasoning_tokens) == (4, 2) + assert (second.id, second.object, second.status) == ("batch_def456", "batch", "queued") diff --git a/tests/unit/files/test_main.py b/tests/unit/files/test_main.py index e91951fb5ba..44aadc9014e 100644 --- a/tests/unit/files/test_main.py +++ b/tests/unit/files/test_main.py @@ -1,3 +1,4 @@ +from collections.abc import Awaitable, Callable from types import MappingProxyType from typing import Final from urllib.parse import parse_qs, urlparse @@ -124,6 +125,61 @@ async def test_afile_retrieve_rejects_a_provider_file_without_its_size(): assert [error["loc"] for error in exc_info.value.errors()] == [("bytes",)] +VLLM_ROUTER_API_BASE: Final = "http://vllm-router.test/v1" +VLLM_ROUTER_FILE: Final = MappingProxyType( + {"id": "file-b73f4c", "bytes": 156, "created_at": 1793238707, "filename": "batch_requests.jsonl", "purpose": "batch"} +) + + +async def _upload_to_vllm_router() -> OpenAIFileObject: + return await litellm.acreate_file( + file=("batch_requests.jsonl", b'{"custom_id": "request-1"}'), + purpose="batch", + custom_llm_provider="hosted_vllm", + api_key="sk-test", + api_base=VLLM_ROUTER_API_BASE, + ) + + +async def _retrieve_from_vllm_router() -> OpenAIFileObject: + return await litellm.afile_retrieve( + file_id="file-b73f4c", custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE + ) + + +VLLM_ROUTER_FILE_CALLS: Final = ( + pytest.param("POST", "/files", _upload_to_vllm_router, id="acreate_file"), + pytest.param("GET", "/files/file-b73f4c", _retrieve_from_vllm_router, id="afile_retrieve"), +) + + +@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_FILE_CALLS) +@respx.mock +async def test_hosted_vllm_file_without_an_object_type_comes_back_as_a_file( + method: str, path: str, call_router: Callable[[], Awaitable[OpenAIFileObject]] +): + respx.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(200, json=dict(VLLM_ROUTER_FILE)) + + file_object: Final = await call_router() + + assert file_object.model_dump(exclude_none=True) == {**VLLM_ROUTER_FILE, "object": "file"} + + +@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_FILE_CALLS) +@respx.mock +async def test_hosted_vllm_file_with_a_different_object_type_is_rejected( + method: str, path: str, call_router: Callable[[], Awaitable[OpenAIFileObject]] +): + respx.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond( + 200, json={**VLLM_ROUTER_FILE, "object": "batch"} + ) + + with pytest.raises(ValidationError) as exc_info: + await call_router() + + assert [error["loc"] for error in exc_info.value.errors()] == [("object",)] + + _FILE_BODY: Final = b'{"prompt": "Hello", "completion": "Hi"}' _FINE_TUNE_FILE_JSON: Final = MappingProxyType( { diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 3e1cbad9308..77a7c3b83fd 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -24,6 +24,7 @@ from litellm.llms.openai.openai import ( AsyncAssistantEventHandler, AsyncCursorPage, MessageData, + OpenAIBatchesAPI, OpenAIChatCompletion, OpenAIConfig, OpenAIMessage as Message, @@ -2320,3 +2321,43 @@ def test_openai_config_hoists_a_later_developer_message_only_for_a_model_off_ope model_info=None, ) assert [message["role"] for message in messages] == expected_roles + + +@pytest.mark.asyncio +async def test_sync_batch_list_given_an_async_client_returns_a_page_the_caller_can_await( + http_backend: ModuleType, +): + def respond(request: _Request) -> httpx.Response | SDKResponse: + assert str(request.url).startswith("http://vllm-router.test/v1/batches") + return http_backend.Response( + 200, + json={ + "object": "list", + "data": [ + { + "id": "batch_5f8dac", + "status": "pending", + "input_file_id": "file-b73f4c", + "created_at": 1793239207, + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + } + ], + "first_id": "batch_5f8dac", + "last_id": "batch_5f8dac", + "has_more": False, + }, + ) + + async with http_backend.AsyncClient(transport=http_backend.MockTransport(respond)) as http_client: + page: Final = OpenAIBatchesAPI().list_batches( + _is_async=False, + api_key="sk-test", + api_base="http://vllm-router.test/v1", + timeout=5.0, + max_retries=0, + organization=None, + client=AsyncOpenAI(api_key="sk-test", base_url="http://vllm-router.test/v1", http_client=http_client), + ) + + assert [(batch.id, batch.status) for batch in (await page).data] == [("batch_5f8dac", "pending")] diff --git a/tests/unit/types/llms/test_types_llms_openai.py b/tests/unit/types/llms/test_types_llms_openai.py index 1b7a9693f7b..05801c9feec 100644 --- a/tests/unit/types/llms/test_types_llms_openai.py +++ b/tests/unit/types/llms/test_types_llms_openai.py @@ -607,6 +607,23 @@ def test_openai_file_object_accepts_pending_status(): assert file_obj.status == "pending" +@pytest.mark.parametrize("object_type", ({}, {"object": None}), ids=["missing", "null"]) +def test_openai_file_object_without_an_object_type_validates_as_a_file(object_type: dict[str, None]) -> None: + from litellm.types.llms.openai import OpenAIFileObject + + server_file: Final = { + "id": "file-b73f4c", + "bytes": 156, + "created_at": 1793238707, + "filename": "batch_requests.jsonl", + "purpose": "batch", + } + + file_obj: Final = OpenAIFileObject.model_validate({**server_file, **object_type}) + + assert file_obj.model_dump(exclude_none=True) == {**server_file, "object": "file"} + + class TestOpenAIFileObjectBatchGuardrailSerialization: """The proxy-only `litellm_batch_guardrail` key must reach the wire only when something set it.""" From a392a39627f51bf0369944556c22268ee8cffe83 Mon Sep 17 00:00:00 2001 From: simonp9 <349053457+simonp9@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:14:51 -0700 Subject: [PATCH 08/15] fix(vertex_ai): bill Claude web search per request on deployments priced in model_info (#49160) * fix(vertex_ai): bill Claude web search per request on deployments priced in model_info A vertex_ai deployment priced in its model_info is registered under its hashed id, so the web search cost check on the model key never saw "claude" and billed Claude searches with the Gemini grounding calculator: one search per call, or Gemini's $0.035 default with no search price. The check now also reads the model the response names. * fix(vertex_ai): bill token-priced Claude deployments at the served model's search price A Vertex Claude deployment priced in model_info without a search price now bills each web search at the served model's cost map price, read at cost time and never written into the deployment entry. An explicit search price, zero included, still wins. * fix(vertex_ai): price web search on the Vertex Claude Sonnet 4.5 and Haiku 4.5 rows Google bills Claude web search at $10 per 1,000 searches on these models, so token-priced deployments of them no longer bill searches at 0 * fix(cost): bill served-model search price on token-only Anthropic and Azure AI Claude deployments Token-only Claude deployments on anthropic and azure_ai billed web search at 0 because the deployment entry has no search price. They now fall back to the served model's cost map search price, as vertex_ai already does. Vertex routing reads the served model first, so a Gemini deployment whose id contains claude keeps per-prompt grounding * fix(proxy): price streamed usage frames under the deployment's model so usage.cost matches the spend row * test(proxy): give the Vertex streamRawPredict logging doubles the request's model, as a real pass-through Logging has --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../llm_cost_calc/tool_call_cost_tracking.py | 3 + litellm/llms/__init__.py | 47 +++- ...odel_prices_and_context_window_backup.json | 20 ++ litellm/proxy/common_request_processing.py | 3 +- model_prices_and_context_window.json | 20 ++ .../cost_calculation/cases/vertex_ai.py | 237 +++++++++++++++++- .../test_tool_call_cost_tracking.py | 172 ++++++++++++- .../test_streaming_handler_interrupt.py | 5 +- .../test_success_handler.py | 2 + .../proxy/test_common_request_processing.py | 67 +++++ 10 files changed, 566 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 045869e4547..54213f40ac0 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -208,11 +208,14 @@ class StandardBuiltInToolCostTracking: usage=usage, response_object=response_object ) + response_model: Final = _output_item_field(response_object, "model") if model_info is not None and resolved_usage is not None and resolved_provider is not None: result: Final = get_cost_for_web_search_request( custom_llm_provider=resolved_provider, usage=resolved_usage, model_info=model_info, + response_model=response_model if isinstance(response_model, str) else None, + lookup_model_info=StandardBuiltInToolCostTracking._safe_get_model_info, ) if result is not None: return result diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index c18ce8cea68..84c91be9386 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -9,6 +9,8 @@ from litellm.types.utils import CallTypes from . import * if TYPE_CHECKING: + from collections.abc import Callable + from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, ) @@ -30,7 +32,29 @@ def get_cost_for_google_maps_grounding_request( return cost_per_google_maps_grounding_request(usage=usage, model_info=model_info) -def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo") -> float | None: +def _with_served_model_search_pricing( + model_info: "ModelInfo", + response_model: str | None, + served_model_provider: str, + lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None", +) -> "ModelInfo": + if model_info.get("search_context_cost_per_query") is not None or response_model is None: + return model_info + served_model_info: Final = ( + lookup_model_info(response_model, served_model_provider) if lookup_model_info is not None else None + ) + if served_model_info is None: + return model_info + return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")} + + +def get_cost_for_web_search_request( + custom_llm_provider: str, + usage: "Usage", + model_info: "ModelInfo", + response_model: str | None = None, + lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None" = None, +) -> float | None: """ Get the cost for a web search request for a given model. @@ -38,6 +62,8 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo custom_llm_provider: The custom LLM provider. usage: The usage object. model_info: The model info. + response_model: The model the provider response names. + lookup_model_info: Looks up a model's cost map entry, returning None when it is not mapped. """ if custom_llm_provider == "gemini": from .gemini.cost_calculator import cost_per_web_search_request @@ -46,17 +72,25 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo elif custom_llm_provider == "anthropic": from .anthropic.cost_calculation import get_cost_for_anthropic_web_search - return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage) + return get_cost_for_anthropic_web_search( + model_info=_with_served_model_search_pricing(model_info, response_model, "anthropic", lookup_model_info), + usage=usage, + ) elif custom_llm_provider.startswith("vertex_ai"): # Anthropic Claude models on Vertex AI populate server_tool_use.web_search_requests # (same as the direct Anthropic API), not prompt_tokens_details.web_search_requests # (which is the Gemini field). Route claude-* models to the Anthropic calculator. model_key: Final[str] = model_info.get("key", "") if model_info else "" - if "claude" in model_key.lower(): + if "claude" in (response_model or model_key).lower(): from .anthropic.cost_calculation import get_cost_for_anthropic_web_search verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator") - return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage) + return get_cost_for_anthropic_web_search( + model_info=_with_served_model_search_pricing( + model_info, response_model, "vertex_ai", lookup_model_info + ), + usage=usage, + ) from .vertex_ai.gemini.cost_calculator import ( cost_per_web_search_request as cost_per_web_search_request_vertex_ai, @@ -71,7 +105,10 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo return None from .anthropic.cost_calculation import get_cost_for_anthropic_web_search - return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage) + return get_cost_for_anthropic_web_search( + model_info=_with_served_model_search_pricing(model_info, response_model, "azure_ai", lookup_model_info), + usage=usage, + ) elif custom_llm_provider == "perplexity": # Perplexity handles search costs internally in its own cost calculator # Return 0.0 to indicate costs are already accounted for diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4d503952600..cbe502e5cae 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -51387,6 +51387,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -51415,6 +51420,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -52161,6 +52171,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -52266,6 +52281,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, "regional_endpoint_uplift_multiplier": 1.1, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index c9b6fff3a00..5599086d5c1 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -5031,8 +5031,9 @@ class ProxyBaseLLMRequestProcessing: if stream_usage is None: return None service_tier: Final = obj.get("service_tier") + served_model: Final = litellm_logging_obj.model if litellm_logging_obj is not None else None cost_val: Final = ProxyBaseLLMRequestProcessing._streamed_usage_cost( - ModelResponse(usage=stream_usage), + ModelResponse(model=served_model, usage=stream_usage), model_name, service_tier if isinstance(service_tier, str) else None, litellm_logging_obj, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4d503952600..cbe502e5cae 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -51387,6 +51387,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -51415,6 +51420,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -52161,6 +52171,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -52266,6 +52281,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, "regional_endpoint_uplift_multiplier": 1.1, diff --git a/tests/integration/cost_calculation/cases/vertex_ai.py b/tests/integration/cost_calculation/cases/vertex_ai.py index 2fb005dbeca..8590a19646b 100644 --- a/tests/integration/cost_calculation/cases/vertex_ai.py +++ b/tests/integration/cost_calculation/cases/vertex_ai.py @@ -3,7 +3,7 @@ PARITY pairs the streamed case with its plain twin where both bill the same row.""" import json -from typing import Final, cast +from typing import Final, Literal, cast from pydantic import JsonValue @@ -41,6 +41,7 @@ from integration.cost_calculation.cases.anthropic import ( from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs from integration.cost_calculation.cost_tracking_case import ( CostTrackingTestCase, + Deployment, ExactExpected, JsonResponse, RecountExpected, @@ -3387,6 +3388,232 @@ VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS: Final = bridge_pairs( ) +VERTEX_AI_CLAUDE_DEPLOYMENT_RATES: Final = Deployment( + input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.01 +) + + +def _vertex_claude_web_search_message(web_search_requests: int) -> JsonResponse: + return JsonResponse( + content_type="application/json", + body={ + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "scripted answer for cost fixtures"}], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "server_tool_use": {"web_search_requests": web_search_requests}, + }, + }, + ) + + +def _vertex_claude_web_search_stream(web_search_requests: int) -> SseResponse: + return SseResponse( + content_type="text/event-stream", + frames=sse_frames( + { + "type": "message_start", + "message": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 1840}, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "scripted answer for cost fixtures"}, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 412, "server_tool_use": {"web_search_requests": web_search_requests}}, + }, + {"type": "message_stop"}, + ), + ) + + +def _vertex_claude_deployment_web_search_case( + name: str, + endpoint: Literal["/v1/chat/completions", "/v1/responses", "/v1/messages"], + request: dict[str, JsonValue], + response: JsonResponse | SseResponse, + tool_usage_cost: float, + deployment: Deployment = VERTEX_AI_CLAUDE_DEPLOYMENT_RATES, +) -> CostTrackingTestCase: + return CostTrackingTestCase( + name=name, + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="vertex_ai/claude-sonnet-5-5", + endpoint=endpoint, + deployment=deployment, + request={"model": "$MODEL", **request}, + response=response, + expected=ExactExpected( + spend=round(0.0039 + tool_usage_cost, 10), + input_cost=0.00184, + output_cost=0.00206, + prompt_tokens=1840, + completion_tokens=412, + tool_usage_cost=tool_usage_cost, + ), + ) + + +VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL: Final[dict[str, JsonValue]] = { + "type": "web_search_20250305", + "name": "web_search", + "max_uses": 3, +} + +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests", + "/v1/messages", + { + "max_tokens": 412, + "messages": [{"role": "user", "content": "vertex claude deployment rates messages web search"}], + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + }, + _vertex_claude_web_search_message(2), + 0.02, +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests_stream", + "/v1/messages", + { + "max_tokens": 412, + "messages": [{"role": "user", "content": "vertex claude deployment rates messages web search stream"}], + "stream": True, + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + }, + _vertex_claude_web_search_stream(2), + 0.02, +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE: Final = ( + _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_zero_search_rate_messages_two_requests", + "/v1/messages", + { + "max_tokens": 412, + "messages": [{"role": "user", "content": "vertex claude deployment zero search rate messages"}], + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + }, + _vertex_claude_web_search_message(2), + 0.0, + Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.0), + ) +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY: Final = ( + _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_token_rates_only_messages_web_search_two_requests", + "/v1/messages", + { + "max_tokens": 412, + "messages": [{"role": "user", "content": "vertex claude deployment token rates only messages web search"}], + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + }, + _vertex_claude_web_search_message(2), + 0.034, + Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06), + ) +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_three_requests", + "/v1/chat/completions", + { + "messages": [{"role": "user", "content": "vertex claude deployment rates chat web search"}], + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + "allowed_openai_params": ["web_search_options"], + }, + _vertex_claude_web_search_message(3), + 0.03, +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_two_requests_stream", + "/v1/chat/completions", + { + "messages": [{"role": "user", "content": "vertex claude deployment rates chat web search stream"}], + "stream": True, + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + "allowed_openai_params": ["web_search_options"], + }, + _vertex_claude_web_search_stream(2), + 0.02, +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_rates_responses_web_search_two_requests", + "/v1/responses", + {"input": "vertex claude deployment rates responses web search", "tools": [{"type": "web_search"}]}, + _vertex_claude_web_search_message(2), + 0.02, +) + +GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = CostTrackingTestCase( + name="vertex_ai/gemini-3.1-pro-preview-deployment_rates_messages_web_search_two_queries_stream", + covers="quota_management.spend_tracking.scripted_wire.logs_cost", + model="vertex_ai/gemini-3.1-pro-preview", + endpoint="/v1/messages", + deployment=Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.03), + request={ + "model": "$MODEL", + "max_tokens": 412, + "messages": [{"role": "user", "content": "vertex gemini deployment rates messages web search stream"}], + "stream": True, + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + }, + response=SseResponse( + content_type="text/event-stream", + frames=sse_frames( + { + "candidates": [ + { + "content": {"parts": [{"text": "scripted answer for cost fixtures"}], "role": "model"}, + "index": 0, + } + ], + "modelVersion": "gemini-3.1-pro-preview", + }, + { + "candidates": [ + { + "content": {"parts": [{"text": ""}], "role": "model"}, + "finishReason": "STOP", + "index": 0, + "groundingMetadata": {"webSearchQueries": ["query 0", "query 1"]}, + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}], + }, + "modelVersion": "gemini-3.1-pro-preview", + }, + ), + ), + expected=ExactExpected( + spend=0.0339, + input_cost=0.00184, + output_cost=0.00206, + prompt_tokens=1840, + completion_tokens=412, + tool_usage_cost=0.03, + ), +) + GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT: Final = _gemini_case( GEMINI_3_8_FLASH_INPUT_TEXT, "vertex_ai/gemini-3.1-pro-preview-input_text", "vertex_ai/gemini-3.1-pro-preview", ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412), @@ -3725,6 +3952,14 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT, VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_STREAM, *(pair.bridged for pair in VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS), + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH, + GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM, GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, GEMINI_3_1_PRO_PREVIEW_STREAM, GEMINI_3_1_PRO_PREVIEW_CACHE_READ, diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index e2452a928a4..9da2632943c 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -1,3 +1,4 @@ +from collections.abc import Callable from typing import Final import pytest @@ -7,7 +8,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) from litellm.types.llms.openai import FileSearchTool, ResponsesAPIResponse, WebSearchOptions -from litellm.types.utils import ModelResponse, StandardBuiltInToolsParams +from litellm.types.utils import ModelResponse, ServerToolUse, StandardBuiltInToolsParams, Usage def test_web_search_cost_low(): @@ -1052,6 +1053,175 @@ def test_azure_ai_non_claude_web_search_keeps_generic_fallback() -> None: assert get_cost_for_web_search_request(custom_llm_provider="azure_ai", usage=usage, model_info=model_info) is None +def _register_priced_deployment( + model: str, search_cost_per_query: float | None, deployment_id: str | None = None +) -> str: + from litellm import Router + + token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5} + search_rates: Final = ( + {} + if search_cost_per_query is None + else { + "search_context_cost_per_query": { + "search_context_size_low": search_cost_per_query, + "search_context_size_medium": search_cost_per_query, + "search_context_size_high": search_cost_per_query, + } + } + ) + router: Final = Router( + model_list=[ + { + "model_name": "priced-deployment", + "litellm_params": {"model": model, "api_key": "k", "vertex_project": "p", "vertex_location": "global"}, + "model_info": {**token_rates, **search_rates, **({} if deployment_id is None else {"id": deployment_id})}, + } + ] + ) + return router.model_list[0]["model_info"]["id"] + + +def _claude_chat_response( + web_search_requests: int, model: str = "claude-sonnet-5-5" +) -> tuple[object, Usage | None]: + usage: Final = Usage( + prompt_tokens=1840, + completion_tokens=412, + total_tokens=2252, + server_tool_use=ServerToolUse(web_search_requests=web_search_requests), + ) + return ModelResponse(model=model, usage=usage), usage + + +def _claude_messages_response( + web_search_requests: int, model: str = "claude-sonnet-5-5" +) -> tuple[object, Usage | None]: + return { + "id": "msg_vertex_claude_web_search", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "scripted answer"}], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "server_tool_use": {"web_search_requests": web_search_requests}, + }, + }, None + + +@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response]) +@pytest.mark.parametrize(("web_search_requests", "search_cost_per_query"), [(2, 0.01), (3, 0.01), (2, 0.0)]) +def test_vertex_claude_deployment_priced_web_search_billed_per_request( + local_model_cost_map: None, + build_response: Callable[[int], tuple[object, Usage | None]], + web_search_requests: int, + search_cost_per_query: float, +) -> None: + deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", search_cost_per_query) + response_object, usage = build_response(web_search_requests) + + cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=deployment_id, + response_object=response_object, + usage=usage, + custom_llm_provider="vertex_ai", + standard_built_in_tools_params=None, + ) + + assert cost == pytest.approx(web_search_requests * search_cost_per_query) + + +@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response]) +@pytest.mark.parametrize( + ("deployment_model", "served_model", "custom_llm_provider"), + # Google bills Claude web search on Sonnet 4.5 and Haiku 4.5 too: + # https://cloud.google.com/model-garden/partner-models/pricing (read 2026-10-30) + [ + ("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "vertex_ai"), + ("vertex_ai/claude-sonnet-4-5@20250929", "claude-sonnet-4-5-20250929", "vertex_ai"), + ("vertex_ai/claude-haiku-4-5@20251001", "claude-haiku-4-5-20251001", "vertex_ai"), + ("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5", "anthropic"), + ("azure_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "azure_ai"), + ], +) +@pytest.mark.parametrize("deployment_id", [None, "claude-token-priced"], ids=["hashed_id", "claude_in_id"]) +def test_claude_deployment_with_token_rates_only_bills_served_model_search_price( + local_model_cost_map: None, + build_response: Callable[[int, str], tuple[object, Usage | None]], + deployment_model: str, + served_model: str, + custom_llm_provider: str, + deployment_id: str | None, +) -> None: + served_search_pricing: Final = ( + litellm.get_model_info(model=deployment_model).get("search_context_cost_per_query") or {} + ) + served_search_price: Final = served_search_pricing.get("search_context_size_medium") or 0.0 + registered_id: Final = _register_priced_deployment(deployment_model, None, deployment_id) + response_object, usage = build_response(2, served_model) + + cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=registered_id, + response_object=response_object, + usage=usage, + custom_llm_provider=custom_llm_provider, + standard_built_in_tools_params=None, + ) + + assert served_search_price > 0 + assert cost == pytest.approx(2 * served_search_price) + + +def test_vertex_claude_deployment_with_token_rates_only_and_unmapped_served_model_bills_no_search( + local_model_cost_map: None, +) -> None: + deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", None) + usage: Final = Usage( + prompt_tokens=1840, + completion_tokens=412, + total_tokens=2252, + server_tool_use=ServerToolUse(web_search_requests=2), + ) + + cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=deployment_id, + response_object=ModelResponse(model="claude-unmapped-preview", usage=usage), + usage=usage, + custom_llm_provider="vertex_ai", + standard_built_in_tools_params=None, + ) + + assert cost == 0.0 + + +@pytest.mark.parametrize("deployment_id", [None, "claude-code-gemini"], ids=["hashed_id", "claude_in_id"]) +def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt( + local_model_cost_map: None, deployment_id: str | None +) -> None: + search_cost_per_prompt: Final = 0.03 + registered_id: Final = _register_priced_deployment( + "vertex_ai/gemini-2.5-flash", search_cost_per_prompt, deployment_id + ) + usage: Final = Usage( + prompt_tokens=1840, + completion_tokens=412, + total_tokens=2252, + server_tool_use=ServerToolUse(web_search_requests=2), + ) + + cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=registered_id, + response_object=ModelResponse(model="gemini-2.5-flash", usage=usage), + usage=usage, + custom_llm_provider="vertex_ai", + standard_built_in_tools_params=None, + ) + + assert cost == pytest.approx(search_cost_per_prompt) + + @pytest.mark.parametrize( "kwargs", [{"tools": 5}, {"web_search_options": None}, {"web_search_options": 5}], diff --git a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py index ae95c27950c..8127ffe558b 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py +++ b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py @@ -41,12 +41,13 @@ def _make_streaming_response(chunks): return mock -def _unarmed_logging_obj(): +def _unarmed_logging_obj(model: str = "unknown"): """Real Logging objects only carry _on_deferred_stream_complete when the proxy arms deferred dispatch; a bare MagicMock's auto-attribute is truthy and would spuriously trigger the deferral branch.""" obj = MagicMock() obj._on_deferred_stream_complete = None + obj.model = model return obj @@ -409,7 +410,7 @@ async def _collect_openai_passthrough_chunks(chunks, endpoint_type): async for chunk in PassThroughStreamingHandler.chunk_processor( response=response, request_body={"model": "gpt-4o-mini", "stream": True}, - litellm_logging_obj=_unarmed_logging_obj(), + litellm_logging_obj=_unarmed_logging_obj(model="gpt-4o-mini"), endpoint_type=endpoint_type, start_time=datetime(2026, 1, 1), passthrough_success_handler_obj=MagicMock(), diff --git a/tests/unit/proxy/pass_through_endpoints/test_success_handler.py b/tests/unit/proxy/pass_through_endpoints/test_success_handler.py index c721e808ed5..cf00545f331 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_success_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/test_success_handler.py @@ -346,6 +346,7 @@ async def test_vertex_ai_anthropic_streaming_cost_injection_enabled(): # Setup logging object with model info litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj) + litellm_logging_obj.model = "claude-sonnet-4@20250514" litellm_logging_obj.litellm_params = {} litellm_logging_obj.model_call_details = {"model": "claude-sonnet-4@20250514"} litellm_logging_obj.completion_start_time = None @@ -544,6 +545,7 @@ async def test_vertex_ai_anthropic_streaming_model_extraction(): response.aiter_bytes = mock_aiter_bytes litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj) + litellm_logging_obj.model = "claude-sonnet-4@20250514" litellm_logging_obj.litellm_params = {} litellm_logging_obj.model_call_details = {} litellm_logging_obj.completion_start_time = None diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index fe6b894b368..4d1b67f9212 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -74,6 +74,7 @@ from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitErro from litellm.proxy.common_utils.sse_keepalive import ANTHROPIC_PING_SSE_CHUNK from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.proxy._types import Litellm_EntityType, ProxyErrorTypes, ProxyException from litellm.proxy._types import UserAPIKeyAuth as ProxyUserAPIKeyAuth from litellm.proxy.auth.fallback_budget import RouterFallbackBudgetCheck @@ -10221,6 +10222,8 @@ class TestInjectCostIntoUsageDict: discount streamed a ``usage.cost`` that disagreed with the callback's ``response_cost``.""" class _StubLoggingObj: + model = "claude-haiku-4-5" + def __init__(self, cost): self._cost = cost self.captured_result = None @@ -10257,6 +10260,8 @@ class TestInjectCostIntoUsageDict: def test_message_delta_falls_back_to_model_pricing_when_the_logging_obj_returns_no_cost(self): class _StubLoggingObj: + model = "claude-haiku-4-5" + def response_cost_calculator(self, result): return None @@ -10282,6 +10287,8 @@ class TestInjectCostIntoUsageDict: model-name pricing rather than propagating into the response body.""" class _StubLoggingObj: + model = "claude-haiku-4-5" + def response_cost_calculator(self, result): raise ValueError("no pricing for this deployment") @@ -10363,6 +10370,8 @@ class TestInjectCostIntoUsageDict: streaming /v1/chat/completions gets its negotiated price instead of sticker.""" class _StubLoggingObj: + model = "gpt-4o-mini" + def __init__(self, cost): self._cost = cost self.captured_result = None @@ -10391,6 +10400,8 @@ class TestInjectCostIntoUsageDict: def test_openai_chunk_falls_back_to_model_pricing_when_the_logging_obj_returns_no_cost(self): class _StubLoggingObj: + model = "gpt-4o-mini" + def response_cost_calculator(self, result): return None @@ -10406,6 +10417,60 @@ class TestInjectCostIntoUsageDict: assert result is not None assert result["usage"]["cost"] == pytest.approx(self._expected_cost("gpt-4o-mini", 11, 4)) + @pytest.mark.parametrize( + ("deployment_model", "served_model", "custom_llm_provider"), + [ + ("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "vertex_ai"), + ("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5", "anthropic"), + ("azure_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "azure_ai"), + ], + ) + def test_message_delta_bills_search_like_the_logged_response_on_a_token_priced_claude_deployment( + self, local_model_cost_map: None, deployment_model: str, served_model: str, custom_llm_provider: str + ) -> None: + token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5} + router: Final = Router( + model_list=[ + { + "model_name": "priced-claude", + "litellm_params": { + "model": deployment_model, + "api_key": "k", + "vertex_project": "p", + "vertex_location": "global", + }, + "model_info": token_rates, + } + ] + ) + deployment_id: Final = router.model_list[0]["model_info"]["id"] + logging_obj: Final = LiteLLMLoggingObj( + model=deployment_model, + messages=[{"role": "user", "content": "test"}], + stream=True, + call_type="anthropic_messages", + start_time=None, + litellm_call_id="token-priced-claude-search", + function_id="token-priced-claude-search", + ) + logging_obj.update_environment_variables( + litellm_params={"metadata": {"model_info": {"id": deployment_id, **token_rates}}}, optional_params={} + ) + logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider + wire_usage: Final = {"input_tokens": 1840, "output_tokens": 412, "server_tool_use": {"web_search_requests": 2}} + event: Final = {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": wire_usage} + + result: Final = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict(event, "priced-claude", logging_obj) + + stream_usage: Final = AnthropicConfig().calculate_usage(usage_object=wire_usage, reasoning_content=None) + logged_cost: Final = logging_obj.response_cost_calculator( + result=litellm.ModelResponse(model=served_model, usage=stream_usage) + ) + token_cost: Final = 1840 * token_rates["input_cost_per_token"] + 412 * token_rates["output_cost_per_token"] + assert logged_cost is not None and logged_cost > token_cost + assert result is not None + assert result["usage"]["cost"] == pytest.approx(logged_cost) + class TestProcessChunkWithCostInjection: @pytest.mark.parametrize("model", ["claude-opus-5-5", "vertex_ai/claude-opus-5-5"]) @@ -10472,6 +10537,8 @@ class TestProcessChunkWithCostInjection: monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True) class _StubLoggingObj: + model = "claude-haiku-4-5" + def response_cost_calculator(self, result): return 0.00042 From af070e7cd0152acb704c1965ffb5f67ae80eee57 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:12:10 -0700 Subject: [PATCH 09/15] build(deps): bump pyasn1, pypdf and minimatch to clear osv-scan (#49362) * build(deps): bump pyasn1 from 0.6.4 to 0.6.5 Bumps [pyasn1](https://github.com/pyasn1/pyasn1) from 0.6.4 to 0.6.5. - [Release notes](https://github.com/pyasn1/pyasn1/releases) - [Changelog](https://github.com/pyasn1/pyasn1/blob/main/CHANGES.rst) - [Commits](https://github.com/pyasn1/pyasn1/compare/v0.6.4...v0.6.5) --- updated-dependencies: - dependency-name: pyasn1 dependency-version: 0.6.5 dependency-type: indirect ... Signed-off-by: dependabot[bot] * build(deps): bump pyasn1, pypdf and minimatch to clear osv-scan pyasn1 0.6.4 -> 0.6.5 and pypdf 6.21.2 -> 6.21.4 in uv.lock, with a dated exclude-newer-package cutoff that admits pypdf 6.21.4 before it clears the 3-day window. minimatch has no fixed 10.x release, so the dashboard's override and a new vscode-extension override move it to 11.0.2. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- pyproject.toml | 2 ++ ui/litellm-dashboard/package-lock.json | 16 ++++++++++++---- ui/litellm-dashboard/package.json | 2 +- uv.lock | 13 +++++++------ vscode-extension/package-lock.json | 16 ++++++++++++---- vscode-extension/package.json | 3 +++ 6 files changed, 37 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6e9e75a5b7f..5c23a64ebb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -383,6 +383,8 @@ exclude-newer = "3 days" [tool.uv.exclude-newer-package] # TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window brotli = "2026-10-22T14:01:00Z" +# TODO: remove after 2026-10-31, once pypdf 6.21.4 clears the 3-day window +pypdf = "2026-10-28T17:01:00Z" [tool.uv.sources] litellm-proxy-extras = { workspace = true } diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 48def46e3fe..54cd9087c4e 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -2037,6 +2037,13 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@isaacs/nonrecursive-glob-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.2.tgz", + "integrity": "sha512-GJPHIgPPtFl+nLhFHyF0zZoFcOFLsZWbHFFzo+Zr+PWDXJbl9hHxb2w+9SiF2fHmLgKaXIpUzQo3SbBBZ9WObw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -9657,13 +9664,14 @@ } }, "node_modules/minimatch": { - "version": "10.2.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", - "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.2.tgz", + "integrity": "sha512-pWTgz5CN3tjqyZNSK2tePlSmu9IwM41q8yn8Usdqqwe9TIHh+9dEpLLjF4uApsfPEAZY71qtSiA2B2APEr+1Yw==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.2" + "@isaacs/nonrecursive-glob-match": "^1.0.2", + "brace-expansion": "^5.0.12" }, "engines": { "node": "18 || 20 || >=22" diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index f65316439bf..0707496a6bc 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -117,7 +117,7 @@ "js-yaml": "4.3.2", "brace-expansion": "5.0.12", "glob": "13.0.0", - "minimatch": "10.2.4", + "minimatch": "11.0.2", "ws": "8.21.0", "braces": "3.0.3", "axios": "1.13.6", diff --git a/uv.lock b/uv.lock index 8cd31eb0828..19d3a1ffd4f 100644 --- a/uv.lock +++ b/uv.lock @@ -16,6 +16,7 @@ exclude-newer-span = "P3D" [options.exclude-newer-package] brotli = "2026-10-22T14:01:00Z" +pypdf = "2026-10-28T17:01:00Z" [manifest] members = [ @@ -8372,11 +8373,11 @@ wheels = [ [[package]] name = "pyasn1" -version = "0.6.4" +version = "0.6.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/88/7e/3c0da41061a2021cac7cd524fd57092adbe5b515e8bf9005702e366802e2/pyasn1-0.6.5.tar.gz", hash = "sha256:6b11616a6bcf3a40e635f082bb232fa57b5e0dd7f517bd7ddd69ec3794508a47", size = 155152, upload-time = "2026-10-27T06:32:49.125Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, + { url = "https://files.pythonhosted.org/packages/19/1f/3eefd5c84c35cae44549ce1c7f7e58f7945b34f5a6bce07a135438522073/pyasn1-0.6.5-py3-none-any.whl", hash = "sha256:4017b0fce125096b8573b39c989fdb1a4b19cf761e683398184550fea0e66587", size = 84807, upload-time = "2026-10-27T06:32:48.045Z" }, ] [[package]] @@ -8869,14 +8870,14 @@ wheels = [ [[package]] name = "pypdf" -version = "6.21.2" +version = "6.21.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/28/081f1cbcca7042441dafc8de00f48ba60853f89e5bff92c5f26477346d81/pypdf-6.21.2.tar.gz", hash = "sha256:0a3e1c5c886a1be03646c16a04827cc166c2a0575c971b26a4bf89686551c2d4", size = 7144571, upload-time = "2026-10-21T16:44:55.272Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/e1/ec8f4d155c997c45917153a5521e60d6ee6389d68abdcbc8e44e564caffc/pypdf-6.21.4.tar.gz", hash = "sha256:c8817e3f888d47a087872ded69812c1ef30cacb871d6e053e0c00bbaf0bc2163", size = 7109255, upload-time = "2026-10-28T17:00:54.81Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/4d/c9d2d67f204eff463237a1a8f5e25a6923b7b1043982b91fcaf104c8ef60/pypdf-6.21.2-py3-none-any.whl", hash = "sha256:1d44d1a3a200f25062adff0085b02040e4646431ac2ccac22e42679e91466084", size = 407766, upload-time = "2026-10-21T16:44:53.626Z" }, + { url = "https://files.pythonhosted.org/packages/3f/4b/77e8ad4ac42c9d93600f40541a50e9feb3a61d2bbee2d76f659346072051/pypdf-6.21.4-py3-none-any.whl", hash = "sha256:33e4fb41ca352810eb4e9838aa0d82d6888a5f8bf682f43e99cce5faa666255c", size = 410430, upload-time = "2026-10-28T17:00:53.146Z" }, ] [[package]] diff --git a/vscode-extension/package-lock.json b/vscode-extension/package-lock.json index 5f6cf7edf4e..80b868e528b 100644 --- a/vscode-extension/package-lock.json +++ b/vscode-extension/package-lock.json @@ -644,6 +644,13 @@ "node": ">=18" } }, + "node_modules/@isaacs/nonrecursive-glob-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.2.tgz", + "integrity": "sha512-GJPHIgPPtFl+nLhFHyF0zZoFcOFLsZWbHFFzo+Zr+PWDXJbl9hHxb2w+9SiF2fHmLgKaXIpUzQo3SbBBZ9WObw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", @@ -2700,13 +2707,14 @@ } }, "node_modules/minimatch": { - "version": "10.2.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", - "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.2.tgz", + "integrity": "sha512-pWTgz5CN3tjqyZNSK2tePlSmu9IwM41q8yn8Usdqqwe9TIHh+9dEpLLjF4uApsfPEAZY71qtSiA2B2APEr+1Yw==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.8" + "@isaacs/nonrecursive-glob-match": "^1.0.2", + "brace-expansion": "^5.0.12" }, "engines": { "node": "18 || 20 || >=22" diff --git a/vscode-extension/package.json b/vscode-extension/package.json index a3cf2dc7a5f..42626439aca 100644 --- a/vscode-extension/package.json +++ b/vscode-extension/package.json @@ -83,5 +83,8 @@ "esbuild": "^0.28.2", "typescript": "^5.9.3", "vitest": "^4.1.12" + }, + "overrides": { + "minimatch": "11.0.2" } } From e5aa3d32cde109d236881ff5473d3d43ff0a384e Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:16:42 -0700 Subject: [PATCH 10/15] fix(proxy): forward a valid metadata.user_id to Anthropic Messages on pass-through routes (#49333) * fix(proxy): forward the client's own metadata on Anthropic and custom pass-through routes * fix(proxy): forward only a valid metadata.user_id to Anthropic Messages on pass-through routes The whole-object forward sent keys Anthropic refuses and reached other providers, so a request that used to pass could start failing with a 400. Read the client's user_id after hooks, drop it unless it is a string with no email or phone number, and send only that key when the upstream URL path ends in /v1/messages. Share the validator with the translated route, whose email check now matches any embedded address as Anthropic's does. * fix(proxy): forward metadata.user_id only to Anthropic upstreams no guardrail scans Limit the pass-through user_id to Messages create on api.anthropic.com or the configured Anthropic base, mirror the checks Anthropic answers 400 for, and keep the field off any request a pre-call guardrail inspects. The translated route goes back to its own user_id filter. * fix(proxy): judge user_id forwarding on the whole request and cap it for configured Anthropic bases * refactor(proxy): pass the user_id check a plain mapping instead of an optional body * fix(proxy): drop an unprintable metadata.user_id and warn when a refused id is dropped Anthropic's whitespace is not Python's, so a control character before the at sign hid an email from the filter and turned a 200 into a 400. Fireworks and Moonshot also fail on newline ids. Requiring a printable id covers both, and a refused id now logs a warning that never carries the value. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/constants.py | 2 + litellm/llms/anthropic/common_utils.py | 37 +++ .../llm_passthrough_endpoints.py | 8 +- .../pass_through_endpoints.py | 54 +++- litellm/proxy/utils.py | 13 +- .../test_anthropic_passthrough_wire.py | 1 - .../anthropic/test_anthropic_common_utils.py | 92 ++++++ .../test_pass_through_endpoints.py | 268 +++++++++++++++++- .../utils/proxy_logging/test_pre_call_hook.py | 18 ++ 9 files changed, 479 insertions(+), 14 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index c17e776f54b..28e7b4a7fd7 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -634,6 +634,8 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60 BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024) # Anthropic's Messages API rejects thinking.budget_tokens < 1024. ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024 +ANTHROPIC_MAX_USER_ID_LENGTH: Final = 512 +ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH: Final = 255 ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:" REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5) DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index e1f9a7d540f..e0da8943a4a 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -4,6 +4,7 @@ This file contains common utils for anthropic calls. import copy import json +import os import re from collections.abc import Container, Mapping, MutableMapping, Sequence from dataclasses import dataclass @@ -19,6 +20,8 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import ( + ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, + ANTHROPIC_MAX_USER_ID_LENGTH, DEFAULT_MODEL_CREATED_AT_TIME, DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, @@ -178,6 +181,40 @@ def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: boo return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools] +def anthropic_pass_through_api_base() -> str: + return os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com" + + +def _same_host_and_port(url: httpx.URL, other: str) -> bool: + try: + other_url: Final = httpx.URL(other) + except httpx.InvalidURL: + return False + return (url.host, url.port) == (other_url.host, other_url.port) + + +def anthropic_upstream_user_id_limit(url: str) -> int | None: + upstream: Final = httpx.URL(url) + if not upstream.path.removesuffix("/").endswith("/v1/messages"): + return None + if upstream.host == "api.anthropic.com": + return ANTHROPIC_MAX_USER_ID_LENGTH + return ( + ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH + if _same_host_and_port(upstream, anthropic_pass_through_api_base()) + else None + ) + + +def _has_email_shape(token: str) -> bool: + at_sign: Final = token.find("@", 1) + return at_sign != -1 and token.find(".", at_sign + 2, -1) != -1 + + +def anthropic_accepts_user_id(user_id: str, max_length: int) -> bool: + return len(user_id) <= max_length and user_id.isprintable() and not any(map(_has_email_shape, user_id.split())) + + def is_anthropic_messages_url(url: str) -> bool: """Check whether a URL addresses Anthropic's Messages API.""" parsed_url: Final = urlparse(url) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 33b18c3d939..151938d73d7 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -52,7 +52,11 @@ from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization -from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers +from litellm.llms.anthropic.common_utils import ( + AnthropicModelInfo, + anthropic_pass_through_api_base, + merge_anthropic_beta_headers, +) from litellm.llms.azure.passthrough.transformation import ( foreign_azure_deployment, is_azure_body_model_inference_endpoint, @@ -1047,7 +1051,7 @@ async def anthropic_proxy_route( """ [Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion) """ - base_target_url = os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com" + base_target_url = anthropic_pass_through_api_base() encoded_endpoint = httpx.URL(endpoint).path # Ensure endpoint starts with '/' for proper URL construction diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 647e145aac1..bda4f4c2ad2 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -71,7 +71,11 @@ from litellm.litellm_core_utils.litellm_logging import get_masked_values from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.redact_messages import should_redact_message_logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps -from litellm.llms.anthropic.common_utils import is_anthropic_messages_url +from litellm.llms.anthropic.common_utils import ( + anthropic_accepts_user_id, + anthropic_upstream_user_id_limit, + is_anthropic_messages_url, +) from litellm.llms.base_llm.managed_resources.utils import ( resolve_passthrough_managed_id_provider, ) @@ -503,7 +507,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): url: httpx.URL, headers: dict, requested_query_params: dict | None = None, - _parsed_body: dict | None = None, + _parsed_body: Mapping[str, object] | None = None, forward_multipart: bool = False, ) -> httpx.Response: """ @@ -717,8 +721,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): # unenforced. Set after the client merge so a request body cannot supply its own budget. # # Only for the built-in provider routes. `get_model_from_request` returns - # None for a user-defined pass-through, deliberately: its body is forwarded - # verbatim, so `model` there names an UPSTREAM model rather than a + # None for a user-defined pass-through, deliberately: the client's `model` + # goes upstream unchanged, so it names an UPSTREAM model rather than a # LiteLLM-managed one. Enforcement is therefore skipped on those routes, and # charging a counter anyway would track spend that nothing can refuse, and # would attribute it to a budget the operator scoped to a LiteLLM model that @@ -1187,6 +1191,36 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]: return headers +def _json_object(value: object) -> Mapping[str, object]: + return cast("Mapping[str, object]", value) if isinstance(value, Mapping) else MappingProxyType({}) # cast-ok: json + + +def _anthropic_user_id_to_forward( + body: Mapping[str, object], + url: str, + has_pre_call_guardrails: Callable[[Mapping[str, object]], bool], +) -> str | None: + user_id: Final = _json_object(body.get("metadata")).get("user_id") + if not isinstance(user_id, str): + return None + max_length: Final = anthropic_upstream_user_id_limit(url) + if max_length is None: + return None + if not anthropic_accepts_user_id(user_id, max_length): + verbose_proxy_logger.warning("Not forwarding metadata.user_id because the upstream would refuse it") + return None + if has_pre_call_guardrails(body): + verbose_proxy_logger.debug("Not forwarding metadata.user_id because a pre-call guardrail cannot scan it") + return None + return user_id + + +def _with_anthropic_user_id(body: Mapping[str, object] | None, user_id: str | None) -> Mapping[str, object] | None: + if body is None or user_id is None: + return body + return {**body, "metadata": {"user_id": user_id}} + + def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None: return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None @@ -1423,6 +1457,10 @@ async def pass_through_request( request_method=getattr(request, "method", None), cost_per_request=cost_per_request, ) + hooked_body: Final = _json_object(_parsed_body) + client_user_id: Final = _anthropic_user_id_to_forward( + hooked_body, str(url), proxy_logging_obj.request_has_pre_call_guardrails + ) kwargs = HttpPassThroughEndpointHelpers.init_kwargs_for_pass_through_endpoint( # rebind-ok: pre-existing rebinding on a rename-only line user_api_key_dict=user_api_key_dict, _parsed_body=_parsed_body, @@ -1590,11 +1628,13 @@ async def pass_through_request( else: logging_url = str(url) + "?" + requested_query_params_str + outgoing_body: Final = cast("Mapping[str, object] | None", _parsed_body) # cast-ok: json + upstream_body: Final = _with_anthropic_user_id(outgoing_body, client_user_id) logging_obj.pre_call( input=[{"role": "user", "content": safe_dumps(_parsed_body)}], api_key="", additional_args={ - "complete_input_dict": _parsed_body, + "complete_input_dict": upstream_body, "api_base": str(logging_url), "headers": redact_credential_headers(upstream_headers), }, @@ -1634,7 +1674,7 @@ async def pass_through_request( url, params=requested_query_params, headers=upstream_headers, - json=_parsed_body, + json=upstream_body, ) ) @@ -1733,7 +1773,7 @@ async def pass_through_request( url=url, headers=upstream_headers, requested_query_params=requested_query_params, - _parsed_body=_parsed_body, + _parsed_body=upstream_body, forward_multipart=is_multipart, ) verbose_proxy_logger.debug("response.headers= %s", response.headers) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 9e237ffcb57..c7efbf9dacf 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3031,14 +3031,21 @@ class ProxyLogging: hook to consult, but it judges the payload the same way, so a proxy configured only with one of those still has something to say about every record. """ - if request_metadata.get("_guardrail_pipelines"): + return self.request_has_pre_call_guardrails({"metadata": dict(request_metadata)}, event_type) + + def request_has_pre_call_guardrails( + self, + request_data: Mapping[str, object], + event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call, + ) -> bool: + if _policy_state_metadata(request_data).get("_guardrail_pipelines"): return True caps: Final = ProxyLogging.callback_capabilities() if caps.has_content_enforcer: return True - probe: Final = {"metadata": dict(request_metadata)} return any( - isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=event_type) + isinstance(callback, CustomGuardrail) + and callback.should_run_guardrail(data=request_data, event_type=event_type) for callback in caps.resolved_callbacks ) diff --git a/tests/integration/providers/test_anthropic_passthrough_wire.py b/tests/integration/providers/test_anthropic_passthrough_wire.py index a791eb9c2a2..0a16520f626 100644 --- a/tests/integration/providers/test_anthropic_passthrough_wire.py +++ b/tests/integration/providers/test_anthropic_passthrough_wire.py @@ -317,7 +317,6 @@ def test_credentialless_proxy_forwards_the_caller_anthropic_credential_and_never def test_anthropic_sdk_metadata_user_id_reaches_upstream(api_key_proxy: Gateway, wire: Wire) -> None: - pytest.skip("BUG: /anthropic/v1/messages drops the native metadata.user_id field before forwarding upstream") with api_key_proxy.scenario() as scenario: key: Final = scenario.key() sent: Final[list[httpx.Request]] = [] diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index d18ef28c68e..cf0b8a39865 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -56,6 +56,98 @@ def test_is_anthropic_messages_url(url: str, expected: bool) -> None: assert is_anthropic_messages_url(url) is expected +@pytest.mark.parametrize( + ("configured_base", "url", "expected"), + [ + pytest.param(None, "https://api.anthropic.com/v1/messages", 512, id="default-upstream"), + pytest.param(None, "https://api.anthropic.com:443/v1/messages/", 512, id="default-upstream-explicit-port"), + pytest.param("http://relay.test:8080", "https://api.anthropic.com/v1/messages", 512, id="direct-target"), + pytest.param("https://api.anthropic.com", "https://api.anthropic.com/v1/messages", 512, id="configured-default"), + pytest.param("http://relay.test:8080", "http://relay.test:8080/v1/messages", 255, id="configured-upstream"), + pytest.param("http://relay.test:8080/a", "http://relay.test:8080/a/v1/messages", 255, id="configured-prefix"), + pytest.param("https://Relay.Test", "https://relay.test/v1/messages", 255, id="configured-upper-case-host"), + pytest.param("https://relay.test:443", "https://relay.test/v1/messages", 255, id="configured-https-port"), + pytest.param("http://relay.test:80/a", "http://relay.test/a/v1/messages", 255, id="configured-http-port"), + pytest.param("http://relay.test:8080", "http://relay.test:8081/v1/messages", None, id="other-port"), + pytest.param("http://relay.test:notaport", "http://relay.test/v1/messages", None, id="invalid-configured-base"), + pytest.param(None, "https://other-vendor.test/v1/messages", None, id="other-vendor"), + pytest.param(None, "https://api.anthropic.com/v1/messages/count_tokens", None, id="count-tokens"), + ], +) +def test_anthropic_upstream_user_id_limit( + monkeypatch: pytest.MonkeyPatch, configured_base: str | None, url: str, expected: int | None +) -> None: + from litellm.llms.anthropic.common_utils import anthropic_upstream_user_id_limit + + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) + if configured_base is not None: + monkeypatch.setenv("ANTHROPIC_API_BASE", configured_base) + + assert anthropic_upstream_user_id_limit(url) == expected + + +def test_anthropic_pass_through_api_base_prefers_the_api_base_variable(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.anthropic.common_utils import anthropic_pass_through_api_base + + monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://fallback.test") + monkeypatch.setenv("ANTHROPIC_API_BASE", "http://preferred.test") + assert anthropic_pass_through_api_base() == "http://preferred.test" + + monkeypatch.delenv("ANTHROPIC_API_BASE") + assert anthropic_pass_through_api_base() == "http://fallback.test" + + +# Each row was sent to https://api.anthropic.com/v1/messages on 2026-10-30, False rows answered 400. +@pytest.mark.parametrize( + ("user_id", "expected"), + [ + pytest.param("user-1", True, id="opaque"), + pytest.param("", True, id="empty"), + pytest.param('{"device_id":"abc","session_id":"def"}', True, id="claude-code-shape"), + pytest.param("1234567890", True, id="digits"), + pytest.param("+14155550100", True, id="phone-shape"), + pytest.param("a@b", True, id="no-dot"), + pytest.param("@b.co", True, id="nothing-before-at-sign"), + pytest.param("a@.co", True, id="nothing-between-at-sign-and-dot"), + pytest.param("a@b.", True, id="nothing-after-dot"), + pytest.param("a @ b.co", True, id="spaces-around-at-sign"), + pytest.param("a\u00a0@b.co", False, id="no-break-space-before-at-sign"), + pytest.param("a@b\n.co", False, id="newline-before-dot"), + pytest.param("line one\nline two", False, id="newline"), + pytest.param("a\tb", False, id="tab"), + pytest.param("a\x1f@b.co", False, id="unit-separator-before-at-sign"), + pytest.param("a@\x1cb.co", False, id="file-separator-after-at-sign"), + pytest.param("a\u200bb", False, id="zero-width-space"), + pytest.param("a@b.co", False, id="email"), + pytest.param("user a@b.co here", False, id="email-inside-text"), + pytest.param('{"device_id":"a@b.co"}', False, id="email-inside-json"), + pytest.param("a@@example.com", False, id="doubled-at-sign"), + pytest.param("@@example.com", False, id="leading-doubled-at-sign"), + pytest.param("a@b@c.co", False, id="two-at-signs"), + pytest.param("a@b..co", False, id="doubled-dot"), + pytest.param("a@.b.co", False, id="dot-after-at-sign"), + pytest.param("a@b.c d", False, id="email-before-space"), + ], +) +def test_anthropic_accepts_user_id(user_id: str, expected: bool) -> None: + from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH + from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id + + assert anthropic_accepts_user_id(user_id, ANTHROPIC_MAX_USER_ID_LENGTH) is expected + + +@pytest.mark.parametrize("character", ["a", "\u00e9", "\U0001f600"], ids=["ascii", "two-bytes", "four-bytes"]) +def test_anthropic_accepts_user_id_counts_characters_up_to_the_limit(character: str) -> None: + from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH + from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id + + assert anthropic_accepts_user_id(character * ANTHROPIC_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH) is True + assert ( + anthropic_accepts_user_id(character * (ANTHROPIC_MAX_USER_ID_LENGTH + 1), ANTHROPIC_MAX_USER_ID_LENGTH) is False + ) + + def test_anthropic_model_not_found_maps_to_not_found_but_authentication_error_does_not() -> None: with respx.mock(assert_all_called=True) as router: router.post("https://api.anthropic.com/v1/messages").mock( diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index b7e76ce0068..4fe358c38a3 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -31,7 +31,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile import litellm from litellm._logging import verbose_proxy_logger from litellm.caching.dual_cache import DualCache -from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS +from litellm.constants import ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH, DEFAULT_REQUEST_TIMEOUT_SECONDS from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.prometheus import PrometheusLogger @@ -6621,6 +6621,272 @@ async def test_pass_through_body_stays_byte_identical_outside_the_opt_in( assert "x-litellm-call-id" in response.headers +def _echo_upstream_body(upstream_request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, content=upstream_request.content, headers={"content-type": "application/json"}, request=upstream_request + ) + + +async def _relayed_body(response: Response) -> bytes: + if isinstance(response, StreamingResponse): + return b"".join([chunk async for chunk in response.body_iterator]) + return bytes(response.body) + + +async def _relay_through_pass_through( + client_body: Mapping[str, object], + target: str, + guardrails_config: dict[str, object] | None, + custom_body: dict[str, object] | None = None, +) -> Mapping[str, object]: + async def receive() -> Mapping[str, object]: + return {"type": "http.request", "body": json.dumps(dict(client_body)).encode()} + + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": "/my-claude-route", + "query_string": b"", + "headers": [(b"content-type", b"application/json")], + }, + receive, + ) + fake_client, cleanup = _inject_fake_passthrough_client(httpx.MockTransport(_echo_upstream_body), timeout=None) + try: + with ExitStack() as stack: + _enter_relay_logging_mocks(stack, {}) + stack.enter_context( + patch("litellm.proxy.proxy_server.proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache())) + ) + response: Final = await pass_through_request( + request=request, + target=target, + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"), + custom_body=custom_body, + guardrails_config=guardrails_config, + ) + relayed: Final = await _relayed_body(response) + finally: + cleanup() + await fake_client.aclose() + + assert response.status_code == 200 + return MappingProxyType(json.loads(relayed)) + + +_MESSAGES_BODY: Final = MappingProxyType( + {"model": "claude-sonnet-4-5", "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]} +) +_ANTHROPIC_MESSAGES: Final = "https://api.anthropic.com/v1/messages" + + +@pytest.fixture +def no_configured_anthropic_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +@pytest.mark.parametrize( + "client_metadata, target, forwarded_metadata", + [ + ({"user_id": "user-1"}, _ANTHROPIC_MESSAGES, {"user_id": "user-1"}), + ({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/", {"user_id": "user-1"}), + ({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}?beta=true", {"user_id": "user-1"}), + ( + {"user_id": "user-1", "tags": ["a"], "trace_id": "t", "generation_name": "g", "foo": "bar"}, + _ANTHROPIC_MESSAGES, + {"user_id": "user-1"}, + ), + ({"user_id": "1234567890"}, _ANTHROPIC_MESSAGES, {"user_id": "1234567890"}), + ({"tags": ["a"], "foo": "bar"}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "a@example.com"}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "x" * (ANTHROPIC_MAX_USER_ID_LENGTH + 1)}, _ANTHROPIC_MESSAGES, None), + ({"user_id": 42}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "line one\nline two"}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "a\x1f@example.com"}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/count_tokens", None), + ({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/batches", None), + ({"user_id": "user-1"}, "https://api.anthropic.com/v1/models", None), + ({"user_id": "user-1"}, "https://other-vendor.test/anthropic/v1/messages", None), + ({"user_id": "user-1"}, "https://other-vendor.test/v1/responses", None), + ({"user_id": "user-1"}, "https://other-vendor.test/v1/chat/completions", None), + ], + ids=[ + "messages", + "messages-trailing-slash", + "messages-with-query", + "user-id-beside-litellm-and-unknown-keys", + "digits-user-id", + "no-user-id", + "email-user-id", + "over-long-user-id", + "integer-user-id", + "newline-user-id", + "control-character-before-at-sign", + "count-tokens", + "batches", + "other-anthropic-path", + "other-vendor-messages", + "other-vendor-responses", + "other-vendor-chat-completions", + ], +) +async def test_pass_through_forwards_only_an_acceptable_user_id_to_anthropic_messages( + stream: bool, client_metadata: object, target: str, forwarded_metadata: Mapping[str, object] | None +) -> None: + client_body: Final = MappingProxyType( + {**_MESSAGES_BODY, "metadata": client_metadata, **({"stream": True} if stream else {})} + ) + body_without_metadata: Final = {k: v for k, v in client_body.items() if k != "metadata"} + + relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None) + + assert relayed == ( + body_without_metadata if forwarded_metadata is None else {**body_without_metadata, "metadata": forwarded_metadata} + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +@pytest.mark.parametrize("base_env", ["ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"]) +@pytest.mark.parametrize( + "target, forwarded", + [ + ("http://gateway.test:8080/anthropic/v1/messages", True), + ("http://gateway.test:9090/v1/messages", False), + ("http://other-gateway.test:8080/v1/messages", False), + ], + ids=["configured-upstream", "other-port", "other-host"], +) +async def test_pass_through_treats_the_configured_anthropic_base_as_anthropic( + monkeypatch: pytest.MonkeyPatch, base_env: str, target: str, forwarded: bool +) -> None: + monkeypatch.setenv(base_env, "http://gateway.test:8080/anthropic") + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}}) + + relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None) + + assert relayed == (client_body if forwarded else _MESSAGES_BODY) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +@pytest.mark.parametrize( + "user_id_length, target, forwarded", + [ + (ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, "http://gateway.test/v1/messages", True), + (ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH + 1, "http://gateway.test/v1/messages", False), + (ANTHROPIC_MAX_USER_ID_LENGTH, _ANTHROPIC_MESSAGES, True), + ], + ids=["at-the-configured-base-limit", "over-the-configured-base-limit", "direct-anthropic-keeps-its-limit"], +) +async def test_pass_through_caps_the_user_id_for_a_configured_anthropic_compatible_base( + monkeypatch: pytest.MonkeyPatch, user_id_length: int, target: str, forwarded: bool +) -> None: + monkeypatch.setenv("ANTHROPIC_API_BASE", "http://gateway.test") + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "u" * user_id_length}}) + + relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None) + + assert relayed == (client_body if forwarded else _MESSAGES_BODY) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +@pytest.mark.parametrize( + "default_on, event_hook, guardrails_config, client_fields, stream_scope, forwarded", + [ + (False, GuardrailEventHooks.pre_call, None, {}, None, True), + (False, GuardrailEventHooks.pre_call, {"unregistered-guard": {}}, {}, None, True), + (True, GuardrailEventHooks.post_call, None, {}, None, True), + (True, GuardrailEventHooks.pre_call, None, {}, None, False), + (False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {}, None, False), + (False, GuardrailEventHooks.pre_call, None, {"litellm_metadata": {"guardrails": ["pii-guard"]}}, None, False), + (False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {"litellm_metadata": {"guardrails": []}}, None, False), + (True, GuardrailEventHooks.pre_call, None, {"stream": True}, "streaming", False), + (True, GuardrailEventHooks.pre_call, None, {}, "streaming", True), + (True, GuardrailEventHooks.pre_call, None, {"stream": True}, "non_streaming", True), + (True, GuardrailEventHooks.pre_call, None, {}, "non_streaming", False), + ], + ids=[ + "guardrail-not-requested", + "guardrail-not-registered", + "default-on-post-call-guardrail", + "default-on-pre-call-guardrail", + "guardrail-on-the-route", + "guardrail-in-litellm-metadata", + "route-guardrail-beside-an-empty-client-list", + "streaming-guardrail-on-a-stream", + "streaming-guardrail-off-a-stream", + "non-streaming-guardrail-on-a-stream", + "non-streaming-guardrail-off-a-stream", + ], +) +async def test_pass_through_keeps_the_user_id_from_routes_a_guardrail_scans( + default_on: bool, + event_hook: GuardrailEventHooks, + guardrails_config: dict[str, object] | None, + client_fields: Mapping[str, object], + stream_scope: str | None, + forwarded: bool, +) -> None: + guardrail: Final = CustomGuardrail( + guardrail_name="pii-guard", default_on=default_on, event_hook=event_hook, stream_scope=stream_scope + ) + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}, **client_fields}) + body_without_client_fields: Final = {k: v for k, v in client_body.items() if k not in ("metadata", "litellm_metadata")} + + with patch.object(litellm, "callbacks", [guardrail]): + relayed: Final = await _relay_through_pass_through( + client_body, _ANTHROPIC_MESSAGES, guardrails_config=guardrails_config + ) + + assert relayed == ( + {**body_without_client_fields, "metadata": {"user_id": "user-1"}} if forwarded else body_without_client_fields + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +async def test_pass_through_warns_without_the_value_when_it_drops_a_refused_user_id( + caplog: pytest.LogCaptureFixture, +) -> None: + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "person@example.com"}}) + + with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name): + await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None) + + assert "Not forwarding metadata.user_id" in caplog.text + assert "person@example.com" not in caplog.text + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +async def test_pass_through_drops_metadata_that_is_not_an_object() -> None: + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": "user-1"}) + + relayed: Final = await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None) + + assert relayed == _MESSAGES_BODY + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +async def test_pass_through_custom_body_replaces_the_client_user_id() -> None: + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}}) + + relayed: Final = await _relay_through_pass_through( + client_body, _ANTHROPIC_MESSAGES, guardrails_config=None, custom_body=dict(_MESSAGES_BODY) + ) + + assert relayed == _MESSAGES_BODY + + _PARTIAL_RELAY_WARNING_MARKER = "ended before upstream body was fully relayed" diff --git a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py index 5013ecd98a2..26ee9f04d01 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py +++ b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py @@ -996,3 +996,21 @@ async def test_guardrails_only_and_skip_guardrails_are_mutually_exclusive( guardrails_only=True, skip_guardrails=True, ) + + +@pytest.mark.parametrize( + "request_data, expected", + [ + ({"metadata": {"_guardrail_pipelines": ["p1"]}}, True), + ({"metadata": {"user_id": "u"}, "litellm_metadata": {"_guardrail_pipelines": ["p1"]}}, True), + ({"metadata": {"_guardrail_pipelines": 5}}, True), + ({"metadata": {"user_id": "u"}}, False), + ], + ids=["pipeline-in-metadata", "pipeline-in-litellm-metadata", "non-list-pipeline-value", "no-pipeline"], +) +def test_request_has_pre_call_guardrails_reads_pipelines_from_both_metadata_buckets( + proxy_logging, monkeypatch, request_data, expected +): + monkeypatch.setattr(litellm, "callbacks", []) + + assert proxy_logging.request_has_pre_call_guardrails(request_data) is expected From a7b66dd8b9db2c4e2b6e00e9b62faf44d6ddd223 Mon Sep 17 00:00:00 2001 From: Vineeth Sai Varikuntla Date: Sat, 31 Oct 2026 00:29:10 -0700 Subject: [PATCH 11/15] fix(openai): stop stripping cache_control from the caller's own messages and tools (#35602) * fix(openai): stop stripping cache_control from the caller's messages Copy before removing the field, so the caller's own message objects are left alone. * fix(fireworks_ai): strip cache_control on a copy so a fallback keeps the caller's hints * fix: keep the caller's request intact on more providers and trim extra Claude cache hints under drop_params * fix: drop only the oldest extra cache hints inside one message when trimming to the Claude limit * fix: keep replayed session turns yielding their hints as whole turns while a request's own message keeps its newest hints * fix(cache_control): count only hints that reach Claude and cap hints on OpenAI-compatible Claude routes under drop_params * fix(cache_control): keep the capped Claude messages a list in OpenAI-compatible request bodies * fix(caching): cap Claude hints for string drop_params, a Claude base_model, and Converse tool messages The 4-hint cap now reads drop_params through drops_unsupported_params on chat, Converse, and Anthropic routes, names Claude by base_model on OpenAI-compatible routes, and counts a Converse tool message's hints as the one cache point it sends. * fix(caching): trim Claude cache hints in one wire-order pass and reserve a top-level cache_control on OpenAI-compatible routes Dropping the oldest hint one at a time re-walked the whole request per hint, which took seconds on long tool histories. The cap now drops each message's oldest hints in the order the provider counts them, so a string assistant turn loses its text hint before its tool call hints. OpenAI-compatible Claude routes also keep room for a request-level or extra_body cache_control under drop_params. * test: drop narrating comments and stacked generator clauses from cache_control tests * fix(vertex_ai): put every system message in the Gemini context cache so generateContent never sends it beside cachedContent --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../anthropic_cache_control_hook.py | 210 +++++++++++++++++- .../prompt_templates/common_utils.py | 17 +- litellm/llms/anthropic/chat/transformation.py | 40 +++- .../bedrock/chat/converse_transformation.py | 35 ++- litellm/llms/deepinfra/chat/transformation.py | 5 +- .../llms/fireworks_ai/chat/transformation.py | 38 ++-- litellm/llms/groq/chat/transformation.py | 9 +- .../llms/hosted_vllm/chat/transformation.py | 3 +- litellm/llms/mistral/chat/transformation.py | 3 +- .../llms/openai/chat/gpt_transformation.py | 186 +++++++++------- .../openai/chat/o_series_transformation.py | 9 +- litellm/llms/openai/common_utils.py | 44 +++- .../llms/openai/responses/transformation.py | 53 ++--- .../context_caching/transformation.py | 43 ++-- litellm/llms/watsonx/chat/transformation.py | 3 +- litellm/llms/xai/chat/transformation.py | 4 +- .../transformation.py | 105 ++------- ...ore_utils_prompt_templates_common_utils.py | 14 ++ .../test_anthropic_chat_transformation.py | 184 +++++++++++++++ .../chat/test_converse_transformation.py | 83 +++++++ .../test_databricks_chat_transformation.py | 63 ++++++ .../test_deepinfra_chat_transformation.py | 37 +++ .../test_fireworks_ai_chat_transformation.py | 61 +++++ .../chat/test_groq_chat_transformation.py | 17 ++ .../test_hosted_vllm_chat_transformation.py | 42 ++++ .../test_mistral_chat_transformation.py | 18 +- .../chat/test_openai_gpt_transformation.py | 138 ++++++++++++ .../test_openai_responses_transformation.py | 62 +++++- .../openai/test_o_series_transformation.py | 21 ++ .../test_openrouter_chat_transformation.py | 55 +++++ .../test_vertex_ai_context_caching.py | 44 ++++ tests/unit/llms/watsonx/test_watsonx.py | 42 ++++ .../llms/xai/test_xai_chat_transformation.py | 42 ++++ .../test_litellm_completion_responses.py | 11 + 34 files changed, 1463 insertions(+), 278 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index d816bc661f3..39368dee47c 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -14,7 +14,7 @@ import itertools import os import re from collections.abc import Callable, Container, Iterable, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast from urllib.parse import urlparse from pydantic import TypeAdapter, ValidationError @@ -102,6 +102,192 @@ def _validated_object_list(value: object) -> list[object] | None: return None +_MessageT: Final = TypeVar("_MessageT") +_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None] +_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls") + + +def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]: + lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS) + return tuple( + itertools.chain.from_iterable( + _OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list) + ) + ) + + +def _block_carries_cache_control(block: object) -> bool: + return isinstance(block, dict) and _CONTENT_PART_ADAPTER.validate_python(block).get("cache_control") is not None + + +_ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT: Final = frozenset({"tool", "function"}) + + +def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool: + if fields.get("cache_control") is None: + return False + return ( + not isinstance(fields.get("content"), list) + or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT + ) + + +def cache_control_count(message: object) -> int: + if not isinstance(message, dict): + return 0 + fields: Final = _CONTENT_PART_ADAPTER.validate_python(message) + block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block)) + return int(_message_mark_reaches_provider(fields)) + block_marks + + +def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]: + others: Final = {key: value for key, value in fields.items() if key != "cache_control"} + if "cache_control" not in fields: + return others + cache_control: Final = rewrite(fields["cache_control"]) + return others if cache_control is None else {**others, "cache_control": cache_control} + + +def block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object: + if not isinstance(block, dict): + return block + return _fields_with_cache_control(_CONTENT_PART_ADAPTER.validate_python(block), rewrite) + + +def with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT: + if not isinstance(message, dict): + return message + fields: Final = _CONTENT_PART_ADAPTER.validate_python(message) + blocks_rewritten: Final = { + key: ( + [block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)] + if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list) + else value + ) + for key, value in fields.items() + } + return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped + _MessageT, _fields_with_cache_control(blocks_rewritten, rewrite) + ) + + +def dropped_cache_control(_: object) -> None: + return None + + +def _first_marked_index(value: object) -> int | None: + if not isinstance(value, list): + return None + blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value) + return next((index for index, block in enumerate(blocks) if _block_carries_cache_control(block)), None) + + +_MarkSlot: TypeAlias = tuple[str, int] +_MESSAGE_MARK_SLOT: Final[_MarkSlot] = ("cache_control", -1) + + +def _block_mark_slots(fields: Mapping[str, object], key: str) -> tuple[_MarkSlot, ...]: + value: Final = fields.get(key) + if not isinstance(value, list): + return () + blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value) + return tuple((key, index) for index, block in enumerate(blocks) if _block_carries_cache_control(block)) + + +def _mark_slots_in_wire_order(fields: Mapping[str, object]) -> tuple[_MarkSlot, ...]: + content_slots: Final = _block_mark_slots(fields, "content") + tool_call_slots: Final = _block_mark_slots(fields, "tool_calls") + if not _message_mark_reaches_provider(fields): + return content_slots + tool_call_slots + if isinstance(fields.get("content"), list): + return content_slots + (_MESSAGE_MARK_SLOT,) + tool_call_slots + return (_MESSAGE_MARK_SLOT,) + content_slots + tool_call_slots + + +def _blocks_without_marks(key: str, value: object, dropped_slots: frozenset[_MarkSlot]) -> object: + if key not in _MARK_CARRYING_MESSAGE_FIELDS or not isinstance(value, list): + return value + return [ + block_with_cache_control(block, dropped_cache_control) if (key, index) in dropped_slots else block + for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(value)) + ] + + +def _without_oldest_marks(message: _MessageT, dropped: int) -> _MessageT: + if dropped == 0 or not isinstance(message, dict): + return message + fields: Final = _CONTENT_PART_ADAPTER.validate_python(message) + dropped_slots: Final = frozenset(_mark_slots_in_wire_order(fields)[:dropped]) + blocks_unmarked: Final = {key: _blocks_without_marks(key, value, dropped_slots) for key, value in fields.items()} + return cast( # cast-ok: the same message with its oldest wire-order marks dropped + _MessageT, + _fields_with_cache_control(blocks_unmarked, dropped_cache_control) + if _MESSAGE_MARK_SLOT in dropped_slots + else blocks_unmarked, + ) + + +def tool_message_with_single_cache_mark(message: _MessageT) -> _MessageT: + if not isinstance(message, dict): + return message + fields: Final = _CONTENT_PART_ADAPTER.validate_python(message) + content: Final = fields.get("content") + if fields.get("role") != "tool" or not isinstance(content, list): + return message + kept_index: Final = None if fields.get("cache_control") is not None else _first_marked_index(content) + return cast( # cast-ok: the same tool message keeping only the mark its single cache point reads + _MessageT, + { + **fields, + "content": [ + block if index == kept_index else block_with_cache_control(block, dropped_cache_control) + for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(content)) + ], + }, + ) + + +_BudgetSpend: TypeAlias = Callable[[tuple[int, int], int], tuple[int, int]] + + +def _newest_marks_kept(state: tuple[int, int], marks: int) -> tuple[int, int]: + remaining, _ = state + kept: Final = min(marks, remaining) + return remaining - kept, marks - kept + + +def _whole_message_kept(state: tuple[int, int], marks: int) -> tuple[int, int]: + remaining, _ = state + if marks > remaining: + return remaining, marks + return remaining - marks, 0 + + +def _messages_within_budget( + messages: Sequence[_MessageT], reserved_marks: int, spend: _BudgetSpend +) -> Sequence[_MessageT]: + budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0) + newest_first: Final = tuple(reversed(messages)) + budget_after_each: Final = itertools.accumulate( + (cache_control_count(message) for message in newest_first), spend, initial=(budget, 0) + ) + dropped_counts: Final = tuple(dropped for _, dropped in itertools.islice(budget_after_each, 1, None)) + kept_newest_first: Final = tuple( + _without_oldest_marks(message, dropped) for message, dropped in zip(newest_first, dropped_counts) + ) + return tuple(reversed(kept_newest_first)) + + +def newest_marks_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]: + return _messages_within_budget(messages, reserved_marks, _newest_marks_kept) + + +def whole_messages_within_cache_control_limit( + messages: Sequence[_MessageT], reserved_marks: int +) -> Sequence[_MessageT]: + return _messages_within_budget(messages, reserved_marks, _whole_message_kept) + + def _part_with_cache_control_renamed(part: object) -> object: if not _accepts_prompt_cache_breakpoint(part): return part @@ -1439,3 +1625,25 @@ class AnthropicCacheControlHook(CustomPromptManagement): llm_router=None, ) return None + + +def messages_within_cache_control_cap( + messages: Sequence[_MessageT], tools: object, cache_control: object = None, request_kwargs: object = None +) -> Sequence[_MessageT]: + """Under drop_params, drop the oldest message hints the provider's cache_control cap has no room for. + + Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first. + """ + reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints( + _validated_object_list(tools), cache_control, request_kwargs + ) + kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks) + dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept)) + if dropped > 0: + verbose_logger.warning( + "Dropped %d cache_control hint(s) from the oldest messages because drop_params is on and the " + "provider accepts at most %d cache_control blocks per request", + dropped, + MAX_CACHE_CONTROL_BLOCKS, + ) + return kept diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 862e61cb631..e733f48ffd7 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -88,17 +88,22 @@ def handle_any_messages_to_chat_completion_str_messages_conversion( return [{"input": str(messages)}] +def _message_with_str_content(message: AllMessageValues) -> AllMessageValues: + texts: Final = convert_content_list_to_str(message=message) + if not texts: + return message + return cast( # cast-ok: the same message with its content flattened to one string + AllMessageValues, {**message, "content": texts} + ) + + def handle_messages_with_content_list_to_str_conversion( messages: list[AllMessageValues], ) -> list[AllMessageValues]: """ - Handles messages with content list conversion + Returns new messages with each content list flattened to a string, leaving the caller's messages untouched """ - for message in messages: - texts = convert_content_list_to_str(message=message) - if texts: - message["content"] = texts - return messages + return [_message_with_str_content(message) for message in messages] def strip_name_from_message(message: AllMessageValues, allowed_name_roles: list[str] = ["user"]) -> AllMessageValues: diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index d773e31c733..25245c347f4 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -167,6 +167,7 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool) _TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _TOOLS_ADAPTER: Final = TypeAdapter(list[object]) _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset( ("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid") ) @@ -2254,6 +2255,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): headers=headers, ) + @staticmethod + def _messages_within_cache_control_cap( + messages: Sequence[AllMessageValues], optional_params: Mapping[str, object] + ) -> Sequence[AllMessageValues]: + from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap + + return messages_within_cache_control_cap( + messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params + ) + def transform_request( self, model: str, @@ -2271,9 +2282,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): anthropic_messages_pt, ) - drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True + drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params")) + capped_messages: Final = ( + list( + self._messages_within_cache_control_cap( + messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params) + ) + ) + if drop_params + else messages + ) - if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages): + if "tools" not in optional_params and has_tool_call_blocks(capped_messages): optional_params["tools"], _ = self._map_tools(add_dummy_tool(custom_llm_provider="anthropic")) # Drop thinking param if thinking is enabled but thinking_blocks are missing @@ -2285,9 +2305,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # Related issue: https://github.com/BerriAI/litellm/issues/18926 if ( optional_params.get("thinking") is not None - and messages is not None - and last_assistant_with_tool_calls_has_no_thinking_blocks(messages) - and not any_assistant_message_has_thinking_blocks(messages) + and last_assistant_with_tool_calls_has_no_thinking_blocks(capped_messages) + and not any_assistant_message_has_thinking_blocks(capped_messages) ): if litellm.modify_params: optional_params.pop("thinking", None) @@ -2332,15 +2351,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): _name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request( optional_params=optional_params, ) - if _name_forward_map: - messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map) + request_messages: Final = ( + self._rewrite_tool_names_in_messages(capped_messages, _name_forward_map) + if _name_forward_map + else capped_messages + ) if _name_reverse_map and isinstance(litellm_params, dict): litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map # Only the leading system run becomes the top-level system prompt. A later # system message stays in the conversation: hoisting it rewrites the cached # prefix and re-bills the whole history at cache-write pricing (#36559). - leading_system_run, later_messages = split_leading_system_run(messages) + leading_system_run, later_messages = split_leading_system_run(request_messages) anthropic_system_message_list: Final = self.translate_system_message(messages=list(leading_system_run)) # Handling anthropic API Prompt Caching if len(anthropic_system_message_list) > 0: @@ -2362,7 +2384,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): except Exception as e: raise AnthropicError( status_code=400, - message=f"{e}\nReceived Messages={messages}", + message=f"{e}\nReceived Messages={request_messages}", ) # don't use verbose_logger.exception, if exception is raised self.update_headers_with_optional_anthropic_beta( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 3800489ec05..21af97e523e 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -25,6 +25,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.core_helpers import ( as_str_mapping, + drops_unsupported_params, filter_exceptions_from_params, filter_internal_params, map_finish_reason, @@ -2129,6 +2130,24 @@ class AmazonConverseConfig(BaseConfig): return data + @staticmethod + def _claude_messages_within_cache_control_cap( + messages: Sequence[AllMessageValues], + capability_model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + ) -> Sequence[AllMessageValues]: + if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower(): + return messages + from litellm.integrations.anthropic_cache_control_hook import ( + messages_within_cache_control_cap, + tool_message_with_single_cache_mark, + ) + + return messages_within_cache_control_cap( + tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools") + ) + async def _async_transform_request( self, model: str, @@ -2140,7 +2159,13 @@ class AmazonConverseConfig(BaseConfig): base_model: Final = litellm_params.get("base_model") capability_model: Final = bedrock_capability_model(model, base_model) messages, system_content_blocks = self._transform_system_message( - messages, model=capability_model, deployment_model=model + list( + self._claude_messages_within_cache_control_cap( + messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params + ) + ), + model=capability_model, + deployment_model=model, ) # Convert last user message to guarded_text if guardrailConfig is present @@ -2221,7 +2246,13 @@ class AmazonConverseConfig(BaseConfig): base_model: Final = litellm_params.get("base_model") capability_model: Final = bedrock_capability_model(model, base_model) messages, system_content_blocks = self._transform_system_message( - messages, model=capability_model, deployment_model=model + list( + self._claude_messages_within_cache_control_cap( + messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params + ) + ), + model=capability_model, + deployment_model=model, ) # Convert last user message to guarded_text if guardrailConfig is present diff --git a/litellm/llms/deepinfra/chat/transformation.py b/litellm/llms/deepinfra/chat/transformation.py index 004128c0184..7caef58dc79 100644 --- a/litellm/llms/deepinfra/chat/transformation.py +++ b/litellm/llms/deepinfra/chat/transformation.py @@ -130,7 +130,8 @@ class DeepInfraConfig(OpenAIGPTConfig): - Input: {"role": "tool", "content": [{"type": "text", "text": "result"}]} - Output: {"role": "tool", "content": "[{\"type\": \"text\", \"text\": \"result\"}]"} """ - for message in messages: + copied_messages: Final = [message.copy() for message in messages] + for message in copied_messages: if message.get("role") == "tool": content = message.get("content") @@ -149,7 +150,7 @@ class DeepInfraConfig(OpenAIGPTConfig): # For complex content, serialize the entire array as JSON string message["content"] = json.dumps(content) - return messages + return copied_messages @overload def _transform_messages( diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 92c6c8439b3..5a9e5745c40 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -23,6 +23,7 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionToolParam, + ChatCompletionToolParamFunctionChunk, OpenAIChatCompletionToolParam, ) from litellm.types.utils import ( @@ -45,6 +46,7 @@ from ...openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, OpenAIGPTConfig, ) +from ...openai.common_utils import without_cache_control from ..common_utils import ( FIREROUTER, FireworksAIException, @@ -480,15 +482,23 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): return (("response_format", _json_schema_response_format(choice_schema, "choice")),) def _transform_tools(self, tools: list[OpenAIChatCompletionToolParam]) -> list[OpenAIChatCompletionToolParam]: - for tool in tools: - if tool.get("type") != "function": - continue - function = tool["function"] - function.pop("strict", None) - params = function.get("parameters") - if isinstance(params, dict): - unpack_legacy_defs(params) - return tools + return [self._fireworks_tool(tool) for tool in tools] + + @staticmethod + def _fireworks_tool(tool: OpenAIChatCompletionToolParam) -> OpenAIChatCompletionToolParam: + if tool.get("type") != "function": + return tool + function: Final = tool["function"] + params: Final = function.get("parameters") + unstrict_function: Final = cast( # cast-ok: the same function definition minus `strict` + ChatCompletionToolParamFunctionChunk, {key: value for key, value in function.items() if key != "strict"} + ) + new_function: Final[ChatCompletionToolParamFunctionChunk] = ( + {**unstrict_function, "parameters": unpack_legacy_defs(params, copy=True)} + if isinstance(params, dict) + else unstrict_function + ) + return {**tool, "function": new_function} def _transform_messages_helper( self, messages: list[AllMessageValues], model: str, litellm_params: dict @@ -496,10 +506,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): """ Strip fields not permitted by FireworksAI from messages. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - filter_value_from_dict, - ) - supports_vision_value: Final = self._get_model_cost_capability_exact(model=model, capability="supports_vision") for message in messages: if message["role"] == "user": @@ -530,9 +536,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): model=model, llm_provider="fireworks_ai", ) - filter_value_from_dict(cast(dict, message), "cache_control") - return [strip_litellm_response_only_fields(message) for message in messages] + return [ + strip_litellm_response_only_fields(cast(AllMessageValues, without_cache_control(message))) + for message in messages + ] # Cached index of fireworks_ai/* entries from litellm.model_cost. Building # this index requires a full scan of model_cost (tens of thousands of diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 23f68335ea9..fd6b81fce8a 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -140,7 +140,8 @@ class GroqChatConfig(OpenAILikeChatConfig): def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: bool = False ) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]: - for idx, message in enumerate(messages): + transformed_messages: Final = list(messages) + for idx, message in enumerate(transformed_messages): """ 1. Don't pass 'null' function_call assistant message to groq - https://github.com/BerriAI/litellm/issues/5839 """ @@ -154,12 +155,12 @@ class GroqChatConfig(OpenAILikeChatConfig): for k, v in _message.items(): if v is not None: new_message[k] = v - messages[idx] = strip_litellm_internal_message_fields(new_message) + transformed_messages[idx] = strip_litellm_internal_message_fields(new_message) if is_async: - return super()._transform_messages(messages=messages, model=model, is_async=True) + return super()._transform_messages(messages=transformed_messages, model=model, is_async=True) else: - return super()._transform_messages(messages=messages, model=model, is_async=False) + return super()._transform_messages(messages=transformed_messages, model=model, is_async=False) def transform_messages( self, diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 9f9503f93b6..3386cd452da 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -2,6 +2,7 @@ Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions` """ +import copy import json from collections.abc import Coroutine from typing import Final, Literal, cast, overload @@ -88,7 +89,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): ) -> dict: _tools = non_default_params.pop("tools", None) if _tools is not None: - _tools = remove_additional_properties(_tools) + _tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params _tools = remove_strict_from_schema(_tools) if isinstance(_tools, list): _tools = self._convert_custom_tools_to_function_tools(_tools) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index db8dd3fa7f0..1ecef46ffb1 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -755,7 +755,8 @@ class MistralConfig(OpenAIGPTConfig): if _name is not None: # Remove name if not a tool message if message["role"] != "tool" or isinstance(_name, str) and len(_name.strip()) == 0: - message.pop("name", None) + message_without_name: Final = {k: v for k, v in message.items() if k != "name"} + return cast(AllMessageValues, message_without_name) # cast-ok: the same message minus name return message diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 22af51aaabe..9d551f6f859 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -14,7 +14,7 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS -from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( extract_reasoning_content, handle_invalid_parallel_tool_calls, @@ -64,7 +64,7 @@ from litellm.types.utils import ( ) from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support -from ..common_utils import OpenAIError +from ..common_utils import OpenAIError, without_cache_control from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config if TYPE_CHECKING: @@ -128,6 +128,29 @@ def _litellm_params_str(litellm_params: Mapping[str, object] | None, key: str) - return value if isinstance(value, str) else None +def _drops_params(litellm_params: Mapping[str, object]) -> bool: + return drops_unsupported_params(litellm_params.get("drop_params")) + + +def _names_claude(model: str, litellm_params: Mapping[str, object]) -> bool: + base_model: Final = litellm_params.get("base_model") + return "claude" in model.lower() or (isinstance(base_model, str) and "claude" in base_model.lower()) + + +def _claude_messages_within_cache_control_cap( + model: str, + messages: Sequence[AllMessageValues], + tools: Sequence["ChatCompletionToolParam"] | None, + litellm_params: Mapping[str, object], + request_params: Mapping[str, object], +) -> Sequence[AllMessageValues]: + if not _names_claude(model, litellm_params) or not _drops_params(litellm_params): + return messages + from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap + + return list(messages_within_cache_control_cap(messages, tools, request_params.get("cache_control"), request_params)) + + def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | None: try: extra_body: Final = _OBJECT_MAPPING.validate_python(optional_params.get("extra_body")) @@ -311,52 +334,58 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): return content_copy async def _async_handle_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile: - file_id: Final = content_item.get("file_id") + content_copy: Final = content_item.copy() + file_id: Final = content_copy.get("file_id") if file_id is not None: # check for file id being url done in _handle_pdf_url base64_data: Final = await async_convert_url_to_base64(file_id) - content_item["file_data"] = base64_data - content_item["filename"] = "my_file.pdf" - content_item.pop("file_id") - return content_item + content_copy["file_data"] = base64_data + content_copy["filename"] = "my_file.pdf" + content_copy.pop("file_id") + return content_copy def _common_file_data_check(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile: file_data: Final = content_item.get("file_data") filename: Final = content_item.get("filename") if file_data is not None and filename is None: - content_item["filename"] = "my_file.pdf" + return {**content_item, "filename": "my_file.pdf"} return content_item + @staticmethod + def _image_item_for_openai(content_item: ChatCompletionImageObject) -> ChatCompletionImageObject: + image_url: Final = content_item["image_url"] + if isinstance(image_url, str): + return {**content_item, "image_url": {"url": image_url}} + if not isinstance(image_url, dict): + return content_item + new_image_url_obj: Final = cast( + ChatCompletionImageUrlObject, + {k: v for k, v in image_url.items() if k not in {"format", "video_metadata"}}, + ) + return {**content_item, "image_url": new_image_url_obj} + + @staticmethod + def _file_item_for_openai(content_item: ChatCompletionFileObject) -> ChatCompletionFileObject: + litellm_specific_params: Final = {"format", "detail", "video_metadata"} + file_obj: Final = content_item.get("file") + if file_obj is None: + raise litellm.BadRequestError( + message="Content block has type='file' but is missing the required 'file' field", + model=None, + llm_provider="openai", + ) + new_file_obj: Final = ChatCompletionFileObjectFile( + **{k: v for k, v in file_obj.items() if k not in litellm_specific_params} + ) + return {**content_item, "file": new_file_obj} + def _apply_common_transform_content_item( self, content_item: OpenAIMessageContentListBlock, ) -> OpenAIMessageContentListBlock: - litellm_specific_params: Final = {"format", "detail", "video_metadata"} if content_item.get("type") == "image_url": - content_item = cast(ChatCompletionImageObject, content_item) - if isinstance(content_item["image_url"], str): - content_item["image_url"] = { - "url": content_item["image_url"], - } - elif isinstance(content_item["image_url"], dict): - new_image_url_obj: Final = cast( - ChatCompletionImageUrlObject, - {k: v for k, v in content_item["image_url"].items() if k not in {"format", "video_metadata"}}, - ) - content_item["image_url"] = new_image_url_obj - elif content_item.get("type") == "file": - content_item = cast(ChatCompletionFileObject, content_item) - file_obj: Final = content_item.get("file") - if file_obj is None: - raise litellm.BadRequestError( - message="Content block has type='file' but is missing the required 'file' field", - model=None, - llm_provider="openai", - ) - new_file_obj: Final = ChatCompletionFileObjectFile( - **{k: v for k, v in file_obj.items() if k not in litellm_specific_params} - ) - content_item["file"] = new_file_obj - + return self._image_item_for_openai(cast(ChatCompletionImageObject, content_item)) + if content_item.get("type") == "file": + return self._file_item_for_openai(cast(ChatCompletionFileObject, content_item)) return content_item def _transform_content_item( @@ -419,31 +448,34 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): hoisted_messages: Final = hoist_images_from_tool_messages(stripped_messages) async def _async_transform(): - for message in hoisted_messages: - message_content = message.get("content") - message_role = message.get("role") - - if message_role == "user" and message_content and isinstance(message_content, list): - message_content_types = cast(list[OpenAIMessageContentListBlock], message_content) - for i, content_item in enumerate(message_content_types): - message_content_types[i] = await self._async_transform_content_item( - cast(OpenAIMessageContentListBlock, content_item), - ) - return hoisted_messages + return [await self._async_user_message_for_openai(message) for message in hoisted_messages] if is_async: return _async_transform() - else: - for message in hoisted_messages: - message_content = message.get("content") - message_role = message.get("role") - if message_role == "user" and message_content and isinstance(message_content, list): - message_content_types = cast(list[OpenAIMessageContentListBlock], message_content) - for i, content_item in enumerate(message_content): - message_content_types[i] = self._transform_content_item( - cast(OpenAIMessageContentListBlock, content_item) - ) - return hoisted_messages + return [self._user_message_for_openai(message) for message in hoisted_messages] + + @staticmethod + def _user_content_parts(message: AllMessageValues) -> Sequence[OpenAIMessageContentListBlock] | None: + message_content: Final = message.get("content") + if message.get("role") != "user" or not message_content or not isinstance(message_content, list): + return None + return cast(Sequence[OpenAIMessageContentListBlock], message_content) + + def _user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues: + parts: Final = self._user_content_parts(message) + if parts is None: + return message + return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI + AllMessageValues, {**message, "content": [self._transform_content_item(part) for part in parts]} + ) + + async def _async_user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues: + parts: Final = self._user_content_parts(message) + if parts is None: + return message + return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI + AllMessageValues, {**message, "content": [await self._async_transform_content_item(part) for part in parts]} + ) def transform_messages( self, @@ -462,23 +494,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): messages: list[AllMessageValues], tools: list["ChatCompletionToolParam"] | None = None, ) -> tuple[list[AllMessageValues], list["ChatCompletionToolParam"] | None]: - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - filter_value_from_dict, - ) from litellm.types.llms.openai import ChatCompletionToolParam - for i, message in enumerate(messages): - messages[i] = cast( - AllMessageValues, - filter_value_from_dict(message, "cache_control"), - ) - if tools is not None: - for i, tool in enumerate(tools): - tools[i] = cast( - ChatCompletionToolParam, - filter_value_from_dict(tool, "cache_control"), - ) - return messages, tools + new_messages: Final = [cast(AllMessageValues, without_cache_control(message)) for message in messages] + new_tools: Final = ( + [cast(ChatCompletionToolParam, without_cache_control(tool)) for tool in tools] + if tools is not None + else None + ) + return new_messages, new_tools def translate_developer_role_to_system_role( self, @@ -527,6 +551,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): tools: Sequence["ChatCompletionToolParam"] | None, prompt_cache_options: object, litellm_params: Mapping[str, object], + request_params: Mapping[str, object], ) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]: """ A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on @@ -543,10 +568,11 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): model, custom_llm_provider, api_base, prompt_cache_options ) if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base): - return messages, tools - drops_breakpoints: Final = not reads_breakpoints and ( - litellm.drop_params is True or litellm_params.get("drop_params") is True - ) + return ( + _claude_messages_within_cache_control_cap(model, messages, tools, litellm_params, request_params), + tools, + ) + drops_breakpoints: Final = not reads_breakpoints and _drops_params(litellm_params) renamed: Final = ( [AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages] if reads_breakpoints @@ -561,7 +587,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): if drops_breakpoints else list(tools) ) - return self.remove_cache_control_flag_from_messages_and_tools(model=model, messages=renamed, tools=kept_tools) + sent_messages, sent_tools = self.remove_cache_control_flag_from_messages_and_tools( + model=model, messages=renamed, tools=kept_tools + ) + return ( + _claude_messages_within_cache_control_cap(model, sent_messages, sent_tools, litellm_params, request_params), + sent_tools, + ) def _sanitized_tools_update_for_openai( self, @@ -623,6 +655,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): tools=optional_params.get("tools", []), prompt_cache_options=_prompt_cache_options(optional_params), litellm_params=litellm_params, + request_params=optional_params, ) if tools is not None and len(tools) > 0: optional_params["tools"] = tools @@ -654,6 +687,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): tools=optional_params.get("tools", []), prompt_cache_options=_prompt_cache_options(optional_params), litellm_params=litellm_params, + request_params=optional_params, ) if tools is not None and len(tools) > 0: optional_params["tools"] = tools diff --git a/litellm/llms/openai/chat/o_series_transformation.py b/litellm/llms/openai/chat/o_series_transformation.py index 64285aafdb0..4d56a3b38c5 100644 --- a/litellm/llms/openai/chat/o_series_transformation.py +++ b/litellm/llms/openai/chat/o_series_transformation.py @@ -153,12 +153,13 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): - role: system ==> translate to role 'user' """ _supports_system_messages: Final = supports_system_messages(model, "openai") - for i, message in enumerate(messages): + transformed_messages: Final = list(messages) + for i, message in enumerate(transformed_messages): if message["role"] == "system" and not _supports_system_messages: new_message = ChatCompletionUserMessage(content=message["content"], role="user") - messages[i] = new_message # Replace the old message with the new one + transformed_messages[i] = new_message # Replace the old message with the new one if is_async: - return super()._transform_messages(messages, model, is_async=cast(Literal[True], True)) + return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True)) else: - return super()._transform_messages(messages, model, is_async=cast(Literal[False], False)) + return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[False], False)) diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index c113cb351d8..d476cf62626 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -3,6 +3,7 @@ Common helpers / utils across al OpenAI endpoints """ import asyncio +import copy import hashlib import inspect import json @@ -31,7 +32,7 @@ if TYPE_CHECKING: import litellm from litellm.exceptions import error_response_without_credentials from litellm.litellm_core_utils.completion_timeout import CompletionTimeout -from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object +from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, iter_items from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -43,6 +44,47 @@ from litellm.llms.custom_httpx.http_handler import ( ) +def _is_plain_dict(value: object) -> bool: + return isinstance(value, dict) + + +def _is_list(value: object) -> bool: + return isinstance(value, list) + + +def _contains_cache_control(value: object) -> bool: + pending: Final[list[object]] = [value] # mutable-ok: a local worklist, never returned or stored + while pending: + node = pending.pop() + mapping = as_str_mapping(node) + if mapping is not None: + if "cache_control" in mapping: + return True + pending.extend(mapping.values()) + elif _is_list(node): + pending.extend(iter_items(node)) + return False + + +def without_cache_control(item: object) -> object: + """Return `item` with every `cache_control` key removed, without touching the original. + + `filter_value_from_dict` deletes the key in place and recurses into nested dicts and + lists, so the caller's own message, tool or input object has to be copied first: it + may reuse the same list on a provider that does support prompt caching. The copy is + skipped when there is nothing to strip, which keeps the object identity the + pass-through paths rely on and avoids a deep copy on every request. + """ + from litellm.litellm_core_utils.prompt_templates.common_utils import filter_value_from_dict + + if not _is_plain_dict(item) or not _contains_cache_control(item): + return item + copied: Final = copy.deepcopy(item) + if isinstance(copied, dict): + filter_value_from_dict(copied, "cache_control") + return copied + + def _get_client_init_params(cls: type) -> tuple[str, ...]: """Extract __init__ parameter names (excluding 'self') from a class.""" return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self") diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 9b8515a9ecd..7cca31c5d3f 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -52,7 +52,7 @@ from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS -from ..common_utils import OpenAIError +from ..common_utils import OpenAIError, without_cache_control from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 @@ -658,13 +658,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search), tools=mapped_tools, ) - if marker_safe_input is not validated_input: - self._strip_caller_input_as_if_unmarked( - model=model, - caller_input=validated_input, - reads_breakpoints=reads_breakpoints, - maps_anthropic_tool_search=maps_anthropic_tool_search, - ) object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools) sanitized_tools: Final = self._sanitized_tool_schemas_for_openai( model=model, tools=object_schema_tools, litellm_params=litellm_params @@ -679,17 +672,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return marked_input return input_without_unanswered_anthropic_tool_search_calls(marked_input) - def _strip_caller_input_as_if_unmarked( - self, - model: str, - caller_input: str | ResponseInputParam, - reads_breakpoints: bool, - maps_anthropic_tool_search: bool, - ) -> None: - self.remove_cache_control_flag_from_input_and_tools( - model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search) - ) - def _input_with_wire_safe_prompt_cache_breakpoints( self, model: str, @@ -800,24 +782,29 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): the chat path. Strips Anthropic-only `cache_control` markers from Responses API input content blocks and tools. - `filter_value_from_dict` mutates each dict in place, so the same - objects are returned. + `filter_value_from_dict` deletes the key in place and recurses, so each + item is copied first: the caller keeps its own input list and may reuse + it on a provider that does support prompt caching. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - filter_value_from_dict, + new_input: Final = ( + cast( # cast-ok: the comprehension rebuilds the input item for item + ResponseInputParam, + [without_cache_control(item) for item in input], + ) + if isinstance(input, list) + else input ) - if isinstance(input, list): - for item in input: - if isinstance(item, dict): - filter_value_from_dict(cast(dict, item), "cache_control") + new_tools: Final = ( + cast( # cast-ok: the comprehension rebuilds the tool list item for item + Sequence[ALL_RESPONSES_API_TOOL_PARAMS], + [without_cache_control(tool) for tool in tools], + ) + if tools is not None + else None + ) - if tools is not None: - for tool in tools: - if isinstance(tool, dict): - filter_value_from_dict(cast(dict, tool), "cache_control") - - return input, tools + return new_input, new_tools def _drop_bridge_minted_reasoning_items(self, input: str | ResponseInputParam) -> str | ResponseInputParam: if self.custom_llm_provider not in _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING or not isinstance(input, list): diff --git a/litellm/llms/vertex_ai/context_caching/transformation.py b/litellm/llms/vertex_ai/context_caching/transformation.py index f683ad0e377..a4601af944b 100644 --- a/litellm/llms/vertex_ai/context_caching/transformation.py +++ b/litellm/llms/vertex_ai/context_caching/transformation.py @@ -110,38 +110,23 @@ def separate_cached_messages( messages: list[AllMessageValues], ) -> tuple[list[AllMessageValues], list[AllMessageValues]]: """ - Returns separated cached and non-cached messages. - - Args: - messages: List of messages to be separated. - - Returns: - Tuple containing: - - cached_messages: List of cached messages. - - non_cached_messages: List of non-cached messages. + Returns the first continuous block of cached messages, plus every system message once + any message is cached, since generateContent rejects a system_instruction beside a + cachedContent, and the remaining messages. """ - cached_messages: list[AllMessageValues] = [] - non_cached_messages: list[AllMessageValues] = [] + filtered_messages: Final = [(idx, message) for idx, message in enumerate(messages) if is_cached_message(message)] + if not filtered_messages: + return [], messages - # Extract cached messages and their indices - filtered_messages: Final[list[tuple[int, AllMessageValues]]] = [] - for idx, message in enumerate(messages): - if is_cached_message(message=message): - filtered_messages.append((idx, message)) - - # Validate only one block of continuous cached messages last_continuous_block_idx: Final = get_first_continuous_block_idx(filtered_messages) - # Separate messages based on the block of cached messages - if filtered_messages and last_continuous_block_idx is not None: - first_cached_idx: Final = filtered_messages[0][0] - last_cached_idx: Final = filtered_messages[last_continuous_block_idx][0] - - cached_messages = messages[first_cached_idx : last_cached_idx + 1] - non_cached_messages = messages[:first_cached_idx] + messages[last_cached_idx + 1 :] - else: - non_cached_messages = messages - - return cached_messages, non_cached_messages + cached_block: Final = range(filtered_messages[0][0], filtered_messages[last_continuous_block_idx][0] + 1) + goes_in_cache: Final = tuple( + idx in cached_block or message.get("role") == "system" for idx, message in enumerate(messages) + ) + return ( + [message for message, cached in zip(messages, goes_in_cache) if cached], + [message for message, cached in zip(messages, goes_in_cache) if not cached], + ) def cached_messages_end_on_supported_turn(cached_messages: Sequence[AllMessageValues]) -> bool: diff --git a/litellm/llms/watsonx/chat/transformation.py b/litellm/llms/watsonx/chat/transformation.py index 071ff7108b9..25981b0f04c 100644 --- a/litellm/llms/watsonx/chat/transformation.py +++ b/litellm/llms/watsonx/chat/transformation.py @@ -4,6 +4,7 @@ Translation from OpenAI's `/chat/completions` endpoint to IBM WatsonX's `/text/c Docs: https://cloud.ibm.com/apidocs/watsonx-ai#text-chat """ +import copy from typing import Final from litellm import verbose_logger @@ -57,7 +58,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig): _tools = non_default_params.pop("tools", None) if _tools is not None: # remove 'additionalProperties' from tools - _tools = remove_additional_properties(_tools) + _tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params # remove 'strict' from tools _tools = remove_strict_from_schema(_tools) if _tools is not None: diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py index 2b943fa4107..f53b71a5a9a 100644 --- a/litellm/llms/xai/chat/transformation.py +++ b/litellm/llms/xai/chat/transformation.py @@ -1,3 +1,4 @@ +import copy from collections.abc import AsyncIterator, Iterator, Mapping from types import MappingProxyType from typing import Any, Final @@ -208,7 +209,8 @@ class XAIChatConfig(OpenAIGPTConfig): tools = [ filtered_tool for filtered_tool in ( - _FILTERED_TOOL.validate_python(filter_value_from_dict(tool, "strict")) for tool in value + _FILTERED_TOOL.validate_python(filter_value_from_dict(copy.deepcopy(tool), "strict")) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params + for tool in value ) if filtered_tool is not None ] diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 8f0a85d67de..2e156d5abba 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -8,7 +8,7 @@ import uuid from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from functools import partial -from itertools import accumulate, chain, groupby, islice +from itertools import chain, groupby from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -45,7 +45,15 @@ from litellm.constants import ( REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER, ) -from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook +from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + block_with_cache_control, + cache_control_count, + dropped_cache_control, + newest_marks_within_cache_control_limit, + whole_messages_within_cache_control_limit, + with_cache_control, +) from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, @@ -603,10 +611,6 @@ def _validated_cache_control(value: object) -> ChatCompletionCachedContent | Non return None -def _dropped_cache_control(_: object) -> None: - return None - - def _without_cache_control_key(item: object) -> object: if not isinstance(item, dict): return item @@ -694,83 +698,6 @@ def _content_blocks_carrying(content: object, cache_control: ChatCompletionCache return _with_cache_control_on_last_block(_OBJECT_LIST_ADAPTER.validate_python(content), cache_control) -_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls") - - -def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]: - lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS) - return tuple( - chain.from_iterable(_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list)) - ) - - -def _block_carries_cache_control(block: object) -> bool: - return isinstance(block, dict) and _STR_KEY_DICT_ADAPTER.validate_python(block).get("cache_control") is not None - - -def _cache_control_count(message: object) -> int: - if not isinstance(message, dict): - return 0 - fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message) - block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block)) - return int(fields.get("cache_control") is not None) + block_marks - - -_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None] - - -def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]: - others: Final = {key: value for key, value in fields.items() if key != "cache_control"} - if "cache_control" not in fields: - return others - cache_control: Final = rewrite(fields["cache_control"]) - return others if cache_control is None else {**others, "cache_control": cache_control} - - -def _block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object: - if not isinstance(block, dict): - return block - return _fields_with_cache_control(_STR_KEY_DICT_ADAPTER.validate_python(block), rewrite) - - -def _with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT: - if not isinstance(message, dict): - return message - fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message) - blocks_rewritten: Final = { - key: ( - [_block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)] - if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list) - else value - ) - for key, value in fields.items() - } - return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped - _MessageT, _fields_with_cache_control(blocks_rewritten, rewrite) - ) - - -def _kept_within_budget(state: tuple[int, bool], marks: int) -> tuple[int, bool]: - remaining, _ = state - if marks > remaining: - return remaining, False - return remaining - marks, True - - -def _messages_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]: - budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0) - newest_first: Final = tuple(reversed(messages)) - budget_after_each: Final = accumulate( - (_cache_control_count(message) for message in newest_first), _kept_within_budget, initial=(budget, True) - ) - kept_flags: Final = tuple(kept for _, kept in islice(budget_after_each, 1, None)) - kept_newest_first: Final = tuple( - message if kept else _with_cache_control(message, _dropped_cache_control) - for message, kept in zip(newest_first, kept_flags) - ) - return tuple(reversed(kept_newest_first)) - - _SYSTEM_MESSAGE_FIELDS: Final = frozenset(("content", "cache_control")) @@ -799,7 +726,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT: if fields is None: return message if fields.get("role") == "tool": - return _with_cache_control(message, _dropped_cache_control) + return with_cache_control(message, dropped_cache_control) tool_calls: Final = fields.get("tool_calls") if not isinstance(tool_calls, list): return message @@ -808,7 +735,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT: { **fields, "tool_calls": [ - _block_with_cache_control(tool_call, _dropped_cache_control) + block_with_cache_control(tool_call, dropped_cache_control) for tool_call in _OBJECT_LIST_ADAPTER.validate_python(tool_calls) ], }, @@ -822,7 +749,7 @@ class _ItemMarkRules: external_marks: int def sanitized(self, message: _MessageT) -> _MessageT: - valid: Final = _with_cache_control(message, _validated_cache_control) + valid: Final = with_cache_control(message, _validated_cache_control) return _without_tool_block_marks(valid) if self.refuses_tool_block_marks else valid def checked(self, message: _MessageT) -> _MessageT: @@ -845,15 +772,15 @@ def _request_messages_for_cache_points(messages: Sequence[_MessageT], rules: _It as_system: Final = tuple(_developer_message_as_system_message(message) for message in messages) if not rules.drop_params: return as_system - return _messages_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks) + return newest_marks_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks) def _replayed_messages_for_cache_points( replayed_messages: Sequence[_MessageT], current_messages: Sequence[object], rules: _ItemMarkRules ) -> Sequence[_MessageT]: as_system: Final = tuple(_developer_message_as_system_message(message) for message in replayed_messages) - current_marks: Final = sum(_cache_control_count(message) for message in current_messages) - return _messages_within_cache_control_limit( + current_marks: Final = sum(cache_control_count(message) for message in current_messages) + return whole_messages_within_cache_control_limit( tuple(map(rules.sanitized, as_system)), current_marks + rules.external_marks ) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 72e72714cd4..4ce6aee1619 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_semantic_cache_prompt_from_messages, get_str_from_messages, handle_any_messages_to_chat_completion_str_messages_conversion, + handle_messages_with_content_list_to_str_conversion, hoist_images_from_tool_messages, is_encrypted_reasoning_block, merge_consecutive_system_messages, @@ -2675,3 +2676,16 @@ def test_tpm_projection_preserves_non_media_identity_and_opaque_values() -> None projected, media = split_media_for_tpm(request) assert projected is request assert media == () + + +def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_untouched() -> None: + messages: Final = [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}, {"type": "text", "text": " world"}]}, + {"role": "assistant", "content": "Hi"}, + ] + messages_before: Final = copy.deepcopy(messages) + + flattened: Final = handle_messages_with_content_list_to_str_conversion(messages) + + assert flattened == [{"role": "user", "content": "Hello world"}, {"role": "assistant", "content": "Hi"}] + assert messages == messages_before diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index cbb60caa57b..501dd5bd0c6 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,6 +1,7 @@ import copy import json import logging +from collections.abc import Iterator from itertools import chain from typing import Final from unittest.mock import MagicMock, patch @@ -9849,3 +9850,186 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key(): assert "cache_control_injection_points" not in data assert "tools" not in data assert litellm_params["metadata"] == {} + + +def _five_cache_hint_conversation() -> list[dict[str, object]]: + return [ + {"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]}, + {"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]}, + {"role": "assistant", "content": "answer 2"}, + {"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]}, + ] + + +def _cached_texts(node: object) -> Iterator[str]: + if isinstance(node, dict): + yield from ((node["text"],) if "cache_control" in node and "text" in node else ()) + yield from chain.from_iterable(map(_cached_texts, node.values())) + elif isinstance(node, list): + yield from chain.from_iterable(map(_cached_texts, node)) + + +@pytest.mark.parametrize( + ("config", "model"), + [ + (AnthropicConfig(), "claude-haiku-5-5"), + (VertexAIAnthropicConfig(), "claude-haiku-5-5"), + (AmazonAnthropicClaudeConfig(), "anthropic.claude-haiku-5-5"), + (AzureAnthropicConfig(), "claude-haiku-5-5"), + ], + ids=["anthropic", "vertex_ai", "bedrock_invoke", "azure_ai"], +) +@pytest.mark.parametrize( + ("drop_params", "expected_cached_texts"), + [ + (True, ["answer 1", "turn 1", "turn 2", "turn 3"]), + ("true", ["answer 1", "turn 1", "turn 2", "turn 3"]), + (False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]), + ], + ids=[ + "drop_params_keeps_the_newest_four", + "string_drop_params_keeps_the_newest_four", + "without_drop_params_all_five_pass_through", + ], +) +def test_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params( + config: AnthropicConfig, + model: str, + drop_params: bool | str, + expected_cached_texts: list[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = _five_cache_hint_conversation() + messages_before: Final = copy.deepcopy(messages) + + body: Final = config.transform_request( + model=model, + messages=messages, + optional_params={}, + litellm_params={"drop_params": drop_params}, + headers={}, + ) + + assert sorted(_cached_texts(body)) == expected_cached_texts + assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts) + assert messages == messages_before + + +def test_claude_cache_control_cap_keeps_a_cached_tool_and_drops_the_oldest_message_hint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + messages: Final = _five_cache_hint_conversation()[1:] + tool: Final = { + "name": "get_weather", + "description": "Get the weather", + "input_schema": {"type": "object", "properties": {}}, + "cache_control": {"type": "ephemeral"}, + } + + body: Final = AnthropicConfig().transform_request( + model="claude-haiku-5-5", + messages=messages, + optional_params={"tools": [tool]}, + litellm_params={}, + headers={}, + ) + + assert body["tools"][0]["cache_control"] == {"type": "ephemeral"} + assert sorted(_cached_texts(body["messages"])) == ["answer 1", "turn 2", "turn 3"] + + +def test_claude_cache_control_cap_drops_a_string_assistant_turns_text_hint_before_its_tool_call_hints( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + hint: Final = {"type": "ephemeral"} + tool_calls: Final = [ + {"id": f"call_{name}", "type": "function", "function": {"name": name, "arguments": "{}"}, "cache_control": hint} + for name in ("lookup", "search") + ] + follow_up: Final = [ + {"type": "text", "text": "follow-up", "cache_control": hint}, + {"type": "text", "text": "and more", "cache_control": hint}, + ] + messages: Final = [ + {"role": "user", "content": "question"}, + {"role": "assistant", "content": "checking both", "cache_control": hint, "tool_calls": tool_calls}, + {"role": "tool", "tool_call_id": "call_lookup", "content": "found"}, + {"role": "tool", "tool_call_id": "call_search", "content": "none"}, + {"role": "user", "content": follow_up}, + ] + + body: Final = AnthropicConfig().transform_request( + model="claude-haiku-5-5", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assistant_blocks: Final = body["messages"][1]["content"] + assert [block["type"] for block in assistant_blocks] == ["text", "tool_use", "tool_use"] + assert ["cache_control" in block for block in assistant_blocks] == [False, True, True] + assert json.dumps(body).count('"cache_control"') == 4 + assert json.dumps(body).count('"cache_control"') == 4 + + +def test_claude_cache_control_cap_keeps_the_newest_hints_inside_one_message( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + messages: Final = [ + { + "role": "user", + "content": [ + *({"type": "text", "text": f"part {n}", "cache_control": {"type": "ephemeral"}} for n in range(1, 6)), + {"type": "text", "text": "Reply with the single word OK."}, + ], + } + ] + messages_before: Final = copy.deepcopy(messages) + + body: Final = AnthropicConfig().transform_request( + model="claude-haiku-5-5", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert sorted(_cached_texts(body["messages"])) == ["part 2", "part 3", "part 4", "part 5"] + assert json.dumps(body).count('"cache_control"') == 4 + assert messages == messages_before + + +def test_claude_cache_control_cap_does_not_count_a_message_level_hint_that_list_content_never_sends( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = [ + {"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}}, + {"role": "assistant", "content": "answer 1"}, + {"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}}, + {"role": "assistant", "content": "answer 2"}, + { + "role": "user", + "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}], + "cache_control": {"type": "ephemeral"}, + }, + ] + + body: Final = AnthropicConfig().transform_request( + model="claude-haiku-5-5", + messages=messages, + optional_params={}, + litellm_params={"drop_params": True}, + headers={}, + ) + + assert sorted(_cached_texts(body)) == ["system prompt", "turn 1", "turn 2", "turn 3"] + assert json.dumps(body).count('"cache_control"') == 4 diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 524caaba7a1..9ab7060d766 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -11259,3 +11259,86 @@ def test_converse_sends_only_the_header_betas_when_header_and_body_both_carry_be "thinking-binding-controls-2026-08-01", "thinking-display-updates-2026-08-18", ] + + +def _five_cache_hint_conversation() -> list[dict[str, object]]: + return [ + {"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]}, + {"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]}, + {"role": "assistant", "content": "answer 2"}, + {"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]}, + ] + + +@pytest.mark.parametrize( + ("drop_params", "expected_system_cache_points"), + [(True, 0), ("true", 0), (False, 1)], + ids=[ + "drop_params_drops_the_oldest_hint", + "string_drop_params_drops_the_oldest_hint", + "without_drop_params_all_five_pass_through", + ], +) +def test_converse_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params( + drop_params: bool | str, expected_system_cache_points: int, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = _five_cache_hint_conversation() + messages_before: Final = copy.deepcopy(messages) + + request: Final = AmazonConverseConfig().transform_request( + model="anthropic.claude-haiku-5-5", + messages=messages, + optional_params={}, + litellm_params={"drop_params": drop_params}, + headers={}, + ) + + assert len(list(_cache_points(request["system"]))) == expected_system_cache_points + assert [len(list(_cache_points(message))) for message in request["messages"]] == [1, 1, 1, 0, 1] + assert messages == messages_before + + +def test_converse_claude_tool_message_with_several_marked_blocks_counts_as_one_cache_point_against_the_cap( + monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "policy", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "question", "cache_control": {"type": "ephemeral"}}, + ], + }, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [ + {"type": "text", "text": "part 1", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "part 2", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "part 3", "cache_control": {"type": "ephemeral"}}, + ], + }, + ] + messages_before: Final = copy.deepcopy(messages) + + request: Final = AmazonConverseConfig().transform_request( + model="anthropic.claude-haiku-5-5", + messages=messages, + optional_params={ + "tools": [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}] + }, + litellm_params={"drop_params": True}, + headers={}, + ) + + assert len(list(_cache_points(request["messages"]))) == 3 + assert messages == messages_before diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index 19234767b19..7a6f9c7dc18 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -2773,3 +2773,66 @@ def test_claude_json_schema_stream_split_across_json_tools_keeps_the_non_stream_ choices: Final = _json_schema_stream_choices(respx_mock, call, split_stream) assert [choice.finish_reason for choice in choices if choice.finish_reason] == [answer.finish_reason] + + +@pytest.mark.parametrize( + ("model", "base_model"), + [("databricks-claude-haiku-5-5", None), ("support-bot-endpoint", "databricks/databricks-claude-haiku-5-5")], + ids=["claude_in_the_endpoint_name", "claude_named_only_by_base_model"], +) +def test_databricks_claude_request_over_the_cache_control_cap_keeps_the_newest_four_hints_under_drop_params( + model: str, base_model: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + hint: Final = {"type": "ephemeral"} + messages: Final = [ + {"role": "system", "content": "system prompt", "cache_control": hint}, + {"role": "user", "content": "turn 1", "cache_control": hint}, + {"role": "assistant", "content": "answer 1", "cache_control": hint}, + {"role": "user", "content": "turn 2", "cache_control": hint}, + {"role": "assistant", "content": "answer 2"}, + {"role": "user", "content": "turn 3", "cache_control": hint}, + ] + messages_before: Final = copy.deepcopy(messages) + + body: Final = DatabricksConfig().transform_request( + model=model, + messages=messages, + optional_params={}, + litellm_params={"drop_params": True, "custom_llm_provider": "databricks", "base_model": base_model}, + headers={}, + ) + + assert "cache_control" not in json.dumps(body["messages"][0]) + assert json.dumps(body["messages"]).count('"cache_control"') == 4 + assert messages == messages_before + + +@pytest.mark.parametrize( + "optional_params", + [{"cache_control": {"type": "ephemeral"}}, {"extra_body": {"cache_control": {"type": "ephemeral"}}}], + ids=["top_level_cache_control", "extra_body_cache_control"], +) +def test_databricks_claude_cache_control_cap_keeps_room_for_a_top_level_cache_control_under_drop_params( + optional_params: dict[str, object], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + hint: Final = {"type": "ephemeral"} + messages: Final = [ + {"role": "user", "content": "turn 1", "cache_control": hint}, + {"role": "assistant", "content": "answer 1", "cache_control": hint}, + {"role": "user", "content": "turn 2", "cache_control": hint}, + {"role": "assistant", "content": "answer 2", "cache_control": hint}, + {"role": "user", "content": "turn 3"}, + ] + + body: Final = DatabricksConfig().transform_request( + model="databricks-claude-haiku-5-5", + messages=messages, + optional_params=copy.deepcopy(optional_params), + litellm_params={"drop_params": True, "custom_llm_provider": "databricks"}, + headers={}, + ) + + assert "cache_control" not in json.dumps(body["messages"][0]) + assert json.dumps(body["messages"]).count('"cache_control"') == 3 diff --git a/tests/unit/llms/deepinfra/test_deepinfra_chat_transformation.py b/tests/unit/llms/deepinfra/test_deepinfra_chat_transformation.py index 7ec9216df7b..773c1d056f0 100644 --- a/tests/unit/llms/deepinfra/test_deepinfra_chat_transformation.py +++ b/tests/unit/llms/deepinfra/test_deepinfra_chat_transformation.py @@ -1,6 +1,8 @@ import asyncio +import copy import json import os +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -200,3 +202,38 @@ async def test_deepinfra_tool_message_content_transformation_async(): print(f"✓ Async test passed: {tool_message['content']}") print("\n✅ DeepInfra async tool message transformation test passed!") + + +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +async def test_deepinfra_request_stringifies_tool_content_without_touching_the_callers_message(is_async: bool) -> None: + from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig + + config: Final = DeepInfraConfig() + messages: Final = [ + {"role": "user", "content": "Calculate 10 + 10"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "calculator", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": "20"}]}, + ] + messages_before: Final = copy.deepcopy(messages) + request_kwargs: Final = { + "model": "deepseek-ai/DeepSeek-V3.2", + "messages": messages, + "optional_params": {}, + "litellm_params": {}, + "headers": {}, + } + + body: Final = ( + await config.async_transform_request(**request_kwargs) + if is_async + else config.transform_request(**request_kwargs) + ) + + assert body["messages"][2]["content"] == "20" + assert messages == messages_before diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index ed166d206c8..dd352aac66d 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1,3 +1,4 @@ +import copy import json import logging from collections.abc import Callable @@ -537,6 +538,26 @@ def test_tool_search_reply_sent_to_fireworks_drops_litellm_response_fields(): } +def test_cache_control_is_stripped_from_the_request_but_kept_on_the_callers_messages(): + hint: Final = {"type": "ephemeral"} + messages: Final = [ + {"role": "system", "content": [{"type": "text", "text": "a long cached prompt", "cache_control": hint}]}, + {"role": "user", "content": "hi", "cache_control": hint}, + ] + caller_messages_before: Final = json.dumps(messages) + + request: Final = FireworksAIConfig().transform_request( + model="accounts/fireworks/models/gpt-oss-120b", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert "cache_control" not in json.dumps(request["messages"]) + assert json.dumps(messages) == caller_messages_before + + # ----------------------------------------------------------------------------- # Regression tests for legacy / OpenAPI $ref defs in tool parameters. # @@ -2477,3 +2498,43 @@ def test_translate_developer_role_follows_the_deployment_model_info_over_the_cos ) assert [message["role"] for message in messages] == expected_roles + + +def test_fireworks_request_rewrites_tools_without_touching_the_callers_tool() -> None: + tools: Final = [ + { + "type": "function", + "function": { + "name": "create_thing", + "strict": True, + "parameters": { + "type": "object", + "properties": {"thing": {"$ref": "#/definitions/Thing"}}, + "definitions": {"Thing": {"type": "object", "properties": {"id": {"type": "string"}}}}, + }, + }, + } + ] + tools_before: Final = copy.deepcopy(tools) + + request: Final = FireworksAIConfig().transform_request( + model="accounts/fireworks/models/gpt-oss-120b", + messages=[{"role": "user", "content": "Make a thing"}], + optional_params={"tools": tools}, + litellm_params={}, + headers={}, + ) + + assert request["tools"] == [ + { + "type": "function", + "function": { + "name": "create_thing", + "parameters": { + "type": "object", + "properties": {"thing": {"type": "object", "properties": {"id": {"type": "string"}}}}, + }, + }, + } + ] + assert tools == tools_before diff --git a/tests/unit/llms/groq/chat/test_groq_chat_transformation.py b/tests/unit/llms/groq/chat/test_groq_chat_transformation.py index 7aa8c65215e..a493a259f39 100644 --- a/tests/unit/llms/groq/chat/test_groq_chat_transformation.py +++ b/tests/unit/llms/groq/chat/test_groq_chat_transformation.py @@ -1,3 +1,4 @@ +import copy import logging from typing import Final from unittest.mock import patch @@ -589,3 +590,19 @@ async def test_streamed_reasoning_reply_can_be_replayed(is_async: bool, with_too } ] assert reply.reasoning_content == "I should check the weather." + + +def test_groq_request_drops_null_assistant_fields_without_touching_the_callers_messages() -> None: + messages: Final = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello", "function_call": None, "tool_calls": None}, + {"role": "user", "content": "Again"}, + ] + messages_before: Final = copy.deepcopy(messages) + + body: Final = GroqChatConfig().transform_request( + model="openai/gpt-oss-120b", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert body["messages"][1] == {"role": "assistant", "content": "Hello"} + assert messages == messages_before diff --git a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py index c792d5dffcd..fc5ae3631ff 100644 --- a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py +++ b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py @@ -1,4 +1,6 @@ +import copy import json +from typing import Final from unittest.mock import MagicMock, patch import pytest @@ -397,3 +399,43 @@ def test_hosted_vllm_custom_tools_use_top_level_input_schema(): assert tools[0]["function"]["name"] == "search" assert tools[0]["function"]["description"] == "Search docs" assert tools[0]["function"]["parameters"] == input_schema + + +def _strict_weather_tool() -> dict[str, object]: + return { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, + }, + }, + } + + +def test_hosted_vllm_tools_are_rewritten_without_touching_the_callers_tool() -> None: + tools: Final = [_strict_weather_tool()] + tools_before: Final = copy.deepcopy(tools) + + optional_params: Final = HostedVLLMChatConfig().map_openai_params( + non_default_params={"tools": tools}, optional_params={}, model="gpt-oss-120b", drop_params=False + ) + + assert optional_params["tools"] == [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + assert tools == tools_before diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index 720a1b4f458..af18a9caf3d 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -1,3 +1,4 @@ +import copy import json import logging from collections.abc import Mapping @@ -1212,7 +1213,10 @@ def test_mistral_sends_only_the_text_items_of_a_native_thinking_chunk_beside_an_ "content": [ { "type": "thinking", - "thinking": [{"type": "text", "text": "It has whiskers."}, {"type": "reference", "reference_ids": [1]}], + "thinking": [ + {"type": "text", "text": "It has whiskers."}, + {"type": "reference", "reference_ids": [1]}, + ], }, {"type": "text", "text": "A cat."}, ], @@ -1672,6 +1676,18 @@ def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_ assert [m.get("role") for m in request["messages"]] == ["user", "assistant"] +def test_mistral_request_drops_a_user_name_without_touching_the_callers_message() -> None: + messages: Final = [{"role": "user", "content": "Hi"}, {"role": "user", "content": "", "name": "alice"}] + messages_before: Final = copy.deepcopy(messages) + + body: Final = MistralConfig().transform_request( + model="mistral-medium-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert "name" not in json.dumps(body["messages"]) + assert messages == messages_before + + def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: return tuple( r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 5b85a94b24d..79fe7061297 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -1824,6 +1824,94 @@ class TestToolSchemaCombinatorFlatteningForOpenAI: assert request["tools"][0] is tool +class TestCacheControlStrippingDoesNotMutateCallerInput: + """ + Stripping cache_control for a provider that cannot use it must not reach back + into the caller's own message and tool objects. + + filter_value_from_dict deletes the key in place and recurses into nested dicts + and lists, and remove_cache_control_flag_from_messages_and_tools assigned the + result back into the caller's list, so one call to any OpenAI-compatible + provider permanently stripped cache_control from a list the caller still holds. + Reusing that list on Anthropic or Bedrock afterwards then silently lost prompt + caching, with no error and full-price billing. + """ + + def setup_method(self): + self.config = OpenAIGPTConfig() + + @pytest.fixture(autouse=True) + def _clean_openai_base_env(self, monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None, raising=False) + + @staticmethod + def _messages(): + return [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "a long cached system prompt", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + { + "role": "user", + "content": "Hello", + "cache_control": {"type": "ephemeral"}, + }, + ] + + @staticmethod + def _tools(): + return [ + { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object"}}, + "cache_control": {"type": "ephemeral"}, + } + ] + + def _transform(self, messages, tools): + return self.config.transform_request( + model="gpt-4o", + messages=messages, + optional_params={"tools": tools}, + litellm_params={"custom_llm_provider": "openai", "api_base": None}, + headers={}, + ) + + def test_caller_messages_and_tools_keep_cache_control(self): + messages, tools = self._messages(), self._tools() + messages_before, tools_before = copy.deepcopy(messages), copy.deepcopy(tools) + + request = self._transform(messages, tools) + + assert "cache_control" not in json.dumps(request) + assert messages == messages_before + assert tools == tools_before + + def test_a_later_anthropic_call_still_sees_cache_control(self): + """The user-visible consequence: the same message list reused on a provider + that does support caching must still carry the cache breakpoints.""" + messages = self._messages() + + self._transform(messages, self._tools()) + + anthropic_body = litellm.AnthropicConfig().transform_request( + model="claude-3-5-sonnet-20240620", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + assert "cache_control" in json.dumps(anthropic_body) + + def _developer_message_after_a_user_turn() -> list[dict[str, str]]: return [ {"role": "system", "content": "You are terse."}, @@ -1916,3 +2004,53 @@ def test_completion_leaves_the_callers_closing_developer_message_unchanged_on_a_ ) assert json.loads(chat.calls.last.request.content)["messages"][-1]["role"] == "user" assert messages == sent + + +def _user_turn_with_litellm_only_media_fields() -> list[dict[str, object]]: + return [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe these"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png", "format": "image/png"}}, + {"type": "image_url", "image_url": "https://example.com/dog.png"}, + { + "type": "file", + "file": {"file_data": "data:application/pdf;base64,JVBERi0=", "format": "application/pdf"}, + }, + ], + } + ] + + +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +async def test_openai_request_rewrites_media_parts_without_touching_the_callers_message( + is_async: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None, raising=False) + config: Final = OpenAIGPTConfig() + messages: Final = _user_turn_with_litellm_only_media_fields() + messages_before: Final = copy.deepcopy(messages) + request_kwargs: Final = { + "model": "gpt-5.5", + "messages": messages, + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai", "api_base": None}, + "headers": {}, + } + + body: Final = ( + await config.async_transform_request(**request_kwargs) + if is_async + else config.transform_request(**request_kwargs) + ) + + assert body["messages"][0]["content"] == [ + {"type": "text", "text": "Describe these"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + {"type": "image_url", "image_url": {"url": "https://example.com/dog.png"}}, + {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0=", "filename": "my_file.pdf"}}, + ] + assert messages == messages_before diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index f5a33f87fd0..41c54bf6cc9 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -1,3 +1,4 @@ +import copy import json import logging from collections.abc import Callable, Mapping @@ -541,6 +542,50 @@ class TestOpenAIResponsesAPIConfig: assert "cache_control" not in result["tools"][0] assert result["tools"][0]["name"] == "get_weather" + def test_transform_does_not_strip_cache_control_from_the_callers_input(self): + """Stripping for OpenAI must not reach back into the caller's own objects. + + `filter_value_from_dict` deletes the key in place and recurses, and + `_validate_input_param` passes plain dict items through by reference, so + the caller's input list used to lose its cache breakpoints. Reusing that + list on Anthropic or Bedrock afterwards then silently lost prompt caching. + Same defect as the Chat Completions path. + """ + input_with_cache_control = [ + { + "role": "system", + "content": [ + { + "type": "input_text", + "text": "a long cached system prompt", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ] + tools_with_cache_control = [ + { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object"}, + "cache_control": {"type": "ephemeral"}, + } + ] + input_before = copy.deepcopy(input_with_cache_control) + tools_before = copy.deepcopy(tools_with_cache_control) + + result = self.config.transform_responses_api_request( + model=self.model, + input=input_with_cache_control, + response_api_optional_request_params={"tools": tools_with_cache_control}, + litellm_params={}, + headers={}, + ) + + assert "cache_control" not in json.dumps(result) + assert input_with_cache_control == input_before + assert tools_with_cache_control == tools_before + def test_transform_preserves_input_without_cache_control(self): """Inputs without cache_control must pass through unmodified.""" input_clean = [ @@ -4607,10 +4652,10 @@ def test_cache_control_still_marks_a_part_whose_own_marker_is_malformed_under_dr } -def _caller_input_after_transform(model: str, marker: object) -> str: +def _hinted_caller_input(marker: object) -> list[dict[str, object]]: hint: Final = {"type": "ephemeral"} marked: Final = {} if marker is None else {"prompt_cache_breakpoint": marker} - input: Final = [ + return [ { "type": "message", "role": "user", @@ -4626,6 +4671,10 @@ def _caller_input_after_transform(model: str, marker: object) -> str: "content": [{"type": "output_text", "text": "ok", "cache_control": hint, **marked}], }, ] + + +def _caller_input_after_transform(model: str, marker: object) -> str: + input: Final = _hinted_caller_input(marker) _CONFIGS[0].transform_responses_api_request( model=model, input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items @@ -4647,6 +4696,15 @@ def test_a_marker_leaves_the_caller_request_that_fallbacks_and_logs_read_as_an_u assert _caller_input_after_transform(model, marker).replace(marker_json, "") == unmarked +@pytest.mark.parametrize("model", ("gpt-4.1", "gpt-6.1-sol")) +@pytest.mark.parametrize("marker", (None, _VALID_MARKERS[0], "yes"), ids=("unmarked", "explicit", "malformed")) +def test_the_caller_input_keeps_its_cache_control_hints_for_a_fallback_deployment( + monkeypatch: pytest.MonkeyPatch, model: str, marker: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + assert _caller_input_after_transform(model, marker) == json.dumps(_hinted_caller_input(marker)) + + @pytest.mark.asyncio @pytest.mark.usefixtures(httpx_transport.__name__) async def test_aresponses_forwards_previous_response_id_to_openai() -> None: diff --git a/tests/unit/llms/openai/test_o_series_transformation.py b/tests/unit/llms/openai/test_o_series_transformation.py index e78e1a6f12b..9d39dfda60a 100644 --- a/tests/unit/llms/openai/test_o_series_transformation.py +++ b/tests/unit/llms/openai/test_o_series_transformation.py @@ -1,3 +1,4 @@ +import copy from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -198,3 +199,23 @@ def test_translate_developer_role_keeps_developer_messages_untouched() -> None: ) == messages ) + + +def test_o_series_request_turns_system_into_user_without_touching_the_callers_messages( + monkeypatch: pytest.MonkeyPatch, +) -> None: + model: Final = "o-series-test-model-without-system-role" + monkeypatch.setitem( + litellm.model_cost, + model, + {"litellm_provider": "openai", "mode": "chat", "supports_system_messages": False}, + ) + messages: Final = [{"role": "system", "content": "Be terse"}, {"role": "user", "content": "Hi"}] + messages_before: Final = copy.deepcopy(messages) + + body: Final = OpenAIOSeriesConfig().transform_request( + model=model, messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [message["role"] for message in body["messages"]] == ["user", "user"] + assert messages == messages_before diff --git a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py index 3d659641f54..6cef65f8a88 100644 --- a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -1,5 +1,8 @@ +import copy import json +from collections.abc import Iterator +from itertools import chain from typing import Final from unittest.mock import MagicMock @@ -874,6 +877,58 @@ def test_openrouter_streaming_reasoning_is_preserved() -> None: assert assembled.choices[0].message.reasoning_content == "Consider the evidence. Apply the rule." +def _five_message_level_cache_hints() -> list[dict[str, object]]: + return [ + {"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}}, + {"role": "assistant", "content": "answer 1", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}}, + {"role": "assistant", "content": "answer 2"}, + {"role": "user", "content": "turn 3", "cache_control": {"type": "ephemeral"}}, + ] + + +def _cached_texts(node: object) -> Iterator[str]: + if isinstance(node, dict): + yield from ((node["text"],) if "cache_control" in node and "text" in node else ()) + yield from chain.from_iterable(map(_cached_texts, node.values())) + elif isinstance(node, list): + yield from chain.from_iterable(map(_cached_texts, node)) + + +@pytest.mark.parametrize( + ("drop_params", "expected_cached_texts"), + [ + (True, ["answer 1", "turn 1", "turn 2", "turn 3"]), + ("true", ["answer 1", "turn 1", "turn 2", "turn 3"]), + (False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]), + ], + ids=[ + "drop_params_keeps_the_newest_four", + "string_drop_params_keeps_the_newest_four", + "without_drop_params_all_five_pass_through", + ], +) +def test_openrouter_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params( + drop_params: bool | str, expected_cached_texts: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = _five_message_level_cache_hints() + messages_before: Final = copy.deepcopy(messages) + + body: Final = OpenrouterConfig().transform_request( + model="anthropic/claude-haiku-5.5", + messages=messages, + optional_params={}, + litellm_params={"drop_params": drop_params, "custom_llm_provider": "openrouter"}, + headers={}, + ) + + assert sorted(_cached_texts(body["messages"])) == expected_cached_texts + assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts) + assert messages == messages_before + + def _transform_openrouter_response(body: dict[str, object]) -> litellm.ModelResponse: return OpenrouterConfig().transform_response( model="openrouter/openai/gpt-4o-mini", diff --git a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py index 78b9e930001..bc8421c80f8 100644 --- a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py +++ b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py @@ -1,3 +1,4 @@ +import json from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -2507,3 +2508,46 @@ def test_check_and_create_cache_clamps_negative_creation_tokens() -> None: assert cache_name == "cachedContents/created" assert logging_obj.vertex_ai_cache_creation_input_tokens == 0 + + +def _recording_gemini_client(calls: list[httpx.Request]) -> HTTPHandler: + def route(request: httpx.Request) -> httpx.Response: + calls.append(request) + if request.url.path.endswith("/cachedContents"): + return httpx.Response(200, json={} if request.method == "GET" else _created_cache_body(3000)) + return httpx.Response( + 200, + json={ + "candidates": [{"content": {"role": "model", "parts": [{"text": "OK"}]}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 3000, "candidatesTokenCount": 1, "totalTokenCount": 3001}, + }, + ) + + return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(route))) + + +def test_a_plain_system_prompt_goes_into_the_gemini_cache_instead_of_beside_it(local_model_cost_map) -> None: + calls: Final[list[httpx.Request]] = [] + litellm.completion( + model="gemini/gemini-flash-latest", + messages=[ + {"role": "system", "content": "You are terse."}, + { + "role": "user", + "content": [ + {"type": "text", "text": "plain system prefix " * 3000, "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "Reply with OK only."}, + ], + }, + ], + api_key="test-key", + client=_recording_gemini_client(calls), + ) + + created: Final = next( + json.loads(call.content) for call in calls if call.method == "POST" and call.url.path.endswith("/cachedContents") + ) + generated: Final = next(json.loads(call.content) for call in calls if call.url.path.endswith(":generateContent")) + assert created["system_instruction"] == {"parts": [{"text": "You are terse."}]} + assert generated["cachedContent"] == "cachedContents/created" + assert "system_instruction" not in generated diff --git a/tests/unit/llms/watsonx/test_watsonx.py b/tests/unit/llms/watsonx/test_watsonx.py index 4afa717debe..31d2875d2b9 100644 --- a/tests/unit/llms/watsonx/test_watsonx.py +++ b/tests/unit/llms/watsonx/test_watsonx.py @@ -1,4 +1,5 @@ import asyncio +import copy import importlib import json from typing import Final, Optional, cast @@ -10,6 +11,7 @@ import pytest import litellm from litellm import completion, embedding from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.watsonx.chat.transformation import IBMWatsonXChatConfig from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -542,3 +544,43 @@ def test_watsonx_deployment_space_id_embedding(monkeypatch, watsonx_embedding_ca # nor space_id or project_id is required by wx.ai API when inferencing deployment assert "project_id" not in json_data and "space_id" not in json_data + + +def _strict_weather_tool() -> dict[str, object]: + return { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, + }, + }, + } + + +def test_watsonx_chat_tools_are_rewritten_without_touching_the_callers_tool() -> None: + tools: Final = [_strict_weather_tool()] + tools_before: Final = copy.deepcopy(tools) + + optional_params: Final = IBMWatsonXChatConfig().map_openai_params( + non_default_params={"tools": tools}, optional_params={}, model="ibm/granite-4-h-small", drop_params=False + ) + + assert optional_params["tools"] == [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + assert tools == tools_before diff --git a/tests/unit/llms/xai/test_xai_chat_transformation.py b/tests/unit/llms/xai/test_xai_chat_transformation.py index 69e620dbc8a..37086f797fa 100644 --- a/tests/unit/llms/xai/test_xai_chat_transformation.py +++ b/tests/unit/llms/xai/test_xai_chat_transformation.py @@ -1,3 +1,4 @@ +import copy import json import os from typing import Final @@ -581,3 +582,44 @@ def test_xai_grok_4_frequency_penalty_not_supported(model): """ supported_params = XAIChatConfig().get_supported_openai_params(model=model) assert "frequency_penalty" not in supported_params + + +def _strict_weather_tool() -> dict[str, object]: + return { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, + }, + }, + } + + +def test_xai_tools_are_rewritten_without_touching_the_callers_tool() -> None: + tools: Final = [_strict_weather_tool()] + tools_before: Final = copy.deepcopy(tools) + + optional_params: Final = XAIChatConfig().map_openai_params( + non_default_params={"tools": tools}, optional_params={}, model="grok-4.3", drop_params=False + ) + + assert optional_params["tools"] == [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, + }, + }, + } + ] + assert tools == tools_before diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 4250c78c669..b1047a345d7 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -6314,6 +6314,17 @@ def test_marks_past_the_provider_cap_keep_the_newest_under_drop_params(drop_para assert messages[1] == {"role": "user", "content": "turn 0"} +def test_marks_past_the_provider_cap_inside_one_message_keep_the_newest_under_drop_params(): + parts: Final = tuple( + {"type": "input_text", "text": f"part {index}", "cache_control": _EPHEMERAL} for index in range(1, 6) + ) + + messages: Final = _bridged_request({"role": "user", "content": list(parts)}, drop_params=True) + + content: Final = TypeAdapter(list[dict[str, object]]).validate_python(messages[-1]["content"]) + assert ["cache_control" in part for part in content] == [False, True, True, True, True] + + def test_global_drop_params_keeps_the_newest_marks(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(litellm, "drop_params", True) From 57090e94a4050dc684155a635d8a437020ded774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Algirdas=20=C4=8C=2E?= Date: Sat, 31 Oct 2026 09:38:45 +0200 Subject: [PATCH 12/15] feat!: add Hostinger Router as an OpenAI-compatible provider (#44863) * feat: add Hostinger Router as an OpenAI-compatible provider Adds hostinger_router (https://router.hostinger.com/v1) to the JSON-configured openai_like providers with chat completions, responses, messages, and embeddings endpoints, cost-map entries for claude-sonnet-5, gpt-5.2 and the two text-embedding-3 models (credits converted at 1 credit = $0.01, sourced from the public https://router.hostinger.com/v1/models catalog), the endpoint-support matrix, Admin UI add-model form fields and logo, and respx-mocked unit tests covering all declared endpoints. Also routes /v1/embeddings for JSON-configured providers that declare it in providers.json; previously embedding calls for these providers fell through to LiteLLMUnknownProvider. * fix(hostinger_router): match prices to the live catalog, drop retired max_tokens, regenerate the endpoint matrix * fix(hostinger_router): keep api_base detection off, send embeddings headers as HTTP headers, drop generic key fallback Unprefixed models called with Hostinger's api_base keep their own provider, JSON provider embeddings no longer borrow OPENAI_LIKE_API_KEY or litellm.api_key, and the openai_like embedding handler sends extra_headers as request headers instead of in the JSON body * fix(openai_like): route Hostinger messages and responses per model, keep the configured key header Native /v1/messages and /v1/responses calls for a JSON provider now also follow the model's cost-map supported_endpoints, so Hostinger models without the route go through chat completions instead of a 403. Embedding key headers replace extra_headers case-insensitively, and pinstripes stops declaring /v1/embeddings. * fix(openai_like): let embeddings extra_headers override the key's headers as chat does Embeddings now match chat through the OpenAI SDK, so an extra_headers entry replaces the key's header of the same name in any letter case. The Hostinger capability test checks that the supports_* lookups agree with each cost-map row instead of pinning vendor flags * fix(openai_like): send numeric embeddings extra_headers as text and keep Hostinger's 1h cache TTL A config.yaml header like X-Org-Id: 12345 loads as an int, which the embeddings header check rejected with a 500. Int and float values now go out as their decimal text. Hostinger Router accepts and bills a 1h cache_control TTL on /v1/messages, so its provider entry opts into keeping it. * fix(openai_like): send Hostinger's 1h cache TTL as the 5-minute default so cache writes bill at its charge * fix(openai_like): route Hostinger natives per model from providers.json, refuse non-text embeddings headers unless drop_params * fix(openai_like): refuse Host, Content-Length and Transfer-Encoding in embeddings extra_headers unless drop_params * fix(openai_like): refuse malformed embeddings extra_headers and ignore transport headers Header names must be HTTP tokens and values printable ASCII on one line, so a colon in a name or a line break in a value answers 400 unless drop_params drops it. Host, Content-Length and Transfer-Encoding are ignored, since a 400 for them sends the call to the deployment's fallback with the same headers. * fix(openai_like): trim embeddings extra_headers values and drop values HTTP cannot carry HTTP ignores spaces and tabs around a header value, so values are trimmed before they are sent. A value that is not printable ASCII on one line after trimming is dropped with a warning instead of failing the call, since the proxy adds headers itself from key aliases and forwarded client headers that the caller cannot fix. A malformed header name or a non-text value still answers 400 unless drop_params drops it. * fix(openai_like): ignore hop-by-hop extra_headers and trim line breaks around values TE, Upgrade, Connection, Keep-Alive, Proxy-Connection and Trailer in extra_headers are now ignored like Host, Content-Length and Transfer-Encoding, since litellm builds the request and TE: gzip fails HTTP/2 while Upgrade makes some upstreams hang. Line breaks at the ends of a value are trimmed along with spaces and tabs, so a secret read from a file with a trailing newline is sent instead of dropped. The drop warning now names the model. * fix(embeddings): keep one request's extra_headers out of the deployment's headers and ignore Expect embedding() merged a request's extra_headers into the headers dict it got from the deployment's litellm_params. The router only shallow-copies those params, so the merge wrote the caller's headers into the deployment itself, and later embeddings calls through that deployment sent them too. Build a new dict instead, and drop the None checks that can no longer be true. Also treat Expect as a transport header that hostinger_router embeddings never forwards, and name the model with %r in the dropped-value warning. * fix(embeddings): keep name and value pairs working in headers and answer 400 for other non-object values The previous commit merged headers and extra_headers with dict unpacking, which only takes a mapping, so a list of name and value pairs that dict.update used to accept now raised a TypeError and the proxy answered 500. Merge through dict() so pairs keep working, and answer 400 for any other value, or drop it under drop_params, as the litellm convention does for an unsupported param. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/llms/databricks/embed/handler.py | 2 + litellm/llms/openai_like/README.md | 14 + litellm/llms/openai_like/embedding/handler.py | 77 +- litellm/llms/openai_like/json_loader.py | 25 +- litellm/llms/openai_like/providers.json | 12 +- litellm/main.py | 43 +- ...odel_prices_and_context_window_backup.json | 51 ++ .../provider_endpoints_support_backup.json | 26 + .../provider_create_fields.json | 28 + litellm/types/utils.py | 1 + litellm/utils.py | 4 +- model_prices_and_context_window.json | 51 ++ provider_endpoints_support.json | 26 + .../test_hostinger_router_provider.py | 706 ++++++++++++++++++ .../public/assets/logos/hostinger.png | Bin 0 -> 16604 bytes .../src/components/provider_info_helpers.tsx | 5 + 17 files changed, 1047 insertions(+), 25 deletions(-) create mode 100644 tests/unit/llms/openai_like/test_hostinger_router_provider.py create mode 100644 ui/litellm-dashboard/public/assets/logos/hostinger.png diff --git a/litellm/constants.py b/litellm/constants.py index 28e7b4a7fd7..7d99c28deb3 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1165,6 +1165,7 @@ openai_compatible_providers: Final[list[str]] = [ "cortecs", "scx-ai", "prism", + "hostinger_router", "sail", "reka", "streamlake", diff --git a/litellm/llms/databricks/embed/handler.py b/litellm/llms/databricks/embed/handler.py index 8b2fa768968..05807a118c3 100644 --- a/litellm/llms/databricks/embed/handler.py +++ b/litellm/llms/databricks/embed/handler.py @@ -26,6 +26,7 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase): aembedding=None, custom_endpoint: bool | None = None, headers: dict | None = None, + drop_params: bool = False, ) -> EmbeddingResponse: # Check for custom user agent in optional_params or environment # This allows partners building on LiteLLM to set their own telemetry @@ -58,4 +59,5 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase): aembedding=aembedding, custom_endpoint=True, headers=headers, + drop_params=drop_params, ) diff --git a/litellm/llms/openai_like/README.md b/litellm/llms/openai_like/README.md index c56f58b24c9..fb00b07c986 100644 --- a/litellm/llms/openai_like/README.md +++ b/litellm/llms/openai_like/README.md @@ -125,6 +125,20 @@ The provider inherits all request/response handling from OpenAI's Responses API If `supported_endpoints` is omitted, it defaults to `[]` (only chat completions, which is always enabled for JSON providers). +When only some models serve an endpoint natively, list them under `model_supported_endpoints`. Every other model reaches `/v1/responses` and `/v1/messages` through chat completions: + +```json +{ + "your_provider": { + "supported_endpoints": ["/v1/chat/completions"], + "model_supported_endpoints": { + "model-a": ["/v1/messages"], + "model-b": ["/v1/responses"] + } + } +} +``` + ### How It Works 1. `json_loader.py` checks `supported_endpoints` for `/v1/responses` diff --git a/litellm/llms/openai_like/embedding/handler.py b/litellm/llms/openai_like/embedding/handler.py index 77b6b673707..540feeca6e7 100644 --- a/litellm/llms/openai_like/embedding/handler.py +++ b/litellm/llms/openai_like/embedding/handler.py @@ -3,11 +3,16 @@ ## Allows jina ai embedding calls - which don't allow 'encoding_format' in payload. import json +import re +from collections.abc import Mapping +from types import MappingProxyType from typing import Final import httpx +from pydantic import TypeAdapter import litellm +from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -17,6 +22,57 @@ from litellm.types.utils import EmbeddingResponse from ..common_utils import OpenAILikeBase, OpenAILikeError +_KEY_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str]) +_EXTRA_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_TRANSPORT_HEADER_NAMES: Final = frozenset( + { + "host", + "content-length", + "transfer-encoding", + "connection", + "keep-alive", + "proxy-connection", + "te", + "trailer", + "upgrade", + "expect", + } +) +_HEADER_NAME: Final = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+") +_HEADER_VALUE: Final = re.compile(r"[\t\x20-\x7e]*") + + +def _header_text(name: str, value: object) -> str | None: + if isinstance(value, bool) or not isinstance(value, str | int | float): + return None + if _HEADER_NAME.fullmatch(name) is None: + return None + return str(value).strip(" \t\r\n") + + +def _extra_headers_as_text(extra_headers: Mapping[str, object], model: str, drop_params: bool) -> Mapping[str, str]: + caller_set: Final = { + name: value for name, value in extra_headers.items() if name.lower() not in _TRANSPORT_HEADER_NAMES + } + texts: Final = MappingProxyType({name: _header_text(name, value) for name, value in caller_set.items()}) + refused: Final = sorted(name for name, text in texts.items() if text is None) + if refused and not drop_params: + raise litellm.UnsupportedParamsError( + message=f"extra_headers {', '.join(map(repr, refused))} cannot be sent. A header needs a name made of " + "letters, digits or !#$%&'*+-.^_`|~ and a text or number value. " + "To drop them from the call, set `litellm.drop_params = True`.", + model=model, + ) + named: Final = MappingProxyType({name: text for name, text in texts.items() if text is not None}) + unsendable: Final = sorted(name for name, text in named.items() if _HEADER_VALUE.fullmatch(text) is None) + if unsendable: + verbose_logger.warning( + "extra_headers %s dropped for model %r, since a header value must be printable ASCII on one line", + unsendable, + model, + ) + return MappingProxyType({name: text for name, text in named.items() if name not in unsendable}) + class OpenAILikeEmbeddingHandler(OpenAILikeBase): def __init__(self, **kwargs): @@ -94,6 +150,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase): aembedding=None, custom_endpoint: bool | None = None, headers: dict | None = None, + drop_params: bool = False, ) -> EmbeddingResponse: api_base, headers = self._validate_environment( api_base=api_base, @@ -103,8 +160,20 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase): custom_endpoint=custom_endpoint, ) model = model - filtered_optional_params: Final = {k: v for k, v in optional_params.items() if v not in (None, "")} - data: Final = {"model": model, "input": input, **filtered_optional_params} + data: Final = { + "model": model, + "input": input, + **{k: v for k, v in optional_params.items() if k != "extra_headers" and v not in (None, "")}, + } + extra_headers: Final = _extra_headers_as_text( + _EXTRA_HEADERS_ADAPTER.validate_python(optional_params.get("extra_headers") or {}), model, drop_params + ) + key_headers: Final = _KEY_HEADERS_ADAPTER.validate_python(headers) + extra_header_names: Final = frozenset(name.lower() for name in extra_headers) + request_headers: Final = { + **{name: value for name, value in key_headers.items() if name.lower() not in extra_header_names}, + **extra_headers, + } ## LOGGING logging_obj.pre_call( @@ -123,7 +192,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase): api_key=api_key, timeout=timeout, client=client, - headers=headers, + headers=request_headers, ) if client is None or isinstance(client, AsyncHTTPHandler): self.client = HTTPHandler(timeout=timeout) @@ -134,7 +203,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase): try: response: Final = self.client.post( api_base, - headers=headers, + headers=request_headers, data=json.dumps(data), ) diff --git a/litellm/llms/openai_like/json_loader.py b/litellm/llms/openai_like/json_loader.py index 6e99b9e7fec..658e4176f22 100644 --- a/litellm/llms/openai_like/json_loader.py +++ b/litellm/llms/openai_like/json_loader.py @@ -3,7 +3,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers. """ import json -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from pathlib import Path from typing import Final @@ -14,6 +14,7 @@ from litellm.types.utils import ProviderSpecificModelInfo _PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo) _SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool]) +_MODEL_ENDPOINTS_ADAPTER: Final = TypeAdapter(Mapping[str, tuple[str, ...]]) class SimpleProviderConfig: @@ -30,7 +31,10 @@ class SimpleProviderConfig: self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python( data.get("special_handling", {}) ) - self.supported_endpoints = data.get("supported_endpoints", []) + self.supported_endpoints: Sequence[str] = data.get("supported_endpoints", []) + self.model_supported_endpoints: Mapping[str, tuple[str, ...]] = _MODEL_ENDPOINTS_ADAPTER.validate_python( + data.get("model_supported_endpoints", {}) + ) self.provider_info: ProviderSpecificModelInfo = ( _PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {} ) @@ -90,6 +94,23 @@ class JSONProviderRegistry: return False return "/v1/responses" in provider.supported_endpoints + @classmethod + def supports_embeddings(cls, slug: str) -> bool: + provider: Final = cls._providers.get(slug) + if provider is None: + return False + return "/v1/embeddings" in provider.supported_endpoints + + @classmethod + def serves_endpoint(cls, slug: str, model: str | None, endpoint: str) -> bool: + provider: Final = cls._providers.get(slug) + if provider is None: + return False + model_endpoints: Final = ( + provider.model_supported_endpoints.get(model.removeprefix(f"{slug}/"), ()) if model else () + ) + return endpoint in provider.supported_endpoints or endpoint in model_endpoints + @classmethod def list_providers(cls) -> list: """List all registered provider slugs""" diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index ad47b334ed4..a71a9c3bb9b 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -211,7 +211,7 @@ "param_mappings": { "max_completion_tokens": "max_tokens" }, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"] + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"] }, "scx-ai": { "base_url": "https://api.scx.ai/v1", @@ -231,6 +231,16 @@ "api_base_env": "PRISM_API_BASE", "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"] }, + "hostinger_router": { + "base_url": "https://router.hostinger.com/v1", + "api_key_env": "HOSTINGER_ROUTER_API_KEY", + "api_base_env": "HOSTINGER_ROUTER_API_BASE", + "supported_endpoints": ["/v1/chat/completions", "/v1/embeddings"], + "model_supported_endpoints": { + "claude-sonnet-5": ["/v1/messages"], + "gpt-5.2": ["/v1/responses"] + } + }, "sail": { "base_url": "https://api.sailresearch.com/v1", "api_key_env": "SAIL_API_KEY", diff --git a/litellm/main.py b/litellm/main.py index 2320d1a3e01..21ae9657d68 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6809,6 +6809,23 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse: ) +def _embedding_headers(param: str, configured: object, model: str, drop_params: bool) -> Mapping[str, object]: + if not configured: + return {} + if isinstance(configured, Mapping): + return dict(cast(Mapping[str, object], configured)) # cast-ok: header names from untyped request JSON + try: + return dict(cast(Iterable[tuple[str, object]], configured)) # cast-ok: name and value pairs from request JSON + except (TypeError, ValueError): + if drop_params: + return {} + raise litellm.UnsupportedParamsError( + message=f"{param} must be a JSON object of header names to values. " + "To drop it from the call, set `litellm.drop_params = True`.", + model=model, + ) + + # fmt: off # Overload for when aembedding=True (returns coroutine) @@ -6923,11 +6940,10 @@ def embedding( azure_ad_token_provider: Final = kwargs.get("azure_ad_token_provider", None) aembedding: Final[bool | None] = kwargs.get("aembedding", None) extra_headers: Final = kwargs.get("extra_headers", None) - headers = kwargs.get("headers", None) or extra_headers - if headers is None: - headers = {} - if extra_headers is not None: - headers.update(extra_headers) + headers = { + **_embedding_headers("headers", kwargs.get("headers"), model, drop_unsupported_params), + **_embedding_headers("extra_headers", extra_headers, model, drop_unsupported_params), + } # Inject proxy auth headers if configured if litellm.proxy_auth is not None: try: @@ -7091,7 +7107,7 @@ def embedding( # set API KEY api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") - if headers is not None and headers != {}: + if headers: optional_params["extra_headers"] = headers requested_encoding_format: Final = ( @@ -7183,14 +7199,15 @@ def embedding( custom_llm_provider == "openai_like" or custom_llm_provider == "llamafile" or custom_llm_provider == "lm_studio" + or JSONProviderRegistry.supports_embeddings(custom_llm_provider) ): api_base = api_base or litellm.api_base or get_secret_str("OPENAI_LIKE_API_BASE") # set API KEY - if api_key is None: - api_key = api_key or litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY") + if api_key is None and not JSONProviderRegistry.supports_embeddings(custom_llm_provider): + api_key = litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY") - if headers is not None and headers != {}: + if headers: optional_params["extra_headers"] = headers ## EMBEDDING CALL @@ -7205,10 +7222,9 @@ def embedding( optional_params=optional_params, client=client, aembedding=aembedding, + drop_params=drop_unsupported_params, ) elif custom_llm_provider == "oci": - if headers is None: - headers = {} response = base_llm_http_handler.embedding( model=model, input=input, @@ -7233,11 +7249,6 @@ def embedding( or litellm.api_key ) - # Use the merged headers variable (already merged at the top of the function) - # Don't overwrite it with just extra_headers - if headers is None: - headers = {} - response = base_llm_http_handler.embedding( model=model, input=input, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cbe502e5cae..7eede4a395f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -39687,6 +39687,57 @@ "/v1/images/generations" ] }, + "hostinger_router/gpt-5.2": { + "cache_read_input_token_cost": 0.00000021, + "input_cost_per_token": 0.0000021, + "litellm_provider": "hostinger_router", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.0000168, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "source": "https://router.hostinger.com/v1/models" + }, + "hostinger_router/claude-sonnet-5": { + "cache_creation_input_token_cost": 3.3e-06, + "cache_creation_input_token_cost_above_1hr": 5.28e-06, + "cache_read_input_token_cost": 2.64e-07, + "input_cost_per_token": 2.64e-06, + "litellm_provider": "hostinger_router", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.32e-05, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_response_schema": true, + "supported_endpoints": ["/v1/chat/completions", "/v1/messages"], + "source": "https://router.hostinger.com/v1/models" + }, + "hostinger_router/text-embedding-3-large": { + "input_cost_per_token": 1.716e-07, + "litellm_provider": "hostinger_router", + "max_input_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0, + "source": "https://router.hostinger.com/v1/models" + }, + "hostinger_router/text-embedding-3-small": { + "input_cost_per_token": 3e-08, + "litellm_provider": "hostinger_router", + "max_input_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0, + "source": "https://router.hostinger.com/v1/models" + }, "hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": { "input_cost_per_token": 1.2e-07, "litellm_provider": "hyperbolic", diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index d33e61026df..700a35edb06 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -1087,6 +1087,32 @@ "generateContent": false } }, + "hostinger_router": { + "display_name": "Hostinger Router (`hostinger_router`)", + "url": "https://docs.litellm.ai/docs/providers/hostinger_router", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": true, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "crusoe": { "display_name": "Crusoe (`crusoe`)", "url": "https://docs.litellm.ai/docs/providers/crusoe", diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 01a822c5d93..044b8a1de6e 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -3343,6 +3343,34 @@ ], "default_model_placeholder": "prism/deepseek-v4.1-flash" }, + { + "provider": "HOSTINGER_ROUTER", + "provider_display_name": "Hostinger Router", + "litellm_provider": "hostinger_router", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://router.hostinger.com/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "hostinger_router/claude-sonnet-5" + }, { "provider": "RECRAFT", "provider_display_name": "Recraft", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 2f6918d0744..24f4acaf5df 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4683,6 +4683,7 @@ class LlmProviders(str, Enum): PRISM = "prism" OPENINFER = "openinfer" INFERENCEAPIS = "inferenceapis" + HOSTINGER_ROUTER = "hostinger_router" ZEROGPU = "zerogpu" DARKBLOOM = "darkbloom" META = "meta" diff --git a/litellm/utils.py b/litellm/utils.py index 8fe3fd84fbd..f19fa0dbb34 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10159,7 +10159,7 @@ class ProviderConfigManager: from litellm.llms.openai_like.json_loader import JSONProviderRegistry json_provider: Final = JSONProviderRegistry.get(provider.value) - if json_provider is not None and "/v1/messages" in json_provider.supported_endpoints: + if json_provider is not None and JSONProviderRegistry.serves_endpoint(provider.value, model, "/v1/messages"): from litellm.llms.openai_like.messages.transformation import ( JSONProviderAnthropicMessagesConfig, ) @@ -10298,7 +10298,7 @@ class ProviderConfigManager: return result # Fall back to JSON providers (generic OpenAI-compatible) - if JSONProviderRegistry.exists(provider_str) and JSONProviderRegistry.supports_responses_api(provider_str): + if JSONProviderRegistry.serves_endpoint(provider_str, model, "/v1/responses"): provider_config: Final = JSONProviderRegistry.get(provider_str) if provider_config is not None: return create_responses_config_class(provider_config)() diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cbe502e5cae..7eede4a395f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -39687,6 +39687,57 @@ "/v1/images/generations" ] }, + "hostinger_router/gpt-5.2": { + "cache_read_input_token_cost": 0.00000021, + "input_cost_per_token": 0.0000021, + "litellm_provider": "hostinger_router", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.0000168, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "source": "https://router.hostinger.com/v1/models" + }, + "hostinger_router/claude-sonnet-5": { + "cache_creation_input_token_cost": 3.3e-06, + "cache_creation_input_token_cost_above_1hr": 5.28e-06, + "cache_read_input_token_cost": 2.64e-07, + "input_cost_per_token": 2.64e-06, + "litellm_provider": "hostinger_router", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.32e-05, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_response_schema": true, + "supported_endpoints": ["/v1/chat/completions", "/v1/messages"], + "source": "https://router.hostinger.com/v1/models" + }, + "hostinger_router/text-embedding-3-large": { + "input_cost_per_token": 1.716e-07, + "litellm_provider": "hostinger_router", + "max_input_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0, + "source": "https://router.hostinger.com/v1/models" + }, + "hostinger_router/text-embedding-3-small": { + "input_cost_per_token": 3e-08, + "litellm_provider": "hostinger_router", + "max_input_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0, + "source": "https://router.hostinger.com/v1/models" + }, "hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": { "input_cost_per_token": 1.2e-07, "litellm_provider": "hyperbolic", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index d33e61026df..700a35edb06 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1087,6 +1087,32 @@ "generateContent": false } }, + "hostinger_router": { + "display_name": "Hostinger Router (`hostinger_router`)", + "url": "https://docs.litellm.ai/docs/providers/hostinger_router", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": true, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "crusoe": { "display_name": "Crusoe (`crusoe`)", "url": "https://docs.litellm.ai/docs/providers/crusoe", diff --git a/tests/unit/llms/openai_like/test_hostinger_router_provider.py b/tests/unit/llms/openai_like/test_hostinger_router_provider.py new file mode 100644 index 00000000000..cbc16c8567a --- /dev/null +++ b/tests/unit/llms/openai_like/test_hostinger_router_provider.py @@ -0,0 +1,706 @@ +import copy +import json +from typing import Final + +import pytest +import respx + +import litellm +from litellm.caching.llm_caching_handler import LLMClientCache + + +def test_hostinger_router_provider_resolution(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + monkeypatch.setenv("HOSTINGER_ROUTER_API_KEY", "hostinger-test-key") + + model, provider, api_key, api_base = get_llm_provider( + model="hostinger_router/claude-sonnet-5", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "claude-sonnet-5" + assert provider == "hostinger_router" + assert api_key == "hostinger-test-key" + assert api_base == "https://router.hostinger.com/v1" + + +def test_hostinger_router_provider_keeps_explicit_credentials(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + monkeypatch.setenv("HOSTINGER_ROUTER_API_KEY", "hostinger-env-key") + + _, provider, api_key, api_base = get_llm_provider( + model="hostinger_router/claude-sonnet-5", + custom_llm_provider=None, + api_base="https://router.internal.example/v1", + api_key="hostinger-explicit-key", + ) + + assert provider == "hostinger_router" + assert api_key == "hostinger-explicit-key" + assert api_base == "https://router.internal.example/v1" + + +@pytest.mark.parametrize("model", ["gpt-5-nano", "claude-sonnet-5"]) +def test_hostinger_router_api_base_keeps_unprefixed_model_provider(model: str): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + _, provider_without_api_base, _, _ = get_llm_provider(model=model) + _, provider_with_hostinger_api_base, _, _ = get_llm_provider( + model=model, + api_base="https://router.hostinger.com/v1", + api_key="hostinger-test-key", + ) + + assert provider_with_hostinger_api_base == provider_without_api_base + + +HOSTINGER_ROUTER_MODELS = tuple(sorted(name for name in litellm.model_cost if name.startswith("hostinger_router/"))) +HOSTINGER_ROUTER_CHAT_MODELS = tuple( + name for name in HOSTINGER_ROUTER_MODELS if litellm.get_model_info(name)["mode"] == "chat" +) + + +@pytest.mark.parametrize("model", HOSTINGER_ROUTER_CHAT_MODELS) +def test_hostinger_router_model_cost_and_capabilities(model: str): + from litellm.cost_calculator import cost_per_token + + prompt_cost, completion_cost = cost_per_token( + model=model, + prompt_tokens=1_000_000, + completion_tokens=1_000_000, + custom_llm_provider="hostinger_router", + ) + model_info = litellm.get_model_info(model) + + assert prompt_cost == pytest.approx(model_info["input_cost_per_token"] * 1_000_000) + assert completion_cost == pytest.approx(model_info["output_cost_per_token"] * 1_000_000) + assert model_info["output_cost_per_token"] > 0 + assert model_info["max_tokens"] == model_info["max_output_tokens"] <= model_info["max_input_tokens"] + assert model_info["litellm_provider"] == "hostinger_router" + assert model_info["mode"] == "chat" + if model_info.get("supports_prompt_caching"): + assert 0 < model_info["cache_read_input_token_cost"] < model_info["input_cost_per_token"] + capability_checks: Final = ( + ("supports_function_calling", litellm.supports_function_calling), + ("supports_reasoning", litellm.supports_reasoning), + ("supports_response_schema", litellm.supports_response_schema), + ("supports_vision", litellm.supports_vision), + ) + assert {name: check(model) for name, check in capability_checks} == { + name: model_info.get(name) is True for name, _ in capability_checks + } + + +def test_hostinger_router_chat_completions_request(): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond( + 200, + json={ + "id": "chatcmpl_hostinger", + "object": "chat.completion", + "created": 1_789_550_000, + "model": "claude-sonnet-5", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello from Hostinger Router"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}, + }, + ) + response: Final = litellm.completion( + model="hostinger_router/claude-sonnet-5", + messages=[{"role": "user", "content": "Say hello"}], + api_key="hostinger-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert request.headers["authorization"] == "Bearer hostinger-test-key" + assert body["model"] == "claude-sonnet-5" + assert response.choices[0].message.content == "Hello from Hostinger Router" + + +def test_hostinger_router_chat_completions_streaming_request(): + chunks = [ + 'data: {"id":"chatcmpl_hostinger","object":"chat.completion.chunk","created":1789550000,"model":"claude-sonnet-5","choices":[{"index":0,"delta":{"role":"assistant","content":"Hi"},"finish_reason":null}]}\n\n', + 'data: {"id":"chatcmpl_hostinger","object":"chat.completion.chunk","created":1789550000,"model":"claude-sonnet-5","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":4,"completion_tokens":1,"total_tokens":5}}\n\n', + "data: [DONE]\n\n", + ] + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond( + 200, + headers={"content-type": "text/event-stream"}, + content="".join(chunks).encode(), + ) + response: Final = litellm.completion( + model="hostinger_router/claude-sonnet-5", + messages=[{"role": "user", "content": "Say hello"}], + stream=True, + api_key="hostinger-test-key", + ) + collected = [chunk for chunk in response] + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert body["stream"] is True + assert any(chunk.choices and chunk.choices[0].delta.content == "Hi" for chunk in collected) + + +def test_hostinger_router_responses_request(): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/responses").respond( + 200, + json={ + "id": "resp_hostinger", + "object": "response", + "created_at": 1_789_550_000, + "model": "gpt-5.2", + "status": "completed", + "output": [ + { + "id": "msg_hostinger", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Hello from Hostinger Router", "annotations": []}], + } + ], + "usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7}, + }, + ) + response: Final = litellm.responses( + model="hostinger_router/gpt-5.2", + input="Say hello", + api_key="hostinger-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert str(request.url) == "https://router.hostinger.com/v1/responses" + assert request.headers["authorization"] == "Bearer hostinger-test-key" + assert body["model"] == "gpt-5.2" + assert body["input"] == "Say hello" + assert response.output[0].content[0].text == "Hello from Hostinger Router" + + +EMBEDDING_RESPONSE: Final = { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 2, "total_tokens": 2}, +} + + +def test_hostinger_router_embeddings_request(): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + response: Final = litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert request.headers["authorization"] == "Bearer hostinger-test-key" + assert body["model"] == "text-embedding-3-small" + assert response.data[0]["embedding"] == [0.1, 0.2, 0.3] + + +def test_hostinger_router_embeddings_send_extra_headers_as_http_headers(): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={"X-Trace-Id": "trace-123"}, + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert request.headers["x-trace-id"] == "trace-123" + assert request.headers["authorization"] == "Bearer hostinger-test-key" + assert request.headers["content-type"] == "application/json" + assert set(body) == {"model", "input"} + + +def test_hostinger_router_embeddings_extra_headers_replace_key_headers_of_any_case(): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={"authorization": "Bearer caller-key", "x-trace-id": "trace-1"}, + ) + + request: Final = route.calls.last.request + assert request.headers.get_list("authorization") == ["Bearer caller-key"] + assert request.headers.get_list("content-type") == ["application/json"] + assert request.headers.get_list("x-trace-id") == ["trace-1"] + + +@pytest.mark.parametrize(("configured", "sent"), [(12345, "12345"), (1.5, "1.5")]) +def test_hostinger_router_embeddings_send_numeric_extra_headers_as_text(configured: int | float, sent: str): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={"X-Org-Id": configured}, + ) + + assert route.calls.last.request.headers.get_list("x-org-id") == [sent] + + +@pytest.mark.parametrize("configured", [True, None, ["a", "b"]]) +def test_hostinger_router_embeddings_refuse_non_text_extra_headers(configured: object): + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + with pytest.raises(litellm.UnsupportedParamsError, match="X-Debug") as refused: + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={"X-Debug": configured}, + ) + + assert refused.value.status_code == 400 + assert route.call_count == 0 + + +@pytest.mark.parametrize("configured", [True, None, ["a", "b"]]) +def test_hostinger_router_embeddings_drop_non_text_extra_headers_under_drop_params(configured: object): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={"X-Debug": configured, "X-Trace-Id": "trace-1"}, + drop_params=True, + ) + + request: Final = route.calls.last.request + assert "x-debug" not in request.headers + assert request.headers.get_list("x-trace-id") == ["trace-1"] + assert set(json.loads(request.content)) == {"model", "input"} + + +TRANSPORT_EXTRA_HEADERS: Final = [ + ("Host", "other.example"), + ("content-length", "1"), + ("Transfer-Encoding", "chunked"), + ("Connection", "close"), + ("Keep-Alive", "timeout=5"), + ("Proxy-Connection", "close"), + ("TE", "gzip"), + ("Trailer", "X-Checksum"), + ("Upgrade", "websocket"), + ("Expect", "100-continue"), +] + + +@pytest.mark.parametrize(("name", "configured"), TRANSPORT_EXTRA_HEADERS) +def test_hostinger_router_embeddings_ignore_transport_extra_headers(name: str, configured: str): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={name: configured, "X-Trace-Id": "trace-1"}, + ) + + request: Final = route.calls.last.request + assert request.headers.get_list("host") == ["router.hostinger.com"] + assert request.headers.get_list("content-length") == [str(len(request.content))] + assert "transfer-encoding" not in request.headers + assert configured not in request.headers.get_list(name) + assert request.headers.get_list("x-trace-id") == ["trace-1"] + + +MALFORMED_HEADER_NAMES: Final = ["Host:evil.example", "X Note", "X-Caf\u00e9"] + + +@pytest.mark.parametrize("name", MALFORMED_HEADER_NAMES) +def test_hostinger_router_embeddings_refuse_malformed_extra_header_names(name: str): + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + with pytest.raises(litellm.UnsupportedParamsError) as refused: + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={name: "x"}, + ) + + assert refused.value.status_code == 400 + assert repr(name) in refused.value.message + assert route.call_count == 0 + + +@pytest.mark.parametrize("name", MALFORMED_HEADER_NAMES) +def test_hostinger_router_embeddings_drop_malformed_extra_header_names_under_drop_params(name: str): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={name: "x", "X-Trace-Id": "trace-1"}, + drop_params=True, + ) + + request: Final = route.calls.last.request + assert request.headers.get_list("host") == ["router.hostinger.com"] + assert request.headers.get_list("x-trace-id") == ["trace-1"] + assert name.lower() not in request.headers.keys() + + +@pytest.mark.parametrize("configured", ["a\r\nHost: evil.example", "line\nbreak", "caf\u00e9", "a\x7fb"]) +def test_hostinger_router_embeddings_drop_extra_header_values_http_cannot_carry(configured: str): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={"x-litellm-user_api_key_alias": configured, "X-Trace-Id": "trace-1"}, + ) + + request: Final = route.calls.last.request + assert request.headers.get_list("host") == ["router.hostinger.com"] + assert request.headers.get_list("x-trace-id") == ["trace-1"] + assert "x-litellm-user_api_key_alias" not in request.headers + + +@pytest.mark.parametrize( + ("configured", "sent"), + [(" x ", "x"), ("\t", ""), (" a\tb ", "a\tb"), ("env-secret\n", "env-secret"), ("\r\n x \r\n", "x")], +) +def test_hostinger_router_embeddings_trim_whitespace_around_extra_header_values(configured: str, sent: str): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={"X-Rig": configured}, + ) + + assert route.calls.last.request.headers.get_list("x-rig") == [sent] + + +def test_hostinger_router_embeddings_extra_headers_stay_with_their_request(): + router: Final = litellm.Router( + model_list=[ + { + "model_name": "shared", + "litellm_params": { + "model": "hostinger_router/text-embedding-3-small", + "api_key": "hostinger-test-key", + "headers": {"X-Org-Id": "org-1"}, + }, + } + ] + ) + + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + router.embedding( + model="shared", input=["first"], extra_headers={"Authorization": "Bearer caller-a", "X-Leak": "a"} + ) + router.embedding(model="shared", input=["second"]) + + first: Final = route.calls[0].request + second: Final = route.calls[1].request + assert first.headers.get_list("x-leak") == ["a"] + assert second.headers.get_list("authorization") == ["Bearer hostinger-test-key"] + assert "x-leak" not in second.headers + assert second.headers.get_list("x-org-id") == ["org-1"] + + +@pytest.mark.parametrize("param", ["headers", "extra_headers"]) +def test_hostinger_router_embeddings_send_headers_given_as_name_value_pairs(param: str): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + **{param: [["X-Trace-Id", "trace-1"]]}, + ) + + request: Final = route.calls.last.request + assert request.headers.get_list("x-trace-id") == ["trace-1"] + assert request.headers.get_list("authorization") == ["Bearer hostinger-test-key"] + + +@pytest.mark.parametrize("param", ["headers", "extra_headers"]) +def test_hostinger_router_embeddings_refuse_headers_that_are_not_an_object(param: str): + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + with pytest.raises(litellm.UnsupportedParamsError, match=rf"\b{param} must be a JSON object") as refused: + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + **{param: "X-Trace-Id: trace-1"}, + ) + + assert refused.value.status_code == 400 + assert route.call_count == 0 + + +@pytest.mark.parametrize("param", ["headers", "extra_headers"]) +def test_hostinger_router_embeddings_drop_headers_that_are_not_an_object_under_drop_params(param: str): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + drop_params=True, + **{param: "X-Trace-Id: trace-1"}, + ) + + request: Final = route.calls.last.request + assert "x-trace-id" not in request.headers + assert request.headers.get_list("authorization") == ["Bearer hostinger-test-key"] + assert set(json.loads(request.content)) == {"model", "input"} + + +def test_hostinger_router_embeddings_never_borrow_another_providers_key(monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("HOSTINGER_ROUTER_API_KEY", raising=False) + monkeypatch.setenv("OPENAI_LIKE_API_KEY", "other-provider-key") + monkeypatch.setenv("OPENAI_API_KEY", "other-provider-key") + + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + with pytest.raises(litellm.BadRequestError, match="Missing API Key"): + litellm.embedding(model="hostinger_router/text-embedding-3-small", input=["Say hello"]) + + assert route.call_count == 0 + + +def test_json_provider_without_embeddings_endpoint_refuses_embeddings(): + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + slug: Final = next( + slug + for slug in JSONProviderRegistry.list_providers() + if not JSONProviderRegistry.supports_embeddings(slug) and slug not in litellm.provider_list + ) + + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.route().respond(200, json=EMBEDDING_RESPONSE) + with pytest.raises(litellm.BadRequestError): + litellm.embedding(model=f"{slug}/text-embedding-3-small", input=["Say hello"], api_key="test-key") + + assert route.call_count == 0 + + +@pytest.mark.asyncio +async def test_hostinger_router_anthropic_messages_request(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/messages").respond( + 200, + json={ + "id": "msg_hostinger", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [{"type": "text", "text": "Hello from Hostinger Router"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 4, "output_tokens": 3}, + }, + ) + response: Final = await litellm.anthropic.messages.acreate( + model="hostinger_router/claude-sonnet-5", + messages=[{"role": "user", "content": "Say hello"}], + max_tokens=32, + api_key="hostinger-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert str(request.url) == "https://router.hostinger.com/v1/messages" + assert request.headers["authorization"] == "Bearer hostinger-test-key" + assert request.headers["anthropic-version"] == "2023-06-01" + assert body["model"] == "claude-sonnet-5" + assert body["messages"] == [{"role": "user", "content": "Say hello"}] + assert response["content"][0]["text"] == "Hello from Hostinger Router" + + +CHAT_COMPLETION_RESPONSE: Final = { + "id": "chatcmpl_hostinger", + "object": "chat.completion", + "created": 1_789_550_000, + "model": "gpt-5.2", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello from Hostinger Router"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}, +} +UNSUPPORTED_ENDPOINT: Final = {"error": {"message": "Model does not support this endpoint."}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ["gpt-5.2", "gpt-5-nano"]) +async def test_hostinger_router_messages_bridge_models_without_native_messages( + model: str, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + with respx.mock(assert_all_called=False) as upstream: + native: Final = upstream.post("https://router.hostinger.com/v1/messages").respond( + 403, json=UNSUPPORTED_ENDPOINT + ) + chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond( + 200, json=CHAT_COMPLETION_RESPONSE + ) + response: Final = await litellm.anthropic.messages.acreate( + model=f"hostinger_router/{model}", + messages=[{"role": "user", "content": "Say hello"}], + max_tokens=32, + api_key="hostinger-test-key", + ) + + assert native.call_count == 0 + assert chat.call_count == 1 + assert json.loads(chat.calls.last.request.content)["model"] == model + assert response["content"][0]["text"] == "Hello from Hostinger Router" + + +def test_hostinger_router_responses_bridge_models_without_native_responses(): + with respx.mock(assert_all_called=False) as upstream: + native: Final = upstream.post("https://router.hostinger.com/v1/responses").respond( + 403, json=UNSUPPORTED_ENDPOINT + ) + chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond( + 200, json=CHAT_COMPLETION_RESPONSE + ) + response: Final = litellm.responses( + model="hostinger_router/claude-sonnet-5", + input="Say hello", + api_key="hostinger-test-key", + ) + + assert native.call_count == 0 + assert chat.call_count == 1 + assert json.loads(chat.calls.last.request.content)["model"] == "claude-sonnet-5" + assert response.output[0].content[0].text == "Hello from Hostinger Router" + + +ANTHROPIC_MESSAGE_RESPONSE: Final = { + "id": "msg_hostinger", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": "Hello from Hostinger Router"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 4, "output_tokens": 3}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "sibling_model_info", + [ + {"supported_endpoints": ["/v1/chat/completions", "/v1/messages", "/v1/responses"]}, + {"supported_endpoints": "/v1/messages"}, + ], +) +async def test_hostinger_router_endpoint_opt_in_stays_with_its_own_deployment( + sibling_model_info: dict[str, object], monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + deployment_params: Final = {"model": "hostinger_router/gpt-5-nano", "api_key": "hostinger-test-key"} + router: Final = litellm.Router( + model_list=[ + {"model_name": "opted-in", "litellm_params": deployment_params, "model_info": sibling_model_info}, + {"model_name": "plain", "litellm_params": deployment_params}, + ] + ) + + with respx.mock(assert_all_called=False) as upstream: + native_messages: Final = upstream.post("https://router.hostinger.com/v1/messages").respond( + 403, json=UNSUPPORTED_ENDPOINT + ) + native_responses: Final = upstream.post("https://router.hostinger.com/v1/responses").respond( + 403, json=UNSUPPORTED_ENDPOINT + ) + chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond( + 200, json=CHAT_COMPLETION_RESPONSE + ) + messages_response: Final = await router.aanthropic_messages( + model="plain", messages=[{"role": "user", "content": "Say hello"}], max_tokens=32 + ) + responses_response: Final = await router.aresponses(model="plain", input="Say hello") + + assert native_messages.call_count == 0 + assert native_responses.call_count == 0 + assert chat.call_count == 2 + assert messages_response["content"][0]["text"] == "Hello from Hostinger Router" + assert responses_response.output[0].content[0].text == "Hello from Hostinger Router" + + +@pytest.mark.asyncio +async def test_hostinger_router_deployment_cache_ttl_opt_in_keeps_one_hour_ttl(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "opted-in", + "litellm_params": { + "model": "hostinger_router/claude-haiku-4-5", + "api_base": "https://router.hostinger.com/v1", + "api_key": "hostinger-test-key", + }, + "model_info": {"supported_endpoints": ["/v1/messages"], "cache_control_ttl": True}, + } + ] + ) + one_hour_cache: Final = {"type": "ephemeral", "ttl": "1h"} + + with respx.mock() as upstream: + native: Final = upstream.post("https://router.hostinger.com/v1/messages").respond( + 200, json=ANTHROPIC_MESSAGE_RESPONSE + ) + await router.aanthropic_messages( + model="opted-in", + messages=[ + {"role": "user", "content": [{"type": "text", "text": "Say hello", "cache_control": one_hour_cache}]} + ], + max_tokens=32, + ) + + body: Final = json.loads(native.calls.last.request.content) + assert body["messages"][0]["content"][0]["cache_control"] == one_hour_cache diff --git a/ui/litellm-dashboard/public/assets/logos/hostinger.png b/ui/litellm-dashboard/public/assets/logos/hostinger.png new file mode 100644 index 0000000000000000000000000000000000000000..9ae3f4bcca1b0a3d2fa65a23064017950f4926eb GIT binary patch literal 16604 zcmb`u1yoe++b%woC}9B7rKprN3eqSe(jWrTBGTO*q5^_|(hUyMDJ_i%NXXC)LwDEE z{GUC(?>Xmx*0;`D-#NcoYnHQQ$FuK!$8}%#^?X!TlqM#iCV;_U#4^vMp2J|+;3qch z@+I)ko@@Ux3?`8#BPIF5b##5gAGl3k3%8ous#{DH(7RH2 zUau73DMZ~?Jv8nxfeZxq{O^2xISr1NhBnr|XvNl`>lK;bahtW|;N#tC)mDpov_DQjc-WN;ss2;`33bD0^S!WlF59*y@JSNoi zJ%~5TXe!b$YS%I05mvl7ER&()mZ|d8K=P(7TW!CtGn$ZAhIYFI-9B#B)&-sI_Bgy0 z8@d#i**jyh4@yJN8RKvy@>uEYetsbrU_PVJND^8I86|yM^BXrm4FL}}bB$H@ zgW19&D()igF1-+rj!Ja+X(+Eh4kO(@cLTquI!o^`nRCK~ zZj8%b7oq5G*^U|*ZuUgVwc;n>#(c+*G$~4@|oDCUIaWRwGR_>UP2B2p*-mc zKC4n#|5`?puXaKix<|~oz|-)MIP9kL3hfOU+60GY_7cLFJg&~#4bwvAxGQ%r!@dV% z(f6%}Wu?+MnkUhl&31{p_50~Ehq%rUZm=H7c(mGHG8BZ--N2RaAn#$D>VI#^?lh%> zbFtIB=65&c>Z)7Ut0zSFA{k+{hYbCOaT!C-+WF<4Rdij z5w-+TtE~#%3(3ru@PoOB3)`Ff{GWxu4V&JL@~PTs&dFa2f} zb)=&nCkYE|6BpW!%~&o=G7YA2rWwc5?6Vd#T=O7~2oUkK(|UZta1-weHcZGp>S)`E zuf==sHGOU;uv0OMspj;v+O$%pGW2)@tj}%Lr^9We59(Ya&0JXtun*(%co#7b(Ai>JT){Hzamw?AUhh%K zRKyWZvEs&V6M`{@R6@Ft_u}|S^Ml+=qf1zf2k^!s0-*74a(d3x>lN^T2#vTCH@AIm%-epd){uSW?uh6PGqHE-8{-bX~jqxs{$*19p< z@gipLgfD`>rB$xOFmLRLvWuIeI-iB|MU(sa$`|N?=teFEm#;WKgiFF8OLeOS`DGXw zg|52HYL^g-aZ&Z5cOHi=r?Fg4bk3?^ z>;fs~#!XW$g7X=H%uqK6WdAf$55-|t>0_hv8G4N2D@OZa>T$?dQE@Qe;nVN~3Ou@j znXAh@CzEk_3`4Y{TcL3>&PvxH8)=fZTgxwTwI zBe$z+Sz^Twm!51Kx~rehTnpF}bam9|mWMV?pG4`H|@HtaRe zwXIma`raEiZOtDq%LLF0I34mOASrhzBOXyjRGFQf33z^~!*ZNrD={lCt1a}k)O*$aXRY{tJ^$D+U1uIgWY3+Y((mxMdo9p>VhKJ>N`u=$t1s>le2GK1=jO)fC zUPaJ=`ufB@(ftHmbR$mCbn+2vT^%_HMLI7-L}?nf;yzgZDVqMTfhh2r6l<9fXNJdm z=R0dZ@6xo+PjEMuKL=)9s!>}gOvC1T{>j^xG|5n|AHwyLppT0BnD^Po0 zxMiQ~a6#24JGWEGMuh3Y3lkvojdU5)-+z8_mkutLM#t^`1l2?P>N>i*?SIx-Md4G# zGPK(6FYv~1iA^V{;}P)K`XNhBjkZl&r6JSrK{XCdQ1+j=H+0m)=D>@|IK?bWIliiN zvIq0DVn$187RGZvr)cj9#ii^I;P5V8Be4?OwiY`#2TV|x+Z+<)fa0|GtWEr6PG#36 znSW>?>!` zOZcX?uyUZb!2A+QV+U}_4g%>Fx^yPqDr7UVlk??s=a(1aw2LLZR83e*duj_2PtRwg z0lvY!kZPVpvr9=@tviPvbc35=3`v@YM?*{=`XcT8_o2jU73C%|xTmdz^r<)KojO6v zZ7)72y8~k!gg4&N-)st{+SzKSI!ZZpkQkJy^?6A)hrBA$n$H`oKDZ?2pckuKe)B-~nRj zgHq@H!@QZTb03kO%|HAPkh3tyz4y<(e4>s9%GDMk&vH9;=Ud)`M8!oAU$+r(P`zBs zM;PrXkb^92b(P1&n66QmPodq;=KwB6xuwN*Hd0wJtDbE`K|4(8Ja)gSNPjA^{ONm2 z%|q^!$hwmjTM-gi_jGP&0VqXqB_tbU_vsk9ROED5$xGjpnHOo}SjuMD_%4 ztj9UOj3-y%f=S>ju@IXjC?A7&a1q9U*;xb%?*%cLZ^xPik~QYMcndtl%sdU)xeW5 zQDo7jdvOaS2~oGsJ=ZfC4)2k}5qVsqKJuQKoL77rI1(llc+F&ED*2qGu+%hqqRY83 zn75cTAt}_MV1e)T|LY%&(3w!5^&j{l0WJf>1>gU`4_|N@*k$njPyP7snrP}(I-LE+ zpAV3It4P9@l}LD`Eaq(=NPs1A+HP{)?!7)-^jzwfjmbCjc_CS8-2!5;-0PW(|RIj5zNW>U*;Wni;VjH|HR zX+7bib2-Xz*!Ttf@#E{zjc52SQTns`Qx^X)RyhLK%k-*SuKMY#TFu+jbwD%26BX2b z4RahPw&`81QqXN8*zU8lF>od8^WXB(9GU1#8_icJDzW5{HJc~Fh1PyZh^=yi=NGpK z$*2CD!ive`F25PWFZ*uwNE@Qy!A3w~E@1-ZagJak7a%s?FPo z@bH=}N2ET#5rAOCJgsEIbycD!CSjU-Gf$6sDhHIdgQB9s>dFx5n&p_`AqxShxUg+1 zoEO^H0qU{NcFVOl*p8$M7SIzd9~jXMH+4bNyjCH@g`TyRCF_Nh`ns!m^C{xWY~7f5 zE5F8vT++4s+W%ZepNt5;(l0P(?cpQ2J(u`EI3>Z5TC8tTp(4szH^`3>I9?(J9?d)L zB&8w!?KwnmL~r?!mtz1*w!%`07cL20AeEgA&*g#F(B46NoR6!^zNVMCmiqlv%p*I- zAyQejEl~MdFFn`RkPmfus_woh`-EUT_j##_*gLffuZ``pSRL= zZz!~0!9OhQ;D$SDQwkmi`&mgct_`l*R9wu@I#pjJ3ve!0IRxy9?6P(wmSscqi3M?LVTdCXbh zX4%@=jAeDgpxri)HRVE`ayL^Mx0~nWIL(QZ@OGFMyWRM`R&Irk>(|0OFT_Q+;WqC; z=t|%dnTuv9-RJuv_ZH{qCaYp8{NaY4t)|LR%_6m*cA_gGwm;=<6e0FmRZ_Qu&Sc20 zdnTQ?^BwBX&q^Q6@rafO%3Nmz8ELD;OFcI-=(+Jvtkc-O8Sn>WJ2y^O^2&gklKxAr zqb2;Wzv;`wFH6mC@B|pRzHvo51S~G{_Ep+HErcY*pRRAO_ZoC9m?NDmoHlH^ z#EM-3tU*?moxaZXPuHRxf^FOz-->n*-gqYY$|q}E)2t=7E=Gxd;-q(x)JxRir*)d? zybuK1Kwi>iyCkL7zwJ(Prl%CXX3gndl4UummS8c-f;wqt)nWiwNg=>yJ=e+QMvs;R z9Ehw^6*9LGz^Qd^d}VnCWysH*`&lx%Sr*!?-J4{s>3{FMWKN1TFZ7x^8W^$ZON)P*8dCp- z&QZ5EmWc~LXRzNoJ|UGqYm<92AHVowJag4u;*=?=ggey4DgX~a2CPM-nvIqtWg{|_ zn~!fsh3Qr1tSsA%n@-h#>YdGIc%{wSvLqIB_S!M!5!byF_r(trmxBE4=wXz*um-d^ znL;54i5gbsy~wFQl*hqOHcs8u8~=6(Q*|-Bz`3eTKSq$^(N$=2aaEfv`ypfk^0H3H z3Z-anr6d`!(>X;sKZ=gH0s|#W*T%4+Jp7Elk6D;HTI{>!iKu`^KUYBr8u1D=?!WBL ztZSSl2HPrVZj)^MgiF$6LCYVYWK_|JR2Q>}&xdym##dNtsXHT|lyT%@Z|jkD{z(iM z<+RyI^6K-k`;z;3rjkRq;2*o~(BKrXzYatt7(Dv2(0x4NQ=gS}`WBC4T2>%A*zdMK z(Yo8<)X&6K`psyO*Ttk(Bm&d~Zh%qZk;jH%`GS;E)!yQ3pam=!Gq@FM#qKQP*4R!L z<}+RSN9f$I;XI{zRfhK|I?iDZI9~1NwN+v6wTFSv3k+M_R6dh1!cy1edP*&CqliMG=T8wbPh{B{j?x2hPo3|Ry$tWmqu1+H8K#&^TyDPvrk)No1^Yx}r64-lz0T>dfG zR6Q#Jn~_JK5#61X;w@dbOolGj%c4@5Y$yC~&=%$;haBXFK11a2vYdU^kny`(hd_g=nn2 zjExHe7323s0n5^NFZC+i#Q3D#oaCxk@Oi_O_qX@i#6W`k+OjIc|8&1@uZYLXXZLZC zxQVR@86GPZHtakGF8RbyG2bxmTrXh1`lv}0Pp+P7U@@F0W7>+Hpj=!vEDVuq%|&p@ z(XRO&s*@{Ou%_~~QBzAU0@YPoCKaV#(4>Ef6m|~=`<8nXuQpk#ekkHjbwV_Uj-J@& zD8~aEwGA!;^B%-Z0n0FlS!_MG&4vxWQHpn=LF|kaHJue@h8eBMzh*;KtCFXe233_u zy-x7g0OiAebEml41~qc%WLQIicaov>n|5fkWz3%U-kGzFN;kh~x!v_WoXe2;1YdzC zu>eN5N@pt0cAvi4(Xk+^%P>qCTF>|@i1H-Z?_D)=O(WK&2~`{+onl+`PhJw7f=lu! zbl4chHa^pxh)+W5zSP>@62{7W+C1CyQ_JE;&+E7=yv4}ofY5c1W{2$a@o9q!l}P)< zaseI^*n8k)&&Db6hHQUO{;cB{aJ^&UXY);zDl>6Bx}Cw zT=fy0^y$qgluMO*&!zeaz!&|s9w7xD>8YMlaX){dYhjk>q>X?&*La=HqMn=%IUXNr zz+ub$gzdwn$fyn^U)wrrqtWFM(D#4971AYXF46le5XrywBQ& zQmjKu4QU@RG4#1hkM3L4ss>aWHNySa&+h$0|f*ajjYTEq}{= zFLd<0kqa`{n$rqo+Np*$)nj12>1q1xS;b%}?Yktzzw%gqGeMDjYAJ9@O6qjdk$t9I zXw{|O-puX@r;z_c`|WLAvQi_dLc*5NKR;GR_TIle3LU#1fnbCZC2Yq*Tx%Ghyh}f; z+0@lVou-4@>%GXGX=bWeme9uWM_z@322z`H$#SLDPS*f1Vrcm1MEtfEiC^2Up5^N@ z5yuVwHbLz4HE$-mVg^~(&tqbCf2pZ(8>H&SdYpZD=h5jsZa7NfOrvw-$@v6=@va1{ zLDD`Wb!c!DR41XxwN;11$Xj6r$zLu&=WdeJuib$T_A~apiT6%BQAyx*;Uhe~ueW_g ztggc(UAvq)4)T52l6xsjx>|o({);hgk>~0s)4Cn)n@6h!8JqDly8}%ia3RVWsxUaB zs;$Ulmmoh8Yyby*X~*G`9AfMfL2S|wHmlkqpm=61SJ+*n`l%P}s&a zj4!3M6PIsp@S*|=4*>qz2xMR2CB`cwl0=<*Q%`~bJ$l@7YeES*fSAod%Wbo`Knc!?q9CxwzE<3P$HM`!K9+AUF}`{=4nF}+@5C$2A8uP)a=h!K+(%L~pc z0b-n|ajvg2Lb2%1ESehhal^~--_o<MjnOMfN-6qUN5idOpCut!SRBD>wspw(?|^AabSrAM+xud> z5e)ZGAuA5clzpXR!G4z@2Z1qi*4xS#<~L(+0%tX^fNL0~a3 z36WM<*z&%#(@VT0YGz@xf4LqW!1W|KOT`bW^z@SB&g`+4bz@x@!Z^F`|0w5nXbTPU zDAdUduUCe#|Ku%_3ryWMF^c{A%nMVuUx2zjhLsr)4w&-U2v2MGJ2l6eVkE-WdPLte zi+1B%Yf;v03u>3<)GM2fBS~k{p~Y_S+01Ff0>Ne5?ag7cH>KO_N*$w?!sh$;aGn{sdKZP zww4r82U*{o5LQFud>fa8Cob$%fTE9#hX|PO_H8Ix^MPc21|{ntOl^BFlF>@;2ZPb_ zCm$jNNMWbfxN;1FOmA?=$~Un$c_az2i7)1u@RHz%xgh@OJV$j{K_G)^5$|Q!0|gvU zb$@`T=2f5W|Cno*s+KM+8e`Og(>0X>wnPFP_{-z^vwid~`GW3A?k!R8qi;j8f~>W(zF0G>FztarD33xPG@O81L^}`bhDpZH zLssTt+iw3$BWRV*0-;Q72^|VNxRk|Re}ne!(5+Z0EOQj&5s?EeSh)#c8T&Q#bmM!}uR0a(Hz}0*YuVJ^U!27h0qm zlT`p5i|!4M41=C)5HX0w$Qee$E3dLH$;j^Z=4WLMO(AvcgEPe@Zh%>YM)?854Yf(# zH-r5Cx2~c{(3%hoEg%zCJyubcdSe?lg3~gucVVi&lSO*Fhy0_|JHY#3OkB;7DkT*> zFY{TYmg|wFJPIY2vVx?ecBX-|2Bdue*O!1T5x^UzssrLu9OhY>+;tBVkqOcC&Z|)@ z>-MGG&^;_+hucdc$x39M^0YJToz>IDk`qYR2L2jgUs@~5d(wnBFs#%LMHt|#5K*Q* zV7#}MVDkL5*MY45(_a9AM?IUQWyxPh<+hfnf0KC{dm@7$?jIE>F@$Fb6de1O?mrtmTdg9q2vo^% zK}WMKigg(%2Liw7eD>J)8^KE{{HJQuJCU_XV8T-=67Y%6u8=}ESL6Mc}!m9A|V#8IQ_-19+GsBjO1Z6 zX4qR#WDx}{`JR*f+7@@U%~7XkJH;LhIaZo$%p_e-uF}3EsJ)fz z<&W7mE~XxGx(mry7s!Q@-cLPVTnYSD=tBKxJNxZds)-R%tG0;02?ADX#!jF`UynNY z_B%PcEI62LquTPUK;vQ1k|3D%a$o*-=#b}^Xn~|2++EA8O}-;M_ptIda3?S#_JuJ< z!e)zEvOqC|q$f&r2mSthJK~=A1a^nl4mki~1(`t8w+X>UOO#^|w&iaE%wYy0eJ zvPL6i%OdUR?yAIu!Ce+pdzWl(kmAtVtbZ~IT#)XMPW$!u( zywNL6l!$6{H|8CILxBfEdnd3>u~c*1lx2>GL;Dxw<-dW<4*Ol_FQH9Ukk9f`t=zno zbg^)F``7bUs}%qs0hvZQus9_BY1v_9m-ReR(xAdX|6_RAC6KJX;#~J3ep}8}35My# z`H?kDm&VKAI5&o;gXApixGCe|Z?I8q5nZ{8UyAn@;!9#s7Xzm=GEK5<^obk@?NjHB z{JZTvKQXR)54Mxm+?W+4R30@~(TSUFet4X2H1C4s7~AdHaP`^W3JW9Oh~^*Hj}<^y zyen3!9pV=GO9en#;uSVX)lIOiCPeDGLJ2X?8|`(E3QqRwlgX8Z@WBFgvb6-UTMjz) zG&UV=v3p}#_P^{Bo@6Y4CkBB62U~AbqFG|2|V{@*yAh8Ge1};9nEPAnZF~a&uoqy zou|VKpYBz?I-KP4c`{SRGJF6fisTm;A zeXqc<81--@PLjR@TBiMPKUJK6a5e_~&}(QT0}%~qt8>B2dqAdQ1*l^Ibce{srF2~k z5iJVMxS_H&j_1ojj#qY=Pt>6%S>}z-t>@&d-)sibH_qo|{0C>AIK>%gpmS?F6Mf!_ z{Ddj+%4DFyz?kpQK&87BRBI|*bLWB__mxx7#Cwqh^}B$`_@4z|Tu6MeJ(CYA_~TDM z=5S+f{S}vGTbqm@BTJ(9DQr5PIPC#ra#ZM=*EUOE(M%mnR{DBKkgW6@r-43p3Ni(u zwK1y-B_}mg6Tc**VAHRlLW8_!jm;>oMz(k77)1KWz}|%4yb)Tjr0gIm2RM&B1QO=- zTsDf5O~j1iJ{tedcNx=Hk_QDDpm>QIJl|Tl@SLC&{?TK}b8YHD{oBs4a|j6f1h0)1iEO|Yg~;gTs))~xft{U>(0*x6FhI!|na z`vyYEH-^>^>1XX1C;NoY0v#Z78Nw78WP*fk^}!oQ0XL7-Q3l+60N~~aqJsPY$S49J z;}7!3!bboym<4<8{R1+-2;P^$#C(-AY#4PtP>{aQWK2DWAGabJOurm^;&T?|(nihd z4IdsPH8S`BcK-MGd@@5oFV|D{nRM;S)QO!nSvS=g#vBP^OTey|!)7)pUb|ay6c7kL z_6*Y5b)LVr?t1XD|xO@P~g&HcxIR7TzCXNpsLm^d!GI_ zF&PD8xOCl^dNL>(YLNP-a1vw;XG}*GS1B!Uuvf1lp*htW0bCO5+5-IY=2WiC?k?zw zGOk7n6E$6D?p-qN``3-nv3)%YSximyEp4ju9i<*_#QVTin!S{U@E?At)>7eSHnbmv zpaZ%smYX@}=ifvMfhtgq>h#IT`btlM4}R(&$j^2^I1LX9KoLGQuYRgxi=w#<_lHz9 zk{eE8M$BRH)E~>A-J!k%WzfrXu#ZEecp*T9q0rP$3ljA@P_W3g?A6wE8y=4;k@3%< zr5X>;-v!>nG@)P&Mp`#~dcx%Q<#-75l8x`d{Ie>L$>FJ^twd<~# zFCh@~T7_6M=MY3HkdOw_5HEps)y3)a8@y2&IWsZMrzTmmb{-p}e56GQbH%;h3TTlR z{o9(X-hErL@UxkKI$*>>Fzr}HGjQZdxx6LC2Xcqt{kv;W>?9dBDYV11x-Lf;!fHx`8+OaNYPeh*+ zD24q0lE9Ia1e~_7)G8(Fp2FASvUF(L;oi!J`oT6VCKR?viudP0fX#e6R*b4=)$jEC zBMq^LWNt>3UBaE5pNiUH*sx)230gpg)=c>S@9Hl^Dpe-DvApY@fh#z;*uItYE@9BU zP;GVRMVSKo9B5i81=1*}>2a`U8Y~2(>cuF-<8Q^@ZD}3lsI!zPV!72S;66ZO_%Ea7 zK5lq?XYFC7Rp!;_WVj80g7f_a@~p+(1atBUWsrK)JhX;BRQ>GviMoajTAIvH$OI2n zy$Vw6ULm&7u{pqBQ_UE#UBU!13X%GdxDPP+p<~0v87s|%fc@0zUs(%sEL zUHns9VX|Ui0!LtdTuC)2#FE{P!G0uhZLabX)UjhtnW&RF()E>ErbKc6X$%Sh6A$O} zoSe+M5GkDFR+r9gO)|0`#4rB)RD-+YH;x`@h{nZ#_W0IoJ!XE(wn2z>P6pEvZ1H`< zQn~3=lZ*@K`1@sla>Y4wVqK2>Ckm~PwkeD}FTFkMyE+!}`rF;B^4+5Yw=Z3RT}*n_ zPxC}Nt4VH@X%H`qopdRGo8(alUMIzFKwfHQ7R!(iGdqrQe$kL^*Bfz(9-}mq-V?g` zXk_GaAq_tZ7X@s=_{lsN$=V_QRAv0&Z$RN*{pPExYfKq!+VcrZ0@i>OxGeRe!D&K$ zK=>@{QRX*}_>%O4KPWEt>cy{st@T@Xd4QNHVi_jK9qp2aUG2neAW7HdX|`xJ`A&`t z`!&*donB7U#+^6(jJ8z+j#Fm(2mis<;bW^Y^^it} zn}U}w;}1;)8v*_BTK}Wl;ezFxH!l)(E8jdZ*eH{B6K!ijOD{sr*32g`XbMyZ0Q-%$ z8BQ4W;gHWC&X+sO-N`1`NX)b|`G`vL??ZWlvF^1mAl=JCvmi~A+?u%CN>S#+F3OWe z)gN)H#bxU2(u5u_Q*fYbmar`COTTC!7V=2cTMp2*Z(kb9QKCKH@p;1>rr0l3wMpcC z3k)W)U=TH>&~ddIqeQ;8?7gyX^6AVL`Ac%QOJ*BgHhPQ|?+pW;*7YGUv8rYAKL}g) zIO-_biLs;yLnz#_2q2aN`R{Wj7eNr10exOqXcNXvyQnMPv;i&o)d#=6VOrQI?G)*LSyS zwQ_Ej&wkM4#^<#oYRHg_Wp}R#K`n$?jP609^xEud7jDE5AbF zI#4hIw>dxxZ7+;`2u^4{IQwJp_XtiEveGZ4EgWdr6V1e&{{euzuGdB4C2YyI^%K%6Q1GoQw{^1K<0yr9=LjE&9 z{_Ds8KUn7f@FzxSP1p^Max(q%8iZ4e62XHyc-LVz+rSYLq=4X751v&zac_ z&$?y*?XS-!Mx}(uxFG4yXP}CuOb1lLt-Cj`dF^YpZEFI2;{DUcWK@r@XQl-g*O&66 zl=eOtqj30?nf}Z|?3^eLv5ihu06mXNdXyaS5fzM(sQnm;uF(@(FAf{j^L+NicDEH0 z67ko_vjDmJ77-6o7;d8Z{(N)->ZLMo`r@(fMP&-%>!W#NDc3Or*D58WWd{$eT-1M2l3z(zC$cQ7QTmznR3#g&I=bwK+eaAIizI!Jz79s<<{o_Io0S=v)_LgD9)r+WC32o8q z3WkrbU^l!C!OG#dOqoIX`u9RSe4a2UGjZoAW0gQVDLx8%bO7Q~pV-Kc`bp@nR&`5T zaK)orf>f*29(rzPA-hUzlgj~TE(>J&FOSs`V_=8GO=#3d!Nn3ETVJ!5yewN?_HRSf zeG9Mgj8s1|?Toat2vAaGH{nGFwgGs&+oBhcat)a>a9RRIj@zIz*`^y)qer#Ly-My#I>T zFzcA*(P4R>SvTT5c0w4;yqGs~G)~*M1?kAtN1Y%^>nF8^56!BIC!T#yAFwi=jSv$_ zm}l1_BeJ#=)AebN2$s!Ej~eiDjlnQPyx;+n%G*WiO&_0w=~9NYCT3@cVWqAD>OpO- z{op1_+eE&L;5Yx9EkfTTVF`gbtxtS|>Uvyl|5N_)p#)qN|(#Donc zq_}Qj+ch@XWOLA*U`GGDy#&msnPjVrt5)LF3OQ38e^rdxq^vxBA`HC-aQzC6WFBP52}YY7Q%gizL|s)m=&ms#eFJMUG2L-qqPw0kKFqXwAoW6zm|yg z*Wr!p9%92Fqu4EaR-Fql#Xq^J4?tc@ZxysB@c_mX#i zTxsaSSW2}h5wmjuB!{bNRVg-A=kr5yp_}Dy=mde2JhF~*2*g0|pJK3!E7@^xNZUsA z-b}eV_g5hNZUMk5I_fQY=zN9CCO@&q%Vbsn31k7J&^hm~7)9u*I}gnqHAmnB253g) zi9AR|m|Y6@3r|WH_QOfJAZf-EYMe3hO%8%`JS+w9E)3v}3(7xEjxsFt-ShX}cXE{3 z{MCR1w+S-U1j9(0MGB?(9Z+DxoX1-@FpVhpO(+xTkmZ?iT6x**k5quzH4d8~#@J(* zN9WEcBDj6?#NNB-+pYAQVTpHdW7Q{tv4q589`BHs&Y9NoHic)6bys)BtC2-3cw;S; znrs__{Wzv`mU{mPobQ%}XKNp++3M4Zv}mSbcu89S!XZ zL7th1Ru&1$7+t(E;5ecO8m|rN*bZs4D%cUJ+S{~cCi1W3@6hZa}5-nJgG8tf*tKa^cH=Awym%c43_5+cor>;3s34fJ#|)V z8dCbZyP~(^f6+R{3)XxCTBqAp%>Wh1LV*wRDbie@OLV(8Y*3ds>sHaeDrWD59nhhV z+&p{Tp+HDM&Ja^;fIg+Me96Y7=wh?NyiBVmHa{Z?kpdI}Fkvn1~ZFn>1z- zQ^|v6UjcUD-~1Dm>71K)F~bA+MZb8+HNl3T0r))H=Rv(|ZxNsikdd{yo$@ST7fd6i zB5?kweYw@imv%cg;yMnJ7sm}5*GND>wHCQGQTT?YDEqrStsg$EJ|YYb>liJyXbb^6 zEFl-LeYJ?SZYZcKMKtoV*7n2?(Z=lk{ivnM?~AC#Sc@(FAgq`KZd;Otz-cW zK&z)s>xZ4hVYZbl+Ia-~6%feEG&7<8aK#JeWS!Ral|3i+u1?6mn

BhjkFB91D_ z+WXcauH(_uaNMvE_V;0VbQ-eN^{sy#g?zZH>TWf>-~J#v?#(EeK(G%@6YIl-tRIhn z{B4Im%_j_EX1HixKr!6-)L&=<@{h!)R$HgH`@mGX$5RRH87At3KIfm2T}6QClgkcl z0{ghbVkFj_~RI0p8Z@4-4kcOBr6Dk zJtA$N=<%^SYFZ_f3tj(ifX*Fq!#r#*kIqRW9RNBjaHD^LPAgvGXcV;JX7qirlkQSl z83pPB031H{l^C(L5hQNeh=+&ksiTV#@`RAdg5<4x%j1J}z4) z49ZBXaqJs^@~$g+0OJxp!J-G3-U27sVqRSksmj`$+Y72vT0E+uD*pqFlbOdG@C`(k z(QYvFCrv%?&1_le)lwE7O1IM(D(t=ohVR(jv%++x?txvt~yOH;OPaps+9u>&1DLLUO?kAsMuX0ZT~yhvs-Zj$VvbR^Zz7Z z*0@c{Nj8#JH|ad)fx$XJ;>R1dYyz?lDRZPvB-eehgVMV+g&da!2h#sJq2N|!^O0kd zO|on48!au(iB^}Rnq{DK0^5jELELI=xFX(o_q_iJ2#jRffiGH^P~N7rhR9x&r%0P}Lap_s}KyPdiL{j-+U5OzCdV&RX9%bCE zvCzbT9&!jzaep0O^DI8!X@y{>dfj4L_1L{g-4Y;?g%j2fRu=;FJ~q#Wph)ayMa}|c>20vuqs~&2qa+IB4ZsH2A@EG|GgLhX1 z8xHEGEefGsTy9U#@sT<{d4sJ)UTZww!AE7cn=@1<~-?VcRp!M3j4PI7eF>|t=-^Cavy`b%iSFd!i4FzS7ydUs cRiPI+wI&}sUR&O5fKHQvD@qkSe&zT70bxfGz5oCK literal 0 HcmV?d00001 diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index a8491f656a0..a8879eb570b 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -30,6 +30,7 @@ import googleLogo from "../../public/assets/logos/google.svg"; import groqLogo from "../../public/assets/logos/groq.svg"; import huggingfaceLogo from "../../public/assets/logos/huggingface.svg"; import hyperbolicLogo from "../../public/assets/logos/hyperbolic.svg"; +import hostingerLogo from "../../public/assets/logos/hostinger.png"; import infinityLogo from "../../public/assets/logos/infinity.png"; import jinaLogo from "../../public/assets/logos/jina.png"; import lambdaLogo from "../../public/assets/logos/lambda.svg"; @@ -131,6 +132,7 @@ export enum Providers { Hosted_Vllm = "Hosted vLLM", HUGGINGFACE = "Huggingface", HYPERBOLIC = "Hyperbolic", + HOSTINGER_ROUTER = "Hostinger Router", Infinity = "Infinity", JinaAI = "Jina AI", OPENCODE_GO = "OpenCode Go", @@ -263,6 +265,7 @@ export const provider_map: Record = { Hosted_Vllm: "hosted_vllm", HUGGINGFACE: "huggingface", HYPERBOLIC: "hyperbolic", + HOSTINGER_ROUTER: "hostinger_router", Infinity: "infinity", JinaAI: "jina_ai", OPENCODE_GO: "opencode_go", @@ -389,6 +392,7 @@ export const providerLogoMap: Partial> = { [Providers.Hosted_Vllm]: vllmLogo.src, [Providers.HUGGINGFACE]: huggingfaceLogo.src, [Providers.HYPERBOLIC]: hyperbolicLogo.src, + [Providers.HOSTINGER_ROUTER]: hostingerLogo.src, [Providers.Infinity]: infinityLogo.src, [Providers.JinaAI]: jinaLogo.src, [Providers.OPENCODE_GO]: opencodeLogo.src, @@ -493,6 +497,7 @@ const providerPlaceholderMap: Partial> = { [Providers.EDENAI]: "edenai/openai/gpt-mini-latest", [Providers.FalAI]: "fal_ai/fal-ai/flux-pro/v1.1-ultra", [Providers.Google_AI_Studio]: "gemini-pro", + [Providers.HOSTINGER_ROUTER]: "hostinger_router/claude-sonnet-5", [Providers.JinaAI]: "jina_ai/", [Providers.META]: "meta/muse-spark-1.1", [Providers.MICROSOFT_365_COPILOT]: "microsoft_365_copilot/chat", From 3fc7969e8bff396a48639514f334b6ce299c57fd Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:03:04 -0700 Subject: [PATCH 13/15] perf(logging): skip the base64 regex on plain text and copy a few large strings inline (#49228) * perf(logging): skip the base64 data URI scan and thread hop for text without a data URI * perf(logging): check data URI markers before the scan and hop only for data URIs Check for ';', 'data:', and ';base64,' before running the data URI regex, and offload to a worker thread only when a request holds a data URI. Tests now watch every regex call and confirm plain text is copied without yielding to the loop. * perf(logging): keep main's copy and offload, skip the regex for strings without data URI markers * perf(logging): copy a few large plain-text strings inline instead of hopping to a worker thread Type the regex watcher test helpers. * perf(logging): skip the second marker scan when copying a few large plain-text strings inline The async gate already checks every string for the data URI markers before it copies a few large strings inline, so the copy now keeps those strings as they are instead of scanning each one again. SSE-shaped tool output, which holds both ';' and 'data:', was paying for that scan twice on the event loop. * perf(logging): keep huge plain-text payloads on the worker thread --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/constants.py | 2 + litellm/litellm_core_utils/logging_utils.py | 52 +++++-- .../litellm_core_utils/test_logging_utils.py | 130 +++++++++++++++++- 3 files changed, 169 insertions(+), 15 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 7d99c28deb3..1a1e66ecf86 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -177,6 +177,8 @@ MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0 # Set to 0 to disable truncation. MAX_BASE64_LENGTH_FOR_LOGGING: Final = get_env_int("MAX_BASE64_LENGTH_FOR_LOGGING", 64) BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: Final = 256 * 1024 +BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS: Final = 256 +BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS: Final = 8 * 1024 * 1024 REDACTED_BY_LITELLM: Final = "redacted-by-litellm" # in-memory stand-in handed to provider converters for redacted arguments; never stored REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}" diff --git a/litellm/litellm_core_utils/logging_utils.py b/litellm/litellm_core_utils/logging_utils.py index 8f58e8b7051..88d84201beb 100644 --- a/litellm/litellm_core_utils/logging_utils.py +++ b/litellm/litellm_core_utils/logging_utils.py @@ -3,13 +3,15 @@ import functools import inspect import re import time -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast from litellm._logging import format_base64_size, verbose_logger from litellm.constants import ( + BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS, BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS, + BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS, MAX_BASE64_LENGTH_FOR_LOGGING, ) from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs @@ -44,6 +46,8 @@ Helper utils used for logging callbacks # Regex matching data-URI base64 content: "data:;base64," # Captures: group(1)=mime_type, group(2)=base64_payload _DATA_URI_RE: Final = re.compile(r"data:([^;,\s]{1,255});base64,([A-Za-z0-9+/=]+)") +_DATA_URI_PREFIX: Final = "data:" +_DATA_URI_BASE64_MARKER: Final = ";base64," # Maximum nesting depth for _truncate_base64_in_value to guard against # pathological payloads. OpenAI message format is typically 3-4 levels deep. @@ -60,15 +64,20 @@ def _base64_data_uri_replacer(match: re.Match[str]) -> str: return f"data:{mime_type};base64,[base64_data truncated: {size_str}]" +def _may_contain_base64_data_uri(value: str) -> bool: + return ";" in value and _DATA_URI_PREFIX in value and _DATA_URI_BASE64_MARKER in value + + def _truncate_base64_in_string(value: str) -> str: """Replace long base64 data-URI payloads in a string with a size placeholder.""" - if MAX_BASE64_LENGTH_FOR_LOGGING <= 0: + if MAX_BASE64_LENGTH_FOR_LOGGING <= 0 or not _may_contain_base64_data_uri(value): return value return _DATA_URI_RE.sub(_base64_data_uri_replacer, value) def _truncate_base64_in_value( value: str | dict[str, object] | list[object] | None, + truncate_string: Callable[[str], str], ) -> str | dict[str, object] | list[object] | None: """Iteratively truncate base64 data URIs in a JSON-like value (str/list/dict). @@ -78,13 +87,13 @@ def _truncate_base64_in_value( # Stack entries: (source_value, depth, parent_container, key_or_index) # We mutate *copies* of dicts/lists in-place via parent references. if isinstance(value, str): - return _truncate_base64_in_string(value) + return truncate_string(value) if not isinstance(value, (dict, list)): return value # Shallow-copy the root so we don't mutate the caller's data. root: Final = {k: v for k, v in value.items()} if isinstance(value, dict) else list(value) - stack: Final[list] = [(root, 0)] + stack: Final[list[tuple[dict[str, object] | list[object], int]]] = [(root, 0)] # mutable-ok: explicit stack while stack: container, depth = stack.pop() @@ -93,19 +102,19 @@ def _truncate_base64_in_value( if isinstance(container, dict): for k, v in container.items(): if isinstance(v, str): - container[k] = _truncate_base64_in_string(v) + container[k] = truncate_string(v) elif isinstance(v, dict): - copy: dict | list = {ck: cv for ck, cv in v.items()} + copy: dict[str, object] | list[object] = {ck: cv for ck, cv in v.items()} container[k] = copy stack.append((copy, depth + 1)) elif isinstance(v, list): copy = list(v) container[k] = copy stack.append((copy, depth + 1)) - elif isinstance(container, list): + else: for i, v in enumerate(container): if isinstance(v, str): - container[i] = _truncate_base64_in_string(v) + container[i] = truncate_string(v) elif isinstance(v, dict): copy = {ck: cv for ck, cv in v.items()} container[i] = copy @@ -127,8 +136,15 @@ def truncate_base64_in_messages( """ if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0: return messages + return _copy_messages(messages, _truncate_base64_in_string) + + +def _copy_messages( + messages: str | dict[str, object] | list[object], # mutable-ok: same contract as truncate_base64_in_messages + truncate_string: Callable[[str], str], +) -> str | dict[str, object] | list[object] | None: # mutable-ok: same contract as truncate_base64_in_messages try: - return _truncate_base64_in_value(messages) + return _truncate_base64_in_value(messages, truncate_string) except Exception as e: verbose_logger.debug("Failed to truncate base64 in messages: %s", e) return messages @@ -151,19 +167,31 @@ def _iter_string_leaves(value: _StringTree) -> Iterator[str]: pass +def _is_few_plain_strings(leaves: tuple[str, ...]) -> bool: + return len(leaves) < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS and not any( + _may_contain_base64_data_uri(leaf) for leaf in leaves + ) + + async def truncate_base64_in_messages_async( messages: str | list | dict | None, # mutable-ok: same contract as truncate_base64_in_messages ) -> str | list | dict | None: # mutable-ok: same contract as truncate_base64_in_messages """ Same result as truncate_base64_in_messages, but payloads whose string content - reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are scanned in a worker - thread so the regex pass over multi-MB base64 images does not block the event loop. + reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are copied in a worker thread, + so neither the regex pass over multi-MB base64 images nor the copy of a long + history blocks the event loop. A few strings with no data URI, under + BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS in total, are copied inline, since + checking them for data URIs and copying them costs less than the thread hop. """ if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0: return messages - total_chars: Final = sum(len(leaf) for leaf in _iter_string_leaves(messages)) + leaves: Final = tuple(_iter_string_leaves(messages)) + total_chars: Final = sum(len(leaf) for leaf in leaves) if total_chars < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: return truncate_base64_in_messages(messages) + if total_chars < BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS and _is_few_plain_strings(leaves): + return _copy_messages(messages, lambda text: text) return await asyncio.to_thread(truncate_base64_in_messages, messages) diff --git a/tests/unit/litellm_core_utils/test_logging_utils.py b/tests/unit/litellm_core_utils/test_logging_utils.py index edf0dc7960b..a367c02fd6f 100644 --- a/tests/unit/litellm_core_utils/test_logging_utils.py +++ b/tests/unit/litellm_core_utils/test_logging_utils.py @@ -3,7 +3,10 @@ Tests for litellm.litellm_core_utils.logging_utils — base64 truncation helpers """ import asyncio, datetime, importlib, litellm, os, pytest_asyncio +import sys import threading +from itertools import chain +from types import FrameType from unittest.mock import MagicMock import pytest @@ -17,7 +20,7 @@ from litellm.litellm_core_utils.logging_utils import ( truncate_base64_in_messages, truncate_base64_in_messages_async, ) -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable, Coroutine, Sequence from datetime import datetime as datetime_assemble_streaming from litellm import( Choices, @@ -28,6 +31,7 @@ from litellm import( ) from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.types.llms.openai import AllMessageValues from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome from typing import Final @@ -99,8 +103,8 @@ class TestTruncateBase64InString: @pytest.mark.parametrize( "text", [ - 'data: {"choices": [{"delta": {"content": "hi"}}]}\n\n' * 50_000, - "data:" * 200_000, + 'data: {"choices": [{"delta": {"content": "hi"}}]}\n\n' * 50_000 + ";base64,", + "data:" * 200_000 + ";base64,", ], ids=["sse_lines", "whitespace_free_prefixes"], ) @@ -241,6 +245,49 @@ def scan_threads(monkeypatch): return threads +def _run_without_yielding(coroutine: Coroutine[object, object, object]) -> object: + try: + coroutine.send(None) + except StopIteration as finished: + return finished.value + coroutine.close() + raise AssertionError("the coroutine yielded to the event loop") + + +def _regex_calls_while(run: Callable[[], object]) -> tuple[object, tuple[str, ...]]: + calls: Final[list[str]] = [] # mutable-ok: sys.setprofile reports each call to a callback that has to record it + previous: Final = sys.getprofile() + + def record(frame: FrameType, event: str, arg: object) -> None: + name: Final = getattr(arg, "__qualname__", "") + if event == "c_call" and name.startswith("Pattern."): + calls.append(name) + + sys.setprofile(record) + try: + result: Final = run() + finally: + sys.setprofile(previous) + return result, tuple(calls) + + +def _tool_result_messages(tool_output: str) -> list[AllMessageValues]: # mutable-ok: the truncator only walks lists + return [ + {"role": "user", "content": "Summarize the rows the tool returned."}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "dump_rows", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": tool_output}, + ] + + +_PLAIN_TOOL_OUTPUT_CHARS: Final = 400 * 1024 + + class TestTruncateBase64InMessagesAsync: @pytest.mark.asyncio async def test_large_payload_is_scanned_off_the_event_loop(self, monkeypatch, scan_threads): @@ -268,6 +315,83 @@ class TestTruncateBase64InMessagesAsync: assert scan_threads assert set(scan_threads) == {threading.get_ident()} + @pytest.mark.parametrize( + "row", + [ + '{"id": 7, "name": "widget", "status": "ok"}\n', + "data: {\"choices\": [{\"delta\": {\"content\": \"hi\"}}]}\n\n", + "const total = rows.map((r) => r.price).reduce((a, b) => a + b, 0);\n", + 'data: {"choices": [{"delta": {"content": "a; b"}}]}\n\n', + "Content-Type: text/plain;base64,aGVsbG8=\n", + ], + ids=["json_rows", "sse_lines", "code_with_semicolons", "sse_lines_with_semicolons", "base64_marker_without_data"], + ) + def test_tool_message_without_data_uri_is_copied_without_a_regex_scan(self, row: str): + tool_output: Final = (row * (_PLAIN_TOOL_OUTPUT_CHARS // len(row) + 1))[:_PLAIN_TOOL_OUTPUT_CHARS] + messages: Final = _tool_result_messages(tool_output) + + result, regex_calls = _regex_calls_while(lambda: truncate_base64_in_messages(messages)) + + assert result == messages + assert result is not messages + assert regex_calls == () + + def test_small_data_uri_is_truncated_inline_with_a_regex_scan(self): + payload: Final = "L" * 2_000 + messages: Final = _image_messages(payload) + + result, regex_calls = _regex_calls_while( + lambda: _run_without_yielding(truncate_base64_in_messages_async(messages)) + ) + + assert regex_calls + assert payload not in result[0]["content"][1]["image_url"]["url"] + assert "base64_data truncated" in result[0]["content"][1]["image_url"]["url"] + + def test_large_plain_text_is_copied_inline_without_rescanning_or_a_thread_hop( + self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int] + ): + monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000) + messages: Final = _tool_result_messages("plain tool output\n" * 100) + + result: Final = _run_without_yielding(truncate_base64_in_messages_async(messages)) + + assert result == messages + assert result is not messages + assert scan_threads == [] + + @pytest.mark.asyncio + async def test_long_plain_text_history_is_copied_off_the_event_loop( + self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int] + ): + monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000) + monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS", 10) + messages: Final = list(chain.from_iterable(_tool_result_messages(f"rows {turn}\n" * 100) for turn in range(5))) + + result: Final = await truncate_base64_in_messages_async(messages) + offload_threads: Final = tuple(scan_threads) + + assert result == messages + assert result is not messages + assert offload_threads + assert threading.get_ident() not in offload_threads + + @pytest.mark.asyncio + async def test_plain_text_over_the_inline_limit_is_copied_off_the_event_loop( + self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int] + ): + monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000) + monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS", 10_000) + messages: Final = _tool_result_messages('data: {"choices": [{"delta": {"content": "a; b"}}]}\n\n' * 400) + + result: Final = await truncate_base64_in_messages_async(messages) + offload_threads: Final = tuple(scan_threads) + + assert result == messages + assert result is not messages + assert offload_threads + assert threading.get_ident() not in offload_threads + @pytest.mark.asyncio async def test_none_and_disabled_truncation_short_circuit(self, monkeypatch, scan_threads): assert await truncate_base64_in_messages_async(None) is None From c4b01b2ac85b64583c238fd91a43f85ec87a9303 Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:09:46 -0700 Subject: [PATCH 14/15] fix(decisions)!: honor a deployment's own supports_audio_input and supports_video_input flags (#49207) * fix(decisions): honor a deployment's own supports_audio_input and supports_video_input flags A Decisions deployment's model_info flags for audio and video now win over the cost map, per media kind, the same way supports_box_questions already does. An unlisted model flagged true gets its clips sent upstream, and a false flag refuses them (or drops them under drop_params). OpenAI-format providers still refuse clips since their request shape has no field for them, and the SystemOne request builder no longer re-reads the cost map. * fix(decisions): ignore media flags on a deployment whose own mode is not decisions A chat-mode deployment's supports_audio_input describes chat input, so it no longer turns on audio or video clips for Decisions. The cost map decides for it, as before. Adds a proxy integration test for flagged, partly flagged, unflagged, and drop_params deployments on both /v1/decisions and /v1/systemone, and for caller-sent model_info. * fix(decisions): follow the cost map's mode for a flagged deployment without its own mode A chat deployment flagged for audio or video usually leaves mode unset, so its flags described chat input yet still let clips through on /v1/decisions. When the deployment sets no mode, the cost map's mode for the model now decides whether its media flags apply, and an unlisted model still takes them. The audio and video lookups also run only when the request carries a clip. * fix(decisions): read the mode a flagged deployment falls back to from the shipped cost map The router copies each deployment's model_info into the model's shared cost map row, so another deployment's mode there could switch off a flagged deployment's media flags. The fallback now reads the cost map as loaded, untouched by router registrations * fix(decisions): route clips in a load-balanced group to a deployment that takes them A Decisions or SystemOne request carrying audio or video now skips group deployments whose own flags or the cost map refuse that media, and keeps the whole group when none takes it, so the 400 still names the media. * fix(decisions): keep deployments that drop or cannot resolve clips in the media filter * fix(router): skip clip-refusing Decisions deployments after cooldown and tag filtering Running the media filter before cooldown turned the 400 a group answered when its only clip-taking deployment was cooling down into a 429. It now runs next to the image filter, just before order filtering, in both the async and sync selection paths. * fix(router): honor router default drop_params when skipping clip-refusing Decisions deployments * test(router): list the router default params helper in the router coverage check * test(bedrock): stop reloading the cost map module in the GovCloud fixture --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/decisions/call.py | 160 +++++--- .../llms/base_llm/decisions/transformation.py | 34 +- .../llms/openai/decisions/transformation.py | 4 +- litellm/router.py | 84 +++- .../router_code_coverage.py | 3 + ...st_decisions_media_deployment_flag_wire.py | 123 ++++++ tests/unit/decisions/test_main.py | 387 +++++++++++++++++- .../test_base_decisions_transformation.py | 13 +- .../llms/bedrock/test_bedrock_common_utils.py | 3 +- 9 files changed, 720 insertions(+), 91 deletions(-) create mode 100644 tests/integration/providers/test_decisions_media_deployment_flag_wire.py diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index ae01aa8d0f7..871fa8227ea 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -3,10 +3,10 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass, field, replace from functools import partial from types import MappingProxyType -from typing import Final, Literal, TypeAlias +from typing import Annotated, Final, Literal, TypeAlias import httpx -from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError +from pydantic import BaseModel, BeforeValidator, ConfigDict, TypeAdapter, ValidationError from pydantic_core import to_jsonable_python from typing_extensions import assert_never @@ -23,6 +23,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.decisions.transformation import ( BaseDecisionsConfig, DecisionsResponseValidationError, + catalog_mode, image_media_type, image_param, input_images, @@ -61,7 +62,7 @@ from litellm.types.decisions import ( UnsupportedDecisionsRequest, ) from litellm.types.llms.openai import ChatCompletionUserMessage -from litellm.types.utils import LlmProviders, ProviderSpecificHeader +from litellm.types.utils import LlmProviders, ProviderSpecificHeader, is_decisions_model_mode from litellm.utils import ProviderConfigManager, get_model_cost_key SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]] @@ -257,19 +258,105 @@ def _request_safety_identifier( return None -def _fields_without_unsupported_media( - fields: RequestFields, +def _bool_or_none(value: object) -> bool | None: + return value if isinstance(value, bool) else None + + +def _str_or_none(value: object) -> str | None: + return value if isinstance(value, str) else None + + +_DeploymentFlag: TypeAlias = Annotated[bool | None, BeforeValidator(_bool_or_none)] + + +class _DeploymentCapabilities(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + mode: Annotated[str | None, BeforeValidator(_str_or_none)] = None + supports_box_questions: _DeploymentFlag = None + supports_audio_input: _DeploymentFlag = None + supports_video_input: _DeploymentFlag = None + + def decisions_media_flag(self, flag: bool | None, model: str, custom_llm_provider: str) -> bool | None: + if flag is None: + return None + mode: Final = self.mode if self.mode is not None else catalog_mode(model, custom_llm_provider) + return flag if mode is None or is_decisions_model_mode(mode) else None + + +@dataclass(frozen=True, slots=True) +class _ResolvedSupport: + box_questions: bool + audio: bool + videos: bool + + +def _takes_audio( + provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities +) -> bool: + return provider_config.supports_audio_input( + model, + custom_llm_provider, + deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider), + ) + + +def _takes_videos( + provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities +) -> bool: + return provider_config.supports_video_input( + model, + custom_llm_provider, + deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider), + ) + + +def deployment_refuses_media( + model: str, + custom_llm_provider: str, + model_info: Mapping[str, object], + call_kwargs: Mapping[str, object], + *, + audio: bool, + videos: bool, +) -> bool: + if _drops_params(call_kwargs): + return False + provider_config: Final = get_decisions_provider_config(model, custom_llm_provider) + canonical_model: Final = provider_config.canonical_model(model) + deployment: Final = _DeploymentCapabilities.model_validate(model_info) + return (audio and not _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) or ( + videos and not _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment) + ) + + +def _resolved_support( provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, + headers: Mapping[str, str], kwargs: Mapping[str, object], + fields: RequestFields, +) -> _ResolvedSupport: + deployment: Final = _DeploymentCapabilities.model_validate(kwargs.get("model_info") or {}) + return _ResolvedSupport( + box_questions=provider_config.supports_box_questions( + model, custom_llm_provider, headers, deployment.supports_box_questions + ), + audio=bool(fields.audio) and _takes_audio(provider_config, model, custom_llm_provider, deployment), + videos=bool(fields.videos) and _takes_videos(provider_config, model, custom_llm_provider, deployment), + ) + + +def _fields_without_unsupported_media( + fields: RequestFields, support: _ResolvedSupport, kwargs: Mapping[str, object] ) -> RequestFields: if not _drops_params(kwargs): return fields return replace( fields, - audio=fields.audio if provider_config.supports_audio_input(model, custom_llm_provider) else None, - videos=fields.videos if provider_config.supports_video_input(model, custom_llm_provider) else None, + audio=fields.audio if support.audio else None, + videos=fields.videos if support.videos else None, ) @@ -292,10 +379,8 @@ class _MalformedImages: def _unsupported_params( ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig, - model: str, - custom_llm_provider: str, headers: Mapping[str, str], - supports_box_questions: bool, + support: _ResolvedSupport, ) -> tuple[str, ...]: refuses_safety_identifier: Final = ( ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier @@ -304,13 +389,9 @@ def _unsupported_params( None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input) ) refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers) - refuses_box_questions: Final = _has_box_question(ir_request.questions) and not supports_box_questions - refuses_audio: Final = bool(ir_request.audio) and not provider_config.supports_audio_input( - model, custom_llm_provider - ) - refuses_videos: Final = bool(ir_request.videos) and not provider_config.supports_video_input( - model, custom_llm_provider - ) + refuses_box_questions: Final = _has_box_question(ir_request.questions) and not support.box_questions + refuses_audio: Final = bool(ir_request.audio) and not support.audio + refuses_videos: Final = bool(ir_request.videos) and not support.videos return ( *(("safety_identifier",) if refuses_safety_identifier else ()), *(() if refused_image_param is None else (refused_image_param,)), @@ -333,7 +414,7 @@ def _without_unsupported_params( ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig, headers: Mapping[str, str], - supports_box_questions: bool, + support: _ResolvedSupport, ) -> DecisionsIRRequest: return replace( ir_request, @@ -345,9 +426,7 @@ def _without_unsupported_params( safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None, assets=ir_request.assets if provider_config.supports_assets(headers) else MappingProxyType({}), questions=( - ir_request.questions - if supports_box_questions - else tuple(filter(_is_not_box_question, ir_request.questions)) + ir_request.questions if support.box_questions else tuple(filter(_is_not_box_question, ir_request.questions)) ), ) @@ -368,39 +447,20 @@ def _without_malformed_images( return well_formed -class _DeploymentBoxQuestions(BaseModel): - model_config = ConfigDict(frozen=True, from_attributes=True) - - supports_box_questions: StrictBool | None = None - - -def _deployment_supports_box_questions(kwargs: Mapping[str, object]) -> bool | None: - try: - return _DeploymentBoxQuestions.model_validate(kwargs.get("model_info") or {}).supports_box_questions - except ValidationError: - return None - - def _provider_ir_request( ir_request: DecisionsIRRequest, *, provider_config: BaseDecisionsConfig, - model: str, - custom_llm_provider: str, headers: Mapping[str, str], + support: _ResolvedSupport, kwargs: Mapping[str, object], ) -> DecisionsIRRequest | _UnsupportedParams | _NothingLeftAfterDrop: - supports_box_questions: Final = provider_config.supports_box_questions( - model, custom_llm_provider, headers, _deployment_supports_box_questions(kwargs) - ) - unsupported: Final = _unsupported_params( - ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions - ) + unsupported: Final = _unsupported_params(ir_request, provider_config, headers, support) if not unsupported: return ir_request if not _drops_params(kwargs): return _UnsupportedParams(names=unsupported) - supported: Final = _without_unsupported_params(ir_request, provider_config, headers, supports_box_questions) + supported: Final = _without_unsupported_params(ir_request, provider_config, headers, support) if _leaves_no_input(supported.input): return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on") if not supported.questions: @@ -479,10 +539,11 @@ def prepare_call( raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error try: headers: Final = decisions_outbound_headers(extra_headers, provider, kwargs) + support: Final = _resolved_support(provider_config, canonical_model, provider, headers, kwargs, fields) request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs) - request: Final = _fields_without_unsupported_media( - fields, provider_config, canonical_model, provider, kwargs - ).validate(safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs)) + request: Final = _fields_without_unsupported_media(fields, support, kwargs).validate( + safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs) + ) except ValidationError as error: raise litellm.BadRequestError( message=f"Invalid Decisions request: {error}", @@ -523,12 +584,7 @@ def prepare_call( llm_provider=provider, ) ir_request: Final = _provider_ir_request( - caller_ir_request, - provider_config=provider_config, - model=canonical_model, - custom_llm_provider=provider, - headers=headers, - kwargs=kwargs, + caller_ir_request, provider_config=provider_config, headers=headers, support=support, kwargs=kwargs ) if isinstance(ir_request, _UnsupportedParams): raise litellm.UnsupportedParamsError( diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index efd3e3fe9b5..c562c8d0f04 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -12,6 +12,7 @@ from pydantic_core import ErrorDetails from typing_extensions import assert_never import litellm +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str @@ -297,13 +298,7 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]: assert_never(question) -def ir_to_systemone_request( - model: str, request: DecisionsIRRequest, *, supports_audio: bool = False, supports_video: bool = False -) -> Mapping[str, object] | UnsupportedDecisionsRequest: - if request.audio and not supports_audio: - return UnsupportedDecisionsRequest(reason="audio input is not supported by this Decisions model") - if request.videos and not supports_video: - return UnsupportedDecisionsRequest(reason="video input is not supported by this Decisions model") +def ir_to_systemone_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object]: state: Final = _systemone_state(request.input) keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True) images: Final = input_images(request.input) @@ -503,6 +498,16 @@ def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIR ) +def catalog_mode(model: str, custom_llm_provider: str) -> str | None: + try: + key: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)["key"] + except Exception: # noqa: BLE001 # get_model_info raises bare Exception for an unmapped model, which the catalog does not list + return None + entry: Final = GetModelCostMap.loaded_model_cost_map().get(key) + mode: Final = entry.get("mode") if entry is not None else None + return mode if isinstance(mode, str) else None + + def _is_decisions_model(model: str, custom_llm_provider: str) -> bool: try: model_mode: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider).get("mode") @@ -640,12 +645,16 @@ class BaseDecisionsConfig(ABC): model=model, custom_llm_provider=custom_llm_provider ) - def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool: + def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool: + if deployment_flag is not None: + return deployment_flag return _is_decisions_model(model, custom_llm_provider) and litellm.supports_audio_input( model=model, custom_llm_provider=custom_llm_provider ) - def supports_video_input(self, model: str, custom_llm_provider: str) -> bool: + def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool: + if deployment_flag is not None: + return deployment_flag return _is_decisions_model(model, custom_llm_provider) and litellm.supports_video_input( model=model, custom_llm_provider=custom_llm_provider ) @@ -656,12 +665,7 @@ class BaseDecisionsConfig(ABC): request: DecisionsIRRequest, custom_llm_provider: str, ) -> Mapping[str, object] | UnsupportedDecisionsRequest: - return ir_to_systemone_request( - self.request_model(model), - request, - supports_audio=self.supports_audio_input(model, custom_llm_provider), - supports_video=self.supports_video_input(model, custom_llm_provider), - ) + return ir_to_systemone_request(self.request_model(model), request) def unwrap_response(self, payload: object) -> object: return payload diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 4518fe6a77e..369b6959411 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -354,10 +354,10 @@ class OpenAIDecisionsConfig(BaseDecisionsConfig): ) -> str | None: return resolve_openai_bearer_token(api_key=api_key, api_base=api_base, litellm_params=litellm_params) - def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool: + def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool: return False - def supports_video_input(self, model: str, custom_llm_provider: str) -> bool: + def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool: return False def transform_decisions_request( diff --git a/litellm/router.py b/litellm/router.py index 535072d5fcf..6392ea8cd90 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -635,6 +635,7 @@ _PRE_CALL_CHECK_FILTERED_PARAMS: Final = frozenset({"response_format"}) _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"}) _video_call_deployment_id: Final[ContextVar[str | None]] = ContextVar("video_call_deployment_id", default=None) _routed_call_type: Final[ContextVar[str | None]] = ContextVar("routed_call_type", default=None) +_DECISIONS_MEDIA_CALL_TYPES: Final = frozenset({"adecisions", "decisions", "asystemone", "systemone"}) _ROUTE_CAPABILITY_RESOLVERS: Final[Mapping[str, Callable[[str, LlmProviders], object | None]]] = MappingProxyType( { "adecisions": ProviderConfigManager.get_provider_decisions_config, @@ -4153,13 +4154,16 @@ class Router(DeprecatedRouterMembers): raise e.original_exception from e raise e + def _active_default_litellm_params(self) -> Mapping[str, object]: + default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params) + return {key: value for key, value in default_params.items() if value is not None} + def _params_sent_to_deployment( self, request_kwargs: Mapping[str, object], deployment: Deployment | None ) -> Mapping[str, object]: """The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies.""" deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {} - default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params) - defaults: Final = {key: value for key, value in default_params.items() if value is not None} + defaults: Final = self._active_default_litellm_params() deployment_tool_choice: Final = ( {"tool_choice": deployment_params["tool_choice"]} if deployment_params.get("tool_choice") is not None @@ -14911,6 +14915,62 @@ class Router(DeprecatedRouterMembers): return healthy_deployments return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates + @staticmethod + def _decisions_deployment_refuses_media( + deployment: DeploymentTypedDict, group: str, request_kwargs: Mapping[str, object], *, audio: bool, videos: bool + ) -> bool: + from litellm.decisions.call import deployment_refuses_media + + plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts + model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict + deployment_params: Final = cast(Mapping[str, object], deployment["litellm_params"]) # cast-ok: plain dict + try: + model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group) + return deployment_refuses_media( + model, + custom_llm_provider, + model_info, + {**deployment_params, **request_kwargs}, + audio=audio, + videos=videos, + ) + except Exception as e: # noqa: BLE001 # best-effort filter: a deployment it cannot resolve answers for itself + verbose_router_logger.debug( + "litellm.router.py::_decisions_deployment_refuses_media: keeping a deployment of model=%s. Got - %s", + group, + e, + ) + return False + + def _without_decisions_deployments_refusing_media( + self, healthy_deployments: _RouteDeploymentsT, group: str, request_kwargs: Mapping[str, object] | None + ) -> _RouteDeploymentsT: + call_kwargs: Final = request_kwargs or {} + audio: Final = bool(call_kwargs.get("audio")) + videos: Final = bool(call_kwargs.get("videos")) + if ( + _routed_call_type.get() not in _DECISIONS_MEDIA_CALL_TYPES + or not (audio or videos) + or isinstance(healthy_deployments, Mapping) + ): + return healthy_deployments + + sent_kwargs: Final = {**self._active_default_litellm_params(), **call_kwargs} + deployments_taking_media: Final = [ + deployment + for deployment in healthy_deployments + if not Router._decisions_deployment_refuses_media( + cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape + group, + sent_kwargs, + audio=audio, + videos=videos, + ) + ] + if not deployments_taking_media: + return healthy_deployments + return cast(_RouteDeploymentsT, deployments_taking_media) # cast-ok: same list shape as the input candidates + @staticmethod def _without_declared_text_only_deployments( healthy_deployments: _RouteDeploymentsT, @@ -15635,10 +15695,14 @@ class Router(DeprecatedRouterMembers): ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) - batch_candidates: Final = self._without_declared_text_only_deployments( - cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + batch_candidates: Final = self._without_decisions_deployments_refusing_media( + self._without_declared_text_only_deployments( + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + model, + messages, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict + ), model, - messages, cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict ) embedding_batch_admitting: Final = deployments_admitting_embedding_batch( @@ -16878,10 +16942,14 @@ class Router(DeprecatedRouterMembers): ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) - batch_candidates: Final = self._without_declared_text_only_deployments( - cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + batch_candidates: Final = self._without_decisions_deployments_refusing_media( + self._without_declared_text_only_deployments( + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + model, + messages, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict + ), model, - messages, cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict ) embedding_batch_admitting: Final = deployments_admitting_embedding_batch( diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 69ad0c91ee9..3a708ce6d0a 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -183,6 +183,9 @@ ignored_function_names = [ "_aresponses_background_stream_can_release", # Tested through the background Responses stream tests in test_router.py "_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py "_mid_stream_error_before_output", # Tested through the Responses mid-stream fallback and background stream tests + "_without_decisions_deployments_refusing_media", # Tested through the mixed media-flag group tests in tests/unit/decisions/test_main.py + "_decisions_deployment_refuses_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests + "_active_default_litellm_params", # Tested through the router default drop_params test in tests/unit/decisions/test_main.py ] diff --git a/tests/integration/providers/test_decisions_media_deployment_flag_wire.py b/tests/integration/providers/test_decisions_media_deployment_flag_wire.py new file mode 100644 index 00000000000..a2e53ef47ca --- /dev/null +++ b/tests/integration/providers/test_decisions_media_deployment_flag_wire.py @@ -0,0 +1,123 @@ +import json +import uuid +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final, Literal + +import httpx +import pytest +from integration._support.client import Gateway, eventually, object_value +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_API_KEY: Final = "synthetic-decisions-key" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_INPUT: Final = "Listen to the attached clip." +_AUDIO: Final = ("data:audio/wav;base64,UklGRiQAAABXQVZF",) +_VIDEOS: Final = ("data:video/mp4;base64,AAAAIGZ0eXBpc29t",) +_QUESTION: Final[dict[str, JsonValue]] = { + "type": "predicate", + "name": "banana", + "instructions": "Does the clip name a banana?", +} +_SPEND_QUERY: Final = 'SELECT status, model_group FROM "LiteLLM_SpendLogs" WHERE request_id = %s' + +Route = Literal["/v1/decisions", "/v1/systemone"] +Media = Literal["audio", "videos"] + + +@dataclass(frozen=True, slots=True) +class _Upstream: + model: str + + def reply(self, request: Request) -> Reply: + questions: Final = object_value(_JSON_OBJECT.validate_json(request.body)["questions"]) + plain: Final[dict[str, JsonValue]] = { + "model": self.model, + "answers": {name: {"type": "noul", "noul": 0.9} for name in questions}, + "usage": {"input_tokens": 41, "output_tokens": 0}, + } + return Reply(body=json.dumps(plain).encode()) + + +def _unlisted_openrouter_model() -> _Upstream: + return _Upstream(f"openrouter/example/omni-{uuid.uuid4().hex[:12]}") + + +def _decide(gateway: Gateway, model: str, route: Route, media: Mapping[str, JsonValue]) -> httpx.Response: + body: Final[dict[str, JsonValue]] = ( + {"model": model, "input": _INPUT, "questions": [_QUESTION], "num_retries": 0, **media} + if route == "/v1/decisions" + else { + "model": model, + "state": _INPUT, + "questions": {"banana": {"type": "noul", "instructions": _QUESTION["instructions"]}}, + "num_retries": 0, + **media, + } + ) + return gateway.request("POST", route, body) + + +def _sent_media(request: Request) -> dict[str, JsonValue]: + body: Final = _JSON_OBJECT.validate_json(request.body) + return {kind: body[kind] for kind in ("audio", "videos") if kind in body} + + +def _assert_refused(response: httpx.Response, kinds: tuple[Media, ...]) -> None: + assert response.status_code == 400, response.text + assert f"does not support parameters: {list(kinds)}" in response.text, response.text + + +_BOTH: Final[dict[str, JsonValue]] = {"audio": list(_AUDIO), "videos": list(_VIDEOS)} + + +@pytest.mark.parametrize("route", ("/v1/decisions", "/v1/systemone")) +def test_a_deployment_flagged_for_audio_and_video_sends_both_clips_for_a_model_the_cost_map_does_not_list( + gateway: Gateway, route: Route +) -> None: + upstream: Final = _unlisted_openrouter_model() + with wire_server(upstream.reply) as wire, gateway.scenario() as scenario: + flags: Final[dict[str, JsonValue]] = {"supports_audio_input": True, "supports_video_input": True} + flagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags) + flagged_dropping: Final = scenario.model( + model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags, drop_params=True + ) + audio_only: Final = scenario.model( + model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info={"supports_audio_input": True} + ) + unflagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY) + + _assert_refused(_decide(gateway, unflagged, route, _BOTH), ("audio", "videos")) + _assert_refused(_decide(gateway, audio_only, route, _BOTH), ("videos",)) + assert wire.drain() == () + + answered: Final = _decide(gateway, flagged, route, _BOTH) + assert answered.status_code == 200, answered.text + (call,) = wire.drain() + assert _sent_media(call) == _BOTH + rows: Final = eventually( + lambda: read_rows(_SPEND_QUERY, (answered.headers["x-litellm-call-id"],)), + lambda found: len(found) == 1, + seconds=70, + ) + assert (rows[0]["status"], rows[0]["model_group"]) == ("success", flagged) + + kept: Final = _decide(gateway, flagged_dropping, route, _BOTH) + assert kept.status_code == 200, kept.text + (kept_call,) = wire.drain() + assert _sent_media(kept_call) == _BOTH + + +@pytest.mark.parametrize( + "value", + ({"supports_audio_input": True, "supports_video_input": True}, 5, [{"supports_audio_input": True}], "x" * 5000), + ids=("flags", "int", "list", "5kb"), +) +def test_a_caller_sent_model_info_never_turns_media_on(gateway: Gateway, value: JsonValue) -> None: + upstream: Final = _unlisted_openrouter_model() + with wire_server(upstream.reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY) + _assert_refused(_decide(gateway, model, "/v1/decisions", {**_BOTH, "model_info": value}), ("audio", "videos")) + assert wire.drain() == () diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 0c357837d31..789d59bd613 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -6,10 +6,11 @@ import copy import json import re import threading -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from concurrent.futures import wait +from functools import partial from types import MappingProxyType -from typing import Final +from typing import Final, TypeAlias import httpx import pytest @@ -23,6 +24,7 @@ import litellm from litellm.constants import AWS_SIGNING_MAX_THREADS from litellm.cost_calculator import get_response_cost_from_hidden_params from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded, refetch_model_cost_map from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers from litellm.llms.bedrock.base_aws_llm import AWS_SIGNING_EXECUTOR @@ -3702,6 +3704,387 @@ async def test_a_model_info_object_instead_of_a_dict_still_reaches_the_provider( assert response.answers[0].model_dump(mode="json")["name"] == "is_defect" +_AUDIO_CLIPS: Final = ("data:audio/wav;base64,YQ==",) +_VIDEO_CLIPS: Final = ("data:video/mp4;base64,YQ==",) + + +@pytest.mark.asyncio +async def test_a_deployment_that_advertises_media_sends_it_for_a_model_the_cost_map_does_not_list( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + backend: Final = "openrouter/example/omni-preview" + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "hears", + "litellm_params": {"model": backend, "api_key": "k"}, + "model_info": {"id": "media-hears", "supports_audio_input": True}, + }, + { + "model_name": "hears-and-sees", + "litellm_params": {"model": backend, "api_key": "k"}, + "model_info": {"id": "media-both", "supports_audio_input": True, "supports_video_input": True}, + }, + ] + ) + send: Final = partial( + router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS + ) + + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['videos']")): + await send(model="hears") + assert not route.called + + await send(model="hears-and-sees") + + sent: Final = json.loads(route.calls.last.request.content) + assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS)) + + +@pytest.mark.asyncio +async def test_media_flags_on_a_deployment_whose_own_mode_is_not_decisions_never_send_clips( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + advertised: Final = {"supports_audio_input": True, "supports_video_input": True} + router: Final = litellm.Router( + model_list=[ + { + "model_name": mode, + "litellm_params": {"model": f"openrouter/example/omni-{mode}-preview", "api_key": "k"}, + "model_info": {"id": f"media-{mode}", "mode": mode, **advertised}, + } + for mode in ("chat", "decisions") + ] + ) + send: Final = partial( + router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS + ) + + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")): + await send(model="chat") + assert not route.called + + await send(model="decisions") + + sent: Final = json.loads(route.calls.last.request.content) + assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS)) + + +CatalogLoader: TypeAlias = Callable[[Mapping[str, Mapping[str, object]]], Awaitable[None]] + + +@pytest.fixture +def load_catalog_with(monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None) -> Iterator[CatalogLoader]: + async def load(rows: Mapping[str, Mapping[str, object]]) -> None: + catalog: Final = {**litellm.get_model_cost_map(url=""), **rows} + transport: Final = httpx.MockTransport(lambda _: httpx.Response(200, json=catalog)) + monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP") + async with httpx.AsyncClient(transport=transport) as client: + loaded: Final = await refetch_model_cost_map(url="https://catalog.example/prices.json", client=client) + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + assert isinstance(loaded, ModelCostMapReloaded) + monkeypatch.setattr(litellm, "model_cost", loaded.model_cost_map) + litellm.get_model_info.cache_clear() + + yield load + litellm.get_model_cost_map(url="") + + +@pytest.mark.asyncio +async def test_media_flags_on_a_deployment_without_its_own_mode_follow_the_mode_the_catalog_lists( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, load_catalog_with: CatalogLoader +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + await load_catalog_with( + { + f"openrouter/example/omni-{mode}": {"litellm_provider": "openrouter", "mode": mode} + for mode in ("chat", "decisions") + } + ) + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + router: Final = litellm.Router( + model_list=[ + { + "model_name": mode, + "litellm_params": {"model": f"openrouter/example/omni-{mode}", "api_key": "k"}, + "model_info": {"id": f"listed-{mode}", "supports_audio_input": True, "supports_video_input": True}, + } + for mode in ("chat", "decisions") + ] + ) + send: Final = partial( + router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS + ) + + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")): + await send(model="chat") + assert not route.called + + monkeypatch.setattr(litellm, "drop_params", True) + await send(model="chat") + dropped: Final = json.loads(route.calls.last.request.content) + assert ("audio" in dropped, "videos" in dropped) == (False, False) + + await send(model="decisions") + sent: Final = json.loads(route.calls.last.request.content) + assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sibling_mode", ("responses", "embedding")) +async def test_another_deployments_mode_never_switches_off_a_deployments_own_media_flags( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, sibling_mode: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + shared_params: Final = {"model": "openrouter/example/omni-shared", "api_key": "k"} + router: Final = litellm.Router( + model_list=[ + {"model_name": "sibling", "litellm_params": shared_params, "model_info": {"id": "s", "mode": sibling_mode}}, + { + "model_name": "flagged", + "litellm_params": shared_params, + "model_info": {"id": "f", "supports_audio_input": True, "supports_video_input": True}, + }, + ] + ) + + await router.adecisions( + model="flagged", input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS)) + + +_MEDIA_SURFACES: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType( + { + "adecisions": {"input": "review", "questions": _PREDICATE_QUESTIONS}, + "asystemone": {"state": "review", "questions": _QUESTIONS}, + "systemone": {"state": "review", "questions": _QUESTIONS}, + } +) + + +def _group_member( + group: str, model: str, key: str, weight: int, model_info: Mapping[str, object], **params: object +) -> dict[str, object]: + return { + "model_name": group, + "litellm_params": {"model": model, "api_key": key, "weight": weight, **params}, + "model_info": {"id": f"{group}-{key}", **model_info}, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES)) +async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_only_when_none_does( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + monkeypatch.setitem( + litellm.model_cost, + "cloudflare/@cf/cloudflare/clef-omni", + {"litellm_provider": "cloudflare", "mode": "evaluation", "supports_audio_input": True}, + ) + respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + respx_mock.post("https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-omni").respond( + json=_RESPONSE + ) + unlisted: Final = "openrouter/example/omni-preview" + listed: Final = "cloudflare/clef-omni" + router: Final = litellm.Router( + model_list=[ + _group_member("unlisted", unlisted, "refuses", 1, {}), + _group_member("unlisted", unlisted, "takes", 0, {"supports_audio_input": True}), + _group_member("listed", listed, "refuses", 1, {"supports_audio_input": False}), + _group_member("listed", listed, "takes", 0, {}), + _group_member("none", unlisted, "refuses", 1, {}), + _group_member("none", listed, "refuses", 1, {"supports_audio_input": False}), + _group_member("drops", unlisted, "drops", 1, {}, drop_params=True), + _group_member("drops", listed, "takes", 0, {}), + _group_member("unresolvable", "databricks/not a serving endpoint", "unresolvable", 0, {}), + _group_member("unresolvable", listed, "takes", 1, {}), + ] + ) + call: Final = getattr(router, surface) + + async def send(group: str, **media: object) -> object: + response: Final = call(model=group, **_MEDIA_SURFACES[surface], **media) + return await response if asyncio.iscoroutine(response) else response + + for group in ("unlisted", "listed", "unresolvable"): + await send(group, audio=_AUDIO_CLIPS) + assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes" + assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS) + + await send("drops", audio=_AUDIO_CLIPS) + assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops" + assert "audio" not in json.loads(respx_mock.calls.last.request.content) + + await send("unlisted") + assert respx_mock.calls.last.request.headers["authorization"] == "Bearer refuses" + + sent_before_refusal: Final = len(respx_mock.calls) + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")): + await send("none", audio=_AUDIO_CLIPS) + assert len(respx_mock.calls) == sent_before_refusal + + +def _answer_unless_key_is(rejected_key: str) -> Callable[[httpx.Request], httpx.Response]: + def answer(request: httpx.Request) -> httpx.Response: + if request.headers["authorization"] == f"Bearer {rejected_key}": + return httpx.Response(401, json={"error": {"message": "invalid key"}}) + return httpx.Response(200, json=_RESPONSE) + + return answer + + +@pytest.mark.asyncio +async def test_clips_to_a_group_whose_only_taker_is_cooling_down_are_refused_rather_than_rate_limited( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").mock( + side_effect=_answer_unless_key_is("takes") + ) + unlisted: Final = "openrouter/example/omni-preview" + router: Final = litellm.Router( + model_list=[ + _group_member("g", unlisted, "refuses", 1, {}), + _group_member("g", unlisted, "takes", 1, {"supports_audio_input": True}), + ], + num_retries=0, + cooldown_time=120, + ) + with pytest.raises(litellm.AuthenticationError): + await router.adecisions(model="g-takes", input="review", questions=_PREDICATE_QUESTIONS) + sent_before_clips: Final = len(route.calls) + + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")): + await router.adecisions(model="g", input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS) + + assert len(route.calls) == sent_before_clips + + +@pytest.mark.asyncio +@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES)) +async def test_a_router_default_drop_params_overrides_a_deployment_when_choosing_who_takes_clips( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + unlisted: Final = "openrouter/example/omni-preview" + router: Final = litellm.Router( + model_list=[ + _group_member("g", unlisted, "drops", 1, {}, drop_params=True), + _group_member("g", unlisted, "takes", 0, {"supports_audio_input": True}), + ], + default_litellm_params={"drop_params": False}, + ) + call: Final = getattr(router, surface) + + async def send(**params: object) -> object: + response: Final = call(model="g", **_MEDIA_SURFACES[surface], audio=_AUDIO_CLIPS, **params) + return await response if asyncio.iscoroutine(response) else response + + await send() + assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes" + assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS) + + await send(drop_params=True) + assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops" + assert "audio" not in json.loads(respx_mock.calls.last.request.content) + + +@pytest.mark.asyncio +async def test_a_deployment_that_turns_audio_off_is_refused_or_dropped_even_when_the_cost_map_allows_it( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setitem( + litellm.model_cost, + "typesafe/jev-omni", + { + "litellm_provider": "typesafe", + "mode": "decisions", + "supports_audio_input": True, + "supports_video_input": True, + }, + ) + route: Final = respx_mock.post("https://example.com/v1/systemone").respond(json=_RESPONSE) + send: Final = partial( + litellm.asystemone, + model="typesafe/jev-omni", + state="test", + questions=_QUESTIONS, + api_base="https://example.com", + api_key="test", + audio=_AUDIO_CLIPS, + videos=_VIDEO_CLIPS, + model_info={"supports_audio_input": False}, + ) + + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio']")): + await send() + assert not route.called + + await send(drop_params=True) + + sent: Final = json.loads(route.calls.last.request.content) + assert "audio" not in sent + assert sent["videos"] == list(_VIDEO_CLIPS) + + +@pytest.mark.asyncio +async def test_a_media_flag_that_is_not_a_boolean_leaves_the_cost_map_in_charge_of_that_kind_only( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://example.com/v1/systemone").respond(json=_RESPONSE) + + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio']")): + await litellm.asystemone( + model="typesafe/jev-unlisted", + state="test", + questions=_QUESTIONS, + api_base="https://example.com", + api_key="test", + audio=_AUDIO_CLIPS, + videos=_VIDEO_CLIPS, + model_info={"supports_audio_input": "true", "supports_video_input": True}, + ) + + assert not route.called + + +@pytest.mark.asyncio +async def test_an_openai_format_deployment_that_advertises_media_is_still_refused_it( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) + + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")): + await litellm.adecisions( + model="openai/gpt-6-luna", + input="review", + questions=_PREDICATE_QUESTIONS, + api_key="deployment-key", + audio=_AUDIO_CLIPS, + videos=_VIDEO_CLIPS, + model_info={"supports_audio_input": True, "supports_video_input": True}, + ) + + assert not route.called + + def test_dropping_every_question_answers_400_before_http(respx_mock: respx.MockRouter) -> None: route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py index ec6f118837f..041c625eaef 100644 --- a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py +++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py @@ -26,7 +26,6 @@ from litellm.types.decisions import ( DecisionsIRScoreAnswer, DecisionsRequestBody, OpenAIDecisionRequestBody, - UnsupportedDecisionsRequest, ) from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager @@ -279,7 +278,7 @@ def test_a_malformed_systemone_response_names_the_field_at_fault( assert [item["loc"] for item in error.value.errors()] == [field] -def test_each_media_kind_is_gated_by_its_own_support_flag() -> None: +def test_each_media_kind_is_sent_in_its_own_field() -> None: audio: Final = ["data:audio/wav;base64,YQ=="] videos: Final = ["data:video/mp4;base64,YQ=="] audio_only: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "audio": audio})) @@ -287,22 +286,16 @@ def test_each_media_kind_is_gated_by_its_own_support_flag() -> None: _SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "videos": videos}) ) - assert ir_to_systemone_request("clef-omni", audio_only, supports_audio=True) == { + assert ir_to_systemone_request("clef-omni", audio_only) == { "model": "clef-omni", **_SYSTEMONE_REQUEST, "audio": audio, } - assert ir_to_systemone_request("clef-omni", video_only, supports_video=True) == { + assert ir_to_systemone_request("clef-omni", video_only) == { "model": "clef-omni", **_SYSTEMONE_REQUEST, "videos": videos, } - assert isinstance( - ir_to_systemone_request("clef-omni", audio_only, supports_video=True), UnsupportedDecisionsRequest - ) - assert isinstance( - ir_to_systemone_request("clef-omni", video_only, supports_audio=True), UnsupportedDecisionsRequest - ) _ASSET_REQUEST: Final[Mapping[str, object]] = { diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index 20c0e10cb81..38c69e0ce27 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -1,5 +1,5 @@ -import asyncio, importlib, litellm, litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map, pytest +import asyncio, importlib, litellm, pytest from typing import Final @@ -1563,7 +1563,6 @@ def _pr4_bedrock_env(monkeypatch: pytest.MonkeyPatch) -> None: def _use_local_model_cost_map() -> Iterator[None]: with pytest.MonkeyPatch.context() as monkeypatch: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - importlib.reload(bedrock_govcloud_model_cost_map) importlib.reload(litellm) yield From da71a03f6efc34410c067a6ecec6e95e80ccce54 Mon Sep 17 00:00:00 2001 From: Wiley Kestner Date: Sat, 31 Oct 2026 09:22:45 +0100 Subject: [PATCH 15/15] fix(proxy): trace model list requests with the caller's traceparent (#49141) * fix(proxy): trace model list requests with the caller's traceparent The proxy creates an OTEL SERVER span at the auth boundary whenever a request carries traceparent, but /v1/models never closed it. Unended spans are not exported, so callers saw nothing for the model list while inference requests showed up in the same trace Route the endpoint through the management span wrapper so the SERVER span is closed and parented to the caller's trace. The wrapper gains a variant that skips request and response payloads, since attaching the full model list would bloat every poll * refactor(proxy): type the span wrapper with ParamSpec The strict ruff rules ban new typing.cast imports, so the wrapper keeps the wrapped endpoint's exact signature via ParamSpec instead of casting the wrapped callable back to its type. No behavior change * fix(proxy): keep request bodies off the model list span and tidy its tests The quiet span wrapper now skips the request body as well as the response, so no payload reaches the span on success or failure. The wrapper passes the endpoint's kwargs through instead of copying them, and reads the caller's key with an isinstance check. The model list test drives the real endpoint without swapping helpers, and the new payload test checks both wrappers against a real request body * fix(proxy): keep the 5xx status on the model list span and trace model retrieve The span wrapper ended the server span on every failure, so a 500 or 503 from /v1/models lost its status code. It now leaves that span to the proxy's exception handler, which knows the answered status, and only ends it itself for an HTTPException, which no handler closes on these routes. GET /v1/models/{model_id} (the OpenAI SDK's models.retrieve) gets the same wrapper, so it joins the caller's trace like the listing does. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/management_helpers/utils.py | 56 +++-- litellm/proxy/proxy_server.py | 3 + .../test_otel_admin_endpoints.py | 204 ++++++++++++++++++ 3 files changed, 248 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index 30aaaacad04..ea19befccc6 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -1,14 +1,15 @@ # What is this? ## Helper utils for the management endpoints (keys/users/teams) -from collections.abc import Callable, Mapping, MutableMapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, MutableMapping, Sequence from collections.abc import Set as AbstractSet from datetime import datetime from functools import wraps from types import MappingProxyType -from typing import Any, Final, Protocol +from typing import Any, Final, ParamSpec, Protocol, TypeVar from fastapi import HTTPException, Request from pydantic import BaseModel, TypeAdapter +from starlette.exceptions import HTTPException as StarletteHTTPException import litellm from litellm._internal_context import with_service_target @@ -687,6 +688,7 @@ async def _emit_management_endpoint_otel_span( end_time: datetime, result: Any = None, exception: Exception | None = None, + log_payload: bool = True, ) -> None: """Stamp + end the parent OTEL SERVER span for a management endpoint. @@ -716,8 +718,8 @@ async def _emit_management_endpoint_otel_span( ) route = get_request_route(http_request) - request_body: dict = await read_request_body( # rebind-ok: pre-existing rebinding on a rename-only line - request=http_request + request_body: dict = ( # rebind-ok: pre-existing rebinding on a rename-only line + await read_request_body(request=http_request) if log_payload else {} ) else: route = func.__name__ @@ -738,7 +740,7 @@ async def _emit_management_endpoint_otel_span( ) _response: dict[str, object] | None = None - if exception is None and result is not None: + if log_payload and exception is None and result is not None: try: raw: Final[Mapping[str, object]] = dict(result) _response = {k: v for k, v in raw.items() if k not in _CREDENTIAL_FIELDS} @@ -767,37 +769,60 @@ async def _emit_management_endpoint_otel_span( ) -def management_endpoint_wrapper(func): +_P: Final = ParamSpec("_P") +_R: Final = TypeVar("_R") + + +def management_endpoint_wrapper(func: Callable[_P, Awaitable[_R]]) -> Callable[_P, Awaitable[_R]]: """ This wrapper does the following: 1. Log I/O, Exceptions to OTEL 2. Create an Audit log for success calls """ + return _wrap_management_endpoint(func, log_payload=True) - @wraps(func) - async def wrapper(*args, **kwargs): + +def management_endpoint_span_wrapper(func: Callable[_P, Awaitable[_R]]) -> Callable[_P, Awaitable[_R]]: + return _wrap_management_endpoint(func, log_payload=False) + + +def _caller_auth(endpoint_kwargs: Mapping[str, object]) -> UserAPIKeyAuth: + caller: Final = endpoint_kwargs.get("user_api_key_dict") + return caller if isinstance(caller, UserAPIKeyAuth) else UserAPIKeyAuth() + + +def _proxy_exception_handler_ends_server_span(exception: Exception) -> bool: + return not isinstance(exception, StarletteHTTPException) + + +def _wrap_management_endpoint( + endpoint: Callable[_P, Awaitable[_R]], *, log_payload: bool +) -> Callable[_P, Awaitable[_R]]: + @wraps(endpoint) + async def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: start_time: Final = datetime.now() try: - result: Final = await func(*args, **kwargs) + result: Final = await endpoint(*args, **kwargs) end_time = datetime.now() try: - user_api_key_dict: UserAPIKeyAuth = kwargs.get("user_api_key_dict") or UserAPIKeyAuth() + user_api_key_dict: UserAPIKeyAuth = _caller_auth(kwargs) await send_management_endpoint_alert( request_kwargs=kwargs, user_api_key_dict=user_api_key_dict, - function_name=func.__name__, + function_name=endpoint.__name__, ) parent_otel_span: Span | None = getattr(user_api_key_dict, "parent_otel_span", None) if parent_otel_span is not None: await _emit_management_endpoint_otel_span( - func=func, + func=endpoint, kwargs=kwargs, parent_otel_span=parent_otel_span, start_time=start_time, end_time=end_time, result=result, + log_payload=log_payload, ) # Delete updated/deleted info from cache @@ -813,17 +838,18 @@ def management_endpoint_wrapper(func): except Exception as e: end_time = datetime.now() - user_api_key_dict: UserAPIKeyAuth = kwargs.get("user_api_key_dict") or UserAPIKeyAuth() + user_api_key_dict: UserAPIKeyAuth = _caller_auth(kwargs) parent_otel_span = getattr(user_api_key_dict, "parent_otel_span", None) - if parent_otel_span is not None: + if parent_otel_span is not None and (log_payload or not _proxy_exception_handler_ends_server_span(e)): try: await _emit_management_endpoint_otel_span( - func=func, + func=endpoint, kwargs=kwargs, parent_otel_span=parent_otel_span, start_time=start_time, end_time=end_time, exception=e, + log_payload=log_payload, ) except Exception as otel_exc: # Non-Blocking Exception - never let OTEL failures swallow diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a767f5423ed..7d4a42cf025 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -181,6 +181,7 @@ from litellm.proxy.common_utils.token_count_settings import ( ) from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog from litellm.proxy.management_helpers.model_group_custom_fields import model_group_custom_fields +from litellm.proxy.management_helpers.utils import management_endpoint_span_wrapper from litellm.repositories import database as database_runtime from litellm.router_strategy.auto_router.auto_router_model_naming import ( GATED_AUTO_ROUTER_CAPABILITIES, @@ -12339,6 +12340,7 @@ async def _deployment_hidden_by_listing_callbacks(deployment: Deployment, user_a @router.get( "/models", dependencies=[Depends(user_api_key_auth)], tags=["model management"] ) # if project requires model list +@management_endpoint_span_wrapper async def model_list( request: Request = None, # pyright: ignore[reportArgumentType] # FastAPI always injects the Request; the None default only serves direct in-process callers user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -12649,6 +12651,7 @@ async def model_list( dependencies=[Depends(user_api_key_auth)], tags=["model management"], ) +@management_endpoint_span_wrapper async def model_info( model_id: str, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), diff --git a/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py b/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py index ebeb514ffff..1965fba94c6 100644 --- a/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py +++ b/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py @@ -3,10 +3,14 @@ async_management_endpoint_{success,failure}_hook integration points.""" import asyncio from datetime import datetime +from typing import Final from unittest.mock import MagicMock +import httpx import pytest +from fastapi import Request +from litellm import Router from litellm.proxy._types import ( ManagementEndpointLoggingPayload, UserAPIKeyAuth, @@ -340,3 +344,203 @@ def test_management_wrapper_swallows_post_success_errors(server_span_factory, ot result = asyncio.run(fake_fn(data={}, user_api_key_dict=_real_user_api_key_dict(server_span))) assert result == {"ok": True} + + +TEAM_NEW_PATH: Final = "/team/new" +MODEL_LIST_PATH: Final = "/v1/models" +MODEL_RETRIEVE_PATH: Final = "/v1/models/gpt-4o" +INBOUND_TRACEPARENT: Final = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" + + +def _request(method: str, path: str, body: bytes = b"") -> Request: + async def receive() -> dict[str, object]: + return {"type": "http.request", "body": body, "more_body": False} + + return Request( + { + "type": "http", + "method": method, + "path": path, + "root_path": "", + "query_string": b"", + "headers": [(b"content-type", b"application/json")], + }, + receive, + ) + + +async def _answer_like_the_proxy(request: Request, exception: Exception) -> None: + from litellm.proxy.proxy_server import app + + handler: Final = next( + app.exception_handlers[cls] for cls in type(exception).__mro__ if cls in app.exception_handlers + ) + await handler(request, exception) + + +@pytest.mark.parametrize( + ("wrapper_name", "exports_payload"), + [("management_endpoint_wrapper", True), ("management_endpoint_span_wrapper", False)], +) +def test_span_wrapper_omits_payload(server_span_factory, otel_with_exporter, monkeypatch, wrapper_name, exports_payload): + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_helpers import utils as mgmt_utils + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + + server_span: Final = server_span_factory(TEAM_NEW_PATH) + wrap: Final = getattr(mgmt_utils, wrapper_name) + + @wrap + async def fake_fn(http_request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: + return {"data": [{"id": "gpt-4o"}]} + + asyncio.run( + fake_fn( + http_request=_request("POST", TEAM_NEW_PATH, b'{"team_alias": "t"}'), + user_api_key_dict=_real_user_api_key_dict(server_span), + ) + ) + + spans: Final = [s for s in exporter.get_finished_spans() if s.name == TEAM_NEW_PATH] + assert len(spans) == 1 + attribute_keys: Final = tuple(spans[0].attributes) + assert any(key.startswith("request.") for key in attribute_keys) is exports_payload + assert any(key.startswith("response.") for key in attribute_keys) is exports_payload + + +@pytest.mark.parametrize( + ("wrapper_name", "exports_request_body"), + [("management_endpoint_wrapper", True), ("management_endpoint_span_wrapper", False)], +) +def test_span_wrapper_omits_request_body_on_failure( + server_span_factory, otel_with_exporter, monkeypatch, wrapper_name, exports_request_body +): + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_helpers import utils as mgmt_utils + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + + server_span: Final = server_span_factory(TEAM_NEW_PATH) + wrap: Final = getattr(mgmt_utils, wrapper_name) + + @wrap + async def fake_fn(http_request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: + raise make_fastapi_http_exception(400, "team_alias taken") + + with pytest.raises(Exception, match="team_alias taken"): + asyncio.run( + fake_fn( + http_request=_request("POST", TEAM_NEW_PATH, b'{"team_alias": "t"}'), + user_api_key_dict=_real_user_api_key_dict(server_span), + ) + ) + + spans: Final = [s for s in exporter.get_finished_spans() if s.name == TEAM_NEW_PATH] + assert len(spans) == 1 + assert any(key.startswith("request.") for key in spans[0].attributes) is exports_request_body + + +@pytest.mark.parametrize( + ("raised", "status_code"), + [ + (RuntimeError("router settings lookup failed"), 500), + (httpx.ConnectError("All connection attempts failed"), 503), + (make_fastapi_http_exception(400, "Invalid scope parameter"), 400), + ], +) +def test_span_wrapper_failure_keeps_the_answered_status(otel_with_exporter, monkeypatch, raised, status_code): + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_helpers import utils as mgmt_utils + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + + server_span: Final = _server_span(otel, MODEL_LIST_PATH, headers={}) + request: Final = _request("GET", MODEL_LIST_PATH) + request.state.parent_otel_span = server_span + + @mgmt_utils.management_endpoint_span_wrapper + async def failing_model_list(request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: + raise raised + + async def call_and_answer() -> None: + try: + await failing_model_list(request=request, user_api_key_dict=_real_user_api_key_dict(server_span)) + except Exception as exc: + await _answer_like_the_proxy(request, exc) + + asyncio.run(call_and_answer()) + + assert_server_span_attrs( + exporter, + expected_status=status_code, + expected_url_path=MODEL_LIST_PATH, + where=f"model list {type(raised).__name__}", + ) + exported: Final = get_server_span(exporter) + assert exported is not None + assert exported.attributes.get("error.code") == str(status_code) + + +def _server_span(otel, path: str, headers: dict[str, str]): + server_span: Final = otel.create_litellm_proxy_request_started_span(start_time=datetime.now(), headers=headers) + otel.set_proxy_request_route_attributes(server_span, url_path=path, http_route=path) + return server_span + + +def _assert_ended_in_inbound_trace(exporter, path: str, handler_name: str) -> None: + assert_server_span_attrs(exporter, expected_status=200, expected_url_path=path, where=f"{path} success") + exported: Final = get_server_span(exporter) + assert exported is not None + assert exported.context.trace_id == int(INBOUND_TRACEPARENT.split("-")[1], 16) + + endpoint_spans: Final = [s for s in exporter.get_finished_spans() if s.name == handler_name] + assert len(endpoint_spans) == 1 + assert endpoint_spans[0].parent is not None + assert endpoint_spans[0].parent.span_id == exported.context.span_id + assert not any(key.startswith("response.") for key in endpoint_spans[0].attributes) + + +def _install_proxy_state(monkeypatch, router: Router | None) -> None: + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "user_model", None) + + +def test_model_list_ends_server_span_in_inbound_trace(otel_with_exporter, monkeypatch): + import litellm.proxy.proxy_server as proxy_server + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + _install_proxy_state(monkeypatch, None) + server_span: Final = _server_span(otel, MODEL_LIST_PATH, headers={"traceparent": INBOUND_TRACEPARENT}) + + result: Final = asyncio.run(proxy_server.model_list(user_api_key_dict=_real_user_api_key_dict(server_span))) + + assert result == {"data": [], "object": "list"} + _assert_ended_in_inbound_trace(exporter, MODEL_LIST_PATH, "model_list") + + +def test_model_retrieve_ends_server_span_in_inbound_trace(otel_with_exporter, monkeypatch): + import litellm.proxy.proxy_server as proxy_server + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + _install_proxy_state( + monkeypatch, + Router(model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-x"}}]), + ) + server_span: Final = _server_span(otel, MODEL_RETRIEVE_PATH, headers={"traceparent": INBOUND_TRACEPARENT}) + + result: Final = asyncio.run( + proxy_server.model_info(model_id="gpt-4o", user_api_key_dict=_real_user_api_key_dict(server_span)) + ) + + assert result["id"] == "gpt-4o" + _assert_ended_in_inbound_trace(exporter, MODEL_RETRIEVE_PATH, "model_info")