mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge remote-tracking branch 'origin/main' into litellm_responses_fallback_prefill
# Conflicts: # litellm/responses/streaming_iterator.py
This commit is contained in:
commit
2143c6c263
267 changed files with 19685 additions and 3621 deletions
|
|
@ -1291,7 +1291,7 @@ jobs:
|
|||
name: Install MCP SDK1 peer
|
||||
command: |
|
||||
uv venv --python 3.12 .venv-mcp-peer
|
||||
uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
|
||||
uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
|
||||
echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
|
||||
- when:
|
||||
condition:
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ while IFS= read -r file || [ -n "$file" ]; do
|
|||
*) has_backend=true ;;
|
||||
esac
|
||||
case "$file" in
|
||||
model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json | model_prices_and_context_window.schema.json)
|
||||
model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json)
|
||||
has_cost_map=true ;;
|
||||
tests/unit/proxy/*) : ;;
|
||||
*) outside_cost_map_set=true ;;
|
||||
|
|
|
|||
1
.github/workflows/test-unit.yml
vendored
1
.github/workflows/test-unit.yml
vendored
|
|
@ -139,6 +139,7 @@ jobs:
|
|||
test-path: |-
|
||||
tests/unit/batch_completion
|
||||
tests/unit/decisions
|
||||
tests/unit/extraction
|
||||
tests/unit/litellm_core_utils
|
||||
tests/unit/systemone
|
||||
python-version: "3.12"
|
||||
|
|
|
|||
|
|
@ -4,8 +4,7 @@ Every pull request whose diff against its merge base touches one of the two cost
|
|||
checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
|
||||
skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
|
||||
branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
|
||||
get the file checks and additionally may only touch those two files or the legacy schema
|
||||
path and may only add or update models.
|
||||
get the file checks and additionally may only touch those two files and may only add or update models.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -21,7 +20,6 @@ from typing import Final
|
|||
|
||||
COST_MAP_PATH: Final = "model_prices_and_context_window.json"
|
||||
BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
|
||||
LEGACY_SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
|
||||
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
|
||||
BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
|
||||
SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
|
||||
|
|
@ -110,7 +108,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
|
|||
*(
|
||||
f"bot PRs may only change the cost map files, not {path}"
|
||||
for path in changed_files
|
||||
if path not in GUARDED_PATHS and path != LEGACY_SCHEMA_PATH
|
||||
if path not in GUARDED_PATHS
|
||||
),
|
||||
*(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map),
|
||||
*(f"bot PRs may not remove fields: {ref}" for ref in removed_fields),
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tupl
|
|||
from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
|
||||
ManagedObjectCreatorAttribution,
|
||||
ManagedObjectRow,
|
||||
row_metadata,
|
||||
)
|
||||
|
||||
import litellm
|
||||
|
|
@ -90,11 +91,6 @@ def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.Li
|
|||
return TeamRepository(prisma_client).table
|
||||
|
||||
|
||||
def _row_metadata(row: object) -> dict[str, object]:
|
||||
metadata: Final = getattr(row, "metadata", None)
|
||||
return cast(dict[str, object], metadata) if isinstance(metadata, dict) else {} # cast-ok: prisma Json object column
|
||||
|
||||
|
||||
class CheckBatchCost:
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -1063,7 +1059,9 @@ class CheckBatchCost:
|
|||
if (has_key_row and key_row is None) or (team_id and team_row is None):
|
||||
return True
|
||||
settings: Final = get_dynamic_logging_metadata(
|
||||
UserAPIKeyAuth(metadata=_row_metadata(key_row), team_metadata=_row_metadata(team_row), team_id=team_id),
|
||||
UserAPIKeyAuth(
|
||||
metadata=dict(row_metadata(key_row)), team_metadata=dict(row_metadata(team_row)), team_id=team_id
|
||||
),
|
||||
proxy_config,
|
||||
)
|
||||
callback_vars: Final = (settings.callback_vars if settings is not None else None) or {}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,12 @@ attribution, priced against the deployment that served it. The poll's status che
|
|||
with BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN and client reads are marked managed by the
|
||||
managed files hook, so both stay free. The spend event carries the id the client holds, the row's
|
||||
unified_object_id, so its spend log is found under that id with the _background_cost suffix.
|
||||
The spend event reaches the creating key's or team's own logging callbacks the way that creator's
|
||||
requests do. Its message content is redacted when the creator's key or team logging turns message
|
||||
logging off, else as the proxy is set, the way the batch poll decides it. The row is claimed before
|
||||
the creator is read, so reads that lose the race skip the lookups. A creator whose lookup fails, or
|
||||
whose logging settings the logger rejects, is still billed, through the proxy's own callbacks only
|
||||
and with its message content redacted.
|
||||
A read that finds no row, because the create could not store one, stores it under the reading key
|
||||
and bills it the same way, so a response the create left untracked is still billed once. A streamed
|
||||
background create is free on its own; its stream stores the row under the creating key and bills
|
||||
|
|
@ -29,6 +35,7 @@ from typing import TYPE_CHECKING, Final, cast
|
|||
from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
|
||||
ManagedObjectCreatorAttribution,
|
||||
ManagedObjectRow,
|
||||
ResponseCreator,
|
||||
)
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
|
|
@ -54,11 +61,13 @@ from litellm.types.utils import (
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.router import Router
|
||||
|
||||
_STORED_RESPONSE_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
|
||||
_CALLBACK_LIST_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
class _RoutedModelInfo(BaseModel):
|
||||
|
|
@ -73,6 +82,8 @@ class _RoutedDeployment(BaseModel):
|
|||
|
||||
_ROUTED_DEPLOYMENTS: Final = TypeAdapter(tuple[_RoutedDeployment, ...])
|
||||
_RETRIEVE_RESPONSE_CALL_TYPE: Final = "aget_responses"
|
||||
_CALLBACK_LIST_FIELDS: Final = frozenset({"success_callback", "failure_callback", "litellm_disabled_callbacks"})
|
||||
_REJECTED_CREATOR_LOGGING: Final[Mapping[str, object]] = MappingProxyType({"turn_off_message_logging": True})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -89,6 +100,7 @@ class BilledBackgroundResponse:
|
|||
custom_llm_provider: str | None
|
||||
litellm_params: Mapping[str, object]
|
||||
metadata: Mapping[str, object]
|
||||
creator_logging: Mapping[str, object]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -107,11 +119,51 @@ def _without_read_price(response: ResponsesAPIResponse) -> ResponsesAPIResponse:
|
|||
return unpriced
|
||||
|
||||
|
||||
async def log_billed_background_response(billed: BilledBackgroundResponse) -> BilledCharge:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
def _string_callbacks(callbacks: object) -> tuple[str, ...]:
|
||||
if not isinstance(callbacks, list):
|
||||
return ()
|
||||
return tuple(
|
||||
callback for callback in _CALLBACK_LIST_ADAPTER.validate_python(callbacks) if isinstance(callback, str)
|
||||
)
|
||||
|
||||
logging_obj: Final = LiteLLMLogging(
|
||||
|
||||
def _running_proxy_config() -> "ProxyConfig":
|
||||
from litellm.proxy.proxy_server import proxy_config
|
||||
|
||||
return proxy_config
|
||||
|
||||
|
||||
def creator_logging_fields(creator: ResponseCreator, proxy_config: "ProxyConfig | None") -> Mapping[str, object]:
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.litellm_pre_call_utils import key_and_team_logging_fields
|
||||
|
||||
if not creator.settings_read:
|
||||
return _REJECTED_CREATOR_LOGGING
|
||||
try:
|
||||
return key_and_team_logging_fields(
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
metadata=dict(creator.key_metadata),
|
||||
team_id=creator.team_id,
|
||||
team_metadata=dict(creator.team_metadata),
|
||||
),
|
||||
proxy_config=proxy_config if proxy_config is not None else _running_proxy_config(),
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(
|
||||
f"CheckResponsesCost: could not read the creator's logging settings, so only the proxy's "
|
||||
f"callbacks get the billed event: {e}"
|
||||
)
|
||||
return _REJECTED_CREATOR_LOGGING
|
||||
|
||||
|
||||
def _billed_logging_obj(billed: BilledBackgroundResponse, creator_logging: Mapping[str, object]) -> "LiteLLMLogging":
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.utils import split_dynamic_success_callbacks
|
||||
|
||||
sync_callbacks, async_callbacks = split_dynamic_success_callbacks(
|
||||
_string_callbacks(creator_logging.get("success_callback"))
|
||||
)
|
||||
return LiteLLMLogging(
|
||||
model=billed.model,
|
||||
messages=[{"role": "user", "content": "<retrieve_response>"}],
|
||||
stream=False,
|
||||
|
|
@ -119,7 +171,27 @@ async def log_billed_background_response(billed: BilledBackgroundResponse) -> Bi
|
|||
start_time=datetime.now(),
|
||||
litellm_call_id=str(uuid.uuid4()),
|
||||
function_id=str(uuid.uuid4()),
|
||||
dynamic_success_callbacks=list(sync_callbacks),
|
||||
dynamic_async_success_callbacks=list(async_callbacks) or None,
|
||||
kwargs={key: value for key, value in creator_logging.items() if key not in _CALLBACK_LIST_FIELDS},
|
||||
)
|
||||
|
||||
|
||||
def _creator_or_proxy_logging_obj(billed: BilledBackgroundResponse) -> "LiteLLMLogging":
|
||||
try:
|
||||
return _billed_logging_obj(billed, billed.creator_logging)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(
|
||||
f"CheckResponsesCost: the creator's logging settings were rejected, so only the proxy's "
|
||||
f"callbacks get the billed event: {e}"
|
||||
)
|
||||
return _billed_logging_obj(billed, _REJECTED_CREATOR_LOGGING)
|
||||
|
||||
|
||||
async def log_billed_background_response(billed: BilledBackgroundResponse) -> BilledCharge:
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
||||
logging_obj: Final = _creator_or_proxy_logging_obj(billed)
|
||||
logging_obj.update_environment_variables(
|
||||
litellm_params={**billed.litellm_params, "metadata": dict(billed.metadata)},
|
||||
optional_params={},
|
||||
|
|
@ -187,6 +259,7 @@ class CheckResponsesCost:
|
|||
prisma_client: "PrismaClient",
|
||||
llm_router: "Router",
|
||||
log_billed_response: BilledResponseLogger = log_billed_background_response,
|
||||
proxy_config: "ProxyConfig | None" = None,
|
||||
):
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.router import Router
|
||||
|
|
@ -196,6 +269,7 @@ class CheckResponsesCost:
|
|||
self.llm_router: Router = llm_router
|
||||
self._creator_attribution: Final = ManagedObjectCreatorAttribution(prisma_client, "CheckResponsesCost")
|
||||
self._log_billed_response: Final = log_billed_response
|
||||
self._proxy_config: Final = proxy_config
|
||||
|
||||
def _resolve_deployment(self, response_id: str) -> Deployment | None:
|
||||
model_id: str | None = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
|
||||
|
|
@ -307,11 +381,12 @@ class CheckResponsesCost:
|
|||
data={"status": job.status or PENDING_RESPONSE_STATUSES[0]},
|
||||
)
|
||||
|
||||
async def _billed_response(
|
||||
def _billed_response(
|
||||
self,
|
||||
job: ManagedObjectRow,
|
||||
target: _PollTarget,
|
||||
response: ResponsesAPIResponse,
|
||||
creator: ResponseCreator,
|
||||
) -> BilledBackgroundResponse:
|
||||
deployment: Final = target.deployment
|
||||
return BilledBackgroundResponse(
|
||||
|
|
@ -323,13 +398,14 @@ class CheckResponsesCost:
|
|||
litellm_params=_deployment_litellm_params(deployment) if deployment is not None else MappingProxyType({}),
|
||||
metadata=MappingProxyType(
|
||||
{
|
||||
**(await self._creator_attribution.response_metadata(job, job.unified_object_id)),
|
||||
**creator.metadata,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
|
||||
**target.model_metadata,
|
||||
**(_deployment_metadata(deployment) if deployment is not None else {}),
|
||||
**self._requested_model_group(target, job.team_id),
|
||||
}
|
||||
),
|
||||
creator_logging=creator_logging_fields(creator, self._proxy_config),
|
||||
)
|
||||
|
||||
def _requested_model_group(self, target: _PollTarget, team_id: str | None) -> Mapping[str, object]:
|
||||
|
|
@ -354,7 +430,8 @@ class CheckResponsesCost:
|
|||
verbose_proxy_logger.info(f"Response {job.unified_object_id} was already billed elsewhere, skipping")
|
||||
return None
|
||||
try:
|
||||
charge: Final = await self._log_billed_response(await self._billed_response(job, target, response))
|
||||
creator: Final = await self._creator_attribution.response_creator(job, job.unified_object_id)
|
||||
charge: Final = await self._log_billed_response(self._billed_response(job, target, response, creator))
|
||||
except BaseException:
|
||||
await self._release_billing_claim(job)
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -10,12 +10,16 @@ from __future__ import annotations
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Final, Protocol, cast
|
||||
from typing import TYPE_CHECKING, Final, Protocol, TypeGuard, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import CLI_SESSION_KEY_PREFIX
|
||||
from litellm.repositories.table_repositories import (
|
||||
DeletedVerificationTokenRepository,
|
||||
DeprecatedVerificationTokenRepository,
|
||||
)
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
from litellm.repositories.verification_token_repository import VerificationTokenRepository
|
||||
|
|
@ -70,6 +74,7 @@ class ManagedObjectRow(Protocol):
|
|||
|
||||
_TAG_LIST_ADAPTER: Final[TypeAdapter[list[object]]] = TypeAdapter(list[object])
|
||||
_KEY_OWNER_SCOPE_PREFIX: Final = "key:"
|
||||
_MAX_ROTATION_HOPS: Final = 16
|
||||
|
||||
|
||||
def _string_tags(request_tags: object) -> list[str] | None:
|
||||
|
|
@ -96,6 +101,20 @@ def _team_table(prisma_client: PrismaClient) -> TableActions[prisma_models.LiteL
|
|||
return TeamRepository(prisma_client).table
|
||||
|
||||
|
||||
def _rotated_token_table(
|
||||
prisma_client: PrismaClient,
|
||||
) -> TableActions[prisma_models.LiteLLM_DeprecatedVerificationToken]:
|
||||
return DeprecatedVerificationTokenRepository(prisma_client).table
|
||||
|
||||
|
||||
def _deleted_token_table(prisma_client: PrismaClient) -> TableActions[prisma_models.LiteLLM_DeletedVerificationToken]:
|
||||
return DeletedVerificationTokenRepository(prisma_client).table
|
||||
|
||||
|
||||
def _deleted_team_table(prisma_client: PrismaClient) -> TableActions[prisma_models.LiteLLM_DeletedTeamTable]:
|
||||
return TeamRepository(prisma_client).deleted_table
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _CreatorRows:
|
||||
key: prisma_models.LiteLLM_VerificationToken | None
|
||||
|
|
@ -103,6 +122,15 @@ class _CreatorRows:
|
|||
user: prisma_models.LiteLLM_UserTable | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ResponseCreator:
|
||||
metadata: Mapping[str, object]
|
||||
key_metadata: Mapping[str, object]
|
||||
team_id: str | None
|
||||
team_metadata: Mapping[str, object]
|
||||
settings_read: bool = True
|
||||
|
||||
|
||||
class ManagedObjectCreatorAttribution:
|
||||
"""Rebuilds the spend-tracking metadata of whoever created a managed object, so the poll
|
||||
job's spend row is attributed like the creating request: key hash, team, organization,
|
||||
|
|
@ -117,7 +145,12 @@ class ManagedObjectCreatorAttribution:
|
|||
A background response also carries its end user, the creating key's project and its alias,
|
||||
and the key, team, and user budgets and per-model budgets, the way the read that billed it
|
||||
before did, and never names a key-scoped creator as its user. A key that sets no budget of
|
||||
its own takes its budget tier's, the same rule auth applies.
|
||||
its own takes its budget tier's, the same rule auth applies. Its creator's key and team
|
||||
metadata come along too, so the billed event reaches the key's and team's logging callbacks
|
||||
like that read did. A rotated key's metadata is read from the live key its rotations lead
|
||||
to, and a deleted key's or team's from its deleted row. A lookup that fails leaves the
|
||||
creator's settings unread and the attribution to the rows that still load, so the
|
||||
response still bills and still counts toward their budgets.
|
||||
"""
|
||||
|
||||
def __init__(self, prisma_client: PrismaClient, poller_name: str) -> None:
|
||||
|
|
@ -128,31 +161,70 @@ class ManagedObjectCreatorAttribution:
|
|||
rows: Final = await self._rows(job, object_id)
|
||||
return {"user_api_key_user_id": job.created_by, **_shared_metadata(job, rows)}
|
||||
|
||||
async def response_metadata(self, job: ManagedObjectRow, object_id: str) -> Mapping[str, object]:
|
||||
rows: Final = await self._rows(job, object_id)
|
||||
end_user_id: Final = getattr(job, "end_user_id", None)
|
||||
return {
|
||||
"user_api_key_user_id": _creator_user_id(job.created_by),
|
||||
**_shared_metadata(job, rows),
|
||||
**({"user_api_key_end_user_id": end_user_id} if end_user_id else {}),
|
||||
"user_api_key_project_id": getattr(rows.key, "project_id", None),
|
||||
"user_api_key_project_alias": getattr(
|
||||
getattr(rows.key, "litellm_project_table", None), "project_alias", None
|
||||
async def response_creator(self, job: ManagedObjectRow, object_id: str) -> ResponseCreator:
|
||||
try:
|
||||
return await self._read_response_creator(job, object_id)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(
|
||||
f"{self._poller_name}: could not read the creator of {object_id}, so it bills without their "
|
||||
f"settings: {e}"
|
||||
)
|
||||
return ResponseCreator(
|
||||
metadata=_response_creator_metadata(job, await self._rows(job, object_id)),
|
||||
key_metadata={},
|
||||
team_id=getattr(job, "team_id", None),
|
||||
team_metadata={},
|
||||
settings_read=False,
|
||||
)
|
||||
|
||||
async def _read_response_creator(self, job: ManagedObjectRow, object_id: str) -> ResponseCreator:
|
||||
api_key: Final = getattr(job, "api_key", None)
|
||||
team_id: Final = getattr(job, "team_id", None)
|
||||
rows: Final = _CreatorRows(
|
||||
key=await self._live_key_row(api_key, job.created_by),
|
||||
team=await self._live_team_row(team_id),
|
||||
user=await self._user_row(object_id, _creator_user_id(job.created_by)),
|
||||
)
|
||||
return ResponseCreator(
|
||||
metadata=_response_creator_metadata(job, rows),
|
||||
key_metadata=row_metadata(rows.key) if rows.key is not None else await self._retired_key_metadata(api_key),
|
||||
team_id=team_id,
|
||||
team_metadata=(
|
||||
row_metadata(rows.team) if rows.team is not None else await self._deleted_team_metadata(team_id)
|
||||
),
|
||||
**_key_budget_fields(rows.key),
|
||||
**_budget_fields(
|
||||
rows.team,
|
||||
"user_api_key_team_spend",
|
||||
"user_api_key_team_max_budget",
|
||||
"user_api_key_team_model_max_budget",
|
||||
),
|
||||
**_budget_fields(
|
||||
rows.user,
|
||||
"user_api_key_user_spend",
|
||||
"user_api_key_user_max_budget",
|
||||
"user_api_key_user_model_max_budget",
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
async def _retired_key_metadata(self, api_key: str | None) -> Mapping[str, object]:
|
||||
if not _names_a_key_row(api_key):
|
||||
return {}
|
||||
current_token: Final = await self._token_rotated_to(api_key, _MAX_ROTATION_HOPS)
|
||||
current_key: Final = (
|
||||
await _token_table(self._prisma_client).find_unique(where={"token": current_token})
|
||||
if current_token != api_key
|
||||
else None
|
||||
)
|
||||
if current_key is not None:
|
||||
return row_metadata(current_key)
|
||||
return row_metadata(
|
||||
await _deleted_token_table(self._prisma_client).find_first(
|
||||
where={"token": current_token}, order={"deleted_at": "desc"}
|
||||
)
|
||||
)
|
||||
|
||||
async def _token_rotated_to(self, token: str, hops_left: int) -> str:
|
||||
rotation: Final = await _rotated_token_table(self._prisma_client).find_unique(where={"token": token})
|
||||
if rotation is None or hops_left == 0:
|
||||
return token
|
||||
return await self._token_rotated_to(rotation.active_token_id, hops_left - 1)
|
||||
|
||||
async def _deleted_team_metadata(self, team_id: str | None) -> Mapping[str, object]:
|
||||
if not team_id:
|
||||
return {}
|
||||
return row_metadata(
|
||||
await _deleted_team_table(self._prisma_client).find_first(
|
||||
where={"team_id": team_id}, order={"deleted_at": "desc"}
|
||||
)
|
||||
)
|
||||
|
||||
async def _rows(self, job: ManagedObjectRow, object_id: str) -> _CreatorRows:
|
||||
return _CreatorRows(
|
||||
|
|
@ -175,25 +247,57 @@ class ManagedObjectCreatorAttribution:
|
|||
async def _key_row(
|
||||
self, object_id: str, api_key: str | None, created_by: str | None
|
||||
) -> prisma_models.LiteLLM_VerificationToken | None:
|
||||
if not api_key or _is_cli_session_key(api_key, created_by):
|
||||
return None
|
||||
try:
|
||||
return await _token_table(self._prisma_client).find_unique(
|
||||
where={"token": api_key}, include={"litellm_budget_table": True, "litellm_project_table": True}
|
||||
)
|
||||
return await self._live_key_row(api_key, created_by)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(f"{self._poller_name}: could not look up the creating key for {object_id}: {e}")
|
||||
return None
|
||||
|
||||
async def _team_row(self, object_id: str, team_id: str | None) -> prisma_models.LiteLLM_TeamTable | None:
|
||||
if not team_id:
|
||||
async def _live_key_row(
|
||||
self, api_key: str | None, created_by: str | None
|
||||
) -> prisma_models.LiteLLM_VerificationToken | None:
|
||||
if not api_key or _is_cli_session_key(api_key, created_by):
|
||||
return None
|
||||
return await _token_table(self._prisma_client).find_unique(
|
||||
where={"token": api_key}, include={"litellm_budget_table": True, "litellm_project_table": True}
|
||||
)
|
||||
|
||||
async def _team_row(self, object_id: str, team_id: str | None) -> prisma_models.LiteLLM_TeamTable | None:
|
||||
try:
|
||||
return await _team_table(self._prisma_client).find_unique(where={"team_id": team_id})
|
||||
return await self._live_team_row(team_id)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(f"{self._poller_name}: could not look up team {team_id} for {object_id}: {e}")
|
||||
return None
|
||||
|
||||
async def _live_team_row(self, team_id: str | None) -> prisma_models.LiteLLM_TeamTable | None:
|
||||
if not team_id:
|
||||
return None
|
||||
return await _team_table(self._prisma_client).find_unique(where={"team_id": team_id})
|
||||
|
||||
|
||||
def _response_creator_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str, object]:
|
||||
end_user_id: Final = getattr(job, "end_user_id", None)
|
||||
return {
|
||||
"user_api_key_user_id": _creator_user_id(job.created_by),
|
||||
**_shared_metadata(job, rows),
|
||||
**({"user_api_key_end_user_id": end_user_id} if end_user_id else {}),
|
||||
"user_api_key_project_id": getattr(rows.key, "project_id", None),
|
||||
"user_api_key_project_alias": getattr(getattr(rows.key, "litellm_project_table", None), "project_alias", None),
|
||||
**_key_budget_fields(rows.key),
|
||||
**_budget_fields(
|
||||
rows.team,
|
||||
"user_api_key_team_spend",
|
||||
"user_api_key_team_max_budget",
|
||||
"user_api_key_team_model_max_budget",
|
||||
),
|
||||
**_budget_fields(
|
||||
rows.user,
|
||||
"user_api_key_user_spend",
|
||||
"user_api_key_user_max_budget",
|
||||
"user_api_key_user_model_max_budget",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _shared_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str, object]:
|
||||
api_key: Final = getattr(job, "api_key", None)
|
||||
|
|
@ -221,6 +325,19 @@ def _shared_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str,
|
|||
}
|
||||
|
||||
|
||||
def row_metadata(row: object) -> Mapping[str, object]:
|
||||
metadata: Final = getattr(row, "metadata", None)
|
||||
if not isinstance(metadata, dict):
|
||||
return {}
|
||||
return cast(Mapping[str, object], metadata) # cast-ok: prisma Json object column
|
||||
|
||||
|
||||
def _names_a_key_row(api_key: object) -> TypeGuard[str]:
|
||||
from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash
|
||||
|
||||
return isinstance(api_key, str) and is_valid_sha256_hash(api_key)
|
||||
|
||||
|
||||
def _is_cli_session_key(api_key: str, created_by: str | None) -> bool:
|
||||
return bool(created_by) and api_key == f"{CLI_SESSION_KEY_PREFIX}-{created_by}"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.78"
|
||||
version = "0.1.79"
|
||||
description = "Package for LiteLLM Enterprise features"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.1.78"
|
||||
version = "0.1.79"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-enterprise==",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""Path allowlist for the gateway component.
|
||||
|
||||
The gateway exposes the LLM data-plane surface: chat/completions, embeddings,
|
||||
audio, batches, files, fine-tuning, rerank, decisions, ocr, rag, video, search, image,
|
||||
audio, batches, files, fine-tuning, rerank, decisions, extraction, ocr, rag, video, search, image,
|
||||
responses, vector stores, passthrough providers, realtime websockets, MCP
|
||||
tool-call endpoints, and operational endpoints (/health, /metrics, and the
|
||||
/debug/memory/summary read of the serving worker's RSS).
|
||||
|
|
@ -64,6 +64,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
|
|||
"/decisions",
|
||||
"/v1/systemone",
|
||||
"/systemone",
|
||||
"/v1/extraction",
|
||||
"/extraction",
|
||||
"/v1/ocr",
|
||||
"/ocr",
|
||||
"/v1/rag/",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.110"
|
||||
version = "0.4.111"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.110"
|
||||
version = "0.4.111"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-proxy-extras==",
|
||||
|
|
|
|||
|
|
@ -1505,6 +1505,7 @@ from .rag.main import *
|
|||
from .sandbox.main import *
|
||||
from .decisions.main import *
|
||||
from .systemone.main import *
|
||||
from .extraction.main import *
|
||||
from .tool_loop import ToolLoopMaxRoundsExceeded, arun_tool_loop, run_tool_loop
|
||||
from .search.main import *
|
||||
from .realtime_api.main import (
|
||||
|
|
@ -1711,6 +1712,9 @@ if TYPE_CHECKING:
|
|||
from .llms.cloudflare.decisions.transformation import (
|
||||
CloudflareDecisionsConfig as CloudflareDecisionsConfig,
|
||||
)
|
||||
from .llms.cloudflare.extraction.transformation import (
|
||||
CloudflareExtractionConfig as CloudflareExtractionConfig,
|
||||
)
|
||||
from .llms.strands_decider.decisions.transformation import (
|
||||
StrandsDeciderDecisionsConfig as StrandsDeciderDecisionsConfig,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -166,6 +166,7 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"OpenRouterDecisionsConfig",
|
||||
"FireworksAIDecisionsConfig",
|
||||
"CloudflareDecisionsConfig",
|
||||
"CloudflareExtractionConfig",
|
||||
"StrandsDeciderDecisionsConfig",
|
||||
"DatabricksDecisionsConfig",
|
||||
"HostedVLLMDecisionsConfig",
|
||||
|
|
@ -735,6 +736,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
"OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
|
||||
"FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),
|
||||
"CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"),
|
||||
"CloudflareExtractionConfig": (".llms.cloudflare.extraction.transformation", "CloudflareExtractionConfig"),
|
||||
"StrandsDeciderDecisionsConfig": (
|
||||
".llms.strands_decider.decisions.transformation",
|
||||
"StrandsDeciderDecisionsConfig",
|
||||
|
|
|
|||
|
|
@ -95,7 +95,10 @@ class InMemoryCache(BaseCache):
|
|||
"""
|
||||
Check if a specific key is expired
|
||||
"""
|
||||
return key in self.ttl_dict and self._clock() > self.ttl_dict[key]
|
||||
try:
|
||||
return self._clock() > self.ttl_dict[key]
|
||||
except KeyError:
|
||||
return False
|
||||
|
||||
def _cache_keys(self) -> tuple[str, ...]:
|
||||
return tuple(self.cache_dict) # pyright: ignore[reportUnknownVariableType] # cache_dict is untyped legacy state
|
||||
|
|
@ -227,7 +230,10 @@ class InMemoryCache(BaseCache):
|
|||
if key in self.cache_dict:
|
||||
if self.evict_element_if_expired(key):
|
||||
return None
|
||||
original_cached_response: Final = self.cache_dict[key]
|
||||
try:
|
||||
original_cached_response: Final = self.cache_dict[key]
|
||||
except KeyError:
|
||||
return None
|
||||
try:
|
||||
cached_response = json.loads(original_cached_response)
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -156,6 +156,9 @@ DEFAULT_REPLICATE_POLLING_RETRIES: Final = get_env_int("DEFAULT_REPLICATE_POLLIN
|
|||
DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)
|
||||
DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250)
|
||||
HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
|
||||
HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS: Final = get_env_int_in_range(
|
||||
"HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 300, 1, 86400
|
||||
)
|
||||
|
||||
# Maximum wall-clock seconds a streaming response is allowed to run.
|
||||
# Streams exceeding this duration are terminated with a Timeout error.
|
||||
|
|
@ -258,6 +261,9 @@ MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS: Final = get_env_int("MCP_PER_USER_TOKE
|
|||
|
||||
# MCP timeout defaults (seconds). Override via env vars for slow/custom MCP servers.
|
||||
MCP_CLIENT_TIMEOUT: Final = get_env_float("LITELLM_MCP_CLIENT_TIMEOUT", 60.0)
|
||||
MCP_MAX_SSE_EVENT_SIZE_BYTES: Final = get_env_int_in_range(
|
||||
"LITELLM_MCP_MAX_SSE_EVENT_SIZE_BYTES", 64 * 1024 * 1024, 1, sys.maxsize
|
||||
)
|
||||
MCP_TOOL_LISTING_TIMEOUT: Final = get_env_float("LITELLM_MCP_TOOL_LISTING_TIMEOUT", 30.0)
|
||||
MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES: Final = get_env_int("LITELLM_MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES", 3)
|
||||
MCP_UPSTREAM_RATE_LIMIT_BASE_DELAY_SECONDS: Final = get_env_float(
|
||||
|
|
@ -1651,6 +1657,7 @@ BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
|
|||
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
|
||||
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
|
||||
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
|
||||
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
|
||||
SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
|
||||
|
|
|
|||
|
|
@ -112,6 +112,7 @@ from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_toke
|
|||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.types.agents import LiteLLMSendMessageResponse
|
||||
from litellm.types.decisions import DecisionsResponse, DecisionsUsage, OpenAIDecisionResponse, OpenAIDecisionUsage
|
||||
from litellm.types.extraction import ExtractionResponse, ExtractionUsage
|
||||
from litellm.types.llms.base import CachedTokensDetails, LiteLLMBaseModel
|
||||
from litellm.types.llms.openai import (
|
||||
HttpxBinaryResponseContent,
|
||||
|
|
@ -1171,6 +1172,8 @@ def get_usage_object(
|
|||
|
||||
if isinstance(completion_response, (DecisionsResponse, OpenAIDecisionResponse)):
|
||||
return None if completion_response.usage is None else _decisions_usage(completion_response.usage)
|
||||
if isinstance(completion_response, ExtractionResponse):
|
||||
return _extraction_usage(completion_response.usage)
|
||||
if isinstance(completion_response, HttpxBinaryResponseContent):
|
||||
return completion_response.usage
|
||||
if usage_obj is None:
|
||||
|
|
@ -1227,6 +1230,14 @@ def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage:
|
|||
)
|
||||
|
||||
|
||||
def _extraction_usage(usage: ExtractionUsage) -> Usage:
|
||||
return Usage(
|
||||
prompt_tokens=usage.input_tokens,
|
||||
completion_tokens=usage.output_tokens,
|
||||
total_tokens=usage.input_tokens + usage.output_tokens,
|
||||
)
|
||||
|
||||
|
||||
def _is_known_usage_objects(usage_obj):
|
||||
"""Returns True if the usage obj is a known Usage type"""
|
||||
return (
|
||||
|
|
@ -1234,6 +1245,7 @@ def _is_known_usage_objects(usage_obj):
|
|||
or isinstance(usage_obj, ResponseAPIUsage)
|
||||
or isinstance(usage_obj, DecisionsUsage)
|
||||
or isinstance(usage_obj, OpenAIDecisionUsage)
|
||||
or isinstance(usage_obj, ExtractionUsage)
|
||||
or TranscriptionUsageObjectTransformation.is_transcription_usage_object(usage_obj)
|
||||
)
|
||||
|
||||
|
|
@ -1263,6 +1275,8 @@ def infer_call_type(call_type: CallTypesLiteral | None, completion_response: obj
|
|||
return "send_message"
|
||||
elif isinstance(completion_response, (DecisionsResponse, OpenAIDecisionResponse)):
|
||||
return "adecisions"
|
||||
elif isinstance(completion_response, ExtractionResponse):
|
||||
return "aextraction"
|
||||
|
||||
return call_type
|
||||
|
||||
|
|
@ -1659,8 +1673,10 @@ def completion_cost(
|
|||
verbose_logger.debug("selected model name for cost calculation: %s", model)
|
||||
|
||||
if completion_response is not None and (
|
||||
isinstance(completion_response, BaseModel) or isinstance(completion_response, dict)
|
||||
): # tts returns a custom class
|
||||
isinstance(completion_response, (BaseModel, dict))
|
||||
or isinstance(completion_response, HttpxBinaryResponseContent)
|
||||
and completion_response.usage is not None
|
||||
):
|
||||
if isinstance(completion_response, dict):
|
||||
usage_obj: dict | Usage | None = completion_response.get("usage", {})
|
||||
else:
|
||||
|
|
@ -1674,6 +1690,8 @@ def completion_cost(
|
|||
)
|
||||
if isinstance(usage_obj, (DecisionsUsage, OpenAIDecisionUsage)):
|
||||
_usage = _decisions_usage(usage_obj).model_dump()
|
||||
elif isinstance(usage_obj, ExtractionUsage):
|
||||
_usage = _extraction_usage(usage_obj).model_dump()
|
||||
elif usage_obj is None:
|
||||
_usage = {}
|
||||
elif isinstance(usage_obj, BaseModel):
|
||||
|
|
@ -2199,7 +2217,8 @@ def response_cost_calculator(
|
|||
| Response
|
||||
| SearchResponse
|
||||
| DecisionsResponse
|
||||
| OpenAIDecisionResponse,
|
||||
| OpenAIDecisionResponse
|
||||
| ExtractionResponse,
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
call_type: Literal[
|
||||
|
|
@ -2225,6 +2244,8 @@ def response_cost_calculator(
|
|||
"adecisions",
|
||||
"systemone",
|
||||
"asystemone",
|
||||
"extraction",
|
||||
"aextraction",
|
||||
],
|
||||
optional_params: dict,
|
||||
cache_hit: bool | None = None,
|
||||
|
|
|
|||
|
|
@ -122,6 +122,7 @@ class SpeechToCompletionBridgeHandler:
|
|||
if isinstance(result, ModelResponse):
|
||||
return self.transformation_handler.transform_response(
|
||||
model_response=result,
|
||||
model=model,
|
||||
response_format=requested_response_format if isinstance(requested_response_format, str) else None,
|
||||
)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Final, cast
|
|||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache, set_hidden_param
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -38,6 +39,11 @@ class ChatAudioParam(TypedDict):
|
|||
format: ReadOnly[NotRequired[str]]
|
||||
|
||||
|
||||
class ChatSpeechConfigAudioParam(TypedDict):
|
||||
speech_config: ReadOnly[Mapping[str, object]]
|
||||
format: ReadOnly[str]
|
||||
|
||||
|
||||
class SpeechToCompletionBridgeTransformationHandler:
|
||||
def _validate_response_format(
|
||||
self, model: str, custom_llm_provider: str, optional_params: Mapping[str, object]
|
||||
|
|
@ -49,6 +55,20 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
return
|
||||
from litellm.exceptions import BadRequestError
|
||||
|
||||
if custom_llm_provider.startswith("vertex_ai"):
|
||||
from litellm.llms.vertex_ai.text_to_speech.transformation import VertexAITextToSpeechConfig
|
||||
|
||||
vertex_supported: Final = ", ".join(
|
||||
sorted({*GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS, *VertexAITextToSpeechConfig.GEMINI_FORMAT_MAPPINGS})
|
||||
)
|
||||
raise BadRequestError(
|
||||
message=(
|
||||
f"Vertex AI Gemini TTS does not support response_format='{response_format}'."
|
||||
f" Supported response formats: {vertex_supported}."
|
||||
),
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
)
|
||||
supported: Final = ", ".join(sorted(GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS))
|
||||
raise BadRequestError(
|
||||
message=(
|
||||
|
|
@ -75,8 +95,27 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
return response_format if isinstance(response_format, str) else None
|
||||
|
||||
def _chat_audio_param(
|
||||
self, model: str, voice: str | Mapping[str, object] | None, optional_params: Mapping[str, object]
|
||||
) -> ChatAudioParam | None:
|
||||
self,
|
||||
model: str,
|
||||
voice: str | Mapping[str, object] | None,
|
||||
optional_params: Mapping[str, object],
|
||||
drop_params: bool,
|
||||
custom_llm_provider: str,
|
||||
) -> ChatAudioParam | ChatSpeechConfigAudioParam | None:
|
||||
import litellm
|
||||
|
||||
if isinstance(voice, Mapping) and self._is_gemini_tts_model(litellm.model_alias_map.get(model, model)):
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
normalize_gemini_speech_config,
|
||||
reject_unknown_gemini_tts_voice_keys,
|
||||
)
|
||||
|
||||
reject_unknown_gemini_tts_voice_keys(model, voice, drop_params, custom_llm_provider)
|
||||
speech_audio: Final[ChatSpeechConfigAudioParam] = {
|
||||
"speech_config": normalize_gemini_speech_config(voice, drops_unsupported_params(drop_params)),
|
||||
"format": GEMINI_TTS_CHAT_AUDIO_FORMAT,
|
||||
}
|
||||
return speech_audio
|
||||
if not isinstance(voice, str):
|
||||
return None
|
||||
audio_format: Final = self._chat_audio_format(model, optional_params)
|
||||
|
|
@ -90,9 +129,9 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
self,
|
||||
model: str,
|
||||
input: str,
|
||||
voice: str | dict | None,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
voice: str | Mapping[str, object] | None,
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object],
|
||||
headers: dict,
|
||||
litellm_logging_obj: "LiteLLMLoggingObj",
|
||||
custom_llm_provider: str,
|
||||
|
|
@ -104,7 +143,13 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
"messages": [user_message],
|
||||
"modalities": ["audio"],
|
||||
**self._chat_completion_params(optional_params),
|
||||
"audio": self._chat_audio_param(model, voice, optional_params),
|
||||
"audio": self._chat_audio_param(
|
||||
model,
|
||||
voice,
|
||||
optional_params,
|
||||
drops_unsupported_params(litellm_params.get("drop_params")),
|
||||
custom_llm_provider,
|
||||
),
|
||||
**litellm_params,
|
||||
"headers": headers,
|
||||
"litellm_logging_obj": litellm_logging_obj,
|
||||
|
|
@ -153,8 +198,9 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
return wav_header + pcm_data
|
||||
|
||||
def _is_gemini_tts_model(self, model: str) -> bool:
|
||||
"""Check if the model is a Gemini TTS model that returns PCM16 data."""
|
||||
return "gemini" in model.lower() and ("tts" in model.lower() or "preview-tts" in model.lower())
|
||||
from litellm.utils import is_gemini_tts_model
|
||||
|
||||
return is_gemini_tts_model(model)
|
||||
|
||||
def _gemini_tts_response_body(self, decoded_audio: bytes, response_format: str | None) -> tuple[bytes, str]:
|
||||
if response_format == GEMINI_TTS_RAW_RESPONSE_FORMAT:
|
||||
|
|
@ -162,7 +208,7 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
return self._convert_pcm16_to_wav(decoded_audio), "audio/wav"
|
||||
|
||||
def transform_response(
|
||||
self, model_response: "ModelResponse", response_format: str | None
|
||||
self, model_response: "ModelResponse", model: str, response_format: str | None
|
||||
) -> "HttpxBinaryResponseContent":
|
||||
import base64
|
||||
|
||||
|
|
@ -176,10 +222,9 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
raise ValueError("No audio part found in the response")
|
||||
decoded_audio: Final = base64.b64decode(audio_part.data)
|
||||
|
||||
model: Final = getattr(model_response, "model", "")
|
||||
content, content_type = (
|
||||
self._gemini_tts_response_body(decoded_audio, response_format)
|
||||
if self._is_gemini_tts_model(model)
|
||||
if self._is_gemini_tts_model(model) or self._is_gemini_tts_model(model_response.model or "")
|
||||
else (decoded_audio, "audio/mpeg")
|
||||
)
|
||||
response: Final = httpx.Response(
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@ from pydantic import AnyUrl, TypeAdapter
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import (
|
||||
MCP_CLIENT_TIMEOUT,
|
||||
MCP_MAX_SSE_EVENT_SIZE_BYTES,
|
||||
MCP_NPM_CACHE_DIR,
|
||||
MCP_TOOL_LISTING_MAX_PAGES,
|
||||
MCP_TOOL_LISTING_TIMEOUT,
|
||||
|
|
@ -647,6 +648,7 @@ class MCPClient:
|
|||
transport_ctx: Final = streamable_http_client(
|
||||
url=self.server_url,
|
||||
http_client=http_client,
|
||||
max_sse_event_size=MCP_MAX_SSE_EVENT_SIZE_BYTES,
|
||||
)
|
||||
return transport_ctx, http_client
|
||||
|
||||
|
|
|
|||
3
litellm/extraction/__init__.py
Normal file
3
litellm/extraction/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from litellm.extraction.main import aextraction, extraction
|
||||
|
||||
__all__ = ["aextraction", "extraction"]
|
||||
351
litellm/extraction/main.py
Normal file
351
litellm/extraction/main.py
Normal file
|
|
@ -0,0 +1,351 @@
|
|||
import asyncio
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.decisions.call import log_caller_request
|
||||
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tool_fields import DropPolicy, drop_policy
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.extraction.transformation import BaseExtractionConfig
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.types.extraction import (
|
||||
EXTRACTION_STATE_REWRITTEN_METADATA_KEY,
|
||||
ExtractionAnswer,
|
||||
ExtractionBase64Image,
|
||||
ExtractionField,
|
||||
ExtractionOptions,
|
||||
ExtractionRequestBody,
|
||||
ExtractionResponse,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders, is_litellm_owned_kwarg
|
||||
from litellm.utils import (
|
||||
ProviderConfigManager,
|
||||
client, # pyright: ignore[reportUnknownVariableType] # client is an untyped decorator
|
||||
get_model_cost_key,
|
||||
)
|
||||
|
||||
ExtractionFields: TypeAlias = Mapping[str, ExtractionField | Mapping[str, object]]
|
||||
ExtractionImages: TypeAlias = Sequence[str | ExtractionBase64Image | Mapping[str, object]]
|
||||
ExtractionOptionsParam: TypeAlias = ExtractionOptions | Mapping[str, object]
|
||||
|
||||
_REQUEST_ADAPTER: Final[TypeAdapter[ExtractionRequestBody]] = TypeAdapter(ExtractionRequestBody)
|
||||
_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
_HANDLER: Final = BaseLLMHTTPHandler()
|
||||
_LOCAL_KWARGS: Final = frozenset(
|
||||
(
|
||||
*GenericLiteLLMParams.model_fields,
|
||||
"extra_headers",
|
||||
"base_url",
|
||||
"default_headers",
|
||||
"deployment_id",
|
||||
"additional_drop_params",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, repr=False)
|
||||
class _ExtractionCall:
|
||||
model: str
|
||||
requested_model: str
|
||||
custom_llm_provider: str
|
||||
provider_config: BaseExtractionConfig
|
||||
body: Mapping[str, object]
|
||||
api_base: str
|
||||
api_key: str | None
|
||||
logging_obj: LiteLLMLoggingObj | None
|
||||
headers: Mapping[str, str]
|
||||
timeout: float | httpx.Timeout | None
|
||||
litellm_params: Mapping[str, object]
|
||||
client: HTTPHandler | AsyncHTTPHandler | None
|
||||
|
||||
|
||||
def supported_extraction_providers() -> tuple[str, ...]:
|
||||
return tuple(
|
||||
provider.value
|
||||
for provider in LlmProviders
|
||||
if ProviderConfigManager.get_provider_extraction_config(model="", provider=provider) is not None
|
||||
)
|
||||
|
||||
|
||||
def _provider_config(model: str, custom_llm_provider: str) -> BaseExtractionConfig:
|
||||
provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None)
|
||||
provider_config: Final = (
|
||||
None if provider is None else ProviderConfigManager.get_provider_extraction_config(model, provider)
|
||||
)
|
||||
if provider_config is None:
|
||||
raise litellm.BadRequestError(
|
||||
message=(
|
||||
f"Unknown Extraction provider '{custom_llm_provider}'. "
|
||||
f"Supported providers: {', '.join(supported_extraction_providers())}"
|
||||
),
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
)
|
||||
return provider_config
|
||||
|
||||
|
||||
def _request_fields(
|
||||
*,
|
||||
state: str | None,
|
||||
fields: ExtractionFields | None,
|
||||
images: ExtractionImages | None,
|
||||
reference_date: str | None,
|
||||
options: ExtractionOptionsParam | None,
|
||||
) -> Mapping[str, object]:
|
||||
sent: Final[Mapping[str, object | None]] = MappingProxyType(
|
||||
{"state": state, "fields": fields, "images": images, "reference_date": reference_date, "options": options}
|
||||
)
|
||||
return MappingProxyType({name: value for name, value in sent.items() if value is not None})
|
||||
|
||||
|
||||
def _is_unsupported(name: str, value: object, policy: DropPolicy) -> bool:
|
||||
return (
|
||||
value is not None
|
||||
and value is not False
|
||||
and name not in _LOCAL_KWARGS
|
||||
and not is_litellm_owned_kwarg(name)
|
||||
and not policy.drops(name)
|
||||
)
|
||||
|
||||
|
||||
def _unsupported_params(kwargs: Mapping[str, object]) -> tuple[str, ...]:
|
||||
policy: Final = drop_policy(kwargs)
|
||||
return tuple(sorted(name for name, value in kwargs.items() if _is_unsupported(name, value, policy)))
|
||||
|
||||
|
||||
def _prepare_call(
|
||||
*,
|
||||
model: str,
|
||||
request_fields: Mapping[str, object],
|
||||
api_key: str | None,
|
||||
api_base: str | None,
|
||||
timeout: float | httpx.Timeout | None,
|
||||
custom_llm_provider: str | None,
|
||||
extra_headers: Mapping[str, str] | None,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> _ExtractionCall:
|
||||
log_caller_request(kwargs, request_fields)
|
||||
upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
)
|
||||
provider_config: Final = _provider_config(model, provider)
|
||||
unsupported: Final = _unsupported_params(kwargs)
|
||||
if unsupported:
|
||||
raise litellm.UnsupportedParamsError(
|
||||
message=(
|
||||
f"{provider} does not support parameters: {list(unsupported)}, for model={model}. "
|
||||
"To drop these, set `litellm.drop_params=True` or for proxy:\n\n"
|
||||
"`litellm_settings:\n drop_params: true`\n"
|
||||
),
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
try:
|
||||
request: Final = _REQUEST_ADAPTER.validate_python(request_fields)
|
||||
except ValidationError as error:
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Invalid Extraction request: {error}", model=model, llm_provider=provider
|
||||
) from error
|
||||
canonical_model: Final = provider_config.canonical_model(upstream_model)
|
||||
resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base)
|
||||
if resolved_api_base is None:
|
||||
raise litellm.BadRequestError(
|
||||
message=provider_config.missing_api_base_message(provider), model=model, llm_provider=provider
|
||||
)
|
||||
resolved_api_key: Final = provider_config.resolve_api_key(dynamic_api_key or api_key)
|
||||
if resolved_api_key is None:
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Missing API key for Extraction provider '{provider}'", model=model, llm_provider=provider
|
||||
)
|
||||
logging_obj: Final = kwargs.get("litellm_logging_obj")
|
||||
http_client: Final = kwargs.get("client")
|
||||
if isinstance(logging_obj, LiteLLMLoggingObj):
|
||||
logging_obj.update_from_kwargs( # pyright: ignore[reportUnknownMemberType] # Logging takes untyped dicts
|
||||
kwargs=dict(kwargs),
|
||||
model=canonical_model,
|
||||
litellm_params={
|
||||
"litellm_call_id": kwargs.get("litellm_call_id"),
|
||||
"api_base": provider_config.get_complete_url(resolved_api_base, canonical_model),
|
||||
},
|
||||
custom_llm_provider=provider,
|
||||
)
|
||||
return _ExtractionCall(
|
||||
model=canonical_model,
|
||||
requested_model=model,
|
||||
custom_llm_provider=provider,
|
||||
provider_config=provider_config,
|
||||
body=provider_config.transform_extraction_request(canonical_model, request),
|
||||
api_base=resolved_api_base,
|
||||
api_key=resolved_api_key,
|
||||
logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None,
|
||||
headers=MappingProxyType(dict(extra_headers or {})),
|
||||
timeout=timeout,
|
||||
litellm_params=kwargs,
|
||||
client=http_client if isinstance(http_client, HTTPHandler | AsyncHTTPHandler) else None,
|
||||
)
|
||||
|
||||
|
||||
def _state_rewritten_by_guardrail(litellm_params: Mapping[str, object]) -> bool:
|
||||
metadata: Final = litellm_params.get(get_metadata_variable_name_from_kwargs(litellm_params))
|
||||
return (
|
||||
isinstance(metadata, Mapping)
|
||||
and _METADATA_ADAPTER.validate_python(metadata).get(EXTRACTION_STATE_REWRITTEN_METADATA_KEY) is True
|
||||
)
|
||||
|
||||
|
||||
def _without_spans(answers: Mapping[str, ExtractionAnswer]) -> Mapping[str, ExtractionAnswer]:
|
||||
return {name: answer.model_copy(update={"span": None, "spans": None}) for name, answer in answers.items()}
|
||||
|
||||
|
||||
def _formatted_response(response: ExtractionResponse, call: _ExtractionCall) -> ExtractionResponse:
|
||||
spans_index_hidden_text: Final = response.state_text is None and _state_rewritten_by_guardrail(call.litellm_params)
|
||||
formatted: Final = response.model_copy(
|
||||
update={
|
||||
"model": call.requested_model,
|
||||
**({"answers": _without_spans(response.answers)} if spans_index_hidden_text else {}),
|
||||
}
|
||||
)
|
||||
formatted.set_hidden_params(
|
||||
{
|
||||
"model": f"{call.custom_llm_provider}/{call.model}",
|
||||
"custom_llm_provider": call.custom_llm_provider,
|
||||
"provider_response_model": (
|
||||
get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
|
||||
or f"{call.custom_llm_provider}/{call.model}"
|
||||
),
|
||||
}
|
||||
)
|
||||
return formatted
|
||||
|
||||
|
||||
def _upstream_exception(error: Exception, call: _ExtractionCall) -> Exception:
|
||||
if isinstance(error, BaseLLMException) and error.status_code_is_synthesized:
|
||||
return litellm.APIConnectionError(
|
||||
message=f"{call.custom_llm_provider.capitalize()}Exception - {error.message}",
|
||||
llm_provider=call.custom_llm_provider,
|
||||
model=f"{call.custom_llm_provider}/{call.model}",
|
||||
)
|
||||
return litellm.exception_type( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped
|
||||
model=f"{call.custom_llm_provider}/{call.model}",
|
||||
custom_llm_provider=call.custom_llm_provider,
|
||||
original_exception=error,
|
||||
)
|
||||
|
||||
|
||||
async def _asend(call: _ExtractionCall) -> ExtractionResponse:
|
||||
try:
|
||||
response: Final = await _HANDLER.aextraction(
|
||||
model=call.model,
|
||||
custom_llm_provider=call.custom_llm_provider,
|
||||
logging_obj=call.logging_obj,
|
||||
provider_config=call.provider_config,
|
||||
body=call.body,
|
||||
api_base=call.api_base,
|
||||
api_key=call.api_key,
|
||||
headers=call.headers,
|
||||
timeout=call.timeout,
|
||||
litellm_params=call.litellm_params,
|
||||
client=call.client if isinstance(call.client, AsyncHTTPHandler) else None,
|
||||
)
|
||||
except Exception as error:
|
||||
raise _upstream_exception(error, call) from error
|
||||
return _formatted_response(response, call)
|
||||
|
||||
|
||||
def _send(call: _ExtractionCall) -> ExtractionResponse:
|
||||
try:
|
||||
response: Final = _HANDLER.extraction(
|
||||
model=call.model,
|
||||
custom_llm_provider=call.custom_llm_provider,
|
||||
logging_obj=call.logging_obj,
|
||||
provider_config=call.provider_config,
|
||||
body=call.body,
|
||||
api_base=call.api_base,
|
||||
api_key=call.api_key,
|
||||
headers=call.headers,
|
||||
timeout=call.timeout,
|
||||
litellm_params=call.litellm_params,
|
||||
client=call.client if isinstance(call.client, HTTPHandler) else None,
|
||||
)
|
||||
except Exception as error:
|
||||
raise _upstream_exception(error, call) from error
|
||||
return _formatted_response(response, call)
|
||||
|
||||
|
||||
@client
|
||||
async def aextraction(
|
||||
model: str,
|
||||
fields: ExtractionFields,
|
||||
state: str | None = None,
|
||||
images: ExtractionImages | None = None,
|
||||
reference_date: str | None = None,
|
||||
options: ExtractionOptionsParam | None = None,
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
extra_headers: Mapping[str, str] | None = None,
|
||||
**kwargs: object, # kwargs-ok: litellm_params forwarded by @client
|
||||
) -> ExtractionResponse:
|
||||
call: Final = await asyncio.to_thread(
|
||||
partial(
|
||||
_prepare_call,
|
||||
model=model,
|
||||
request_fields=_request_fields(
|
||||
state=state, fields=fields, images=images, reference_date=reference_date, options=options
|
||||
),
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
timeout=timeout,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
extra_headers=extra_headers,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
)
|
||||
return await _asend(call)
|
||||
|
||||
|
||||
@client
|
||||
def extraction(
|
||||
model: str,
|
||||
fields: ExtractionFields,
|
||||
state: str | None = None,
|
||||
images: ExtractionImages | None = None,
|
||||
reference_date: str | None = None,
|
||||
options: ExtractionOptionsParam | None = None,
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
extra_headers: Mapping[str, str] | None = None,
|
||||
**kwargs: object, # kwargs-ok: litellm_params forwarded by @client
|
||||
) -> ExtractionResponse:
|
||||
call: Final = _prepare_call(
|
||||
model=model,
|
||||
request_fields=_request_fields(
|
||||
state=state, fields=fields, images=images, reference_date=reference_date, options=options
|
||||
),
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
timeout=timeout,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
extra_headers=extra_headers,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
return _send(call)
|
||||
|
||||
|
||||
__all__ = ["aextraction", "extraction"]
|
||||
|
|
@ -37,6 +37,7 @@ from litellm.harness.context import GatewayTarget
|
|||
from litellm.harness.errors import HarnessError, HarnessInstallFailed
|
||||
from litellm.harness.types import Harness, Usage
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
|
||||
from litellm.types.llms.custom_http import httpxSpecialProvider
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -690,18 +691,21 @@ class ModelEndpoint:
|
|||
async def _invoke_sdk(
|
||||
self,
|
||||
route: str,
|
||||
kwargs: dict[str, Any], # mutable-ok: injects stream_options into the SDK kwargs
|
||||
kwargs: dict[str, Any], # mutable-ok: SDK call kwargs built by _sdk_kwargs
|
||||
) -> object:
|
||||
if route == ROUTE_MESSAGES:
|
||||
return await litellm.anthropic.messages.acreate(**kwargs)
|
||||
if route == ROUTE_CHAT:
|
||||
if kwargs.get("stream"):
|
||||
stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field
|
||||
kwargs["stream_options"] = { # mutable-ok: JSON field sent to litellm.acompletion
|
||||
"include_usage": True,
|
||||
**stream_options,
|
||||
if not kwargs.get("stream"):
|
||||
return await litellm.acompletion(**kwargs)
|
||||
stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field
|
||||
return await litellm.acompletion(
|
||||
**{
|
||||
**kwargs,
|
||||
"stream_options": {"include_usage": True, **stream_options},
|
||||
STREAM_USAGE_INJECTED_FIELD: "include_usage" not in stream_options,
|
||||
}
|
||||
return await litellm.acompletion(**kwargs)
|
||||
)
|
||||
return await litellm.aresponses(**kwargs)
|
||||
|
||||
async def _call_sdk(self, route: str, body: Mapping[str, object]) -> Response:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ Helper functions for health check calls.
|
|||
|
||||
import base64
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal
|
||||
|
||||
from litellm.llms.base_llm.ocr.transformation import DocumentType
|
||||
|
|
@ -25,6 +26,9 @@ IMAGE_EDIT_HEALTH_CHECK_PROMPT: Final = (
|
|||
)
|
||||
|
||||
ANTHROPIC_MESSAGES_HEALTH_CHECK_MAX_TOKENS: Final = 16
|
||||
EXTRACTION_HEALTH_CHECK_FIELDS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType(
|
||||
{"reachable": MappingProxyType({"type": "text", "instructions": "Any word from the text"})}
|
||||
)
|
||||
|
||||
|
||||
def native_health_check_mode(model: str, custom_llm_provider: str | None) -> Literal["anthropic_messages"] | None:
|
||||
|
|
@ -215,6 +219,7 @@ class HealthCheckHelpers:
|
|||
"anthropic_messages",
|
||||
"ocr",
|
||||
"decisions",
|
||||
"extraction",
|
||||
],
|
||||
Callable,
|
||||
]:
|
||||
|
|
@ -237,7 +242,11 @@ class HealthCheckHelpers:
|
|||
from litellm.litellm_core_utils.audio_utils.utils import (
|
||||
get_audio_file_for_health_check,
|
||||
)
|
||||
from litellm.litellm_core_utils.health_check_utils import DECISIONS_CALL_PARAMS, _filter_model_params
|
||||
from litellm.litellm_core_utils.health_check_utils import (
|
||||
DECISIONS_CALL_PARAMS,
|
||||
EXTRACTION_CALL_PARAMS,
|
||||
_filter_model_params,
|
||||
)
|
||||
from litellm.realtime_api.main import realtime_health_check
|
||||
|
||||
return {
|
||||
|
|
@ -322,4 +331,13 @@ class HealthCheckHelpers:
|
|||
}
|
||||
)
|
||||
),
|
||||
"extraction": lambda: litellm.aextraction(
|
||||
**EXTRACTION_CALL_PARAMS.validate_python(
|
||||
{
|
||||
"state": prompt or "health check",
|
||||
"fields": EXTRACTION_HEALTH_CHECK_FIELDS,
|
||||
**_filter_model_params(model_params=model_params),
|
||||
}
|
||||
)
|
||||
),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,8 +8,10 @@ from typing import Final
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.types.decisions import DecisionsCallParams
|
||||
from litellm.types.extraction import ExtractionCallParams
|
||||
|
||||
DECISIONS_CALL_PARAMS: Final[TypeAdapter[DecisionsCallParams]] = TypeAdapter(DecisionsCallParams)
|
||||
EXTRACTION_CALL_PARAMS: Final[TypeAdapter[ExtractionCallParams]] = TypeAdapter(ExtractionCallParams)
|
||||
OPTIONAL_STR: Final[TypeAdapter[str | None]] = TypeAdapter(str | None)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import subprocess
|
|||
import sys
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime as dt_object
|
||||
from functools import lru_cache
|
||||
|
|
@ -132,11 +132,13 @@ from litellm.responses.utils import ResponseAPILoggingUtils
|
|||
from litellm.types.agents import LiteLLMSendMessageResponse
|
||||
from litellm.types.containers.main import ContainerObject
|
||||
from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse
|
||||
from litellm.types.extraction import ExtractionResponse
|
||||
from litellm.types.integrations.s3_v2 import S3PartitionGranularity
|
||||
from litellm.types.interactions import (
|
||||
InteractionsAPIResponse,
|
||||
InteractionsAPIStreamingResponse,
|
||||
)
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
Batch,
|
||||
|
|
@ -573,6 +575,27 @@ class UncachedPrice:
|
|||
cost_breakdown: CostBreakdown | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BilledRoundsSpend:
|
||||
usage: Usage
|
||||
cost: float
|
||||
cost_breakdown: CostBreakdown | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BilledRoundsDispatch:
|
||||
"""A parked success log for a stream bridged through chat completions that billed several rounds, and the spend
|
||||
its failure log bills when a post-call hook rejects the response, as it does for a rejected one-round bridged
|
||||
stream."""
|
||||
|
||||
dispatch: Coroutine[object, object, None]
|
||||
spend: BilledRoundsSpend
|
||||
|
||||
|
||||
def _fixed_margin(cost_breakdown: CostBreakdown | None) -> float:
|
||||
return 0.0 if cost_breakdown is None else cost_breakdown.get("margin_fixed_amount", 0.0)
|
||||
|
||||
|
||||
def _timestamp_seconds(moment: object) -> float | None:
|
||||
if isinstance(moment, datetime.datetime):
|
||||
return moment.timestamp()
|
||||
|
|
@ -594,6 +617,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
prometheusLogger, \
|
||||
slack_app
|
||||
custom_pricing: bool = False
|
||||
replaced_global_callback_keys: frozenset[str] = frozenset()
|
||||
stream_options = None
|
||||
litellm_request_debug: bool = False
|
||||
streamed_anthropic_message_id: str | None = None
|
||||
|
|
@ -648,11 +672,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
litellm_call_id: str,
|
||||
function_id: str,
|
||||
litellm_trace_id: str | None = None,
|
||||
dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = None,
|
||||
dynamic_input_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
|
||||
dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
|
||||
dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
|
||||
dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = None,
|
||||
dynamic_async_failure_callbacks: list[str | Callable | CustomLogger] | None = None,
|
||||
dynamic_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
|
||||
dynamic_async_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
|
||||
applied_guardrails: list[str] | None = None,
|
||||
kwargs: dict | None = None,
|
||||
log_raw_request_response: bool = False,
|
||||
|
|
@ -697,6 +721,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self._pre_call_session_id: str = session_id_var.get()
|
||||
_sid: Final = kwargs.get("litellm_session_id") if kwargs else None
|
||||
self.litellm_session_id: str = str(_sid) if _sid else ""
|
||||
self.stream_usage_injected: Final = kwargs is not None and kwargs.get(STREAM_USAGE_INJECTED_FIELD) is True
|
||||
# supports_correlation_logging is False for calls originating from the
|
||||
# sync client entry point (wrapper() in utils.py): a plain OS thread
|
||||
# has no per-call context isolation the way an asyncio Task does, and
|
||||
|
|
@ -725,15 +750,17 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.raw_request_only = raw_request_only
|
||||
|
||||
# Initialize dynamic callbacks
|
||||
self.dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = dynamic_input_callbacks
|
||||
self.dynamic_input_callbacks: list[str | Callable[..., object] | CustomLogger] | None = dynamic_input_callbacks
|
||||
self.dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
|
||||
dynamic_success_callbacks
|
||||
)
|
||||
self.dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
|
||||
dynamic_async_success_callbacks
|
||||
)
|
||||
self.dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = dynamic_failure_callbacks
|
||||
self.dynamic_async_failure_callbacks: list[str | Callable | CustomLogger] | None = (
|
||||
self.dynamic_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
|
||||
dynamic_failure_callbacks
|
||||
)
|
||||
self.dynamic_async_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
|
||||
dynamic_async_failure_callbacks
|
||||
)
|
||||
|
||||
|
|
@ -850,10 +877,32 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.dynamic_async_failure_callbacks, callback
|
||||
)
|
||||
|
||||
def replace_global_callback(self, callback: CustomLogger) -> None:
|
||||
self.add_dynamic_callback(callback)
|
||||
self.replaced_global_callback_keys = self.replaced_global_callback_keys | {
|
||||
litellm.logging_callback_manager.get_custom_logger_key(callback)
|
||||
}
|
||||
|
||||
def add_dynamic_function_callbacks(
|
||||
self,
|
||||
*,
|
||||
async_success: Callable[..., object],
|
||||
failure: Callable[..., object],
|
||||
async_failure: Callable[..., object],
|
||||
) -> None:
|
||||
self.dynamic_async_success_callbacks = self._with_dynamic_callback(
|
||||
self.dynamic_async_success_callbacks, async_success
|
||||
)
|
||||
self.dynamic_failure_callbacks = self._with_dynamic_callback(self.dynamic_failure_callbacks, failure)
|
||||
self.dynamic_async_failure_callbacks = self._with_dynamic_callback(
|
||||
self.dynamic_async_failure_callbacks, async_failure
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _with_dynamic_callback(
|
||||
callbacks: Sequence[str | Callable | CustomLogger] | None, callback: CustomLogger
|
||||
) -> list[str | Callable | CustomLogger]:
|
||||
callbacks: Sequence[str | Callable[..., object] | CustomLogger] | None,
|
||||
callback: Callable[..., object] | CustomLogger,
|
||||
) -> list[str | Callable[..., object] | CustomLogger]:
|
||||
existing: Final = tuple(callbacks or ())
|
||||
return [*existing, *(() if callback in existing else (callback,))]
|
||||
|
||||
|
|
@ -890,9 +939,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
def _process_dynamic_callback_list(
|
||||
self,
|
||||
callback_list: list[str | Callable | CustomLogger] | None,
|
||||
callback_list: list[str | Callable[..., object] | CustomLogger] | None,
|
||||
dynamic_callbacks_type: Literal["input", "success", "failure", "async_success", "async_failure"],
|
||||
) -> list[str | Callable | CustomLogger] | None:
|
||||
) -> list[str | Callable[..., object] | CustomLogger] | None:
|
||||
"""
|
||||
Helper function to initialize CustomLogger compatible callbacks in self.dynamic_* callbacks
|
||||
|
||||
|
|
@ -904,7 +953,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if callback_list is None:
|
||||
return None
|
||||
|
||||
processed_list: Final[list[str | Callable | CustomLogger]] = []
|
||||
processed_list: Final[list[str | Callable[..., object] | CustomLogger]] = []
|
||||
for callback in callback_list:
|
||||
if isinstance(callback, str) and callback in litellm._known_custom_logger_compatible_callbacks:
|
||||
for callback_instance in self._resolve_dynamic_callback_string(callback):
|
||||
|
|
@ -1565,7 +1614,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"api_call_start_time"
|
||||
]
|
||||
# Input Integration Logging -> If you want to log the fact that an attempt to call the model was made
|
||||
callbacks: Final = litellm.input_callback + (self.dynamic_input_callbacks or [])
|
||||
callbacks: Final = (
|
||||
*self._without_replaced_global_callbacks(litellm.input_callback),
|
||||
*(self.dynamic_input_callbacks or ()),
|
||||
)
|
||||
for callback in callbacks:
|
||||
try:
|
||||
if callback == "supabase" and supabaseClient is not None:
|
||||
|
|
@ -1738,7 +1790,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
# Input Integration Logging -> If you want to log the fact that an attempt to call the model was made
|
||||
|
||||
callbacks: Final = litellm.input_callback + (self.dynamic_input_callbacks or [])
|
||||
callbacks: Final = (
|
||||
*self._without_replaced_global_callbacks(litellm.input_callback),
|
||||
*(self.dynamic_input_callbacks or ()),
|
||||
)
|
||||
for callback in callbacks:
|
||||
try:
|
||||
if callback == "sentry" and add_breadcrumb:
|
||||
|
|
@ -2209,20 +2264,23 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
usage=usage,
|
||||
)
|
||||
|
||||
def custom_pricing_for(self, result: object) -> bool:
|
||||
litellm_params: Final = getattr(self, "litellm_params", None)
|
||||
def spilled_over_ptu_for(self, result: object) -> bool:
|
||||
result_hidden_params: Final = getattr(result, HIDDEN_PARAMS_ATTR, None) or MappingProxyType({})
|
||||
additional_headers: Final = (
|
||||
result_hidden_params.get("additional_headers")
|
||||
if isinstance(result_hidden_params, dict)
|
||||
else getattr(result_hidden_params, "additional_headers", None)
|
||||
)
|
||||
spilled_over: Final = is_spilled_over_ptu_request(
|
||||
model_info=_deployment_model_info(litellm_params),
|
||||
return is_spilled_over_ptu_request(
|
||||
model_info=_deployment_model_info(getattr(self, "litellm_params", None)),
|
||||
response_headers=self.model_call_details.get("response_headers"),
|
||||
additional_headers=additional_headers,
|
||||
)
|
||||
return False if spilled_over else use_custom_pricing_for_model(litellm_params=litellm_params)
|
||||
|
||||
def custom_pricing_for(self, result: object) -> bool:
|
||||
if self.spilled_over_ptu_for(result):
|
||||
return False
|
||||
return use_custom_pricing_for_model(litellm_params=getattr(self, "litellm_params", None))
|
||||
|
||||
def _prompt_for_cost_calculation(self) -> str:
|
||||
"""
|
||||
|
|
@ -2384,6 +2442,31 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.model_call_details["combined_usage_object"] = usage
|
||||
self.model_call_details["response_cost"] = response_cost
|
||||
|
||||
def record_failure_after_earlier_rounds(
|
||||
self, earlier: BilledRoundsSpend, failed_usage: Usage, failed_cost: float
|
||||
) -> None:
|
||||
"""Bill a failed round on the failure log after the rounds of its request before it were billed on the success
|
||||
log.
|
||||
|
||||
The fixed margin is a per-request fee, so it is charged once: the failed round's margin is dropped when the
|
||||
earlier rounds already paid one."""
|
||||
from litellm.responses.mcp.round_billing import breakdown_less_repeated_fixed_margin
|
||||
|
||||
failed_breakdown: Final = None if self.cost_breakdown is earlier.cost_breakdown else self.cost_breakdown
|
||||
repeated_fixed_margin: Final = (
|
||||
0.0
|
||||
if failed_breakdown is None
|
||||
else min(_fixed_margin(earlier.cost_breakdown), _fixed_margin(failed_breakdown), earlier.cost, failed_cost)
|
||||
)
|
||||
self.model_call_details["combined_usage_object"] = failed_usage
|
||||
self.model_call_details["response_cost"] = failed_cost - repeated_fixed_margin
|
||||
self.cost_breakdown = breakdown_less_repeated_fixed_margin(failed_breakdown, repeated_fixed_margin)
|
||||
|
||||
def record_billed_rounds_for_failure(self, spend: BilledRoundsSpend) -> None:
|
||||
"""Bill every round of a fully streamed request on the failure log when a post-call hook rejects it."""
|
||||
self.record_partial_usage_for_failure(spend.usage, spend.cost)
|
||||
self.cost_breakdown = spend.cost_breakdown
|
||||
|
||||
def attempt_scoped_copy(self) -> "Logging":
|
||||
"""A copy with its own model_call_details, so logging one attempt of a request through it leaves
|
||||
the state the request's other attempts share untouched."""
|
||||
|
|
@ -2919,6 +3002,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
or isinstance(logging_result, SearchResponse) # Search API
|
||||
or isinstance(logging_result, DecisionsResponse)
|
||||
or isinstance(logging_result, OpenAIDecisionResponse)
|
||||
or isinstance(logging_result, ExtractionResponse)
|
||||
or (
|
||||
isinstance(logging_result, InteractionsAPIResponse)
|
||||
and logging_result.usage is not None
|
||||
|
|
@ -4511,14 +4595,26 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
_filtered_failure_callbacks = self._remove_internal_litellm_callbacks(_filtered_failure_callbacks)
|
||||
return len(_filtered_failure_callbacks) > 0
|
||||
|
||||
def _without_replaced_global_callbacks(
|
||||
self, global_callbacks: Iterable[str | Callable[..., object] | CustomLogger]
|
||||
) -> tuple[str | Callable[..., object] | CustomLogger, ...]:
|
||||
if not self.replaced_global_callback_keys:
|
||||
return tuple(global_callbacks)
|
||||
return tuple(
|
||||
litellm.logging_callback_manager.without_custom_logger_keys(
|
||||
global_callbacks, self.replaced_global_callback_keys
|
||||
)
|
||||
)
|
||||
|
||||
def get_combined_callback_list(
|
||||
self,
|
||||
dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None,
|
||||
global_callbacks: list[str | Callable[..., object] | CustomLogger],
|
||||
) -> list[str | Callable[..., object] | CustomLogger]:
|
||||
kept_global_callbacks: Final = self._without_replaced_global_callbacks(global_callbacks)
|
||||
if dynamic_success_callbacks is None:
|
||||
return list(global_callbacks)
|
||||
return list(dict.fromkeys(dynamic_success_callbacks + global_callbacks))
|
||||
return list(kept_global_callbacks)
|
||||
return list(dict.fromkeys((*dynamic_success_callbacks, *kept_global_callbacks)))
|
||||
|
||||
def _remove_internal_litellm_callbacks(self, callbacks: list) -> list:
|
||||
"""
|
||||
|
|
@ -7245,13 +7341,16 @@ def get_standard_logging_object_payload(
|
|||
saved_cache_cost: Final = (uncached_price.cost if uncached_price is not None else None) or 0.0
|
||||
|
||||
## Get model cost information ##
|
||||
base_model = get_base_model_from_metadata(model_call_details=kwargs)
|
||||
# The router overrides completion_response.model to the model-group alias before
|
||||
# this payload is built, so cost-map lookup via that alias always misses.
|
||||
# Fall back to the actual deployment model set by the router in metadata.
|
||||
if base_model is None:
|
||||
base_model = metadata.get("deployment")
|
||||
custom_pricing: Final = logging_obj.custom_pricing_for(init_response_obj)
|
||||
configured_base_model: Final = get_base_model_from_metadata(model_call_details=kwargs)
|
||||
spilled_over: Final = logging_obj.spilled_over_ptu_for(init_response_obj)
|
||||
# The router overrides completion_response.model to the model-group alias before this payload is
|
||||
# built, so cost-map lookup via that alias misses and the deployment model stands in for base_model.
|
||||
# A PTU request Azure spilled onto pay-as-you-go capacity is billed at the served model's rates,
|
||||
# which the deployment's zeroed PTU entry would misreport, so that fallback is skipped there.
|
||||
base_model: Final = (
|
||||
metadata.get("deployment") if configured_base_model is None and not spilled_over else configured_base_model
|
||||
)
|
||||
custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj)
|
||||
raw_response_cost: Final = kwargs.get("response_cost")
|
||||
llm_response_cost: Final[float] = raw_response_cost or 0.0
|
||||
guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from functools import reduce
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.constants import LITELLM_DETAILED_TIMING
|
||||
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers
|
||||
|
|
@ -101,6 +102,32 @@ def response_timing_metrics(
|
|||
return {"_response_ms": total_response_time_ms, "litellm_overhead_time_ms": overhead_ms}
|
||||
|
||||
|
||||
_HIDDEN_PARAMS_DICT: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def _hidden_params_mapping(hidden_params: object) -> Mapping[str, object]:
|
||||
try:
|
||||
return _HIDDEN_PARAMS_DICT.validate_python(hidden_params)
|
||||
except ValidationError:
|
||||
return {}
|
||||
|
||||
|
||||
def _existing_provider_response_model(result: object) -> str | None:
|
||||
"""The pricing model a response already carries, like the decisions path sets, wins over the response model."""
|
||||
hidden_params: Final[object] = getattr(result, "_hidden_params", None)
|
||||
carried: Final[object] = _hidden_params_mapping(hidden_params).get("provider_response_model") or getattr(
|
||||
hidden_params, "provider_response_model", None
|
||||
)
|
||||
return carried if isinstance(carried, str) and carried else None
|
||||
|
||||
|
||||
def _provider_response_model(result: object) -> str | None:
|
||||
"""The model the provider answered with, kept because the proxy later rewrites ``result.model`` to the alias the
|
||||
client asked for and cost-map lookups for the standard logging payload would otherwise miss."""
|
||||
model: Final = getattr(result, "model", None) if isinstance(result, ModelResponse) else None
|
||||
return model if isinstance(model, str) and model else None
|
||||
|
||||
|
||||
class ResponseMetadata:
|
||||
"""
|
||||
Handles setting and managing `_hidden_params`, `response_time_ms`, and `litellm_overhead_time_ms` for LiteLLM responses
|
||||
|
|
@ -147,6 +174,8 @@ class ResponseMetadata:
|
|||
preserve_litellm_internal_headers=True,
|
||||
),
|
||||
"litellm_model_name": model,
|
||||
"provider_response_model": _existing_provider_response_model(self.result)
|
||||
or _provider_response_model(self.result),
|
||||
}
|
||||
self._update_hidden_params(new_params)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
from collections.abc import Callable
|
||||
import threading
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from types import MethodType
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import litellm
|
||||
|
|
@ -15,6 +17,7 @@ else:
|
|||
_custom_logger_compatible_callbacks_literal = str
|
||||
|
||||
_generic_api_logger_cache: Final[dict[str, GenericAPILogger]] = {}
|
||||
_callback_lists_lock: Final = threading.RLock()
|
||||
|
||||
|
||||
class LoggingCallbackManager:
|
||||
|
|
@ -43,16 +46,18 @@ class LoggingCallbackManager:
|
|||
Add a input callback to litellm.input_callback.
|
||||
Auto-routes async callbacks to litellm._async_input_callback.
|
||||
"""
|
||||
if not isinstance(callback, str) and self._is_async_callable(callback):
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_input_callback)
|
||||
else:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.input_callback)
|
||||
with _callback_lists_lock:
|
||||
if not isinstance(callback, str) and self._is_async_callable(callback):
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_input_callback)
|
||||
else:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.input_callback)
|
||||
|
||||
def add_litellm_service_callback(self, callback: CustomLogger | str | Callable[..., object]):
|
||||
"""
|
||||
Add a service callback to litellm.service_callback
|
||||
"""
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.service_callback)
|
||||
with _callback_lists_lock:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.service_callback)
|
||||
|
||||
def add_litellm_callback(self, callback: CustomLogger | str | Callable[..., object]):
|
||||
"""
|
||||
|
|
@ -60,10 +65,11 @@ class LoggingCallbackManager:
|
|||
|
||||
Ensures no duplicates are added.
|
||||
"""
|
||||
self._safe_add_callback_to_list(
|
||||
callback=callback,
|
||||
parent_list=litellm.callbacks,
|
||||
)
|
||||
with _callback_lists_lock:
|
||||
self._safe_add_callback_to_list(
|
||||
callback=callback,
|
||||
parent_list=litellm.callbacks,
|
||||
)
|
||||
|
||||
def add_litellm_success_callback(self, callback: CustomLogger | str | Callable[..., object]):
|
||||
"""
|
||||
|
|
@ -71,37 +77,41 @@ class LoggingCallbackManager:
|
|||
Auto-routes async callbacks to litellm._async_success_callback.
|
||||
Special-cases 'dynamodb' and 'openmeter' as async callbacks.
|
||||
"""
|
||||
if (
|
||||
isinstance(callback, str)
|
||||
and callback in ("dynamodb", "openmeter")
|
||||
or not isinstance(callback, str)
|
||||
and self._is_async_callable(callback)
|
||||
):
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback)
|
||||
else:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.success_callback)
|
||||
with _callback_lists_lock:
|
||||
if (
|
||||
isinstance(callback, str)
|
||||
and callback in ("dynamodb", "openmeter")
|
||||
or not isinstance(callback, str)
|
||||
and self._is_async_callable(callback)
|
||||
):
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback)
|
||||
else:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.success_callback)
|
||||
|
||||
def add_litellm_failure_callback(self, callback: CustomLogger | str | Callable[..., object]):
|
||||
"""
|
||||
Add a failure callback to `litellm.failure_callback`.
|
||||
Auto-routes async callbacks to litellm._async_failure_callback.
|
||||
"""
|
||||
if not isinstance(callback, str) and self._is_async_callable(callback):
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback)
|
||||
else:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.failure_callback)
|
||||
with _callback_lists_lock:
|
||||
if not isinstance(callback, str) and self._is_async_callable(callback):
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback)
|
||||
else:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.failure_callback)
|
||||
|
||||
def add_litellm_async_success_callback(self, callback: CustomLogger | Callable[..., object] | str):
|
||||
"""
|
||||
Add a success callback to litellm._async_success_callback
|
||||
"""
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback)
|
||||
with _callback_lists_lock:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback)
|
||||
|
||||
def add_litellm_async_failure_callback(self, callback: CustomLogger | Callable[..., object] | str):
|
||||
"""
|
||||
Add a failure callback to litellm._async_failure_callback
|
||||
"""
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback)
|
||||
with _callback_lists_lock:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback)
|
||||
|
||||
def remove_callback_from_list_by_object(self, callback_list, obj, require_self=True):
|
||||
"""
|
||||
|
|
@ -110,13 +120,14 @@ class LoggingCallbackManager:
|
|||
if not isinstance(callback_list, list): # Not list -> do nothing
|
||||
return
|
||||
|
||||
if require_self:
|
||||
remove_list = [c for c in callback_list if hasattr(c, "__self__") and c.__self__ == obj]
|
||||
else:
|
||||
remove_list = [c for c in callback_list if c == obj]
|
||||
with _callback_lists_lock:
|
||||
if require_self:
|
||||
remove_list = [c for c in callback_list if hasattr(c, "__self__") and c.__self__ == obj]
|
||||
else:
|
||||
remove_list = [c for c in callback_list if c == obj]
|
||||
|
||||
for c in remove_list:
|
||||
callback_list.remove(c)
|
||||
for c in remove_list:
|
||||
callback_list.remove(c)
|
||||
|
||||
def remove_callbacks_by_type(self, callback_list, callback_type):
|
||||
"""
|
||||
|
|
@ -134,10 +145,11 @@ class LoggingCallbackManager:
|
|||
if not isinstance(callback_list, list):
|
||||
return
|
||||
|
||||
remove_list: Final = [c for c in callback_list if isinstance(c, callback_type)]
|
||||
with _callback_lists_lock:
|
||||
remove_list: Final = [c for c in callback_list if isinstance(c, callback_type)]
|
||||
|
||||
for c in remove_list:
|
||||
callback_list.remove(c)
|
||||
for c in remove_list:
|
||||
callback_list.remove(c)
|
||||
|
||||
def _add_string_callback_to_list(
|
||||
self, callback: str, parent_list: list[CustomLogger | Callable[..., object] | str]
|
||||
|
|
@ -336,6 +348,55 @@ class LoggingCallbackManager:
|
|||
|
||||
return "-".join(key_parts)
|
||||
|
||||
get_custom_logger_key = _get_custom_logger_key
|
||||
callback_lists_lock: Final = _callback_lists_lock
|
||||
|
||||
def drop_callbacks(self, callbacks: Iterable[object], method_owners: Iterable[object] = ()) -> frozenset[str]:
|
||||
dropped_ids: Final = frozenset(map(id, callbacks))
|
||||
owner_ids: Final = frozenset(map(id, method_owners))
|
||||
|
||||
def keep(callback: object) -> bool:
|
||||
if id(callback) in dropped_ids:
|
||||
return False
|
||||
return not (isinstance(callback, MethodType) and id(callback.__self__) in owner_ids)
|
||||
|
||||
with _callback_lists_lock:
|
||||
vacated_keys: Final = frozenset(
|
||||
self._get_custom_logger_key(callback)
|
||||
for callback in litellm.callbacks
|
||||
if isinstance(callback, CustomLogger) and id(callback) in dropped_ids
|
||||
)
|
||||
litellm.callbacks = [callback for callback in litellm.callbacks if keep(callback)]
|
||||
litellm.input_callback = [callback for callback in litellm.input_callback if keep(callback)]
|
||||
litellm.success_callback = [callback for callback in litellm.success_callback if keep(callback)]
|
||||
litellm.failure_callback = [callback for callback in litellm.failure_callback if keep(callback)]
|
||||
litellm.service_callback = [callback for callback in litellm.service_callback if keep(callback)]
|
||||
litellm._async_success_callback = [
|
||||
callback for callback in litellm._async_success_callback if keep(callback)
|
||||
]
|
||||
litellm._async_failure_callback = [
|
||||
callback for callback in litellm._async_failure_callback if keep(callback)
|
||||
]
|
||||
return vacated_keys
|
||||
|
||||
def with_custom_logger_keys(
|
||||
self, candidates: Iterable[CustomLogger], keys: frozenset[str]
|
||||
) -> tuple[CustomLogger, ...]:
|
||||
if not keys:
|
||||
return ()
|
||||
class_names: Final = frozenset(key.split("-", 1)[0] for key in keys)
|
||||
same_class: Final = (candidate for candidate in candidates if type(candidate).__name__ in class_names)
|
||||
return tuple(candidate for candidate in same_class if self._get_custom_logger_key(candidate) in keys)
|
||||
|
||||
def without_custom_logger_keys(
|
||||
self, callbacks: Iterable[str | Callable[..., object] | CustomLogger], keys: frozenset[str]
|
||||
) -> Iterator[str | Callable[..., object] | CustomLogger]:
|
||||
return (
|
||||
callback
|
||||
for callback in callbacks
|
||||
if not (isinstance(callback, CustomLogger) and self._get_custom_logger_key(callback) in keys)
|
||||
)
|
||||
|
||||
def _reset_all_callbacks(self):
|
||||
"""
|
||||
Reset all callbacks to an empty list
|
||||
|
|
@ -363,19 +424,21 @@ class LoggingCallbackManager:
|
|||
|
||||
_get_all_callbacks = get_all_callbacks
|
||||
|
||||
def remove_callback_from_all_lists(self, obj, require_self=False) -> None:
|
||||
def remove_callback_from_all_lists(self, obj: object, require_self: bool = False) -> None:
|
||||
"""
|
||||
Remove a callback object from every callback list it may have been
|
||||
promoted into, so a re-initialized callback leaves no stale instance behind.
|
||||
"""
|
||||
for callback_list in (
|
||||
litellm.callbacks,
|
||||
litellm.success_callback,
|
||||
litellm.failure_callback,
|
||||
litellm._async_success_callback,
|
||||
litellm._async_failure_callback,
|
||||
):
|
||||
self.remove_callback_from_list_by_object(callback_list, obj, require_self=require_self)
|
||||
with _callback_lists_lock:
|
||||
for callback_list in (
|
||||
litellm.callbacks,
|
||||
litellm.input_callback,
|
||||
litellm.success_callback,
|
||||
litellm.failure_callback,
|
||||
litellm._async_success_callback,
|
||||
litellm._async_failure_callback,
|
||||
):
|
||||
self.remove_callback_from_list_by_object(callback_list, obj, require_self=require_self)
|
||||
|
||||
def get_active_additional_logging_utils_from_custom_logger(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import traceback
|
|||
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, NoReturn, Protocol, TypeVar, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeVar, cast
|
||||
|
||||
import anyio
|
||||
import httpx
|
||||
|
|
@ -60,6 +60,9 @@ from .exception_mapping_utils import exception_type, with_deployment_api_base
|
|||
from .llm_response_utils.get_api_base import get_api_base
|
||||
from .rules import Rules
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .litellm_logging import BilledRoundsSpend
|
||||
|
||||
# Constants for special delta attribute names
|
||||
AUDIO_ATTRIBUTE: Final = "audio"
|
||||
IMAGE_ATTRIBUTE: Final = "images"
|
||||
|
|
@ -259,6 +262,8 @@ def _provider_hidden_params(
|
|||
|
||||
|
||||
class CustomStreamWrapper:
|
||||
_earlier_rounds_spend: "BilledRoundsSpend | None" = None
|
||||
|
||||
@property
|
||||
def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation
|
||||
return self._hidden_params
|
||||
|
|
@ -342,6 +347,10 @@ class CustomStreamWrapper:
|
|||
self.messages = getattr(logging_obj, "messages", None)
|
||||
self.sent_stream_usage = False
|
||||
self.send_stream_usage = True if self.check_send_stream_usage(self.stream_options) else False
|
||||
self.caller_requested_stream_usage: Final = (
|
||||
self.check_send_stream_usage(getattr(logging_obj, "stream_options", None))
|
||||
and getattr(logging_obj, "stream_usage_injected", False) is not True
|
||||
)
|
||||
self.tool_call = False
|
||||
# keep track of the returned chunks - used for calculating the input/output tokens for stream options
|
||||
self.chunks: list[ModelResponseStream] = []
|
||||
|
|
@ -2070,7 +2079,7 @@ class CustomStreamWrapper:
|
|||
except StopIteration:
|
||||
if self.sent_last_chunk is True:
|
||||
complete_streaming_response: Final = self._build_complete_streaming_response()
|
||||
response = self.model_response_creator()
|
||||
response = self._new_usage_chunk()
|
||||
if complete_streaming_response is not None:
|
||||
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)
|
||||
|
||||
|
|
@ -2222,27 +2231,10 @@ class CustomStreamWrapper:
|
|||
)
|
||||
|
||||
if _has_usage:
|
||||
# Store a copy ONLY when usage stripping below will mutate
|
||||
# the chunk. For non-usage chunks (vast majority), store
|
||||
# directly to avoid expensive model_copy() per chunk.
|
||||
self.chunks.append(processed_chunk.model_copy())
|
||||
|
||||
# Strip usage from the outgoing chunk so it's not sent twice
|
||||
# (once in the chunk, once in _hidden_params).
|
||||
obj_dict = processed_chunk.model_dump()
|
||||
if "usage" in obj_dict:
|
||||
del obj_dict["usage"]
|
||||
processed_chunk = self.model_response_creator(
|
||||
chunk=obj_dict,
|
||||
hidden_params=cast( # cast-ok: preserve mapping operations on dynamic storage
|
||||
Mapping[str, object], getattr(processed_chunk, HIDDEN_PARAMS_ATTR)
|
||||
),
|
||||
)
|
||||
is_empty = is_model_response_stream_empty(
|
||||
model_response=cast(ModelResponseStream, processed_chunk)
|
||||
)
|
||||
if is_empty:
|
||||
usage_free_chunk = self._record_and_strip_usage_chunk(processed_chunk)
|
||||
if usage_free_chunk is None:
|
||||
continue
|
||||
processed_chunk = usage_free_chunk
|
||||
else:
|
||||
# No usage data — safe to store directly without copying
|
||||
self.chunks.append(processed_chunk)
|
||||
|
|
@ -2273,13 +2265,18 @@ class CustomStreamWrapper:
|
|||
if processed_chunk is None:
|
||||
continue
|
||||
|
||||
choice = processed_chunk.choices[0]
|
||||
if isinstance(choice, StreamingChoices):
|
||||
self._accumulate_streamed_delta(choice.delta)
|
||||
if processed_chunk.choices:
|
||||
choice = processed_chunk.choices[0]
|
||||
if isinstance(choice, StreamingChoices):
|
||||
self._accumulate_streamed_delta(choice.delta)
|
||||
self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
|
||||
# RETURN RESULT
|
||||
self.chunks.append(processed_chunk)
|
||||
return processed_chunk
|
||||
if not self.send_stream_usage or getattr(processed_chunk, "usage", None) is None:
|
||||
self.chunks.append(processed_chunk)
|
||||
return processed_chunk
|
||||
usage_free_chunk = self._record_and_strip_usage_chunk(processed_chunk)
|
||||
if usage_free_chunk is None:
|
||||
continue
|
||||
return usage_free_chunk
|
||||
except (StopAsyncIteration, StopIteration):
|
||||
return await self._finalize_completed_stream(cache_hit=cache_hit)
|
||||
except httpx.TimeoutException as e: # if httpx read timeout error occues
|
||||
|
|
@ -2306,7 +2303,7 @@ class CustomStreamWrapper:
|
|||
async def _finalize_completed_stream(self, cache_hit: bool) -> "ModelResponseStream":
|
||||
if self.sent_last_chunk is True:
|
||||
complete_streaming_response: Final = await self._async_build_complete_streaming_response()
|
||||
response: Final = self.model_response_creator()
|
||||
response: Final = self._new_usage_chunk()
|
||||
if complete_streaming_response is not None:
|
||||
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)
|
||||
|
||||
|
|
@ -2407,6 +2404,9 @@ class CustomStreamWrapper:
|
|||
return
|
||||
await asyncio.wait((self._failure_logging,), timeout=timeout)
|
||||
|
||||
def bill_failure_after_earlier_rounds(self, earlier: "BilledRoundsSpend") -> None:
|
||||
self._earlier_rounds_spend = earlier
|
||||
|
||||
def _record_partial_usage_for_failure(self) -> None:
|
||||
"""
|
||||
A stream that breaks mid-flight still billed the provider for the chunks
|
||||
|
|
@ -2433,10 +2433,12 @@ class CustomStreamWrapper:
|
|||
if self.model:
|
||||
partial_response.model = self.model
|
||||
backfill_missing_cache_usage_fields(usage)
|
||||
self.logging_obj.model_call_details["combined_usage_object"] = usage
|
||||
self.logging_obj.model_call_details["response_cost"] = (
|
||||
self.logging_obj.response_cost_calculator(result=partial_response) or 0.0
|
||||
)
|
||||
partial_cost: Final = self.logging_obj.response_cost_calculator(result=partial_response) or 0.0
|
||||
if self._earlier_rounds_spend is None:
|
||||
self.logging_obj.model_call_details["combined_usage_object"] = usage
|
||||
self.logging_obj.model_call_details["response_cost"] = partial_cost
|
||||
return
|
||||
self.logging_obj.record_failure_after_earlier_rounds(self._earlier_rounds_spend, usage, partial_cost)
|
||||
except Exception as recover_error:
|
||||
verbose_logger.debug(
|
||||
"could not recover partial usage for interrupted stream: %s",
|
||||
|
|
@ -2463,6 +2465,22 @@ class CustomStreamWrapper:
|
|||
return False
|
||||
return any(get(field) for field in cls._CONTINUATION_DISQUALIFYING_DELTA_FIELDS)
|
||||
|
||||
def _new_usage_chunk(self) -> ModelResponseStream:
|
||||
usage_chunk: Final = self.model_response_creator()
|
||||
if self.caller_requested_stream_usage:
|
||||
usage_chunk.choices = []
|
||||
return usage_chunk
|
||||
|
||||
def _record_and_strip_usage_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream | None:
|
||||
self.chunks.append(chunk.model_copy())
|
||||
usage_free_chunk: Final = self.model_response_creator(
|
||||
chunk={key: value for key, value in chunk.model_dump().items() if key != "usage"},
|
||||
hidden_params=cast( # cast-ok: preserve mapping operations on dynamic storage
|
||||
Mapping[str, object], getattr(chunk, HIDDEN_PARAMS_ATTR)
|
||||
),
|
||||
)
|
||||
return None if is_model_response_stream_empty(model_response=usage_free_chunk) else usage_free_chunk
|
||||
|
||||
def _accumulate_streamed_delta(self, delta: object) -> None:
|
||||
# Shared by the sync, async, and non-aiohttp iteration sites so answer
|
||||
# text and the disqualifying-content latch stay in step across all three.
|
||||
|
|
|
|||
3
litellm/llms/base_llm/extraction/__init__.py
Normal file
3
litellm/llms/base_llm/extraction/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from .transformation import BaseExtractionConfig
|
||||
|
||||
__all__ = ["BaseExtractionConfig"]
|
||||
86
litellm/llms/base_llm/extraction/transformation.py
Normal file
86
litellm/llms/base_llm/extraction/transformation.py
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.extraction import ExtractionRequestBody, ExtractionResponse
|
||||
|
||||
_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object)
|
||||
_RESPONSE_ADAPTER: Final[TypeAdapter[ExtractionResponse]] = TypeAdapter(ExtractionResponse)
|
||||
_RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"})
|
||||
|
||||
|
||||
class BaseExtractionConfig(ABC):
|
||||
api_key_env: tuple[str, ...] = ()
|
||||
api_base_env: tuple[str, ...] = ()
|
||||
|
||||
def get_default_api_base(self) -> str | None:
|
||||
return None
|
||||
|
||||
def missing_api_base_message(self, custom_llm_provider: str) -> str:
|
||||
return f"api_base is required for Extraction provider '{custom_llm_provider}'"
|
||||
|
||||
def resolve_api_base(self, api_base: str | None) -> str | None:
|
||||
return api_base or self._first_secret(self.api_base_env) or self.get_default_api_base()
|
||||
|
||||
def resolve_api_key(self, api_key: str | None) -> str | None:
|
||||
return api_key or self._first_secret(self.api_key_env)
|
||||
|
||||
@staticmethod
|
||||
def _first_secret(names: tuple[str, ...]) -> str | None:
|
||||
return next((value for value in (get_secret_str(name) for name in names) if value), None)
|
||||
|
||||
def canonical_model(self, model: str) -> str:
|
||||
return model
|
||||
|
||||
def request_model(self, model: str) -> str:
|
||||
return model
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
model: str,
|
||||
api_key: str | None,
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> Mapping[str, str]:
|
||||
return {
|
||||
**{name: value for name, value in headers.items() if name.lower() not in _RESERVED_HEADERS},
|
||||
**({"Authorization": f"Bearer {api_key}"} if api_key is not None else {}),
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
@abstractmethod
|
||||
def get_complete_url(self, api_base: str, model: str) -> str: ...
|
||||
|
||||
def transform_extraction_request(self, model: str, request: ExtractionRequestBody) -> Mapping[str, object]:
|
||||
return {**request.model_dump(mode="json", exclude_unset=True), "model": self.request_model(model)}
|
||||
|
||||
def unwrap_response(self, payload: object) -> object:
|
||||
return payload
|
||||
|
||||
def transform_extraction_response(
|
||||
self,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
raw_response: httpx.Response,
|
||||
) -> ExtractionResponse:
|
||||
payload: Final[object] = _PAYLOAD_ADAPTER.validate_json(raw_response.content)
|
||||
try:
|
||||
return _RESPONSE_ADAPTER.validate_python(self.unwrap_response(payload))
|
||||
except ValidationError as error:
|
||||
raise BaseLLMException(
|
||||
status_code=500,
|
||||
message=f"Extraction provider '{custom_llm_provider}' returned an unexpected response: {error}",
|
||||
) from error
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: Mapping[str, str] | httpx.Headers,
|
||||
) -> BaseLLMException:
|
||||
return BaseLLMException(status_code=status_code, message=error_message, headers=httpx.Headers(headers))
|
||||
53
litellm/llms/cloudflare/common_utils.py
Normal file
53
litellm/llms/cloudflare/common_utils.py
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.secret_managers.main import (
|
||||
get_secret_str,
|
||||
normalize_nonempty_secret_str,
|
||||
)
|
||||
|
||||
CLOUDFLARE_MISSING_API_BASE_MESSAGE: Final = (
|
||||
"Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID or pass api_base explicitly"
|
||||
)
|
||||
_RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def workers_ai_default_api_base() -> str | None:
|
||||
account_id: Final = normalize_nonempty_secret_str(get_secret_str("CLOUDFLARE_ACCOUNT_ID"))
|
||||
if account_id is None:
|
||||
return None
|
||||
return f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run"
|
||||
|
||||
|
||||
def workers_ai_canonical_model(model: str) -> str:
|
||||
if model.startswith("@cf/"):
|
||||
return model
|
||||
return f"@cf/cloudflare/{model}"
|
||||
|
||||
|
||||
def workers_ai_request_model(model: str) -> str:
|
||||
return model.rsplit("/", maxsplit=1)[-1]
|
||||
|
||||
|
||||
def workers_ai_run_url(api_base: str, model: str) -> str:
|
||||
normalized_api_base: Final = api_base.rstrip("/")
|
||||
if normalized_api_base.endswith("/ai/v1"):
|
||||
return f"{normalized_api_base.removesuffix('/ai/v1')}/ai/run/{model}"
|
||||
if normalized_api_base.endswith("/ai/run"):
|
||||
return f"{normalized_api_base}/{model}"
|
||||
return f"{normalized_api_base}/ai/run/{model}"
|
||||
|
||||
|
||||
def unwrap_workers_ai_result(payload: object) -> object:
|
||||
try:
|
||||
response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload)
|
||||
except ValidationError:
|
||||
return payload
|
||||
if "answers" in response_mapping:
|
||||
return response_mapping
|
||||
try:
|
||||
return _RESPONSE_MAPPING_ADAPTER.validate_python(response_mapping.get("result"))
|
||||
except ValidationError:
|
||||
return response_mapping
|
||||
|
|
@ -1,16 +1,13 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
from litellm.secret_managers.main import (
|
||||
get_secret_str,
|
||||
normalize_nonempty_secret_str,
|
||||
from litellm.llms.cloudflare.common_utils import (
|
||||
CLOUDFLARE_MISSING_API_BASE_MESSAGE,
|
||||
unwrap_workers_ai_result,
|
||||
workers_ai_canonical_model,
|
||||
workers_ai_default_api_base,
|
||||
workers_ai_request_model,
|
||||
workers_ai_run_url,
|
||||
)
|
||||
|
||||
_RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
class CloudflareDecisionsConfig(BaseDecisionsConfig):
|
||||
api_key_env = ("CLOUDFLARE_API_KEY",)
|
||||
|
|
@ -23,38 +20,19 @@ class CloudflareDecisionsConfig(BaseDecisionsConfig):
|
|||
inlines_remote_images = True
|
||||
|
||||
def get_default_api_base(self) -> str | None:
|
||||
account_id: Final = normalize_nonempty_secret_str(get_secret_str("CLOUDFLARE_ACCOUNT_ID"))
|
||||
if account_id is None:
|
||||
return None
|
||||
return f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run"
|
||||
return workers_ai_default_api_base()
|
||||
|
||||
def missing_api_base_message(self, custom_llm_provider: str) -> str:
|
||||
return "Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID or pass api_base explicitly"
|
||||
return CLOUDFLARE_MISSING_API_BASE_MESSAGE
|
||||
|
||||
def canonical_model(self, model: str) -> str:
|
||||
if model.startswith("@cf/"):
|
||||
return model
|
||||
return f"@cf/cloudflare/{model}"
|
||||
return workers_ai_canonical_model(model)
|
||||
|
||||
def request_model(self, model: str) -> str:
|
||||
return model.rsplit("/", maxsplit=1)[-1]
|
||||
return workers_ai_request_model(model)
|
||||
|
||||
def get_complete_url(self, api_base: str, model: str) -> str:
|
||||
normalized_api_base: Final = api_base.rstrip("/")
|
||||
if normalized_api_base.endswith("/ai/v1"):
|
||||
return f"{normalized_api_base.removesuffix('/ai/v1')}/ai/run/{model}"
|
||||
if normalized_api_base.endswith("/ai/run"):
|
||||
return f"{normalized_api_base}/{model}"
|
||||
return f"{normalized_api_base}/ai/run/{model}"
|
||||
return workers_ai_run_url(api_base, model)
|
||||
|
||||
def unwrap_response(self, payload: object) -> object:
|
||||
try:
|
||||
response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload)
|
||||
except ValidationError:
|
||||
return payload
|
||||
if "answers" in response_mapping:
|
||||
return response_mapping
|
||||
try:
|
||||
return _RESPONSE_MAPPING_ADAPTER.validate_python(response_mapping.get("result"))
|
||||
except ValidationError:
|
||||
return response_mapping
|
||||
return unwrap_workers_ai_result(payload)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.llms.cloudflare.extraction.guardrail_translation.handler import ExtractionHandler
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
guardrail_translation_mappings: Final = {
|
||||
CallTypes.extraction: ExtractionHandler,
|
||||
CallTypes.aextraction: ExtractionHandler,
|
||||
}
|
||||
|
||||
__all__ = ["ExtractionHandler", "guardrail_translation_mappings"]
|
||||
|
|
@ -0,0 +1,276 @@
|
|||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain, groupby
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias, TypeVar
|
||||
|
||||
from pydantic import Field, JsonValue, TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
|
||||
from litellm.types.extraction import (
|
||||
EXTRACTION_STATE_REWRITTEN_METADATA_KEY,
|
||||
ExtractionAnswer,
|
||||
ExtractionBase64Image,
|
||||
ExtractionResponse,
|
||||
)
|
||||
from litellm.types.utils import GenericGuardrailAPIInputs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
||||
|
||||
_FIELDS_ADAPTER: Final[TypeAdapter[Mapping[str, Mapping[str, object]]]] = TypeAdapter(
|
||||
Mapping[str, Mapping[str, object]]
|
||||
)
|
||||
_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
_IMAGES_ADAPTER: Final[TypeAdapter[tuple[ExtractionBase64Image | JsonValue, ...]]] = TypeAdapter(
|
||||
tuple[Annotated[ExtractionBase64Image | JsonValue, Field(union_mode="left_to_right")], ...]
|
||||
)
|
||||
|
||||
|
||||
_AnswerKey: TypeAlias = Literal["value", "guess", "mean", "median", "p10", "p90", "quantiles"]
|
||||
_ESTIMATE_KEYS: Final[tuple[_AnswerKey, ...]] = ("value", "guess", "mean", "median", "p10", "p90", "quantiles")
|
||||
_STATISTIC_KEYS: Final[frozenset[_AnswerKey]] = frozenset(("mean", "median", "p10", "p90", "quantiles"))
|
||||
_NULLED_ANSWER: Final[Mapping[str, None]] = MappingProxyType(
|
||||
dict.fromkeys(("value", "guess", "mean", "median", "p10", "p90", "quantiles", "steps", "span", "spans"), None)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _AnswerText:
|
||||
name: str
|
||||
key: _AnswerKey
|
||||
item: int | None
|
||||
text: str
|
||||
|
||||
|
||||
_SPEC_TEXT_KEYS: Final = ("instructions", "unit")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SpecText:
|
||||
name: str
|
||||
key: str
|
||||
text: str
|
||||
|
||||
|
||||
def _spec_texts_of(name: str, spec: Mapping[str, object]) -> tuple[_SpecText, ...]:
|
||||
return tuple(
|
||||
_SpecText(name=name, key=key, text=text) for key in _SPEC_TEXT_KEYS if isinstance(text := spec.get(key), str)
|
||||
)
|
||||
|
||||
|
||||
def _spec_texts(fields: Mapping[str, Mapping[str, object]]) -> tuple[_SpecText, ...]:
|
||||
return tuple(chain.from_iterable(_spec_texts_of(name, spec) for name, spec in fields.items()))
|
||||
|
||||
|
||||
_NamedText = TypeVar("_NamedText", _AnswerText, _SpecText)
|
||||
|
||||
|
||||
def _scans_by_name(scanned: Iterable[tuple[_NamedText, str]]) -> Mapping[str, tuple[tuple[_NamedText, str], ...]]:
|
||||
return {name: tuple(group) for name, group in groupby(scanned, key=lambda pair: pair[0].name)}
|
||||
|
||||
|
||||
def _guardrailed_spec(spec: Mapping[str, object], scanned: Sequence[tuple[_SpecText, str]]) -> Mapping[str, object]:
|
||||
return {**spec, **{text.key: guardrailed for text, guardrailed in scanned}}
|
||||
|
||||
|
||||
def _metadata_key(data: Mapping[str, object]) -> Literal["metadata", "litellm_metadata"]:
|
||||
return get_metadata_variable_name_from_kwargs(data)
|
||||
|
||||
|
||||
def _metadata(data: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return _METADATA_ADAPTER.validate_python(data.get(_metadata_key(data)) or {})
|
||||
|
||||
|
||||
def _is_text_answer(answer: ExtractionAnswer, spec: Mapping[str, object] | None) -> bool:
|
||||
return answer.type == "text" or (answer.type == "list" and (spec is None or spec.get("items") == "text"))
|
||||
|
||||
|
||||
def _image_urls(images: object) -> tuple[str, ...]:
|
||||
if not isinstance(images, list):
|
||||
return ()
|
||||
return tuple(
|
||||
f"data:{image.content_type};base64,{image.base64}" if isinstance(image, ExtractionBase64Image) else image
|
||||
for image in _IMAGES_ADAPTER.validate_python(images)
|
||||
if isinstance(image, (str, ExtractionBase64Image))
|
||||
)
|
||||
|
||||
|
||||
def _scalar_text(value: JsonValue, whole: bool) -> str | None:
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, float) and whole:
|
||||
return format(value, ".0f")
|
||||
return str(value) if isinstance(value, (str, int, float)) else None
|
||||
|
||||
|
||||
def _item_texts(items: Sequence[JsonValue], whole: bool) -> tuple[tuple[int, str], ...]:
|
||||
return tuple((index, text) for index, item in enumerate(items) if (text := _scalar_text(item, whole)) is not None)
|
||||
|
||||
|
||||
def _value_texts(name: str, key: _AnswerKey, value: JsonValue) -> tuple[_AnswerText, ...]:
|
||||
whole: Final = key in _STATISTIC_KEYS
|
||||
if isinstance(value, list):
|
||||
return tuple(
|
||||
_AnswerText(name=name, key=key, item=index, text=text) for index, text in _item_texts(value, whole)
|
||||
)
|
||||
text: Final = _scalar_text(value, whole)
|
||||
return () if text is None else (_AnswerText(name=name, key=key, item=None, text=text),)
|
||||
|
||||
|
||||
def _step_texts(name: str, step: JsonValue) -> tuple[_AnswerText, ...]:
|
||||
if not isinstance(step, dict):
|
||||
return ()
|
||||
return tuple(chain.from_iterable(_value_texts(name, key, step.get(key)) for key in _ESTIMATE_KEYS))
|
||||
|
||||
|
||||
def _answer_texts(name: str, answer: ExtractionAnswer) -> tuple[_AnswerText, ...]:
|
||||
return (
|
||||
*_value_texts(name, "value", answer.value),
|
||||
*_value_texts(name, "guess", answer.guess),
|
||||
*_value_texts(name, "mean", answer.mean),
|
||||
*_value_texts(name, "median", answer.median),
|
||||
*_value_texts(name, "p10", answer.p10),
|
||||
*_value_texts(name, "p90", answer.p90),
|
||||
*_value_texts(name, "quantiles", list(answer.quantiles or ())),
|
||||
*chain.from_iterable(_step_texts(name, step) for step in answer.steps or ()),
|
||||
)
|
||||
|
||||
|
||||
def _guardrailed_value(value: JsonValue, scanned: Mapping[int | None, str]) -> JsonValue:
|
||||
if isinstance(value, list):
|
||||
return [scanned.get(index, item) for index, item in enumerate(value)]
|
||||
return scanned.get(None, value)
|
||||
|
||||
|
||||
def _scans_of(key: _AnswerKey, scanned: Sequence[tuple[_AnswerText, str]]) -> Mapping[int | None, str]:
|
||||
return {text.item: guardrailed for text, guardrailed in scanned if text.key == key}
|
||||
|
||||
|
||||
def _guardrailed_answer(
|
||||
answer: ExtractionAnswer,
|
||||
scanned: Sequence[tuple[_AnswerText, str]],
|
||||
drop_spans: bool,
|
||||
is_text: bool,
|
||||
) -> ExtractionAnswer:
|
||||
if not is_text and any(text.text != guardrailed for text, guardrailed in scanned):
|
||||
return answer.model_copy(update=_NULLED_ANSWER)
|
||||
value_scans: Final = _scans_of("value", scanned if is_text else ())
|
||||
guess_scans: Final = _scans_of("guess", scanned if is_text else ())
|
||||
update: Final = {
|
||||
**({"value": _guardrailed_value(answer.value, value_scans)} if value_scans else {}),
|
||||
**({"guess": _guardrailed_value(answer.guess, guess_scans)} if guess_scans else {}),
|
||||
**({"span": None, "spans": None} if drop_spans else {}),
|
||||
}
|
||||
return answer.model_copy(update=update) if update else answer
|
||||
|
||||
|
||||
async def _scanned_texts(
|
||||
texts: Sequence[str],
|
||||
model: object,
|
||||
guardrail_to_apply: "CustomGuardrail",
|
||||
request_data: dict[str, object], # mutable-ok: CustomGuardrail.apply_guardrail takes the request as a plain dict
|
||||
input_type: Literal["request", "response"],
|
||||
litellm_logging_obj: "LiteLLMLoggingObj | None",
|
||||
images: Sequence[str] = (),
|
||||
) -> tuple[str, ...]:
|
||||
unique_texts: Final = tuple(dict.fromkeys(texts))
|
||||
inputs: Final = GenericGuardrailAPIInputs(texts=list(unique_texts), model=model if isinstance(model, str) else None)
|
||||
if images:
|
||||
inputs["images"] = list(images)
|
||||
guardrailed: Final = await guardrail_to_apply.apply_guardrail( # pyright: ignore[reportUnknownMemberType] # CustomGuardrail.apply_guardrail types request_data as a bare dict
|
||||
inputs=inputs,
|
||||
request_data=request_data,
|
||||
input_type=input_type,
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
returned: Final = guardrailed.get("texts", [])
|
||||
guardrailed_by_text: Final = MappingProxyType(
|
||||
{text: returned[index] if index < len(returned) else text for index, text in enumerate(unique_texts)}
|
||||
)
|
||||
return tuple(guardrailed_by_text[text] for text in texts)
|
||||
|
||||
|
||||
class ExtractionHandler(BaseTranslation):
|
||||
async def process_input_messages(
|
||||
self,
|
||||
data: dict[str, object], # mutable-ok: BaseTranslation hands over the proxy's request dict
|
||||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
|
||||
) -> Mapping[str, object]:
|
||||
state: Final = data.get("state")
|
||||
states: Final = (state,) if isinstance(state, str) and state else ()
|
||||
fields: Final = _FIELDS_ADAPTER.validate_python(data.get("fields") or {})
|
||||
spec_texts: Final = _spec_texts(fields)
|
||||
images: Final = _image_urls(data.get("images"))
|
||||
if not states and not spec_texts and not images:
|
||||
return data
|
||||
scanned: Final = await _scanned_texts(
|
||||
(*states, *(text.text for text in spec_texts)),
|
||||
data.get("model"),
|
||||
guardrail_to_apply,
|
||||
data,
|
||||
"request",
|
||||
litellm_logging_obj,
|
||||
images,
|
||||
)
|
||||
spec_scans: Final = _scans_by_name(zip(spec_texts, scanned[len(states) :]))
|
||||
state_rewritten: Final = bool(states) and scanned[0] != states[0]
|
||||
return {
|
||||
**data,
|
||||
**({"state": scanned[0]} if states else {}),
|
||||
**(
|
||||
{_metadata_key(data): {**_metadata(data), EXTRACTION_STATE_REWRITTEN_METADATA_KEY: True}}
|
||||
if state_rewritten
|
||||
else {}
|
||||
),
|
||||
"fields": {name: _guardrailed_spec(spec, spec_scans.get(name, ())) for name, spec in fields.items()},
|
||||
}
|
||||
|
||||
async def process_output_response(
|
||||
self,
|
||||
response: object,
|
||||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
|
||||
user_api_key_dict: "UserAPIKeyAuth | None" = None,
|
||||
request_data: dict[str, object] | None = None, # mutable-ok: BaseTranslation hands over the request dict
|
||||
) -> object:
|
||||
if not isinstance(response, ExtractionResponse):
|
||||
return response
|
||||
texts: Final = tuple(
|
||||
chain.from_iterable(_answer_texts(name, answer) for name, answer in response.answers.items())
|
||||
)
|
||||
state_texts: Final = (response.state_text,) if response.state_text else ()
|
||||
request: Final = request_data or {}
|
||||
scanned: Final = (
|
||||
await _scanned_texts(
|
||||
(*(text.text for text in texts), *state_texts),
|
||||
response.model,
|
||||
guardrail_to_apply,
|
||||
request,
|
||||
"response",
|
||||
litellm_logging_obj,
|
||||
)
|
||||
if texts or state_texts
|
||||
else ()
|
||||
)
|
||||
answer_scans: Final = _scans_by_name(zip(texts, scanned))
|
||||
state_text: Final = next(iter(scanned[len(texts) :]), response.state_text)
|
||||
specs: Final = _FIELDS_ADAPTER.validate_python(request.get("fields") or {})
|
||||
return response.model_copy(
|
||||
update={
|
||||
"answers": {
|
||||
name: _guardrailed_answer(
|
||||
answer,
|
||||
answer_scans.get(name, ()),
|
||||
drop_spans=state_text != response.state_text,
|
||||
is_text=_is_text_answer(answer, specs.get(name)),
|
||||
)
|
||||
for name, answer in response.answers.items()
|
||||
},
|
||||
**({"state_text": state_text} if state_texts else {}),
|
||||
}
|
||||
)
|
||||
32
litellm/llms/cloudflare/extraction/transformation.py
Normal file
32
litellm/llms/cloudflare/extraction/transformation.py
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
from litellm.llms.base_llm.extraction.transformation import BaseExtractionConfig
|
||||
from litellm.llms.cloudflare.common_utils import (
|
||||
CLOUDFLARE_MISSING_API_BASE_MESSAGE,
|
||||
unwrap_workers_ai_result,
|
||||
workers_ai_canonical_model,
|
||||
workers_ai_default_api_base,
|
||||
workers_ai_request_model,
|
||||
workers_ai_run_url,
|
||||
)
|
||||
|
||||
|
||||
class CloudflareExtractionConfig(BaseExtractionConfig):
|
||||
api_key_env = ("CLOUDFLARE_API_KEY",)
|
||||
api_base_env = ("CLOUDFLARE_API_BASE",)
|
||||
|
||||
def get_default_api_base(self) -> str | None:
|
||||
return workers_ai_default_api_base()
|
||||
|
||||
def missing_api_base_message(self, custom_llm_provider: str) -> str:
|
||||
return CLOUDFLARE_MISSING_API_BASE_MESSAGE
|
||||
|
||||
def canonical_model(self, model: str) -> str:
|
||||
return workers_ai_canonical_model(model)
|
||||
|
||||
def request_model(self, model: str) -> str:
|
||||
return workers_ai_request_model(model)
|
||||
|
||||
def get_complete_url(self, api_base: str, model: str) -> str:
|
||||
return workers_ai_run_url(api_base, model)
|
||||
|
||||
def unwrap_response(self, payload: object) -> object:
|
||||
return unwrap_workers_ai_result(payload)
|
||||
|
|
@ -76,6 +76,7 @@ from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
|
|||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
|
||||
from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig
|
||||
from litellm.llms.base_llm.extraction.transformation import BaseExtractionConfig
|
||||
from litellm.llms.base_llm.files.transformation import (
|
||||
BaseFilesConfig,
|
||||
BaseFileUploadStream,
|
||||
|
|
@ -128,6 +129,7 @@ from litellm.types.containers.main import (
|
|||
DeleteContainerResult,
|
||||
)
|
||||
from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse
|
||||
from litellm.types.extraction import ExtractionResponse
|
||||
from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig
|
||||
from litellm.types.integrations.custom_logger import (
|
||||
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
|
||||
|
|
@ -1700,6 +1702,115 @@ class BaseLLMHTTPHandler:
|
|||
model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request
|
||||
)
|
||||
|
||||
def _prepare_extraction_request(
|
||||
self,
|
||||
model: str,
|
||||
logging_obj: LiteLLMLoggingObj | None,
|
||||
provider_config: BaseExtractionConfig,
|
||||
body: Mapping[str, object],
|
||||
api_base: str,
|
||||
api_key: str | None,
|
||||
headers: Mapping[str, str],
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> tuple[str, Mapping[str, str]]:
|
||||
outbound_headers: Final = provider_config.validate_environment(
|
||||
headers=headers,
|
||||
model=model,
|
||||
api_key=api_key,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
url: Final = provider_config.get_complete_url(api_base=api_base, model=model)
|
||||
if logging_obj is not None:
|
||||
logging_obj.pre_call( # pyright: ignore[reportUnknownMemberType] # Logging.pre_call is untyped
|
||||
input=dict(body),
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
additional_args={"api_base": url, "complete_input_dict": dict(body), "headers": outbound_headers},
|
||||
)
|
||||
return url, outbound_headers
|
||||
|
||||
def extraction(
|
||||
self,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
logging_obj: LiteLLMLoggingObj | None,
|
||||
provider_config: BaseExtractionConfig,
|
||||
body: Mapping[str, object],
|
||||
api_base: str,
|
||||
api_key: str | None,
|
||||
headers: Mapping[str, str],
|
||||
timeout: float | httpx.Timeout | None,
|
||||
litellm_params: Mapping[str, object],
|
||||
client: HTTPHandler | None = None,
|
||||
) -> ExtractionResponse:
|
||||
url, outbound_headers = self._prepare_extraction_request(
|
||||
model=model,
|
||||
logging_obj=logging_obj,
|
||||
provider_config=provider_config,
|
||||
body=body,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
headers=headers,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
sync_httpx_client: Final = client if client is not None else get_httpx_client()
|
||||
try:
|
||||
response: Final = sync_httpx_client.post( # pyright: ignore[reportUnknownMemberType] # untyped
|
||||
url,
|
||||
json=dict(body),
|
||||
headers=dict(outbound_headers),
|
||||
timeout=timeout,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
except httpx.HTTPError as e:
|
||||
raise self._handle_error(e=e, provider_config=provider_config)
|
||||
return provider_config.transform_extraction_response(
|
||||
model=model, custom_llm_provider=custom_llm_provider, raw_response=response
|
||||
)
|
||||
|
||||
async def aextraction(
|
||||
self,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
logging_obj: LiteLLMLoggingObj | None,
|
||||
provider_config: BaseExtractionConfig,
|
||||
body: Mapping[str, object],
|
||||
api_base: str,
|
||||
api_key: str | None,
|
||||
headers: Mapping[str, str],
|
||||
timeout: float | httpx.Timeout | None,
|
||||
litellm_params: Mapping[str, object],
|
||||
client: AsyncHTTPHandler | None = None,
|
||||
) -> ExtractionResponse:
|
||||
url, outbound_headers = self._prepare_extraction_request(
|
||||
model=model,
|
||||
logging_obj=logging_obj,
|
||||
provider_config=provider_config,
|
||||
body=body,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
headers=headers,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
async_httpx_client: Final = (
|
||||
client
|
||||
if client is not None
|
||||
else get_async_httpx_client(llm_provider=litellm.LlmProviders(custom_llm_provider))
|
||||
)
|
||||
try:
|
||||
response: Final = await async_httpx_client.post( # pyright: ignore[reportUnknownMemberType] # untyped
|
||||
url,
|
||||
json=dict(body),
|
||||
headers=dict(outbound_headers),
|
||||
timeout=timeout,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
except httpx.HTTPError as e:
|
||||
raise self._handle_error(e=e, provider_config=provider_config)
|
||||
return provider_config.transform_extraction_response(
|
||||
model=model, custom_llm_provider=custom_llm_provider, raw_response=response
|
||||
)
|
||||
|
||||
def _prepare_audio_transcription_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -6462,6 +6573,7 @@ class BaseLLMHTTPHandler:
|
|||
provider_config: Union[
|
||||
BaseConfig,
|
||||
BaseDecisionsConfig,
|
||||
BaseExtractionConfig,
|
||||
BaseRerankConfig,
|
||||
BaseResponsesAPIConfig,
|
||||
BaseImageEditConfig,
|
||||
|
|
|
|||
|
|
@ -82,9 +82,9 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
|
|||
return "tts" in model
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
sampling_params: Final = ("temperature", "top_p") if self.supports_sampling_params(model) else ()
|
||||
supported_params: Final = [
|
||||
"temperature",
|
||||
"top_p",
|
||||
*sampling_params,
|
||||
"max_tokens",
|
||||
"max_completion_tokens",
|
||||
"stream",
|
||||
|
|
|
|||
|
|
@ -16,7 +16,9 @@ The handler is intentionally a thin orchestration layer:
|
|||
3. Construct ``riva.client.Auth`` honoring NVCF (function-id metadata + TLS)
|
||||
vs self-hosted (any host:port, optional TLS) modes.
|
||||
4. Stream the audio through Riva's ``streaming_response_generator`` and
|
||||
aggregate ``is_final`` results into a single transcript.
|
||||
aggregate ``is_final`` results into a single transcript, or, when
|
||||
``riva_offline`` is set, send the whole buffer through the unary
|
||||
``offline_recognize`` RPC for NIMs that only ship offline engines.
|
||||
5. Return a normalized ``TranscriptionResponse`` with ``duration`` exposed
|
||||
on ``_hidden_params`` so cost calculation works.
|
||||
|
||||
|
|
@ -26,13 +28,19 @@ without the optional STT extras installed.
|
|||
|
||||
import asyncio
|
||||
import inspect
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm.constants import COMPLETION_HTTP_FALLBACK_SECONDS
|
||||
from litellm.litellm_core_utils.audio_utils.utils import (
|
||||
get_audio_file_name,
|
||||
process_audio_file,
|
||||
)
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.llms.nvidia_riva.audio_transcription.audio_utils import (
|
||||
resample_to_riva_pcm,
|
||||
)
|
||||
|
|
@ -45,6 +53,7 @@ from litellm.llms.nvidia_riva.common_utils import (
|
|||
NvidiaRivaException,
|
||||
grpc_error_to_litellm_exception,
|
||||
)
|
||||
from litellm.secret_managers.main import str_to_bool
|
||||
from litellm.types.utils import FileTypes, TranscriptionResponse
|
||||
from litellm.utils import convert_to_model_response_object
|
||||
|
||||
|
|
@ -67,10 +76,20 @@ class _RivaAuth(Protocol):
|
|||
"""Opaque ``riva.client.Auth`` handle."""
|
||||
|
||||
|
||||
class _RecognizeFuture(Protocol):
|
||||
def result(self, timeout: float | None = None) -> object: ...
|
||||
|
||||
def done(self) -> bool: ...
|
||||
|
||||
def cancel(self) -> bool: ...
|
||||
|
||||
|
||||
class _AsrService(Protocol):
|
||||
@property
|
||||
def streaming_response_generator(self) -> Callable[..., Iterable[object]]: ...
|
||||
|
||||
def offline_recognize(self, audio_bytes: bytes, config: object, future: bool) -> _RecognizeFuture: ...
|
||||
|
||||
|
||||
class _EndpointingConfig(Protocol):
|
||||
"""Opaque ``EndpointingConfig`` protobuf message."""
|
||||
|
|
@ -128,6 +147,38 @@ def _audio_encoding(riva_asr_module: _RivaAsrModule) -> _AudioEncoding:
|
|||
return riva_asr_module.AudioEncoding
|
||||
|
||||
|
||||
def _parse_riva_offline(value: object, drop_params: bool) -> bool:
|
||||
match value:
|
||||
case None:
|
||||
return False
|
||||
case bool():
|
||||
return value
|
||||
case str() if (parsed := str_to_bool(value)) is not None:
|
||||
return parsed
|
||||
case _ if drop_params:
|
||||
return False
|
||||
case _:
|
||||
raise NvidiaRivaException(
|
||||
status_code=400,
|
||||
message=f"riva_offline must be true or false, got {value!r}. Set drop_params to ignore it.",
|
||||
)
|
||||
|
||||
|
||||
def _deadline_seconds(timeout: float | str | httpx.Timeout, drop_params: bool) -> float:
|
||||
try:
|
||||
limit: Final = CompletionTimeout.as_limit(timeout)
|
||||
except ValueError:
|
||||
if drop_params:
|
||||
return COMPLETION_HTTP_FALLBACK_SECONDS
|
||||
raise NvidiaRivaException(
|
||||
status_code=400,
|
||||
message=f"timeout must be a number of seconds, got {timeout!r}. Set drop_params to ignore it.",
|
||||
) from None
|
||||
if not isinstance(limit, httpx.Timeout):
|
||||
return limit
|
||||
return COMPLETION_HTTP_FALLBACK_SECONDS if limit.read is None else limit.read
|
||||
|
||||
|
||||
class NvidiaRivaAudioTranscription:
|
||||
"""Sync + async entry point for Riva ASR."""
|
||||
|
||||
|
|
@ -138,7 +189,7 @@ class NvidiaRivaAudioTranscription:
|
|||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
model_response: TranscriptionResponse,
|
||||
timeout: float,
|
||||
timeout: float | str | httpx.Timeout | None,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
api_key: str | None,
|
||||
api_base: str | None,
|
||||
|
|
@ -183,7 +234,7 @@ class NvidiaRivaAudioTranscription:
|
|||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
model_response: TranscriptionResponse,
|
||||
timeout: float,
|
||||
timeout: float | str | httpx.Timeout | None,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
api_key: str | None,
|
||||
api_base: str | None,
|
||||
|
|
@ -213,7 +264,7 @@ class NvidiaRivaAudioTranscription:
|
|||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
model_response: TranscriptionResponse,
|
||||
timeout: float,
|
||||
timeout: float | str | httpx.Timeout | None,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
api_key: str | None,
|
||||
api_base: str | None,
|
||||
|
|
@ -259,6 +310,10 @@ class NvidiaRivaAudioTranscription:
|
|||
|
||||
response_format: Final = request_payload.get("response_format") or "json"
|
||||
timestamp_granularities: Final = request_payload.get("timestamp_granularities")
|
||||
drop_params: Final = (
|
||||
"drop_params" in litellm_params and litellm_params["drop_params"] is True
|
||||
) or litellm.drop_params is True
|
||||
riva_offline: Final = _parse_riva_offline(optional_params.get("riva_offline"), drop_params=drop_params)
|
||||
|
||||
riva_module, riva_asr_module = _import_riva()
|
||||
auth_obj: Final = self._construct_auth(
|
||||
|
|
@ -272,10 +327,6 @@ class NvidiaRivaAudioTranscription:
|
|||
riva_asr_module=riva_asr_module,
|
||||
recognition_config_dict=recognition_config_dict,
|
||||
)
|
||||
streaming_config: Final[_StreamingRecognitionConfig] = riva_asr_module.StreamingRecognitionConfig(
|
||||
config=recognition_config, interim_results=False
|
||||
)
|
||||
|
||||
logging_obj.pre_call(
|
||||
input=None,
|
||||
api_key=api_key,
|
||||
|
|
@ -286,24 +337,31 @@ class NvidiaRivaAudioTranscription:
|
|||
"recognition_config": recognition_config_dict,
|
||||
"nvcf_function_id_set": bool(optional_params.get("nvcf_function_id")),
|
||||
"use_ssl": optional_params.get("use_ssl"),
|
||||
"riva_offline": riva_offline,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
asr_service: Final[_AsrService] = riva_module.ASRService(auth_obj)
|
||||
audio_chunks: Final = self._iter_audio_chunks(resampled.pcm_bytes)
|
||||
stream_kwargs: Final[dict[str, object]] = {
|
||||
"audio_chunks": audio_chunks,
|
||||
"streaming_config": streaming_config,
|
||||
}
|
||||
# Forward the deadline so the stream cannot block forever if the
|
||||
# server stalls. Older riva-client versions do not accept a
|
||||
# ``timeout`` kwarg, so pass it only when supported.
|
||||
if timeout is not None and self._supports_timeout_kwarg(asr_service.streaming_response_generator):
|
||||
stream_kwargs["timeout"] = float(timeout)
|
||||
stream: Final = asr_service.streaming_response_generator(**stream_kwargs)
|
||||
final_results: Final = self._collect_final_results(stream)
|
||||
responses: Final = (
|
||||
self._offline_responses(
|
||||
asr_service=asr_service,
|
||||
pcm_bytes=resampled.pcm_bytes,
|
||||
recognition_config=recognition_config,
|
||||
timeout=None if timeout is None else _deadline_seconds(timeout, drop_params),
|
||||
)
|
||||
if riva_offline
|
||||
else self._streaming_responses(
|
||||
asr_service=asr_service,
|
||||
riva_asr_module=riva_asr_module,
|
||||
pcm_bytes=resampled.pcm_bytes,
|
||||
recognition_config=recognition_config,
|
||||
timeout=timeout,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
)
|
||||
final_results: Final = self._collect_final_results(responses, require_final=not riva_offline)
|
||||
except NvidiaRivaException:
|
||||
raise
|
||||
except Exception as e:
|
||||
|
|
@ -409,6 +467,54 @@ class NvidiaRivaAudioTranscription:
|
|||
|
||||
return config
|
||||
|
||||
def _streaming_responses(
|
||||
self,
|
||||
asr_service: _AsrService,
|
||||
riva_asr_module: _RivaAsrModule,
|
||||
pcm_bytes: bytes,
|
||||
recognition_config: _RecognitionConfig,
|
||||
timeout: float | str | httpx.Timeout | None,
|
||||
drop_params: bool,
|
||||
) -> Iterable[object]:
|
||||
streaming_config: Final[_StreamingRecognitionConfig] = riva_asr_module.StreamingRecognitionConfig(
|
||||
config=recognition_config, interim_results=False
|
||||
)
|
||||
return asr_service.streaming_response_generator(
|
||||
audio_chunks=self._iter_audio_chunks(pcm_bytes),
|
||||
streaming_config=streaming_config,
|
||||
**self._timeout_kwargs(asr_service.streaming_response_generator, timeout, drop_params),
|
||||
)
|
||||
|
||||
def _offline_responses(
|
||||
self,
|
||||
asr_service: _AsrService,
|
||||
pcm_bytes: bytes,
|
||||
recognition_config: _RecognitionConfig,
|
||||
timeout: float | None,
|
||||
) -> Iterable[object]:
|
||||
pending: Final = asr_service.offline_recognize(audio_bytes=pcm_bytes, config=recognition_config, future=True)
|
||||
try:
|
||||
return (pending.result(timeout=timeout),)
|
||||
except Exception as e:
|
||||
if pending.done():
|
||||
raise
|
||||
pending.cancel()
|
||||
raise NvidiaRivaException(
|
||||
status_code=408,
|
||||
message=f"NVIDIA Riva offline recognition did not finish within {timeout} seconds",
|
||||
) from e
|
||||
|
||||
@classmethod
|
||||
def _timeout_kwargs(
|
||||
cls, rpc: Callable[..., object], timeout: float | str | httpx.Timeout | None, drop_params: bool
|
||||
) -> Mapping[str, float]:
|
||||
# Forward the deadline so the call cannot block forever if the
|
||||
# server stalls. Older riva-client versions do not accept a
|
||||
# ``timeout`` kwarg, so pass it only when supported.
|
||||
if timeout is None or not cls._supports_timeout_kwarg(rpc):
|
||||
return MappingProxyType({})
|
||||
return MappingProxyType({"timeout": _deadline_seconds(timeout, drop_params)})
|
||||
|
||||
@staticmethod
|
||||
def _supports_timeout_kwarg(callable_obj: Callable[..., object]) -> bool:
|
||||
try:
|
||||
|
|
@ -429,9 +535,9 @@ class NvidiaRivaAudioTranscription:
|
|||
yield chunk
|
||||
|
||||
@staticmethod
|
||||
def _collect_final_results(stream) -> list[dict[str, object]]:
|
||||
def _collect_final_results(stream: Iterable[object], require_final: bool = True) -> list[dict[str, object]]:
|
||||
"""
|
||||
Walk the gRPC stream, ignore empty / non-final chunks, and return a
|
||||
Walk the gRPC responses, ignore empty / non-final chunks, and return a
|
||||
list of normalized final-result dicts. Matching the user's note: the
|
||||
``id`` blocks with no ``results`` are streaming heartbeats and must
|
||||
be skipped.
|
||||
|
|
@ -440,7 +546,7 @@ class NvidiaRivaAudioTranscription:
|
|||
for response in stream:
|
||||
results = getattr(response, "results", None) or []
|
||||
for result in results:
|
||||
if not getattr(result, "is_final", False):
|
||||
if require_final and not getattr(result, "is_final", False):
|
||||
continue
|
||||
alternatives = getattr(result, "alternatives", None) or []
|
||||
if not alternatives:
|
||||
|
|
|
|||
|
|
@ -41,6 +41,9 @@ _GRPC_STATUS_CODE_TO_HTTP: Final[dict] = {
|
|||
}
|
||||
|
||||
|
||||
_GRPC_MESSAGE_TOO_LARGE: Final = "message larger than max"
|
||||
|
||||
|
||||
def _extract_grpc_status_name(error: object) -> str | None:
|
||||
"""
|
||||
Best-effort extraction of a gRPC StatusCode name from an arbitrary error.
|
||||
|
|
@ -79,8 +82,16 @@ def grpc_error_to_litellm_exception(error: Exception) -> NvidiaRivaException:
|
|||
a ``NvidiaRivaException`` with an appropriate HTTP-equivalent status code.
|
||||
"""
|
||||
status_name: Final = _extract_grpc_status_name(error)
|
||||
http_status: Final = _GRPC_STATUS_CODE_TO_HTTP.get(status_name or "", 500)
|
||||
|
||||
detail: Final = _extract_grpc_details(error) or str(error)
|
||||
if status_name == "RESOURCE_EXHAUSTED" and _GRPC_MESSAGE_TOO_LARGE in detail:
|
||||
return NvidiaRivaException(
|
||||
status_code=413,
|
||||
message=(
|
||||
f"NVIDIA Riva gRPC error ({status_name}): {detail}. The audio is too large for one offline "
|
||||
"request to this Riva server: send shorter audio, or drop riva_offline on models that support streaming."
|
||||
),
|
||||
)
|
||||
|
||||
http_status: Final = _GRPC_STATUS_CODE_TO_HTTP.get(status_name or "", 500)
|
||||
message = f"NVIDIA Riva gRPC error ({status_name}): {detail}" if status_name else f"NVIDIA Riva error: {detail}"
|
||||
return NvidiaRivaException(status_code=http_status, message=message)
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import time
|
|||
from collections.abc import AsyncGenerator, Callable, Iterable, Iterator, Mapping
|
||||
from contextlib import aclosing
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import IO, Any, Final, TypedDict
|
||||
from urllib.parse import quote, unquote
|
||||
|
|
@ -30,6 +31,7 @@ from litellm.litellm_core_utils.cloud_storage_security import (
|
|||
split_configured_cloud_bucket_name,
|
||||
validate_managed_cloud_file_id,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
extract_file_data,
|
||||
|
|
@ -1103,21 +1105,26 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
def _map_openai_to_vertex_params(
|
||||
self,
|
||||
openai_request_body: dict[str, Any],
|
||||
deployment_drop_params: object = None,
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
wrapper to call VertexGeminiConfig.map_openai_params
|
||||
"""
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
VertexGeminiConfig,
|
||||
)
|
||||
|
||||
config: Final = VertexGeminiConfig()
|
||||
_model: Final = openai_request_body.get("model", "")
|
||||
raw_model: Final = openai_request_body.get("model", "")
|
||||
model: Final = raw_model if isinstance(raw_model, str) else ""
|
||||
sampling_params: Final = tuple(
|
||||
p for p in ("temperature", "top_p", "top_k") if openai_request_body.get(p) is not None
|
||||
)
|
||||
if sampling_params and not VertexGeminiConfig.supports_sampling_params(model):
|
||||
VertexGeminiConfig.reject_unsupported_sampling_param(
|
||||
model=model, param=", ".join(sampling_params), drop_params=False
|
||||
)
|
||||
vertex_params: Final = config.map_openai_params(
|
||||
model=_model,
|
||||
model=model,
|
||||
non_default_params=openai_request_body,
|
||||
optional_params={},
|
||||
drop_params=False,
|
||||
drop_params=drops_unsupported_params(openai_request_body.get("drop_params", deployment_drop_params)),
|
||||
allow_integer_enums=False,
|
||||
)
|
||||
return vertex_params
|
||||
|
|
@ -1153,11 +1160,12 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
create_file_data=create_file_data,
|
||||
content_type=content_type,
|
||||
):
|
||||
deployment_drop_params: Final[object] = litellm_params.get("drop_params")
|
||||
return {
|
||||
"streaming_media_upload": StreamingMediaUploadConfig(
|
||||
body_stream=_OpenAIToVertexBatchUploadStream(
|
||||
file_data,
|
||||
self._map_openai_to_vertex_params,
|
||||
partial(self._map_openai_to_vertex_params, deployment_drop_params=deployment_drop_params),
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,17 +2,20 @@ import json
|
|||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import accumulate, chain
|
||||
from typing import Final, TypeAlias, TypeVar, cast # noqa: TID251 # sum_usage_metadata casts computed keys
|
||||
from typing import Final, Literal, TypeAlias, TypeVar, cast # noqa: TID251 # sum_usage_metadata casts computed keys
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import Timeout
|
||||
from litellm.litellm_core_utils.env_utils import get_env_int_or_none
|
||||
from litellm.types.llms.vertex_ai import (
|
||||
Candidates,
|
||||
Citation,
|
||||
CitationMetadata,
|
||||
GenerateContentResponseBody,
|
||||
GenerationConfig,
|
||||
GroundingMetadata,
|
||||
GroundingSegment,
|
||||
GroundingSupport,
|
||||
|
|
@ -48,8 +51,14 @@ HOP_FAILURES: Final = (VertexAIError, httpx.HTTPError, Timeout)
|
|||
HOP_END_FINISH_REASON: Final = "CONTINUATION"
|
||||
CUT_OFF_FINISH_REASONS: Final = frozenset({"MAX_TOKENS", HOP_END_FINISH_REASON})
|
||||
|
||||
DEFAULT_OUTPUT_BUDGET_ENV: Final = "LITELLM_GEMINI_DEFAULT_MAX_OUTPUT_TOKENS"
|
||||
_OUTPUT_LIMIT_KEYS: Final = frozenset({"max_output_tokens", "maxOutputTokens"})
|
||||
_REQUEST_KEYS_THAT_SKIP_THE_DEFAULT_BUDGET: Final = frozenset({"generation_config", "continuationToken"})
|
||||
_TEXT_ONLY_MODALITIES: Final = TypeAdapter(list[Literal["TEXT"]])
|
||||
|
||||
_VERTEX_V1_PATH: Final = "/v1/projects/"
|
||||
_VERTEX_PATH_ACCEPTING_CONTINUATION_TOKEN: Final = "/v1beta1/projects/"
|
||||
_CANDIDATE_COUNT_KEYS: Final = ("candidate_count", "candidateCount")
|
||||
|
||||
LongDecodeHops: TypeAlias = tuple[GenerateContentResponseBody, ...]
|
||||
AsyncHopPost: TypeAlias = Callable[[RequestBody], Awaitable[httpx.Response]]
|
||||
|
|
@ -76,15 +85,53 @@ def _int_or_none(value: object) -> int | None:
|
|||
return value if isinstance(value, int) else None
|
||||
|
||||
|
||||
def _asks_for_one_candidate(generation_config: Mapping[str, object]) -> bool:
|
||||
return all(
|
||||
_int_or_none(generation_config[key] or 1) == 1 for key in _CANDIDATE_COUNT_KEYS if key in generation_config
|
||||
)
|
||||
|
||||
|
||||
def continuation_cap_of(request_body: RequestBody) -> int | None:
|
||||
generation_config: Final = request_body.get("generationConfig")
|
||||
if generation_config is None or "max_output_tokens" not in generation_config:
|
||||
return None
|
||||
if "candidate_count" in generation_config and _int_or_none(generation_config["candidate_count"]) != 1:
|
||||
if not _asks_for_one_candidate(generation_config):
|
||||
return None
|
||||
return _int_or_none(generation_config["max_output_tokens"])
|
||||
|
||||
|
||||
def _asks_for_text_only(generation_config: Mapping[str, object]) -> bool:
|
||||
if "response_modalities" in generation_config:
|
||||
return False
|
||||
try:
|
||||
_ = _TEXT_ONLY_MODALITIES.validate_python(generation_config.get("responseModalities", []))
|
||||
except ValidationError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def with_default_output_budget(
|
||||
request_body: RequestBody, model_max_output_tokens: Callable[[], int | None]
|
||||
) -> RequestBody:
|
||||
generation_config: Final = request_body.get("generationConfig", GenerationConfig())
|
||||
request_json: Final[Mapping[str, object]] = request_body
|
||||
budget: Final = get_env_int_or_none(DEFAULT_OUTPUT_BUDGET_ENV)
|
||||
if budget is None or budget <= 0:
|
||||
return request_body
|
||||
if not _REQUEST_KEYS_THAT_SKIP_THE_DEFAULT_BUDGET.isdisjoint(request_body):
|
||||
return request_body
|
||||
if not isinstance(request_json.get("generationConfig", {}), dict):
|
||||
return request_body
|
||||
if not _OUTPUT_LIMIT_KEYS.isdisjoint(generation_config):
|
||||
return request_body
|
||||
if not _asks_for_text_only(generation_config):
|
||||
return request_body
|
||||
model_max: Final = model_max_output_tokens()
|
||||
if model_max is None:
|
||||
return request_body
|
||||
return {**request_body, "generationConfig": {**generation_config, "max_output_tokens": min(budget, model_max)}}
|
||||
|
||||
|
||||
def client_continuation_token(candidate: Candidates) -> str | None:
|
||||
if candidate.get("finishReason") not in CUT_OFF_FINISH_REASONS:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -7,15 +7,19 @@ Why separate file? Make it easy to see how transformation works
|
|||
import os
|
||||
import re
|
||||
from collections.abc import Callable, Mapping
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, cast
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from pydantic import BaseModel, PositiveInt, TypeAdapter, ValidationError
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_image_mime_type_from_url,
|
||||
)
|
||||
|
|
@ -64,6 +68,7 @@ from ..common_utils import (
|
|||
get_supports_response_schema,
|
||||
get_supports_system_message,
|
||||
)
|
||||
from .long_decode import with_default_output_budget
|
||||
|
||||
# VertexBase is imported only for type checking to avoid introducing a
|
||||
# module-load-time cyclic import to vertex_llm_base. The instance is lazily
|
||||
|
|
@ -868,7 +873,7 @@ def gemini_convert_messages_with_history(
|
|||
"""
|
||||
has_text_in_content = check_text_in_content(user_content)
|
||||
if has_text_in_content is False:
|
||||
verbose_logger.warning(
|
||||
verbose_logger.debug(
|
||||
"No text in user content. Adding a blank text to user content, to ensure Gemini doesn't fail the request. Relevant Issue - https://github.com/BerriAI/litellm/issues/5515"
|
||||
)
|
||||
user_content.append(
|
||||
|
|
@ -1189,7 +1194,18 @@ def transform_request_body(
|
|||
# labels: optional explicit param and/or metadata.requester_metadata (OpenAI metadata)
|
||||
labels: Final = pop_vertex_request_labels(optional_params, litellm_params)
|
||||
|
||||
filtered_params = {k: v for k, v in optional_params.items() if _get_equivalent_key(k, set(config_fields))}
|
||||
from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig
|
||||
|
||||
drops_top_k: Final = "top_k" in optional_params and not VertexGeminiConfig.supports_sampling_params(model)
|
||||
if drops_top_k:
|
||||
request_params: Final = cast(Mapping[str, object], litellm_params) # cast-ok: untyped shared signature
|
||||
VertexGeminiConfig.reject_unsupported_sampling_param(
|
||||
model=model,
|
||||
param="top_k",
|
||||
drop_params=drops_unsupported_params(request_params.get("drop_params")),
|
||||
)
|
||||
config_keys: Final = set(config_fields) - {"top_k"} if drops_top_k else set(config_fields)
|
||||
filtered_params = {k: v for k, v in optional_params.items() if _get_equivalent_key(k, config_keys)}
|
||||
|
||||
generation_config: Final[GenerationConfig | None] = GenerationConfig(**filtered_params)
|
||||
|
||||
|
|
@ -1292,13 +1308,75 @@ def sync_transform_request_body(
|
|||
vertex_auth_header=vertex_auth_header,
|
||||
)
|
||||
|
||||
return transform_request_body(
|
||||
messages=messages,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
cached_content=cached_content,
|
||||
optional_params=optional_params,
|
||||
return _with_default_output_budget(
|
||||
transform_request_body(
|
||||
messages=messages,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
cached_content=cached_content,
|
||||
optional_params=optional_params,
|
||||
),
|
||||
model,
|
||||
custom_llm_provider,
|
||||
litellm_params.get("model_info"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is an untyped dict
|
||||
)
|
||||
|
||||
|
||||
class _CostMapOutputModalities(TypedDict):
|
||||
supported_output_modalities: NotRequired[ReadOnly[tuple[str, ...]]]
|
||||
|
||||
|
||||
_COST_MAP_OUTPUT_MODALITIES: Final = TypeAdapter(_CostMapOutputModalities)
|
||||
|
||||
|
||||
def _answers_text(cost_map_key: str) -> bool:
|
||||
try:
|
||||
row: Final = _COST_MAP_OUTPUT_MODALITIES.validate_python(litellm.model_cost.get(cost_map_key)) # pyright: ignore[reportUnknownMemberType] # cost map rows are untyped JSON
|
||||
except ValidationError:
|
||||
return False
|
||||
return "text" in row.get("supported_output_modalities", ("text",))
|
||||
|
||||
|
||||
class _OutputLimit(TypedDict):
|
||||
max_output_tokens: NotRequired[ReadOnly[PositiveInt | None]]
|
||||
|
||||
|
||||
_OUTPUT_LIMIT: Final = TypeAdapter(_OutputLimit)
|
||||
|
||||
|
||||
def _output_limit(fields: object) -> int | None:
|
||||
try:
|
||||
return _OUTPUT_LIMIT.validate_python(fields).get("max_output_tokens")
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _catalog_capped_output_limit(cost_map_key: str, override: object) -> int | None:
|
||||
limits: Final = (
|
||||
_output_limit(GetModelCostMap.loaded_model_cost_map().get(cost_map_key)),
|
||||
_output_limit(override),
|
||||
)
|
||||
return min((limit for limit in limits if limit is not None), default=None)
|
||||
|
||||
|
||||
def _text_chat_model_max_output_tokens(
|
||||
model: str, custom_llm_provider: str, deployment_model_info: object
|
||||
) -> int | None:
|
||||
info: Final = litellm.utils.lookup_model_info_for_registration(model, custom_llm_provider)
|
||||
if isinstance(info, litellm.utils.ModelInfoLookupMiss) or info.get("mode") != "chat":
|
||||
return None
|
||||
if not _answers_text(info["key"]):
|
||||
return None
|
||||
return _catalog_capped_output_limit(info["key"], deployment_model_info or info)
|
||||
|
||||
|
||||
def _with_default_output_budget(
|
||||
request_body: RequestBody, model: str, custom_llm_provider: str, deployment_model_info: object
|
||||
) -> RequestBody:
|
||||
return with_default_output_budget(
|
||||
request_body,
|
||||
partial(_text_chat_model_max_output_tokens, model, custom_llm_provider, deployment_model_info),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1368,21 +1446,8 @@ async def async_transform_request_body(
|
|||
messages, should_inline=should_inline_remote_media(custom_llm_provider)
|
||||
)
|
||||
|
||||
if _openai_messages_may_need_sync_gcs_metadata_fetch(inlined_messages):
|
||||
# _transform_request_body may issue a sync httpx.get (up to 5s timeout)
|
||||
# via _get_gcs_object_content_type to fetch GCS object metadata. Run the
|
||||
# whole sync transformation on a worker thread so it does not block the
|
||||
# async event loop.
|
||||
return await asyncify(transform_request_body)(
|
||||
messages=inlined_messages,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
cached_content=cached_content,
|
||||
optional_params=optional_params,
|
||||
)
|
||||
|
||||
return transform_request_body(
|
||||
transform: Final = partial(
|
||||
transform_request_body,
|
||||
messages=inlined_messages,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
@ -1390,6 +1455,21 @@ async def async_transform_request_body(
|
|||
cached_content=cached_content,
|
||||
optional_params=optional_params,
|
||||
)
|
||||
# _transform_request_body may issue a sync httpx.get (up to 5s timeout)
|
||||
# via _get_gcs_object_content_type to fetch GCS object metadata. Run the
|
||||
# whole sync transformation on a worker thread so it does not block the
|
||||
# async event loop.
|
||||
request_body: Final[RequestBody] = (
|
||||
await asyncify(transform)()
|
||||
if _openai_messages_may_need_sync_gcs_metadata_fetch(inlined_messages)
|
||||
else transform()
|
||||
)
|
||||
return _with_default_output_budget(
|
||||
request_body,
|
||||
model,
|
||||
custom_llm_provider,
|
||||
litellm_params.get("model_info"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is an untyped dict
|
||||
)
|
||||
|
||||
|
||||
def _default_user_message_when_system_message_passed() -> ChatCompletionUserMessage:
|
||||
|
|
|
|||
|
|
@ -3,15 +3,16 @@
|
|||
## Initial implementation - covers gemini + image gen calls
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from copy import deepcopy
|
||||
from functools import partial
|
||||
from functools import lru_cache, partial
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
|
||||
|
||||
import httpx
|
||||
from pydantic import JsonValue
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm import verbose_logger
|
||||
|
|
@ -27,7 +28,8 @@ from litellm.constants import (
|
|||
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO,
|
||||
)
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
encode_tool_call_id_with_signature,
|
||||
|
|
@ -196,6 +198,29 @@ def _unsupported_reasoning_effort(reasoning_effort: str) -> UnsupportedParamsErr
|
|||
)
|
||||
|
||||
|
||||
_SAMPLING_DEPRECATION_UNMAPPED_MODELS: Final = "models outside litellm's model map"
|
||||
_sampling_deprecation_lock: Final = threading.Lock()
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _log_gemini_sampling_params_deprecated(model: str) -> None:
|
||||
verbose_logger.warning(
|
||||
"DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.",
|
||||
model,
|
||||
)
|
||||
|
||||
|
||||
def _sampling_deprecation_notice_key(model: str) -> str:
|
||||
catalog: Final = GetModelCostMap.loaded_model_cost_map()
|
||||
is_cataloged: Final = any(name in catalog for name in (model, f"vertex_ai/{model}", f"gemini/{model}"))
|
||||
return model if is_cataloged else _SAMPLING_DEPRECATION_UNMAPPED_MODELS
|
||||
|
||||
|
||||
def _warn_gemini_sampling_params_deprecated(model: str) -> None:
|
||||
with _sampling_deprecation_lock:
|
||||
_log_gemini_sampling_params_deprecated(_sampling_deprecation_notice_key(model))
|
||||
|
||||
|
||||
def _cache_creation_tokens(logging_obj: LoggingClass) -> int:
|
||||
recorded: Final[object] = getattr(logging_obj, "vertex_ai_cache_creation_input_tokens", 0)
|
||||
return recorded if isinstance(recorded, int) and recorded > 0 else 0
|
||||
|
|
@ -207,6 +232,274 @@ def _served_model_name(model_version: object) -> str | None:
|
|||
return model_version.split("@", 1)[0]
|
||||
|
||||
|
||||
_GEMINI_SPEECH_CONFIG_KEY_MAP: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
"multi_speaker_voice_config": "multiSpeakerVoiceConfig",
|
||||
"speaker_voice_configs": "speakerVoiceConfigs",
|
||||
"voice_config": "voiceConfig",
|
||||
"prebuilt_voice_config": "prebuiltVoiceConfig",
|
||||
"voice_name": "voiceName",
|
||||
"replicated_voice_config": "replicatedVoiceConfig",
|
||||
"mime_type": "mimeType",
|
||||
"voice_sample_audio": "voiceSampleAudio",
|
||||
"language_code": "languageCode",
|
||||
"speaker_alias": "speakerAlias",
|
||||
"speaker_id": "speakerId",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_SPEECH_CONFIG_ITEMS: Final = TypeAdapter(list[object])
|
||||
|
||||
|
||||
def _normalize_gemini_speech_config_item(value: Mapping[str, object]) -> Mapping[str, object]:
|
||||
normalized: Final[dict[str, object]] = {} # mutable-ok: iterative traversal fills nested output without recursion
|
||||
|
||||
pending: Final[ # mutable-ok: the explicit worklist must grow while avoiding recursive traversal
|
||||
list[tuple[dict[str, object] | list[object], Mapping[str, object] | list[object]]]
|
||||
] = [(normalized, value)]
|
||||
while pending:
|
||||
target, source = pending.pop()
|
||||
items = source.items() if isinstance(source, Mapping) else enumerate(source)
|
||||
for key, item in items:
|
||||
if isinstance(source, Mapping):
|
||||
if not isinstance(key, str):
|
||||
continue
|
||||
normalized_key: str | int = _GEMINI_SPEECH_CONFIG_KEY_MAP.get(key, key)
|
||||
else:
|
||||
normalized_key = len(target)
|
||||
|
||||
child: object
|
||||
if (mapping_item := as_str_mapping(item)) is not None:
|
||||
pending.append((child_mapping := dict[str, object](), mapping_item))
|
||||
child = child_mapping
|
||||
elif isinstance(item, list):
|
||||
pending.append((child_items := list[object](), _SPEECH_CONFIG_ITEMS.validate_python(item)))
|
||||
child = child_items
|
||||
else:
|
||||
child = (
|
||||
OPENAI_TO_GEMINI_TTS_VOICES.get(item.lower(), item)
|
||||
if normalized_key == "voiceName" and isinstance(item, str)
|
||||
else item
|
||||
)
|
||||
|
||||
if isinstance(target, dict):
|
||||
assert isinstance(normalized_key, str)
|
||||
target[normalized_key] = child
|
||||
else:
|
||||
target.append(child)
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def _gemini_speaker_voice_config(item: object) -> object:
|
||||
speaker_config: Final = as_str_mapping(item)
|
||||
if speaker_config is None:
|
||||
return item
|
||||
speaker: Final = speaker_config.get("speakerAlias")
|
||||
voice_name: Final = speaker_config.get("speakerId")
|
||||
if not isinstance(speaker, str) or not isinstance(voice_name, str):
|
||||
return item
|
||||
return {
|
||||
**{key: value for key, value in speaker_config.items() if key not in ("speakerAlias", "speakerId")},
|
||||
"speaker": speaker,
|
||||
"voiceConfig": {
|
||||
"prebuiltVoiceConfig": {"voiceName": OPENAI_TO_GEMINI_TTS_VOICES.get(voice_name.lower(), voice_name)}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _with_gemini_speaker_voice_configs(config: Mapping[str, object]) -> Mapping[str, object]:
|
||||
multi_speaker_config: Final = as_str_mapping(config.get("multiSpeakerVoiceConfig"))
|
||||
if multi_speaker_config is None:
|
||||
return config
|
||||
speaker_voice_configs: Final = multi_speaker_config.get("speakerVoiceConfigs")
|
||||
if not isinstance(speaker_voice_configs, list):
|
||||
return config
|
||||
return {
|
||||
**config,
|
||||
"multiSpeakerVoiceConfig": {
|
||||
**multi_speaker_config,
|
||||
"speakerVoiceConfigs": [
|
||||
_gemini_speaker_voice_config(item)
|
||||
for item in _SPEECH_CONFIG_ITEMS.validate_python(speaker_voice_configs)
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
_GEMINI_SPEECH_CONFIG_FIELDS: Final = frozenset({"voiceConfig", "multiSpeakerVoiceConfig", "languageCode"})
|
||||
|
||||
_GEMINI_VOICE_CONFIG_FIELDS: Final = MappingProxyType(
|
||||
{
|
||||
"prebuiltVoiceConfig": frozenset({"voiceName"}),
|
||||
"replicatedVoiceConfig": frozenset({"mimeType", "voiceSampleAudio"}),
|
||||
"voice": None,
|
||||
}
|
||||
)
|
||||
|
||||
_GEMINI_SPEAKER_VOICE_CONFIG_FIELDS: Final = frozenset({"speaker", "voiceConfig"})
|
||||
|
||||
|
||||
def _known_leaf_fields(value: object, fields: frozenset[str] | None) -> object:
|
||||
mapping: Final = as_str_mapping(value)
|
||||
if fields is None or mapping is None:
|
||||
return value
|
||||
return {key: item for key, item in mapping.items() if key in fields}
|
||||
|
||||
|
||||
def _known_voice_config_fields(value: object) -> object:
|
||||
mapping: Final = as_str_mapping(value)
|
||||
if mapping is None:
|
||||
return value
|
||||
return {
|
||||
key: _known_leaf_fields(item, _GEMINI_VOICE_CONFIG_FIELDS[key])
|
||||
for key, item in mapping.items()
|
||||
if key in _GEMINI_VOICE_CONFIG_FIELDS
|
||||
}
|
||||
|
||||
|
||||
def _known_speaker_voice_config_fields(value: object) -> object:
|
||||
mapping: Final = as_str_mapping(value)
|
||||
if mapping is None:
|
||||
return value
|
||||
return {
|
||||
key: _known_voice_config_fields(item) if key == "voiceConfig" else item
|
||||
for key, item in mapping.items()
|
||||
if key in _GEMINI_SPEAKER_VOICE_CONFIG_FIELDS
|
||||
}
|
||||
|
||||
|
||||
def _known_speaker_voice_configs(value: object) -> object:
|
||||
if not isinstance(value, list):
|
||||
return value
|
||||
return [_known_speaker_voice_config_fields(item) for item in _SPEECH_CONFIG_ITEMS.validate_python(value)]
|
||||
|
||||
|
||||
def _known_multi_speaker_voice_config_fields(value: object) -> object:
|
||||
mapping: Final = as_str_mapping(value)
|
||||
if mapping is None:
|
||||
return value
|
||||
return {key: _known_speaker_voice_configs(item) for key, item in mapping.items() if key == "speakerVoiceConfigs"}
|
||||
|
||||
|
||||
def _known_speech_config_field(key: str, value: object) -> object:
|
||||
match key:
|
||||
case "voiceConfig":
|
||||
return _known_voice_config_fields(value)
|
||||
case "multiSpeakerVoiceConfig":
|
||||
return _known_multi_speaker_voice_config_fields(value)
|
||||
case _:
|
||||
return value
|
||||
|
||||
|
||||
def _known_gemini_speech_fields(
|
||||
config: Mapping[str, object],
|
||||
) -> dict[str, object]: # mutable-ok: provider request serialization requires a concrete dict
|
||||
return {
|
||||
key: _known_speech_config_field(key, item)
|
||||
for key, item in config.items()
|
||||
if key in _GEMINI_SPEECH_CONFIG_FIELDS
|
||||
}
|
||||
|
||||
|
||||
_GEMINI_TTS_VOICE_KEYS: Final = frozenset(
|
||||
{
|
||||
"format",
|
||||
"name",
|
||||
"voice",
|
||||
"voiceName",
|
||||
"voice_name",
|
||||
"languageCode",
|
||||
"language_code",
|
||||
"modelName",
|
||||
"model_name",
|
||||
"speechConfig",
|
||||
"speech_config",
|
||||
"voiceConfig",
|
||||
"voice_config",
|
||||
"multiSpeakerVoiceConfig",
|
||||
"multi_speaker_voice_config",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_NAMED_UNKNOWN_VOICE_KEYS: Final = 10
|
||||
|
||||
_NAMED_VOICE_KEY_CHARS: Final = 100
|
||||
|
||||
|
||||
def _gemini_speech_config_wrapper(value: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
camel_config: Final = as_str_mapping(value.get("speechConfig"))
|
||||
return camel_config if camel_config is not None else as_str_mapping(value.get("speech_config"))
|
||||
|
||||
|
||||
def named_voice_keys(keys: Sequence[str]) -> str:
|
||||
named: Final = ", ".join(key[:_NAMED_VOICE_KEY_CHARS] for key in keys[:_NAMED_UNKNOWN_VOICE_KEYS])
|
||||
unnamed_count: Final = len(keys) - _NAMED_UNKNOWN_VOICE_KEYS
|
||||
return f"{named}, and {unnamed_count} more" if unnamed_count > 0 else named
|
||||
|
||||
|
||||
def reject_unknown_gemini_tts_voice_keys(
|
||||
model: str, voice: Mapping[str, object], drop_params: bool, llm_provider: str | None
|
||||
) -> None:
|
||||
nested_config: Final = _gemini_speech_config_wrapper(voice) or {}
|
||||
ignored_wrapper: Final = (
|
||||
("speech_config",)
|
||||
if as_str_mapping(voice.get("speechConfig")) is not None and voice.get("speech_config") is not None
|
||||
else ()
|
||||
)
|
||||
unknown_keys: Final = sorted(
|
||||
{*(key for key in (*voice, *nested_config) if key not in _GEMINI_TTS_VOICE_KEYS), *ignored_wrapper}
|
||||
)
|
||||
if not unknown_keys or drops_unsupported_params(drop_params):
|
||||
return
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} does not support the voice keys {named_voice_keys(unknown_keys)}. Remove them, or set "
|
||||
"drop_params to ignore them."
|
||||
),
|
||||
model=model,
|
||||
llm_provider=llm_provider,
|
||||
)
|
||||
|
||||
|
||||
def normalize_gemini_speech_config(
|
||||
value: Mapping[str, object], drop_unknown_fields: bool = False
|
||||
) -> dict[str, object]: # mutable-ok: provider request serialization requires a concrete dict
|
||||
speech_config: Final = {**_with_gemini_speaker_voice_configs(mapped_gemini_speech_config(value))}
|
||||
return _known_gemini_speech_fields(speech_config) if drop_unknown_fields else speech_config
|
||||
|
||||
|
||||
def _merge_gemini_speech_config_wrapper(value: Mapping[str, object]) -> Mapping[str, object]:
|
||||
nested_config: Final = _gemini_speech_config_wrapper(value)
|
||||
if nested_config is None:
|
||||
return value
|
||||
return {
|
||||
**{key: item for key, item in value.items() if key not in ("speechConfig", "speech_config")},
|
||||
**nested_config,
|
||||
}
|
||||
|
||||
|
||||
def mapped_gemini_speech_config(
|
||||
value: Mapping[str, object],
|
||||
) -> dict[str, object]: # mutable-ok: provider request serialization requires a concrete dict
|
||||
normalized: Final = _normalize_gemini_speech_config_item(_merge_gemini_speech_config_wrapper(value))
|
||||
speech_config: Final = {key: item for key, item in normalized.items() if key in _GEMINI_SPEECH_CONFIG_FIELDS}
|
||||
voice_name: Final = next(
|
||||
(normalized[key] for key in ("name", "voiceName", "voice") if isinstance(normalized.get(key), str)),
|
||||
None,
|
||||
)
|
||||
if "voiceConfig" in speech_config or "multiSpeakerVoiceConfig" in speech_config or not isinstance(voice_name, str):
|
||||
return speech_config
|
||||
return {
|
||||
**speech_config,
|
||||
"voiceConfig": {
|
||||
"prebuiltVoiceConfig": {"voiceName": OPENAI_TO_GEMINI_TTS_VOICES.get(voice_name.lower(), voice_name)}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class VertexAIBaseConfig:
|
||||
def get_mapped_special_auth_params(self) -> dict:
|
||||
"""
|
||||
|
|
@ -404,11 +697,43 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
return False
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def supports_sampling_params(model: str) -> bool:
|
||||
"""Whether Google still honors ``temperature`` / ``top_p`` / ``top_k`` for ``model``.
|
||||
|
||||
Driven by ``supports_sampling_params`` in the model map: Gemini 3.6 Flash, 3.5 Flash-Lite
|
||||
and later accept the fields but ignore them, and a later generation will 400 on them.
|
||||
A model the map says nothing about keeps forwarding them. ``temperature`` and ``top_p``
|
||||
leave ``get_supported_openai_params`` when the flag is off, so the shared unsupported-param
|
||||
check raises or drops them and ``allowed_openai_params`` still forwards them verbatim;
|
||||
``top_k`` is a provider-specific kwarg that bypasses that check and is gated while the
|
||||
request body is built."""
|
||||
rows: Final = cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: JSON rows by model id
|
||||
bare: Final = model.split("/")[-1]
|
||||
for candidate in (model, bare, f"vertex_ai/{bare}", f"gemini/{bare}"):
|
||||
if isinstance(flag := rows.get(candidate, {}).get("supports_sampling_params"), bool):
|
||||
return flag
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def reject_unsupported_sampling_param(model: str, param: str, drop_params: bool) -> None:
|
||||
"""Return under ``drop_params`` so the caller drops ``param``, else raise the client-side 400
|
||||
that names the escape hatch."""
|
||||
if litellm.drop_params or drop_params:
|
||||
return
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} does not support {param}: the model map flags it `supports_sampling_params: false` "
|
||||
"because Google ignores sampling params on this model. "
|
||||
"To drop unsupported params, set `litellm.drop_params = True`."
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
sampling_params: Final = ("temperature", "top_p", "top_k") if self.supports_sampling_params(model) else ()
|
||||
supported_params: Final = [
|
||||
"temperature",
|
||||
"top_p",
|
||||
"top_k",
|
||||
*sampling_params,
|
||||
"max_tokens",
|
||||
"max_completion_tokens",
|
||||
"stream",
|
||||
|
|
@ -1180,7 +1505,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
status_code=400,
|
||||
)
|
||||
|
||||
def _map_audio_params(self, value: dict) -> dict:
|
||||
def _map_audio_params(
|
||||
self, value: Mapping[str, object], drop_unknown_fields: bool = False
|
||||
) -> dict[str, object]: # mutable-ok: the public forwarding interface returns a concrete request dictionary
|
||||
"""
|
||||
Expected input:
|
||||
{
|
||||
|
|
@ -1199,12 +1526,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
languageCode: "en-US",
|
||||
}
|
||||
"""
|
||||
from litellm.types.llms.vertex_ai import (
|
||||
PrebuiltVoiceConfig,
|
||||
SpeechConfig,
|
||||
VoiceConfig,
|
||||
)
|
||||
|
||||
# Validate audio format - Gemini TTS only supports pcm16
|
||||
audio_format: Final = value.get("format")
|
||||
if audio_format is not None and audio_format != "pcm16":
|
||||
|
|
@ -1214,22 +1535,46 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
f"Please set audio format to 'pcm16'."
|
||||
)
|
||||
|
||||
# Map OpenAI audio parameter to Gemini speech config
|
||||
speech_config: Final[SpeechConfig] = {}
|
||||
camel_speech_config: Final = value.get("speechConfig")
|
||||
snake_speech_config: Final = value.get("speech_config")
|
||||
speech_config_value: Final = (
|
||||
camel_speech_config if isinstance(camel_speech_config, Mapping) else snake_speech_config
|
||||
)
|
||||
language_code: Final = value.get("language_code", value.get("languageCode"))
|
||||
if isinstance(speech_config_value, Mapping):
|
||||
return normalize_gemini_speech_config(value, drop_unknown_fields)
|
||||
|
||||
if "voice" in value:
|
||||
voice: Final = value["voice"]
|
||||
voice_name: Final = (
|
||||
OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice) if isinstance(voice, str) else voice
|
||||
)
|
||||
prebuilt_voice_config: Final[PrebuiltVoiceConfig] = {"voiceName": voice_name}
|
||||
voice_config: Final[VoiceConfig] = {"prebuiltVoiceConfig": prebuilt_voice_config}
|
||||
speech_config["voiceConfig"] = voice_config
|
||||
voice: Final = value.get("voice")
|
||||
speech_config_without_language: Final[ # mutable-ok: provider request serialization requires a concrete dict
|
||||
dict[str, object]
|
||||
] = (
|
||||
{
|
||||
"voiceConfig": {
|
||||
"prebuiltVoiceConfig": {
|
||||
"voiceName": OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice)
|
||||
if isinstance(voice, str)
|
||||
else voice,
|
||||
}
|
||||
}
|
||||
}
|
||||
if "voice" in value
|
||||
else {}
|
||||
)
|
||||
|
||||
if "language_code" in value:
|
||||
speech_config["languageCode"] = value["language_code"]
|
||||
if language_code is not None:
|
||||
return {
|
||||
**speech_config_without_language,
|
||||
"languageCode": language_code,
|
||||
}
|
||||
|
||||
return cast(dict, speech_config)
|
||||
return speech_config_without_language
|
||||
|
||||
def _map_chat_audio_params(
|
||||
self, model: str, value: Mapping[str, object], drop_params: bool
|
||||
) -> dict[str, object]: # mutable-ok: the public forwarding interface returns a concrete request dictionary
|
||||
if _gemini_speech_config_wrapper(value) is not None:
|
||||
reject_unknown_gemini_tts_voice_keys(model, value, drop_params, self.custom_llm_provider)
|
||||
return self._map_audio_params(value, drop_unknown_fields=drops_unsupported_params(drop_params))
|
||||
|
||||
def map_audio_params(
|
||||
self,
|
||||
|
|
@ -1266,8 +1611,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
allow_integer_enums: bool = True,
|
||||
) -> dict:
|
||||
self._apply_include_server_side_tool_invocations(non_default_params, optional_params)
|
||||
gemini_sampling_params_warned: bool = False
|
||||
supports_sampling_params: Final = self.supports_sampling_params(model)
|
||||
for param, value in non_default_params.items():
|
||||
if param in ("temperature", "top_p", "top_k") and not supports_sampling_params:
|
||||
continue
|
||||
if param == "temperature":
|
||||
if VertexGeminiConfig._is_gemini_3_or_newer(model):
|
||||
if value is not None and value < 1.0:
|
||||
|
|
@ -1275,28 +1622,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
"Warning: Setting temperature < 1.0 for Gemini 3 models (%s) can cause infinite loops, degraded reasoning performance, and failure on complex tasks. Strongly recommended to use temperature = 1.0 (default).",
|
||||
model,
|
||||
)
|
||||
if not gemini_sampling_params_warned:
|
||||
verbose_logger.warning(
|
||||
"DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.",
|
||||
model,
|
||||
)
|
||||
gemini_sampling_params_warned = True
|
||||
_warn_gemini_sampling_params_deprecated(model)
|
||||
optional_params["temperature"] = value
|
||||
elif param == "top_p":
|
||||
if VertexGeminiConfig._is_gemini_3_or_newer(model) and not gemini_sampling_params_warned:
|
||||
verbose_logger.warning(
|
||||
"DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.",
|
||||
model,
|
||||
)
|
||||
gemini_sampling_params_warned = True
|
||||
if VertexGeminiConfig._is_gemini_3_or_newer(model):
|
||||
_warn_gemini_sampling_params_deprecated(model)
|
||||
optional_params["top_p"] = value
|
||||
elif param == "top_k":
|
||||
if VertexGeminiConfig._is_gemini_3_or_newer(model) and not gemini_sampling_params_warned:
|
||||
verbose_logger.warning(
|
||||
"DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.",
|
||||
model,
|
||||
)
|
||||
gemini_sampling_params_warned = True
|
||||
if VertexGeminiConfig._is_gemini_3_or_newer(model):
|
||||
_warn_gemini_sampling_params_deprecated(model)
|
||||
optional_params["top_k"] = value
|
||||
elif (
|
||||
param == "stream" and value is True
|
||||
|
|
@ -1305,7 +1639,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
elif param == "n":
|
||||
optional_params["candidate_count"] = value
|
||||
elif param == "audio" and isinstance(value, dict):
|
||||
optional_params["speechConfig"] = self._map_audio_params(value)
|
||||
optional_params["speechConfig"] = self._map_chat_audio_params(model, value, drop_params)
|
||||
elif param == "stop":
|
||||
if isinstance(value, str):
|
||||
optional_params["stop_sequences"] = [value]
|
||||
|
|
|
|||
|
|
@ -6,20 +6,25 @@ Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/s
|
|||
"""
|
||||
|
||||
import base64
|
||||
from collections.abc import Coroutine, Mapping
|
||||
import math
|
||||
from collections.abc import Coroutine, Iterator, Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, TypeAlias, Union
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Final, TypeAlias, Union
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.audio_utils.utils import (
|
||||
DEFAULT_SPEECH_MEDIA_TYPE,
|
||||
calculate_request_duration,
|
||||
speech_media_type_from_audio_bytes,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
|
||||
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.text_to_speech.transformation import (
|
||||
BaseTextToSpeechConfig,
|
||||
TextToSpeechRequestData,
|
||||
|
|
@ -33,6 +38,7 @@ from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
|
|||
from litellm.types.llms.vertex_ai_text_to_speech import (
|
||||
VertexTextToSpeechAudioConfig,
|
||||
VertexTextToSpeechInput,
|
||||
VertexTextToSpeechSpeakerVoiceConfig,
|
||||
VertexTextToSpeechVoice,
|
||||
)
|
||||
|
||||
|
|
@ -49,6 +55,166 @@ _LyriaVoice: TypeAlias = str | dict | None
|
|||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...], config=ConfigDict(hide_input_in_errors=True))
|
||||
_STR: Final = TypeAdapter(str, config=ConfigDict(hide_input_in_errors=True))
|
||||
_GEMINI_CLOUD_TTS_LOCATIONS: Final = MappingProxyType(
|
||||
{
|
||||
**dict.fromkeys(
|
||||
("us", "us-central1", "us-east1", "us-east4", "us-east5", "us-south1", "us-west1", "us-west4"), "us"
|
||||
),
|
||||
**dict.fromkeys(
|
||||
("eu", "europe-central2", "europe-north1", "europe-southwest1", "europe-west1", "europe-west4"), "eu"
|
||||
),
|
||||
"northamerica-northeast1": "northamerica-northeast1",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_DEFAULT_GEMINI_TTS_VOICE: Final = "Kore"
|
||||
|
||||
|
||||
class _GeminiCloudTtsInput(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
text: str | None = None
|
||||
ssml: str | None = None
|
||||
prompt: str | None = None
|
||||
|
||||
|
||||
class _GeminiCloudTtsAudioConfig(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
audioEncoding: str = ""
|
||||
sampleRateHertz: int | None = None
|
||||
|
||||
|
||||
class _GeminiCloudTtsRequest(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
input: _GeminiCloudTtsInput
|
||||
audioConfig: _GeminiCloudTtsAudioConfig
|
||||
|
||||
|
||||
def _fallback_gemini_tts_audio_duration(audio: bytes, encoding: str, sample_rate: int) -> float | None:
|
||||
if encoding == "PCM":
|
||||
return len(audio) / (2 * sample_rate) if sample_rate > 0 else None
|
||||
if encoding in ("ALAW", "MULAW") and not (audio[:4] == b"RIFF" and audio[8:12] == b"WAVE"):
|
||||
return len(audio) / sample_rate if sample_rate > 0 else None
|
||||
|
||||
if encoding in ("LINEAR16", "ALAW", "MULAW") and audio[:4] == b"RIFF" and audio[8:12] == b"WAVE":
|
||||
format_offset: Final = audio.find(b"fmt ", 12)
|
||||
data_offset: Final = audio.find(b"data", 12)
|
||||
if format_offset >= 0 and data_offset >= 0 and format_offset + 20 <= len(audio):
|
||||
byte_rate: Final = int.from_bytes(audio[format_offset + 16 : format_offset + 20], "little")
|
||||
data_size: Final = int.from_bytes(audio[data_offset + 4 : data_offset + 8], "little")
|
||||
if byte_rate > 0 and data_size <= len(audio) - data_offset - 8:
|
||||
return data_size / byte_rate
|
||||
|
||||
if encoding == "MP3" and speech_media_type_from_audio_bytes(audio) == "audio/mpeg":
|
||||
return len(audio) / 4000
|
||||
|
||||
if encoding == "OGG_OPUS" and audio[:4] == b"OggS":
|
||||
opus_header: Final = audio.find(b"OpusHead")
|
||||
last_page: Final = audio.rfind(b"OggS")
|
||||
if opus_header >= 0 and opus_header + 12 <= len(audio) and last_page + 14 <= len(audio):
|
||||
pre_skip: Final = int.from_bytes(audio[opus_header + 10 : opus_header + 12], "little")
|
||||
granule: Final = int.from_bytes(audio[last_page + 6 : last_page + 14], "little")
|
||||
if pre_skip <= granule < 2**64 - 1:
|
||||
return (granule - pre_skip) / 48000
|
||||
|
||||
return None
|
||||
|
||||
|
||||
_SPEAKER_CONFIG_ITEMS: Final = TypeAdapter(list[object])
|
||||
|
||||
_CloudTtsVoiceFields: TypeAlias = "Mapping[str, _CloudTtsVoiceFields]"
|
||||
|
||||
_CLOUD_TTS_VOICE_LEAF: Final[_CloudTtsVoiceFields] = MappingProxyType({})
|
||||
|
||||
_CLOUD_TTS_PREBUILT_VOICE_FIELDS: Final[_CloudTtsVoiceFields] = MappingProxyType(
|
||||
{"prebuiltVoiceConfig": MappingProxyType({"voiceName": _CLOUD_TTS_VOICE_LEAF})}
|
||||
)
|
||||
|
||||
_CLOUD_TTS_SPEECH_CONFIG_FIELDS: Final[_CloudTtsVoiceFields] = MappingProxyType(
|
||||
{
|
||||
"languageCode": _CLOUD_TTS_VOICE_LEAF,
|
||||
"voiceConfig": _CLOUD_TTS_PREBUILT_VOICE_FIELDS,
|
||||
"multiSpeakerVoiceConfig": MappingProxyType(
|
||||
{
|
||||
"speakerVoiceConfigs": MappingProxyType(
|
||||
{
|
||||
"speaker": _CLOUD_TTS_VOICE_LEAF,
|
||||
"speakerAlias": _CLOUD_TTS_VOICE_LEAF,
|
||||
"speakerId": _CLOUD_TTS_VOICE_LEAF,
|
||||
"voiceConfig": _CLOUD_TTS_PREBUILT_VOICE_FIELDS,
|
||||
}
|
||||
)
|
||||
}
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_CLOUD_TTS_LIST_FIELDS: Final = frozenset({"speakerVoiceConfigs"})
|
||||
|
||||
_GEMINI_TTS_VOICE_NAME_KEYS: Final = ("name", "voiceName", "voice_name", "voice")
|
||||
|
||||
|
||||
def _fields_cloud_tts_cannot_send(
|
||||
config: Mapping[str, object], fields: _CloudTtsVoiceFields, path: str
|
||||
) -> Iterator[str]:
|
||||
for key, value in config.items():
|
||||
yield from _field_cloud_tts_cannot_send(f"{path}{key}", value, fields.get(key), key in _CLOUD_TTS_LIST_FIELDS)
|
||||
|
||||
|
||||
def _field_cloud_tts_cannot_send(
|
||||
path: str, value: object, fields: "_CloudTtsVoiceFields | None", is_list_field: bool
|
||||
) -> Iterator[str]:
|
||||
if fields is None:
|
||||
yield path
|
||||
return
|
||||
if value is None:
|
||||
return
|
||||
if not fields:
|
||||
yield from () if isinstance(value, str) else (path,)
|
||||
return
|
||||
items: Final = (
|
||||
_SPEAKER_CONFIG_ITEMS.validate_python(value) if is_list_field and isinstance(value, list) else (value,)
|
||||
)
|
||||
for item in items:
|
||||
mapping = as_str_mapping(item)
|
||||
yield from (path,) if mapping is None else _fields_cloud_tts_cannot_send(mapping, fields, f"{path}.")
|
||||
|
||||
|
||||
def _non_string_voice_names(config: Mapping[str, object], path: str) -> Iterator[str]:
|
||||
return (
|
||||
f"{path}{key}"
|
||||
for key in _GEMINI_TTS_VOICE_NAME_KEYS
|
||||
if config.get(key) is not None and not isinstance(config.get(key), str)
|
||||
)
|
||||
|
||||
|
||||
def _non_mapping_speech_config_wrappers(voice: Mapping[str, object]) -> Iterator[str]:
|
||||
return (
|
||||
key
|
||||
for key in ("speechConfig", "speech_config")
|
||||
if voice.get(key) is not None and as_str_mapping(voice.get(key)) is None
|
||||
)
|
||||
|
||||
|
||||
def _speaker_entries(config: Mapping[str, object]) -> Iterator[Mapping[str, object]]:
|
||||
multi_speaker_config: Final = as_str_mapping(config.get("multiSpeakerVoiceConfig"))
|
||||
speakers: Final = multi_speaker_config.get("speakerVoiceConfigs") if multi_speaker_config is not None else None
|
||||
items: Final = _SPEAKER_CONFIG_ITEMS.validate_python(speakers) if isinstance(speakers, list) else ()
|
||||
return (entry for item in items if (entry := as_str_mapping(item)) is not None)
|
||||
|
||||
|
||||
def _prebuilt_voices_replaced_by_speaker_id(config: Mapping[str, object]) -> Iterator[str]:
|
||||
return (
|
||||
"multiSpeakerVoiceConfig.speakerVoiceConfigs.voiceConfig.prebuiltVoiceConfig"
|
||||
for entry in _speaker_entries(config)
|
||||
if entry.get("speakerId") is not None
|
||||
and (voice_config := as_str_mapping(entry.get("voiceConfig"))) is not None
|
||||
and voice_config.get("prebuiltVoiceConfig") is not None
|
||||
)
|
||||
|
||||
|
||||
class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
|
||||
|
|
@ -81,14 +247,19 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
|
|||
}
|
||||
|
||||
# Response format mappings from OpenAI to Google Cloud audio encoding
|
||||
FORMAT_MAPPINGS = {
|
||||
"mp3": "MP3",
|
||||
"opus": "OGG_OPUS",
|
||||
"aac": "MP3", # Google doesn't have AAC, use MP3
|
||||
"flac": "FLAC",
|
||||
"wav": "LINEAR16",
|
||||
"pcm": "LINEAR16",
|
||||
}
|
||||
FORMAT_MAPPINGS: ClassVar[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
"mp3": "MP3",
|
||||
"opus": "OGG_OPUS",
|
||||
"aac": "MP3", # Google doesn't have AAC, use MP3
|
||||
"flac": "FLAC",
|
||||
"wav": "LINEAR16",
|
||||
"pcm": "LINEAR16",
|
||||
}
|
||||
)
|
||||
GEMINI_FORMAT_MAPPINGS: ClassVar[Mapping[str, str]] = MappingProxyType(
|
||||
{"mp3": "MP3", "opus": "OGG_OPUS", "alaw": "ALAW", "mulaw": "MULAW"}
|
||||
)
|
||||
|
||||
def __init__(self) -> None:
|
||||
BaseTextToSpeechConfig.__init__(self)
|
||||
|
|
@ -96,7 +267,9 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
|
|||
|
||||
def _map_voice_to_vertex_format(
|
||||
self,
|
||||
voice: str | dict | None,
|
||||
voice: str | dict[str, object] | None,
|
||||
model: str | None = None,
|
||||
drop_params: bool = False,
|
||||
) -> tuple[str | None, dict | None]:
|
||||
"""
|
||||
Map voice to Vertex AI format.
|
||||
|
|
@ -111,6 +284,9 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
|
|||
- voice_str: Original string voice (for interface compatibility)
|
||||
- voice_dict: Vertex AI format dict with languageCode and name
|
||||
"""
|
||||
if model is not None and self._is_gemini_tts_model(model):
|
||||
return self._map_gemini_tts_voice_to_vertex_format(model=model, voice=voice, drop_params=drop_params)
|
||||
|
||||
if voice is None:
|
||||
return None, None
|
||||
|
||||
|
|
@ -142,11 +318,225 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
|
|||
|
||||
return voice_str, voice_dict
|
||||
|
||||
@staticmethod
|
||||
def _is_gemini_tts_model(model: str) -> bool:
|
||||
from litellm.utils import is_gemini_tts_model
|
||||
|
||||
return is_gemini_tts_model(model)
|
||||
|
||||
@staticmethod
|
||||
def _get_str_value(
|
||||
source: Mapping[str, object],
|
||||
*keys: str,
|
||||
) -> str | None:
|
||||
for key in keys:
|
||||
value = source.get(key)
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_dict_value(
|
||||
source: Mapping[str, object],
|
||||
*keys: str,
|
||||
) -> Mapping[str, object] | None:
|
||||
for key in keys:
|
||||
value = source.get(key)
|
||||
if (mapping := as_str_mapping(value)) is not None:
|
||||
return mapping
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _resolve_gemini_tts_voice_name(model: str, voice_name: str, drop_params: bool) -> str:
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import OPENAI_TO_GEMINI_TTS_VOICES
|
||||
|
||||
gemini_voice_name: Final = OPENAI_TO_GEMINI_TTS_VOICES.get(voice_name.lower(), voice_name)
|
||||
if "-" not in gemini_voice_name:
|
||||
return gemini_voice_name
|
||||
if drop_params:
|
||||
return _DEFAULT_GEMINI_TTS_VOICE
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} speaks Gemini voices such as {_DEFAULT_GEMINI_TTS_VOICE}, and {voice_name} is a Cloud "
|
||||
"Text-to-Speech voice. Use a Gemini voice, or set drop_params to use the default Gemini voice."
|
||||
),
|
||||
model=model,
|
||||
llm_provider="vertex_ai",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _reject_malformed_gemini_tts_voice(model: str, drop_params: bool) -> None:
|
||||
if drop_params:
|
||||
return
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} needs voice to be a Gemini voice name such as {_DEFAULT_GEMINI_TTS_VOICE} or a speech "
|
||||
"config object. Send one, or set drop_params to use the default Gemini voice."
|
||||
),
|
||||
model=model,
|
||||
llm_provider="vertex_ai",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _reject_voice_fields_cloud_tts_cannot_send(model: str, voice: Mapping[str, object], drop_params: bool) -> None:
|
||||
if drop_params:
|
||||
return
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
mapped_gemini_speech_config,
|
||||
named_voice_keys,
|
||||
)
|
||||
|
||||
speech_config: Final = VertexAITextToSpeechConfig._get_dict_value(voice, "speechConfig", "speech_config")
|
||||
mapped_config: Final = mapped_gemini_speech_config(voice)
|
||||
unsendable: Final = sorted(
|
||||
{
|
||||
*_fields_cloud_tts_cannot_send(mapped_config, _CLOUD_TTS_SPEECH_CONFIG_FIELDS, ""),
|
||||
*_prebuilt_voices_replaced_by_speaker_id(mapped_config),
|
||||
*_non_mapping_speech_config_wrappers(voice),
|
||||
*_non_string_voice_names(voice, ""),
|
||||
*_non_string_voice_names(speech_config or {}, "speechConfig."),
|
||||
}
|
||||
)
|
||||
if not unsendable:
|
||||
return
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} cannot send the voice fields {named_voice_keys(unsendable)} as mp3, opus, alaw, or mulaw. "
|
||||
"Request wav or pcm, remove them, or set drop_params to ignore them."
|
||||
),
|
||||
model=model,
|
||||
llm_provider="vertex_ai",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _reject_malformed_gemini_tts_speaker(model: str, drop_params: bool) -> None:
|
||||
if drop_params:
|
||||
return
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} needs each entry of speaker_voice_configs to name a speaker and a voice, as speaker with "
|
||||
"voice_config.prebuilt_voice_config.voice_name, or speakerAlias with speakerId."
|
||||
),
|
||||
model=model,
|
||||
llm_provider="vertex_ai",
|
||||
)
|
||||
|
||||
def _extract_gemini_tts_speaker_configs(
|
||||
self,
|
||||
model: str,
|
||||
speech_config: Mapping[str, object],
|
||||
drop_params: bool,
|
||||
) -> Sequence[VertexTextToSpeechSpeakerVoiceConfig]:
|
||||
multi_speaker_config: Final = self._get_dict_value(speech_config, "multiSpeakerVoiceConfig")
|
||||
if multi_speaker_config is None:
|
||||
return []
|
||||
raw_speaker_configs: Final = multi_speaker_config.get("speakerVoiceConfigs", [])
|
||||
try:
|
||||
validated_speaker_configs: Final = _SPEAKER_CONFIG_ITEMS.validate_python(raw_speaker_configs, strict=True)
|
||||
except ValidationError:
|
||||
self._reject_malformed_gemini_tts_speaker(model, drop_params)
|
||||
return []
|
||||
speaker_configs: Final[ # mutable-ok: validated payloads are accumulated for serialization
|
||||
list[VertexTextToSpeechSpeakerVoiceConfig]
|
||||
] = []
|
||||
for raw_item in validated_speaker_configs:
|
||||
if (raw_config := as_str_mapping(raw_item)) is None:
|
||||
self._reject_malformed_gemini_tts_speaker(model, drop_params)
|
||||
continue
|
||||
speaker_alias = self._get_str_value(raw_config, "speakerAlias", "speaker")
|
||||
speaker_id = self._get_str_value(raw_config, "speakerId") or self._prebuilt_voice_name(raw_config)
|
||||
if speaker_alias is None or speaker_id is None:
|
||||
self._reject_malformed_gemini_tts_speaker(model, drop_params)
|
||||
continue
|
||||
speaker_configs.append(
|
||||
{
|
||||
"speakerAlias": speaker_alias,
|
||||
"speakerId": self._resolve_gemini_tts_voice_name(model, speaker_id, drop_params),
|
||||
}
|
||||
)
|
||||
return speaker_configs
|
||||
|
||||
def _prebuilt_voice_name(self, config: Mapping[str, object]) -> str | None:
|
||||
voice_config: Final = self._get_dict_value(config, "voiceConfig")
|
||||
prebuilt_voice_config: Final = (
|
||||
self._get_dict_value(voice_config, "prebuiltVoiceConfig") if voice_config is not None else None
|
||||
)
|
||||
if prebuilt_voice_config is None:
|
||||
return None
|
||||
return self._get_str_value(prebuilt_voice_config, "voiceName")
|
||||
|
||||
def _map_gemini_tts_voice_to_vertex_format(
|
||||
self,
|
||||
model: str,
|
||||
voice: object,
|
||||
drop_params: bool = False,
|
||||
) -> tuple[str | None, dict[str, object]]: # mutable-ok: provider request serialization requires a concrete dict
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
mapped_gemini_speech_config,
|
||||
reject_unknown_gemini_tts_voice_keys,
|
||||
)
|
||||
|
||||
default_voice: Final[dict[str, object]] = {
|
||||
"languageCode": self.DEFAULT_LANGUAGE_CODE,
|
||||
"modelName": model,
|
||||
"name": _DEFAULT_GEMINI_TTS_VOICE,
|
||||
}
|
||||
if voice is None:
|
||||
return None, default_voice
|
||||
|
||||
if isinstance(voice, str) and voice:
|
||||
return voice, {
|
||||
"languageCode": self.DEFAULT_LANGUAGE_CODE,
|
||||
"modelName": model,
|
||||
"name": self._resolve_gemini_tts_voice_name(model, voice, drop_params),
|
||||
}
|
||||
|
||||
voice_config: Final = as_str_mapping(voice)
|
||||
if voice_config is None:
|
||||
self._reject_malformed_gemini_tts_voice(model, drop_params)
|
||||
return None, default_voice
|
||||
reject_unknown_gemini_tts_voice_keys(model, voice_config, drop_params, "vertex_ai")
|
||||
|
||||
speech_config: Final = mapped_gemini_speech_config(voice_config)
|
||||
language_code: Final = self._get_str_value(speech_config, "languageCode") or self.DEFAULT_LANGUAGE_CODE
|
||||
model_name: Final = model
|
||||
speaker_configs: Final = self._extract_gemini_tts_speaker_configs(model, speech_config, drop_params)
|
||||
self._reject_voice_fields_cloud_tts_cannot_send(model, voice_config, drop_params)
|
||||
if speaker_configs:
|
||||
return None, {
|
||||
"languageCode": language_code,
|
||||
"modelName": model_name,
|
||||
"multiSpeakerVoiceConfig": {
|
||||
"speakerVoiceConfigs": speaker_configs,
|
||||
},
|
||||
}
|
||||
voice_name: Final = self._prebuilt_voice_name(speech_config)
|
||||
if voice_name is not None:
|
||||
return None, {
|
||||
"languageCode": language_code,
|
||||
"modelName": model_name,
|
||||
"name": self._resolve_gemini_tts_voice_name(model, voice_name, drop_params),
|
||||
}
|
||||
return None, {
|
||||
"languageCode": language_code,
|
||||
"modelName": model_name,
|
||||
"name": _DEFAULT_GEMINI_TTS_VOICE,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _dispatch_voice_name(voice: str | Mapping[str, object] | None) -> str | None:
|
||||
if isinstance(voice, str):
|
||||
return voice
|
||||
if not isinstance(voice, Mapping):
|
||||
return None
|
||||
name: Final = voice.get("name")
|
||||
return name if isinstance(name, str) else None
|
||||
|
||||
def dispatch_text_to_speech(
|
||||
self,
|
||||
model: str,
|
||||
input: str,
|
||||
voice: str | dict | None,
|
||||
voice: str | dict[str, object] | None,
|
||||
optional_params: dict[str, object],
|
||||
litellm_params_dict: dict[str, object],
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
|
|
@ -175,14 +565,17 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
|
|||
vertex_project: Final = self.safe_get_vertex_ai_project(litellm_params_dict)
|
||||
vertex_location: Final = self.safe_get_vertex_ai_location(litellm_params_dict)
|
||||
|
||||
# Convert voice to string if it's a dict (extract name)
|
||||
# Actual voice mapping happens in map_openai_params
|
||||
voice_str: str | None = None
|
||||
if isinstance(voice, str):
|
||||
voice_str = voice
|
||||
elif isinstance(voice, dict):
|
||||
# Extract voice name from dict if needed
|
||||
voice_str = voice.get("name") if voice else None
|
||||
mapped_voice, mapped_params = (
|
||||
self.map_openai_params(
|
||||
model=model,
|
||||
voice=voice,
|
||||
optional_params=optional_params,
|
||||
drop_params=drops_unsupported_params(kwargs.get("drop_params")),
|
||||
kwargs=kwargs,
|
||||
)
|
||||
if self._is_gemini_tts_model(model)
|
||||
else (self._dispatch_voice_name(voice), optional_params)
|
||||
)
|
||||
|
||||
# Store credentials in litellm_params for use in transform methods
|
||||
litellm_params_dict.update(
|
||||
|
|
@ -198,9 +591,11 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
|
|||
response: Final = base_llm_http_handler.text_to_speech_handler(
|
||||
model=model,
|
||||
input=input,
|
||||
voice=voice_str,
|
||||
text_to_speech_provider_config=self,
|
||||
text_to_speech_optional_params=optional_params,
|
||||
voice=mapped_voice,
|
||||
text_to_speech_provider_config=(
|
||||
VertexAIGeminiCloudTextToSpeechConfig() if self._is_gemini_tts_model(model) else self
|
||||
),
|
||||
text_to_speech_optional_params=mapped_params,
|
||||
custom_llm_provider="vertex_ai",
|
||||
litellm_params=litellm_params_dict,
|
||||
logging_obj=logging_obj,
|
||||
|
|
@ -219,13 +614,13 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
|
|||
Note: Vertex AI also supports additional parameters like audioConfig
|
||||
which can be passed but are not part of the OpenAI spec
|
||||
"""
|
||||
return ["voice", "response_format", "speed"]
|
||||
return ["voice", "response_format", "speed", *(["instructions"] if self._is_gemini_tts_model(model) else [])]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
voice: str | dict | None = None,
|
||||
voice: str | dict[str, object] | None = None,
|
||||
drop_params: bool = False,
|
||||
kwargs: dict = {},
|
||||
) -> tuple[str | None, dict]:
|
||||
|
|
@ -244,19 +639,28 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
|
|||
Tuple of (mapped_voice_str, mapped_params)
|
||||
"""
|
||||
mapped_params: Final[dict[str, object]] = {}
|
||||
openai_params: Final = _JSON_OBJECT.validate_python(optional_params)
|
||||
drops_params: Final = drop_params or litellm.drop_params is True
|
||||
|
||||
##########################################################
|
||||
# Map voice using helper
|
||||
##########################################################
|
||||
mapped_voice_str, voice_dict = self._map_voice_to_vertex_format(voice)
|
||||
mapped_voice_str, voice_dict = self._map_voice_to_vertex_format(
|
||||
voice=voice,
|
||||
model=model,
|
||||
drop_params=drops_params,
|
||||
)
|
||||
if voice_dict is not None:
|
||||
mapped_params["vertex_voice_dict"] = voice_dict
|
||||
|
||||
# Map response format
|
||||
if "response_format" in optional_params:
|
||||
format_name: Final = optional_params["response_format"]
|
||||
if format_name in self.FORMAT_MAPPINGS:
|
||||
mapped_params["audioEncoding"] = self.FORMAT_MAPPINGS[format_name]
|
||||
format_mappings: Final = (
|
||||
self.GEMINI_FORMAT_MAPPINGS if self._is_gemini_tts_model(model) else self.FORMAT_MAPPINGS
|
||||
)
|
||||
if format_name in format_mappings:
|
||||
mapped_params["audioEncoding"] = format_mappings[format_name]
|
||||
else:
|
||||
# Try to use it directly as Google Cloud format
|
||||
mapped_params["audioEncoding"] = format_name
|
||||
|
|
@ -265,10 +669,19 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
|
|||
mapped_params["audioEncoding"] = self.DEFAULT_AUDIO_ENCODING
|
||||
|
||||
# Map speed (OpenAI: 0.25-4.0, Vertex AI: speakingRate 0.25-4.0)
|
||||
if "speed" in optional_params:
|
||||
speed: Final = optional_params["speed"]
|
||||
if speed is not None:
|
||||
mapped_params["speakingRate"] = str(speed)
|
||||
speed: Final = openai_params.get("speed")
|
||||
if speed is not None:
|
||||
mapped_params["speakingRate"] = str(speed)
|
||||
|
||||
instructions: Final = openai_params.get("instructions")
|
||||
if isinstance(instructions, str) and self._is_gemini_tts_model(model):
|
||||
mapped_params["prompt"] = instructions
|
||||
elif instructions is not None and self._is_gemini_tts_model(model) and not drops_params:
|
||||
raise UnsupportedParamsError(
|
||||
message=f"{model} needs instructions to be a string. Send one, or set drop_params to ignore it.",
|
||||
model=model,
|
||||
llm_provider="vertex_ai",
|
||||
)
|
||||
|
||||
# Pass through Vertex AI-specific parameters from kwargs
|
||||
if "audioConfig" in kwargs:
|
||||
|
|
@ -311,15 +724,48 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
|
|||
|
||||
Google Cloud TTS endpoint: https://texttospeech.googleapis.com/v1/text:synthesize
|
||||
"""
|
||||
if self._is_gemini_tts_model(model):
|
||||
return self._get_gemini_tts_url(
|
||||
model=model, api_base=api_base, litellm_params=_JSON_OBJECT.validate_python(litellm_params)
|
||||
)
|
||||
|
||||
if api_base:
|
||||
return api_base
|
||||
|
||||
return self.TTS_API_URL
|
||||
|
||||
def _get_gemini_tts_url(self, model: str, api_base: str | None, litellm_params: Mapping[str, object]) -> str:
|
||||
if api_base:
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} serves mp3, opus, alaw and mulaw through Cloud Text-to-Speech, which does not use "
|
||||
"this deployment's api_base. Request wav or pcm, or use a deployment without api_base."
|
||||
),
|
||||
model=model,
|
||||
llm_provider="vertex_ai",
|
||||
)
|
||||
location: Final = litellm_params.get("vertex_location")
|
||||
vertex_location: Final = location if isinstance(location, str) else self.get_default_vertex_location()
|
||||
if vertex_location == "global":
|
||||
return self.TTS_API_URL
|
||||
cloud_tts_location: Final = _GEMINI_CLOUD_TTS_LOCATIONS.get(vertex_location)
|
||||
if cloud_tts_location is None:
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} serves mp3, opus, alaw and mulaw through Cloud Text-to-Speech, which has no Gemini TTS "
|
||||
f"location for vertex_location {vertex_location}. Use a deployment in global, us, eu, "
|
||||
"northamerica-northeast1, or a US or EU Vertex AI region."
|
||||
),
|
||||
model=model,
|
||||
llm_provider="vertex_ai",
|
||||
)
|
||||
return f"https://{cloud_tts_location}-texttospeech.googleapis.com/v1/text:synthesize"
|
||||
|
||||
def _validate_vertex_input(
|
||||
self,
|
||||
input_data: VertexTextToSpeechInput,
|
||||
optional_params: dict,
|
||||
detect_ssml: bool,
|
||||
) -> VertexTextToSpeechInput:
|
||||
"""
|
||||
Validate and transform input for Vertex AI TTS
|
||||
|
|
@ -343,7 +789,7 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
|
|||
else:
|
||||
# LiteLLM will auto-detect if text is in ssml format
|
||||
# check if "text" is an ssml - in this case we should pass it as ssml instead of text
|
||||
if input_data:
|
||||
if detect_ssml and input_data:
|
||||
_text: Final = input_data.get("text", None) or ""
|
||||
if "<speak>" in _text:
|
||||
input_data["ssml"] = input_data.pop("text")
|
||||
|
|
@ -403,8 +849,13 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
|
|||
headers["x-goog-user-project"] = vertex_project
|
||||
|
||||
####### Build the request ################
|
||||
vertex_input = VertexTextToSpeechInput(text=input)
|
||||
vertex_input = self._validate_vertex_input(vertex_input, optional_params)
|
||||
prompt: Final = _JSON_OBJECT.validate_python(optional_params).get("prompt")
|
||||
validated_input: Final = self._validate_vertex_input(
|
||||
VertexTextToSpeechInput(text=input), optional_params, detect_ssml=not self._is_gemini_tts_model(model)
|
||||
)
|
||||
vertex_input: Final[VertexTextToSpeechInput] = (
|
||||
{**validated_input, "prompt": prompt} if isinstance(prompt, str) else validated_input
|
||||
)
|
||||
|
||||
# Build voice configuration
|
||||
# Check for voice dict stored in:
|
||||
|
|
@ -412,8 +863,8 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
|
|||
# 2. optional_params by map_openai_params
|
||||
voice_dict: Final = litellm_params.get("vertex_voice_dict") or optional_params.get("vertex_voice_dict")
|
||||
if voice_dict is not None and isinstance(voice_dict, dict):
|
||||
vertex_voice = VertexTextToSpeechVoice(**voice_dict)
|
||||
elif voice is not None and isinstance(voice, str):
|
||||
vertex_voice = _JSON_OBJECT.validate_python(voice_dict)
|
||||
elif voice is not None:
|
||||
# Handle string voice (shouldn't normally happen if dispatch was called)
|
||||
parts: Final = voice.split("-")
|
||||
if len(parts) >= 2:
|
||||
|
|
@ -485,8 +936,44 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
|
|||
content=binary_data,
|
||||
)
|
||||
|
||||
# Initialize the HttpxBinaryResponseContent instance
|
||||
return HttpxBinaryResponseContent(response)
|
||||
binary_response: Final = HttpxBinaryResponseContent(response)
|
||||
if self._is_gemini_tts_model(model):
|
||||
from litellm.types.utils import CompletionTokensDetailsWrapper, Usage
|
||||
|
||||
request: Final = _GeminiCloudTtsRequest.model_validate_json(raw_response.request.content)
|
||||
input_data: Final = request.input
|
||||
audio_config: Final = request.audioConfig
|
||||
container_duration: Final = calculate_request_duration(binary_data)
|
||||
sample_rate: Final = audio_config.sampleRateHertz or 24000
|
||||
duration: Final = (
|
||||
container_duration
|
||||
if container_duration is not None
|
||||
else _fallback_gemini_tts_audio_duration(binary_data, audio_config.audioEncoding, sample_rate)
|
||||
)
|
||||
if duration is None:
|
||||
verbose_logger.warning(
|
||||
"Could not measure Gemini TTS audio duration for %s, so its audio tokens are not billed", model
|
||||
)
|
||||
return binary_response
|
||||
input_text: Final = " ".join(
|
||||
value for value in (input_data.text, input_data.ssml, input_data.prompt) if value
|
||||
)
|
||||
prompt_tokens: Final = litellm.token_counter(model=model, text=input_text)
|
||||
audio_tokens: Final = math.ceil(duration * 25)
|
||||
binary_response.usage = Usage(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=audio_tokens,
|
||||
total_tokens=prompt_tokens + audio_tokens,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=audio_tokens),
|
||||
)
|
||||
return binary_response
|
||||
|
||||
|
||||
class VertexAIGeminiCloudTextToSpeechConfig(VertexAITextToSpeechConfig):
|
||||
def get_error_class(self, error_message: str, status_code: int, headers: dict[str, str]) -> BaseLLMException:
|
||||
return super().get_error_class(
|
||||
error_message=error_message, status_code=400 if status_code == 404 else status_code, headers=headers
|
||||
)
|
||||
|
||||
|
||||
class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig):
|
||||
|
|
|
|||
|
|
@ -8846,8 +8846,11 @@ def speech(
|
|||
|
||||
generic_optional_params: Final = GenericLiteLLMParams.model_validate(kwargs)
|
||||
|
||||
# Handle Gemini models separately (they use speech_to_completion_bridge)
|
||||
if "gemini" in model:
|
||||
needs_cloud_text_to_speech: Final = (
|
||||
litellm.utils.is_gemini_tts_model(model)
|
||||
and response_format in VertexAITextToSpeechConfig.GEMINI_FORMAT_MAPPINGS
|
||||
)
|
||||
if "gemini" in model and not needs_cloud_text_to_speech:
|
||||
from .endpoints.speech.speech_to_completion_bridge.handler import (
|
||||
speech_to_completion_bridge_handler,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -16580,6 +16580,24 @@
|
|||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/",
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/clairic": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "cloudflare",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "extraction",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/models/clairic/",
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/@cf/cloudflare/clairic": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "cloudflare",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "extraction",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/models/clairic/",
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/@cf/meta/llama-2-7b-chat-fp16": {
|
||||
"input_cost_per_token": 1.923e-06,
|
||||
"litellm_provider": "cloudflare",
|
||||
|
|
@ -21903,6 +21921,7 @@
|
|||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -21927,6 +21946,7 @@
|
|||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -21958,6 +21978,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -22021,6 +22042,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -29417,6 +29439,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -30139,6 +30162,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -30200,6 +30224,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -30268,6 +30293,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -30317,6 +30343,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": false,
|
||||
"supports_url_context": false,
|
||||
|
|
@ -31743,6 +31770,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -31927,6 +31955,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -31989,6 +32018,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -32051,6 +32081,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -32420,6 +32451,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -32481,6 +32513,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -32549,6 +32582,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -32598,6 +32632,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": false,
|
||||
"supports_url_context": false,
|
||||
|
|
@ -33192,6 +33227,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_vision": true
|
||||
},
|
||||
"github_copilot/gemini-3.8-flash": {
|
||||
|
|
@ -33210,6 +33246,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_vision": true
|
||||
},
|
||||
"github_copilot/gpt-3.5-turbo": {
|
||||
|
|
@ -52605,6 +52642,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -65823,6 +65861,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_vision": true,
|
||||
"source": "https://deepinfra.com/pricing",
|
||||
"supports_audio_input": true
|
||||
|
|
@ -69264,6 +69303,7 @@
|
|||
"perplexity/google/gemini-3.5-flash-lite": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_sampling_params": false,
|
||||
"supports_web_search": true,
|
||||
"supports_function_calling": true,
|
||||
"input_cost_per_token": 3e-07,
|
||||
|
|
@ -69274,6 +69314,7 @@
|
|||
"perplexity/google/gemini-3.6-flash": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_sampling_params": false,
|
||||
"supports_web_search": true,
|
||||
"supports_function_calling": true,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
|
|
@ -69284,6 +69325,7 @@
|
|||
"perplexity/google/gemini-3.7-flash": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_sampling_params": false,
|
||||
"supports_web_search": true,
|
||||
"supports_function_calling": true,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
|
|
@ -71375,6 +71417,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true
|
||||
},
|
||||
|
|
@ -71392,6 +71435,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true
|
||||
},
|
||||
|
|
@ -74322,6 +74366,7 @@
|
|||
"perplexity/google/gemini-3.8-flash": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_sampling_params": false,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
"output_cost_per_token": 3.75e-06,
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
|
|
@ -75064,12 +75109,10 @@
|
|||
"gemini-nano-banana-2.1": {
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"cache_read_input_token_cost_batches": 7.5e-08,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_read_input_token_cost_priority": 2.7e-07,
|
||||
"input_cost_per_image": 0.00168,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
"input_cost_per_token_batches": 7.5e-07,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
"input_cost_per_token_priority": 2.7e-06,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -75080,7 +75123,6 @@
|
|||
"output_cost_per_image_token": 3e-05,
|
||||
"output_cost_per_token": 7.5e-06,
|
||||
"output_cost_per_token_batches": 3.75e-06,
|
||||
"output_cost_per_token_flex": 3.75e-06,
|
||||
"output_cost_per_token_priority": 1.35e-05,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
|
|
@ -75113,12 +75155,10 @@
|
|||
"vertex_ai/gemini-nano-banana-2.1": {
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"cache_read_input_token_cost_batches": 7.5e-08,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_read_input_token_cost_priority": 2.7e-07,
|
||||
"input_cost_per_image": 0.00168,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
"input_cost_per_token_batches": 7.5e-07,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
"input_cost_per_token_priority": 2.7e-06,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -75129,7 +75169,6 @@
|
|||
"output_cost_per_image_token": 3e-05,
|
||||
"output_cost_per_token": 7.5e-06,
|
||||
"output_cost_per_token_batches": 3.75e-06,
|
||||
"output_cost_per_token_flex": 3.75e-06,
|
||||
"output_cost_per_token_priority": 1.35e-05,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
|
|
@ -76929,5 +76968,83 @@
|
|||
"supported_output_modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash-1017-us": {
|
||||
"cache_read_input_token_cost": 9e-09,
|
||||
"cache_read_input_token_cost_priority": 1.125e-08,
|
||||
"input_cost_per_token": 4.5e-07,
|
||||
"input_cost_per_token_priority": 5.625e-07,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 393216,
|
||||
"max_tokens": 393216,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.8e-06,
|
||||
"output_cost_per_token_priority": 2.25e-06,
|
||||
"source": "https://docs.fireworks.ai/serverless/us-only-serverless",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"wandb/reflection/Beam-1.0-501B-A23B": {
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "wandb",
|
||||
"max_input_tokens": 1049000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-06,
|
||||
"source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"wandb/zai-org/GLM-5.3": {
|
||||
"cache_read_input_token_cost": 1.9e-07,
|
||||
"input_cost_per_token": 1.2e-06,
|
||||
"litellm_provider": "wandb",
|
||||
"max_input_tokens": 1049000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4e-06,
|
||||
"source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"azure_ai/MAI-Code-1.1-Flash": {
|
||||
"cache_read_input_token_cost": 2e-08,
|
||||
"deprecation_date": "2026-12-09",
|
||||
"input_cost_per_token": 2e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 256000,
|
||||
"max_output_tokens": 64000,
|
||||
"max_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/responses"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4361,6 +4361,7 @@ class MCPServerManager:
|
|||
raw_headers: Mapping[str, str] | None = None,
|
||||
client_ip: str | None = None,
|
||||
protocol_version_override: MCPUpstreamProtocol | None = None,
|
||||
for_tool_listing: bool = False,
|
||||
) -> MCPClient:
|
||||
self.catalog.assert_current(server)
|
||||
record_auth_resolution(server.server_id, AuthResolution.unresolved)
|
||||
|
|
@ -4394,6 +4395,7 @@ class MCPServerManager:
|
|||
else None
|
||||
),
|
||||
upstream_transport=self._upstream_transport,
|
||||
for_tool_listing=for_tool_listing,
|
||||
)
|
||||
|
||||
_create_mcp_client = create_mcp_client
|
||||
|
|
@ -4540,6 +4542,7 @@ class MCPServerManager:
|
|||
user_api_key_auth=user_api_key_auth,
|
||||
raw_headers=raw_headers,
|
||||
client_ip=client_ip,
|
||||
for_tool_listing=True,
|
||||
)
|
||||
|
||||
## HANDLE OPENAPI TOOLS
|
||||
|
|
|
|||
|
|
@ -460,6 +460,8 @@ def validate_static_credential(
|
|||
headers: Mapping[str, str],
|
||||
upstream_token_header: str | None = None,
|
||||
static_header_names: Iterable[str] = (),
|
||||
*,
|
||||
allow_absent: bool = False,
|
||||
) -> Result[None, CredError]:
|
||||
if auth_type not in _STATIC_MODES:
|
||||
return Ok(None)
|
||||
|
|
@ -475,17 +477,23 @@ def validate_static_credential(
|
|||
)
|
||||
)
|
||||
values: Final = tuple((name.lower(), value.strip()) for name, value in headers.items() if name.lower() in slots)
|
||||
if allow_absent and not values:
|
||||
return Ok(None)
|
||||
if any(_usable_credential_value(auth_type, name, value) for name, value in values):
|
||||
return Ok(None)
|
||||
return Error(CredError.of_misconfigured(f"{auth_type} requires a usable upstream credential"))
|
||||
|
||||
|
||||
async def prepare_mcp_client(server: MCPServer, client: MCPClient) -> MCPClient:
|
||||
async def prepare_mcp_client(server: MCPServer, client: MCPClient, *, for_tool_listing: bool = False) -> MCPClient:
|
||||
if server.auth_type not in _STATIC_MODES or client.transport_type == MCPTransport.stdio:
|
||||
return client
|
||||
request: Final = await client.prepare_request_auth()
|
||||
match validate_static_credential(
|
||||
server.auth_type, request.headers, server.upstream_token_header, server.static_headers or ()
|
||||
server.auth_type,
|
||||
request.headers,
|
||||
server.upstream_token_header,
|
||||
server.static_headers or (),
|
||||
allow_absent=for_tool_listing and server.auth_type == MCPAuth.authorization,
|
||||
):
|
||||
case Error(error):
|
||||
raise_public(error)
|
||||
|
|
|
|||
|
|
@ -208,6 +208,7 @@ async def prepare_upstream_client(
|
|||
sampling_callback: SamplingCallback | None = None,
|
||||
elicitation_callback: ElicitationCallback | None = None,
|
||||
upstream_transport: httpx2.AsyncBaseTransport | None = None,
|
||||
for_tool_listing: bool = False,
|
||||
) -> MCPClient:
|
||||
transport: Final[MCPTransportType] = server.transport or MCPTransport.sse
|
||||
server_spec: Final = None if transport == MCPTransport.stdio else to_server_spec_fail_closed(server)
|
||||
|
|
@ -272,6 +273,7 @@ async def prepare_upstream_client(
|
|||
sampling_callback=sampling_callback,
|
||||
elicitation_callback=elicitation_callback,
|
||||
),
|
||||
for_tool_listing=for_tool_listing,
|
||||
)
|
||||
aws_auth: Final = (
|
||||
MCPSigV4Auth(
|
||||
|
|
@ -316,4 +318,5 @@ async def prepare_upstream_client(
|
|||
sampling_callback=sampling_callback,
|
||||
elicitation_callback=elicitation_callback,
|
||||
),
|
||||
for_tool_listing=for_tool_listing,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -275,7 +275,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
|
|||
LazyFeature(
|
||||
name="decisions",
|
||||
module_path="litellm.proxy.decisions_endpoints.endpoints",
|
||||
path_prefixes=("/v1/decisions", "/decisions", "/v1/systemone", "/systemone"),
|
||||
path_prefixes=("/v1/decisions", "/decisions", "/v1/systemone", "/systemone", "/v1/extraction", "/extraction"),
|
||||
),
|
||||
LazyFeature(
|
||||
name="claude_code_marketplace",
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -377,6 +377,8 @@ class LiteLLMRoutes(enum.Enum):
|
|||
"/v1/decisions",
|
||||
"/systemone",
|
||||
"/v1/systemone",
|
||||
"/extraction",
|
||||
"/v1/extraction",
|
||||
# OCR
|
||||
"/ocr",
|
||||
"/v1/ocr",
|
||||
|
|
@ -3729,6 +3731,7 @@ class SpendLogsMetadata(TypedDict):
|
|||
routing_decision: StandardLoggingRoutingDecision | None
|
||||
kubernetes_pod_routing: ReadOnly[StandardLoggingKubernetesPodRouting | None]
|
||||
internal_call_origin: InternalCallOrigin | None
|
||||
billed_retry_attempt: ReadOnly[NotRequired[bool | None]]
|
||||
litellm_roi_estimator: ReadOnly[NotRequired[bool | None]]
|
||||
guardrail_information: list[StandardLoggingGuardrailInformation] | None
|
||||
eval_information: Any | None
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ _MANAGED_MODEL_ROUTES: Final = frozenset(
|
|||
"rerank",
|
||||
"decisions",
|
||||
"systemone",
|
||||
"extraction",
|
||||
"ocr",
|
||||
),
|
||||
)
|
||||
|
|
@ -77,6 +78,7 @@ _MODEL_ROUTE_KINDS: Final[
|
|||
"/rerank": "body",
|
||||
"/decisions": "body",
|
||||
"/systemone": "body",
|
||||
"/extraction": "body",
|
||||
"/messages/count_tokens": "body",
|
||||
":countTokens": "path",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -446,6 +446,7 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = (
|
|||
# the request away from the admin's pinned configuration.
|
||||
"nvcf_function_id",
|
||||
"use_ssl",
|
||||
"riva_offline",
|
||||
# Per-deployment opt-in that hands the whole call to the Rust core. It is a
|
||||
# deployment decision, not a request one: the Rust path uses its own client
|
||||
# rather than the one the deployment configured, and reports no post_call,
|
||||
|
|
|
|||
|
|
@ -494,7 +494,7 @@ Each command resolves your LiteLLM key (logging in via SSO when none is stored a
|
|||
|
||||
The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-<UTF-8 hex of the group name>` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. Codex gets the same list as a catalog file, `$CODEX_HOME/litellm-models.json` (default `~/.codex/`), passed as `-c model_catalog_json=<path>` so `/model` lists exactly the proxy's chat models; a proxy model the installed Codex already knows (`gpt-5.5`, say) keeps that Codex's own entry, reasoning levels and prompt included, and only its place in the picker comes from the proxy, while a model Codex does not know gets the plain entry Codex uses for an unknown `-m` slug. Before launching, `lite codex` asks the installed Codex for its own list and then has it read the written file back (both through `codex debug models`), and when the fetch, either of those or the write fails (Codex releases older than 0.130 have no such command) it says so on stderr and launches with Codex's built-in catalog, leaving a rejected file in place.
|
||||
|
||||
pi ignores base-URL environment variables entirely, so `lite pi` (kept out of the `lite --help` command listing for now, but fully functional) wires it up differently: before handoff it fetches the models your key can use from the proxy's `/v1/models` (plus each model's context window and output cap from `/model_group/info`, when available) and syncs them into a `litellm` provider entry in pi's `~/.pi/agent/models.json` (honoring `PI_CODING_AGENT_DIR`), then starts pi on that provider's first model via an injected `--model litellm/<id>`. Only that one provider entry is rewritten; the rest of the file, including any other custom providers, is left alone. The entry references the key as `$LITELLM_PROXY_API_KEY`, which the wrapper exports for the session, so the token itself never lands on disk and plain `pi` outside the wrapper simply shows the litellm models as unavailable. Your own flags come after the injected pin, so `lite pi --model litellm/<other-id>` wins, and inside the TUI the `/model` picker lists every synced litellm model.
|
||||
pi ignores base-URL environment variables entirely, so `lite pi` (kept out of the `lite --help` command listing for now, but fully functional) wires it up differently: before handoff it fetches the models your key can use from the proxy's `/v1/models` (plus each model's context window and output cap from `/model_group/info`, when available) and syncs them into a `litellm` provider entry in pi's `~/.pi/agent/models.json` (honoring `PI_CODING_AGENT_DIR`), then starts pi on that provider's first model via an injected `--model litellm/<id>`. Only that one provider entry is rewritten; the rest of the file, including any other custom providers, is left alone. The entry references the key as `$LITELLM_PROXY_API_KEY`, which the wrapper exports for the session, so the token itself never lands on disk and plain `pi` outside the wrapper simply shows the litellm models as unavailable. The entry also turns on pi's session header (`x-session-id`, sent by pi 0.80.7 and later), so every chat turn of one pi session shares one session id in the logs and sticks to one deployment when session affinity is on. A custom `pi --session-id` must be at least 8 letters, digits, `_`, or `-` for the proxy to read it. Your own flags come after the injected pin, so `lite pi --model litellm/<other-id>` wins, and inside the TUI the `/model` picker lists every synced litellm model.
|
||||
|
||||
Options (these belong to the wrapper, so put them before the agent's own flags):
|
||||
|
||||
|
|
|
|||
|
|
@ -191,7 +191,7 @@ def provider_block(
|
|||
Real contextWindow/maxTokens matter: pi otherwise assumes 128k/16384, which
|
||||
breaks compaction thresholds and over-asks models with smaller output caps.
|
||||
pi sniffs compat from the base URL, and one gateway URL fronts models with
|
||||
different capabilities, so both flags are pinned off.
|
||||
different capabilities, so the store and long-cache flags are pinned off.
|
||||
"""
|
||||
return {
|
||||
"baseUrl": base_url.rstrip("/") + "/v1",
|
||||
|
|
@ -199,6 +199,8 @@ def provider_block(
|
|||
"compat": {
|
||||
"supportsStore": False,
|
||||
"supportsLongCacheRetention": False,
|
||||
"sendSessionAffinityHeaders": True,
|
||||
"sessionAffinityFormat": "openrouter",
|
||||
},
|
||||
"apiKey": f"${LITELLM_PROXY_API_KEY_ENV}",
|
||||
"models": [_model_entry(model_id, limits) for model_id in model_ids],
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ from litellm.litellm_core_utils.get_supported_openai_params import (
|
|||
get_supported_openai_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
|
||||
|
|
@ -133,12 +134,14 @@ from litellm.router import Router
|
|||
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model
|
||||
from litellm.types.extraction import ExtractionResponse
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.router import RouterRateLimitError
|
||||
from litellm.types.router_weights import validate_router_weights
|
||||
|
||||
_LateResponseT = TypeVar("_LateResponseT", bound=Response)
|
||||
_LlmCallT = TypeVar("_LlmCallT")
|
||||
_ResponseT: Final = TypeVar("_ResponseT")
|
||||
_ChunkT: Final = TypeVar("_ChunkT")
|
||||
|
||||
KNOWN_PROXY_ROUTES: Final = frozenset(
|
||||
|
|
@ -190,6 +193,7 @@ ProxyRouteType: TypeAlias = Literal[
|
|||
"asearch",
|
||||
"adecisions",
|
||||
"asystemone",
|
||||
"aextraction",
|
||||
"avideo_generation",
|
||||
"avideo_list",
|
||||
"avideo_status",
|
||||
|
|
@ -262,6 +266,7 @@ from litellm.proxy.litellm_pre_call_utils import (
|
|||
sanitize_for_log,
|
||||
)
|
||||
from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
|
||||
from litellm.types.utils import (
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
|
|
@ -548,22 +553,29 @@ def model_deployments_support_stream_options(
|
|||
return all(_litellm_model_supports_stream_options(m) for m in candidate_models)
|
||||
|
||||
|
||||
_STREAM_USAGE_FLAGS: Final = ("_litellm_strip_stream_usage", STREAM_USAGE_INJECTED_FIELD)
|
||||
|
||||
|
||||
def stream_usage_tracking_updates(
|
||||
data: Mapping[str, object],
|
||||
general_settings: Mapping[str, object],
|
||||
route_type: str,
|
||||
supports_stream_options: Callable[[], bool],
|
||||
) -> Mapping[str, object]:
|
||||
scrub: Final = {"_litellm_strip_stream_usage": False} if "_litellm_strip_stream_usage" in data else {}
|
||||
scrub: Final = {flag: False for flag in _STREAM_USAGE_FLAGS if flag in data}
|
||||
if data.get("stream", False) is not True:
|
||||
return scrub
|
||||
always_include: Final = general_settings.get("always_include_stream_usage")
|
||||
stream_options: Final = data.get("stream_options")
|
||||
if always_include is True:
|
||||
if "stream_options" not in data:
|
||||
return {**scrub, "stream_options": {"include_usage": True}}
|
||||
return {**scrub, "stream_options": {"include_usage": True}, STREAM_USAGE_INJECTED_FIELD: True}
|
||||
if isinstance(stream_options, dict) and "include_usage" not in stream_options:
|
||||
return {**scrub, "stream_options": {**stream_options, "include_usage": True}}
|
||||
return {
|
||||
**scrub,
|
||||
"stream_options": {**stream_options, "include_usage": True},
|
||||
STREAM_USAGE_INJECTED_FIELD: True,
|
||||
}
|
||||
return scrub
|
||||
if always_include is False and route_type == "agenerate_content_stream" and "stream_options" not in data:
|
||||
return {**scrub, "stream_options": {"include_usage": False}}
|
||||
|
|
@ -577,6 +589,7 @@ def stream_usage_tracking_updates(
|
|||
return {
|
||||
"stream_options": {**merged_stream_options, "include_usage": True},
|
||||
"_litellm_strip_stream_usage": True,
|
||||
STREAM_USAGE_INJECTED_FIELD: True,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1481,11 +1494,11 @@ _is_azure_model_router_request: Final = is_azure_model_router_request
|
|||
|
||||
def _override_openai_response_model(
|
||||
*,
|
||||
response_obj: object,
|
||||
response_obj: _ResponseT,
|
||||
requested_model: str,
|
||||
log_context: str,
|
||||
return_raw_model_name: bool = False,
|
||||
) -> None:
|
||||
) -> _ResponseT:
|
||||
"""
|
||||
Force the OpenAI-compatible `model` field in the response to match what the client requested.
|
||||
|
||||
|
|
@ -1509,7 +1522,7 @@ def _override_openai_response_model(
|
|||
model group name instead of the comma-separated list the client sent.
|
||||
"""
|
||||
if return_raw_model_name or not requested_model:
|
||||
return
|
||||
return response_obj
|
||||
|
||||
hidden_params: Final = get_hidden_params_dict(response_obj)
|
||||
if isinstance(hidden_params, dict):
|
||||
|
|
@ -1522,7 +1535,7 @@ def _override_openai_response_model(
|
|||
log_context,
|
||||
attempted_fallbacks,
|
||||
)
|
||||
return
|
||||
return response_obj
|
||||
|
||||
# For fastest_response batch completions, use the winning model's group
|
||||
# name rather than the comma-separated list the client sent.
|
||||
|
|
@ -1541,7 +1554,7 @@ def _override_openai_response_model(
|
|||
"%s: fastest_response detected but no model group header found, preserving actual model from response.",
|
||||
log_context,
|
||||
)
|
||||
return
|
||||
return response_obj
|
||||
|
||||
# Check if this is an Azure Model Router request - if so, preserve the actual model used
|
||||
if is_azure_model_router_request(requested_model, hidden_params):
|
||||
|
|
@ -1549,11 +1562,12 @@ def _override_openai_response_model(
|
|||
"%s: Azure Model Router detected - preserving actual model used from response instead of overriding to router model.",
|
||||
log_context,
|
||||
)
|
||||
return
|
||||
return response_obj
|
||||
|
||||
unchanged_response: Final = response_obj
|
||||
if isinstance(response_obj, dict):
|
||||
if "model" not in response_obj:
|
||||
return
|
||||
return unchanged_response
|
||||
downstream_model = response_obj.get("model")
|
||||
if downstream_model != requested_model:
|
||||
verbose_proxy_logger.debug(
|
||||
|
|
@ -1563,10 +1577,10 @@ def _override_openai_response_model(
|
|||
downstream_model,
|
||||
)
|
||||
response_obj["model"] = requested_model
|
||||
return
|
||||
return unchanged_response
|
||||
|
||||
if not hasattr(response_obj, "model"):
|
||||
return
|
||||
return response_obj
|
||||
|
||||
downstream_model = getattr(response_obj, "model", None)
|
||||
if downstream_model != requested_model:
|
||||
|
|
@ -1577,6 +1591,9 @@ def _override_openai_response_model(
|
|||
downstream_model,
|
||||
)
|
||||
|
||||
if isinstance(response_obj, ExtractionResponse):
|
||||
return response_obj.model_copy(update={"model": requested_model})
|
||||
|
||||
try:
|
||||
setattr(response_obj, "model", requested_model)
|
||||
except Exception as e:
|
||||
|
|
@ -1588,6 +1605,7 @@ def _override_openai_response_model(
|
|||
str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
return response_obj
|
||||
|
||||
|
||||
_METADATA_BUCKET_KEYS: Final = ("metadata", "litellm_metadata")
|
||||
|
|
@ -2200,6 +2218,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
"asearch",
|
||||
"adecisions",
|
||||
"asystemone",
|
||||
"aextraction",
|
||||
"avideo_generation",
|
||||
"avideo_list",
|
||||
"avideo_status",
|
||||
|
|
@ -3226,7 +3245,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
# Always return the client-requested model name (not provider-prefixed internal identifiers)
|
||||
# for OpenAI-compatible responses.
|
||||
if requested_model_from_client:
|
||||
_override_openai_response_model(
|
||||
response = _override_openai_response_model( # rebind-ok: a frozen response comes back as a copy
|
||||
response_obj=response,
|
||||
requested_model=requested_model_from_client,
|
||||
log_context=f"litellm_call_id={logging_obj.litellm_call_id}",
|
||||
|
|
@ -3978,6 +3997,8 @@ class ProxyBaseLLMRequestProcessing:
|
|||
|
||||
async def _on_deferred_native_stream_complete(*args: object) -> None:
|
||||
match args:
|
||||
case (BilledRoundsDispatch(dispatch=logging_coroutine),):
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine)
|
||||
case (logging_coroutine,) if asyncio.iscoroutine(logging_coroutine):
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine)
|
||||
case (assembled_response, cache_hit):
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from litellm._internal_context import service_target, with_service_target
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching import RedisCache
|
||||
from litellm.constants import (
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY,
|
||||
DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME,
|
||||
DB_SPEND_UPDATE_JOB_NAME,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
|
|
@ -312,6 +313,11 @@ def _spend_tables_left_to_send(
|
|||
return cast(DBSpendUpdateTransactions, remaining) if any(remaining.values()) else None
|
||||
|
||||
|
||||
def _is_billed_retry_attempt(metadata: SpendLogsMetadata) -> bool:
|
||||
"""An earlier billed retry attempt's own spend row, not a request the caller sent."""
|
||||
return BILLED_RETRY_ATTEMPT_METADATA_KEY in metadata and metadata[BILLED_RETRY_ATTEMPT_METADATA_KEY] is True
|
||||
|
||||
|
||||
def _timed_request_duration_ms(
|
||||
payload: dict | SpendLogsPayload,
|
||||
request_status: Literal["success", "failure"],
|
||||
|
|
@ -1651,6 +1657,8 @@ class DBSpendUpdateWriter:
|
|||
window_spend_update_queue=self.window_spend_update_queue,
|
||||
)
|
||||
|
||||
await self._flush_tool_discovery_queue(prisma_client=prisma_client)
|
||||
|
||||
# Only commit from redis to db if this pod is the leader
|
||||
if await self.pod_lock_manager.acquire_lock(
|
||||
cronjob_id=DB_SPEND_UPDATE_JOB_NAME,
|
||||
|
|
@ -2803,11 +2811,12 @@ class DBSpendUpdateWriter:
|
|||
if not date:
|
||||
return
|
||||
metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
|
||||
internal_origin: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults
|
||||
request_error_accumulator.record(
|
||||
payload=payload,
|
||||
request_status=prisma_client.get_request_status(payload), # pyright: ignore[reportUnknownMemberType] # legacy payload union; this caller supplies a typed spend payload
|
||||
date=date.split("T")[0],
|
||||
is_internal_call=bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)), # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults
|
||||
is_internal_call=internal_origin or _is_billed_retry_attempt(metadata),
|
||||
)
|
||||
|
||||
async def _common_add_spend_log_transaction_to_daily_transaction(
|
||||
|
|
@ -2872,6 +2881,7 @@ class DBSpendUpdateWriter:
|
|||
endpoint = ROUTE_ENDPOINT_MAPPING.get(call_type, None)
|
||||
|
||||
is_internal_call: Final = bool(_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY))
|
||||
caller_request: Final = not is_internal_call and not _is_billed_retry_attempt(_metadata)
|
||||
cache_read_input_tokens: Final = extract_cache_read_tokens(usage_obj)
|
||||
compression_saved_tokens: Final = extract_compression_saved_tokens(_metadata)
|
||||
savings_spend: Final = compute_savings_spend(
|
||||
|
|
@ -2906,9 +2916,9 @@ class DBSpendUpdateWriter:
|
|||
# requests the caller made: counting them inflates request-volume
|
||||
# readers, and an auto-router savings figure computed on a shadow
|
||||
# duplicate credits savings for traffic no user sent.
|
||||
api_requests=0 if is_internal_call else 1,
|
||||
successful_requests=1 if not is_internal_call and request_status == "success" else 0,
|
||||
failed_requests=1 if not is_internal_call and request_status != "success" else 0,
|
||||
api_requests=1 if caller_request else 0,
|
||||
successful_requests=1 if caller_request and request_status == "success" else 0,
|
||||
failed_requests=1 if caller_request and request_status != "success" else 0,
|
||||
cache_read_input_tokens=cache_read_input_tokens,
|
||||
cache_creation_input_tokens=extract_cache_creation_tokens(usage_obj),
|
||||
compression_saved_tokens=compression_saved_tokens,
|
||||
|
|
|
|||
|
|
@ -10,8 +10,9 @@ worker on every call. ``SELECT DISTINCT ON`` keeps the transfer at one row per
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from datetime import datetime, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
from pydantic import ConfigDict, JsonValue, TypeAdapter, field_validator
|
||||
|
|
@ -113,3 +114,13 @@ async def fetch_latest_health_checks_for_models(
|
|||
except Exception as query_err: # noqa: BLE001 # a paged model list must not fail on its health decoration
|
||||
verbose_proxy_logger.error("Error getting latest health checks for models: %s", query_err)
|
||||
return ()
|
||||
|
||||
|
||||
def latest_by_model_name(rows: Iterable[LatestHealthCheckRow]) -> Mapping[str, LatestHealthCheckRow]:
|
||||
"""The newest row per model name.
|
||||
|
||||
The queries above return one row per (model_id, model_name), so a model group with several
|
||||
deployments comes back as several rows, plus one without a model_id if it was ever checked by
|
||||
name. They arrive in model_id order with NULL last, which says nothing about which check is newest.
|
||||
"""
|
||||
return MappingProxyType({row.model_name: row for row in sorted(rows, key=lambda row: row.checked_at)})
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Final, Literal
|
||||
from typing import Annotated, Final, Literal, TypeAlias
|
||||
|
||||
from fastapi import APIRouter, Depends, Request, Response
|
||||
from fastapi.responses import ORJSONResponse # pyright: ignore[reportDeprecated] # required endpoint contract
|
||||
|
|
@ -18,6 +18,7 @@ from litellm.types.decisions import (
|
|||
OpenAIDecisionRequestBody,
|
||||
OpenAIDecisionResponse,
|
||||
)
|
||||
from litellm.types.extraction import ExtractionRequest, ExtractionRequestBody, ExtractionResponse
|
||||
|
||||
router: Final = APIRouter()
|
||||
_REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
|
||||
|
|
@ -25,6 +26,7 @@ _DECISIONS_REQUEST_BODY_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = Type
|
|||
_OPENAI_DECISION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(
|
||||
OpenAIDecisionRequestBody
|
||||
)
|
||||
_EXTRACTION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[ExtractionRequestBody]] = TypeAdapter(ExtractionRequestBody)
|
||||
_GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
|
||||
_OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None)
|
||||
_OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None)
|
||||
|
|
@ -73,16 +75,35 @@ _DECISIONS_REQUEST_BODY: Final = inline_request_body(
|
|||
],
|
||||
},
|
||||
)
|
||||
_EXTRACTION_REQUEST_BODY: Final = inline_request_body(
|
||||
ExtractionRequest,
|
||||
{
|
||||
"model": "clairic",
|
||||
"state": "Invoice #4471 from Acme Tools, dated 2026-03-03. 12 hammers. Total $174. Payment due in 30 days.",
|
||||
"fields": {
|
||||
"vendor": {"type": "text", "instructions": "Who sent the invoice?"},
|
||||
"invoice_date": {"type": "date"},
|
||||
"total": {"type": "number", "unit": "USD"},
|
||||
"items": {"type": "list", "items": "text", "instructions": "What was bought?"},
|
||||
"days_to_pay": {"type": "estimate", "min": 0, "max": 90},
|
||||
},
|
||||
},
|
||||
)
|
||||
_ApiName: TypeAlias = Literal["Decisions", "Extraction"]
|
||||
_RouteType: TypeAlias = Literal["adecisions", "asystemone", "aextraction"]
|
||||
|
||||
|
||||
async def _invalid_request(
|
||||
raw_data: Mapping[str, object], error: ValidationError, user_api_key_dict: UserAPIKeyAuth
|
||||
raw_data: Mapping[str, object],
|
||||
error: ValidationError,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
api_name: _ApiName,
|
||||
) -> Exception:
|
||||
from litellm.proxy.proxy_server import proxy_logging_obj, version
|
||||
|
||||
return await ProxyBaseLLMRequestProcessing(data=dict(raw_data)).handle_llm_api_exception(
|
||||
e=BadRequestError(
|
||||
message=f"Invalid Decisions request: {error}",
|
||||
message=f"Invalid {api_name} request: {error}",
|
||||
model=str(raw_data.get("model", "")),
|
||||
llm_provider="",
|
||||
),
|
||||
|
|
@ -95,19 +116,21 @@ async def _invalid_request(
|
|||
_FIELDS_CHECKED_AFTER_ROUTING: Final = frozenset({"safety_identifier", "images", "assets", "audio", "videos"})
|
||||
|
||||
|
||||
def _fields_checked_before_routing(data: Mapping[str, object]) -> Mapping[str, object]:
|
||||
def _fields_checked_before_routing(data: Mapping[str, object], route_type: _RouteType) -> Mapping[str, object]:
|
||||
if route_type == "aextraction":
|
||||
return data
|
||||
return MappingProxyType({key: value for key, value in data.items() if key not in _FIELDS_CHECKED_AFTER_ROUTING})
|
||||
|
||||
|
||||
_DEPLOYMENT_ONLY_FIELDS: Final = frozenset({"model_info"})
|
||||
|
||||
|
||||
async def _request_data(request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]:
|
||||
async def _request_data(request: Request, user_api_key_dict: UserAPIKeyAuth, api_name: _ApiName) -> dict[str, object]:
|
||||
body: Final = await request.body()
|
||||
try:
|
||||
data: Final = _REQUEST_DATA_ADAPTER.validate_json(body)
|
||||
except ValidationError as error:
|
||||
raise await _invalid_request(raw_data={}, error=error, user_api_key_dict=user_api_key_dict)
|
||||
raise await _invalid_request(raw_data={}, error=error, user_api_key_dict=user_api_key_dict, api_name=api_name)
|
||||
return {key: value for key, value in data.items() if key not in _DEPLOYMENT_ONLY_FIELDS}
|
||||
|
||||
|
||||
|
|
@ -115,8 +138,10 @@ async def _process_decisions(
|
|||
request: Request,
|
||||
fastapi_response: Response,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
body_adapter: TypeAdapter[DecisionsRequestBody] | TypeAdapter[OpenAIDecisionRequestBody],
|
||||
route_type: Literal["adecisions", "asystemone"],
|
||||
body_adapter: (
|
||||
TypeAdapter[DecisionsRequestBody] | TypeAdapter[OpenAIDecisionRequestBody] | TypeAdapter[ExtractionRequestBody]
|
||||
),
|
||||
route_type: _RouteType,
|
||||
) -> object:
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings as proxy_general_settings,
|
||||
|
|
@ -139,15 +164,17 @@ async def _process_decisions(
|
|||
user_temperature as proxy_user_temperature,
|
||||
)
|
||||
|
||||
data: Final = await _request_data(request, user_api_key_dict)
|
||||
api_name: Final[_ApiName] = "Extraction" if route_type == "aextraction" else "Decisions"
|
||||
data: Final = await _request_data(request, user_api_key_dict, api_name)
|
||||
try:
|
||||
body_adapter.validate_python(_fields_checked_before_routing(data))
|
||||
body_adapter.validate_python(_fields_checked_before_routing(data, route_type))
|
||||
except ValidationError as error:
|
||||
raise await _invalid_request(raw_data=data, error=error, user_api_key_dict=user_api_key_dict)
|
||||
raise await _invalid_request(raw_data=data, error=error, user_api_key_dict=user_api_key_dict, api_name=api_name)
|
||||
general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(proxy_general_settings)
|
||||
user_api_base: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_api_base)
|
||||
user_model: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_model)
|
||||
user_temperature: Final = _OPTIONAL_FLOAT_ADAPTER.validate_python(proxy_user_temperature)
|
||||
takes_generation_settings: Final = route_type != "aextraction"
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(data=data)
|
||||
try:
|
||||
return await processor.base_process_llm_request(
|
||||
|
|
@ -162,9 +189,9 @@ async def _process_decisions(
|
|||
select_data_generator=None,
|
||||
model=None,
|
||||
user_model=user_model,
|
||||
user_temperature=user_temperature,
|
||||
user_temperature=user_temperature if takes_generation_settings else None,
|
||||
user_request_timeout=user_request_timeout,
|
||||
user_max_tokens=user_max_tokens,
|
||||
user_max_tokens=user_max_tokens if takes_generation_settings else None,
|
||||
user_api_base=user_api_base,
|
||||
version=version,
|
||||
)
|
||||
|
|
@ -263,3 +290,53 @@ async def decisions(
|
|||
body_adapter=_OPENAI_DECISION_REQUEST_BODY_ADAPTER,
|
||||
route_type="adecisions",
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/extraction",
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract
|
||||
tags=["extraction"],
|
||||
summary="Extract typed fields from a state",
|
||||
responses={200: {"model": ExtractionResponse}},
|
||||
response_model=None,
|
||||
openapi_extra={"requestBody": _EXTRACTION_REQUEST_BODY},
|
||||
)
|
||||
@router.post(
|
||||
"/extraction",
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract
|
||||
tags=["extraction"],
|
||||
summary="Extract typed fields from a state",
|
||||
responses={200: {"model": ExtractionResponse}},
|
||||
response_model=None,
|
||||
openapi_extra={"requestBody": _EXTRACTION_REQUEST_BODY},
|
||||
)
|
||||
async def extraction(
|
||||
request: Request,
|
||||
fastapi_response: Response,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
) -> object:
|
||||
"""
|
||||
Read a `state` text and up to 4 images and get one typed answer per named field in `fields`.
|
||||
|
||||
Field types are `text`, `number`, `date`, `time`, `datetime`, `duration`, `list` (set `items`), `estimate`
|
||||
(set `min` and `max`) and `forecast` (set `min`, `max` and `horizon`). Each answer carries its `value`, which is
|
||||
null when the input does not determine it, plus `confidence`, `p_null`, text `span`s into the state and, for
|
||||
estimates, `mean`, `median`, `p10` and `p90`. Images are data URLs or `{content_type, base64}` objects, remote
|
||||
URLs are not accepted. An unknown field type, a field name that is not 1 to 64 letters, digits, `_`, `.` or `-`
|
||||
starting with a letter or `_`, a remote image URL or more than 4 images is rejected with a 400 before any
|
||||
provider is called. Any other top-level key, `stream: true` and OpenAI generation settings such as `temperature`
|
||||
included, is rejected with a 400 naming it unless `drop_params` or the deployment's `additional_drop_params`
|
||||
drops it. Settings inside `fields` and `options` go to the provider as sent, so a field missing a setting its
|
||||
type needs or an empty `state` with no images comes back with the provider's own error. `model` is any
|
||||
extraction model in the proxy model list, such as Cloudflare Clairic. The request and the answers keep the
|
||||
provider's format.
|
||||
"""
|
||||
return await _process_decisions(
|
||||
request=request,
|
||||
fastapi_response=fastapi_response,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
body_adapter=_EXTRACTION_REQUEST_BODY_ADAPTER,
|
||||
route_type="aextraction",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -494,7 +494,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail):
|
|||
sse: Final = AnthropicSSERestorer(step)
|
||||
events: Final = ResponsesStreamRestorer(step, rehydrate)
|
||||
carries: Final[CarryWindows] = {}
|
||||
last_chunk = None # rebind-ok: tracks the most recent chunk for the final flush.
|
||||
last_chunk = None # rebind-ok: tracks the most recent chunk with choices for the final flush.
|
||||
|
||||
async for chunk in response:
|
||||
if isinstance(chunk, (bytes, str)):
|
||||
|
|
@ -506,7 +506,8 @@ class LLMShieldProxyGuardrail(CustomGuardrail):
|
|||
yield event
|
||||
continue
|
||||
restored_chunk = detached(chunk)
|
||||
last_chunk = restored_chunk
|
||||
if getattr(restored_chunk, "choices", None):
|
||||
last_chunk = restored_chunk
|
||||
for choice in attribute_items(restored_chunk, "choices"):
|
||||
await self._restore_choice(choice, carries, session_id)
|
||||
yield restored_chunk
|
||||
|
|
|
|||
|
|
@ -2155,6 +2155,7 @@ async def test_model_connection(
|
|||
"ocr",
|
||||
"evaluation",
|
||||
"decisions",
|
||||
"extraction",
|
||||
]
|
||||
| None = fastapi.Body(
|
||||
None,
|
||||
|
|
|
|||
|
|
@ -254,39 +254,45 @@ class KeyManagementEventHooks:
|
|||
litellm_changed_by: str | None = None,
|
||||
) -> None:
|
||||
from litellm.proxy.management_helpers.audit_logs import (
|
||||
create_audit_log_for_update,
|
||||
create_audit_logs_for_updates,
|
||||
get_audit_log_changed_by,
|
||||
is_audit_logging_enabled,
|
||||
track_audit_task,
|
||||
)
|
||||
from litellm.proxy.proxy_server import litellm_proxy_admin_name
|
||||
|
||||
if not is_audit_logging_enabled():
|
||||
return
|
||||
for key in keys_being_deleted:
|
||||
key_row = LiteLLM_VerificationToken.model_validate(key, from_attributes=True)
|
||||
if key_row.token is None:
|
||||
continue
|
||||
_key_row = key_row.model_dump_json(exclude_none=True)
|
||||
|
||||
asyncio.create_task(
|
||||
create_audit_log_for_update(
|
||||
request_data=LiteLLM_AuditLogs(
|
||||
id=str(uuid.uuid4()),
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
changed_by=get_audit_log_changed_by(
|
||||
litellm_changed_by=litellm_changed_by,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
litellm_proxy_admin_name=litellm_proxy_admin_name,
|
||||
),
|
||||
changed_by_api_key=user_api_key_dict.token,
|
||||
table_name=LitellmTableNames.KEY_TABLE_NAME,
|
||||
object_id=key_row.token,
|
||||
action="deleted",
|
||||
updated_values="{}",
|
||||
before_value=_key_row,
|
||||
)
|
||||
)
|
||||
changed_by: Final = get_audit_log_changed_by(
|
||||
litellm_changed_by=litellm_changed_by,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
litellm_proxy_admin_name=litellm_proxy_admin_name,
|
||||
)
|
||||
key_rows: Final = (
|
||||
LiteLLM_VerificationToken.model_validate(key, from_attributes=True) for key in keys_being_deleted
|
||||
)
|
||||
audit_rows: Final = tuple(
|
||||
LiteLLM_AuditLogs(
|
||||
id=str(uuid.uuid4()),
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
changed_by=changed_by,
|
||||
changed_by_api_key=user_api_key_dict.token,
|
||||
table_name=LitellmTableNames.KEY_TABLE_NAME,
|
||||
object_id=key_row.token,
|
||||
action="deleted",
|
||||
updated_values="{}",
|
||||
before_value=key_row.model_dump_json(exclude_none=True),
|
||||
)
|
||||
for key_row in key_rows
|
||||
if key_row.token is not None
|
||||
)
|
||||
if not audit_rows:
|
||||
return
|
||||
track_audit_task(
|
||||
asyncio.create_task(
|
||||
create_audit_logs_for_updates(audit_rows), name=f"{len(audit_rows)} deleted key audit logs"
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _store_virtual_key_in_secret_manager(secret_name: str, secret_token: str, team_id: str | None = None):
|
||||
|
|
|
|||
|
|
@ -374,6 +374,8 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
|||
self._redis_increment_queue_lock = asyncio.Lock()
|
||||
self._redis_increment_flush_lock = asyncio.Lock()
|
||||
self._detached_increment_operations = None
|
||||
self._retired = False
|
||||
self._flush_after_retire = None
|
||||
self.deployment_budget_config = None
|
||||
|
||||
@with_service_target("model_budgets")
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ This is currently in development and not yet ready for production.
|
|||
import asyncio
|
||||
import binascii
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
|
|
@ -42,6 +43,7 @@ from litellm.caching.redis_batch import (
|
|||
)
|
||||
from litellm.caching.redis_cache import log_redis_failure
|
||||
from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY
|
||||
from litellm.cost_calculator import get_usage_object
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_str_from_messages,
|
||||
|
|
@ -78,6 +80,7 @@ from litellm.router_utils.add_retry_fallback_headers import (
|
|||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.router_utils.ptu_shares import PTUTeamCeiling
|
||||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
from litellm.types.extraction import ExtractionResponse
|
||||
from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage
|
||||
from litellm.types.mcp_server.mcp_server_manager import MCPServer
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
|
||||
|
|
@ -490,6 +493,7 @@ RESPONSES_API_CALL_TYPES: Final = ("aresponses", "responses")
|
|||
EMBEDDING_API_CALL_TYPES: Final = ("aembedding", "embedding")
|
||||
TEXT_COMPLETION_API_CALL_TYPES: Final = ("atext_completion", "text_completion")
|
||||
RERANK_API_CALL_TYPES: Final = (CallTypes.rerank.value, CallTypes.arerank.value)
|
||||
EXTRACTION_API_CALL_TYPES: Final = (CallTypes.extraction.value, CallTypes.aextraction.value)
|
||||
GOOGLE_GENAI_NATIVE_CALL_TYPES: Final = (
|
||||
CallTypes.generate_content.value,
|
||||
CallTypes.agenerate_content.value,
|
||||
|
|
@ -691,6 +695,7 @@ class RequestRateLimiterStash:
|
|||
owner_litellm_call_id: str | None = None
|
||||
rate_limit_response: RateLimitResponse | None = None
|
||||
parallel_slot: ParallelSlotAcquisition | None = None
|
||||
parallel_slot_reused_at: datetime | None = None
|
||||
parallel_slot_release_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False, compare=False)
|
||||
reserved_tokens: int = 0
|
||||
reserved_model: RateLimitedModel | None = None
|
||||
|
|
@ -787,6 +792,19 @@ def claim_request_stash_for_data(data: dict) -> RequestRateLimiterStash:
|
|||
return stash
|
||||
|
||||
|
||||
def _ended_before_slot_reuse(ended_at: datetime | None, reused_at: datetime | None) -> bool:
|
||||
if ended_at is None or reused_at is None or (ended_at.tzinfo is None) != (reused_at.tzinfo is None):
|
||||
return False
|
||||
return ended_at <= reused_at
|
||||
|
||||
|
||||
def _callback_kwargs_mapping(kwargs: object) -> Mapping[str, object]:
|
||||
try:
|
||||
return _REQUEST_RATE_LIMIT_DATA.validate_python(kwargs)
|
||||
except ValidationError:
|
||||
return MappingProxyType({})
|
||||
|
||||
|
||||
def get_request_stash_for_call(litellm_call_id: str | None) -> RequestRateLimiterStash | None:
|
||||
stash: Final = _request_stash.get()
|
||||
if stash is None:
|
||||
|
|
@ -1094,6 +1112,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
or PROXY_MaxParallelRequestsHandler_v3._has_explicit_output_cap(data, call_type)
|
||||
or is_embedding
|
||||
or endpoint_type == EndpointType.DECISIONS
|
||||
or call_type in EXTRACTION_API_CALL_TYPES
|
||||
):
|
||||
return
|
||||
effective_cap: Final = max(capped_floor, configured_output_tokens or 0)
|
||||
|
|
@ -1218,6 +1237,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
if call_type in RESPONSES_API_CALL_TYPES or call_type in EMBEDDING_API_CALL_TYPES
|
||||
else (None, estimable_data.get("prompt"), None)
|
||||
if call_type in TEXT_COMPLETION_API_CALL_TYPES
|
||||
else (None, self._extraction_input_text(_REQUEST_RATE_LIMIT_DATA.validate_python(estimable_data)), None)
|
||||
if call_type in EXTRACTION_API_CALL_TYPES
|
||||
else (estimable_data.get("messages"), None, None)
|
||||
if call_type
|
||||
else (
|
||||
|
|
@ -1245,7 +1266,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
estimated_input_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0
|
||||
|
||||
explicit_max_tokens: Final = self._get_explicit_output_cap(data, call_type)
|
||||
is_embedding: Final = self._is_embedding_request(data, call_type)
|
||||
generates_no_output: Final = (
|
||||
self._is_embedding_request(data, call_type) or call_type in EXTRACTION_API_CALL_TYPES
|
||||
)
|
||||
|
||||
base_output_floor: Final = self.no_max_tokens_output_floor(min_configured_tpm_limit)
|
||||
output_floor: Final = (
|
||||
|
|
@ -1255,7 +1278,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
max_tokens_estimate: Final = (
|
||||
0
|
||||
if is_embedding or (explicit_max_tokens is None and total_chars == 0 and configured_output_tokens is None)
|
||||
if generates_no_output
|
||||
or (explicit_max_tokens is None and total_chars == 0 and configured_output_tokens is None)
|
||||
else explicit_max_tokens
|
||||
if explicit_max_tokens is not None
|
||||
else configured_output_tokens
|
||||
|
|
@ -1642,6 +1666,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
read_only: bool = False,
|
||||
skip_tpm_check: bool = False,
|
||||
parallel_slot_id: str | None = None,
|
||||
skip_parallel_check: bool = False,
|
||||
) -> RateLimitResponse:
|
||||
"""
|
||||
Check if any of the rate limit descriptors should be rate limited.
|
||||
|
|
@ -1658,6 +1683,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
``reserve_tpm_tokens`` reservation path should set this to
|
||||
avoid the +1-per-key Lua / in-memory increment double-charging
|
||||
the tokens counter.
|
||||
skip_parallel_check: If True, ``max_parallel_requests`` gauges are
|
||||
neither checked nor acquired, for a call that runs inside a
|
||||
request already holding its slot.
|
||||
|
||||
``max_parallel_requests`` descriptors are enforced by the dedicated
|
||||
concurrency-gauge path (``_check_parallel_request_gauges``), never by
|
||||
|
|
@ -1750,7 +1778,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
if windowed_response["overall_code"] == "OVER_LIMIT":
|
||||
return windowed_response
|
||||
|
||||
if not gauges:
|
||||
if not gauges or skip_parallel_check:
|
||||
return windowed_response
|
||||
|
||||
gauge_response: Final = await self._check_parallel_request_gauges(
|
||||
|
|
@ -2013,12 +2041,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
parent_otel_span: Span | None,
|
||||
*,
|
||||
in_logging_callback: bool = False,
|
||||
round_ended_at: datetime | None = None,
|
||||
) -> None:
|
||||
if stash is None:
|
||||
return
|
||||
async with stash.parallel_slot_release_lock:
|
||||
acquisition: Final = stash.parallel_slot
|
||||
if acquisition is None:
|
||||
if acquisition is None or _ended_before_slot_reuse(round_ended_at, stash.parallel_slot_reused_at):
|
||||
return
|
||||
deferred: Final = in_logging_callback and await self._defer_parallel_slot_release(
|
||||
acquisition, parent_otel_span
|
||||
|
|
@ -3834,6 +3863,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
if isinstance(part, (str, dict))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extraction_input_text(data: Mapping[str, object]) -> str:
|
||||
state: Final = data.get("state")
|
||||
return f"{state if isinstance(state, str) else ''}\n{json.dumps(data.get('fields'), default=str)}"
|
||||
|
||||
def _estimate_precise_input_tokens(self, data: object, model: str | None, call_type: str | None = None) -> int:
|
||||
"""
|
||||
Model-aware input token estimate for the project ITPM reservation,
|
||||
|
|
@ -3892,6 +3926,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
if call_type in RERANK_API_CALL_TYPES
|
||||
else (None, prompt, data.get("tools"), data.get("tool_choice"))
|
||||
if call_type in TEXT_COMPLETION_API_CALL_TYPES
|
||||
else (None, self._extraction_input_text(_REQUEST_RATE_LIMIT_DATA.validate_python(data)), None, None)
|
||||
if call_type in EXTRACTION_API_CALL_TYPES
|
||||
else (data.get("messages"), fallback_text, data.get("tools"), data.get("tool_choice"))
|
||||
)
|
||||
messages, selected_text, countable_tools, countable_tool_choice = selected_inputs
|
||||
|
|
@ -3930,6 +3966,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
tpm_reservation_amount: int,
|
||||
call_type: str | None = None,
|
||||
endpoint_type: EndpointType = EndpointType.GENERIC,
|
||||
reuses_request_slot: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Reserve project-scoped ITPM/OTPM tokens (Bedrock Mantle-style
|
||||
|
|
@ -4009,7 +4046,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
)
|
||||
stash.reservation_released = True
|
||||
await self._release_stashed_parallel_slot(stash, user_api_key_dict.parent_otel_span)
|
||||
if not reuses_request_slot:
|
||||
await self._release_stashed_parallel_slot(stash, user_api_key_dict.parent_otel_span)
|
||||
self._handle_rate_limit_error(
|
||||
response=io_response,
|
||||
descriptors=descriptors,
|
||||
|
|
@ -4170,6 +4208,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
verbose_proxy_logger.debug("Inside Rate Limit Pre-Call Hook")
|
||||
|
||||
stash: Final = claim_request_stash_for_data(data)
|
||||
reuses_request_slot: Final = call_type == CallTypes.call_mcp_tool.value and stash.parallel_slot is not None
|
||||
if reuses_request_slot:
|
||||
stash.parallel_slot_reused_at = self._get_current_time()
|
||||
|
||||
#########################################################
|
||||
# Check if the call type has a specific rate limiter
|
||||
|
|
@ -4214,11 +4255,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
# then because _reserve_project_io_tokens_or_raise below charges
|
||||
# them unconditionally and counting them here too would
|
||||
# double-charge every request.
|
||||
parallel_counter_keys: Final = [
|
||||
self.create_rate_limit_keys(d["key"], d["value"], "max_parallel_requests")
|
||||
for d in descriptors
|
||||
if (d.get("rate_limit") or {}).get("max_parallel_requests") is not None
|
||||
]
|
||||
parallel_counter_keys: Final = (
|
||||
[]
|
||||
if reuses_request_slot
|
||||
else [
|
||||
self.create_rate_limit_keys(d["key"], d["value"], "max_parallel_requests")
|
||||
for d in descriptors
|
||||
if (d.get("rate_limit") or {}).get("max_parallel_requests") is not None
|
||||
]
|
||||
)
|
||||
parallel_slot_id: Final = uuid.uuid4().hex if parallel_counter_keys else None
|
||||
|
||||
first_pass_descriptors: Final = (
|
||||
|
|
@ -4233,6 +4278,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
skip_tpm_check=self.tpm_reservation_enabled,
|
||||
parallel_slot_id=parallel_slot_id,
|
||||
skip_parallel_check=reuses_request_slot,
|
||||
)
|
||||
|
||||
if response["overall_code"] == "OVER_LIMIT":
|
||||
|
|
@ -4242,7 +4288,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
requested_model=requested_model,
|
||||
)
|
||||
else:
|
||||
stash.rate_limit_response = response
|
||||
if not reuses_request_slot:
|
||||
stash.rate_limit_response = response
|
||||
if parallel_slot_id is not None:
|
||||
stash.parallel_slot = ParallelSlotAcquisition(
|
||||
slot_id=parallel_slot_id,
|
||||
|
|
@ -4340,7 +4387,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
raw_estimate=estimated_tokens,
|
||||
)
|
||||
if stash.ptu_ceiling is not None and ptu_estimated_tokens > stash.ptu_ceiling.tpm_limit:
|
||||
await self._release_stashed_parallel_slot(stash, user_api_key_dict.parent_otel_span)
|
||||
if not reuses_request_slot:
|
||||
await self._release_stashed_parallel_slot(stash, user_api_key_dict.parent_otel_span)
|
||||
raise PTUShareExceededError(stash.ptu_ceiling, ptu_estimated_tokens)
|
||||
tpm_response: Final = await self.reserve_tpm_tokens(
|
||||
descriptors=descriptors,
|
||||
|
|
@ -4350,7 +4398,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
|
||||
if tpm_response["overall_code"] == "OVER_LIMIT":
|
||||
await self._release_stashed_parallel_slot(stash, user_api_key_dict.parent_otel_span)
|
||||
if not reuses_request_slot:
|
||||
await self._release_stashed_parallel_slot(stash, user_api_key_dict.parent_otel_span)
|
||||
self._handle_rate_limit_error(
|
||||
response=tpm_response,
|
||||
descriptors=descriptors,
|
||||
|
|
@ -4401,6 +4450,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
tpm_reservation_amount=tpm_reservation_amount,
|
||||
call_type=call_type,
|
||||
endpoint_type=endpoint_type,
|
||||
reuses_request_slot=reuses_request_slot,
|
||||
)
|
||||
|
||||
def _create_pipeline_operations(
|
||||
|
|
@ -4512,6 +4562,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
) -> Usage | ResponseAPIUsage | Mapping[str, object] | None:
|
||||
if isinstance(response_obj, (Usage, ResponseAPIUsage)):
|
||||
return response_obj
|
||||
if isinstance(response_obj, ExtractionResponse):
|
||||
return get_usage_object(response_obj)
|
||||
if isinstance(
|
||||
response_obj,
|
||||
(ModelResponse, EmbeddingResponse, TextCompletionResponse, BaseLiteLLMOpenAIResponseObject),
|
||||
|
|
@ -5143,21 +5195,24 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
# than parsed out of the body) carry their usage in
|
||||
# ``combined_usage_object`` instead, and would otherwise never charge
|
||||
# the TPM window.
|
||||
_usage: Usage | dict[str, object] | None = None
|
||||
if isinstance(
|
||||
response_obj,
|
||||
(
|
||||
ModelResponse,
|
||||
EmbeddingResponse,
|
||||
TextCompletionResponse,
|
||||
BaseLiteLLMOpenAIResponseObject,
|
||||
),
|
||||
):
|
||||
_usage = getattr(response_obj, "usage", None)
|
||||
else:
|
||||
_combined_usage: Final = kwargs.get("combined_usage_object")
|
||||
if isinstance(_combined_usage, Usage):
|
||||
_usage = _combined_usage
|
||||
_combined_usage: Final = kwargs.get("combined_usage_object")
|
||||
_usage: Final[Usage | dict[str, object] | None] = (
|
||||
getattr(response_obj, "usage", None)
|
||||
if isinstance(
|
||||
response_obj,
|
||||
(
|
||||
ModelResponse,
|
||||
EmbeddingResponse,
|
||||
TextCompletionResponse,
|
||||
BaseLiteLLMOpenAIResponseObject,
|
||||
),
|
||||
)
|
||||
else get_usage_object(response_obj)
|
||||
if isinstance(response_obj, ExtractionResponse)
|
||||
else _combined_usage
|
||||
if isinstance(_combined_usage, Usage)
|
||||
else None
|
||||
)
|
||||
total_tokens = self._get_total_tokens_from_usage(usage=_usage, rate_limit_type=rate_limit_type)
|
||||
if total_tokens == 0:
|
||||
total_tokens = self._aggregate_only_total_tokens(usage=_usage)
|
||||
|
|
@ -5310,8 +5365,16 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
try:
|
||||
verbose_proxy_logger.debug("INSIDE parallel request limiter ASYNC SUCCESS LOGGING")
|
||||
|
||||
stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs))
|
||||
await self._release_stashed_parallel_slot(stash, litellm_parent_otel_span, in_logging_callback=True)
|
||||
callback_kwargs: Final = _callback_kwargs_mapping(kwargs)
|
||||
stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(callback_kwargs))
|
||||
await self._release_stashed_parallel_slot(
|
||||
stash,
|
||||
litellm_parent_otel_span,
|
||||
in_logging_callback=True,
|
||||
round_ended_at=end_time
|
||||
if isinstance(end_time, datetime) and not callback_kwargs.get("stream")
|
||||
else None,
|
||||
)
|
||||
|
||||
pipeline_operations: Final = self._build_success_event_pipeline_operations(
|
||||
kwargs=kwargs,
|
||||
|
|
|
|||
|
|
@ -5,14 +5,14 @@ from contextlib import contextmanager
|
|||
from contextvars import ContextVar
|
||||
from datetime import datetime, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Protocol, cast
|
||||
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, cast
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.batches.batch_utils import batch_cost_is_final
|
||||
from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED
|
||||
from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED, BILLED_RETRY_ATTEMPT_METADATA_KEY
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
budget_reservation_from_metadata,
|
||||
|
|
@ -54,11 +54,13 @@ from litellm.proxy.spend_tracking.spend_log_error_logger import (
|
|||
)
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 # legacy module exports
|
||||
_sanitize_error_information_for_spend_logs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
deployment_params_provider,
|
||||
get_request_model_access_groups,
|
||||
sanitize_error_information_for_spend_logs,
|
||||
should_store_prompts_and_responses_in_spend_logs,
|
||||
)
|
||||
from litellm.proxy.utils import BilledFailedAttempt, ProxyUpdateSpend
|
||||
from litellm.types.router import Deployment
|
||||
from litellm.types.utils import (
|
||||
CallTypes,
|
||||
LiteLLMBatch,
|
||||
|
|
@ -187,6 +189,14 @@ def _proxy_spend_writer() -> DBSpendUpdateWriter:
|
|||
return proxy_logging_obj.db_spend_update_writer
|
||||
|
||||
|
||||
DeploymentLookup: TypeAlias = Callable[[str], Deployment | None]
|
||||
|
||||
|
||||
def _proxy_deployment(model_id: str) -> Deployment | None:
|
||||
router: Final = get_llm_router()
|
||||
return None if router is None else router.get_deployment(model_id=model_id)
|
||||
|
||||
|
||||
def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledFailedAttempt, ...]:
|
||||
"""The earlier billed attempts ``_failure_fields_to_lift`` stashed on the request, if any."""
|
||||
attempts: Final = request_data.get("billed_failed_attempts")
|
||||
|
|
@ -197,7 +207,10 @@ def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledF
|
|||
|
||||
|
||||
def _failure_spend_rows(
|
||||
request_data: Mapping[str, object], final_cost: float, late_guardrails_snapshot: object
|
||||
request_data: Mapping[str, object],
|
||||
final_cost: float,
|
||||
late_guardrails_snapshot: object,
|
||||
deployment_lookup: DeploymentLookup,
|
||||
) -> tuple[tuple[float, Mapping[str, object]], ...]:
|
||||
"""One ``(response_cost, kwargs)`` per spend row a failure writes: the final attempt under the
|
||||
request id, then every earlier attempt the provider also billed under its own row."""
|
||||
|
|
@ -209,25 +222,59 @@ def _failure_spend_rows(
|
|||
return (
|
||||
(final_cost, final_row),
|
||||
*(
|
||||
(attempt.response_cost, _attempt_spend_row_kwargs(request_data, attempt, position))
|
||||
(
|
||||
attempt.response_cost,
|
||||
_attempt_spend_row_kwargs(
|
||||
request_data, attempt, position, _attempt_deployment(attempt, deployment_lookup)
|
||||
),
|
||||
)
|
||||
for position, attempt in enumerate(_billed_failed_attempts(request_data))
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _attempt_deployment(attempt: BilledFailedAttempt, deployment_lookup: DeploymentLookup) -> Deployment | None:
|
||||
return None if attempt.deployment_id is None else deployment_lookup(attempt.deployment_id)
|
||||
|
||||
|
||||
def _deployment_attribution(deployment: Deployment) -> Mapping[str, object]:
|
||||
"""The spend-log attribution fields of the deployment that served an attempt: the model string it
|
||||
was called with, its provider, its configured ``api_base`` and its model group."""
|
||||
return MappingProxyType(
|
||||
{
|
||||
"model": deployment.litellm_params.model,
|
||||
"custom_llm_provider": deployment_params_provider(deployment.litellm_params),
|
||||
"api_base": deployment.litellm_params.api_base,
|
||||
"model_group": deployment.model_name,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_NO_ATTRIBUTION: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _attempt_spend_row_kwargs(
|
||||
request_data: Mapping[str, object], attempt: BilledFailedAttempt, position: int
|
||||
request_data: Mapping[str, object], attempt: BilledFailedAttempt, position: int, deployment: Deployment | None
|
||||
) -> Mapping[str, object]:
|
||||
"""The final attempt's row kwargs re-keyed for one earlier billed attempt: its own
|
||||
``request_id`` (``<id>-attempt-<position>``, since ``request_id`` is the spend-log primary key
|
||||
and the retry count restarts after a fallback hop), its estimated usage and cost, its retry
|
||||
count under ``attempted_retries`` and the deployment that billed it under ``model_info.id``."""
|
||||
count under ``attempted_retries``, the deployment that billed it under ``model_info.id`` and,
|
||||
when the router still knows that deployment, its model, provider, ``api_base`` and model group.
|
||||
The row is stamped ``billed_retry_attempt`` so the daily spend tables keep its spend, tokens
|
||||
and model usage without counting it as a request the caller sent."""
|
||||
suffix: Final = f"-attempt-{position}"
|
||||
attribution: Final = _NO_ATTRIBUTION if deployment is None else _deployment_attribution(deployment)
|
||||
raw_params: Final = cast("Mapping[str, object]", request_data.get("litellm_params") or {}) # cast-ok: request dict
|
||||
litellm_params: Final = dict(raw_params)
|
||||
litellm_params: Final = {
|
||||
**raw_params,
|
||||
**({} if deployment is None else {"api_base": attribution["api_base"]}),
|
||||
}
|
||||
metadata: Final = {
|
||||
**cast("Mapping[str, object]", litellm_params.get("metadata") or {}), # cast-ok: request dict
|
||||
"attempted_retries": attempt.attempted_retries,
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY: True,
|
||||
**({} if deployment is None else {"model_group": attribution["model_group"]}),
|
||||
}
|
||||
metadata.pop("previous_models", None)
|
||||
if attempt.deployment_id is not None:
|
||||
|
|
@ -238,6 +285,7 @@ def _attempt_spend_row_kwargs(
|
|||
litellm_params["metadata"] = metadata
|
||||
row: Final = {
|
||||
**request_data,
|
||||
**({} if deployment is None else {k: attribution[k] for k in ("model", "custom_llm_provider")}),
|
||||
"litellm_params": litellm_params,
|
||||
"combined_usage_object": attempt.usage,
|
||||
"response_cost": attempt.response_cost,
|
||||
|
|
@ -250,7 +298,11 @@ def _attempt_spend_row_kwargs(
|
|||
if isinstance(standard_logging_object, dict):
|
||||
payload: Final = cast("Mapping[str, object]", standard_logging_object) # cast-ok: StandardLoggingPayload dict
|
||||
if isinstance(payload.get("id"), str):
|
||||
row["standard_logging_object"] = {**payload, "id": f"{payload['id']}{suffix}"}
|
||||
row["standard_logging_object"] = {
|
||||
**payload,
|
||||
"id": f"{payload['id']}{suffix}",
|
||||
**({} if deployment is None else {**attribution, "model_id": attempt.deployment_id}),
|
||||
}
|
||||
return row
|
||||
|
||||
|
||||
|
|
@ -262,10 +314,12 @@ class _ProxyDBLogger(CustomLogger):
|
|||
turn_off_message_logging: bool = False,
|
||||
message_logging: bool = True,
|
||||
spend_writer: Callable[[], DBSpendUpdateWriter] = _proxy_spend_writer,
|
||||
deployment_lookup: DeploymentLookup = _proxy_deployment,
|
||||
) -> None:
|
||||
super().__init__(turn_off_message_logging=turn_off_message_logging, message_logging=message_logging)
|
||||
self.spend_event_producer = spend_event_producer
|
||||
self._spend_writer: Final = spend_writer
|
||||
self._deployment_lookup: Final = deployment_lookup
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: ObjectMapping, response_obj: object, start_time: datetime, end_time: datetime
|
||||
|
|
@ -439,6 +493,7 @@ class _ProxyDBLogger(CustomLogger):
|
|||
cast("Mapping[str, object]", request_data), # cast-ok: request dict, copied per row below
|
||||
recovered_response_cost,
|
||||
failure_snapshot_with_late_guardrails,
|
||||
self._deployment_lookup,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from litellm._service_logger import ServiceLogging
|
|||
from litellm._uuid import uuid
|
||||
from litellm.constants import (
|
||||
BAGGAGE_MAPPED_TAGS_METADATA_KEY,
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY,
|
||||
CALLER_SESSION_ID_METADATA_KEY,
|
||||
CLIENT_OUTPUT_CEILING_METADATA_KEY,
|
||||
CONSUMED_REQUEST_TAGS_METADATA_KEY,
|
||||
|
|
@ -267,7 +268,7 @@ def sanitize_for_log(value: object) -> str:
|
|||
|
||||
from litellm.router import Router
|
||||
from litellm.secret_managers.main import get_secret_bool
|
||||
from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES
|
||||
from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES, STREAM_USAGE_INJECTED_FIELD
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_API_HEADERS
|
||||
from litellm.types.router import BaggageMappedTagsStamp, holds_api_key_secret_reference
|
||||
from litellm.types.services import ServiceTypes
|
||||
|
|
@ -426,6 +427,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = (
|
|||
CONSUMED_REQUEST_TAGS_METADATA_KEY,
|
||||
ROUTING_REQUEST_TAGS_METADATA_KEY,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY,
|
||||
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
|
||||
"standard_logging_object",
|
||||
"litellm_roi_estimator",
|
||||
|
|
@ -1218,6 +1220,32 @@ def get_dynamic_logging_metadata(
|
|||
return callback_settings_obj
|
||||
|
||||
|
||||
def _callback_settings_fields(callback_settings_obj: TeamCallbackMetadata) -> Mapping[str, object]:
|
||||
callback_vars: Final = callback_settings_obj.callback_vars
|
||||
return {
|
||||
"success_callback": callback_settings_obj.success_callback,
|
||||
"failure_callback": callback_settings_obj.failure_callback,
|
||||
**({**callback_vars, TRUSTED_CALLBACK_VARS_FIELD: callback_vars} if callback_vars is not None else {}),
|
||||
}
|
||||
|
||||
|
||||
def key_and_team_logging_fields(user_api_key_dict: UserAPIKeyAuth, proxy_config: ProxyConfig) -> Mapping[str, object]:
|
||||
callback_settings_obj: Final = get_dynamic_logging_metadata(
|
||||
user_api_key_dict=user_api_key_dict, proxy_config=proxy_config
|
||||
)
|
||||
disabled_callbacks: Final = (user_api_key_dict.metadata or {}).get("litellm_disabled_callbacks")
|
||||
return MappingProxyType(
|
||||
{
|
||||
**(_callback_settings_fields(callback_settings_obj) if callback_settings_obj is not None else {}),
|
||||
**(
|
||||
{"litellm_disabled_callbacks": disabled_callbacks}
|
||||
if disabled_callbacks and isinstance(disabled_callbacks, list)
|
||||
else {}
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_get_dynamic_logging_metadata: Final = get_dynamic_logging_metadata
|
||||
|
||||
|
||||
|
|
@ -2342,7 +2370,15 @@ def refresh_proxy_server_request_body_snapshot(
|
|||
if not isinstance(proxy_server_request, dict):
|
||||
return
|
||||
_body_snapshot_exclude: Final = (
|
||||
frozenset({"secret_fields", "proxy_server_request", "litellm_logging_obj", VECTOR_STORE_CALLER_METADATA_KWARG})
|
||||
frozenset(
|
||||
{
|
||||
"secret_fields",
|
||||
"proxy_server_request",
|
||||
"litellm_logging_obj",
|
||||
VECTOR_STORE_CALLER_METADATA_KWARG,
|
||||
STREAM_USAGE_INJECTED_FIELD,
|
||||
}
|
||||
)
|
||||
| _TRANSPORT_ONLY_CREDENTIAL_KEYS
|
||||
| _CALLBACK_CREDENTIAL_KEYS
|
||||
)
|
||||
|
|
@ -2841,27 +2877,7 @@ async def add_litellm_data_to_request(
|
|||
request=request, request_data=data, metadata_key=_metadata_variable_name
|
||||
)
|
||||
|
||||
# Team Callbacks controls
|
||||
callback_settings_obj: Final = get_dynamic_logging_metadata(
|
||||
user_api_key_dict=user_api_key_dict, proxy_config=proxy_config
|
||||
)
|
||||
if callback_settings_obj is not None:
|
||||
data["success_callback"] = callback_settings_obj.success_callback
|
||||
data["failure_callback"] = callback_settings_obj.failure_callback
|
||||
|
||||
if callback_settings_obj.callback_vars is not None:
|
||||
# unpack callback_vars in data
|
||||
for k, v in callback_settings_obj.callback_vars.items():
|
||||
data[k] = v
|
||||
# Callbacks that must not honour request-supplied credentials read this
|
||||
# proxy-owned field instead of the raw request kwargs.
|
||||
data[TRUSTED_CALLBACK_VARS_FIELD] = callback_settings_obj.callback_vars
|
||||
|
||||
# Add disabled callbacks from key metadata
|
||||
if user_api_key_dict.metadata and "litellm_disabled_callbacks" in user_api_key_dict.metadata:
|
||||
disabled_callbacks: Final = user_api_key_dict.metadata["litellm_disabled_callbacks"]
|
||||
if disabled_callbacks and isinstance(disabled_callbacks, list):
|
||||
data["litellm_disabled_callbacks"] = disabled_callbacks
|
||||
data.update(key_and_team_logging_fields(user_api_key_dict=user_api_key_dict, proxy_config=proxy_config))
|
||||
|
||||
# Guardrails from key/team metadata and policy engine
|
||||
await move_guardrails_to_metadata(
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from dataclasses import dataclass
|
|||
from functools import partial, reduce
|
||||
from itertools import chain, groupby
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, NamedTuple, NoReturn, Protocol, TypeAlias, overload
|
||||
from typing import TYPE_CHECKING, Annotated, Final, NamedTuple, NoReturn, Protocol, TypeAlias, overload
|
||||
|
||||
from fastapi import (
|
||||
APIRouter,
|
||||
|
|
@ -53,7 +53,12 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
|
|||
delete_cache_team_object,
|
||||
)
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
|
||||
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
|
||||
broadcast_evictions,
|
||||
evict_and_broadcast,
|
||||
evict_local,
|
||||
evict_shared,
|
||||
)
|
||||
from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports
|
||||
_safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
safe_get_request_headers,
|
||||
|
|
@ -64,6 +69,7 @@ from litellm.proxy.management_endpoints.scim.scim_transformations import (
|
|||
ScimTransformations,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.team_endpoints import (
|
||||
delete_validated_team_rows,
|
||||
new_team,
|
||||
team_member_add,
|
||||
team_member_delete,
|
||||
|
|
@ -104,6 +110,7 @@ from litellm.types.llms.base import LiteLLMBaseModel
|
|||
from litellm.types.proxy.management_endpoints.scim_v2 import *
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prisma import Prisma
|
||||
from prisma.models import LiteLLM_VerificationToken as PrismaVerificationToken
|
||||
|
||||
|
||||
|
|
@ -317,11 +324,15 @@ async def _check_team_exists(team_id: str) -> LiteLLM_TeamTable:
|
|||
team: Final = await _table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id})
|
||||
|
||||
if not team:
|
||||
raise HTTPException(status_code=404, detail={"error": f"Group not found with ID: {team_id}"})
|
||||
raise _group_not_found(team_id)
|
||||
|
||||
return team
|
||||
|
||||
|
||||
def _group_not_found(team_id: str) -> HTTPException:
|
||||
return HTTPException(status_code=404, detail={"error": f"Group not found with ID: {team_id}"})
|
||||
|
||||
|
||||
def _extract_scim_user_data(user: SCIMUser) -> ScimUserData:
|
||||
"""Extract common data from SCIMUser object."""
|
||||
user_email = None
|
||||
|
|
@ -482,10 +493,7 @@ async def _recompute_scim_member_roles(prisma_client: PrismaClient, user_ids: It
|
|||
Recompute and persist each user's global proxy role from their resulting team
|
||||
membership. No-op unless scim_admin_group is configured, so a SCIM group write
|
||||
that drops a member from the admin group demotes them just like the user
|
||||
endpoints do, and the role is left untouched when the feature is off. The users'
|
||||
rows are locked through ``USER_ROWS_LOCK_SQL`` before their teams are read, the
|
||||
same lock a roster write takes, so a concurrent group push cannot change a
|
||||
member's teams between the read and the role write.
|
||||
endpoints do, and the role is left untouched when the feature is off.
|
||||
"""
|
||||
admin_group: Final = await _get_scim_admin_group()
|
||||
if admin_group is None:
|
||||
|
|
@ -494,28 +502,46 @@ async def _recompute_scim_member_roles(prisma_client: PrismaClient, user_ids: It
|
|||
ids: Final = tuple(dict.fromkeys(user_ids))
|
||||
if not ids:
|
||||
return
|
||||
default_role: Final = _default_scim_user_role()
|
||||
async with prisma_client.tx(timeout=ROSTER_TX_TIMEOUT) as tx:
|
||||
await tx.query_raw(USER_ROWS_LOCK_SQL, sorted(ids))
|
||||
rows: Final = await find_many_in(user_tx_db(tx), "user_id", ids)
|
||||
team_ids: Final = tuple(dict.fromkeys(chain.from_iterable(row.teams for row in rows)))
|
||||
teams: Final = await find_many_in(team_tx_db(tx), "team_id", team_ids)
|
||||
alias_of: Final = MappingProxyType({team.team_id: team.team_alias for team in teams})
|
||||
resolved: Final = tuple(
|
||||
(
|
||||
row.user_id,
|
||||
_resolve_scim_user_role(
|
||||
[SCIMUserGroup(value=team_id, display=alias_of.get(team_id)) for team_id in row.teams],
|
||||
admin_group,
|
||||
default_role,
|
||||
),
|
||||
)
|
||||
for row in rows
|
||||
await _write_scim_member_roles_tx(tx, ids, admin_group=admin_group)
|
||||
|
||||
from litellm.proxy.proxy_server import user_api_key_cache
|
||||
|
||||
evict_local(cache_keys=ids, user_api_key_cache=user_api_key_cache)
|
||||
await evict_shared(cache_keys=ids, cache=user_api_key_cache)
|
||||
await broadcast_evictions(cache_keys=ids)
|
||||
|
||||
|
||||
async def _write_scim_member_roles_tx(tx: "Prisma", user_ids: Sequence[str], *, admin_group: str) -> None:
|
||||
"""
|
||||
Write each user's role resolved from the teams they are in, on ``tx``. The users'
|
||||
rows are locked through ``USER_ROWS_LOCK_SQL`` before their teams are read, the
|
||||
same lock a roster write takes, so a concurrent group push cannot change a
|
||||
member's teams between the read and the role write.
|
||||
"""
|
||||
if not user_ids:
|
||||
return
|
||||
default_role: Final = _default_scim_user_role()
|
||||
await tx.query_raw(USER_ROWS_LOCK_SQL, sorted(user_ids))
|
||||
rows: Final = await find_many_in(user_tx_db(tx), "user_id", user_ids)
|
||||
team_ids: Final = tuple(dict.fromkeys(chain.from_iterable(row.teams for row in rows)))
|
||||
teams: Final = await find_many_in(team_tx_db(tx), "team_id", team_ids)
|
||||
alias_of: Final = MappingProxyType({team.team_id: team.team_alias for team in teams})
|
||||
resolved: Final = tuple(
|
||||
(
|
||||
row.user_id,
|
||||
_resolve_scim_user_role(
|
||||
[SCIMUserGroup(value=team_id, display=alias_of.get(team_id)) for team_id in row.teams],
|
||||
admin_group,
|
||||
default_role,
|
||||
),
|
||||
)
|
||||
for row in rows
|
||||
)
|
||||
for role in dict.fromkeys(role for _, role in resolved):
|
||||
await tx.execute_raw(
|
||||
_SET_USER_ROLE_SQL, role, sorted(user_id for user_id, user_role in resolved if user_role == role)
|
||||
)
|
||||
for role in dict.fromkeys(role for _, role in resolved):
|
||||
await tx.execute_raw(
|
||||
_SET_USER_ROLE_SQL, role, sorted(user_id for user_id, user_role in resolved if user_role == role)
|
||||
)
|
||||
|
||||
|
||||
class _ResolvedUserMember(NamedTuple):
|
||||
|
|
@ -3051,7 +3077,8 @@ async def update_group(
|
|||
dependencies=[Depends(user_api_key_auth)],
|
||||
)
|
||||
async def delete_group(
|
||||
group_id: str = Path(..., title="Group ID"),
|
||||
group_id: Annotated[str, Path(title="Group ID")],
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
):
|
||||
"""
|
||||
Delete a group according to SCIM v2 protocol
|
||||
|
|
@ -3064,23 +3091,22 @@ async def delete_group(
|
|||
existing_team: Final = await _check_team_exists(group_id)
|
||||
raise_if_teams_hold_ptu_shares((group_id,), llm_router)
|
||||
|
||||
member_ids: Final = await _get_team_member_user_ids_from_team(existing_team)
|
||||
|
||||
# For each member, remove this team from their teams list
|
||||
for member_id in member_ids:
|
||||
user = await _table(UserRepository(prisma_client)).find_unique(where={"user_id": member_id})
|
||||
if user:
|
||||
current_teams = user.teams or []
|
||||
if group_id in current_teams:
|
||||
new_teams = [t for t in current_teams if t != group_id]
|
||||
await _table(UserRepository(prisma_client)).update(
|
||||
where={"user_id": member_id}, data={"teams": new_teams}
|
||||
)
|
||||
|
||||
await _recompute_scim_member_roles(prisma_client, member_ids)
|
||||
|
||||
# Delete team
|
||||
await _table(TeamRepository(prisma_client)).delete(where={"team_id": group_id})
|
||||
admin_group: Final = await _get_scim_admin_group()
|
||||
deleted: Final = await delete_validated_team_rows(
|
||||
teams=(LiteLLM_TeamTable.model_validate(existing_team.model_dump()),),
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN,
|
||||
api_key=user_api_key_dict.api_key,
|
||||
user_id=user_api_key_dict.user_id,
|
||||
),
|
||||
litellm_changed_by=None,
|
||||
before_commit=(
|
||||
None if admin_group is None else partial(_write_scim_member_roles_tx, admin_group=admin_group)
|
||||
),
|
||||
)
|
||||
if group_id not in deleted.team_ids:
|
||||
raise _group_not_found(group_id)
|
||||
|
||||
return Response(status_code=204)
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import copy
|
|||
import json
|
||||
import math
|
||||
import traceback
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from collections.abc import Set as AbstractSet
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
|
@ -34,7 +34,7 @@ from typing import (
|
|||
|
||||
import fastapi
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field, JsonValue, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict, assert_never
|
||||
|
||||
import litellm
|
||||
|
|
@ -102,6 +102,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
|
|||
OrganizationNotFoundError,
|
||||
_cache_team_object, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
allowed_route_check_inside_route,
|
||||
await_inflight_team_membership_load,
|
||||
cache_team_object,
|
||||
can_org_access_model,
|
||||
delete_cache_key_objects,
|
||||
|
|
@ -112,6 +113,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
|
|||
get_team_object,
|
||||
get_user_object,
|
||||
invalidate_team_member_spend_state,
|
||||
jwt_key_mapping_cache_key,
|
||||
)
|
||||
from litellm.proxy.auth.auth_utils import (
|
||||
enforce_batch_limits_are_admin_only,
|
||||
|
|
@ -119,12 +121,19 @@ from litellm.proxy.auth.auth_utils import (
|
|||
)
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
|
||||
broadcast_evictions,
|
||||
evict_and_broadcast,
|
||||
evict_local,
|
||||
evict_shared,
|
||||
publish_auth_cache_invalidation,
|
||||
)
|
||||
from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars
|
||||
from litellm.proxy.common_utils.json_merge_patch import apply_json_merge_patch
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.common_utils.user_api_key_cache import (
|
||||
UserApiKeyCache,
|
||||
team_membership_auth_cache_key,
|
||||
team_membership_reservation_cache_key,
|
||||
)
|
||||
from litellm.proxy.db.db_span import db_span
|
||||
from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
|
||||
from litellm.proxy.hooks.model_max_budget_limiter import (
|
||||
|
|
@ -207,7 +216,7 @@ from litellm.proxy.management_helpers.utils import (
|
|||
)
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging, handle_exception_on_proxy
|
||||
from litellm.repositories.budget_repository import BudgetRepository
|
||||
from litellm.repositories.chunked_in import find_many_in
|
||||
from litellm.repositories.chunked_in import delete_many_in, find_many_in
|
||||
from litellm.repositories.organization_repository import OrganizationRepository
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.repositories.table_repositories import (
|
||||
|
|
@ -417,6 +426,23 @@ class _TeamDeleteTx(AccessGroupSyncTx, Protocol):
|
|||
@property
|
||||
def litellm_teammembership(self) -> "TableActions[prisma_models.LiteLLM_TeamMembership]": ...
|
||||
|
||||
@property
|
||||
def litellm_deletedteamtable(self) -> "TableActions[prisma_models.LiteLLM_DeletedTeamTable]": ...
|
||||
|
||||
@property
|
||||
def litellm_verificationtoken(self) -> "TableActions[prisma_models.LiteLLM_VerificationToken]": ...
|
||||
|
||||
@property
|
||||
def litellm_jwtkeymapping(self) -> "TableActions[prisma_models.LiteLLM_JWTKeyMapping]": ...
|
||||
|
||||
|
||||
def _team_delete_tx(tx: "Prisma") -> _TeamDeleteTx:
|
||||
return tx # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do
|
||||
|
||||
|
||||
_LOCK_DELETED_TEAM_MEMBERS_SQL: Final = """
|
||||
SELECT user_id, teams FROM "LiteLLM_UserTable" WHERE teams && $1::text[] ORDER BY user_id COLLATE "C" FOR UPDATE
|
||||
"""
|
||||
|
||||
_STRIP_DELETED_TEAMS_FROM_USERS_SQL: Final = """
|
||||
UPDATE "LiteLLM_UserTable"
|
||||
|
|
@ -430,6 +456,22 @@ WHERE user_id IN (
|
|||
)
|
||||
"""
|
||||
|
||||
|
||||
class _DetachedMember(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
user_id: str
|
||||
teams: tuple[str, ...]
|
||||
|
||||
|
||||
_DetachedMembers: Final = TypeAdapter(tuple[_DetachedMember, ...])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeletedTeams:
|
||||
team_ids: tuple[str, ...]
|
||||
|
||||
|
||||
_INCLUDE_MODEL_TABLE: Final = MappingProxyType({"litellm_model_table": True})
|
||||
|
||||
|
||||
|
|
@ -4526,18 +4568,7 @@ async def delete_team(
|
|||
}'
|
||||
```
|
||||
"""
|
||||
from litellm.proxy.management_helpers.audit_logs import (
|
||||
get_audit_log_changed_by,
|
||||
is_audit_logging_enabled,
|
||||
)
|
||||
from litellm.proxy.proxy_server import (
|
||||
create_audit_log_for_update,
|
||||
litellm_proxy_admin_name,
|
||||
llm_router,
|
||||
prisma_client,
|
||||
proxy_logging_obj,
|
||||
user_api_key_cache,
|
||||
)
|
||||
from litellm.proxy.proxy_server import llm_router, prisma_client
|
||||
|
||||
if prisma_client is None:
|
||||
raise HTTPException(status_code=500, detail={"error": "No db connected"})
|
||||
|
|
@ -4567,47 +4598,32 @@ async def delete_team(
|
|||
|
||||
raise_if_teams_hold_ptu_shares(data.team_ids, llm_router)
|
||||
|
||||
await _persist_deleted_team_records(
|
||||
await delete_validated_teams(
|
||||
teams=team_rows,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
litellm_changed_by=litellm_changed_by,
|
||||
)
|
||||
deleted_teams: Final[_DeletedTeamsResult] = {"deleted_teams": data.team_ids}
|
||||
return deleted_teams
|
||||
|
||||
# we do this after the first for loop, since first for loop is for validation. we only want this inserted after validation passes
|
||||
if is_audit_logging_enabled():
|
||||
# make an audit log for each team deleted
|
||||
for team_id in data.team_ids:
|
||||
team_row: LiteLLM_TeamTable | None = await prisma_client.get_data(
|
||||
team_id=team_id, table_name="team", query_type="find_unique"
|
||||
)
|
||||
|
||||
if team_row is None:
|
||||
continue
|
||||
async def delete_validated_teams(
|
||||
teams: Sequence[LiteLLM_TeamTable],
|
||||
prisma_client: PrismaClient,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
litellm_changed_by: str | None,
|
||||
) -> DeletedTeams:
|
||||
"""
|
||||
What ``/team/delete`` does once the teams exist and the caller may delete them: every key
|
||||
and BYOK model of the teams, then ``delete_validated_team_rows``.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
proxy_logging_obj,
|
||||
user_api_key_cache,
|
||||
)
|
||||
|
||||
_team_row = team_row.json(exclude_none=True)
|
||||
|
||||
asyncio.create_task(
|
||||
create_audit_log_for_update(
|
||||
request_data=LiteLLM_AuditLogs(
|
||||
id=str(uuid.uuid4()),
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
changed_by=get_audit_log_changed_by(
|
||||
litellm_changed_by=litellm_changed_by,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
litellm_proxy_admin_name=litellm_proxy_admin_name,
|
||||
),
|
||||
changed_by_api_key=user_api_key_dict.api_key,
|
||||
table_name=LitellmTableNames.TEAM_TABLE_NAME,
|
||||
object_id=team_id,
|
||||
action="deleted",
|
||||
updated_values="{}",
|
||||
before_value=_team_row,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
# End of Audit logging
|
||||
team_ids: Final = [team.team_id for team in teams]
|
||||
|
||||
## DELETE ASSOCIATED KEYS
|
||||
# Fetch keys before deletion to persist them
|
||||
|
|
@ -4615,7 +4631,7 @@ async def delete_team(
|
|||
persist_deleted_verification_tokens,
|
||||
)
|
||||
|
||||
keys_to_delete: Final = await _tokens_db(prisma_client).find_many(where={"team_id": {"in": data.team_ids}})
|
||||
keys_to_delete: Final = await _tokens_db(prisma_client).find_many(where={"team_id": {"in": team_ids}})
|
||||
jwt_mapping_cache_keys: Final = await get_jwt_key_mapping_cache_keys_for_tokens(
|
||||
hashed_tokens=tuple(key.token for key in keys_to_delete),
|
||||
prisma_client=prisma_client,
|
||||
|
|
@ -4629,7 +4645,7 @@ async def delete_team(
|
|||
litellm_changed_by=litellm_changed_by,
|
||||
)
|
||||
|
||||
await prisma_client.delete_data(team_id_list=data.team_ids, table_name="key")
|
||||
await prisma_client.delete_data(team_id_list=team_ids, table_name="key")
|
||||
|
||||
if keys_to_delete:
|
||||
KeyManagementEventHooks.create_key_deleted_audit_logs(
|
||||
|
|
@ -4654,18 +4670,41 @@ async def delete_team(
|
|||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
await delete_team_models(
|
||||
team_ids=data.team_ids,
|
||||
team_ids=team_ids,
|
||||
prisma_client=prisma_client,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
|
||||
await _sweep_deleted_team_references(team_ids=data.team_ids, prisma_client=prisma_client)
|
||||
|
||||
member_ids_per_team: Final = await _resolve_deleted_team_member_user_ids(
|
||||
teams=team_rows,
|
||||
return await delete_validated_team_rows(
|
||||
teams=teams,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
litellm_changed_by=litellm_changed_by,
|
||||
)
|
||||
|
||||
|
||||
async def delete_validated_team_rows(
|
||||
teams: Sequence[LiteLLM_TeamTable],
|
||||
prisma_client: PrismaClient,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
litellm_changed_by: str | None,
|
||||
before_commit: Callable[["Prisma", tuple[str, ...]], Awaitable[None]] | None = None,
|
||||
) -> DeletedTeams:
|
||||
"""
|
||||
Under every team's advisory lock, delete the teams still there with the keys their rostered
|
||||
members hold on them, their membership rows and user references, and write their deleted-team
|
||||
records; then evict caches and write audit rows. Keys of users off the roster and the teams'
|
||||
models stay. ``team_ids`` names only the teams this call deleted. ``before_commit`` runs on the
|
||||
delete's transaction with the members of the rosters read under the lock, so its writes commit
|
||||
or roll back with the delete.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
proxy_logging_obj,
|
||||
user_api_key_cache,
|
||||
)
|
||||
|
||||
team_ids: Final = [team.team_id for team in teams]
|
||||
|
||||
## DELETE TEAMS
|
||||
# Both the delete and the reconcile sweep run under every team's advisory lock
|
||||
# (TEAM_ADVISORY_LOCK_SQL, the same one /team/member_add takes before its own writes),
|
||||
|
|
@ -4674,14 +4713,28 @@ async def delete_team(
|
|||
# the lock before this transaction starts, in which case this sweep reaches what it wrote,
|
||||
# or is still waiting on the lock, in which case its own re-read happens after this commits
|
||||
# and sees the row gone before it writes anything.
|
||||
delete_filter: Final[_TeamIdInFilter] = {"team_id": {"in": data.team_ids}}
|
||||
async with prisma_client.tx(timeout=timedelta(seconds=TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS)) as tx:
|
||||
for team_id in sorted(data.team_ids):
|
||||
tx_budget: Final = timedelta(seconds=TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS)
|
||||
async with prisma_client.tx(max_wait=tx_budget, timeout=tx_budget) as tx:
|
||||
for team_id in sorted(team_ids):
|
||||
await tx.query_raw(TEAM_ADVISORY_LOCK_SQL, team_id)
|
||||
await tx.litellm_teamtable.delete_many(where=delete_filter)
|
||||
await _sweep_deleted_team_references_tx(team_ids=data.team_ids, tx=tx)
|
||||
team_tx: Final = _team_delete_tx(tx)
|
||||
locked: Final = await _locked_teams_tx(team_ids=team_ids, tx=team_tx)
|
||||
member_keys, jwt_mapping_cache_keys = await _delete_roster_member_keys_tx(
|
||||
locked=locked,
|
||||
tx=tx,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
litellm_changed_by=litellm_changed_by,
|
||||
)
|
||||
await _delete_locked_teams_tx(
|
||||
locked=locked, tx=team_tx, user_api_key_dict=user_api_key_dict, litellm_changed_by=litellm_changed_by
|
||||
)
|
||||
detached_per_team: Final = await _sweep_deleted_team_references_tx(team_ids=team_ids, tx=team_tx)
|
||||
if before_commit is not None:
|
||||
await before_commit(tx, _locked_roster_user_ids(locked))
|
||||
|
||||
deleted_teams: Final[_DeletedTeamsResult] = {"deleted_teams": data.team_ids}
|
||||
rostered_per_team: Final = await _resolve_deleted_team_member_user_ids(teams=locked, prisma_client=prisma_client)
|
||||
member_ids_per_team: Final = _members_per_deleted_team(rostered_per_team, detached_per_team)
|
||||
|
||||
# Evict AFTER the rows are gone. Both writers of these keys (`_cache_team_object` and
|
||||
# `get_team_object_by_alias`) hydrate from the db, so evicting first leaves a window where a
|
||||
|
|
@ -4689,7 +4742,7 @@ async def delete_team(
|
|||
# invalidated anything. Nothing fallible runs between the delete and this, or a failure there
|
||||
# would strand the deleted team in cache.
|
||||
await _invalidate_deleted_team_cache(
|
||||
teams=team_rows,
|
||||
teams=locked,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
|
@ -4697,44 +4750,167 @@ async def delete_team(
|
|||
member_ids_per_team=member_ids_per_team,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
)
|
||||
await _invalidate_deleted_key_cache(
|
||||
keys=member_keys,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
await evict_and_broadcast(cache_keys=jwt_mapping_cache_keys, user_api_key_cache=user_api_key_cache)
|
||||
|
||||
for deleted_team in team_rows:
|
||||
_schedule_deleted_team_audit_rows(
|
||||
teams=locked, user_api_key_dict=user_api_key_dict, litellm_changed_by=litellm_changed_by
|
||||
)
|
||||
if member_keys:
|
||||
KeyManagementEventHooks.create_key_deleted_audit_logs(
|
||||
keys_being_deleted=member_keys,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
litellm_changed_by=litellm_changed_by,
|
||||
)
|
||||
|
||||
for deleted_team in locked:
|
||||
_emit_team_members_metric(deleted_team.model_copy(update={"members_with_roles": ()}))
|
||||
await sync_team_access_group_membership(prisma_client=prisma_client, team_id=deleted_team.team_id)
|
||||
|
||||
return deleted_teams
|
||||
return DeletedTeams(team_ids=tuple(team.team_id for team in locked))
|
||||
|
||||
|
||||
async def _sweep_deleted_team_references(team_ids: Sequence[str], prisma_client: PrismaClient) -> None:
|
||||
def _locked_roster_user_ids(locked: Sequence[LiteLLM_TeamTable]) -> tuple[str, ...]:
|
||||
return tuple(dict.fromkeys(chain.from_iterable(_roster_user_ids(team) for team in locked)))
|
||||
|
||||
|
||||
def _members_per_deleted_team(
|
||||
rostered: Sequence[tuple[str, Sequence[str]]], detached: Sequence[tuple[str, Sequence[str]]]
|
||||
) -> tuple[tuple[str, tuple[str, ...]], ...]:
|
||||
detached_by_team: Final = MappingProxyType({team_id: user_ids for team_id, user_ids in detached})
|
||||
return tuple(
|
||||
(team_id, tuple(dict.fromkeys((*user_ids, *detached_by_team[team_id])))) for team_id, user_ids in rostered
|
||||
)
|
||||
|
||||
|
||||
def _roster_user_ids(team: LiteLLM_TeamTable) -> tuple[str, ...]:
|
||||
return tuple(member.user_id for member in team.members_with_roles if member.user_id is not None)
|
||||
|
||||
|
||||
def _is_roster_member_key(
|
||||
key: "prisma_models.LiteLLM_VerificationToken", roster_by_team: Mapping[str, frozenset[str]]
|
||||
) -> bool:
|
||||
return key.team_id is not None and key.user_id in roster_by_team.get(key.team_id, frozenset())
|
||||
|
||||
|
||||
async def _locked_teams_tx(team_ids: Sequence[str], tx: _TeamDeleteTx) -> tuple[LiteLLM_TeamTable, ...]:
|
||||
rows: Final = await find_many_in(tx.litellm_teamtable, "team_id", team_ids)
|
||||
return tuple(LiteLLM_TeamTable.model_validate(row.model_dump()) for row in rows)
|
||||
|
||||
|
||||
async def _delete_roster_member_keys_tx(
|
||||
locked: Sequence[LiteLLM_TeamTable],
|
||||
tx: "Prisma",
|
||||
prisma_client: PrismaClient,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
litellm_changed_by: str | None,
|
||||
) -> tuple[tuple["prisma_models.LiteLLM_VerificationToken", ...], tuple[str, ...]]:
|
||||
from litellm.proxy.management_endpoints.key_management_endpoints import (
|
||||
persist_deleted_verification_tokens,
|
||||
)
|
||||
|
||||
team_tx: Final = _team_delete_tx(tx)
|
||||
roster_by_team: Final = MappingProxyType({team.team_id: frozenset(_roster_user_ids(team)) for team in locked})
|
||||
team_keys: Final = await find_many_in(team_tx.litellm_verificationtoken, "team_id", tuple(roster_by_team))
|
||||
member_keys: Final = tuple(key for key in team_keys if _is_roster_member_key(key, roster_by_team))
|
||||
if not member_keys:
|
||||
return (), ()
|
||||
tokens: Final = tuple(key.token for key in member_keys)
|
||||
jwt_mappings: Final = await find_many_in(team_tx.litellm_jwtkeymapping, "token", tokens)
|
||||
await persist_deleted_verification_tokens(
|
||||
keys=member_keys, # pyright: ignore[reportArgumentType] # generated row model carries the same columns as LiteLLM_VerificationToken
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
litellm_changed_by=litellm_changed_by,
|
||||
tx=tx,
|
||||
)
|
||||
await delete_many_in(team_tx.litellm_verificationtoken, "token", tokens, atomicity="caller_transaction")
|
||||
return member_keys, tuple(
|
||||
jwt_key_mapping_cache_key(mapping.jwt_claim_name, mapping.jwt_claim_value, mapping.jwt_issuer)
|
||||
for mapping in jwt_mappings
|
||||
)
|
||||
|
||||
|
||||
async def _delete_locked_teams_tx(
|
||||
locked: Sequence[LiteLLM_TeamTable],
|
||||
tx: _TeamDeleteTx,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
litellm_changed_by: str | None,
|
||||
) -> None:
|
||||
await _persist_deleted_team_records(
|
||||
teams=locked,
|
||||
deleted_teams=tx.litellm_deletedteamtable,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
litellm_changed_by=litellm_changed_by,
|
||||
)
|
||||
await delete_many_in(
|
||||
tx.litellm_teamtable,
|
||||
"team_id",
|
||||
tuple(team.team_id for team in locked),
|
||||
atomicity="caller_transaction",
|
||||
)
|
||||
|
||||
|
||||
def _schedule_deleted_team_audit_rows(
|
||||
teams: Sequence[LiteLLM_TeamTable],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
litellm_changed_by: str | None,
|
||||
) -> None:
|
||||
from litellm.proxy.management_helpers.audit_logs import (
|
||||
get_audit_log_changed_by,
|
||||
is_audit_logging_enabled,
|
||||
)
|
||||
from litellm.proxy.proxy_server import (
|
||||
create_audit_log_for_update,
|
||||
litellm_proxy_admin_name,
|
||||
)
|
||||
|
||||
if not is_audit_logging_enabled():
|
||||
return
|
||||
for team in teams:
|
||||
asyncio.create_task(
|
||||
create_audit_log_for_update(
|
||||
request_data=LiteLLM_AuditLogs(
|
||||
id=str(uuid.uuid4()),
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
changed_by=get_audit_log_changed_by(
|
||||
litellm_changed_by=litellm_changed_by,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
litellm_proxy_admin_name=litellm_proxy_admin_name,
|
||||
),
|
||||
changed_by_api_key=user_api_key_dict.api_key,
|
||||
table_name=LitellmTableNames.TEAM_TABLE_NAME,
|
||||
object_id=team.team_id,
|
||||
action="deleted",
|
||||
updated_values="{}",
|
||||
before_value=team.model_dump_json(exclude_none=True),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _sweep_deleted_team_references_tx(
|
||||
team_ids: Sequence[str], tx: _TeamDeleteTx
|
||||
) -> tuple[tuple[str, tuple[str, ...]], ...]:
|
||||
"""
|
||||
Strip the deleted team ids from every user row and team-membership row that still references them.
|
||||
|
||||
The per-member `team_member_delete` pass above only reaches users listed in the team's
|
||||
`members_with_roles`, so a user row that outlived its roster entry is invisible to it and keeps
|
||||
surfacing the team on `/user/info` after the team is gone.
|
||||
|
||||
#36839 closed the route that created that drift, by resolving member removal off the roster
|
||||
entry's `user_id` rather than the identifier the caller happened to pass. It does not backfill
|
||||
rows that already drifted, which is the state this was reported against, so the sweep still has
|
||||
to run on delete.
|
||||
|
||||
One statement filters the array in place rather than read-filter-write: rewriting the whole
|
||||
array from a snapshot read outside a transaction drops any team a concurrent `/team/member_add`
|
||||
appended in between. It locks the user rows in the order `USER_ROWS_LOCK_SQL` takes them, so it
|
||||
never waits on a SCIM group push in a cycle.
|
||||
Strip the deleted team ids from every user and team-membership row still referencing them, on
|
||||
the transaction holding the teams' advisory locks. User rows are locked first in
|
||||
``USER_ROWS_LOCK_SQL`` order so the sweep never deadlocks with a SCIM group push, and one
|
||||
statement filters each array in place so a team a concurrent ``/team/member_add`` appended is
|
||||
kept. Returns, per team, the users read under the lock as referencing it.
|
||||
"""
|
||||
_ = await prisma_client.db.execute_raw(_STRIP_DELETED_TEAMS_FROM_USERS_SQL, list(team_ids))
|
||||
|
||||
_ = await _team_membership_db(prisma_client).delete_many(where=_TeamIdInFilter(team_id={"in": tuple(team_ids)}))
|
||||
|
||||
|
||||
async def _sweep_deleted_team_references_tx(team_ids: Sequence[str], tx: _TeamDeleteTx) -> None:
|
||||
"""Same sweep as `_sweep_deleted_team_references`, run on the transaction that holds
|
||||
every id's advisory lock and deletes the team rows, so it commits or rolls back with them."""
|
||||
locked: Final = _DetachedMembers.validate_python(await tx.query_raw(_LOCK_DELETED_TEAM_MEMBERS_SQL, list(team_ids)))
|
||||
_ = await tx.execute_raw(_STRIP_DELETED_TEAMS_FROM_USERS_SQL, list(team_ids))
|
||||
|
||||
membership_filter: Final[_TeamIdInFilter] = {"team_id": {"in": tuple(team_ids)}}
|
||||
_ = await tx.litellm_teammembership.delete_many(where=membership_filter)
|
||||
return tuple(
|
||||
(team_id, tuple(member.user_id for member in locked if team_id in member.teams)) for team_id in team_ids
|
||||
)
|
||||
|
||||
|
||||
async def _invalidate_deleted_key_cache(
|
||||
|
|
@ -4791,17 +4967,20 @@ async def _evict_deleted_team_member_cache(
|
|||
member_user_ids: Sequence[str],
|
||||
user_api_key_cache: UserApiKeyCache,
|
||||
) -> None:
|
||||
await evict_and_broadcast(cache_keys=tuple(member_user_ids), user_api_key_cache=user_api_key_cache)
|
||||
await asyncio.gather(
|
||||
*(
|
||||
invalidate_team_member_spend_state(
|
||||
user_id=user_id,
|
||||
team_id=team_id,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
)
|
||||
for user_id in member_user_ids
|
||||
)
|
||||
_ = await asyncio.gather(
|
||||
*(await_inflight_team_membership_load(user_id=user_id, team_id=team_id) for user_id in member_user_ids)
|
||||
)
|
||||
cache_keys: Final = tuple(member_cache_keys(team_id, member_user_ids))
|
||||
evict_local(cache_keys=cache_keys, user_api_key_cache=user_api_key_cache)
|
||||
await evict_shared(cache_keys=cache_keys, cache=user_api_key_cache)
|
||||
await broadcast_evictions(cache_keys=cache_keys)
|
||||
|
||||
|
||||
def member_cache_keys(team_id: str, user_ids: Sequence[str]) -> Iterator[str]:
|
||||
for user_id in user_ids:
|
||||
yield user_id
|
||||
yield team_membership_auth_cache_key(team_id=team_id, user_id=user_id)
|
||||
yield team_membership_reservation_cache_key(user_id=user_id, team_id=team_id)
|
||||
|
||||
|
||||
async def _resolve_deleted_team_member_user_ids(
|
||||
|
|
@ -4832,7 +5011,7 @@ async def _deleted_team_member_user_ids(team: LiteLLM_TeamTable, prisma_client:
|
|||
|
||||
|
||||
def _transform_teams_to_deleted_records(
|
||||
teams: list[LiteLLM_TeamTable],
|
||||
teams: Sequence[LiteLLM_TeamTable],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
litellm_changed_by: str | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
|
|
@ -4881,17 +5060,17 @@ def _transform_teams_to_deleted_records(
|
|||
|
||||
async def _save_deleted_team_records(
|
||||
records: list[dict[str, object]],
|
||||
prisma_client: PrismaClient,
|
||||
deleted_teams: "TableActions[prisma_models.LiteLLM_DeletedTeamTable]",
|
||||
) -> None:
|
||||
"""Save deleted team records to the database."""
|
||||
if not records:
|
||||
return
|
||||
await _deleted_team_db(prisma_client).create_many(data=records)
|
||||
await deleted_teams.create_many(data=records)
|
||||
|
||||
|
||||
async def _persist_deleted_team_records(
|
||||
teams: list[LiteLLM_TeamTable],
|
||||
prisma_client: PrismaClient,
|
||||
teams: Sequence[LiteLLM_TeamTable],
|
||||
deleted_teams: "TableActions[prisma_models.LiteLLM_DeletedTeamTable]",
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
litellm_changed_by: str | None = None,
|
||||
) -> None:
|
||||
|
|
@ -4903,7 +5082,7 @@ async def _persist_deleted_team_records(
|
|||
)
|
||||
await _save_deleted_team_records(
|
||||
records=records,
|
||||
prisma_client=prisma_client,
|
||||
deleted_teams=deleted_teams,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ Functions to create audit logs for LiteLLM Proxy
|
|||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Awaitable, Sequence
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final, TypeVar
|
||||
|
||||
|
|
@ -258,12 +259,19 @@ async def create_object_audit_log(
|
|||
)
|
||||
|
||||
|
||||
async def _insert_audit_log_row(repository: AuditLogRepository, request_data: LiteLLM_AuditLogs) -> None:
|
||||
async def _insert_audit_logs(insert: Awaitable[object], description: str) -> None:
|
||||
try:
|
||||
await repository.table.create(data={**request_data.model_dump(exclude_none=True)})
|
||||
await insert
|
||||
except Exception as e:
|
||||
# [Non-Blocking Exception. Do not allow blocking LLM API call]
|
||||
verbose_proxy_logger.error("Failed Creating audit log %s", e)
|
||||
verbose_proxy_logger.error("Failed Creating %s %s", description, e)
|
||||
|
||||
|
||||
def _json_encoded(request_data: LiteLLM_AuditLogs) -> LiteLLM_AuditLogs:
|
||||
values: Final = (("updated_values", request_data.updated_values), ("before_value", request_data.before_value))
|
||||
return request_data.model_copy(
|
||||
update={field: json.dumps(value) for field, value in values if isinstance(value, dict)}
|
||||
)
|
||||
|
||||
|
||||
async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs):
|
||||
|
|
@ -279,15 +287,10 @@ async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs):
|
|||
return
|
||||
|
||||
verbose_proxy_logger.debug("creating audit log for %s", request_data)
|
||||
|
||||
if isinstance(request_data.updated_values, dict):
|
||||
request_data.updated_values = json.dumps(request_data.updated_values)
|
||||
|
||||
if isinstance(request_data.before_value, dict):
|
||||
request_data.before_value = json.dumps(request_data.before_value)
|
||||
encoded: Final = _json_encoded(request_data)
|
||||
|
||||
# Dispatch to external audit log callbacks regardless of DB availability
|
||||
await _dispatch_audit_log_to_callbacks(request_data)
|
||||
await _dispatch_audit_log_to_callbacks(encoded)
|
||||
|
||||
if prisma_client is None:
|
||||
verbose_proxy_logger.error("prisma_client is None, cannot write audit log to DB")
|
||||
|
|
@ -295,7 +298,41 @@ async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs):
|
|||
|
||||
await track_audit_task(
|
||||
asyncio.create_task(
|
||||
_insert_audit_log_row(AuditLogRepository(prisma_client), request_data),
|
||||
name=_audit_task_name(request_data, "the database"),
|
||||
_insert_audit_logs(
|
||||
AuditLogRepository(prisma_client).table.create(data={**encoded.model_dump(exclude_none=True)}),
|
||||
"audit log",
|
||||
),
|
||||
name=_audit_task_name(encoded, "the database"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def create_audit_logs_for_updates(rows: Sequence[LiteLLM_AuditLogs]) -> None:
|
||||
"""Create audit logs for many objects with one insert."""
|
||||
if not rows or not is_audit_logging_enabled():
|
||||
return
|
||||
|
||||
from litellm.proxy.proxy_server import premium_user, prisma_client
|
||||
|
||||
if premium_user is not True:
|
||||
return
|
||||
|
||||
encoded: Final = tuple(_json_encoded(row) for row in rows)
|
||||
for row in encoded:
|
||||
await _dispatch_audit_log_to_callbacks(row)
|
||||
|
||||
if prisma_client is None:
|
||||
verbose_proxy_logger.error("prisma_client is None, cannot write audit log to DB")
|
||||
return
|
||||
|
||||
await track_audit_task(
|
||||
asyncio.create_task(
|
||||
_insert_audit_logs(
|
||||
AuditLogRepository(prisma_client).table.create_many(
|
||||
data=tuple(row.model_dump(exclude_none=True) for row in encoded)
|
||||
),
|
||||
f"{len(encoded)} audit logs",
|
||||
),
|
||||
name=f"{len(encoded)} audit logs to the database",
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
import math
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
CustomMetadataValue: TypeAlias = str | int | float | bool
|
||||
|
||||
_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[object, object])
|
||||
_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
|
||||
|
||||
|
||||
def _string_tags(tags: object) -> Iterator[str]:
|
||||
if isinstance(tags, list):
|
||||
yield from (tag for tag in _OBJECT_LIST_ADAPTER.validate_python(tags) if isinstance(tag, str))
|
||||
|
||||
|
||||
def _is_non_finite(value: object) -> bool:
|
||||
return isinstance(value, float) and not math.isfinite(value)
|
||||
|
||||
|
||||
def _scalar_metadata(metadata: object) -> Iterator[tuple[str, CustomMetadataValue]]:
|
||||
if isinstance(metadata, dict):
|
||||
yield from (
|
||||
(key, value)
|
||||
for key, value in _OBJECT_MAPPING_ADAPTER.validate_python(metadata).items()
|
||||
if isinstance(key, str) and isinstance(value, (str, int, float, bool)) and not _is_non_finite(value)
|
||||
)
|
||||
|
||||
|
||||
def _custom_tag_values(model_infos: Sequence[Mapping[object, object]]) -> Iterator[str]:
|
||||
for model_info in model_infos:
|
||||
yield from _string_tags(model_info.get("custom_tags"))
|
||||
|
||||
|
||||
def _custom_metadata_entries(
|
||||
model_infos: Sequence[Mapping[object, object]],
|
||||
) -> Iterator[tuple[str, CustomMetadataValue]]:
|
||||
for model_info in model_infos:
|
||||
yield from _scalar_metadata(model_info.get("custom_metadata"))
|
||||
|
||||
|
||||
def _merged_custom_tags(model_infos: Sequence[Mapping[object, object]]) -> tuple[str, ...] | None:
|
||||
return tuple(dict.fromkeys(_custom_tag_values(model_infos))) or None
|
||||
|
||||
|
||||
def _merged_custom_metadata(
|
||||
model_infos: Sequence[Mapping[object, object]],
|
||||
) -> Mapping[str, CustomMetadataValue] | None:
|
||||
entries: Final = tuple(_custom_metadata_entries(model_infos))
|
||||
first_seen: Final = MappingProxyType(dict(reversed(entries)))
|
||||
return MappingProxyType({key: first_seen[key] for key in dict.fromkeys(key for key, _ in entries)}) or None
|
||||
|
||||
|
||||
def _model_info(deployment: Mapping[str, object]) -> Mapping[object, object]:
|
||||
model_info: Final = deployment.get("model_info")
|
||||
return _OBJECT_MAPPING_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else MappingProxyType({})
|
||||
|
||||
|
||||
def model_group_custom_fields(deployments: Sequence[Mapping[str, object]]) -> Mapping[str, object]:
|
||||
model_infos: Final = tuple(_model_info(deployment) for deployment in deployments)
|
||||
return MappingProxyType(
|
||||
{
|
||||
"custom_tags": _merged_custom_tags(model_infos),
|
||||
"custom_metadata": _merged_custom_metadata(model_infos),
|
||||
}
|
||||
)
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import asyncio
|
||||
import json
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import timedelta
|
||||
from types import MappingProxyType
|
||||
|
|
@ -23,8 +23,6 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
|
|||
)
|
||||
from litellm.proxy.common_utils.user_api_key_cache import (
|
||||
UserApiKeyCache,
|
||||
team_membership_auth_cache_key,
|
||||
team_membership_reservation_cache_key,
|
||||
)
|
||||
from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
|
||||
from litellm.proxy.management_endpoints.key_management_endpoints import (
|
||||
|
|
@ -33,6 +31,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import (
|
|||
from litellm.proxy.management_endpoints.team_endpoints import (
|
||||
_emit_team_members_metric, # pyright: ignore[reportPrivateUsage] # same gauge /team/member_add moves
|
||||
_schedule_team_membership_audit_log, # pyright: ignore[reportPrivateUsage] # same audit row /team/member_add writes
|
||||
member_cache_keys,
|
||||
)
|
||||
from litellm.proxy.management_helpers.access_group_team_sync import (
|
||||
DETACH_TEAM_FROM_USERS_SQL,
|
||||
|
|
@ -277,7 +276,7 @@ async def _settle_caches(
|
|||
evicted: Final = (
|
||||
*deleted_tokens,
|
||||
*removal.jwt_mapping_cache_keys,
|
||||
*_member_cache_keys(team.team_id, changed_user_ids),
|
||||
*member_cache_keys(team.team_id, changed_user_ids),
|
||||
)
|
||||
evict_local(cache_keys=evicted, user_api_key_cache=user_api_key_cache)
|
||||
await evict_shared(cache_keys=evicted, cache=user_api_key_cache)
|
||||
|
|
@ -292,13 +291,6 @@ async def _settle_caches(
|
|||
await broadcast_evictions(cache_keys=evicted)
|
||||
|
||||
|
||||
def _member_cache_keys(team_id: str, user_ids: Sequence[str]) -> Iterator[str]:
|
||||
for user_id in user_ids:
|
||||
yield user_id
|
||||
yield team_membership_auth_cache_key(team_id=team_id, user_id=user_id)
|
||||
yield team_membership_reservation_cache_key(user_id=user_id, team_id=team_id)
|
||||
|
||||
|
||||
async def sync_team_roster(
|
||||
prisma_client: PrismaClient,
|
||||
team_id: str,
|
||||
|
|
|
|||
|
|
@ -477,6 +477,18 @@ async def get_authorized_credentials_for_model(
|
|||
)
|
||||
|
||||
|
||||
def deployment_drop_params(
|
||||
llm_router: Optional["Router"], model_id: str, request: Mapping[str, object]
|
||||
) -> Mapping[str, bool | str]:
|
||||
deployment: Final = (
|
||||
llm_router.get_credential_deployment(model_id=model_id)
|
||||
if llm_router is not None and "drop_params" not in request
|
||||
else None
|
||||
)
|
||||
drop_params: Final = deployment.litellm_params.drop_params if deployment is not None else None
|
||||
return MappingProxyType({}) if drop_params is None else MappingProxyType({"drop_params": drop_params})
|
||||
|
||||
|
||||
def _key_allowlist_covers(public_model_name: str, key_model_allowlist: Sequence[str]) -> bool:
|
||||
from litellm.proxy.auth.auth_checks import model_matches_any_wildcard_pattern_in_list
|
||||
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( # noqa: F401 #
|
|||
add_internal_model_credentials,
|
||||
apply_team_provider_credentials,
|
||||
authorize_model_for_key,
|
||||
deployment_drop_params,
|
||||
encode_file_id_with_model,
|
||||
extract_file_creation_params,
|
||||
get_authorized_credentials_for_model,
|
||||
|
|
@ -457,7 +458,7 @@ async def route_create_file(
|
|||
# Merge credentials into the request
|
||||
prepare_data_with_credentials(
|
||||
data=_create_file_request,
|
||||
credentials=credentials,
|
||||
credentials={**credentials, **deployment_drop_params(llm_router, model, _create_file_request)},
|
||||
)
|
||||
|
||||
# Create the file with model credentials
|
||||
|
|
|
|||
|
|
@ -880,8 +880,8 @@ def _build_passthrough_failure_request_payload(
|
|||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _TeamCallbackWiring:
|
||||
success_callbacks: "list[str | Callable | CustomLogger] | None" = None # mutable-ok: Logging.__init__ arg
|
||||
failure_callbacks: "list[str | Callable | CustomLogger] | None" = None # mutable-ok: Logging.__init__ arg
|
||||
success_callbacks: "list[str | Callable[..., object] | CustomLogger] | None" = None # mutable-ok: Logging init arg
|
||||
failure_callbacks: "list[str | Callable[..., object] | CustomLogger] | None" = None # mutable-ok: Logging init arg
|
||||
logging_kwargs: dict[str, str | dict[str, str]] | None = None # mutable-ok: Logging.__init__ arg
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -166,6 +166,7 @@ from litellm.proxy.common_utils.token_count_settings import (
|
|||
resolve_token_count_settings,
|
||||
)
|
||||
from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog
|
||||
from litellm.proxy.management_helpers.model_group_custom_fields import model_group_custom_fields
|
||||
from litellm.router_utils.access_windows import access_windows_config_error
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
get_fallback_errors_from_headers,
|
||||
|
|
@ -975,6 +976,7 @@ from litellm.secret_managers.main import (
|
|||
)
|
||||
from litellm.tracing.config import is_lens_tracing_enabled
|
||||
from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingArgs
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
|
||||
from litellm.types.llms.anthropic import (
|
||||
AnthropicMessagesRequest,
|
||||
AnthropicResponse,
|
||||
|
|
@ -17747,10 +17749,7 @@ def get_model_group_info(
|
|||
) -> list[ModelGroupInfoProxy]:
|
||||
model_groups: Final[list[ModelGroupInfoProxy]] = []
|
||||
|
||||
unique_models: Final = []
|
||||
for model in all_models_str:
|
||||
if model not in unique_models:
|
||||
unique_models.append(model)
|
||||
unique_models: Final = tuple(dict.fromkeys(all_models_str))
|
||||
|
||||
for model in unique_models:
|
||||
if model_group is not None and model_group != model:
|
||||
|
|
@ -17759,7 +17758,18 @@ def get_model_group_info(
|
|||
_model_group_info = llm_router.get_model_group_info(model_group=model, team_id=team_id)
|
||||
|
||||
if _model_group_info is not None:
|
||||
model_groups.append(ModelGroupInfoProxy.model_validate(_model_group_info.model_dump()))
|
||||
model_groups.append(
|
||||
ModelGroupInfoProxy.model_validate(
|
||||
{
|
||||
**_model_group_info.model_dump(),
|
||||
**model_group_custom_fields(
|
||||
llm_router.get_model_list_of_routed_group(
|
||||
resolve_model_group_alias(llm_router.model_group_alias, model) or model
|
||||
)
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
else:
|
||||
model_group_info = ModelGroupInfoProxy(
|
||||
model_group=model,
|
||||
|
|
@ -18210,6 +18220,7 @@ async def async_queue_request(
|
|||
try:
|
||||
data = await request.json()
|
||||
data.pop("_litellm_strip_stream_usage", None)
|
||||
data.pop(STREAM_USAGE_INJECTED_FIELD, None)
|
||||
|
||||
# Include original request and headers in the data
|
||||
data["proxy_server_request"] = {
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
|||
from litellm.proxy._types import (
|
||||
CommonProxyErrors,
|
||||
)
|
||||
from litellm.proxy.db.health_check_latest import latest_by_model_name
|
||||
from litellm.proxy.utils import get_custom_url
|
||||
from litellm.repositories.table_repositories import ClaudeCodePluginRepository
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import FusePresetCatalog, get_fuse_presets
|
||||
|
|
@ -295,13 +296,8 @@ async def public_model_hub():
|
|||
if prisma_client is not None:
|
||||
try:
|
||||
latest_checks: Final = await prisma_client.get_all_latest_health_checks()
|
||||
for check in latest_checks:
|
||||
key = check.model_id if check.model_id else check.model_name
|
||||
if key:
|
||||
health_check_dict = convert_health_check_to_dict(check)
|
||||
health_checks_map[key] = health_check_dict
|
||||
if check.model_name:
|
||||
health_checks_map[check.model_name] = health_check_dict
|
||||
for model_name, check in latest_by_model_name(latest_checks).items():
|
||||
health_checks_map[model_name] = convert_health_check_to_dict(check)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import litellm
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.db.health_check_latest import latest_by_model_name
|
||||
from litellm.proxy.list_api.common import PROBLEM_TYPE_BASE, ManagementProblem
|
||||
from litellm.proxy.list_api.in_memory import Cells, InMemoryListExecutor
|
||||
from litellm.proxy.list_api.list_framework import (
|
||||
|
|
@ -59,12 +60,12 @@ class PrismaHealthSnapshotLookup:
|
|||
checks: Final = await self.prisma_client.get_latest_health_checks_for_models(model_groups)
|
||||
return MappingProxyType(
|
||||
{
|
||||
check.model_name: HealthSnapshot(
|
||||
model_group: HealthSnapshot(
|
||||
status=check.status,
|
||||
response_time_ms=check.response_time_ms,
|
||||
checked_at=check.checked_at.isoformat() if check.checked_at else None,
|
||||
)
|
||||
for check in checks
|
||||
for model_group, check in latest_by_model_name(checks).items()
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,13 @@
|
|||
import asyncio
|
||||
from collections.abc import Mapping
|
||||
import hashlib
|
||||
import json
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Annotated, Any, Final, Literal
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
from fastapi import HTTPException, status
|
||||
|
|
@ -10,6 +15,7 @@ from pydantic import BeforeValidator, TypeAdapter, ValidationError
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
|
||||
from litellm.router_utils.common_utils import is_proxy_admin_request
|
||||
|
||||
|
|
@ -54,7 +60,34 @@ _ROUTER_ONLY_SETTINGS: Final = frozenset(
|
|||
)
|
||||
|
||||
|
||||
def _route_user_config_request(data: dict, route_type: str):
|
||||
class UserConfigRouters:
|
||||
def __init__(self, max_size: int) -> None:
|
||||
self._max_size: Final = max_size
|
||||
self._routers: Final[OrderedDict[str, LitellmRouter]] = OrderedDict() # mutable-ok: LRU evicts in place
|
||||
|
||||
def get(self, config: Mapping[str, object], build: Callable[[], LitellmRouter]) -> LitellmRouter:
|
||||
key: Final = hashlib.sha256(json.dumps(config, sort_keys=True, default=str).encode()).hexdigest()
|
||||
cached: Final = self._routers.get(key)
|
||||
if cached is not None:
|
||||
self._routers.move_to_end(key)
|
||||
return cached
|
||||
router: Final = build()
|
||||
router.serve_through_request_logging()
|
||||
self._routers[key] = router
|
||||
if len(self._routers) > self._max_size:
|
||||
self._routers.popitem(last=False)[1].discard()
|
||||
return router
|
||||
|
||||
def clear(self) -> None:
|
||||
for router in self._routers.values():
|
||||
router.discard()
|
||||
self._routers.clear()
|
||||
|
||||
|
||||
user_config_routers: Final = UserConfigRouters(max_size=32)
|
||||
|
||||
|
||||
async def _route_user_config_request(data: dict, route_type: str):
|
||||
"""Route a request using the user-provided router config."""
|
||||
router_config: Final = data.pop("user_config")
|
||||
|
||||
|
|
@ -63,10 +96,24 @@ def _route_user_config_request(data: dict, route_type: str):
|
|||
valid_args: Final = litellm.Router.get_valid_args()
|
||||
filtered_config: Final = {k: v for k, v in router_config.items() if k in valid_args}
|
||||
|
||||
user_router: Final = litellm.Router(**filtered_config)
|
||||
ret_val: Final = getattr(user_router, f"{route_type}")(**data)
|
||||
user_router.discard()
|
||||
return ret_val
|
||||
logging_obj: Final = _request_logging_obj(JSON_OBJECT_ADAPTER.validate_python(data), route_type)
|
||||
cached_router: Final = user_config_routers.get(filtered_config, lambda: litellm.Router(**filtered_config))
|
||||
cached_router.bind_to_request_logging(logging_obj)
|
||||
return await getattr(cached_router, f"{route_type}")(**{**data, "litellm_logging_obj": logging_obj})
|
||||
|
||||
|
||||
def _request_logging_obj(data: Mapping[str, object], route_type: str) -> LiteLLMLoggingObj:
|
||||
arrived_with: Final = data.get("litellm_logging_obj")
|
||||
if isinstance(arrived_with, LiteLLMLoggingObj):
|
||||
return arrived_with
|
||||
logging_obj, _ = litellm.utils.function_setup(
|
||||
original_function=route_type,
|
||||
rules_obj=litellm.utils.Rules(),
|
||||
start_time=datetime.now(), # noqa: DTZ005 # Logging subtracts it from naive end times
|
||||
is_async_call=True,
|
||||
**{"litellm_call_id": str(uuid4()), **data},
|
||||
)
|
||||
return logging_obj
|
||||
|
||||
|
||||
def _is_a2a_agent_model(model_name: object) -> bool:
|
||||
|
|
@ -111,6 +158,7 @@ ROUTE_ENDPOINT_MAPPING: Final = {
|
|||
"asearch": "/search",
|
||||
"adecisions": "/decisions",
|
||||
"asystemone": "/systemone",
|
||||
"aextraction": "/extraction",
|
||||
"avideo_generation": "/videos",
|
||||
"avideo_list": "/videos",
|
||||
"avideo_status": "/videos/{video_id}",
|
||||
|
|
@ -573,6 +621,7 @@ RouteType = Literal[
|
|||
"asearch",
|
||||
"adecisions",
|
||||
"asystemone",
|
||||
"aextraction",
|
||||
"avideo_generation",
|
||||
"avideo_list",
|
||||
"avideo_status",
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import (
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY,
|
||||
CALLER_SESSION_ID_METADATA_KEY,
|
||||
CLI_SESSION_KEY_PREFIX,
|
||||
EMPTY_MAPPING,
|
||||
|
|
@ -187,6 +188,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset(
|
|||
"used_client_oauth_token",
|
||||
KUBERNETES_POD_ROUTING_KEY,
|
||||
"litellm_roi_estimator",
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY,
|
||||
"user_api_key_user_email",
|
||||
)
|
||||
)
|
||||
|
|
@ -249,6 +251,7 @@ def _get_spend_logs_metadata(
|
|||
guardrail_information=None,
|
||||
internal_call_origin=None,
|
||||
litellm_roi_estimator=False,
|
||||
billed_retry_attempt=False,
|
||||
eval_information=None,
|
||||
cold_storage_object_key=cold_storage_object_key,
|
||||
litellm_overhead_time_ms=None,
|
||||
|
|
@ -270,6 +273,8 @@ def _get_spend_logs_metadata(
|
|||
)
|
||||
verbose_proxy_logger.debug("getting payload for SpendLogs, available keys in metadata: %s", [*metadata])
|
||||
|
||||
retry_key: Final = BILLED_RETRY_ATTEMPT_METADATA_KEY
|
||||
billed_retry_attempt: Final = retry_key in metadata and metadata[retry_key] is True
|
||||
# Filter the metadata dictionary to include only the specified keys
|
||||
clean_metadata: Final = SpendLogsMetadata(
|
||||
**MappingProxyType(
|
||||
|
|
@ -283,6 +288,7 @@ def _get_spend_logs_metadata(
|
|||
used_client_oauth_token=used_client_oauth_token,
|
||||
kubernetes_pod_routing=kubernetes_pod_routing,
|
||||
litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True,
|
||||
billed_retry_attempt=billed_retry_attempt,
|
||||
user_api_key_user_email=_stored_user_email(metadata),
|
||||
)
|
||||
_raw_key: Final = clean_metadata.get("user_api_key")
|
||||
|
|
@ -453,7 +459,12 @@ def _sl_attribution_fallback(
|
|||
|
||||
|
||||
def _deployment_provider(deployment: DeploymentTypedDict) -> str | None:
|
||||
litellm_params: Final = LiteLLM_Params.model_validate(deployment["litellm_params"])
|
||||
litellm_params: Final[LiteLLM_Params] = LiteLLM_Params.model_validate(deployment["litellm_params"])
|
||||
return deployment_params_provider(litellm_params)
|
||||
|
||||
|
||||
def deployment_params_provider(litellm_params: LiteLLM_Params) -> str | None:
|
||||
"""The provider a deployment's ``litellm_params`` authenticate against, None when its model cannot be resolved."""
|
||||
if litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default(litellm_params=litellm_params):
|
||||
return LlmProviders.LITELLM_PROXY.value
|
||||
declared: Final = declared_authenticating_provider(litellm_params.model, litellm_params.custom_llm_provider)
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
)
|
||||
from litellm.litellm_core_utils.duration_parser import get_budget_window_start
|
||||
from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging, deployment_pricing_model_info
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, Logging, deployment_pricing_model_info
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
|
|
@ -1161,6 +1161,43 @@ def _timed_out_earlier_records(
|
|||
return tuple(record for identity, record in by_identity.items() if identity != final_identity)
|
||||
|
||||
|
||||
def _record_pricing(
|
||||
model_call_details: Mapping[str, object], record: Mapping[str, object], base_model: str | None
|
||||
) -> tuple[Mapping[str, object], str | None]:
|
||||
"""The ``(model_call_details, base_model)`` to price one earlier attempt with: the record's own
|
||||
deployment model and ``base_model`` when it served a different model than the final attempt
|
||||
(the provider is then inferred from that model), else the final attempt's."""
|
||||
record_model: Final = record.get("model")
|
||||
if not isinstance(record_model, str) or record_model == model_call_details.get("model"):
|
||||
return model_call_details, base_model
|
||||
record_base_model: Final = record.get("base_model")
|
||||
return (
|
||||
{**model_call_details, "model": record_model, "custom_llm_provider": None},
|
||||
record_base_model if isinstance(record_base_model, str) else None,
|
||||
)
|
||||
|
||||
|
||||
def _billed_earlier_attempt(
|
||||
model_call_details: Mapping[str, object],
|
||||
record: Mapping[str, object],
|
||||
index: int,
|
||||
estimated_usage: Usage,
|
||||
base_model: str | None,
|
||||
) -> BilledFailedAttempt:
|
||||
pricing_details, pricing_base_model = _record_pricing(model_call_details, record, base_model)
|
||||
return BilledFailedAttempt(
|
||||
attempted_retries=_attempt_number(record, index),
|
||||
deployment_id=_record_deployment_id(record),
|
||||
usage=estimated_usage,
|
||||
response_cost=_estimated_input_cost(
|
||||
pricing_details,
|
||||
estimated_usage.prompt_tokens,
|
||||
deployment_id=_record_deployment_id(record),
|
||||
base_model=pricing_base_model,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _billed_earlier_attempts(
|
||||
model_call_details: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object] | None,
|
||||
|
|
@ -1174,23 +1211,14 @@ def _billed_earlier_attempts(
|
|||
request's own deployment and retry count (``attempted_retries`` restarts at 0 after a fallback
|
||||
hop, so the count alone cannot tell the groups apart). An earlier attempt that ended in a
|
||||
``Timeout`` had the same prompt accepted by its deployment, so it is billed like the final
|
||||
attempt, at that deployment's input rate (else the cost map's rate for ``base_model`` or the
|
||||
model). Only the exception class survives on the record, so an earlier connect timeout is
|
||||
billed too; any other failure class is not."""
|
||||
attempt, at that deployment's input rate (else the cost map's rate for the record's own
|
||||
``base_model`` or model, which a fallback hop can make a different model than the final one).
|
||||
Only the exception class survives on the record, so an earlier connect timeout is billed too;
|
||||
any other failure class is not."""
|
||||
final_attempt: Final = _retry_metadata_bucket(litellm_params).get("attempted_retries")
|
||||
final_identity: Final = (deployment_id, final_attempt if isinstance(final_attempt, int) else None)
|
||||
return tuple(
|
||||
BilledFailedAttempt(
|
||||
attempted_retries=_attempt_number(record, index),
|
||||
deployment_id=_record_deployment_id(record),
|
||||
usage=estimated_usage,
|
||||
response_cost=_estimated_input_cost(
|
||||
model_call_details,
|
||||
estimated_usage.prompt_tokens,
|
||||
deployment_id=_record_deployment_id(record),
|
||||
base_model=base_model,
|
||||
),
|
||||
)
|
||||
_billed_earlier_attempt(model_call_details, record, index, estimated_usage, base_model)
|
||||
for index, record in enumerate(_timed_out_earlier_records(litellm_params, final_identity))
|
||||
)
|
||||
|
||||
|
|
@ -4704,19 +4732,25 @@ class ProxyLogging:
|
|||
|
||||
@staticmethod
|
||||
def _discard_deferred_stream_logging_for_failure(request_data: Mapping[str, object], error: Exception) -> bool:
|
||||
"""Drop the parked success dispatch for an assembled chat stream that ends in an error
|
||||
``post_call_failure_hook`` logs as a failure, billing its usage on the failure row instead.
|
||||
"""Drop the parked success dispatch for an assembled chat stream or a stream that billed several rounds when it
|
||||
ends in an error ``post_call_failure_hook`` logs as a failure, billing its usage on the failure row instead.
|
||||
Returns False when the parked dispatch should still be flushed by the caller."""
|
||||
logging_obj: Final = request_data.get("litellm_logging_obj")
|
||||
if not isinstance(logging_obj, Logging):
|
||||
return False
|
||||
_args: Final[tuple[object, ...] | None] = getattr(logging_obj, "_deferred_stream_complete_args", None)
|
||||
assembled: Final = _args[0] if _args else None
|
||||
if not isinstance(error, _PROXY_ONLY_LLM_API_ERRORS) or not isinstance(assembled, ModelResponse):
|
||||
if not isinstance(error, _PROXY_ONLY_LLM_API_ERRORS):
|
||||
return False
|
||||
match _args:
|
||||
case (ModelResponse() as assembled, *_):
|
||||
logging_obj.record_assembled_response_for_failure(assembled)
|
||||
case (BilledRoundsDispatch() as billed,):
|
||||
billed.dispatch.close()
|
||||
logging_obj.record_billed_rounds_for_failure(billed.spend)
|
||||
case _:
|
||||
return False
|
||||
logging_obj._on_deferred_stream_complete = None
|
||||
logging_obj._deferred_stream_complete_args = None
|
||||
logging_obj.record_assembled_response_for_failure(assembled)
|
||||
return True
|
||||
|
||||
async def arelease_max_parallel_requests_on_disconnect(
|
||||
|
|
@ -4804,6 +4838,7 @@ def jsonify_object(data: dict) -> dict:
|
|||
_deprecated_key_cache: Final[LimitedSizeOrderedDict] = LimitedSizeOrderedDict(max_size=1000)
|
||||
_DEPRECATED_KEY_CACHE_TTL_SECONDS: Final = 60
|
||||
_PRISMA_DEFAULT_TX_TIMEOUT: Final = timedelta(seconds=5)
|
||||
_PRISMA_DEFAULT_TX_MAX_WAIT: Final = timedelta(seconds=2)
|
||||
|
||||
|
||||
async def _lookup_deprecated_key(
|
||||
|
|
@ -5210,13 +5245,17 @@ class PrismaClient:
|
|||
return self.db.read_target
|
||||
return self.db
|
||||
|
||||
def tx(self, *, timeout: timedelta = _PRISMA_DEFAULT_TX_TIMEOUT) -> "TransactionManager":
|
||||
def tx(
|
||||
self, *, timeout: timedelta = _PRISMA_DEFAULT_TX_TIMEOUT, max_wait: timedelta = _PRISMA_DEFAULT_TX_MAX_WAIT
|
||||
) -> "TransactionManager":
|
||||
"""Open an interactive transaction on the writer.
|
||||
|
||||
Callers go through this instead of reaching into ``self.db`` so writer
|
||||
selection and read-replica routing stay encapsulated in the wrapper.
|
||||
"""
|
||||
return cast("TransactionManager", self.db.tx(timeout=timeout)) # cast-ok: untyped __getattr__ delegate
|
||||
return cast( # cast-ok: untyped __getattr__ delegate
|
||||
"TransactionManager", self.db.tx(max_wait=max_wait, timeout=timeout)
|
||||
)
|
||||
|
||||
def get_request_status(self, payload: dict | SpendLogsPayload) -> Literal["success", "failure"]:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@ from itertools import filterfalse
|
|||
from typing import Any, Final, cast
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params
|
||||
from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
|
||||
from litellm.main import stream_chunk_builder
|
||||
from litellm.responses.litellm_completion_transformation.custom_tools import (
|
||||
build_tool_call_item_kwargs,
|
||||
|
|
@ -1364,6 +1365,17 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
return False
|
||||
return not any(getattr(part, "text", None) for part in getattr(item, "content", None) or ())
|
||||
|
||||
def finished_round_event(self) -> ResponseCompletedEvent | ResponseIncompleteEvent | None:
|
||||
if not isinstance(self.litellm_model_response, ModelResponse):
|
||||
return None
|
||||
finished: Final = self._emit_terminal_response_event(self.litellm_model_response)
|
||||
if finished is not None and served_from_cache(self.litellm_custom_stream_wrapper):
|
||||
finished.response.hidden_params = {**finished.response.hidden_params, "cache_hit": True}
|
||||
return finished
|
||||
|
||||
def bill_failure_after_earlier_rounds(self, earlier: BilledRoundsSpend) -> None:
|
||||
self.litellm_custom_stream_wrapper.bill_failure_after_earlier_rounds(earlier)
|
||||
|
||||
def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None:
|
||||
"""
|
||||
The response.incomplete event to bill when the client went away mid-stream.
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@ from typing import TYPE_CHECKING, Any, Final, cast
|
|||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.responses.litellm_completion_transformation.streaming_iterator import LiteLLMCompletionStreamingIterator
|
||||
from litellm.responses.mcp.request_context import MCPRequestContext
|
||||
from litellm.responses.mcp.stream_round_billing import StreamedRoundBilling
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
|
||||
from litellm.types.llms.openai import (
|
||||
BaseLiteLLMOpenAIResponseObject,
|
||||
|
|
@ -352,6 +354,9 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
self._round_max_output_index = -1
|
||||
self._composed_output: list[object] = [] # mutable-ok: grows as each round finishes
|
||||
self._pending_mcp_call_items: list[dict[str, object]] = [] # mutable-ok: grows per executed tool
|
||||
self._iteration_started = False
|
||||
self._round_billing: StreamedRoundBilling | None = None
|
||||
self._bridged_round_billed: LiteLLMCompletionStreamingIterator | None = None
|
||||
|
||||
def _extract_mcp_headers_from_params(self) -> None:
|
||||
"""Extract MCP headers from original request params to pass to tool calls"""
|
||||
|
|
@ -436,7 +441,15 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
return self
|
||||
|
||||
async def __anext__(self) -> ResponsesAPIStreamingResponse:
|
||||
chunk: Final = await self._anext_impl()
|
||||
if not self._iteration_started:
|
||||
self._iteration_started = True
|
||||
self._round_billing = self._bill_rounds_on_one_row()
|
||||
try:
|
||||
chunk: Final = await self._anext_impl()
|
||||
except Exception:
|
||||
if self._round_billing is not None:
|
||||
self._round_billing.finish()
|
||||
raise
|
||||
sequence_number: Final = getattr(chunk, "sequence_number", None)
|
||||
if isinstance(sequence_number, int):
|
||||
if sequence_number <= self._last_sequence_number and self._last_sequence_number > 0:
|
||||
|
|
@ -446,6 +459,49 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
self._last_sequence_number = max(self._last_sequence_number, sequence_number)
|
||||
return chunk
|
||||
|
||||
def __del__(self) -> None:
|
||||
round_billing: Final[object] = getattr(self, "_round_billing", None)
|
||||
if isinstance(round_billing, StreamedRoundBilling):
|
||||
round_billing.release_abandoned()
|
||||
|
||||
def _bill_rounds_on_one_row(self) -> StreamedRoundBilling | None:
|
||||
if not self.should_auto_execute:
|
||||
return None
|
||||
if isinstance(self.base_iterator, LiteLLMCompletionStreamingIterator):
|
||||
return StreamedRoundBilling(self.base_iterator.litellm_logging_obj)
|
||||
if isinstance(self.base_iterator, BaseResponsesAPIStreamingIterator):
|
||||
return StreamedRoundBilling(self.base_iterator.logging_obj)
|
||||
return None
|
||||
|
||||
def _restore_round_headers(self, logged_event: ResponsesAPIStreamingResponse) -> None:
|
||||
if isinstance(self.base_iterator, LiteLLMCompletionStreamingIterator):
|
||||
return
|
||||
if isinstance(self.base_iterator, BaseResponsesAPIStreamingIterator):
|
||||
self.base_iterator._restore_provider_response_headers(logged_event)
|
||||
|
||||
def _round_billed_early(self) -> bool:
|
||||
return self._bridged_round_billed is not None and self._bridged_round_billed is self.base_iterator
|
||||
|
||||
def _bill_finished_bridged_round(self) -> None:
|
||||
base: Final = self.base_iterator
|
||||
if self._round_billing is None or not isinstance(base, LiteLLMCompletionStreamingIterator):
|
||||
return
|
||||
if self._round_billed_early():
|
||||
return
|
||||
finished: Final = base.finished_round_event()
|
||||
if finished is None:
|
||||
return
|
||||
self._bridged_round_billed = base
|
||||
round_output: Final = _output_items(finished.response)
|
||||
self._round_billing.complete_round(
|
||||
finished,
|
||||
finished.response,
|
||||
round_output
|
||||
if finished.type == ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE
|
||||
else (*self._composed_output, *round_output),
|
||||
self._restore_round_headers,
|
||||
)
|
||||
|
||||
async def _anext_impl(self) -> ResponsesAPIStreamingResponse:
|
||||
"""
|
||||
Phase-based streaming:
|
||||
|
|
@ -617,6 +673,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
follow-up rounds past the items already emitted, and lists every
|
||||
round's items on the final response.completed.
|
||||
"""
|
||||
self._bill_finished_bridged_round()
|
||||
chunk_type: Final[object] = getattr(chunk, "type", None)
|
||||
if self._round_index > 0 and chunk_type in (
|
||||
ResponsesAPIStreamEvents.RESPONSE_CREATED,
|
||||
|
|
@ -630,12 +687,21 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
if self._output_index_offset:
|
||||
_set_event_field(chunk, "output_index", output_index + self._output_index_offset)
|
||||
|
||||
if self.should_auto_execute and chunk_type == ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE:
|
||||
return self._charge_incomplete_round(chunk)
|
||||
if not (self.should_auto_execute and self._is_response_completed(chunk)):
|
||||
return chunk
|
||||
|
||||
response_obj: Final[object] = getattr(chunk, "response", None)
|
||||
if isinstance(response_obj, ResponsesAPIResponse):
|
||||
self.collected_response = response_obj
|
||||
if self._round_billing is not None and not self._round_billed_early():
|
||||
self._round_billing.complete_round(
|
||||
chunk,
|
||||
response_obj,
|
||||
(*self._composed_output, *_output_items(response_obj)),
|
||||
self._restore_round_headers,
|
||||
)
|
||||
# Move to tool execution phase after this chunk
|
||||
self.phase = "tool_execution"
|
||||
await self._generate_tool_execution_events()
|
||||
|
|
@ -645,13 +711,33 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
if self._follow_up_pending():
|
||||
self._absorb_round(response_obj)
|
||||
return None
|
||||
if self._composed_output:
|
||||
merged_output: Final[list[object]] = [ # mutable-ok: the response model declares output as a list
|
||||
*self._composed_output,
|
||||
*_output_items(response_obj),
|
||||
]
|
||||
merged_response: Final = response_obj.model_copy(update={"output": merged_output})
|
||||
_set_event_field(chunk, "response", merged_response)
|
||||
merged_response: Final = (
|
||||
response_obj.model_copy(update={"output": [*self._composed_output, *_output_items(response_obj)]})
|
||||
if self._composed_output
|
||||
else response_obj
|
||||
)
|
||||
client_response: Final = (
|
||||
merged_response
|
||||
if self._round_billing is None
|
||||
else self._round_billing.charge_final_round(chunk, merged_response, self._restore_round_headers)
|
||||
)
|
||||
if client_response is not response_obj:
|
||||
_set_event_field(chunk, "response", client_response)
|
||||
return chunk
|
||||
|
||||
def _charge_incomplete_round(self, chunk: ResponsesAPIStreamingResponse) -> ResponsesAPIStreamingResponse:
|
||||
response_obj: Final[object] = getattr(chunk, "response", None)
|
||||
if self._round_billing is None or not isinstance(response_obj, ResponsesAPIResponse):
|
||||
return chunk
|
||||
if not self._round_billed_early():
|
||||
self._round_billing.complete_round(
|
||||
chunk, response_obj, _output_items(response_obj), self._restore_round_headers
|
||||
)
|
||||
client_response: Final = self._round_billing.charge_final_round(
|
||||
chunk, response_obj, self._restore_round_headers
|
||||
)
|
||||
if client_response is not response_obj:
|
||||
_set_event_field(chunk, "response", client_response)
|
||||
return chunk
|
||||
|
||||
async def _process_base_iterator_chunk(self) -> ResponsesAPIStreamingResponse:
|
||||
|
|
@ -949,6 +1035,8 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
self.collected_response = None
|
||||
self._cached_response_id = None
|
||||
self._round_index += 1
|
||||
if self._round_billing is not None:
|
||||
self._round_billing.record_billed_rounds(follow_up_response)
|
||||
|
||||
except Exception as e:
|
||||
verbose_logger.error("Error creating follow-up iterator: %s", e)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from contextlib import contextmanager
|
|||
from types import MappingProxyType
|
||||
from typing import Final, TypeVar
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from pydantic_core import to_jsonable_python
|
||||
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache, set_hidden_params
|
||||
|
|
@ -46,15 +46,18 @@ def _summed_mapping(first: Mapping[str, object], final: Mapping[str, object]) ->
|
|||
return MappingProxyType({key: _summed_entry(key, first, final) for key in keys})
|
||||
|
||||
|
||||
def _summed_usage(first: BaseModel, final: BaseModel) -> object:
|
||||
return to_jsonable_python(_summed_mapping(first.model_dump(), final.model_dump()), fallback=dict)
|
||||
|
||||
|
||||
def merged_round_usage(first: ResponseAPIUsage | None, final: ResponseAPIUsage | None) -> ResponseAPIUsage | None:
|
||||
if first is None or final is None:
|
||||
return final if first is None else first
|
||||
return ResponseAPIUsage.model_validate(_summed_mapping(first.model_dump(), final.model_dump()))
|
||||
return ResponseAPIUsage.model_validate(_summed_usage(first, final))
|
||||
|
||||
|
||||
def _merged_chat_usage(first: Usage, final: Usage) -> Usage:
|
||||
summed: Final = _summed_mapping(first.model_dump(), final.model_dump())
|
||||
return Usage.model_validate(to_jsonable_python(summed, fallback=dict))
|
||||
def merged_chat_usage(first: Usage, final: Usage) -> Usage:
|
||||
return Usage.model_validate(_summed_usage(first, final))
|
||||
|
||||
|
||||
def _chat_usage(round_response: ResponsesAPIResponse | ModelResponse) -> Usage | None:
|
||||
|
|
@ -82,7 +85,7 @@ def _usage_of_both_rounds(
|
|||
final_usage: Final = _chat_usage(final)
|
||||
if first_usage is None or final_usage is None:
|
||||
return final_usage if first_usage is None else first_usage
|
||||
return _merged_chat_usage(first_usage, final_usage)
|
||||
return merged_chat_usage(first_usage, final_usage)
|
||||
|
||||
|
||||
def _repeated_fixed_margin(
|
||||
|
|
@ -105,6 +108,14 @@ def _less_repeated_fixed_margin(summed: Mapping[str, object], repeated: float) -
|
|||
return MappingProxyType({**summed, **charged_once})
|
||||
|
||||
|
||||
def breakdown_less_repeated_fixed_margin(
|
||||
breakdown: CostBreakdown | None, repeated_fixed_margin: float
|
||||
) -> CostBreakdown | None:
|
||||
if breakdown is None:
|
||||
return None
|
||||
return _COST_BREAKDOWN.validate_python(_less_repeated_fixed_margin(breakdown, repeated_fixed_margin))
|
||||
|
||||
|
||||
def summed_cost_breakdown(
|
||||
first: ResponsesAPIResponse | ModelResponse,
|
||||
first_breakdown: CostBreakdown | None,
|
||||
|
|
|
|||
376
litellm/responses/mcp/stream_round_billing.py
Normal file
376
litellm/responses/mcp/stream_round_billing.py
Normal file
|
|
@ -0,0 +1,376 @@
|
|||
import asyncio
|
||||
from collections.abc import Callable, Coroutine, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Final, Protocol, TypeAlias
|
||||
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, BilledRoundsSpend, UncachedPrice
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.responses.mcp.round_billing import billed_for_every_round, merged_chat_usage, summed_cost_breakdown
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator, spawn_logging_task
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStreamingResponse
|
||||
from litellm.types.utils import CostBreakdown, ModelResponse, Usage
|
||||
|
||||
Parked: TypeAlias = tuple[object, ...]
|
||||
RestoresHeaders: TypeAlias = Callable[[ResponsesAPIStreamingResponse], None]
|
||||
|
||||
_PARKED_DISPATCH: Final = "_deferred_stream_complete_args"
|
||||
_DISPATCH_RELEASE: Final = "_on_deferred_stream_complete"
|
||||
_SERVED_FROM_CACHE: Final = UncachedPrice(cost=0.0, cost_breakdown=None)
|
||||
|
||||
|
||||
class _DispatchesSuccessHandlers(Protocol):
|
||||
async def dispatch_success_handlers(
|
||||
self,
|
||||
result: object = None,
|
||||
start_time: object = None,
|
||||
end_time: object = None,
|
||||
cache_hit: object = None,
|
||||
prefer_async_handlers: bool = False,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
def _success_dispatcher(logger: LiteLLMLoggingObj) -> _DispatchesSuccessHandlers:
|
||||
return logger
|
||||
|
||||
|
||||
def _event_loop_running() -> bool:
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _priced(response: ResponsesAPIResponse, price: UncachedPrice) -> ResponsesAPIResponse:
|
||||
priced: Final = response.model_copy()
|
||||
priced.hidden_params = {**response.hidden_params, "response_cost": price.cost}
|
||||
return priced
|
||||
|
||||
|
||||
def _assembled(parked: Parked | None) -> ModelResponse | None:
|
||||
match parked:
|
||||
case (ModelResponse() as assembled, bool()):
|
||||
return assembled
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _bridged_round_usage(parked: Parked | None) -> Usage | None:
|
||||
usage: Final[object] = getattr(_assembled(parked), "usage", None)
|
||||
return usage if isinstance(usage, Usage) else None
|
||||
|
||||
|
||||
def _usage_of_bridged_rounds(billed: Usage | None, round_usage: Usage | None, cost: object) -> Usage | None:
|
||||
if billed is None or round_usage is None:
|
||||
return None
|
||||
return merged_chat_usage(billed, round_usage).model_copy(update={"cost": cost})
|
||||
|
||||
|
||||
def _logged_cache_hit(response: ResponsesAPIResponse, chat_usage: Usage | None) -> bool | None:
|
||||
if chat_usage is not None:
|
||||
return served_from_cache(response)
|
||||
return served_from_cache(response) or None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _RoundRow:
|
||||
logged: ResponsesAPIStreamingResponse | ModelResponse
|
||||
cache_hit: bool | None
|
||||
end_time: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SuccessLog:
|
||||
logger: LiteLLMLoggingObj
|
||||
row: _RoundRow
|
||||
chat_usage: Usage | None
|
||||
|
||||
def carry_bridged_usage(self) -> None:
|
||||
if self.chat_usage is not None:
|
||||
self.logger.model_call_details["combined_usage_object"] = self.chat_usage
|
||||
|
||||
|
||||
def _round_row(
|
||||
completed: ResponsesAPIStreamingResponse,
|
||||
response: ResponsesAPIResponse,
|
||||
chat_usage: Usage | None,
|
||||
restore_headers: RestoresHeaders,
|
||||
) -> _RoundRow:
|
||||
logged_response: Final = response.model_copy()
|
||||
logged_response.hidden_params = {**response.hidden_params}
|
||||
logged: Final = completed.model_copy(update={"response": logged_response})
|
||||
restore_headers(logged)
|
||||
return _RoundRow(
|
||||
logged,
|
||||
_logged_cache_hit(response, chat_usage),
|
||||
datetime.now(), # noqa: DTZ005 # naive to match the logging pipeline start_time
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _BilledRounds:
|
||||
response: ResponsesAPIResponse
|
||||
breakdown: CostBreakdown | None
|
||||
chat_usage: Usage | None
|
||||
rounds: int = 1
|
||||
|
||||
@property
|
||||
def stamped_cost(self) -> float | None:
|
||||
cost: Final = self.response.hidden_params.get("response_cost")
|
||||
return float(cost) if isinstance(cost, (int, float)) else None
|
||||
|
||||
@property
|
||||
def cost(self) -> float:
|
||||
return self.stamped_cost or 0.0
|
||||
|
||||
@property
|
||||
def spend(self) -> BilledRoundsSpend:
|
||||
usage: Final = (
|
||||
ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(self.response.usage)
|
||||
if self.chat_usage is None
|
||||
else self.chat_usage
|
||||
)
|
||||
return BilledRoundsSpend(usage, self.cost, self.breakdown)
|
||||
|
||||
def with_round(
|
||||
self, priced: ResponsesAPIResponse, breakdown: CostBreakdown | None, chat_usage: Usage | None
|
||||
) -> "_BilledRounds":
|
||||
merged: Final = billed_for_every_round(self.response, self.breakdown, priced, breakdown)
|
||||
total: Final = merged.hidden_params.get("response_cost")
|
||||
charged: Final = merged.model_copy(
|
||||
update={"usage": None if merged.usage is None else merged.usage.model_copy(update={"cost": total})}
|
||||
)
|
||||
charged.hidden_params = {
|
||||
**merged.hidden_params,
|
||||
"cache_hit": served_from_cache(self.response) and served_from_cache(priced),
|
||||
}
|
||||
return _BilledRounds(
|
||||
charged,
|
||||
summed_cost_breakdown(self.response, self.breakdown, priced, breakdown),
|
||||
_usage_of_bridged_rounds(self.chat_usage, chat_usage, total),
|
||||
self.rounds + 1,
|
||||
)
|
||||
|
||||
def row(
|
||||
self,
|
||||
completed: ResponsesAPIStreamingResponse,
|
||||
client_output: Sequence[object],
|
||||
restore_headers: RestoresHeaders,
|
||||
) -> _RoundRow:
|
||||
return _round_row(
|
||||
completed,
|
||||
self.response.model_copy(update={"output": list(client_output)}),
|
||||
self.chat_usage,
|
||||
restore_headers,
|
||||
)
|
||||
|
||||
def bridged_row(self, assembled: ModelResponse | None, logged_id: str) -> _RoundRow | None:
|
||||
if assembled is None or self.chat_usage is None:
|
||||
return None
|
||||
logged: Final = assembled.model_copy(update={"id": logged_id, "usage": self.chat_usage})
|
||||
logged.hidden_params = {
|
||||
**(get_hidden_params(assembled) or {}),
|
||||
"response_cost": self.stamped_cost,
|
||||
}
|
||||
return _RoundRow(
|
||||
logged,
|
||||
served_from_cache(self.response),
|
||||
datetime.now(), # noqa: DTZ005 # naive to match the logging pipeline start_time
|
||||
)
|
||||
|
||||
|
||||
class StreamedRoundBilling:
|
||||
"""One success row per streamed auto-execute request, billed for every round it ran.
|
||||
|
||||
Each round's iterator parks its success dispatch on the shared logging object rather than sending it, either as a
|
||||
ready dispatch or, for a stream bridged through chat completions, as the assembled response and its cache hit.
|
||||
When a round completes, before its tools run, it is folded into the rounds before it and a dispatch for every
|
||||
round so far stays parked, so a client disconnect or a later round that fails still bills them. Once a later round
|
||||
starts, that dispatch logs from a copy of the logging state taken then, so the later round's failure, logged on the
|
||||
request's own state where the proxy reads its failure row, never reaches their success row. Only the latest
|
||||
dispatch is sent, through the proxy's deferred release when the proxy armed one."""
|
||||
|
||||
def __init__(self, logging_obj: LiteLLMLoggingObj) -> None:
|
||||
self._logging_obj: Final = logging_obj
|
||||
self._billed: _BilledRounds | None = None
|
||||
self._banked: Parked | None = None
|
||||
self._banked_row: _RoundRow | None = None
|
||||
if getattr(logging_obj, _DISPATCH_RELEASE, None) is None:
|
||||
setattr(logging_obj, _DISPATCH_RELEASE, self._run_parked)
|
||||
|
||||
def complete_round(
|
||||
self,
|
||||
completed: ResponsesAPIStreamingResponse,
|
||||
response: ResponsesAPIResponse,
|
||||
client_output: Sequence[object],
|
||||
restore_headers: RestoresHeaders,
|
||||
) -> None:
|
||||
round_parked: Final = self._take_parked()
|
||||
price: Final = self._price(response, round_parked)
|
||||
priced: Final = _priced(response, price)
|
||||
chat_usage: Final = _bridged_round_usage(round_parked)
|
||||
if self._billed is None:
|
||||
first_round: Final = _BilledRounds(priced, price.cost_breakdown, chat_usage)
|
||||
self._billed = first_round
|
||||
self._bank(
|
||||
round_parked,
|
||||
first_round.bridged_row(_assembled(round_parked), priced.id)
|
||||
or first_round.row(completed, client_output, restore_headers),
|
||||
)
|
||||
return
|
||||
billed: Final = self._billed.with_round(priced, price.cost_breakdown, chat_usage)
|
||||
self._close(round_parked)
|
||||
self._close(self._banked)
|
||||
self._logging_obj.model_call_details.pop("combined_usage_object", None)
|
||||
self._logging_obj.model_call_details.pop("response_cost", None)
|
||||
self._billed = billed
|
||||
self._logging_obj.cost_breakdown = billed.breakdown
|
||||
row: Final = billed.bridged_row(_assembled(round_parked), billed.response.id) or billed.row(
|
||||
completed, client_output, restore_headers
|
||||
)
|
||||
self._bank(None if round_parked is None else (self._dispatch(row, billed, self._logging_obj),), row)
|
||||
|
||||
def record_billed_rounds(self, follow_up: object) -> None:
|
||||
if self._billed is None or not isinstance(follow_up, BaseResponsesAPIStreamingIterator):
|
||||
return
|
||||
follow_up.bill_failure_after_earlier_rounds(self._billed.spend)
|
||||
self._bank_apart_from_later_rounds(self._billed)
|
||||
|
||||
def charge_final_round(
|
||||
self, completed: ResponsesAPIStreamingResponse, response: ResponsesAPIResponse, restore_headers: RestoresHeaders
|
||||
) -> ResponsesAPIResponse:
|
||||
parked: Final = self._take_parked()
|
||||
self._banked = None
|
||||
if self._billed is None or self._billed.rounds == 1:
|
||||
if parked is not None:
|
||||
self._release(parked)
|
||||
return response
|
||||
self._close(parked)
|
||||
charged: Final = response.model_copy(update={"usage": self._billed.response.usage})
|
||||
charged.hidden_params = {**self._billed.response.hidden_params}
|
||||
if parked is not None:
|
||||
final_row: Final = self._billed.bridged_row(self._latest_assembled(), charged.id) or _round_row(
|
||||
completed, charged, self._billed.chat_usage, restore_headers
|
||||
)
|
||||
self._release((self._dispatch(final_row, self._billed, self._logging_obj),))
|
||||
return charged
|
||||
|
||||
def finish(self) -> None:
|
||||
parked: Final = self._take_parked()
|
||||
self._banked = None
|
||||
if parked is not None:
|
||||
self._release(parked)
|
||||
return
|
||||
self._disarm()
|
||||
|
||||
def release_abandoned(self) -> None:
|
||||
if not self._owns_release():
|
||||
return
|
||||
if _event_loop_running():
|
||||
self.finish()
|
||||
return
|
||||
self.abandon()
|
||||
|
||||
def abandon(self) -> None:
|
||||
self._close(self._take_parked())
|
||||
self._close(self._banked)
|
||||
self._banked = None
|
||||
self._disarm()
|
||||
|
||||
def _latest_assembled(self) -> ModelResponse | None:
|
||||
row: Final = self._banked_row
|
||||
return row.logged if row is not None and isinstance(row.logged, ModelResponse) else None
|
||||
|
||||
def _price(self, response: ResponsesAPIResponse, round_parked: Parked | None) -> UncachedPrice:
|
||||
if served_from_cache(response):
|
||||
return _SERVED_FROM_CACHE
|
||||
match round_parked:
|
||||
case (ModelResponse() as assembled, bool()):
|
||||
return self._logging_obj.uncached_price(assembled)
|
||||
case _:
|
||||
return self._logging_obj.uncached_price(response)
|
||||
|
||||
def _bank(self, parked: Parked | None, row: _RoundRow) -> None:
|
||||
self._banked = parked
|
||||
self._banked_row = row
|
||||
if parked is not None:
|
||||
setattr(self._logging_obj, _PARKED_DISPATCH, parked)
|
||||
|
||||
def _bank_apart_from_later_rounds(self, billed: _BilledRounds) -> None:
|
||||
row: Final = self._banked_row
|
||||
if row is None:
|
||||
return
|
||||
parked: Final = self._take_parked()
|
||||
if parked is None:
|
||||
return
|
||||
self._close(parked)
|
||||
self._bank((self._dispatch(row, billed, self._logging_obj.attempt_scoped_copy()),), row)
|
||||
|
||||
def _dispatch(
|
||||
self, row: _RoundRow, billed: _BilledRounds, logger: LiteLLMLoggingObj
|
||||
) -> BilledRoundsDispatch | Coroutine[object, object, None]:
|
||||
dispatch: Final = self._log_success(_SuccessLog(logger, row, billed.chat_usage))
|
||||
if billed.chat_usage is None:
|
||||
return dispatch
|
||||
return BilledRoundsDispatch(dispatch, billed.spend)
|
||||
|
||||
@staticmethod
|
||||
async def _log_success(success: _SuccessLog) -> None:
|
||||
success.carry_bridged_usage()
|
||||
await _success_dispatcher(success.logger).dispatch_success_handlers(
|
||||
success.row.logged,
|
||||
start_time=success.logger.start_time,
|
||||
end_time=success.row.end_time,
|
||||
cache_hit=success.row.cache_hit,
|
||||
prefer_async_handlers=True,
|
||||
)
|
||||
|
||||
def _take_parked(self) -> Parked | None:
|
||||
parked: Final[Parked | None] = getattr(self._logging_obj, _PARKED_DISPATCH, None)
|
||||
if not isinstance(parked, tuple):
|
||||
return None
|
||||
setattr(self._logging_obj, _PARKED_DISPATCH, None)
|
||||
return parked
|
||||
|
||||
async def _run_parked(self, *parked: object) -> None:
|
||||
match parked:
|
||||
case (BilledRoundsDispatch(dispatch=dispatch),):
|
||||
await dispatch
|
||||
case (dispatch,) if asyncio.iscoroutine(dispatch):
|
||||
await dispatch
|
||||
case (assembled, cache_hit):
|
||||
await _success_dispatcher(self._logging_obj).dispatch_success_handlers(
|
||||
assembled, cache_hit=cache_hit, start_time=None, end_time=None, prefer_async_handlers=True
|
||||
)
|
||||
case _:
|
||||
return
|
||||
|
||||
def _owns_release(self) -> bool:
|
||||
release: Final[object] = getattr(self._logging_obj, _DISPATCH_RELEASE, None)
|
||||
return release == self._run_parked
|
||||
|
||||
def _release(self, parked: Parked) -> None:
|
||||
if getattr(self._logging_obj, _DISPATCH_RELEASE, None) is not None and not self._owns_release():
|
||||
setattr(self._logging_obj, _PARKED_DISPATCH, parked)
|
||||
return
|
||||
setattr(self._logging_obj, _DISPATCH_RELEASE, None)
|
||||
spawn_logging_task(
|
||||
asyncio.get_running_loop(), self._run_parked(*parked), task_name="Responses MCP stream success logging"
|
||||
)
|
||||
|
||||
def _disarm(self) -> None:
|
||||
if self._owns_release():
|
||||
setattr(self._logging_obj, _DISPATCH_RELEASE, None)
|
||||
|
||||
@staticmethod
|
||||
def _close(parked: Parked | None) -> None:
|
||||
match parked:
|
||||
case (BilledRoundsDispatch(dispatch=dispatch),):
|
||||
dispatch.close()
|
||||
case (dispatch,) if asyncio.iscoroutine(dispatch):
|
||||
dispatch.close()
|
||||
case _:
|
||||
return
|
||||
|
|
@ -42,6 +42,7 @@ from litellm.litellm_core_utils.asyncify import run_async_function
|
|||
from litellm.litellm_core_utils.core_helpers import process_response_headers
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base
|
||||
from litellm.litellm_core_utils.internal_call_metadata import without_budget_reservation
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers
|
||||
|
|
@ -364,6 +365,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
|
||||
_pending_logging_tasks: tuple[asyncio.Task[object], ...] = ()
|
||||
_started_tool_call: bool = False
|
||||
_earlier_rounds_spend: BilledRoundsSpend | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -708,18 +710,22 @@ class BaseResponsesAPIStreamingIterator:
|
|||
if usage_obj is None:
|
||||
return
|
||||
try:
|
||||
self.logging_obj.model_call_details["combined_usage_object"] = (
|
||||
ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage_obj)
|
||||
)
|
||||
failed_usage: Final = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage_obj)
|
||||
except (TypeError, ValueError) as usage_error:
|
||||
verbose_logger.debug(
|
||||
"could not record usage for failed responses stream: %s",
|
||||
usage_error,
|
||||
)
|
||||
return
|
||||
self.logging_obj.model_call_details["response_cost"] = (
|
||||
self.logging_obj.response_cost_calculator(result=response_obj) or 0.0
|
||||
)
|
||||
failed_cost: Final = self.logging_obj.response_cost_calculator(result=response_obj) or 0.0
|
||||
if self._earlier_rounds_spend is None:
|
||||
self.logging_obj.model_call_details["combined_usage_object"] = failed_usage
|
||||
self.logging_obj.model_call_details["response_cost"] = failed_cost
|
||||
return
|
||||
self.logging_obj.record_failure_after_earlier_rounds(self._earlier_rounds_spend, failed_usage, failed_cost)
|
||||
|
||||
def bill_failure_after_earlier_rounds(self, earlier: BilledRoundsSpend) -> None:
|
||||
self._earlier_rounds_spend = earlier
|
||||
|
||||
def _map_error_event_exception(self, error_obj: object) -> Exception:
|
||||
return _map_stream_error_to_exception(
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ from contextvars import ContextVar
|
|||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from functools import lru_cache, partial
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Generic, Literal, NoReturn, Optional, TypeAlias, TypeVar, Union, cast
|
||||
|
||||
|
|
@ -366,10 +367,11 @@ from litellm.router_utils.routing_groups import (
|
|||
validate_routing_strategy,
|
||||
)
|
||||
from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET, RoutingPrefetch, RoutingReadBatch
|
||||
from litellm.router_utils.stream_usage_injection import router_config_requests_stream_usage
|
||||
from litellm.scheduler import FlowItem, Scheduler
|
||||
from litellm.secret_managers.main import str_to_bool
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.litellm_params import RoutingStrategyName
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD, RoutingStrategyName
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionToolParam,
|
||||
|
|
@ -557,6 +559,7 @@ _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT")
|
|||
_CallbackT = TypeVar("_CallbackT")
|
||||
_DeploymentT = TypeVar("_DeploymentT", bound=Mapping[str, object])
|
||||
|
||||
|
||||
_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset(
|
||||
{"model", "api_base", "api_key", "api_version", "responses_api_version"}
|
||||
)
|
||||
|
|
@ -1051,6 +1054,8 @@ class RoutingArgs(enum.Enum):
|
|||
# as the per-request one built from a caller-supplied user_config, drops out on its
|
||||
# own rather than leaving entries behind that nothing can withdraw.
|
||||
_live_routers: Final["weakref.WeakSet[Router]"] = weakref.WeakSet()
|
||||
_routers_holding_global_callbacks: Final["weakref.WeakSet[Router]"] = weakref.WeakSet()
|
||||
_routers_holding_global_callbacks_lock: Final = threading.Lock()
|
||||
|
||||
|
||||
def _replay_live_router_model_cost() -> None:
|
||||
|
|
@ -1583,22 +1588,18 @@ class Router:
|
|||
self._init_routing_groups(self._routing_groups_input)
|
||||
self._override_selectors: dict[str, RouterStrategySelector | None] = {}
|
||||
self._override_selectors_lock = threading.Lock()
|
||||
self._request_scoped_selectors: tuple[CustomLogger, ...] = ()
|
||||
self._request_scoped_policies: tuple[CustomLogger, ...] = ()
|
||||
self.access_groups = None
|
||||
with _routers_holding_global_callbacks_lock:
|
||||
_routers_holding_global_callbacks.add(self)
|
||||
## USAGE TRACKING ##
|
||||
if isinstance(litellm._async_success_callback, list):
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(self.deployment_callback_on_success)
|
||||
else:
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(self.deployment_callback_on_success)
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(self.deployment_callback_on_success)
|
||||
if isinstance(litellm.success_callback, list):
|
||||
litellm.logging_callback_manager.add_litellm_success_callback(self.sync_deployment_callback_on_success)
|
||||
else:
|
||||
litellm.success_callback = [self.sync_deployment_callback_on_success]
|
||||
if isinstance(litellm._async_failure_callback, list):
|
||||
litellm.logging_callback_manager.add_litellm_async_failure_callback(
|
||||
self.async_deployment_callback_on_failure
|
||||
)
|
||||
else:
|
||||
litellm._async_failure_callback = [self.async_deployment_callback_on_failure]
|
||||
litellm.logging_callback_manager.add_litellm_async_failure_callback(self.async_deployment_callback_on_failure)
|
||||
## COOLDOWNS ##
|
||||
if isinstance(litellm.failure_callback, list):
|
||||
litellm.logging_callback_manager.add_litellm_failure_callback(self.deployment_callback_on_failure)
|
||||
|
|
@ -1694,20 +1695,116 @@ class Router:
|
|||
# Stop contributing to cost-map rebuilds straight away rather than waiting
|
||||
# for this router to be collected.
|
||||
_live_routers.discard(self)
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm._async_success_callback, self)
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.success_callback, self)
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm._async_failure_callback, self)
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.failure_callback, self)
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.input_callback, self)
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.service_callback, self)
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.callbacks, self)
|
||||
self.retire_routing_state()
|
||||
self._release_global_callbacks()
|
||||
|
||||
# Remove ForwardClientSideHeadersByModelGroup if it exists
|
||||
if self.optional_callbacks is not None:
|
||||
for callback in self.optional_callbacks:
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(
|
||||
litellm.callbacks, callback, require_self=False
|
||||
)
|
||||
def _release_global_callbacks(self) -> None:
|
||||
with _routers_holding_global_callbacks_lock:
|
||||
_routers_holding_global_callbacks.discard(self)
|
||||
slack_alerting_loggers: Final = self._slack_alerting_loggers()
|
||||
adaptive_router_hooks: Final = self._adaptive_router_hooks()
|
||||
vacated_keys: Final = litellm.logging_callback_manager.drop_callbacks(
|
||||
(
|
||||
*self._callback_selectors(),
|
||||
*self._override_selectors.values(),
|
||||
*(self.optional_callbacks or ()),
|
||||
*slack_alerting_loggers,
|
||||
*adaptive_router_hooks,
|
||||
),
|
||||
method_owners=(self, *slack_alerting_loggers),
|
||||
)
|
||||
with _routers_holding_global_callbacks_lock:
|
||||
survivors: Final = tuple(_routers_holding_global_callbacks)
|
||||
candidates: Final = chain.from_iterable(router._global_callback_candidates() for router in survivors)
|
||||
for candidate in litellm.logging_callback_manager.with_custom_logger_keys(candidates, vacated_keys):
|
||||
self._register_router_selector(candidate)
|
||||
adaptive_keys: Final = frozenset(
|
||||
map(litellm.logging_callback_manager.get_custom_logger_key, adaptive_router_hooks)
|
||||
)
|
||||
adaptive_survivor: Final = next((router for router in survivors if router.adaptive_routers), None)
|
||||
if adaptive_survivor is not None and vacated_keys & adaptive_keys:
|
||||
adaptive_survivor._sync_adaptive_router_hooks()
|
||||
|
||||
def serve_through_request_logging(self) -> None:
|
||||
self._request_scoped_selectors = (*self._callback_selectors(), *self._request_adaptive_router_hooks())
|
||||
self._request_scoped_policies = (*self._optional_loggers(), *self._slack_alerting_loggers())
|
||||
self._release_global_callbacks()
|
||||
|
||||
def bind_to_request_logging(self, logging_obj: LiteLLMLogging) -> None:
|
||||
logging_obj.add_dynamic_function_callbacks(
|
||||
async_success=self.deployment_callback_on_success,
|
||||
failure=self.deployment_callback_on_failure,
|
||||
async_failure=self.async_deployment_callback_on_failure,
|
||||
)
|
||||
for selector in self._request_scoped_selectors:
|
||||
logging_obj.replace_global_callback(selector)
|
||||
for policy in self._request_policies_not_set_globally():
|
||||
logging_obj.add_dynamic_callback(policy)
|
||||
|
||||
def _request_policies_not_set_globally(self) -> tuple[CustomLogger, ...]:
|
||||
global_keys: Final = frozenset(
|
||||
map(
|
||||
litellm.logging_callback_manager.get_custom_logger_key,
|
||||
(callback for callback in litellm.callbacks if isinstance(callback, CustomLogger)),
|
||||
)
|
||||
)
|
||||
return tuple(
|
||||
policy
|
||||
for policy in self._request_scoped_policies
|
||||
if litellm.logging_callback_manager.get_custom_logger_key(policy) not in global_keys
|
||||
)
|
||||
|
||||
def _request_adaptive_router_hooks(self) -> tuple[CustomLogger, ...]:
|
||||
from litellm.router_strategy.adaptive_router.hooks import (
|
||||
AdaptiveRouterPostCallHook,
|
||||
)
|
||||
|
||||
return tuple(
|
||||
AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy)
|
||||
for tagged in chain.from_iterable(self.adaptive_routers.values())
|
||||
)
|
||||
|
||||
def _routing_callbacks(self) -> Iterable[object]:
|
||||
if not (self._request_scoped_selectors or self._request_scoped_policies):
|
||||
return litellm.callbacks
|
||||
replaced_keys: Final = frozenset(
|
||||
map(litellm.logging_callback_manager.get_custom_logger_key, self._request_scoped_selectors)
|
||||
)
|
||||
return chain(
|
||||
litellm.logging_callback_manager.without_custom_logger_keys(litellm.callbacks, replaced_keys),
|
||||
self._request_scoped_selectors,
|
||||
self._request_policies_not_set_globally(),
|
||||
)
|
||||
|
||||
def retire_routing_state(self) -> None:
|
||||
for strategy in (*self._global_callback_candidates(), *self._override_selectors.values()):
|
||||
if isinstance(strategy, BaseRoutingStrategy | RouterBudgetLimiting):
|
||||
strategy.retire()
|
||||
|
||||
def _slack_alerting_loggers(self) -> tuple[CustomLogger, ...]:
|
||||
slack_alerting_logger: Final = getattr(self, "slack_alerting_logger", None)
|
||||
return (slack_alerting_logger,) if isinstance(slack_alerting_logger, CustomLogger) else ()
|
||||
|
||||
def _adaptive_router_hooks(self) -> tuple[CustomLogger, ...]:
|
||||
from litellm.router_strategy.adaptive_router.hooks import (
|
||||
AdaptiveRouterPostCallHook,
|
||||
)
|
||||
|
||||
own_strategy_ids: Final = frozenset(
|
||||
id(tagged.strategy) for tagged in chain.from_iterable(self.adaptive_routers.values())
|
||||
)
|
||||
hooks: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(AdaptiveRouterPostCallHook)
|
||||
return tuple(
|
||||
hook
|
||||
for hook in hooks
|
||||
if isinstance(hook, AdaptiveRouterPostCallHook) and id(hook.adaptive_router) in own_strategy_ids
|
||||
)
|
||||
|
||||
def _optional_loggers(self) -> tuple[CustomLogger, ...]:
|
||||
return tuple(callback for callback in self.optional_callbacks or () if isinstance(callback, CustomLogger))
|
||||
|
||||
def _global_callback_candidates(self) -> tuple[CustomLogger, ...]:
|
||||
return (*self._callback_selectors(), *self._optional_loggers(), *self._slack_alerting_loggers())
|
||||
|
||||
@staticmethod
|
||||
def _create_redis_cache(
|
||||
|
|
@ -1813,7 +1910,7 @@ class Router:
|
|||
return selector
|
||||
|
||||
@staticmethod
|
||||
def _register_router_selector(selector: RouterStrategySelector) -> None:
|
||||
def _register_router_selector(selector: CustomLogger) -> None:
|
||||
if isinstance(selector, LeastBusyLoggingHandler):
|
||||
if isinstance(litellm.input_callback, list):
|
||||
litellm.logging_callback_manager.add_litellm_input_callback(selector)
|
||||
|
|
@ -1822,23 +1919,28 @@ class Router:
|
|||
if isinstance(litellm.callbacks, list):
|
||||
litellm.logging_callback_manager.add_litellm_callback(selector)
|
||||
|
||||
def _callback_selectors(self) -> tuple[RouterStrategySelector, ...]:
|
||||
return tuple(
|
||||
selector
|
||||
for selector in (
|
||||
*(getattr(self, attr, None) for attr in self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.values()),
|
||||
*chain.from_iterable(selectors.values() for selectors in self._group_selectors.values()),
|
||||
)
|
||||
if selector is not None
|
||||
)
|
||||
|
||||
def _unregister_router_selectors(self, selectors: Sequence[object]) -> None:
|
||||
"""
|
||||
Drop router-owned strategy selectors from litellm's global callback
|
||||
lists by identity. Used before re-init (`routing_strategy_init` /
|
||||
`_init_routing_groups`) so repeated `update_settings` calls don't
|
||||
accumulate dead selectors that keep receiving callback events.
|
||||
lists. Used before re-init (`routing_strategy_init` /
|
||||
`_init_routing_groups`) and by `discard` so dead selectors stop
|
||||
receiving callback events. `function_setup` copies `litellm.callbacks`
|
||||
into the per-event lists on every request, so those are purged too.
|
||||
"""
|
||||
for selector in selectors:
|
||||
if isinstance(selector, BaseRoutingStrategy):
|
||||
selector.retire()
|
||||
selector_ids: Final = {id(s) for s in selectors if s is not None}
|
||||
if not selector_ids:
|
||||
return
|
||||
if isinstance(litellm.callbacks, list):
|
||||
litellm.callbacks = [c for c in litellm.callbacks if id(c) not in selector_ids]
|
||||
if isinstance(litellm.input_callback, list):
|
||||
litellm.input_callback = [c for c in litellm.input_callback if id(c) not in selector_ids]
|
||||
litellm.logging_callback_manager.drop_callbacks(selectors)
|
||||
|
||||
def _apply_updated_routing_strategy_args(self) -> None:
|
||||
"""
|
||||
|
|
@ -2530,6 +2632,11 @@ class Router:
|
|||
self.asystemone = self.factory_function(asystemone, call_type="asystemone") # pyright: ignore[reportUnknownMemberType] # factory_function takes a bare Callable
|
||||
self.systemone = self.factory_function(systemone, call_type="systemone") # pyright: ignore[reportUnknownMemberType] # factory_function takes a bare Callable
|
||||
|
||||
from litellm.extraction import aextraction, extraction
|
||||
|
||||
self.aextraction = self.factory_function(aextraction, call_type="aextraction") # pyright: ignore[reportUnknownMemberType] # factory_function takes a bare Callable
|
||||
self.extraction = self.factory_function(extraction, call_type="extraction") # pyright: ignore[reportUnknownMemberType] # factory_function takes a bare Callable
|
||||
|
||||
def _initialize_video_endpoints(self):
|
||||
"""Initialize video endpoints."""
|
||||
from litellm.videos import (
|
||||
|
|
@ -4657,7 +4764,8 @@ class Router:
|
|||
deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ())
|
||||
):
|
||||
kwargs.pop(key, None)
|
||||
self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=self.default_litellm_params)
|
||||
defaults: Final = _LITELLM_PARAMS_ADAPTER.validate_python(self.default_litellm_params)
|
||||
self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=defaults)
|
||||
|
||||
model_info = deployment.get("model_info", {}).copy()
|
||||
deployment_litellm_model_name = deployment["litellm_params"]["model"]
|
||||
|
|
@ -4801,6 +4909,19 @@ class Router:
|
|||
data=_DEPLOYMENT_PARAMS_ADAPTER.validate_python(deployment["litellm_params"]),
|
||||
)
|
||||
|
||||
deployment_litellm_params: Final = _DEPLOYMENT_PARAMS_ADAPTER.validate_python(deployment["litellm_params"])
|
||||
if router_config_requests_stream_usage(
|
||||
_REQUEST_KWARGS_ADAPTER.validate_python(kwargs),
|
||||
(
|
||||
_LITELLM_PARAMS_ADAPTER.validate_python(
|
||||
deployment_litellm_params
|
||||
if isinstance(deployment_litellm_params, Mapping)
|
||||
else deployment_litellm_params.model_dump(exclude_none=True)
|
||||
),
|
||||
defaults,
|
||||
),
|
||||
):
|
||||
kwargs[STREAM_USAGE_INJECTED_FIELD] = True
|
||||
self._update_kwargs_with_default_litellm_params(kwargs=kwargs, metadata_variable_name=metadata_variable_name)
|
||||
hop_kwargs: Final = _FALLBACK_HOP_ADAPTER.validate_python(kwargs)
|
||||
if _is_fallback_hop(hop_kwargs):
|
||||
|
|
@ -7699,6 +7820,8 @@ class Router:
|
|||
"decisions",
|
||||
"asystemone",
|
||||
"systemone",
|
||||
"aextraction",
|
||||
"extraction",
|
||||
"aadapter_generate_content",
|
||||
"avideo_generation",
|
||||
"video_generation",
|
||||
|
|
@ -7774,6 +7897,7 @@ class Router:
|
|||
"search",
|
||||
"decisions",
|
||||
"systemone",
|
||||
"extraction",
|
||||
"video_generation",
|
||||
"video_list",
|
||||
"video_status",
|
||||
|
|
@ -7943,6 +8067,7 @@ class Router:
|
|||
"ocr",
|
||||
"adecisions",
|
||||
"asystemone",
|
||||
"aextraction",
|
||||
"avideo_generation",
|
||||
"avideo_list",
|
||||
"avideo_status",
|
||||
|
|
@ -9492,17 +9617,24 @@ class Router:
|
|||
When a retry or fallback happens, record which model group, deployment and attempt just failed and why,
|
||||
and count it toward the request-wide num_retries_per_request cap
|
||||
"""
|
||||
from litellm.litellm_core_utils.get_litellm_params import get_base_model_from_litellm_call_metadata
|
||||
from litellm.types.router import RetryAttemptRecord
|
||||
|
||||
_metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"
|
||||
request_metadata: Final[Mapping[str, object]] = kwargs[_metadata_var]
|
||||
request_metadata: Final = cast("Mapping[str, object]", kwargs[_metadata_var]) # cast-ok: untyped kwargs
|
||||
model_group: Final = kwargs.get("model")
|
||||
model_info: Final = request_metadata.get("model_info")
|
||||
deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None
|
||||
base_model: Final = get_base_model_from_litellm_call_metadata(
|
||||
metadata=request_metadata
|
||||
) or self._deployment_params_base_model(deployment_id if isinstance(deployment_id, str) else None)
|
||||
deployment_model: Final = request_metadata.get("deployment")
|
||||
attempted_retries: Final = request_metadata.get("attempted_retries")
|
||||
attempt_record: Final[RetryAttemptRecord] = {
|
||||
"model_group": model_group if isinstance(model_group, str) else None,
|
||||
"deployment_id": deployment_id if isinstance(deployment_id, str) else None,
|
||||
"model": deployment_model if isinstance(deployment_model, str) else None,
|
||||
"base_model": base_model if isinstance(base_model, str) else None,
|
||||
"exception_type": type(e).__name__,
|
||||
"exception_string": str(e),
|
||||
"attempted_retries": attempted_retries if type(attempted_retries) is int else None,
|
||||
|
|
@ -9533,6 +9665,18 @@ class Router:
|
|||
kwargs[_metadata_var]["request_retry_count"] = request_retry_count # rebind-ok: same dict, read by the cap
|
||||
return kwargs
|
||||
|
||||
def _deployment_params_base_model(self, deployment_id: str | None) -> str | None:
|
||||
"""The ``base_model`` a deployment declares under ``litellm_params`` (the Azure named-deployment
|
||||
shape), the same key the success path's ``get_base_model_from_metadata`` reads."""
|
||||
deployment: Final = self.get_deployment(model_id=deployment_id) if deployment_id is not None else None
|
||||
if deployment is None:
|
||||
return None
|
||||
extra_params: Final = cast( # cast-ok: pydantic extras are untyped, base_model is isinstance-checked below
|
||||
"Mapping[str, object]", deployment.litellm_params.model_extra or {}
|
||||
)
|
||||
base_model: Final = extra_params.get("base_model")
|
||||
return base_model if isinstance(base_model, str) else None
|
||||
|
||||
def _update_usage(self, deployment_id: str, parent_otel_span: Span | None) -> int:
|
||||
"""
|
||||
Update deployment rpm for that minute
|
||||
|
|
@ -9755,7 +9899,7 @@ class Router:
|
|||
Raises:
|
||||
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
|
||||
"""
|
||||
for _callback in litellm.callbacks:
|
||||
for _callback in self._routing_callbacks():
|
||||
if isinstance(_callback, CustomLogger):
|
||||
_callback.pre_call_check(deployment)
|
||||
|
||||
|
|
@ -9776,7 +9920,7 @@ class Router:
|
|||
Raises:
|
||||
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
|
||||
"""
|
||||
for _callback in litellm.callbacks:
|
||||
for _callback in self._routing_callbacks():
|
||||
if isinstance(_callback, CustomLogger):
|
||||
try:
|
||||
await _callback.async_pre_call_check(deployment, parent_otel_span)
|
||||
|
|
@ -9913,7 +10057,7 @@ class Router:
|
|||
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
|
||||
"""
|
||||
returned_healthy_deployments = healthy_deployments
|
||||
for _callback in litellm.callbacks:
|
||||
for _callback in self._routing_callbacks():
|
||||
if isinstance(_callback, CustomLogger):
|
||||
try:
|
||||
returned_healthy_deployments = await _callback.async_filter_deployments(
|
||||
|
|
@ -10568,6 +10712,7 @@ class Router:
|
|||
changes, otherwise a released router keeps recording turns through its hook."""
|
||||
from litellm.router_strategy.adaptive_router.hooks import (
|
||||
AdaptiveRouterPostCallHook,
|
||||
AdaptiveRouterResponseHeaderHook,
|
||||
)
|
||||
|
||||
for callback in litellm.logging_callback_manager.get_custom_loggers_for_type(AdaptiveRouterPostCallHook):
|
||||
|
|
@ -10577,6 +10722,8 @@ class Router:
|
|||
litellm.logging_callback_manager.add_litellm_callback(
|
||||
AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy)
|
||||
)
|
||||
if self.adaptive_routers:
|
||||
litellm.logging_callback_manager.add_litellm_callback(AdaptiveRouterResponseHeaderHook())
|
||||
|
||||
def init_adaptive_router_deployment(self, deployment: Deployment) -> None:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
"""Adaptive router strategy. See README.md for design overview."""
|
||||
|
||||
from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter
|
||||
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
|
||||
from litellm.router_strategy.adaptive_router.hooks import (
|
||||
AdaptiveRouterPostCallHook,
|
||||
AdaptiveRouterResponseHeaderHook,
|
||||
)
|
||||
|
||||
__all__ = ["AdaptiveRouter", "AdaptiveRouterPostCallHook"]
|
||||
__all__ = ["AdaptiveRouter", "AdaptiveRouterPostCallHook", "AdaptiveRouterResponseHeaderHook"]
|
||||
|
|
|
|||
|
|
@ -170,12 +170,7 @@ def _assistant_content_and_tool_calls(response_obj: Any) -> tuple[object, Sequen
|
|||
return content, tool_calls
|
||||
|
||||
|
||||
class AdaptiveRouterPostCallHook(CustomLogger):
|
||||
"""One hook instance per AdaptiveRouter. Registered into litellm.callbacks."""
|
||||
|
||||
def __init__(self, adaptive_router: AdaptiveRouter) -> None:
|
||||
self.adaptive_router = adaptive_router
|
||||
|
||||
class AdaptiveRouterResponseHeaderHook(CustomLogger):
|
||||
async def async_post_call_response_headers_hook(
|
||||
self,
|
||||
data: Mapping[str, object],
|
||||
|
|
@ -200,6 +195,13 @@ class AdaptiveRouterPostCallHook(CustomLogger):
|
|||
return None
|
||||
return {ADAPTIVE_ROUTER_RESPONSE_HEADER: chosen}
|
||||
|
||||
|
||||
class AdaptiveRouterPostCallHook(CustomLogger):
|
||||
"""One hook instance per AdaptiveRouter. Registered into litellm.callbacks."""
|
||||
|
||||
def __init__(self, adaptive_router: AdaptiveRouter) -> None:
|
||||
self.adaptive_router = adaptive_router
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
await self._record(kwargs, response_obj, response_status=200)
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ class BaseRoutingStrategy(ABC):
|
|||
self.dual_cache = dual_cache
|
||||
self.redis_increment_operation_queue: list[RedisPipelineIncrementOperation] = []
|
||||
self._sync_task: asyncio.Task[None] | None = None
|
||||
self._retired = False
|
||||
self._flush_after_retire: asyncio.Task[None] | None = None
|
||||
if should_batch_redis_writes:
|
||||
self.setup_sync_task(default_sync_interval)
|
||||
|
||||
|
|
@ -47,13 +49,25 @@ class BaseRoutingStrategy(ABC):
|
|||
|
||||
def retire(self) -> None:
|
||||
self.cancel_sync_task()
|
||||
if not self.redis_increment_operation_queue:
|
||||
self._retired = True
|
||||
self._schedule_flush_after_retire()
|
||||
|
||||
def _schedule_flush_after_retire(self) -> None:
|
||||
if self._flush_after_retire is not None and not self._flush_after_retire.done():
|
||||
return
|
||||
if not self.redis_increment_operation_queue or self.dual_cache.redis_cache is None:
|
||||
return
|
||||
try:
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return
|
||||
loop.create_task(self._push_in_memory_increments_to_redis())
|
||||
self._flush_after_retire = loop.create_task(self._drain_increments_after_retire())
|
||||
|
||||
async def _drain_increments_after_retire(self) -> None:
|
||||
if not self.redis_increment_operation_queue:
|
||||
return
|
||||
await self._push_in_memory_increments_to_redis()
|
||||
await self._drain_increments_after_retire()
|
||||
|
||||
async def cleanup(self):
|
||||
"""Cleanup method to be called when shutting down"""
|
||||
|
|
@ -98,6 +112,8 @@ class BaseRoutingStrategy(ABC):
|
|||
|
||||
self.redis_increment_operation_queue.append(increment_op)
|
||||
self.add_to_in_memory_keys_to_update(key=key)
|
||||
if self._retired:
|
||||
self._schedule_flush_after_retire()
|
||||
return result
|
||||
|
||||
@with_service_target("router_usage")
|
||||
|
|
|
|||
|
|
@ -128,7 +128,9 @@ class RouterBudgetLimiting(CustomLogger):
|
|||
self._redis_increment_queue_lock = asyncio.Lock()
|
||||
self._redis_increment_flush_lock = asyncio.Lock()
|
||||
self._detached_increment_operations: tuple[RedisPipelineIncrementOperation, ...] | None = None
|
||||
asyncio.create_task(self.periodic_sync_in_memory_spend_with_redis())
|
||||
self._sync_task: asyncio.Task[None] = asyncio.create_task(self.periodic_sync_in_memory_spend_with_redis())
|
||||
self._retired = False
|
||||
self._flush_after_retire: asyncio.Task[None] | None = None
|
||||
self.provider_budget_config: GenericBudgetConfigType | None = provider_budget_config
|
||||
self.deployment_budget_config: GenericBudgetConfigType | None = None
|
||||
self.tag_budget_config: GenericBudgetConfigType | None = None
|
||||
|
|
@ -140,6 +142,30 @@ class RouterBudgetLimiting(CustomLogger):
|
|||
if isinstance(litellm.callbacks, list):
|
||||
litellm.logging_callback_manager.add_litellm_callback(self)
|
||||
|
||||
def retire(self) -> None:
|
||||
self._sync_task.cancel()
|
||||
self._retired = True
|
||||
self._schedule_flush_after_retire()
|
||||
|
||||
def _schedule_flush_after_retire(self) -> None:
|
||||
if self._flush_after_retire is not None and not self._flush_after_retire.done():
|
||||
return
|
||||
if not self._has_unwritten_increments():
|
||||
return
|
||||
try:
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return
|
||||
self._flush_after_retire = loop.create_task(self._drain_increments_after_retire())
|
||||
|
||||
def _has_unwritten_increments(self) -> bool:
|
||||
return bool(self.redis_increment_operation_queue) or self._detached_increment_operations is not None
|
||||
|
||||
async def _drain_increments_after_retire(self) -> None:
|
||||
while self._has_unwritten_increments():
|
||||
if not await self._push_in_memory_increments_to_redis():
|
||||
await asyncio.sleep(DEFAULT_REDIS_SYNC_INTERVAL)
|
||||
|
||||
@with_service_target("router_budgets")
|
||||
async def async_filter_deployments(
|
||||
self,
|
||||
|
|
@ -427,6 +453,8 @@ class RouterBudgetLimiting(CustomLogger):
|
|||
if self.dual_cache.redis_cache is None:
|
||||
return
|
||||
self.redis_increment_operation_queue.append(increment_op)
|
||||
if self._retired:
|
||||
self._schedule_flush_after_retire()
|
||||
|
||||
def _get_redis_increment_queue_lock(self) -> asyncio.Lock:
|
||||
return self._redis_increment_queue_lock
|
||||
|
|
|
|||
17
litellm/router_utils/stream_usage_injection.py
Normal file
17
litellm/router_utils/stream_usage_injection.py
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
|
||||
|
||||
def router_config_requests_stream_usage(
|
||||
request_kwargs: Mapping[str, object], configured_params: Sequence[Mapping[str, object]]
|
||||
) -> bool:
|
||||
if request_kwargs.get("stream") is not True or "stream_options" in request_kwargs:
|
||||
return False
|
||||
return any(_includes_usage(params.get("stream_options")) for params in configured_params)
|
||||
|
||||
|
||||
def _includes_usage(stream_options: object) -> bool:
|
||||
match stream_options:
|
||||
case {"include_usage": True}:
|
||||
return True
|
||||
case _:
|
||||
return False
|
||||
162
litellm/types/extraction.py
Normal file
162
litellm/types/extraction.py
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Annotated, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import (
|
||||
ConfigDict,
|
||||
Field,
|
||||
JsonValue,
|
||||
PrivateAttr,
|
||||
StrictBool,
|
||||
StrictFloat,
|
||||
StrictInt,
|
||||
StrictStr,
|
||||
model_validator,
|
||||
with_config,
|
||||
)
|
||||
from typing_extensions import ReadOnly, Required, Self, TypedDict
|
||||
|
||||
from litellm.types.llms.base import LiteLLMPydanticObjectBase
|
||||
|
||||
ExtractionFieldType: TypeAlias = Literal[
|
||||
"text", "number", "date", "time", "datetime", "duration", "list", "estimate", "forecast"
|
||||
]
|
||||
ExtractionFieldName: TypeAlias = Annotated[str, Field(pattern=r"^[A-Za-z_][A-Za-z0-9_.\-]{0,63}$")]
|
||||
JsonNumber: TypeAlias = StrictInt | StrictFloat
|
||||
ExtractionBound: TypeAlias = JsonNumber | StrictStr
|
||||
EXTRACTION_STATE_REWRITTEN_METADATA_KEY: Final = "extraction_state_rewritten_by_guardrail"
|
||||
|
||||
|
||||
def _is_none(value: object) -> bool:
|
||||
return value is None
|
||||
|
||||
|
||||
_OMITTED_WHEN_NONE: Final = Field(exclude_if=_is_none)
|
||||
OptionalJson: TypeAlias = Annotated[JsonValue, _OMITTED_WHEN_NONE]
|
||||
OptionalFloat: TypeAlias = Annotated[float | None, _OMITTED_WHEN_NONE]
|
||||
OptionalInt: TypeAlias = Annotated[int | None, _OMITTED_WHEN_NONE]
|
||||
|
||||
|
||||
class _ClairicObject(LiteLLMPydanticObjectBase):
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
|
||||
class ExtractionField(_ClairicObject):
|
||||
type: Annotated[ExtractionFieldType, Field(description="The answer type for this field")]
|
||||
instructions: Annotated[str | None, Field(description="What the field should contain")] = None
|
||||
unit: Annotated[str | None, Field(description="Unit label. For durations, seconds through years")] = None
|
||||
integer: Annotated[StrictBool | None, Field(description="Numbers and durations only")] = None
|
||||
decimals: Annotated[JsonNumber | None, Field(description="Numbers, durations and number lists only, 0 to 6")] = None
|
||||
items: Annotated[
|
||||
Literal["text", "number", "date"] | None, Field(description="Item type, required for list fields")
|
||||
] = None
|
||||
min: Annotated[ExtractionBound | None, Field(description="Estimate and forecast lower bound")] = None
|
||||
max: Annotated[ExtractionBound | None, Field(description="Estimate and forecast upper bound")] = None
|
||||
step: Annotated[ExtractionBound | None, Field(description="Estimate grid step, or month or year")] = None
|
||||
scale: Literal["linear", "log"] | None = None
|
||||
null_threshold: Annotated[
|
||||
JsonNumber | None, Field(description="Probability of null above which the value is null")
|
||||
] = None
|
||||
horizon: Annotated[JsonNumber | None, Field(description="Forecast only, number of steps")] = None
|
||||
interval: Literal["step", "minute", "hour", "day", "week", "month", "quarter", "year"] | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _list_fields_name_their_item_type(self) -> Self:
|
||||
if self.type == "list" and self.items is None:
|
||||
raise ValueError("list fields need items set to text, number or date")
|
||||
return self
|
||||
|
||||
|
||||
class ExtractionBase64Image(_ClairicObject):
|
||||
content_type: Annotated[str, Field(description="The image media type, such as image/png")]
|
||||
base64: Annotated[str, Field(description="Base64-encoded image bytes")]
|
||||
|
||||
|
||||
ExtractionImage: TypeAlias = Annotated[str, Field(pattern=r"^[Dd][Aa][Tt][Aa]:")] | ExtractionBase64Image
|
||||
|
||||
|
||||
class ExtractionOptions(_ClairicObject):
|
||||
include_quantiles: Annotated[StrictBool | None, Field(description="Add 99 percentiles to estimates")] = None
|
||||
return_state_text: Annotated[StrictBool | None, Field(description="Return the text that spans index into")] = None
|
||||
|
||||
|
||||
class ExtractionRequestBody(_ClairicObject):
|
||||
state: Annotated[
|
||||
str,
|
||||
Field(description="The text to read. Must be non-empty unless images are sent. Text answers are spans of it"),
|
||||
] = ""
|
||||
fields: Annotated[
|
||||
Mapping[ExtractionFieldName, ExtractionField],
|
||||
Field(min_length=1, description="Field name to typed field spec. Each name becomes a key in the answers"),
|
||||
]
|
||||
images: Annotated[
|
||||
tuple[ExtractionImage, ...] | None,
|
||||
Field(max_length=4, description="Up to 4 images as data URLs or content_type plus base64, no remote URLs"),
|
||||
] = None
|
||||
reference_date: Annotated[str | None, Field(description="ISO date that anchors dates written without a year")] = (
|
||||
None
|
||||
)
|
||||
options: ExtractionOptions | None = None
|
||||
|
||||
|
||||
class ExtractionRequest(ExtractionRequestBody):
|
||||
model: Annotated[str, Field(description="An extraction model from the proxy model_list")]
|
||||
|
||||
|
||||
class ExtractionAnswer(_ClairicObject):
|
||||
type: str
|
||||
value: Annotated[JsonValue, Field(description="The answer, null when the input does not determine it")]
|
||||
guess: OptionalJson = None
|
||||
p_null: OptionalFloat = None
|
||||
confidence: OptionalFloat = None
|
||||
span: Annotated[tuple[int, ...] | None, _OMITTED_WHEN_NONE] = None
|
||||
spans: Annotated[tuple[tuple[int, ...], ...] | None, _OMITTED_WHEN_NONE] = None
|
||||
mean: OptionalJson = None
|
||||
median: OptionalJson = None
|
||||
p10: OptionalJson = None
|
||||
p90: OptionalJson = None
|
||||
p_below: OptionalFloat = None
|
||||
p_above: OptionalFloat = None
|
||||
quantiles: Annotated[tuple[JsonValue, ...] | None, _OMITTED_WHEN_NONE] = None
|
||||
steps: Annotated[tuple[JsonValue, ...] | None, _OMITTED_WHEN_NONE] = None
|
||||
|
||||
|
||||
class ExtractionUsage(_ClairicObject):
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
state_tokens: OptionalInt = None
|
||||
image_tokens: OptionalInt = None
|
||||
schema_tokens: OptionalInt = None
|
||||
passes: OptionalInt = None
|
||||
|
||||
|
||||
class ExtractionResponse(_ClairicObject):
|
||||
model: str
|
||||
answers: Mapping[str, ExtractionAnswer]
|
||||
truncated: bool
|
||||
state_text: Annotated[str | None, _OMITTED_WHEN_NONE] = None
|
||||
usage: ExtractionUsage
|
||||
|
||||
_hidden_params: dict[str, object] = PrivateAttr( # mutable-ok: @client and the proxy write cost into it in place
|
||||
default_factory=dict
|
||||
)
|
||||
|
||||
@property
|
||||
def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation
|
||||
return self._hidden_params
|
||||
|
||||
def set_hidden_params(self, params: Mapping[str, object]) -> None:
|
||||
self._hidden_params.update(params)
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow"))
|
||||
class ExtractionCallParams(TypedDict, total=False):
|
||||
model: Required[ReadOnly[str]]
|
||||
state: Required[ReadOnly[str]]
|
||||
fields: Required[ReadOnly[Mapping[str, Mapping[str, object]]]]
|
||||
images: ReadOnly[Sequence[str | Mapping[str, object]] | None]
|
||||
reference_date: ReadOnly[str | None]
|
||||
api_key: ReadOnly[str | None]
|
||||
api_base: ReadOnly[str | None]
|
||||
timeout: ReadOnly[float | None]
|
||||
custom_llm_provider: ReadOnly[str | None]
|
||||
extra_headers: ReadOnly[Mapping[str, str] | None]
|
||||
|
|
@ -54,6 +54,7 @@ RoutingStrategyName: TypeAlias = Literal[
|
|||
|
||||
TRUSTED_CALLBACK_VARS_FIELD: Final = "litellm_trusted_callback_vars"
|
||||
ADDRESSED_RESPONSE_ID_FIELD: Final = "_litellm_addressed_response_id"
|
||||
STREAM_USAGE_INJECTED_FIELD: Final = "_litellm_stream_usage_injected"
|
||||
|
||||
WIRE_NAME: Final = "wire_name"
|
||||
|
||||
|
|
@ -397,6 +398,7 @@ class ProxyRequestState:
|
|||
trusted_callback_vars: Mapping[str, str] | None = field(default=None, metadata=wire(TRUSTED_CALLBACK_VARS_FIELD))
|
||||
addressed_response_id: str | None = field(default=None, metadata=wire(ADDRESSED_RESPONSE_ID_FIELD))
|
||||
strip_stream_usage: bool | None = field(default=None, metadata=wire("_litellm_strip_stream_usage"))
|
||||
stream_usage_injected: bool | None = field(default=None, metadata=wire(STREAM_USAGE_INJECTED_FIELD))
|
||||
client_side_timeout: bool | None = None
|
||||
model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None
|
||||
|
||||
|
|
|
|||
|
|
@ -239,8 +239,18 @@ class VoiceConfig(TypedDict):
|
|||
prebuiltVoiceConfig: PrebuiltVoiceConfig
|
||||
|
||||
|
||||
class SpeakerVoiceConfig(TypedDict):
|
||||
speaker: ReadOnly[str]
|
||||
voiceConfig: ReadOnly[VoiceConfig]
|
||||
|
||||
|
||||
class MultiSpeakerVoiceConfig(TypedDict):
|
||||
speakerVoiceConfigs: ReadOnly[Sequence[SpeakerVoiceConfig]]
|
||||
|
||||
|
||||
class SpeechConfig(TypedDict, total=False):
|
||||
voiceConfig: VoiceConfig
|
||||
multiSpeakerVoiceConfig: ReadOnly[MultiSpeakerVoiceConfig]
|
||||
languageCode: str
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,9 @@ Type definitions for Vertex AI Text-to-Speech API
|
|||
Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/synthesize
|
||||
"""
|
||||
|
||||
from typing_extensions import TypedDict
|
||||
from collections.abc import Sequence
|
||||
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
|
||||
class VertexTextToSpeechInput(TypedDict, total=False):
|
||||
|
|
@ -16,6 +18,16 @@ class VertexTextToSpeechInput(TypedDict, total=False):
|
|||
|
||||
text: str | None
|
||||
ssml: str | None
|
||||
prompt: ReadOnly[str | None]
|
||||
|
||||
|
||||
class VertexTextToSpeechSpeakerVoiceConfig(TypedDict):
|
||||
speakerAlias: ReadOnly[str]
|
||||
speakerId: ReadOnly[str]
|
||||
|
||||
|
||||
class VertexTextToSpeechMultiSpeakerVoiceConfig(TypedDict):
|
||||
speakerVoiceConfigs: ReadOnly[Sequence[VertexTextToSpeechSpeakerVoiceConfig]]
|
||||
|
||||
|
||||
class VertexTextToSpeechVoice(TypedDict, total=False):
|
||||
|
|
@ -29,6 +41,8 @@ class VertexTextToSpeechVoice(TypedDict, total=False):
|
|||
|
||||
languageCode: str
|
||||
name: str
|
||||
modelName: ReadOnly[str]
|
||||
multiSpeakerVoiceConfig: ReadOnly[VertexTextToSpeechMultiSpeakerVoiceConfig]
|
||||
|
||||
|
||||
class VertexTextToSpeechAudioConfig(TypedDict, total=False):
|
||||
|
|
@ -42,6 +56,7 @@ class VertexTextToSpeechAudioConfig(TypedDict, total=False):
|
|||
|
||||
audioEncoding: str
|
||||
speakingRate: str
|
||||
sampleRateHertz: ReadOnly[int]
|
||||
|
||||
|
||||
class VertexTextToSpeechRequest(TypedDict, total=False):
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from collections.abc import Mapping
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -13,6 +14,8 @@ class ModelGroupInfoProxy(ModelGroupInfo):
|
|||
health_status: str | None = Field(default=None)
|
||||
health_response_time: float | None = Field(default=None)
|
||||
health_checked_at: str | None = Field(default=None)
|
||||
custom_tags: tuple[str, ...] | None = None
|
||||
custom_metadata: Mapping[str, str | int | float | bool] | None = None
|
||||
|
||||
|
||||
class UpdateUsefulLinksRequest(LiteLLMBaseModel):
|
||||
|
|
|
|||
|
|
@ -1184,6 +1184,8 @@ class RouterModelGroupAliasItem(TypedDict):
|
|||
class RetryAttemptRecord(TypedDict):
|
||||
model_group: ReadOnly[str | None]
|
||||
deployment_id: ReadOnly[str | None]
|
||||
model: ReadOnly[str | None]
|
||||
base_model: ReadOnly[str | None]
|
||||
exception_type: ReadOnly[str]
|
||||
exception_string: ReadOnly[str]
|
||||
attempted_retries: ReadOnly[int | None]
|
||||
|
|
|
|||
|
|
@ -212,6 +212,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
|
|||
supports_parallel_function_calling: bool | None
|
||||
supports_web_search: bool | None
|
||||
supports_reasoning: bool | None
|
||||
supports_sampling_params: ReadOnly[bool | None]
|
||||
supports_adaptive_thinking: bool | None
|
||||
supports_legacy_thinking: ReadOnly[bool | None]
|
||||
thinking_always_on: ReadOnly[bool | None]
|
||||
|
|
@ -458,6 +459,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
"responses",
|
||||
"decisions",
|
||||
"evaluation",
|
||||
"extraction",
|
||||
"ocr",
|
||||
"realtime",
|
||||
]
|
||||
|
|
@ -518,6 +520,8 @@ class CallTypes(str, Enum):
|
|||
adecisions = "adecisions"
|
||||
systemone = "systemone"
|
||||
asystemone = "asystemone"
|
||||
extraction = "extraction"
|
||||
aextraction = "aextraction"
|
||||
arealtime = "_arealtime"
|
||||
aresponses_websocket = "_aresponses_websocket"
|
||||
create_batch = "create_batch"
|
||||
|
|
@ -708,6 +712,8 @@ CallTypesLiteral = Literal[
|
|||
"adecisions",
|
||||
"systemone",
|
||||
"asystemone",
|
||||
"extraction",
|
||||
"aextraction",
|
||||
"_arealtime",
|
||||
"_aresponses_websocket",
|
||||
"create_batch",
|
||||
|
|
@ -821,6 +827,8 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = {
|
|||
"/v1/decisions": [CallTypes.adecisions, CallTypes.decisions],
|
||||
"/systemone": [CallTypes.asystemone, CallTypes.systemone],
|
||||
"/v1/systemone": [CallTypes.asystemone, CallTypes.systemone],
|
||||
"/extraction": [CallTypes.aextraction, CallTypes.extraction],
|
||||
"/v1/extraction": [CallTypes.aextraction, CallTypes.extraction],
|
||||
# Batches
|
||||
"/batches": [CallTypes.acreate_batch, CallTypes.create_batch],
|
||||
"/v1/batches": [CallTypes.acreate_batch, CallTypes.create_batch],
|
||||
|
|
|
|||
291
litellm/utils.py
291
litellm/utils.py
|
|
@ -71,6 +71,7 @@ from litellm.constants import (
|
|||
DEFAULT_TRIM_RATIO,
|
||||
FUNCTION_DEFINITION_TOKEN_COUNT,
|
||||
HF_CONFIG_FETCH_TIMEOUT_SECONDS,
|
||||
HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS,
|
||||
INITIAL_RETRY_DELAY,
|
||||
JITTER,
|
||||
MAX_RETRY_DELAY,
|
||||
|
|
@ -390,6 +391,7 @@ if TYPE_CHECKING:
|
|||
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
|
||||
from litellm.llms.base_llm.extraction.transformation import BaseExtractionConfig
|
||||
from litellm.llms.base_llm.files.transformation import BaseFilesConfig
|
||||
from litellm.llms.base_llm.google_genai.transformation import (
|
||||
BaseGoogleGenAIGenerateContentConfig,
|
||||
|
|
@ -858,6 +860,11 @@ def _is_gemini_model(model: str | None, custom_llm_provider: str | None) -> bool
|
|||
return model is not None and "gemini" in model.lower()
|
||||
|
||||
|
||||
def is_gemini_tts_model(model: str) -> bool:
|
||||
model_name: Final = model.lower()
|
||||
return "gemini" in model_name and "tts" in model_name
|
||||
|
||||
|
||||
def _remove_thought_signature_from_id(tool_call_id: str, separator: str) -> str:
|
||||
"""
|
||||
Remove thought signature from a tool call ID.
|
||||
|
|
@ -1124,83 +1131,90 @@ def function_setup(
|
|||
dynamic_callbacks: Final[list[str | Callable | CustomLogger] | None] = kwargs.pop("callbacks", None)
|
||||
all_callbacks: Final = get_dynamic_callbacks(dynamic_callbacks=dynamic_callbacks)
|
||||
|
||||
if len(all_callbacks) > 0:
|
||||
for callback in all_callbacks:
|
||||
# check if callback is a string - e.g. "lago", "openmeter"
|
||||
if isinstance(callback, str):
|
||||
callback = litellm.litellm_core_utils.litellm_logging._init_custom_logger_compatible_class(
|
||||
callback,
|
||||
internal_usage_cache=None,
|
||||
llm_router=None,
|
||||
)
|
||||
if callback is None or any(
|
||||
type(cb) is type(callback) for cb in litellm._async_success_callback
|
||||
): # don't double add a callback
|
||||
continue
|
||||
if callback not in litellm.input_callback:
|
||||
litellm.input_callback.append(callback)
|
||||
if callback not in litellm.success_callback:
|
||||
litellm.logging_callback_manager.add_litellm_success_callback(callback)
|
||||
if callback not in litellm.failure_callback:
|
||||
litellm.logging_callback_manager.add_litellm_failure_callback(callback)
|
||||
if callback not in litellm._async_success_callback:
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
|
||||
if callback not in litellm._async_failure_callback:
|
||||
litellm.logging_callback_manager.add_litellm_async_failure_callback(callback)
|
||||
print_verbose(f"Initialized litellm callbacks, Async Success Callbacks: {litellm._async_success_callback}")
|
||||
with litellm.logging_callback_manager.callback_lists_lock:
|
||||
if len(all_callbacks) > 0:
|
||||
for callback in all_callbacks:
|
||||
# check if callback is a string - e.g. "lago", "openmeter"
|
||||
if isinstance(callback, str):
|
||||
callback = litellm.litellm_core_utils.litellm_logging._init_custom_logger_compatible_class(
|
||||
callback,
|
||||
internal_usage_cache=None,
|
||||
llm_router=None,
|
||||
)
|
||||
if callback is None or any(
|
||||
type(cb) is type(callback) for cb in litellm._async_success_callback
|
||||
): # don't double add a callback
|
||||
continue
|
||||
if callback not in litellm.input_callback:
|
||||
litellm.input_callback.append(callback)
|
||||
if callback not in litellm.success_callback:
|
||||
litellm.logging_callback_manager.add_litellm_success_callback(callback)
|
||||
if callback not in litellm.failure_callback:
|
||||
litellm.logging_callback_manager.add_litellm_failure_callback(callback)
|
||||
if callback not in litellm._async_success_callback:
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
|
||||
if callback not in litellm._async_failure_callback:
|
||||
litellm.logging_callback_manager.add_litellm_async_failure_callback(callback)
|
||||
print_verbose(
|
||||
f"Initialized litellm callbacks, Async Success Callbacks: {litellm._async_success_callback}"
|
||||
)
|
||||
|
||||
if (
|
||||
len(litellm.input_callback) > 0 or len(litellm.success_callback) > 0 or len(litellm.failure_callback) > 0
|
||||
) and len(callback_list) == 0:
|
||||
callback_list = list(set(litellm.input_callback + litellm.success_callback + litellm.failure_callback))
|
||||
callbacks_to_set_up: Final = (
|
||||
list(set(litellm.input_callback + litellm.success_callback + litellm.failure_callback))
|
||||
if len(callback_list) == 0
|
||||
else []
|
||||
)
|
||||
if callbacks_to_set_up:
|
||||
callback_list = callbacks_to_set_up
|
||||
## ASYNC CALLBACKS - safety net for callbacks added via direct append
|
||||
if len(litellm.input_callback) > 0:
|
||||
removed_async_items = []
|
||||
for index, callback in enumerate(litellm.input_callback):
|
||||
if coroutine_checker.is_async_callable(callback):
|
||||
litellm._async_input_callback.append(callback)
|
||||
removed_async_items.append(index)
|
||||
|
||||
# Pop the async items from input_callback in reverse order to avoid index issues
|
||||
for index in reversed(removed_async_items):
|
||||
litellm.input_callback.pop(index)
|
||||
if len(litellm.success_callback) > 0:
|
||||
removed_async_items = []
|
||||
for index, callback in enumerate(litellm.success_callback):
|
||||
if coroutine_checker.is_async_callable(callback):
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
|
||||
removed_async_items.append(index)
|
||||
elif callback == "dynamodb" or callback == "openmeter":
|
||||
# dynamo is an async callback, it's used for the proxy and needs to be async
|
||||
# we only support async dynamo db logging for acompletion/aembedding since that's used on proxy
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
|
||||
removed_async_items.append(index)
|
||||
elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str):
|
||||
add_custom_logger_callback_to_specific_event(callback, "success")
|
||||
|
||||
# Pop the async items from success_callback in reverse order to avoid index issues
|
||||
for index in reversed(removed_async_items):
|
||||
litellm.success_callback.pop(index)
|
||||
|
||||
if len(litellm.failure_callback) > 0:
|
||||
removed_async_items = []
|
||||
for index, callback in enumerate(litellm.failure_callback):
|
||||
if coroutine_checker.is_async_callable(callback):
|
||||
litellm.logging_callback_manager.add_litellm_async_failure_callback(callback)
|
||||
removed_async_items.append(index)
|
||||
elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str):
|
||||
add_custom_logger_callback_to_specific_event(callback, "failure")
|
||||
|
||||
# Pop the async items from failure_callback in reverse order to avoid index issues
|
||||
for index in reversed(removed_async_items):
|
||||
litellm.failure_callback.pop(index)
|
||||
if callbacks_to_set_up:
|
||||
get_set_callbacks: Final[Callable[[], Callable[..., None]]] = litellm_utils.get_set_callbacks
|
||||
get_set_callbacks()(callback_list=callback_list, function_id=function_id)
|
||||
## ASYNC CALLBACKS - safety net for callbacks added via direct append
|
||||
if len(litellm.input_callback) > 0:
|
||||
removed_async_items = []
|
||||
for index, callback in enumerate(litellm.input_callback):
|
||||
if coroutine_checker.is_async_callable(callback):
|
||||
litellm._async_input_callback.append(callback)
|
||||
removed_async_items.append(index)
|
||||
|
||||
# Pop the async items from input_callback in reverse order to avoid index issues
|
||||
for index in reversed(removed_async_items):
|
||||
litellm.input_callback.pop(index)
|
||||
if len(litellm.success_callback) > 0:
|
||||
removed_async_items = []
|
||||
for index, callback in enumerate(litellm.success_callback):
|
||||
if coroutine_checker.is_async_callable(callback):
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
|
||||
removed_async_items.append(index)
|
||||
elif callback == "dynamodb" or callback == "openmeter":
|
||||
# dynamo is an async callback, it's used for the proxy and needs to be async
|
||||
# we only support async dynamo db logging for acompletion/aembedding since that's used on proxy
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
|
||||
removed_async_items.append(index)
|
||||
elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str):
|
||||
add_custom_logger_callback_to_specific_event(callback, "success")
|
||||
|
||||
# Pop the async items from success_callback in reverse order to avoid index issues
|
||||
for index in reversed(removed_async_items):
|
||||
litellm.success_callback.pop(index)
|
||||
|
||||
if len(litellm.failure_callback) > 0:
|
||||
removed_async_items = []
|
||||
for index, callback in enumerate(litellm.failure_callback):
|
||||
if coroutine_checker.is_async_callable(callback):
|
||||
litellm.logging_callback_manager.add_litellm_async_failure_callback(callback)
|
||||
removed_async_items.append(index)
|
||||
elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str):
|
||||
add_custom_logger_callback_to_specific_event(callback, "failure")
|
||||
|
||||
# Pop the async items from failure_callback in reverse order to avoid index issues
|
||||
for index in reversed(removed_async_items):
|
||||
litellm.failure_callback.pop(index)
|
||||
get_set_callbacks()(callback_list=callbacks_to_set_up, function_id=function_id)
|
||||
### DYNAMIC CALLBACKS ###
|
||||
dynamic_success_callbacks: list[str | Callable | CustomLogger] | None = None
|
||||
dynamic_async_success_callbacks: list[str | Callable | CustomLogger] | None = None
|
||||
dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = None
|
||||
dynamic_async_failure_callbacks: Final[list[str | Callable | CustomLogger] | None] = None
|
||||
dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None
|
||||
dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None
|
||||
dynamic_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None
|
||||
dynamic_async_failure_callbacks: Final[list[str | Callable[..., object] | CustomLogger] | None] = None
|
||||
if kwargs.get("success_callback", None) is not None and isinstance(kwargs["success_callback"], list):
|
||||
removed_async_items = []
|
||||
for index, callback in enumerate(kwargs["success_callback"]):
|
||||
|
|
@ -1579,6 +1593,22 @@ def _get_wrapper_timeout(kwargs: dict[str, object]) -> object:
|
|||
return kwargs.get("timeout") if stream_timeout is None else stream_timeout
|
||||
|
||||
|
||||
def _runs_as_async_success_callback(callback: str | Callable[..., object] | CustomLogger) -> bool:
|
||||
get_coroutine_checker: Final = litellm_utils.get_coroutine_checker
|
||||
return get_coroutine_checker().is_async_callable(callback) or callback in ("dynamodb", "s3")
|
||||
|
||||
|
||||
def split_dynamic_success_callbacks(
|
||||
callbacks: Sequence[str | Callable[..., object] | CustomLogger],
|
||||
) -> tuple[
|
||||
tuple[str | Callable[..., object] | CustomLogger, ...], tuple[str | Callable[..., object] | CustomLogger, ...]
|
||||
]:
|
||||
return (
|
||||
tuple(callback for callback in callbacks if not _runs_as_async_success_callback(callback)),
|
||||
tuple(callback for callback in callbacks if _runs_as_async_success_callback(callback)),
|
||||
)
|
||||
|
||||
|
||||
def check_coroutine(value) -> bool:
|
||||
get_coroutine_checker: Final = litellm_utils.get_coroutine_checker
|
||||
return get_coroutine_checker().is_async_callable(value)
|
||||
|
|
@ -2166,7 +2196,11 @@ def client(original_function):
|
|||
|
||||
# Sync logging_obj.stream after deployment hooks (they may convert it).
|
||||
_hook_stream: Final = kwargs.get("stream")
|
||||
if _hook_stream is not None and logging_obj.stream != _hook_stream:
|
||||
if (
|
||||
_hook_stream is not None
|
||||
and logging_obj.stream != _hook_stream
|
||||
and call_type not in _NEVER_STREAMING_CALL_TYPES
|
||||
):
|
||||
logging_obj.stream = _hook_stream
|
||||
|
||||
kwargs["litellm_logging_obj"] = logging_obj
|
||||
|
|
@ -2510,6 +2544,15 @@ _STREAMING_CALL_TYPES: Final = frozenset(
|
|||
}
|
||||
)
|
||||
|
||||
_NEVER_STREAMING_CALL_TYPES: Final = frozenset(
|
||||
{
|
||||
CallTypes.extraction,
|
||||
CallTypes.aextraction,
|
||||
CallTypes.extraction.value,
|
||||
CallTypes.aextraction.value,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _is_streaming_request(
|
||||
kwargs: dict[str, object],
|
||||
|
|
@ -2521,6 +2564,8 @@ def _is_streaming_request(
|
|||
- if "stream=True" in kwargs (litellm chat completion, litellm text completion, litellm messages)
|
||||
- if call_type is generate_content_stream or agenerate_content_stream (litellm google genai)
|
||||
"""
|
||||
if call_type in _NEVER_STREAMING_CALL_TYPES:
|
||||
return False
|
||||
if "stream" in kwargs and kwargs["stream"] is True:
|
||||
return True
|
||||
return call_type in _STREAMING_CALL_TYPES
|
||||
|
|
@ -2565,19 +2610,14 @@ def _select_tokenizer_helper(model: str) -> SelectTokenizerResponse:
|
|||
|
||||
try:
|
||||
result: Final = _return_huggingface_tokenizer(model)
|
||||
if result is not None:
|
||||
if result is not None and not isinstance(result, _HuggingFaceTokenizerLoadFailure):
|
||||
return result
|
||||
except Exception as e:
|
||||
from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted, ProcessReservedForForking
|
||||
|
||||
if isinstance(e, (ForkedAfterNativeRuntimeStarted, ProcessReservedForForking)):
|
||||
raise
|
||||
verbose_logger.warning(
|
||||
"Falling back to tiktoken for %s; token counts may be approximate. "
|
||||
"For Python Hugging Face tokenization, install tokenizers and huggingface-hub. Error: %s",
|
||||
json.dumps(redact_secrets(model)),
|
||||
json.dumps(redact_secrets(str(e))),
|
||||
)
|
||||
_warn_huggingface_tokenizer_fallback(model, str(e))
|
||||
|
||||
# default - tiktoken
|
||||
return _return_openai_tokenizer(model)
|
||||
|
|
@ -2617,15 +2657,41 @@ def huggingface_tokenizer_kind(model: str) -> HuggingFaceTokenizerKind | None:
|
|||
return None
|
||||
|
||||
|
||||
def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | None:
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _HuggingFaceTokenizerLoadFailure:
|
||||
reason: str
|
||||
|
||||
|
||||
def _warn_huggingface_tokenizer_fallback(model: str, reason: str) -> None:
|
||||
verbose_logger.warning(
|
||||
"Falling back to tiktoken for %s; token counts may be approximate. "
|
||||
"For Python Hugging Face tokenization, install tokenizers and huggingface-hub. Error: %s",
|
||||
json.dumps(redact_secrets(model)),
|
||||
json.dumps(redact_secrets(reason)),
|
||||
)
|
||||
|
||||
|
||||
def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None:
|
||||
kind: Final = huggingface_tokenizer_kind(model)
|
||||
if kind is None:
|
||||
return None
|
||||
backend: Final = _huggingface_tokenizer_backend()
|
||||
key: Final = _huggingface_tokenizer_failure_key(kind, backend)
|
||||
with _BUNDLED_TOKENIZER_LOAD_LOCKS.get(kind, nullcontext()):
|
||||
tokenizer: Final = _load_huggingface_tokenizer(kind, _huggingface_tokenizer_backend())
|
||||
cached_failure: Final = _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.get_cache(key)
|
||||
if isinstance(cached_failure, _HuggingFaceTokenizerLoadFailure):
|
||||
return cached_failure
|
||||
tokenizer: Final = _load_huggingface_tokenizer_recording_failure(kind, backend, key)
|
||||
if isinstance(tokenizer, _HuggingFaceTokenizerLoadFailure):
|
||||
_warn_huggingface_tokenizer_fallback(model, tokenizer.reason)
|
||||
return tokenizer
|
||||
return {"type": "huggingface_tokenizer", "tokenizer": tokenizer}
|
||||
|
||||
|
||||
def _huggingface_tokenizer_failure_key(kind: HuggingFaceTokenizerKind, backend: Decision) -> str:
|
||||
return f"{kind}:{backend.name}"
|
||||
|
||||
|
||||
@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
|
||||
def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind, backend: Decision) -> HuggingFace:
|
||||
"""One tokenizer per kind and backend; `backend` is the cache key, the dispatch re-derives it."""
|
||||
|
|
@ -2648,6 +2714,21 @@ def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind, backend: Decisio
|
|||
assert_never(kind)
|
||||
|
||||
|
||||
def _load_huggingface_tokenizer_recording_failure(
|
||||
kind: HuggingFaceTokenizerKind, backend: Decision, key: str
|
||||
) -> HuggingFace | _HuggingFaceTokenizerLoadFailure:
|
||||
try:
|
||||
return _load_huggingface_tokenizer(kind, backend)
|
||||
except Exception as error:
|
||||
from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted, ProcessReservedForForking
|
||||
|
||||
if isinstance(error, (ForkedAfterNativeRuntimeStarted, ProcessReservedForForking)):
|
||||
raise
|
||||
failure: Final = _HuggingFaceTokenizerLoadFailure(reason=str(error))
|
||||
_HUGGINGFACE_TOKENIZER_LOAD_FAILURES.set_cache(key, failure, ttl=HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS)
|
||||
return failure
|
||||
|
||||
|
||||
def encode(model="", text="", custom_tokenizer: dict | None = None):
|
||||
"""
|
||||
Encodes the given text using the specified model.
|
||||
|
|
@ -7893,12 +7974,16 @@ class TextCompletionStreamWrapper:
|
|||
text_choices: Final = TextChoices()
|
||||
if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices
|
||||
raise Exception
|
||||
delta: Final = chunk["choices"][0]["delta"]
|
||||
text_choices["text"] = delta["content"]
|
||||
text_choices["reasoning_content"] = delta.get("reasoning_content")
|
||||
text_choices["index"] = chunk["choices"][0]["index"]
|
||||
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
|
||||
response["choices"] = [text_choices]
|
||||
|
||||
if chunk["choices"]:
|
||||
delta: Final = chunk["choices"][0]["delta"]
|
||||
text_choices["text"] = delta["content"]
|
||||
text_choices["reasoning_content"] = delta.get("reasoning_content")
|
||||
text_choices["index"] = chunk["choices"][0]["index"]
|
||||
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
|
||||
response["choices"] = [text_choices]
|
||||
else:
|
||||
response["choices"] = []
|
||||
|
||||
# only pass usage when stream_options["include_usage"] is True
|
||||
if self.stream_options and self.stream_options.get("include_usage", False) is True:
|
||||
|
|
@ -8263,6 +8348,26 @@ def trim_messages(
|
|||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
|
||||
|
||||
class _HuggingFaceTokenizerLoadFailureCache(Protocol):
|
||||
def get_cache(self, key: str) -> object | None: ...
|
||||
|
||||
def set_cache(self, key: str, value: object, ttl: float | None = None) -> None: ...
|
||||
|
||||
def flush_cache(self) -> None: ...
|
||||
|
||||
|
||||
_HUGGINGFACE_TOKENIZER_LOAD_FAILURES: Final[_HuggingFaceTokenizerLoadFailureCache] = (
|
||||
cast( # cast-ok: cache methods are untyped
|
||||
_HuggingFaceTokenizerLoadFailureCache,
|
||||
InMemoryCache(
|
||||
max_size_in_memory=16,
|
||||
default_ttl=HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS,
|
||||
clock=lambda: time.monotonic(),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class AvailableModelsCache(InMemoryCache):
|
||||
def __init__(self, ttl_seconds: int = 300, max_size: int = 1000):
|
||||
super().__init__(ttl_seconds, max_size)
|
||||
|
|
@ -9539,6 +9644,12 @@ class ProviderConfigManager:
|
|||
return litellm.AzureAIDecisionsConfig()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_provider_extraction_config(model: str, provider: LlmProviders) -> BaseExtractionConfig | None:
|
||||
if provider == LlmProviders.CLOUDFLARE:
|
||||
return litellm.CloudflareExtractionConfig()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_provider_anthropic_messages_config(
|
||||
model: str,
|
||||
|
|
|
|||
|
|
@ -16580,6 +16580,24 @@
|
|||
"source": "https://developers.cloudflare.com/workers-ai/platform/pricing/",
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/clairic": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "cloudflare",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "extraction",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/models/clairic/",
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/@cf/cloudflare/clairic": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "cloudflare",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "extraction",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://developers.cloudflare.com/workers-ai/models/clairic/",
|
||||
"supports_vision": true
|
||||
},
|
||||
"cloudflare/@cf/meta/llama-2-7b-chat-fp16": {
|
||||
"input_cost_per_token": 1.923e-06,
|
||||
"litellm_provider": "cloudflare",
|
||||
|
|
@ -21903,6 +21921,7 @@
|
|||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -21927,6 +21946,7 @@
|
|||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -21958,6 +21978,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -22021,6 +22042,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -29417,6 +29439,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -30139,6 +30162,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -30200,6 +30224,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -30268,6 +30293,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -30317,6 +30343,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": false,
|
||||
"supports_url_context": false,
|
||||
|
|
@ -31743,6 +31770,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -31927,6 +31955,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -31989,6 +32018,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -32051,6 +32081,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -32420,6 +32451,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -32481,6 +32513,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -32549,6 +32582,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -32598,6 +32632,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": false,
|
||||
"supports_url_context": false,
|
||||
|
|
@ -33192,6 +33227,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_vision": true
|
||||
},
|
||||
"github_copilot/gemini-3.8-flash": {
|
||||
|
|
@ -33210,6 +33246,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_vision": true
|
||||
},
|
||||
"github_copilot/gpt-3.5-turbo": {
|
||||
|
|
@ -52605,6 +52642,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_url_context": true,
|
||||
|
|
@ -65823,6 +65861,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_vision": true,
|
||||
"source": "https://deepinfra.com/pricing",
|
||||
"supports_audio_input": true
|
||||
|
|
@ -69264,6 +69303,7 @@
|
|||
"perplexity/google/gemini-3.5-flash-lite": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_sampling_params": false,
|
||||
"supports_web_search": true,
|
||||
"supports_function_calling": true,
|
||||
"input_cost_per_token": 3e-07,
|
||||
|
|
@ -69274,6 +69314,7 @@
|
|||
"perplexity/google/gemini-3.6-flash": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_sampling_params": false,
|
||||
"supports_web_search": true,
|
||||
"supports_function_calling": true,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
|
|
@ -69284,6 +69325,7 @@
|
|||
"perplexity/google/gemini-3.7-flash": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_sampling_params": false,
|
||||
"supports_web_search": true,
|
||||
"supports_function_calling": true,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
|
|
@ -71375,6 +71417,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true
|
||||
},
|
||||
|
|
@ -71392,6 +71435,7 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true
|
||||
},
|
||||
|
|
@ -74322,6 +74366,7 @@
|
|||
"perplexity/google/gemini-3.8-flash": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_sampling_params": false,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
"output_cost_per_token": 3.75e-06,
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
|
|
@ -75064,12 +75109,10 @@
|
|||
"gemini-nano-banana-2.1": {
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"cache_read_input_token_cost_batches": 7.5e-08,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_read_input_token_cost_priority": 2.7e-07,
|
||||
"input_cost_per_image": 0.00168,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
"input_cost_per_token_batches": 7.5e-07,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
"input_cost_per_token_priority": 2.7e-06,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -75080,7 +75123,6 @@
|
|||
"output_cost_per_image_token": 3e-05,
|
||||
"output_cost_per_token": 7.5e-06,
|
||||
"output_cost_per_token_batches": 3.75e-06,
|
||||
"output_cost_per_token_flex": 3.75e-06,
|
||||
"output_cost_per_token_priority": 1.35e-05,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
|
|
@ -75113,12 +75155,10 @@
|
|||
"vertex_ai/gemini-nano-banana-2.1": {
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"cache_read_input_token_cost_batches": 7.5e-08,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_read_input_token_cost_priority": 2.7e-07,
|
||||
"input_cost_per_image": 0.00168,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
"input_cost_per_token_batches": 7.5e-07,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
"input_cost_per_token_priority": 2.7e-06,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -75129,7 +75169,6 @@
|
|||
"output_cost_per_image_token": 3e-05,
|
||||
"output_cost_per_token": 7.5e-06,
|
||||
"output_cost_per_token_batches": 3.75e-06,
|
||||
"output_cost_per_token_flex": 3.75e-06,
|
||||
"output_cost_per_token_priority": 1.35e-05,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
|
|
@ -76929,5 +76968,83 @@
|
|||
"supported_output_modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash-1017-us": {
|
||||
"cache_read_input_token_cost": 9e-09,
|
||||
"cache_read_input_token_cost_priority": 1.125e-08,
|
||||
"input_cost_per_token": 4.5e-07,
|
||||
"input_cost_per_token_priority": 5.625e-07,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 393216,
|
||||
"max_tokens": 393216,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.8e-06,
|
||||
"output_cost_per_token_priority": 2.25e-06,
|
||||
"source": "https://docs.fireworks.ai/serverless/us-only-serverless",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"wandb/reflection/Beam-1.0-501B-A23B": {
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "wandb",
|
||||
"max_input_tokens": 1049000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-06,
|
||||
"source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"wandb/zai-org/GLM-5.3": {
|
||||
"cache_read_input_token_cost": 1.9e-07,
|
||||
"input_cost_per_token": 1.2e-06,
|
||||
"litellm_provider": "wandb",
|
||||
"max_input_tokens": 1049000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4e-06,
|
||||
"source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"azure_ai/MAI-Code-1.1-Flash": {
|
||||
"cache_read_input_token_cost": 2e-08,
|
||||
"deprecation_date": "2026-12-09",
|
||||
"input_cost_per_token": 2e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 256000,
|
||||
"max_output_tokens": 64000,
|
||||
"max_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/responses"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue