scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-28 06:23:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 94

Merge remote-tracking branch 'origin/main' into litellm_stream_eof_without_finish_reason

Browse source 
# Conflicts:
#	tests/unit/llms/bedrock/chat/agentcore/test_agentcore_transformation.py
This commit is contained in:
yucheng-berri 2026-10-21 01:50:05 -07:00
commit 693b8cb014
1342 changed files with 44323 additions and 13056 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.circleci/config.yml
View file

@ -1291,7 +1291,7 @@ jobs:
name: Install MCP SDK1 peer
command: |
uv venv --python 3.12 .venv-mcp-peer
uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
- when:
condition:

2
.circleci/scripts/classify_changes.sh
View file

@ -33,7 +33,7 @@ while IFS= read -r file || [ -n "$file" ]; do
*) has_backend=true ;;
esac
case "$file" in
model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json | model_prices_and_context_window.schema.json)
model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json)
has_cost_map=true ;;
tests/unit/proxy/*) : ;;
*) outside_cost_map_set=true ;;

2
.github/PULL_REQUEST_TEMPLATE/general.md vendored
View file

@ -4,8 +4,6 @@
Drop every section you have nothing to put in, heading included: a bare "## Relevant issues" or
"## Affected release" with nothing under it must not appear in the final description -->
## Description
<!-- Two or three plain sentences a teammate can read in 10 seconds, written the way you'd explain it in Slack
Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
No headings, bullets, tables, or code names here. Everything else goes inside the Details block below -->

10
.github/workflows/test-unit.yml vendored
View file

@ -139,6 +139,7 @@ jobs:
test-path: |-
tests/unit/batch_completion
tests/unit/decisions
tests/unit/extraction
tests/unit/litellm_core_utils
tests/unit/systemone
python-version: "3.12"
@ -215,10 +216,12 @@ jobs:
- shard: llms-providers
test-path: |-
tests/unit/llms/test_api_base_trailing_slash_routes.py
tests/unit/llms/test_cache_control_and_reasoning.py
tests/unit/llms/test_custom_llm.py
tests/unit/llms/test_file_content_block.py
tests/unit/llms/test_file_search_responses.py
tests/unit/llms/test_get_complete_url_trailing_slash.py
tests/unit/llms/test_lifecycle_fix.py
tests/unit/llms/test_oss_decision.py
tests/unit/llms/test_polling_url_origin_match.py
@ -433,6 +436,13 @@ jobs:
- shard: rust-bridge-harness
test-path: >-
tests/test_litellm_rust/test_dispatch.py
tests/test_litellm_rust/test_public_entries.py
tests/test_litellm_rust/chat_completions/test_dispatch.py
tests/test_litellm_rust/embeddings/test_dispatch.py
tests/test_litellm_rust/messages/test_dispatch.py
tests/test_litellm_rust/ocr/test_dispatch.py
tests/test_litellm_rust/responses/test_dispatch.py
tests/unit/compression
tests/unit/harness
tests/unit/rust_bridge

6
ci_cd/cost_map_guard.py
View file

@ -4,8 +4,7 @@ Every pull request whose diff against its merge base touches one of the two cost
checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
get the file checks and additionally may only touch those two files or the legacy schema
path and may only add or update models.
get the file checks and additionally may only touch those two files and may only add or update models.
"""
from __future__ import annotations
@ -21,7 +20,6 @@ from typing import Final
COST_MAP_PATH: Final = "model_prices_and_context_window.json"
BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
LEGACY_SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
@ -110,7 +108,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
*(
f"bot PRs may only change the cost map files, not {path}"
for path in changed_files
if path not in GUARDED_PATHS and path != LEGACY_SCHEMA_PATH
if path not in GUARDED_PATHS
),
*(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map),
*(f"bot PRs may not remove fields: {ref}" for ref in removed_fields),

41
cookbook/litellm_proxy_server/batch_api/bedrock/README.md Normal file
View file

@ -0,0 +1,41 @@
# Bedrock batch endpoints
Set `aws_bedrock_control_plane_endpoint` on a deployment when batch create, retrieve, and cancel requests must use a custom Bedrock control-plane endpoint. The proxy signs these requests for the configured URL. Without this setting, batch requests use the standard regional Bedrock endpoint
```yaml
model_list:
- model_name: bedrock-batch
litellm_params:
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_region_name: us-east-1
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_batch_role_arn: os.environ/AWS_BATCH_ROLE_ARN
s3_output_bucket_name: os.environ/AWS_S3_OUTPUT_BUCKET_NAME
aws_bedrock_control_plane_endpoint: os.environ/BEDROCK_CONTROL_PLANE_ENDPOINT
```
Set `BEDROCK_CONTROL_PLANE_ENDPOINT` to your Bedrock control-plane VPC endpoint or compatible gateway base URL, without `/model-invocation-job`. The proxy adds the operation path. The endpoint must be reachable from the proxy, and the configured AWS region must match the endpoint's signing region
`api_base` and `aws_bedrock_runtime_endpoint` configure runtime inference separately. They do not override the batch control-plane endpoint. `s3_endpoint_url` configures S3 file access separately
The control-plane setting belongs in deployment configuration. Sending `aws_bedrock_control_plane_endpoint` in an API request body is rejected. There is no automatic `AWS_BEDROCK_CONTROL_PLANE_ENDPOINT` environment fallback, so reference the desired environment variable explicitly as shown above
Start the proxy with this configuration and set `LITELLM_API_BASE` to its URL. Set `BEDROCK_BATCH_INPUT_URI` to a valid input JSONL file in S3 that the batch service role can read
```bash
curl -sS "$LITELLM_API_BASE/v1/batches" \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H 'Content-Type: application/json' \
-d "{\"model\":\"bedrock-batch\",\"input_file_id\":\"$BEDROCK_BATCH_INPUT_URI\",\"endpoint\":\"/v1/chat/completions\",\"completion_window\":\"24h\"}"
```
Use the returned `id` as `BATCH_ID` to retrieve or cancel the same job through the deployment's configured endpoint
```bash
curl -sS "$LITELLM_API_BASE/v1/batches/$BATCH_ID?model=bedrock-batch" \
-H "Authorization: Bearer $LITELLM_MASTER_KEY"
curl -sS -X POST "$LITELLM_API_BASE/v1/batches/$BATCH_ID/cancel?model=bedrock-batch" \
-H "Authorization: Bearer $LITELLM_MASTER_KEY"
```

46
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tupl
from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
ManagedObjectCreatorAttribution,
ManagedObjectRow,
row_metadata,
)
import litellm
@ -90,11 +91,6 @@ def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.Li
return TeamRepository(prisma_client).table
def _row_metadata(row: object) -> dict[str, object]:
metadata: Final = getattr(row, "metadata", None)
return cast(dict[str, object], metadata) if isinstance(metadata, dict) else {} # cast-ok: prisma Json object column
class CheckBatchCost:
def __init__(
self,
@ -852,20 +848,26 @@ class CheckBatchCost:
)
return None
try:
await logging_obj.async_success_handler(
result=response,
batch_cost=batch_result.cost,
batch_usage=batch_result.usage,
batch_models=batch_result.models,
batch_successful_requests=batch_result.successful_requests,
batch_failed_requests=batch_result.failed_requests,
batch_prompt_cost=batch_result.prompt_cost,
batch_completion_cost=batch_result.completion_cost,
)
except Exception:
await self._release_job_claim(job, original_status, original_file_object)
raise
from litellm.proxy.hooks.proxy_track_cost_callback import track_batch_accounting
with track_batch_accounting(logging_obj.litellm_call_id) as accounting:
try:
await logging_obj.async_success_handler(
result=response,
batch_cost=batch_result.cost,
batch_usage=batch_result.usage,
batch_models=batch_result.models,
batch_successful_requests=batch_result.successful_requests,
batch_failed_requests=batch_result.failed_requests,
batch_prompt_cost=batch_result.prompt_cost,
batch_completion_cost=batch_result.completion_cost,
)
finally:
if accounting.retryable_failure:
await self._release_job_claim(job, original_status, original_file_object)
if accounting.retryable_failure:
return None
# Record batch duration (completed_at - created_at)
if prom_logger and response.completed_at and response.created_at:
@ -1057,7 +1059,9 @@ class CheckBatchCost:
if (has_key_row and key_row is None) or (team_id and team_row is None):
return True
settings: Final = get_dynamic_logging_metadata(
UserAPIKeyAuth(metadata=_row_metadata(key_row), team_metadata=_row_metadata(team_row), team_id=team_id),
UserAPIKeyAuth(
metadata=dict(row_metadata(key_row)), team_metadata=dict(row_metadata(team_row)), team_id=team_id
),
proxy_config,
)
callback_vars: Final = (settings.callback_vars if settings is not None else None) or {}
@ -1186,7 +1190,7 @@ class CheckBatchCost:
continue
verbose_proxy_logger.error(
f"CheckBatchCost: failed to track cost for batch {batch_id} "
f"(job {job.id}); leaving it unprocessed so the next poll retries: {tracking_err}"
f"(job {job.id}): {tracking_err}"
)
self._record_error(prom_logger, "cost_tracking_error")
continue

91
enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py
View file

@ -7,6 +7,12 @@ attribution, priced against the deployment that served it. The poll's status che
with BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN and client reads are marked managed by the
managed files hook, so both stay free. The spend event carries the id the client holds, the row's
unified_object_id, so its spend log is found under that id with the _background_cost suffix.
The spend event reaches the creating key's or team's own logging callbacks the way that creator's
requests do. Its message content is redacted when the creator's key or team logging turns message
logging off, else as the proxy is set, the way the batch poll decides it. The row is claimed before
the creator is read, so reads that lose the race skip the lookups. A creator whose lookup fails, or
whose logging settings the logger rejects, is still billed, through the proxy's own callbacks only
and with its message content redacted.
A read that finds no row, because the create could not store one, stores it under the reading key
and bills it the same way, so a response the create left untracked is still billed once. A streamed
background create is free on its own; its stream stores the row under the creating key and bills
@ -29,6 +35,7 @@ from typing import TYPE_CHECKING, Final, cast
from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
ManagedObjectCreatorAttribution,
ManagedObjectRow,
ResponseCreator,
)
from pydantic import BaseModel, ConfigDict, TypeAdapter
@ -54,11 +61,13 @@ from litellm.types.utils import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.proxy.proxy_server import ProxyConfig
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.repositories.prisma_protocols import TableActions
from litellm.router import Router
_STORED_RESPONSE_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
_CALLBACK_LIST_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...])
class _RoutedModelInfo(BaseModel):
@ -73,6 +82,8 @@ class _RoutedDeployment(BaseModel):
_ROUTED_DEPLOYMENTS: Final = TypeAdapter(tuple[_RoutedDeployment, ...])
_RETRIEVE_RESPONSE_CALL_TYPE: Final = "aget_responses"
_CALLBACK_LIST_FIELDS: Final = frozenset({"success_callback", "failure_callback", "litellm_disabled_callbacks"})
_REJECTED_CREATOR_LOGGING: Final[Mapping[str, object]] = MappingProxyType({"turn_off_message_logging": True})
@dataclass(frozen=True, slots=True)
@ -89,6 +100,7 @@ class BilledBackgroundResponse:
custom_llm_provider: str | None
litellm_params: Mapping[str, object]
metadata: Mapping[str, object]
creator_logging: Mapping[str, object]
@dataclass(frozen=True, slots=True)
@ -107,11 +119,51 @@ def _without_read_price(response: ResponsesAPIResponse) -> ResponsesAPIResponse:
return unpriced
async def log_billed_background_response(billed: BilledBackgroundResponse) -> BilledCharge:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
def _string_callbacks(callbacks: object) -> tuple[str, ...]:
if not isinstance(callbacks, list):
return ()
return tuple(
callback for callback in _CALLBACK_LIST_ADAPTER.validate_python(callbacks) if isinstance(callback, str)
)
logging_obj: Final = LiteLLMLogging(
def _running_proxy_config() -> "ProxyConfig":
from litellm.proxy.proxy_server import proxy_config
return proxy_config
def creator_logging_fields(creator: ResponseCreator, proxy_config: "ProxyConfig | None") -> Mapping[str, object]:
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.litellm_pre_call_utils import key_and_team_logging_fields
if not creator.settings_read:
return _REJECTED_CREATOR_LOGGING
try:
return key_and_team_logging_fields(
user_api_key_dict=UserAPIKeyAuth(
metadata=dict(creator.key_metadata),
team_id=creator.team_id,
team_metadata=dict(creator.team_metadata),
),
proxy_config=proxy_config if proxy_config is not None else _running_proxy_config(),
)
except Exception as e:
verbose_proxy_logger.error(
f"CheckResponsesCost: could not read the creator's logging settings, so only the proxy's "
f"callbacks get the billed event: {e}"
)
return _REJECTED_CREATOR_LOGGING
def _billed_logging_obj(billed: BilledBackgroundResponse, creator_logging: Mapping[str, object]) -> "LiteLLMLogging":
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.utils import split_dynamic_success_callbacks
sync_callbacks, async_callbacks = split_dynamic_success_callbacks(
_string_callbacks(creator_logging.get("success_callback"))
)
return LiteLLMLogging(
model=billed.model,
messages=[{"role": "user", "content": "<retrieve_response>"}],
stream=False,
@ -119,7 +171,27 @@ async def log_billed_background_response(billed: BilledBackgroundResponse) -> Bi
start_time=datetime.now(),
litellm_call_id=str(uuid.uuid4()),
function_id=str(uuid.uuid4()),
dynamic_success_callbacks=list(sync_callbacks),
dynamic_async_success_callbacks=list(async_callbacks) or None,
kwargs={key: value for key, value in creator_logging.items() if key not in _CALLBACK_LIST_FIELDS},
)
def _creator_or_proxy_logging_obj(billed: BilledBackgroundResponse) -> "LiteLLMLogging":
try:
return _billed_logging_obj(billed, billed.creator_logging)
except Exception as e:
verbose_proxy_logger.error(
f"CheckResponsesCost: the creator's logging settings were rejected, so only the proxy's "
f"callbacks get the billed event: {e}"
)
return _billed_logging_obj(billed, _REJECTED_CREATOR_LOGGING)
async def log_billed_background_response(billed: BilledBackgroundResponse) -> BilledCharge:
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
logging_obj: Final = _creator_or_proxy_logging_obj(billed)
logging_obj.update_environment_variables(
litellm_params={**billed.litellm_params, "metadata": dict(billed.metadata)},
optional_params={},
@ -187,6 +259,7 @@ class CheckResponsesCost:
prisma_client: "PrismaClient",
llm_router: "Router",
log_billed_response: BilledResponseLogger = log_billed_background_response,
proxy_config: "ProxyConfig | None" = None,
):
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.router import Router
@ -196,6 +269,7 @@ class CheckResponsesCost:
self.llm_router: Router = llm_router
self._creator_attribution: Final = ManagedObjectCreatorAttribution(prisma_client, "CheckResponsesCost")
self._log_billed_response: Final = log_billed_response
self._proxy_config: Final = proxy_config
def _resolve_deployment(self, response_id: str) -> Deployment | None:
model_id: str | None = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
@ -307,11 +381,12 @@ class CheckResponsesCost:
data={"status": job.status or PENDING_RESPONSE_STATUSES[0]},
)
async def _billed_response(
def _billed_response(
self,
job: ManagedObjectRow,
target: _PollTarget,
response: ResponsesAPIResponse,
creator: ResponseCreator,
) -> BilledBackgroundResponse:
deployment: Final = target.deployment
return BilledBackgroundResponse(
@ -323,13 +398,14 @@ class CheckResponsesCost:
litellm_params=_deployment_litellm_params(deployment) if deployment is not None else MappingProxyType({}),
metadata=MappingProxyType(
{
**(await self._creator_attribution.response_metadata(job, job.unified_object_id)),
**creator.metadata,
INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
**target.model_metadata,
**(_deployment_metadata(deployment) if deployment is not None else {}),
**self._requested_model_group(target, job.team_id),
}
),
creator_logging=creator_logging_fields(creator, self._proxy_config),
)
def _requested_model_group(self, target: _PollTarget, team_id: str | None) -> Mapping[str, object]:
@ -354,7 +430,8 @@ class CheckResponsesCost:
verbose_proxy_logger.info(f"Response {job.unified_object_id} was already billed elsewhere, skipping")
return None
try:
charge: Final = await self._log_billed_response(await self._billed_response(job, target, response))
creator: Final = await self._creator_attribution.response_creator(job, job.unified_object_id)
charge: Final = await self._log_billed_response(self._billed_response(job, target, response, creator))
except BaseException:
await self._release_billing_claim(job)
raise

185
enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py
View file

@ -10,12 +10,16 @@ from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import datetime
from typing import TYPE_CHECKING, Final, Protocol, cast
from typing import TYPE_CHECKING, Final, Protocol, TypeGuard, cast
from pydantic import TypeAdapter
from litellm._logging import verbose_proxy_logger
from litellm.constants import CLI_SESSION_KEY_PREFIX
from litellm.repositories.table_repositories import (
DeletedVerificationTokenRepository,
DeprecatedVerificationTokenRepository,
)
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
@ -70,6 +74,7 @@ class ManagedObjectRow(Protocol):
_TAG_LIST_ADAPTER: Final[TypeAdapter[list[object]]] = TypeAdapter(list[object])
_KEY_OWNER_SCOPE_PREFIX: Final = "key:"
_MAX_ROTATION_HOPS: Final = 16
def _string_tags(request_tags: object) -> list[str] | None:
@ -96,6 +101,20 @@ def _team_table(prisma_client: PrismaClient) -> TableActions[prisma_models.LiteL
return TeamRepository(prisma_client).table
def _rotated_token_table(
prisma_client: PrismaClient,
) -> TableActions[prisma_models.LiteLLM_DeprecatedVerificationToken]:
return DeprecatedVerificationTokenRepository(prisma_client).table
def _deleted_token_table(prisma_client: PrismaClient) -> TableActions[prisma_models.LiteLLM_DeletedVerificationToken]:
return DeletedVerificationTokenRepository(prisma_client).table
def _deleted_team_table(prisma_client: PrismaClient) -> TableActions[prisma_models.LiteLLM_DeletedTeamTable]:
return TeamRepository(prisma_client).deleted_table
@dataclass(frozen=True, slots=True)
class _CreatorRows:
key: prisma_models.LiteLLM_VerificationToken | None
@ -103,6 +122,15 @@ class _CreatorRows:
user: prisma_models.LiteLLM_UserTable | None
@dataclass(frozen=True, slots=True)
class ResponseCreator:
metadata: Mapping[str, object]
key_metadata: Mapping[str, object]
team_id: str | None
team_metadata: Mapping[str, object]
settings_read: bool = True
class ManagedObjectCreatorAttribution:
"""Rebuilds the spend-tracking metadata of whoever created a managed object, so the poll
job's spend row is attributed like the creating request: key hash, team, organization,
@ -117,7 +145,12 @@ class ManagedObjectCreatorAttribution:
A background response also carries its end user, the creating key's project and its alias,
and the key, team, and user budgets and per-model budgets, the way the read that billed it
before did, and never names a key-scoped creator as its user. A key that sets no budget of
its own takes its budget tier's, the same rule auth applies.
its own takes its budget tier's, the same rule auth applies. Its creator's key and team
metadata come along too, so the billed event reaches the key's and team's logging callbacks
like that read did. A rotated key's metadata is read from the live key its rotations lead
to, and a deleted key's or team's from its deleted row. A lookup that fails leaves the
creator's settings unread and the attribution to the rows that still load, so the
response still bills and still counts toward their budgets.
"""
def __init__(self, prisma_client: PrismaClient, poller_name: str) -> None:
@ -128,31 +161,70 @@ class ManagedObjectCreatorAttribution:
rows: Final = await self._rows(job, object_id)
return {"user_api_key_user_id": job.created_by, **_shared_metadata(job, rows)}
async def response_metadata(self, job: ManagedObjectRow, object_id: str) -> Mapping[str, object]:
rows: Final = await self._rows(job, object_id)
end_user_id: Final = getattr(job, "end_user_id", None)
return {
"user_api_key_user_id": _creator_user_id(job.created_by),
**_shared_metadata(job, rows),
**({"user_api_key_end_user_id": end_user_id} if end_user_id else {}),
"user_api_key_project_id": getattr(rows.key, "project_id", None),
"user_api_key_project_alias": getattr(
getattr(rows.key, "litellm_project_table", None), "project_alias", None
async def response_creator(self, job: ManagedObjectRow, object_id: str) -> ResponseCreator:
try:
return await self._read_response_creator(job, object_id)
except Exception as e:
verbose_proxy_logger.error(
f"{self._poller_name}: could not read the creator of {object_id}, so it bills without their "
f"settings: {e}"
)
return ResponseCreator(
metadata=_response_creator_metadata(job, await self._rows(job, object_id)),
key_metadata={},
team_id=getattr(job, "team_id", None),
team_metadata={},
settings_read=False,
)
async def _read_response_creator(self, job: ManagedObjectRow, object_id: str) -> ResponseCreator:
api_key: Final = getattr(job, "api_key", None)
team_id: Final = getattr(job, "team_id", None)
rows: Final = _CreatorRows(
key=await self._live_key_row(api_key, job.created_by),
team=await self._live_team_row(team_id),
user=await self._user_row(object_id, _creator_user_id(job.created_by)),
)
return ResponseCreator(
metadata=_response_creator_metadata(job, rows),
key_metadata=row_metadata(rows.key) if rows.key is not None else await self._retired_key_metadata(api_key),
team_id=team_id,
team_metadata=(
row_metadata(rows.team) if rows.team is not None else await self._deleted_team_metadata(team_id)
),
**_key_budget_fields(rows.key),
**_budget_fields(
rows.team,
"user_api_key_team_spend",
"user_api_key_team_max_budget",
"user_api_key_team_model_max_budget",
),
**_budget_fields(
rows.user,
"user_api_key_user_spend",
"user_api_key_user_max_budget",
"user_api_key_user_model_max_budget",
),
}
)
async def _retired_key_metadata(self, api_key: str | None) -> Mapping[str, object]:
if not _names_a_key_row(api_key):
return {}
current_token: Final = await self._token_rotated_to(api_key, _MAX_ROTATION_HOPS)
current_key: Final = (
await _token_table(self._prisma_client).find_unique(where={"token": current_token})
if current_token != api_key
else None
)
if current_key is not None:
return row_metadata(current_key)
return row_metadata(
await _deleted_token_table(self._prisma_client).find_first(
where={"token": current_token}, order={"deleted_at": "desc"}
)
)
async def _token_rotated_to(self, token: str, hops_left: int) -> str:
rotation: Final = await _rotated_token_table(self._prisma_client).find_unique(where={"token": token})
if rotation is None or hops_left == 0:
return token
return await self._token_rotated_to(rotation.active_token_id, hops_left - 1)
async def _deleted_team_metadata(self, team_id: str | None) -> Mapping[str, object]:
if not team_id:
return {}
return row_metadata(
await _deleted_team_table(self._prisma_client).find_first(
where={"team_id": team_id}, order={"deleted_at": "desc"}
)
)
async def _rows(self, job: ManagedObjectRow, object_id: str) -> _CreatorRows:
return _CreatorRows(
@ -175,25 +247,57 @@ class ManagedObjectCreatorAttribution:
async def _key_row(
self, object_id: str, api_key: str | None, created_by: str | None
) -> prisma_models.LiteLLM_VerificationToken | None:
if not api_key or _is_cli_session_key(api_key, created_by):
return None
try:
return await _token_table(self._prisma_client).find_unique(
where={"token": api_key}, include={"litellm_budget_table": True, "litellm_project_table": True}
)
return await self._live_key_row(api_key, created_by)
except Exception as e:
verbose_proxy_logger.error(f"{self._poller_name}: could not look up the creating key for {object_id}: {e}")
return None
async def _team_row(self, object_id: str, team_id: str | None) -> prisma_models.LiteLLM_TeamTable | None:
if not team_id:
async def _live_key_row(
self, api_key: str | None, created_by: str | None
) -> prisma_models.LiteLLM_VerificationToken | None:
if not api_key or _is_cli_session_key(api_key, created_by):
return None
return await _token_table(self._prisma_client).find_unique(
where={"token": api_key}, include={"litellm_budget_table": True, "litellm_project_table": True}
)
async def _team_row(self, object_id: str, team_id: str | None) -> prisma_models.LiteLLM_TeamTable | None:
try:
return await _team_table(self._prisma_client).find_unique(where={"team_id": team_id})
return await self._live_team_row(team_id)
except Exception as e:
verbose_proxy_logger.error(f"{self._poller_name}: could not look up team {team_id} for {object_id}: {e}")
return None
async def _live_team_row(self, team_id: str | None) -> prisma_models.LiteLLM_TeamTable | None:
if not team_id:
return None
return await _team_table(self._prisma_client).find_unique(where={"team_id": team_id})
def _response_creator_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str, object]:
end_user_id: Final = getattr(job, "end_user_id", None)
return {
"user_api_key_user_id": _creator_user_id(job.created_by),
**_shared_metadata(job, rows),
**({"user_api_key_end_user_id": end_user_id} if end_user_id else {}),
"user_api_key_project_id": getattr(rows.key, "project_id", None),
"user_api_key_project_alias": getattr(getattr(rows.key, "litellm_project_table", None), "project_alias", None),
**_key_budget_fields(rows.key),
**_budget_fields(
rows.team,
"user_api_key_team_spend",
"user_api_key_team_max_budget",
"user_api_key_team_model_max_budget",
),
**_budget_fields(
rows.user,
"user_api_key_user_spend",
"user_api_key_user_max_budget",
"user_api_key_user_model_max_budget",
),
}
def _shared_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str, object]:
api_key: Final = getattr(job, "api_key", None)
@ -221,6 +325,19 @@ def _shared_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str,
}
def row_metadata(row: object) -> Mapping[str, object]:
metadata: Final = getattr(row, "metadata", None)
if not isinstance(metadata, dict):
return {}
return cast(Mapping[str, object], metadata) # cast-ok: prisma Json object column
def _names_a_key_row(api_key: object) -> TypeGuard[str]:
from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash
return isinstance(api_key, str) and is_valid_sha256_hash(api_key)
def _is_cli_session_key(api_key: str, created_by: str | None) -> bool:
return bool(created_by) and api_key == f"{CLI_SESSION_KEY_PREFIX}-{created_by}"

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.78"
version = "0.1.79"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.78"
version = "0.1.79"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

4
gateway/routes/allowlist.py
View file

@ -1,7 +1,7 @@
"""Path allowlist for the gateway component.
The gateway exposes the LLM data-plane surface: chat/completions, embeddings,
audio, batches, files, fine-tuning, rerank, decisions, ocr, rag, video, search, image,
audio, batches, files, fine-tuning, rerank, decisions, extraction, ocr, rag, video, search, image,
responses, vector stores, passthrough providers, realtime websockets, MCP
tool-call endpoints, and operational endpoints (/health, /metrics, and the
/debug/memory/summary read of the serving worker's RSS).
@ -64,6 +64,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
"/decisions",
"/v1/systemone",
"/systemone",
"/v1/extraction",
"/extraction",
"/v1/ocr",
"/ocr",
"/v1/rag/",

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.110"
version = "0.4.111"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.110"
version = "0.4.111"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

2
litellm-rust/crates/python-bridge/src/cache/mod.rs vendored
View file

@ -4,7 +4,7 @@ mod selection;
pub(crate) use native::NativeCacheHandle;
pub(crate) use python::{CacheCall, PythonCache};
pub(crate) use selection::{Cached, Selection, admit_native, configure, configured_native};
pub(crate) use selection::{Cached, Selection, configure, configured_native};
use litellm_cache::Error;
use pyo3::{

2
litellm-rust/crates/python-bridge/src/cache/native/v2.rs vendored
View file

@ -279,7 +279,7 @@ pub(in crate::cache) fn configured(
)> {
let handle = native_handle(configured)?.ok_or_else(|| {
pyo3::exceptions::PyRuntimeError::new_err(
"the configured cache changed to a Python cache after native admission",
"native inference needs a v2 cache, the catalog keeps Python caches on Python",
)
})?;
let cache = handle.extract::<PyRef<'_, NativeCacheHandle>>()?;

15
litellm-rust/crates/python-bridge/src/cache/selection.rs vendored
View file

@ -76,21 +76,6 @@ fn selected_cache<'py>(
Ok(Some(configured))
}
pub(crate) fn admit_native(
py: Python<'_>,
kwargs: &Bound<'_, PyDict>,
call_type: &str,
) -> PyResult<()> {
if let Some(configured) = selected_cache(py, kwargs, call_type)?
&& native::v2::native_handle(&configured)?.is_none()
{
return Err(crate::errors::RustBridgeDeclined::new_err(
"the configured cache requires Python inference",
));
}
Ok(())
}
pub(crate) fn configured_native(
py: Python<'_>,
kwargs: &Bound<'_, PyDict>,

1
litellm-rust/crates/python-bridge/src/routes/inference.rs
View file

@ -177,7 +177,6 @@ where
} else {
R::SYNC_CALL_TYPE
};
crate::cache::admit_native(py, &call.kwargs, call_type)?;
let (arguments, hooks) = super::call_hooks(py, R::OPERATION, &call, asynchronous)?;
super::run_public_call(
py,

40
litellm-rust/crates/python-bridge/src/routes/responses/mod.rs
View file

@ -8,59 +8,19 @@ use litellm_auth::AuthServices;
use litellm_cache_response::{CachePolicy, ScopedCache};
use litellm_callbacks_legacy_python::LoggingOperation;
use litellm_host::{call::HostedMachine, protocol::Protocol};
use litellm_host_python::present;
use litellm_inference_responses::{ResponsesRoute, route::Responses};
use litellm_secrets::source::SecretSource;
use pyo3::prelude::*;
use serde_json::Value;
pub(crate) use websocket::ResponsesWebSocketConnection;
use super::{
NativeCall,
inference::{InferenceHost, InferenceRoute, run_inference},
};
use crate::errors::RustBridgeDeclined;
const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.responses";
fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyResult<Py<PyAny>> {
let resolved = call.resolved()?;
if let Some(reason) = py
.import(ROUTE_HOST_MODULE)?
.getattr("decline_reason")?
.call1((&resolved,))?
.extract::<Option<String>>()?
{
return Err(RustBridgeDeclined::new_err(reason));
}
let argument = |name: &str| present(&call.kwargs, &call.base, name);
let model = argument("model")?
.ok_or_else(|| pyo3::exceptions::PyValueError::new_err("model is required"))?
.extract::<String>()?;
let provider = argument("custom_llm_provider")?
.map(|value| value.extract::<String>())
.transpose()?;
if provider
.as_deref()
.is_some_and(|provider| provider != "openai")
|| model
.strip_prefix("openai/")
.unwrap_or(&model)
.contains('/')
{
return Err(RustBridgeDeclined::new_err(
"native HTTP responses provider",
));
}
if argument("stream")?
.map(|value| litellm_host_python::from_py::<Value>(&value))
.transpose()?
.is_some_and(|value| value == Value::Bool(true))
{
return Err(RustBridgeDeclined::new_err(
"native Python responses streaming",
));
}
let host = InferenceHost::new(call.view()?, ROUTE_HOST_MODULE);
run_inference::<ResponsesRoute, _>(py, call, asynchronous, ResponsesPythonHost::new(host))
}

4
litellm/__init__.py
View file

@ -1505,6 +1505,7 @@ from .rag.main import *
from .sandbox.main import *
from .decisions.main import *
from .systemone.main import *
from .extraction.main import *
from .tool_loop import ToolLoopMaxRoundsExceeded, arun_tool_loop, run_tool_loop
from .search.main import *
from .realtime_api.main import (
@ -1711,6 +1712,9 @@ if TYPE_CHECKING:
from .llms.cloudflare.decisions.transformation import (
CloudflareDecisionsConfig as CloudflareDecisionsConfig,
)
from .llms.cloudflare.extraction.transformation import (
CloudflareExtractionConfig as CloudflareExtractionConfig,
)
from .llms.strands_decider.decisions.transformation import (
StrandsDeciderDecisionsConfig as StrandsDeciderDecisionsConfig,
)

2
litellm/_lazy_imports_registry.py
View file

@ -166,6 +166,7 @@ LLM_CONFIG_NAMES: Final = (
"OpenRouterDecisionsConfig",
"FireworksAIDecisionsConfig",
"CloudflareDecisionsConfig",
"CloudflareExtractionConfig",
"StrandsDeciderDecisionsConfig",
"DatabricksDecisionsConfig",
"HostedVLLMDecisionsConfig",
@ -735,6 +736,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
"FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),
"CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"),
"CloudflareExtractionConfig": (".llms.cloudflare.extraction.transformation", "CloudflareExtractionConfig"),
"StrandsDeciderDecisionsConfig": (
".llms.strands_decider.decisions.transformation",
"StrandsDeciderDecisionsConfig",

10
litellm/caching/in_memory_cache.py
View file

@ -95,7 +95,10 @@ class InMemoryCache(BaseCache):
"""
Check if a specific key is expired
"""
return key in self.ttl_dict and self._clock() > self.ttl_dict[key]
try:
return self._clock() > self.ttl_dict[key]
except KeyError:
return False
def _cache_keys(self) -> tuple[str, ...]:
return tuple(self.cache_dict) # pyright: ignore[reportUnknownVariableType] # cache_dict is untyped legacy state
@ -227,7 +230,10 @@ class InMemoryCache(BaseCache):
if key in self.cache_dict:
if self.evict_element_if_expired(key):
return None
original_cached_response: Final = self.cache_dict[key]
try:
original_cached_response: Final = self.cache_dict[key]
except KeyError:
return None
try:
cached_response = json.loads(original_cached_response)
except Exception:

35
litellm/chat_completions/dispatch.py
View file

@ -1,15 +1,19 @@
import inspect
from collections.abc import Awaitable, Callable, Coroutine, Mapping
from types import MappingProxyType
from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable
from litellm import main
from litellm.rust_bridge.catalog import Route, RouteContext
from litellm.rust_bridge import detectors
from litellm.rust_bridge.catalog import Feature, Route, RouteContext
from litellm.rust_bridge.chat_completions import detectors as route_detectors
from litellm.rust_bridge.chat_completions.entrypoints import (
NATIVE_ACOMPLETION,
NATIVE_COMPLETION,
)
from litellm.rust_bridge.dispatch import PublicDispatch
from litellm.rust_bridge.public_call import (
AsyncFlag,
NativeCall,
bind,
native_call,
@ -44,6 +48,7 @@ def _python_acompletion() -> PythonAcompletion:
_PYTHON_COMPLETION: Final = _python_completion()
_COMPLETION: Final = signature(_PYTHON_COMPLETION)
_FLAG: Final = AsyncFlag("acompletion", sync=_COMPLETION)
_PYTHON_ACOMPLETION: Final = _python_acompletion()
_ACOMPLETION: Final = signature(_PYTHON_ACOMPLETION)
@ -61,11 +66,32 @@ def _public_request(
return native_call(legacy, args, kwargs)
_FEATURES: Final[detectors.Detectors] = MappingProxyType(
{
**detectors.COMMON,
Feature.MOCK_RESPONSE: route_detectors.chat_mock_response,
Feature.AGENTIC_LOOP_HOOK: route_detectors.chat_agentic_loop_hook,
Feature.RESPONSE_CACHE: detectors.response_cache,
Feature.STREAMING: detectors.streaming,
Feature.DROP_PARAMS: detectors.drop_params,
Feature.MODIFY_PARAMS: detectors.modify_params,
Feature.EXTRA_BODY: detectors.extra_body,
Feature.RETRIES_FALLBACKS: route_detectors.chat_retries_fallbacks,
Feature.PROMPT_MANAGEMENT: detectors.prompt_management,
Feature.CACHE_CONTROL_INJECTION: detectors.cache_control_injection,
Feature.MODEL_PARAM_GATING: route_detectors.chat_model_param_gating,
Feature.MODEL_LIST: route_detectors.model_list,
Feature.MODEL_ALIAS_MAP: route_detectors.model_alias_map,
}
)
def _context(request: NativeCall) -> RouteContext:
return RouteContext(
Route.CHAT_COMPLETIONS,
provider=optional_str(request.resolved.get("custom_llm_provider")),
model=str(request.resolved["model"]),
features=detectors.detected(_FEATURES, request),
)
@ -73,7 +99,6 @@ _DISPATCH: Final = PublicDispatch(
route=Route.CHAT_COMPLETIONS,
request=lambda args, kwargs: _public_request(_COMPLETION, args, kwargs),
context=_context,
bypass=lambda request: request.kwargs.get("acompletion") is True,
)
_ADISPATCH: Final = PublicDispatch(
@ -88,9 +113,11 @@ def completion(
**kwargs: object, # kwargs-ok: preserve the public chat completions call shape
) -> ChatResult | Coroutine[object, object, ChatResult]:
python: Final = _PYTHON_COMPLETION
if _FLAG.requested(kwargs):
return acompletion(**_FLAG.keywords(args, kwargs))
return _DISPATCH.run(
args,
kwargs,
_FLAG.dropped(kwargs),
python=python,
binding=NATIVE_COMPLETION,
native=native_call_hook,
@ -101,7 +128,7 @@ async def acompletion(*args: object, **kwargs: object) -> ChatResult: # kwargs-
python: Final = _PYTHON_ACOMPLETION
return await _ADISPATCH.arun(
args,
kwargs,
_FLAG.dropped(kwargs),
python=python,
binding=NATIVE_ACOMPLETION,
native=native_call_hook,

11
litellm/constants.py
View file

@ -25,6 +25,8 @@ CLAUDE_CODE_CLIENT: Final = "claude-code"
MICROSOFT_365_COPILOT_DEFAULT_TIME_ZONE: Final = "UTC"
GITHUB_COPILOT_PER_USER_AUTH_TYPE: Final = "per_user_oauth"
GITHUB_COPILOT_AUTH_TYPE_KEY: Final = "github_copilot_auth_type"
GITHUB_COPILOT_HOST_KEY: Final = "github_copilot_host"
GITHUB_COPILOT_DEFAULT_HOST: Final = "github.com"
GITHUB_COPILOT_USER_TOKEN_SAFETY_MARGIN_SECONDS: Final = 60
GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS: Final = 60
GITHUB_COPILOT_DEVICE_FLOW_CACHE_PREFIX: Final = "github_copilot_device_flow"
@ -154,6 +156,9 @@ DEFAULT_REPLICATE_POLLING_RETRIES: Final = get_env_int("DEFAULT_REPLICATE_POLLIN
DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)
DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250)
HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS: Final = get_env_int_in_range(
"HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 300, 1, 86400
)
# Maximum wall-clock seconds a streaming response is allowed to run.
# Streams exceeding this duration are terminated with a Timeout error.
@ -256,6 +261,9 @@ MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS: Final = get_env_int("MCP_PER_USER_TOKE
# MCP timeout defaults (seconds). Override via env vars for slow/custom MCP servers.
MCP_CLIENT_TIMEOUT: Final = get_env_float("LITELLM_MCP_CLIENT_TIMEOUT", 60.0)
MCP_MAX_SSE_EVENT_SIZE_BYTES: Final = get_env_int_in_range(
"LITELLM_MCP_MAX_SSE_EVENT_SIZE_BYTES", 64 * 1024 * 1024, 1, sys.maxsize
)
MCP_TOOL_LISTING_TIMEOUT: Final = get_env_float("LITELLM_MCP_TOOL_LISTING_TIMEOUT", 30.0)
MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES: Final = get_env_int("LITELLM_MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES", 3)
MCP_UPSTREAM_RATE_LIMIT_BASE_DELAY_SECONDS: Final = get_env_float(
@ -1649,6 +1657,7 @@ BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
@ -1812,6 +1821,7 @@ LITELLM_EXECUTED_BATCH_CONCURRENCY: Final = max(1, get_env_int("LITELLM_EXECUTED
########################### CLI SSO AUTHENTICATION CONSTANTS ###########################
LITELLM_CLI_SOURCE_IDENTIFIER: Final = "litellm-cli"
LITELLM_CLI_SESSION_TOKEN_PREFIX: Final = "litellm-session-token"
CP_RETURN_TO_COOKIE_PREFIX: Final = "return_to.v1."
CLI_SSO_SESSION_CACHE_KEY_PREFIX: Final = "cli_sso_session"
CLI_SSO_SESSION_TTL_SECONDS: Final = 600
CLI_SESSION_KEY_PREFIX: Final = "cli-session"
@ -2320,6 +2330,7 @@ HARNESS_STDERR_TAIL_LINES: Final = 40
HARNESS_STREAM_READ_CHUNK_BYTES: Final = 64 * 1024
HARNESS_EVENT_QUEUE_MAX_SIZE: Final = 1024
HARNESS_PROCESS_KILL_GRACE_SECONDS: Final = 5.0
HARNESS_RESUME_COMMAND_TIMEOUT_SECONDS: Final = 30.0
HARNESS_SNAPSHOT_SKIP_DIRS: Final = frozenset(
{
".git",

43
litellm/cost_calculator.py
View file

@ -112,6 +112,7 @@ from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_toke
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.agents import LiteLLMSendMessageResponse
from litellm.types.decisions import DecisionsResponse, DecisionsUsage, OpenAIDecisionResponse, OpenAIDecisionUsage
from litellm.types.extraction import ExtractionResponse, ExtractionUsage
from litellm.types.llms.base import CachedTokensDetails, LiteLLMBaseModel
from litellm.types.llms.openai import (
HttpxBinaryResponseContent,
@ -1171,6 +1172,8 @@ def get_usage_object(
if isinstance(completion_response, (DecisionsResponse, OpenAIDecisionResponse)):
return None if completion_response.usage is None else _decisions_usage(completion_response.usage)
if isinstance(completion_response, ExtractionResponse):
return _extraction_usage(completion_response.usage)
if isinstance(completion_response, HttpxBinaryResponseContent):
return completion_response.usage
if usage_obj is None:
@ -1227,6 +1230,14 @@ def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage:
)
def _extraction_usage(usage: ExtractionUsage) -> Usage:
return Usage(
prompt_tokens=usage.input_tokens,
completion_tokens=usage.output_tokens,
total_tokens=usage.input_tokens + usage.output_tokens,
)
def _is_known_usage_objects(usage_obj):
"""Returns True if the usage obj is a known Usage type"""
return (
@ -1234,6 +1245,7 @@ def _is_known_usage_objects(usage_obj):
or isinstance(usage_obj, ResponseAPIUsage)
or isinstance(usage_obj, DecisionsUsage)
or isinstance(usage_obj, OpenAIDecisionUsage)
or isinstance(usage_obj, ExtractionUsage)
or TranscriptionUsageObjectTransformation.is_transcription_usage_object(usage_obj)
)
@ -1263,6 +1275,8 @@ def infer_call_type(call_type: CallTypesLiteral | None, completion_response: obj
return "send_message"
elif isinstance(completion_response, (DecisionsResponse, OpenAIDecisionResponse)):
return "adecisions"
elif isinstance(completion_response, ExtractionResponse):
return "aextraction"
return call_type
@ -1511,6 +1525,7 @@ def completion_cost(
litellm_model_name: str | None = None,
router_model_id: str | None = None,
litellm_logging_obj: LitellmLoggingObject | None = None,
pricing_provider: str | None = None,
### SERVICE TIER ###
service_tier: str | None = None, # for OpenAI service tier pricing
### DATA RESIDENCY ###
@ -1531,6 +1546,8 @@ def completion_cost(
"""
try:
call_type = infer_call_type(call_type, completion_response) or "completion"
if pricing_provider is not None:
custom_llm_provider = pricing_provider # rebind-ok: model_info.pricing_provider overrides the routing provider for cost-map lookups
if call_type == CallTypes.aresponses_websocket.value and isinstance(
completion_response, LiteLLMRealtimeStreamLoggingObject
@ -1556,6 +1573,7 @@ def completion_cost(
optional_params=optional_params,
custom_pricing=custom_pricing,
base_model=base_model,
pricing_provider=pricing_provider,
standard_built_in_tools_params=standard_built_in_tools_params,
litellm_model_name=litellm_model_name,
router_model_id=router_model_id,
@ -1649,14 +1667,16 @@ def completion_cost(
raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates)
)
for idx, (model, pricing_provider) in enumerate(potential_model_names):
for idx, (model, candidate_pricing_provider) in enumerate(potential_model_names):
try:
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug("selected model name for cost calculation: %s", model)
if completion_response is not None and (
isinstance(completion_response, BaseModel) or isinstance(completion_response, dict)
): # tts returns a custom class
isinstance(completion_response, (BaseModel, dict))
or isinstance(completion_response, HttpxBinaryResponseContent)
and completion_response.usage is not None
):
if isinstance(completion_response, dict):
usage_obj: dict | Usage | None = completion_response.get("usage", {})
else:
@ -1670,6 +1690,8 @@ def completion_cost(
)
if isinstance(usage_obj, (DecisionsUsage, OpenAIDecisionUsage)):
_usage = _decisions_usage(usage_obj).model_dump()
elif isinstance(usage_obj, ExtractionUsage):
_usage = _extraction_usage(usage_obj).model_dump()
elif usage_obj is None:
_usage = {}
elif isinstance(usage_obj, BaseModel):
@ -1716,7 +1738,9 @@ def completion_cost(
hidden_params = getattr(completion_response, "_hidden_params", None)
if hidden_params is not None:
custom_llm_provider = hidden_params.get("custom_llm_provider", custom_llm_provider or None)
custom_llm_provider = pricing_provider or hidden_params.get(
"custom_llm_provider", custom_llm_provider or None
)
region_name = hidden_params.get("region_name", region_name)
else:
@ -2002,7 +2026,7 @@ def completion_cost(
prompt_tokens=prompt_tokens or 0,
completion_tokens=completion_tokens or 0,
custom_llm_provider=_pricing_provider_for_cost_candidate(
candidate_provider=pricing_provider,
candidate_provider=candidate_pricing_provider,
request_provider=request_pricing_provider,
current_provider=custom_llm_provider,
),
@ -2106,7 +2130,7 @@ def completion_cost(
_token_type_breakdown = get_token_type_cost_breakdown(
model=model,
custom_llm_provider=_pricing_provider_for_cost_candidate(
candidate_provider=pricing_provider,
candidate_provider=candidate_pricing_provider,
request_provider=request_pricing_provider,
current_provider=custom_llm_provider,
),
@ -2193,7 +2217,8 @@ def response_cost_calculator(
| Response
| SearchResponse
| DecisionsResponse
| OpenAIDecisionResponse,
| OpenAIDecisionResponse
| ExtractionResponse,
model: str,
custom_llm_provider: str | None,
call_type: Literal[
@ -2219,6 +2244,8 @@ def response_cost_calculator(
"adecisions",
"systemone",
"asystemone",
"extraction",
"aextraction",
],
optional_params: dict,
cache_hit: bool | None = None,
@ -2229,6 +2256,7 @@ def response_cost_calculator(
litellm_model_name: str | None = None,
router_model_id: str | None = None,
litellm_logging_obj: LitellmLoggingObject | None = None,
pricing_provider: str | None = None,
### SERVICE TIER ###
service_tier: str | None = None, # for OpenAI service tier pricing
### DATA RESIDENCY ###
@ -2269,6 +2297,7 @@ def response_cost_calculator(
litellm_model_name=litellm_model_name,
router_model_id=router_model_id,
litellm_logging_obj=litellm_logging_obj,
pricing_provider=pricing_provider,
service_tier=service_tier,
data_residency=data_residency,
vertex_location=vertex_location,

18
litellm/embeddings/dispatch.py
View file

@ -9,7 +9,15 @@ from litellm.rust_bridge.embeddings.entrypoints import (
NATIVE_AEMBEDDING,
NATIVE_EMBEDDING,
)
from litellm.rust_bridge.public_call import NativeCall, bind, native_call, native_call_hook, optional_str, signature
from litellm.rust_bridge.public_call import (
AsyncFlag,
NativeCall,
bind,
native_call,
native_call_hook,
optional_str,
signature,
)
from litellm.types.utils import EmbeddingResponse
__all__ = ("aembedding", "embedding")
@ -24,6 +32,7 @@ _PYTHON_AEMBEDDING: Final = cast( # cast-ok: [LIT006] preserve the legacy publi
PythonAembedding, main.aembedding
)
_EMBEDDING_SIGNATURE: Final = signature(_PYTHON_EMBEDDING)
_FLAG: Final = AsyncFlag("aembedding", sync=_EMBEDDING_SIGNATURE)
def _public_request(
@ -50,7 +59,6 @@ _DISPATCH: Final = PublicDispatch(
route=Route.EMBEDDINGS,
request=lambda args, kwargs: _public_request(_EMBEDDING_SIGNATURE, args, kwargs),
context=_context,
bypass=lambda request: request.kwargs.get("aembedding") is True,
)
_ADISPATCH: Final = PublicDispatch(
@ -64,9 +72,11 @@ def embedding(
*args: object,
**kwargs: object, # kwargs-ok: preserve the public embedding call shape
) -> EmbeddingResponse | Coroutine[object, object, EmbeddingResponse]:
if _FLAG.requested(kwargs):
return aembedding(**_FLAG.keywords(args, kwargs))
return _DISPATCH.run(
args,
kwargs,
_FLAG.dropped(kwargs),
python=_PYTHON_EMBEDDING,
binding=NATIVE_EMBEDDING,
native=native_call_hook,
@ -76,7 +86,7 @@ def embedding(
async def aembedding(*args: object, **kwargs: object) -> EmbeddingResponse: # kwargs-ok: preserve the public call shape
return await _ADISPATCH.arun(
args,
kwargs,
_FLAG.dropped(kwargs),
python=_PYTHON_AEMBEDDING,
binding=NATIVE_AEMBEDDING,
native=native_call_hook,

1
litellm/endpoints/speech/speech_to_completion_bridge/handler.py
View file

@ -122,6 +122,7 @@ class SpeechToCompletionBridgeHandler:
if isinstance(result, ModelResponse):
return self.transformation_handler.transform_response(
model_response=result,
model=model,
response_format=requested_response_format if isinstance(requested_response_format, str) else None,
)
else:

67
litellm/endpoints/speech/speech_to_completion_bridge/transformation.py
View file

@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Final, cast
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache, set_hidden_param
if TYPE_CHECKING:
@ -38,6 +39,11 @@ class ChatAudioParam(TypedDict):
format: ReadOnly[NotRequired[str]]
class ChatSpeechConfigAudioParam(TypedDict):
speech_config: ReadOnly[Mapping[str, object]]
format: ReadOnly[str]
class SpeechToCompletionBridgeTransformationHandler:
def _validate_response_format(
self, model: str, custom_llm_provider: str, optional_params: Mapping[str, object]
@ -49,6 +55,20 @@ class SpeechToCompletionBridgeTransformationHandler:
return
from litellm.exceptions import BadRequestError
if custom_llm_provider.startswith("vertex_ai"):
from litellm.llms.vertex_ai.text_to_speech.transformation import VertexAITextToSpeechConfig
vertex_supported: Final = ", ".join(
sorted({*GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS, *VertexAITextToSpeechConfig.GEMINI_FORMAT_MAPPINGS})
)
raise BadRequestError(
message=(
f"Vertex AI Gemini TTS does not support response_format='{response_format}'."
f" Supported response formats: {vertex_supported}."
),
model=model,
llm_provider=custom_llm_provider,
)
supported: Final = ", ".join(sorted(GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS))
raise BadRequestError(
message=(
@ -75,8 +95,27 @@ class SpeechToCompletionBridgeTransformationHandler:
return response_format if isinstance(response_format, str) else None
def _chat_audio_param(
self, model: str, voice: str | Mapping[str, object] | None, optional_params: Mapping[str, object]
) -> ChatAudioParam | None:
self,
model: str,
voice: str | Mapping[str, object] | None,
optional_params: Mapping[str, object],
drop_params: bool,
custom_llm_provider: str,
) -> ChatAudioParam | ChatSpeechConfigAudioParam | None:
import litellm
if isinstance(voice, Mapping) and self._is_gemini_tts_model(litellm.model_alias_map.get(model, model)):
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
normalize_gemini_speech_config,
reject_unknown_gemini_tts_voice_keys,
)
reject_unknown_gemini_tts_voice_keys(model, voice, drop_params, custom_llm_provider)
speech_audio: Final[ChatSpeechConfigAudioParam] = {
"speech_config": normalize_gemini_speech_config(voice, drops_unsupported_params(drop_params)),
"format": GEMINI_TTS_CHAT_AUDIO_FORMAT,
}
return speech_audio
if not isinstance(voice, str):
return None
audio_format: Final = self._chat_audio_format(model, optional_params)
@ -90,9 +129,9 @@ class SpeechToCompletionBridgeTransformationHandler:
self,
model: str,
input: str,
voice: str | dict | None,
optional_params: dict,
litellm_params: dict,
voice: str | Mapping[str, object] | None,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
headers: dict,
litellm_logging_obj: "LiteLLMLoggingObj",
custom_llm_provider: str,
@ -104,7 +143,13 @@ class SpeechToCompletionBridgeTransformationHandler:
"messages": [user_message],
"modalities": ["audio"],
**self._chat_completion_params(optional_params),
"audio": self._chat_audio_param(model, voice, optional_params),
"audio": self._chat_audio_param(
model,
voice,
optional_params,
drops_unsupported_params(litellm_params.get("drop_params")),
custom_llm_provider,
),
**litellm_params,
"headers": headers,
"litellm_logging_obj": litellm_logging_obj,
@ -153,8 +198,9 @@ class SpeechToCompletionBridgeTransformationHandler:
return wav_header + pcm_data
def _is_gemini_tts_model(self, model: str) -> bool:
"""Check if the model is a Gemini TTS model that returns PCM16 data."""
return "gemini" in model.lower() and ("tts" in model.lower() or "preview-tts" in model.lower())
from litellm.utils import is_gemini_tts_model
return is_gemini_tts_model(model)
def _gemini_tts_response_body(self, decoded_audio: bytes, response_format: str | None) -> tuple[bytes, str]:
if response_format == GEMINI_TTS_RAW_RESPONSE_FORMAT:
@ -162,7 +208,7 @@ class SpeechToCompletionBridgeTransformationHandler:
return self._convert_pcm16_to_wav(decoded_audio), "audio/wav"
def transform_response(
self, model_response: "ModelResponse", response_format: str | None
self, model_response: "ModelResponse", model: str, response_format: str | None
) -> "HttpxBinaryResponseContent":
import base64
@ -176,10 +222,9 @@ class SpeechToCompletionBridgeTransformationHandler:
raise ValueError("No audio part found in the response")
decoded_audio: Final = base64.b64decode(audio_part.data)
model: Final = getattr(model_response, "model", "")
content, content_type = (
self._gemini_tts_response_body(decoded_audio, response_format)
if self._is_gemini_tts_model(model)
if self._is_gemini_tts_model(model) or self._is_gemini_tts_model(model_response.model or "")
else (decoded_audio, "audio/mpeg")
)
response: Final = httpx.Response(

13
litellm/experimental_mcp_client/client.py
View file

@ -95,6 +95,7 @@ from pydantic import AnyUrl, TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import (
MCP_CLIENT_TIMEOUT,
MCP_MAX_SSE_EVENT_SIZE_BYTES,
MCP_NPM_CACHE_DIR,
MCP_TOOL_LISTING_MAX_PAGES,
MCP_TOOL_LISTING_TIMEOUT,
@ -292,6 +293,7 @@ class _MCPResponseStream(httpx2.AsyncByteStream):
class _MCPHTTPClient(httpx2.AsyncClient):
cleanup_scope: anyio.CancelScope | None = None
cleanup_errors: tuple[Exception, ...] = ()
post_failure: asyncio.Future[Exception] | None = None
def _record_cleanup_error(self, error: Exception) -> None:
if self.cleanup_scope is not None and self.cleanup_scope.shield:
@ -319,7 +321,12 @@ class _MCPHTTPClient(httpx2.AsyncClient):
response: Final = await super().send(request, stream=stream, auth=auth, follow_redirects=follow_redirects)
if request.method == "POST" and response.is_error and response.status_code != 404:
await response.aclose()
response.raise_for_status()
try:
response.raise_for_status()
except httpx2.HTTPStatusError as error:
if self.post_failure is not None and not self.post_failure.done():
self.post_failure.set_result(error)
raise
if stream:
response.stream = _MCPResponseStream(
cast(httpx2.AsyncByteStream, response.stream), self._record_cleanup_error
@ -641,6 +648,7 @@ class MCPClient:
transport_ctx: Final = streamable_http_client(
url=self.server_url,
http_client=http_client,
max_sse_event_size=MCP_MAX_SSE_EVENT_SIZE_BYTES,
)
return transport_ctx, http_client
@ -757,6 +765,8 @@ class MCPClient:
read_stream: Final = transport[0]
write_stream: Final = transport[1]
stream_error: Final[asyncio.Future[Exception]] = asyncio.get_running_loop().create_future()
if isinstance(http_client, _MCPHTTPClient):
http_client.post_failure = stream_error # rebind-ok: operation-scoped stream error
_notify_stream_error(on_stream_error, stream_error)
session_kwargs: Final = (
@ -823,6 +833,7 @@ class MCPClient:
if isinstance(http_client, _MCPHTTPClient):
http_client.cleanup_errors = ()
http_client.cleanup_scope = None
http_client.post_failure = None # rebind-ok: clear operation-scoped stream error
await anyio.lowlevel.checkpoint_if_cancelled()
if cleanup_scope.cancel_called:
raise (

3
litellm/extraction/__init__.py Normal file
View file

@ -0,0 +1,3 @@
from litellm.extraction.main import aextraction, extraction
__all__ = ["aextraction", "extraction"]

351
litellm/extraction/main.py Normal file
View file

@ -0,0 +1,351 @@
import asyncio
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from functools import partial
from types import MappingProxyType
from typing import Final, TypeAlias
import httpx
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.decisions.call import log_caller_request
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.tool_fields import DropPolicy, drop_policy
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.extraction.transformation import BaseExtractionConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.types.extraction import (
EXTRACTION_STATE_REWRITTEN_METADATA_KEY,
ExtractionAnswer,
ExtractionBase64Image,
ExtractionField,
ExtractionOptions,
ExtractionRequestBody,
ExtractionResponse,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders, is_litellm_owned_kwarg
from litellm.utils import (
ProviderConfigManager,
client, # pyright: ignore[reportUnknownVariableType] # client is an untyped decorator
get_model_cost_key,
)
ExtractionFields: TypeAlias = Mapping[str, ExtractionField | Mapping[str, object]]
ExtractionImages: TypeAlias = Sequence[str | ExtractionBase64Image | Mapping[str, object]]
ExtractionOptionsParam: TypeAlias = ExtractionOptions | Mapping[str, object]
_REQUEST_ADAPTER: Final[TypeAdapter[ExtractionRequestBody]] = TypeAdapter(ExtractionRequestBody)
_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
_HANDLER: Final = BaseLLMHTTPHandler()
_LOCAL_KWARGS: Final = frozenset(
(
*GenericLiteLLMParams.model_fields,
"extra_headers",
"base_url",
"default_headers",
"deployment_id",
"additional_drop_params",
)
)
@dataclass(frozen=True, slots=True, repr=False)
class _ExtractionCall:
model: str
requested_model: str
custom_llm_provider: str
provider_config: BaseExtractionConfig
body: Mapping[str, object]
api_base: str
api_key: str | None
logging_obj: LiteLLMLoggingObj | None
headers: Mapping[str, str]
timeout: float | httpx.Timeout | None
litellm_params: Mapping[str, object]
client: HTTPHandler | AsyncHTTPHandler | None
def supported_extraction_providers() -> tuple[str, ...]:
return tuple(
provider.value
for provider in LlmProviders
if ProviderConfigManager.get_provider_extraction_config(model="", provider=provider) is not None
)
def _provider_config(model: str, custom_llm_provider: str) -> BaseExtractionConfig:
provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None)
provider_config: Final = (
None if provider is None else ProviderConfigManager.get_provider_extraction_config(model, provider)
)
if provider_config is None:
raise litellm.BadRequestError(
message=(
f"Unknown Extraction provider '{custom_llm_provider}'. "
f"Supported providers: {', '.join(supported_extraction_providers())}"
),
model=model,
llm_provider=custom_llm_provider,
)
return provider_config
def _request_fields(
*,
state: str | None,
fields: ExtractionFields | None,
images: ExtractionImages | None,
reference_date: str | None,
options: ExtractionOptionsParam | None,
) -> Mapping[str, object]:
sent: Final[Mapping[str, object | None]] = MappingProxyType(
{"state": state, "fields": fields, "images": images, "reference_date": reference_date, "options": options}
)
return MappingProxyType({name: value for name, value in sent.items() if value is not None})
def _is_unsupported(name: str, value: object, policy: DropPolicy) -> bool:
return (
value is not None
and value is not False
and name not in _LOCAL_KWARGS
and not is_litellm_owned_kwarg(name)
and not policy.drops(name)
)
def _unsupported_params(kwargs: Mapping[str, object]) -> tuple[str, ...]:
policy: Final = drop_policy(kwargs)
return tuple(sorted(name for name, value in kwargs.items() if _is_unsupported(name, value, policy)))
def _prepare_call(
*,
model: str,
request_fields: Mapping[str, object],
api_key: str | None,
api_base: str | None,
timeout: float | httpx.Timeout | None,
custom_llm_provider: str | None,
extra_headers: Mapping[str, str] | None,
kwargs: Mapping[str, object],
) -> _ExtractionCall:
log_caller_request(kwargs, request_fields)
upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
api_key=api_key,
)
provider_config: Final = _provider_config(model, provider)
unsupported: Final = _unsupported_params(kwargs)
if unsupported:
raise litellm.UnsupportedParamsError(
message=(
f"{provider} does not support parameters: {list(unsupported)}, for model={model}. "
"To drop these, set `litellm.drop_params=True` or for proxy:\n\n"
"`litellm_settings:\n drop_params: true`\n"
),
model=model,
llm_provider=provider,
)
try:
request: Final = _REQUEST_ADAPTER.validate_python(request_fields)
except ValidationError as error:
raise litellm.BadRequestError(
message=f"Invalid Extraction request: {error}", model=model, llm_provider=provider
) from error
canonical_model: Final = provider_config.canonical_model(upstream_model)
resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base)
if resolved_api_base is None:
raise litellm.BadRequestError(
message=provider_config.missing_api_base_message(provider), model=model, llm_provider=provider
)
resolved_api_key: Final = provider_config.resolve_api_key(dynamic_api_key or api_key)
if resolved_api_key is None:
raise litellm.BadRequestError(
message=f"Missing API key for Extraction provider '{provider}'", model=model, llm_provider=provider
)
logging_obj: Final = kwargs.get("litellm_logging_obj")
http_client: Final = kwargs.get("client")
if isinstance(logging_obj, LiteLLMLoggingObj):
logging_obj.update_from_kwargs( # pyright: ignore[reportUnknownMemberType] # Logging takes untyped dicts
kwargs=dict(kwargs),
model=canonical_model,
litellm_params={
"litellm_call_id": kwargs.get("litellm_call_id"),
"api_base": provider_config.get_complete_url(resolved_api_base, canonical_model),
},
custom_llm_provider=provider,
)
return _ExtractionCall(
model=canonical_model,
requested_model=model,
custom_llm_provider=provider,
provider_config=provider_config,
body=provider_config.transform_extraction_request(canonical_model, request),
api_base=resolved_api_base,
api_key=resolved_api_key,
logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None,
headers=MappingProxyType(dict(extra_headers or {})),
timeout=timeout,
litellm_params=kwargs,
client=http_client if isinstance(http_client, HTTPHandler | AsyncHTTPHandler) else None,
)
def _state_rewritten_by_guardrail(litellm_params: Mapping[str, object]) -> bool:
metadata: Final = litellm_params.get(get_metadata_variable_name_from_kwargs(litellm_params))
return (
isinstance(metadata, Mapping)
and _METADATA_ADAPTER.validate_python(metadata).get(EXTRACTION_STATE_REWRITTEN_METADATA_KEY) is True
)
def _without_spans(answers: Mapping[str, ExtractionAnswer]) -> Mapping[str, ExtractionAnswer]:
return {name: answer.model_copy(update={"span": None, "spans": None}) for name, answer in answers.items()}
def _formatted_response(response: ExtractionResponse, call: _ExtractionCall) -> ExtractionResponse:
spans_index_hidden_text: Final = response.state_text is None and _state_rewritten_by_guardrail(call.litellm_params)
formatted: Final = response.model_copy(
update={
"model": call.requested_model,
**({"answers": _without_spans(response.answers)} if spans_index_hidden_text else {}),
}
)
formatted.set_hidden_params(
{
"model": f"{call.custom_llm_provider}/{call.model}",
"custom_llm_provider": call.custom_llm_provider,
"provider_response_model": (
get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
or f"{call.custom_llm_provider}/{call.model}"
),
}
)
return formatted
def _upstream_exception(error: Exception, call: _ExtractionCall) -> Exception:
if isinstance(error, BaseLLMException) and error.status_code_is_synthesized:
return litellm.APIConnectionError(
message=f"{call.custom_llm_provider.capitalize()}Exception - {error.message}",
llm_provider=call.custom_llm_provider,
model=f"{call.custom_llm_provider}/{call.model}",
)
return litellm.exception_type( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped
model=f"{call.custom_llm_provider}/{call.model}",
custom_llm_provider=call.custom_llm_provider,
original_exception=error,
)
async def _asend(call: _ExtractionCall) -> ExtractionResponse:
try:
response: Final = await _HANDLER.aextraction(
model=call.model,
custom_llm_provider=call.custom_llm_provider,
logging_obj=call.logging_obj,
provider_config=call.provider_config,
body=call.body,
api_base=call.api_base,
api_key=call.api_key,
headers=call.headers,
timeout=call.timeout,
litellm_params=call.litellm_params,
client=call.client if isinstance(call.client, AsyncHTTPHandler) else None,
)
except Exception as error:
raise _upstream_exception(error, call) from error
return _formatted_response(response, call)
def _send(call: _ExtractionCall) -> ExtractionResponse:
try:
response: Final = _HANDLER.extraction(
model=call.model,
custom_llm_provider=call.custom_llm_provider,
logging_obj=call.logging_obj,
provider_config=call.provider_config,
body=call.body,
api_base=call.api_base,
api_key=call.api_key,
headers=call.headers,
timeout=call.timeout,
litellm_params=call.litellm_params,
client=call.client if isinstance(call.client, HTTPHandler) else None,
)
except Exception as error:
raise _upstream_exception(error, call) from error
return _formatted_response(response, call)
@client
async def aextraction(
model: str,
fields: ExtractionFields,
state: str | None = None,
images: ExtractionImages | None = None,
reference_date: str | None = None,
options: ExtractionOptionsParam | None = None,
api_key: str | None = None,
api_base: str | None = None,
timeout: float | httpx.Timeout | None = None,
custom_llm_provider: str | None = None,
extra_headers: Mapping[str, str] | None = None,
**kwargs: object, # kwargs-ok: litellm_params forwarded by @client
) -> ExtractionResponse:
call: Final = await asyncio.to_thread(
partial(
_prepare_call,
model=model,
request_fields=_request_fields(
state=state, fields=fields, images=images, reference_date=reference_date, options=options
),
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
)
)
return await _asend(call)
@client
def extraction(
model: str,
fields: ExtractionFields,
state: str | None = None,
images: ExtractionImages | None = None,
reference_date: str | None = None,
options: ExtractionOptionsParam | None = None,
api_key: str | None = None,
api_base: str | None = None,
timeout: float | httpx.Timeout | None = None,
custom_llm_provider: str | None = None,
extra_headers: Mapping[str, str] | None = None,
**kwargs: object, # kwargs-ok: litellm_params forwarded by @client
) -> ExtractionResponse:
call: Final = _prepare_call(
model=model,
request_fields=_request_fields(
state=state, fields=fields, images=images, reference_date=reference_date, options=options
),
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
)
return _send(call)
__all__ = ["aextraction", "extraction"]

18
litellm/harness/endpoint.py
View file

@ -37,6 +37,7 @@ from litellm.harness.context import GatewayTarget
from litellm.harness.errors import HarnessError, HarnessInstallFailed
from litellm.harness.types import Harness, Usage
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
from litellm.types.llms.custom_http import httpxSpecialProvider
if TYPE_CHECKING:
@ -690,18 +691,21 @@ class ModelEndpoint:
async def _invoke_sdk(
self,
route: str,
kwargs: dict[str, Any], # mutable-ok: injects stream_options into the SDK kwargs
kwargs: dict[str, Any], # mutable-ok: SDK call kwargs built by _sdk_kwargs
) -> object:
if route == ROUTE_MESSAGES:
return await litellm.anthropic.messages.acreate(**kwargs)
if route == ROUTE_CHAT:
if kwargs.get("stream"):
stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field
kwargs["stream_options"] = { # mutable-ok: JSON field sent to litellm.acompletion
"include_usage": True,
**stream_options,
if not kwargs.get("stream"):
return await litellm.acompletion(**kwargs)
stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field
return await litellm.acompletion(
**{
**kwargs,
"stream_options": {"include_usage": True, **stream_options},
STREAM_USAGE_INJECTED_FIELD: "include_usage" not in stream_options,
}
return await litellm.acompletion(**kwargs)
)
return await litellm.aresponses(**kwargs)
async def _call_sdk(self, route: str, body: Mapping[str, object]) -> Response:

86
litellm/harness/handlers/cli_handler.py
View file

@ -12,17 +12,27 @@ from __future__ import annotations
import asyncio
import os
from collections import deque
from collections.abc import AsyncIterator, Sequence
from collections.abc import AsyncIterator, Mapping, Sequence
from dataclasses import dataclass
from typing import Final
from litellm._logging import verbose_logger
from litellm.constants import HARNESS_STDERR_TAIL_LINES, HARNESS_STREAM_READ_CHUNK_BYTES
from litellm.constants import (
HARNESS_RESUME_COMMAND_TIMEOUT_SECONDS,
HARNESS_STDERR_TAIL_LINES,
HARNESS_STREAM_READ_CHUNK_BYTES,
)
from litellm.harness.context import SessionContext
from litellm.harness.errors import HarnessInstallFailed, SandboxError
from litellm.harness.errors import HarnessError, HarnessInstallFailed, SandboxError
from litellm.harness.handlers.base import BaseHarnessHandler
from litellm.harness.sandbox.base import Process, Sandbox
from litellm.harness.sandbox.base import CompletedRun, Process, Sandbox
from litellm.harness.types import Event
from litellm.llms.base_llm.harness.transformation import BaseCLIHarnessConfig, HarnessSessionSetup
from litellm.llms.base_llm.harness.transformation import (
BaseCLIHarnessConfig,
HarnessSessionCopy,
HarnessSessionSetup,
HarnessTurnError,
)
from litellm.llms.base_llm.harness.utils import decode_json_line, read_skill_files
# Link <private_dir>/<dir> to a LiteLLM-owned cache dir so a later session can resume.
@ -81,6 +91,52 @@ async def persist_dir(sandbox: Sandbox, link_path: str, cache_subpath: str) -> N
)
@dataclass(frozen=True, slots=True)
class ResumeFailed:
reason: str
async def bounded_resume_run(
sandbox: Sandbox, cmd: Sequence[str], env: Mapping[str, str]
) -> CompletedRun | ResumeFailed:
try:
run: Final = await sandbox.run(cmd, env=env, timeout=HARNESS_RESUME_COMMAND_TIMEOUT_SECONDS)
except SandboxError as exc:
return ResumeFailed(str(exc))
return run if run.exit_code == 0 else ResumeFailed(run.stderr.strip() or f"exit code {run.exit_code}")
async def import_session_copy(sandbox: Sandbox, copy: HarnessSessionCopy, env: Mapping[str, str]) -> str | ResumeFailed:
try:
await sandbox.write(copy.path, copy.data)
except SandboxError as exc:
return ResumeFailed(str(exc))
imported: Final = await bounded_resume_run(sandbox, copy.import_command, env)
return imported if isinstance(imported, ResumeFailed) else copy.session_id
async def resumable_session_id(
config: BaseCLIHarnessConfig[object, object],
sandbox: Sandbox,
setup_env: Mapping[str, str],
private_dir: str,
native_session_id: str,
) -> str | ResumeFailed:
export_cmd: Final = config.transform_resume_export_command(native_session_id)
if export_cmd is None:
return native_session_id
exported: Final = await bounded_resume_run(sandbox, export_cmd, setup_env)
if isinstance(exported, ResumeFailed):
return exported
try:
copy: Final = config.transform_resume_copy(exported.stdout, private_dir, sandbox.workdir)
except HarnessError as exc:
return ResumeFailed(str(exc))
if copy is None:
return native_session_id
return await import_session_copy(sandbox, copy, setup_env)
async def copy_skills(sandbox: Sandbox, skills: Sequence[str], skills_root: str) -> None:
for skill in skills:
name = os.path.basename(os.path.realpath(os.fspath(skill)))
@ -89,13 +145,14 @@ async def copy_skills(sandbox: Sandbox, skills: Sequence[str], skills_root: str)
class CLIHarnessHandler(BaseHarnessHandler):
config: BaseCLIHarnessConfig
config: BaseCLIHarnessConfig[object, object]
def __init__(self, config: BaseCLIHarnessConfig) -> None:
super().__init__(config)
self._private_dir: str | None = None
self._setup: HarnessSessionSetup | None = None
self._native_id: str | None = None
self._pending_resume: str | None = None
self._proc: Process | None = None
async def start(self, ctx: SessionContext) -> None:
@ -119,6 +176,9 @@ class CLIHarnessHandler(BaseHarnessHandler):
async def turn(self, ctx: SessionContext, prompt: str) -> AsyncIterator[Event]:
if self._setup is None or self._private_dir is None:
raise RuntimeError("CLIHarnessHandler.turn() called before start()")
if self._pending_resume is not None:
self._native_id = await self._prepared_session_id(ctx, self._setup, self._private_dir, self._pending_resume)
self._pending_resume = None
request = self.config.transform_turn_request(ctx, self._setup, self._private_dir, prompt, self._native_id)
argv: Final = list(request.argv) # mutable-ok: Sandbox.exec takes list[str]
proc = await ctx.sandbox.exec(argv, env=request.env, cwd=request.cwd)
@ -158,4 +218,16 @@ class CLIHarnessHandler(BaseHarnessHandler):
return self._native_id
async def resume(self, ctx: SessionContext, native_session_id: str) -> None:
self._native_id = native_session_id
if native_session_id != self._native_id:
self._native_id = native_session_id
self._pending_resume = native_session_id
async def _prepared_session_id(
self, ctx: SessionContext, setup: HarnessSessionSetup, private_dir: str, saved_id: str
) -> str:
target: Final = await resumable_session_id(self.config, ctx.sandbox, setup.env, private_dir, saved_id)
match target:
case ResumeFailed(reason=reason):
raise HarnessTurnError(f"could not prepare session {saved_id} for resume: {reason}")
case session_id:
return session_id

16
litellm/harness/sandbox/local.py
View file

@ -14,7 +14,7 @@ from typing import Final
from litellm.constants import HARNESS_PROCESS_KILL_GRACE_SECONDS
from litellm.harness.errors import SandboxError
from litellm.harness.sandbox.base import CompletedRun
from litellm.harness.sandbox.base import CompletedRun, Process
from litellm.harness.sandbox.snapshot import snapshot_local
_SECRET_PREFIXES: Final = (
@ -103,7 +103,7 @@ class SubprocessHandle:
await self._proc.wait()
async def _read_all(handle: SubprocessHandle) -> tuple[bytes, bytes, int]:
async def _read_all(handle: Process) -> tuple[bytes, bytes, int]:
if handle.stdin is not None:
handle.stdin.close()
stdout, stderr = await asyncio.gather(handle.stdout.read(), handle.stderr.read())
@ -111,13 +111,16 @@ async def _read_all(handle: SubprocessHandle) -> tuple[bytes, bytes, int]:
return stdout, stderr, exit_code
async def collect_output(handle: SubprocessHandle, cmd: Sequence[str], timeout: float | None) -> CompletedRun:
"""Close stdin, read stdout/stderr to EOF; kill and raise SandboxError on timeout."""
async def collect_output(handle: Process, cmd: Sequence[str], timeout: float | None) -> CompletedRun:
"""Close stdin, read stdout/stderr to EOF; kill on cancel, and kill and raise SandboxError on timeout."""
try:
stdout, stderr, code = await asyncio.wait_for(_read_all(handle), timeout)
except asyncio.TimeoutError:
await handle.kill()
raise SandboxError(f"command timed out after {timeout}s: {cmd[0]}")
except asyncio.CancelledError:
await handle.kill()
raise
return CompletedRun(
stdout=stdout.decode("utf-8", errors="replace"),
stderr=stderr.decode("utf-8", errors="replace"),
@ -180,14 +183,15 @@ class LocalSandbox:
self._check_open()
if not cmd:
raise SandboxError("exec() needs a non-empty command")
work_dir: Final = self._resolve_cwd(cwd)
try:
proc = await asyncio.create_subprocess_exec(
*cmd,
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=self._resolve_cwd(cwd),
env=self.child_env(env),
cwd=work_dir,
env=self.child_env(MappingProxyType({**(env or {}), "PWD": work_dir})),
start_new_session=True,
)
except (FileNotFoundError, PermissionError) as exc:

7
litellm/litellm_core_utils/core_helpers.py
View file

@ -329,11 +329,14 @@ def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metad
Helper to get litellm metadata for spend tracking
PATCH for issue where both `litellm_metadata` and `metadata` are present in the kwargs
and user_api_key values are in 'metadata'.
and user_api_key values are in 'metadata'. The proxy receive stamp rides along for the same
reason: a guardrail hook can plant `litellm_metadata` on a chat request after the proxy stamped
`metadata`, and the response timing headers anchor on that stamp.
"""
potential_spend_tracking_metadata_substring: Final = "user_api_key"
for key, value in dict(metadata).items():
if potential_spend_tracking_metadata_substring in key:
carry_receive_stamp = key == "litellm_received_at" and key not in litellm_metadata
if potential_spend_tracking_metadata_substring in key or carry_receive_stamp:
litellm_metadata[key] = value
return litellm_metadata

2
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -27,6 +27,7 @@ AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset(
"aws_external_id",
"aws_session_tags",
"aws_bedrock_runtime_endpoint",
"aws_bedrock_control_plane_endpoint",
"aws_bedrock_project_id",
}
)
@ -76,6 +77,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"otpm",
"use_xai_oauth",
"github_copilot_auth_type",
"github_copilot_host",
"github_copilot_user_session",
"fireworks_forward_user_id",
"responses_api_version",

20
litellm/litellm_core_utils/health_check_helpers.py
View file

@ -4,6 +4,7 @@ Helper functions for health check calls.
import base64
from collections.abc import Awaitable, Callable, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal
from litellm.llms.base_llm.ocr.transformation import DocumentType
@ -25,6 +26,9 @@ IMAGE_EDIT_HEALTH_CHECK_PROMPT: Final = (
)
ANTHROPIC_MESSAGES_HEALTH_CHECK_MAX_TOKENS: Final = 16
EXTRACTION_HEALTH_CHECK_FIELDS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType(
{"reachable": MappingProxyType({"type": "text", "instructions": "Any word from the text"})}
)
def native_health_check_mode(model: str, custom_llm_provider: str | None) -> Literal["anthropic_messages"] | None:
@ -215,6 +219,7 @@ class HealthCheckHelpers:
"anthropic_messages",
"ocr",
"decisions",
"extraction",
],
Callable,
]:
@ -237,7 +242,11 @@ class HealthCheckHelpers:
from litellm.litellm_core_utils.audio_utils.utils import (
get_audio_file_for_health_check,
)
from litellm.litellm_core_utils.health_check_utils import DECISIONS_CALL_PARAMS, _filter_model_params
from litellm.litellm_core_utils.health_check_utils import (
DECISIONS_CALL_PARAMS,
EXTRACTION_CALL_PARAMS,
_filter_model_params,
)
from litellm.realtime_api.main import realtime_health_check
return {
@ -322,4 +331,13 @@ class HealthCheckHelpers:
}
)
),
"extraction": lambda: litellm.aextraction(
**EXTRACTION_CALL_PARAMS.validate_python(
{
"state": prompt or "health check",
"fields": EXTRACTION_HEALTH_CHECK_FIELDS,
**_filter_model_params(model_params=model_params),
}
)
),
}

2
litellm/litellm_core_utils/health_check_utils.py
View file

@ -8,8 +8,10 @@ from typing import Final
from pydantic import TypeAdapter
from litellm.types.decisions import DecisionsCallParams
from litellm.types.extraction import ExtractionCallParams
DECISIONS_CALL_PARAMS: Final[TypeAdapter[DecisionsCallParams]] = TypeAdapter(DecisionsCallParams)
EXTRACTION_CALL_PARAMS: Final[TypeAdapter[ExtractionCallParams]] = TypeAdapter(ExtractionCallParams)
OPTIONAL_STR: Final[TypeAdapter[str | None]] = TypeAdapter(str | None)

275
litellm/litellm_core_utils/litellm_logging.py
View file

@ -12,7 +12,7 @@ import subprocess
import sys
import time
import traceback
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime as dt_object
from functools import lru_cache
@ -117,7 +117,8 @@ from litellm.litellm_core_utils.ptu_pricing import is_spilled_over_ptu_request
from litellm.litellm_core_utils.redact_messages import (
redact_message_input_output_from_custom_logger,
redact_message_input_output_from_logging,
redact_streaming_responses_for_custom_logger,
redacted_model_call_details_for_custom_logger,
redacted_response_obj_for_custom_logger,
should_redact_message_logging,
)
from litellm.litellm_core_utils.served_output_texts import (
@ -131,11 +132,13 @@ from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.agents import LiteLLMSendMessageResponse
from litellm.types.containers.main import ContainerObject
from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse
from litellm.types.extraction import ExtractionResponse
from litellm.types.integrations.s3_v2 import S3PartitionGranularity
from litellm.types.interactions import (
InteractionsAPIResponse,
InteractionsAPIStreamingResponse,
)
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
from litellm.types.llms.openai import (
AllMessageValues,
Batch,
@ -337,6 +340,7 @@ def _get_provider_request_id(original_exception: Exception) -> str | None:
# Cache custom pricing keys as frozenset for O(1) lookups instead of looping through 49 keys
_CUSTOM_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields.keys())
_METADATA_MAPPING: Final = TypeAdapter(Mapping[str, object])
_MODEL_INFO_CUSTOM_PRICING_KEYS: Final[frozenset[str]] = _CUSTOM_PRICING_KEYS | DEPLOYMENT_SCOPED_PRICING_FIELDS
_UNSERIALIZABLE_METADATA_KEYS: Final[frozenset[str]] = frozenset(
("user_api_key_auth", "user_api_key_budget_reservation")
@ -571,6 +575,27 @@ class UncachedPrice:
cost_breakdown: CostBreakdown | None
@dataclass(frozen=True, slots=True)
class BilledRoundsSpend:
usage: Usage
cost: float
cost_breakdown: CostBreakdown | None
@dataclass(frozen=True, slots=True)
class BilledRoundsDispatch:
"""A parked success log for a stream bridged through chat completions that billed several rounds, and the spend
its failure log bills when a post-call hook rejects the response, as it does for a rejected one-round bridged
stream."""
dispatch: Coroutine[object, object, None]
spend: BilledRoundsSpend
def _fixed_margin(cost_breakdown: CostBreakdown | None) -> float:
return 0.0 if cost_breakdown is None else cost_breakdown.get("margin_fixed_amount", 0.0)
def _timestamp_seconds(moment: object) -> float | None:
if isinstance(moment, datetime.datetime):
return moment.timestamp()
@ -579,6 +604,10 @@ def _timestamp_seconds(moment: object) -> float | None:
return None
def same_attempt_marker(first: object, second: object) -> bool:
return first is second or first == second
class Logging(LiteLLMLoggingBaseClass):
global \
supabaseClient, \
@ -592,6 +621,7 @@ class Logging(LiteLLMLoggingBaseClass):
prometheusLogger, \
slack_app
custom_pricing: bool = False
replaced_global_callback_keys: frozenset[str] = frozenset()
stream_options = None
litellm_request_debug: bool = False
streamed_anthropic_message_id: str | None = None
@ -599,6 +629,7 @@ class Logging(LiteLLMLoggingBaseClass):
classifier_input: Mapping[str, JsonValue] | None = None
baseline_cache_context: "BaselineCacheContext | None" = None
baseline_observation: "CapturedBaselineObservation | None" = None
_on_failure_payload_built: "Callable[[], None] | None" = None
@property
def _defer_async_logging(self) -> bool:
@ -646,11 +677,11 @@ class Logging(LiteLLMLoggingBaseClass):
litellm_call_id: str,
function_id: str,
litellm_trace_id: str | None = None,
dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = None,
dynamic_input_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = None,
dynamic_async_failure_callbacks: list[str | Callable | CustomLogger] | None = None,
dynamic_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
dynamic_async_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
applied_guardrails: list[str] | None = None,
kwargs: dict | None = None,
log_raw_request_response: bool = False,
@ -695,6 +726,7 @@ class Logging(LiteLLMLoggingBaseClass):
self._pre_call_session_id: str = session_id_var.get()
_sid: Final = kwargs.get("litellm_session_id") if kwargs else None
self.litellm_session_id: str = str(_sid) if _sid else ""
self.stream_usage_injected: Final = kwargs is not None and kwargs.get(STREAM_USAGE_INJECTED_FIELD) is True
# supports_correlation_logging is False for calls originating from the
# sync client entry point (wrapper() in utils.py): a plain OS thread
# has no per-call context isolation the way an asyncio Task does, and
@ -723,15 +755,17 @@ class Logging(LiteLLMLoggingBaseClass):
self.raw_request_only = raw_request_only
# Initialize dynamic callbacks
self.dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = dynamic_input_callbacks
self.dynamic_input_callbacks: list[str | Callable[..., object] | CustomLogger] | None = dynamic_input_callbacks
self.dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
dynamic_success_callbacks
)
self.dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
dynamic_async_success_callbacks
)
self.dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = dynamic_failure_callbacks
self.dynamic_async_failure_callbacks: list[str | Callable | CustomLogger] | None = (
self.dynamic_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
dynamic_failure_callbacks
)
self.dynamic_async_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
dynamic_async_failure_callbacks
)
@ -794,6 +828,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.defer_async_logging: bool = False
self.enqueue_deferred_logging: Callable[[], None] | None = None
self._async_success_scheduled: bool = False
self._async_success_enqueued: bool = False
self._sync_success_scheduled: bool = False
self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
@ -822,6 +857,7 @@ class Logging(LiteLLMLoggingBaseClass):
"litellm_call_id": litellm_call_id,
"start_time": start_time,
"first_api_call_start_time": None,
"current_attempt_api_call_start_time": None,
"litellm_params": litellm_params,
}
new_call.streaming_chunks = []
@ -847,10 +883,32 @@ class Logging(LiteLLMLoggingBaseClass):
self.dynamic_async_failure_callbacks, callback
)
def replace_global_callback(self, callback: CustomLogger) -> None:
self.add_dynamic_callback(callback)
self.replaced_global_callback_keys = self.replaced_global_callback_keys | {
litellm.logging_callback_manager.get_custom_logger_key(callback)
}
def add_dynamic_function_callbacks(
self,
*,
async_success: Callable[..., object],
failure: Callable[..., object],
async_failure: Callable[..., object],
) -> None:
self.dynamic_async_success_callbacks = self._with_dynamic_callback(
self.dynamic_async_success_callbacks, async_success
)
self.dynamic_failure_callbacks = self._with_dynamic_callback(self.dynamic_failure_callbacks, failure)
self.dynamic_async_failure_callbacks = self._with_dynamic_callback(
self.dynamic_async_failure_callbacks, async_failure
)
@staticmethod
def _with_dynamic_callback(
callbacks: Sequence[str | Callable | CustomLogger] | None, callback: CustomLogger
) -> list[str | Callable | CustomLogger]:
callbacks: Sequence[str | Callable[..., object] | CustomLogger] | None,
callback: Callable[..., object] | CustomLogger,
) -> list[str | Callable[..., object] | CustomLogger]:
existing: Final = tuple(callbacks or ())
return [*existing, *(() if callback in existing else (callback,))]
@ -887,9 +945,9 @@ class Logging(LiteLLMLoggingBaseClass):
def _process_dynamic_callback_list(
self,
callback_list: list[str | Callable | CustomLogger] | None,
callback_list: list[str | Callable[..., object] | CustomLogger] | None,
dynamic_callbacks_type: Literal["input", "success", "failure", "async_success", "async_failure"],
) -> list[str | Callable | CustomLogger] | None:
) -> list[str | Callable[..., object] | CustomLogger] | None:
"""
Helper function to initialize CustomLogger compatible callbacks in self.dynamic_* callbacks
@ -901,7 +959,7 @@ class Logging(LiteLLMLoggingBaseClass):
if callback_list is None:
return None
processed_list: Final[list[str | Callable | CustomLogger]] = []
processed_list: Final[list[str | Callable[..., object] | CustomLogger]] = []
for callback in callback_list:
if isinstance(callback, str) and callback in litellm._known_custom_logger_compatible_callbacks:
for callback_instance in self._resolve_dynamic_callback_string(callback):
@ -995,6 +1053,21 @@ class Logging(LiteLLMLoggingBaseClass):
return model_id
return None
def get_router_pricing_provider(self) -> str | None:
"""The deployment's ``model_info.pricing_provider``, the provider its cost-map lookups run under."""
litellm_params: Final = cast( # cast-ok: callback metadata is caller-provided
Mapping[str, object] | None, getattr(self, "litellm_params", None)
)
if not litellm_params:
return None
candidates: Final = (
_METADATA_MAPPING.validate_python(
_METADATA_MAPPING.validate_python(litellm_params.get(key) or {}).get("model_info") or {}
).get("pricing_provider")
for key in ("litellm_metadata", "metadata")
)
return next((provider for provider in candidates if isinstance(provider, str) and provider), None)
def _routed_deployment_model(self) -> str | None:
litellm_params: Final = self._model_call_details_view.get("litellm_params")
metadata: Final = (
@ -1535,7 +1608,9 @@ class Logging(LiteLLMLoggingBaseClass):
# A proxy-gate failure (auth, rate limit, client disconnect) fires pre_call for its callbacks
# without any provider handoff, so it must not stamp the handoff instants.
if LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL not in self.model_call_details:
self.model_call_details["api_call_start_time"] = datetime.datetime.now()
api_call_start_time: Final = datetime.datetime.now()
self.model_call_details["api_call_start_time"] = api_call_start_time
self.model_call_details["current_attempt_api_call_start_time"] = api_call_start_time
# Set-once first provider-handoff instant. api_call_start_time
# is overwritten on every retry, so it can't measure one-time
# preprocessing; pinning the first attempt excludes retry loops
@ -1543,11 +1618,12 @@ class Logging(LiteLLMLoggingBaseClass):
# litellm_params["metadata"] (caller request metadata, typed
# Dict[str, str], echoed downstream; a datetime breaks it).
if self.model_call_details.get("first_api_call_start_time") is None:
self.model_call_details["first_api_call_start_time"] = self.model_call_details[
"api_call_start_time"
]
self.model_call_details["first_api_call_start_time"] = api_call_start_time
# Input Integration Logging -> If you want to log the fact that an attempt to call the model was made
callbacks: Final = litellm.input_callback + (self.dynamic_input_callbacks or [])
callbacks: Final = (
*self._without_replaced_global_callbacks(litellm.input_callback),
*(self.dynamic_input_callbacks or ()),
)
for callback in callbacks:
try:
if callback == "supabase" and supabaseClient is not None:
@ -1720,7 +1796,10 @@ class Logging(LiteLLMLoggingBaseClass):
)
# Input Integration Logging -> If you want to log the fact that an attempt to call the model was made
callbacks: Final = litellm.input_callback + (self.dynamic_input_callbacks or [])
callbacks: Final = (
*self._without_replaced_global_callbacks(litellm.input_callback),
*(self.dynamic_input_callbacks or ()),
)
for callback in callbacks:
try:
if callback == "sentry" and add_breadcrumb:
@ -2020,6 +2099,7 @@ class Logging(LiteLLMLoggingBaseClass):
"prompt": prompt,
"standard_built_in_tools_params": self.standard_built_in_tools_params,
"router_model_id": router_model_id,
"pricing_provider": self.get_router_pricing_provider(),
"litellm_logging_obj": self,
"data_residency": (
self.litellm_params.get("data_residency")
@ -2164,7 +2244,7 @@ class Logging(LiteLLMLoggingBaseClass):
pricing: Final = pricing_entry_for_cost_calc(
model=model,
completion_response=result,
custom_llm_provider=custom_llm_provider,
custom_llm_provider=self.get_router_pricing_provider() or custom_llm_provider,
custom_pricing=self.custom_pricing_for(result),
base_model=get_base_model_from_metadata(model_call_details=self.model_call_details),
router_model_id=router_model_id or self.get_router_model_id(),
@ -2190,20 +2270,23 @@ class Logging(LiteLLMLoggingBaseClass):
usage=usage,
)
def custom_pricing_for(self, result: object) -> bool:
litellm_params: Final = getattr(self, "litellm_params", None)
def spilled_over_ptu_for(self, result: object) -> bool:
result_hidden_params: Final = getattr(result, HIDDEN_PARAMS_ATTR, None) or MappingProxyType({})
additional_headers: Final = (
result_hidden_params.get("additional_headers")
if isinstance(result_hidden_params, dict)
else getattr(result_hidden_params, "additional_headers", None)
)
spilled_over: Final = is_spilled_over_ptu_request(
model_info=_deployment_model_info(litellm_params),
return is_spilled_over_ptu_request(
model_info=_deployment_model_info(getattr(self, "litellm_params", None)),
response_headers=self.model_call_details.get("response_headers"),
additional_headers=additional_headers,
)
return False if spilled_over else use_custom_pricing_for_model(litellm_params=litellm_params)
def custom_pricing_for(self, result: object) -> bool:
if self.spilled_over_ptu_for(result):
return False
return use_custom_pricing_for_model(litellm_params=getattr(self, "litellm_params", None))
def _prompt_for_cost_calculation(self) -> str:
"""
@ -2365,6 +2448,31 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["combined_usage_object"] = usage
self.model_call_details["response_cost"] = response_cost
def record_failure_after_earlier_rounds(
self, earlier: BilledRoundsSpend, failed_usage: Usage, failed_cost: float
) -> None:
"""Bill a failed round on the failure log after the rounds of its request before it were billed on the success
log.
The fixed margin is a per-request fee, so it is charged once: the failed round's margin is dropped when the
earlier rounds already paid one."""
from litellm.responses.mcp.round_billing import breakdown_less_repeated_fixed_margin
failed_breakdown: Final = None if self.cost_breakdown is earlier.cost_breakdown else self.cost_breakdown
repeated_fixed_margin: Final = (
0.0
if failed_breakdown is None
else min(_fixed_margin(earlier.cost_breakdown), _fixed_margin(failed_breakdown), earlier.cost, failed_cost)
)
self.model_call_details["combined_usage_object"] = failed_usage
self.model_call_details["response_cost"] = failed_cost - repeated_fixed_margin
self.cost_breakdown = breakdown_less_repeated_fixed_margin(failed_breakdown, repeated_fixed_margin)
def record_billed_rounds_for_failure(self, spend: BilledRoundsSpend) -> None:
"""Bill every round of a fully streamed request on the failure log when a post-call hook rejects it."""
self.record_partial_usage_for_failure(spend.usage, spend.cost)
self.cost_breakdown = spend.cost_breakdown
def attempt_scoped_copy(self) -> "Logging":
"""A copy with its own model_call_details, so logging one attempt of a request through it leaves
the state the request's other attempts share untouched."""
@ -2372,12 +2480,37 @@ class Logging(LiteLLMLoggingBaseClass):
attempt.model_call_details = dict(self.model_call_details)
return attempt
def dispatch_attempt_failure_handlers(
self, exception: Exception, traceback_exception: str
) -> Coroutine[None, None, None]:
shared_payload: Final = self.model_call_details.get("standard_logging_object")
attempt_marker: Final = self.model_call_details.get("current_attempt_api_call_start_time")
attempt: Final = self.attempt_scoped_copy()
def publish() -> None:
attempt_payload: Final = attempt.model_call_details.get("standard_logging_object")
current_marker: Final = self.model_call_details.get("current_attempt_api_call_start_time")
if (
attempt_payload is not None
and self.model_call_details.get("standard_logging_object") is shared_payload
and same_attempt_marker(current_marker, attempt_marker)
):
self.model_call_details["standard_logging_object"] = attempt_payload
attempt._on_failure_payload_built = publish
async def dispatch() -> None:
await attempt.dispatch_failure_handlers(exception, traceback_exception, prefer_async_handlers=True)
return dispatch()
def carry_failed_attempt_usage(self) -> None:
"""Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its
own usage while the request's failure log still bills every attempt."""
usage: Final = self.model_call_details.pop("combined_usage_object", None)
response_cost: Final = self.model_call_details.pop("response_cost", None)
if isinstance(usage, Usage):
self.model_call_details["current_attempt_api_call_start_time"] = None
self.model_call_details["failed_attempts_usage"] = (
*self.model_call_details.get("failed_attempts_usage", ()),
(usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0),
@ -2447,7 +2580,7 @@ class Logging(LiteLLMLoggingBaseClass):
await self.async_failure_handler(exception, traceback_exception)
finally:
if self._should_run_sync_failure_callbacks_for_async_calls():
executor.submit(self.failure_handler, exception, traceback_exception)
executor.submit(self.failure_handler, exception, traceback_exception, for_async_request=True)
def should_run_logging(
self,
@ -2462,6 +2595,16 @@ class Logging(LiteLLMLoggingBaseClass):
except Exception:
return True
@property
def async_success_log_enqueued(self) -> bool:
"""True once the success log task was actually created, not merely claimed, so a later failure raised
around it (a disconnect cancelling a still-pending guardrail task) must not bill the prompt a second
time. A claimed log parked in the deferred slot never runs once a 499 interrupts, so it stays False."""
return self._async_success_enqueued
def mark_async_success_log_enqueued(self) -> None:
self._async_success_enqueued = True
def claim_async_success_log(self) -> bool:
"""One async success log per request. The innermost @client wrapper always exits first,
since the outer one is awaiting it, so it claims the log here and the outer wrapper gets
@ -2481,6 +2624,10 @@ class Logging(LiteLLMLoggingBaseClass):
"""
return
self.model_call_details[f"has_logged_{event_type}"] = True
if event_type == "async_failure":
self.model_call_details["async_failure_logged_attempt"] = self.model_call_details.get(
"current_attempt_api_call_start_time"
)
return
def has_run_logging(
@ -2890,6 +3037,7 @@ class Logging(LiteLLMLoggingBaseClass):
or isinstance(logging_result, SearchResponse) # Search API
or isinstance(logging_result, DecisionsResponse)
or isinstance(logging_result, OpenAIDecisionResponse)
or isinstance(logging_result, ExtractionResponse)
or (
isinstance(logging_result, InteractionsAPIResponse)
and logging_result.usage is not None
@ -3489,8 +3637,10 @@ class Logging(LiteLLMLoggingBaseClass):
): # custom logger class
if self.stream and complete_streaming_response is None:
callback.log_stream_event(
kwargs=call_details,
response_obj=result,
kwargs=redacted_model_call_details_for_custom_logger(
model_call_details=call_details, custom_logger=callback
),
response_obj=redacted_response_obj_for_custom_logger(result, callback),
start_time=start_time,
end_time=end_time,
)
@ -3500,8 +3650,13 @@ class Logging(LiteLLMLoggingBaseClass):
result = call_details_view["complete_response"]
callback.log_success_event(
kwargs=callback.redact_standard_logging_payload_from_model_call_details(call_details),
response_obj=result,
kwargs=redacted_model_call_details_for_custom_logger(
model_call_details=callback.redact_standard_logging_payload_from_model_call_details(
call_details
),
custom_logger=callback,
),
response_obj=redacted_response_obj_for_custom_logger(result, callback),
start_time=start_time,
end_time=end_time,
)
@ -3844,7 +3999,7 @@ class Logging(LiteLLMLoggingBaseClass):
model_call_details = callback.redact_standard_logging_payload_from_model_call_details(
model_call_details=model_call_details
)
model_call_details = redact_streaming_responses_for_custom_logger(
model_call_details = redacted_model_call_details_for_custom_logger(
model_call_details=model_call_details, custom_logger=callback
)
##################################
@ -3855,7 +4010,7 @@ class Logging(LiteLLMLoggingBaseClass):
kwargs=model_call_details,
response_obj=model_call_details["async_complete_streaming_response"]
if self.stream
else result,
else redacted_response_obj_for_custom_logger(result, callback),
start_time=start_time,
end_time=end_time,
guardrail_snapshot=self.shadow_eval_request_snapshot,
@ -3871,14 +4026,14 @@ class Logging(LiteLLMLoggingBaseClass):
else:
await callback.async_log_stream_event( # [TODO]: move this to being an async log stream event function
kwargs=model_call_details,
response_obj=result,
response_obj=redacted_response_obj_for_custom_logger(result, callback),
start_time=start_time,
end_time=end_time,
)
else:
await callback.async_log_success_event(
kwargs=model_call_details,
response_obj=result,
response_obj=redacted_response_obj_for_custom_logger(result, callback),
start_time=start_time,
end_time=end_time,
)
@ -4054,15 +4209,21 @@ class Logging(LiteLLMLoggingBaseClass):
traceback_exception: str,
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
*,
for_async_request: bool = False,
) -> None:
"""Restores trace_id/session_id contextvars once this attempt's own failure
logging (including any nested calls its callbacks trigger) is fully done."""
logging (including any nested calls its callbacks trigger) is fully done.
``for_async_request`` marks the sync run of a failure whose async run also logs it, so a
CustomLogger that gets ``async_log_failure_event`` from that async run skips its sync hook."""
try:
return self._failure_handler_body(
exception=exception,
traceback_exception=traceback_exception,
start_time=start_time,
end_time=end_time,
for_async_request=for_async_request,
)
finally:
self._restore_correlation_context()
@ -4073,6 +4234,7 @@ class Logging(LiteLLMLoggingBaseClass):
traceback_exception: str,
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
for_async_request: bool = False,
) -> None:
verbose_logger.debug("Logging Details LiteLLM-Failure Call: %s", litellm.failure_callback)
if not self.should_run_logging(event_type="sync_failure"): # prevent double logging
@ -4165,6 +4327,7 @@ class Logging(LiteLLMLoggingBaseClass):
isinstance(callback, CustomLogger)
and is_sync_request
and self.call_type != CallTypes.pass_through.value
and not (for_async_request and self._async_handler_logs_to(callback))
): # custom logger class
callback.log_failure_event(
start_time=start_time,
@ -4282,7 +4445,8 @@ class Logging(LiteLLMLoggingBaseClass):
start_time=start_time,
end_time=end_time,
)
if self._on_failure_payload_built is not None:
self._on_failure_payload_built()
callbacks: Final = self.get_combined_callback_list(
dynamic_success_callbacks=self.dynamic_async_failure_callbacks,
global_callbacks=litellm._async_failure_callback,
@ -4467,14 +4631,26 @@ class Logging(LiteLLMLoggingBaseClass):
_filtered_failure_callbacks = self._remove_internal_litellm_callbacks(_filtered_failure_callbacks)
return len(_filtered_failure_callbacks) > 0
def _without_replaced_global_callbacks(
self, global_callbacks: Iterable[str | Callable[..., object] | CustomLogger]
) -> tuple[str | Callable[..., object] | CustomLogger, ...]:
if not self.replaced_global_callback_keys:
return tuple(global_callbacks)
return tuple(
litellm.logging_callback_manager.without_custom_logger_keys(
global_callbacks, self.replaced_global_callback_keys
)
)
def get_combined_callback_list(
self,
dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None,
global_callbacks: list[str | Callable[..., object] | CustomLogger],
) -> list[str | Callable[..., object] | CustomLogger]:
kept_global_callbacks: Final = self._without_replaced_global_callbacks(global_callbacks)
if dynamic_success_callbacks is None:
return list(global_callbacks)
return list(dict.fromkeys(dynamic_success_callbacks + global_callbacks))
return list(kept_global_callbacks)
return list(dict.fromkeys((*dynamic_success_callbacks, *kept_global_callbacks)))
def _remove_internal_litellm_callbacks(self, callbacks: list) -> list:
"""
@ -4527,6 +4703,12 @@ class Logging(LiteLLMLoggingBaseClass):
cb_name: Final = self._get_callback_name(cb)
return any(prefix in cb_name for prefix in INTERNAL_PREFIXES)
def _async_handler_logs_to(self, logger: CustomLogger) -> bool:
"""True when the async failure run of this call hands ``logger`` its own ``async_log_failure_event``."""
if type(logger).async_log_failure_event is CustomLogger.async_log_failure_event: # pyright: ignore[reportUnknownMemberType] # CustomLogger's hook signatures are untyped
return False
return logger in (self.dynamic_async_failure_callbacks or []) or logger in litellm._async_failure_callback # pyright: ignore[reportUnknownMemberType, reportPrivateUsage] # callback lists are untyped and have no public accessor
def _remove_internal_custom_logger_callbacks(self, callbacks: list) -> list:
"""
Removes internal custom logger callbacks from the list.
@ -7195,13 +7377,16 @@ def get_standard_logging_object_payload(
saved_cache_cost: Final = (uncached_price.cost if uncached_price is not None else None) or 0.0
## Get model cost information ##
base_model = get_base_model_from_metadata(model_call_details=kwargs)
# The router overrides completion_response.model to the model-group alias before
# this payload is built, so cost-map lookup via that alias always misses.
# Fall back to the actual deployment model set by the router in metadata.
if base_model is None:
base_model = metadata.get("deployment")
custom_pricing: Final = logging_obj.custom_pricing_for(init_response_obj)
configured_base_model: Final = get_base_model_from_metadata(model_call_details=kwargs)
spilled_over: Final = logging_obj.spilled_over_ptu_for(init_response_obj)
# The router overrides completion_response.model to the model-group alias before this payload is
# built, so cost-map lookup via that alias misses and the deployment model stands in for base_model.
# A PTU request Azure spilled onto pay-as-you-go capacity is billed at the served model's rates,
# which the deployment's zeroed PTU entry would misreport, so that fallback is skipped there.
base_model: Final = (
metadata.get("deployment") if configured_base_model is None and not spilled_over else configured_base_model
)
custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj)
raw_response_cost: Final = kwargs.get("response_cost")
llm_response_cost: Final[float] = raw_response_cost or 0.0
guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))

29
litellm/litellm_core_utils/llm_response_utils/response_metadata.py
View file

@ -4,6 +4,7 @@ from functools import reduce
from typing import Final
import httpx
from pydantic import TypeAdapter, ValidationError
from litellm.constants import LITELLM_DETAILED_TIMING
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers
@ -101,6 +102,32 @@ def response_timing_metrics(
return {"_response_ms": total_response_time_ms, "litellm_overhead_time_ms": overhead_ms}
_HIDDEN_PARAMS_DICT: Final = TypeAdapter(dict[str, object])
def _hidden_params_mapping(hidden_params: object) -> Mapping[str, object]:
try:
return _HIDDEN_PARAMS_DICT.validate_python(hidden_params)
except ValidationError:
return {}
def _existing_provider_response_model(result: object) -> str | None:
"""The pricing model a response already carries, like the decisions path sets, wins over the response model."""
hidden_params: Final[object] = getattr(result, "_hidden_params", None)
carried: Final[object] = _hidden_params_mapping(hidden_params).get("provider_response_model") or getattr(
hidden_params, "provider_response_model", None
)
return carried if isinstance(carried, str) and carried else None
def _provider_response_model(result: object) -> str | None:
"""The model the provider answered with, kept because the proxy later rewrites ``result.model`` to the alias the
client asked for and cost-map lookups for the standard logging payload would otherwise miss."""
model: Final = getattr(result, "model", None) if isinstance(result, ModelResponse) else None
return model if isinstance(model, str) and model else None
class ResponseMetadata:
"""
Handles setting and managing `_hidden_params`, `response_time_ms`, and `litellm_overhead_time_ms` for LiteLLM responses
@ -147,6 +174,8 @@ class ResponseMetadata:
preserve_litellm_internal_headers=True,
),
"litellm_model_name": model,
"provider_response_model": _existing_provider_response_model(self.result)
or _provider_response_model(self.result),
}
self._update_hidden_params(new_params)

149
litellm/litellm_core_utils/logging_callback_manager.py
View file

@ -1,4 +1,6 @@
from collections.abc import Callable
import threading
from collections.abc import Callable, Iterable, Iterator
from types import MethodType
from typing import TYPE_CHECKING, Final
import litellm
@ -15,6 +17,7 @@ else:
_custom_logger_compatible_callbacks_literal = str
_generic_api_logger_cache: Final[dict[str, GenericAPILogger]] = {}
_callback_lists_lock: Final = threading.RLock()
class LoggingCallbackManager:
@ -43,16 +46,18 @@ class LoggingCallbackManager:
Add a input callback to litellm.input_callback.
Auto-routes async callbacks to litellm._async_input_callback.
"""
if not isinstance(callback, str) and self._is_async_callable(callback):
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_input_callback)
else:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.input_callback)
with _callback_lists_lock:
if not isinstance(callback, str) and self._is_async_callable(callback):
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_input_callback)
else:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.input_callback)
def add_litellm_service_callback(self, callback: CustomLogger | str | Callable[..., object]):
"""
Add a service callback to litellm.service_callback
"""
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.service_callback)
with _callback_lists_lock:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.service_callback)
def add_litellm_callback(self, callback: CustomLogger | str | Callable[..., object]):
"""
@ -60,10 +65,11 @@ class LoggingCallbackManager:
Ensures no duplicates are added.
"""
self._safe_add_callback_to_list(
callback=callback,
parent_list=litellm.callbacks,
)
with _callback_lists_lock:
self._safe_add_callback_to_list(
callback=callback,
parent_list=litellm.callbacks,
)
def add_litellm_success_callback(self, callback: CustomLogger | str | Callable[..., object]):
"""
@ -71,37 +77,41 @@ class LoggingCallbackManager:
Auto-routes async callbacks to litellm._async_success_callback.
Special-cases 'dynamodb' and 'openmeter' as async callbacks.
"""
if (
isinstance(callback, str)
and callback in ("dynamodb", "openmeter")
or not isinstance(callback, str)
and self._is_async_callable(callback)
):
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback)
else:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.success_callback)
with _callback_lists_lock:
if (
isinstance(callback, str)
and callback in ("dynamodb", "openmeter")
or not isinstance(callback, str)
and self._is_async_callable(callback)
):
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback)
else:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.success_callback)
def add_litellm_failure_callback(self, callback: CustomLogger | str | Callable[..., object]):
"""
Add a failure callback to `litellm.failure_callback`.
Auto-routes async callbacks to litellm._async_failure_callback.
"""
if not isinstance(callback, str) and self._is_async_callable(callback):
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback)
else:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.failure_callback)
with _callback_lists_lock:
if not isinstance(callback, str) and self._is_async_callable(callback):
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback)
else:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.failure_callback)
def add_litellm_async_success_callback(self, callback: CustomLogger | Callable[..., object] | str):
"""
Add a success callback to litellm._async_success_callback
"""
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback)
with _callback_lists_lock:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback)
def add_litellm_async_failure_callback(self, callback: CustomLogger | Callable[..., object] | str):
"""
Add a failure callback to litellm._async_failure_callback
"""
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback)
with _callback_lists_lock:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback)
def remove_callback_from_list_by_object(self, callback_list, obj, require_self=True):
"""
@ -110,13 +120,14 @@ class LoggingCallbackManager:
if not isinstance(callback_list, list): # Not list -> do nothing
return
if require_self:
remove_list = [c for c in callback_list if hasattr(c, "__self__") and c.__self__ == obj]
else:
remove_list = [c for c in callback_list if c == obj]
with _callback_lists_lock:
if require_self:
remove_list = [c for c in callback_list if hasattr(c, "__self__") and c.__self__ == obj]
else:
remove_list = [c for c in callback_list if c == obj]
for c in remove_list:
callback_list.remove(c)
for c in remove_list:
callback_list.remove(c)
def remove_callbacks_by_type(self, callback_list, callback_type):
"""
@ -134,10 +145,11 @@ class LoggingCallbackManager:
if not isinstance(callback_list, list):
return
remove_list: Final = [c for c in callback_list if isinstance(c, callback_type)]
with _callback_lists_lock:
remove_list: Final = [c for c in callback_list if isinstance(c, callback_type)]
for c in remove_list:
callback_list.remove(c)
for c in remove_list:
callback_list.remove(c)
def _add_string_callback_to_list(
self, callback: str, parent_list: list[CustomLogger | Callable[..., object] | str]
@ -336,6 +348,55 @@ class LoggingCallbackManager:
return "-".join(key_parts)
get_custom_logger_key = _get_custom_logger_key
callback_lists_lock: Final = _callback_lists_lock
def drop_callbacks(self, callbacks: Iterable[object], method_owners: Iterable[object] = ()) -> frozenset[str]:
dropped_ids: Final = frozenset(map(id, callbacks))
owner_ids: Final = frozenset(map(id, method_owners))
def keep(callback: object) -> bool:
if id(callback) in dropped_ids:
return False
return not (isinstance(callback, MethodType) and id(callback.__self__) in owner_ids)
with _callback_lists_lock:
vacated_keys: Final = frozenset(
self._get_custom_logger_key(callback)
for callback in litellm.callbacks
if isinstance(callback, CustomLogger) and id(callback) in dropped_ids
)
litellm.callbacks = [callback for callback in litellm.callbacks if keep(callback)]
litellm.input_callback = [callback for callback in litellm.input_callback if keep(callback)]
litellm.success_callback = [callback for callback in litellm.success_callback if keep(callback)]
litellm.failure_callback = [callback for callback in litellm.failure_callback if keep(callback)]
litellm.service_callback = [callback for callback in litellm.service_callback if keep(callback)]
litellm._async_success_callback = [
callback for callback in litellm._async_success_callback if keep(callback)
]
litellm._async_failure_callback = [
callback for callback in litellm._async_failure_callback if keep(callback)
]
return vacated_keys
def with_custom_logger_keys(
self, candidates: Iterable[CustomLogger], keys: frozenset[str]
) -> tuple[CustomLogger, ...]:
if not keys:
return ()
class_names: Final = frozenset(key.split("-", 1)[0] for key in keys)
same_class: Final = (candidate for candidate in candidates if type(candidate).__name__ in class_names)
return tuple(candidate for candidate in same_class if self._get_custom_logger_key(candidate) in keys)
def without_custom_logger_keys(
self, callbacks: Iterable[str | Callable[..., object] | CustomLogger], keys: frozenset[str]
) -> Iterator[str | Callable[..., object] | CustomLogger]:
return (
callback
for callback in callbacks
if not (isinstance(callback, CustomLogger) and self._get_custom_logger_key(callback) in keys)
)
def _reset_all_callbacks(self):
"""
Reset all callbacks to an empty list
@ -363,19 +424,21 @@ class LoggingCallbackManager:
_get_all_callbacks = get_all_callbacks
def remove_callback_from_all_lists(self, obj, require_self=False) -> None:
def remove_callback_from_all_lists(self, obj: object, require_self: bool = False) -> None:
"""
Remove a callback object from every callback list it may have been
promoted into, so a re-initialized callback leaves no stale instance behind.
"""
for callback_list in (
litellm.callbacks,
litellm.success_callback,
litellm.failure_callback,
litellm._async_success_callback,
litellm._async_failure_callback,
):
self.remove_callback_from_list_by_object(callback_list, obj, require_self=require_self)
with _callback_lists_lock:
for callback_list in (
litellm.callbacks,
litellm.input_callback,
litellm.success_callback,
litellm.failure_callback,
litellm._async_success_callback,
litellm._async_failure_callback,
):
self.remove_callback_from_list_by_object(callback_list, obj, require_self=require_self)
def get_active_additional_logging_utils_from_custom_logger(
self,

230
litellm/litellm_core_utils/redact_messages.py
View file

@ -10,10 +10,12 @@
import asyncio
import copy
import inspect
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from dataclasses import replace
from typing import TYPE_CHECKING, Any, Final
from pydantic import TypeAdapter
import litellm
from litellm.constants import REDACTED_BY_LITELLM
from litellm.integrations.custom_logger import CustomLogger
@ -40,36 +42,79 @@ else:
LiteLLMLoggingObject = Any
def _opted_out_of_message_logging(custom_logger: CustomLogger) -> bool:
return hasattr(custom_logger, "message_logging") and custom_logger.message_logging is not True
def redact_message_input_output_from_custom_logger(
litellm_logging_obj: LiteLLMLoggingObject, result, custom_logger: CustomLogger
):
if hasattr(custom_logger, "message_logging") and custom_logger.message_logging is not True:
) -> object:
if _opted_out_of_message_logging(custom_logger):
return perform_redaction(litellm_logging_obj.model_call_details, result, redact_streaming_responses=False)
return result
def redact_streaming_responses_for_custom_logger(model_call_details: dict, custom_logger: CustomLogger) -> dict:
def redacted_model_call_details_for_custom_logger(model_call_details: dict, custom_logger: CustomLogger) -> dict:
"""
Returns a copy of model_call_details whose streaming response entries are redacted deepcopies
when the custom logger has opted out of message logging. The shared model_call_details is left
untouched so other callbacks still receive the unredacted response.
Returns a copy of model_call_details with the request messages, instructions and streaming responses
redacted when the custom logger has opted out of message logging. The shared model_call_details is left
untouched so other callbacks and the spend log still receive the unredacted values.
"""
if not (hasattr(custom_logger, "message_logging") and custom_logger.message_logging is not True):
if not _opted_out_of_message_logging(custom_logger):
return model_call_details
redacted_entries: Final = {
streaming_key: _redacted_streaming_response_copy(model_call_details[streaming_key])
for streaming_key in ("complete_streaming_response", "async_complete_streaming_response")
if model_call_details.get(streaming_key) is not None
return {
**{key: value for key, value in model_call_details.items() if key != SERVED_OUTPUT_TEXTS_KEY},
**_redacted_entries_for_opted_out_logger(model_call_details),
}
if not redacted_entries:
return model_call_details
return {**model_call_details, **redacted_entries}
def _redacted_streaming_response_copy(streaming_response):
redacted_response: Final = copy.deepcopy(streaming_response)
_redact_streaming_response(redacted_response)
return redacted_response
def _redacted_entries_for_opted_out_logger(model_call_details: Mapping[str, object]) -> Mapping[str, object]:
optional_params: Final = _as_mapping(model_call_details.get("optional_params"))
litellm_params: Final = _as_mapping(model_call_details.get("litellm_params"))
standard_logging_object: Final = _as_mapping(model_call_details.get("standard_logging_object"))
return {
"messages": [{"role": "user", "content": REDACTED_BY_LITELLM}],
"prompt": "",
"input": "",
**({"instructions": REDACTED_BY_LITELLM} if model_call_details.get("instructions") is not None else {}),
**({"optional_params": _with_redacted_instructions(optional_params)} if optional_params is not None else {}),
**({"litellm_params": redacted_litellm_params(litellm_params)} if litellm_params is not None else {}),
**(
{"standard_logging_object": _redact_standard_logging_object(standard_logging_object)}
if standard_logging_object is not None
else {}
),
**{
streaming_key: _redacted_streaming_response_copy(model_call_details[streaming_key])
for streaming_key in (
"complete_streaming_response",
"async_complete_streaming_response",
"complete_response",
)
if model_call_details.get(streaming_key) is not None
},
}
def redacted_response_obj_for_custom_logger(response_obj: object, custom_logger: CustomLogger) -> object:
if not _opted_out_of_message_logging(custom_logger) or not _is_redactable_response(response_obj):
return response_obj
return perform_redaction({}, response_obj)
def _is_redactable_response(response_obj: object) -> bool:
return isinstance(
response_obj, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse)
) or (isinstance(response_obj, dict) and ("choices" in response_obj or "output" in response_obj))
def _redacted_streaming_response_copy(streaming_response: object) -> object:
redacted_response: Final = (
streaming_response
if isinstance(streaming_response, litellm.ResponsesAPIResponse)
else copy.deepcopy(streaming_response)
)
return _redact_streaming_response(redacted_response)
def _redact_streaming_response(streaming_response):
@ -77,10 +122,13 @@ def _redact_streaming_response(streaming_response):
for choice in streaming_response.choices:
_redact_choice_content(choice)
redact_vertex_ai_metadata_from_logged_object(streaming_response)
elif isinstance(streaming_response, litellm.ResponsesAPIResponse):
return perform_redaction({}, streaming_response)
elif hasattr(streaming_response, "output"):
_redact_responses_api_output(streaming_response.output)
if hasattr(streaming_response, "reasoning") and streaming_response.reasoning is not None:
streaming_response.reasoning = None
return streaming_response
def _redact_tool_calls(tool_calls) -> None:
@ -177,31 +225,75 @@ def _redact_responses_api_output(output_items):
output_item.input = REDACTED_BY_LITELLM
def _redact_responses_api_output_dict(output_items, redacted_str: str):
"""Helper to redact ResponsesAPIResponse output items in dict form."""
for output_item in output_items:
if not isinstance(output_item, dict):
continue
_RESPONSE_ITEMS: Final = TypeAdapter(tuple[object, ...])
if output_item.get("text") is not None:
output_item["text"] = redacted_str
if isinstance(output_item.get("content"), list):
for content_item in output_item["content"]:
if isinstance(content_item, dict) and content_item.get("text") is not None:
content_item["text"] = redacted_str
if isinstance(content_item, dict) and content_item.get("refusal") is not None:
content_item["refusal"] = redacted_str
def _redacted_responses_api_output_part(part: object, fields: tuple[str, ...], redacted_str: str) -> object:
if (part_mapping := _as_mapping(part)) is None:
return part
return {key: redacted_str if key in fields and value is not None else value for key, value in part_mapping.items()}
if output_item.get("type") == "reasoning" and isinstance(output_item.get("summary"), list):
for summary_item in output_item["summary"]:
if isinstance(summary_item, dict) and summary_item.get("text") is not None:
summary_item["text"] = redacted_str
if output_item.get("type") == "function_call" and "arguments" in output_item:
output_item["arguments"] = redacted_str
if output_item.get("type") == "custom_tool_call" and "input" in output_item:
output_item["input"] = redacted_str
def _redacted_responses_api_output_item(item: object, redacted_str: str) -> object:
if (item_mapping := _as_mapping(item)) is None:
return item
content: Final = item_mapping.get("content")
summary: Final = item_mapping.get("summary")
return {
**item_mapping,
**({"text": redacted_str} if item_mapping.get("text") is not None else {}),
**(
{
"content": [
_redacted_responses_api_output_part(part, ("text", "refusal"), redacted_str)
for part in _RESPONSE_ITEMS.validate_python(content)
]
}
if isinstance(content, list)
else {}
),
**(
{
"summary": [
_redacted_responses_api_output_part(part, ("text",), redacted_str)
for part in _RESPONSE_ITEMS.validate_python(summary)
]
}
if item_mapping.get("type") == "reasoning" and isinstance(summary, list)
else {}
),
**(
{"arguments": redacted_str}
if item_mapping.get("type") == "function_call" and "arguments" in item_mapping
else {}
),
**(
{"input": redacted_str}
if item_mapping.get("type") == "custom_tool_call" and "input" in item_mapping
else {}
),
}
def _redact_responses_api_output_dict(output_items: Sequence[object], redacted_str: str) -> Sequence[object]:
return [_redacted_responses_api_output_item(item, redacted_str) for item in output_items]
def _with_redacted_instructions(params: Mapping[str, object]) -> Mapping[str, object]:
return params if params.get("instructions") is None else {**params, "instructions": REDACTED_BY_LITELLM}
def _redacted_responses_api_response(
response: Mapping[str, object], *, redact_instructions: bool = True
) -> Mapping[str, object]:
output: Final = response.get("output")
redacted: Final = {
**response,
"output": _redact_responses_api_output_dict(_RESPONSE_ITEMS.validate_python(output), REDACTED_BY_LITELLM)
if isinstance(output, list)
else output,
}
return _with_redacted_instructions(redacted) if redact_instructions else redacted
def redacted_standard_logging_payload(payload: Mapping[str, object]) -> Mapping[str, object]:
@ -215,20 +307,25 @@ def redacted_standard_logging_payload(payload: Mapping[str, object]) -> Mapping[
return _redact_standard_logging_object(payload)
def _redact_standard_logging_object(payload: Mapping[str, object]) -> dict[str, object]:
def _redact_standard_logging_object(
payload: Mapping[str, object], *, redact_instructions: bool = True
) -> dict[str, object]:
standard_logging_object: Final = copy.deepcopy(without_classifier_audit(payload))
redacted_str: Final = REDACTED_BY_LITELLM
if standard_logging_object.get("messages") is not None:
standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}]
model_parameters: Final = _as_mapping(standard_logging_object.get("model_parameters"))
if model_parameters is not None and redact_instructions:
standard_logging_object["model_parameters"] = _with_redacted_instructions(model_parameters)
response: Final = standard_logging_object.get("response")
if response is not None:
if isinstance(response, dict) and "output" in response:
# ResponsesAPIResponse format - redact content in output items
if isinstance(response.get("output"), list):
_redact_responses_api_output_dict(response["output"], redacted_str)
redact_vertex_ai_metadata_from_logged_object(response)
standard_logging_object["response"] = _redacted_responses_api_response(
response, redact_instructions=redact_instructions
)
redact_vertex_ai_metadata_from_logged_object(standard_logging_object["response"])
elif isinstance(response, dict) and "choices" in response:
# ModelResponse dict format - redact content in choices
if isinstance(response.get("choices"), list):
@ -320,11 +417,11 @@ def _redact_hidden_provider_specific_fields(params: Mapping[str, object]) -> Non
def redacted_litellm_params(params: Mapping[str, object], *, redact_request_body: bool = True) -> dict[str, object]:
"""
Copy of litellm_params without classifier audit fields or baseline route request parameters.
redact_request_body also redacts the nested proxy_server_request body.
redact_request_body also redacts the request instructions and the nested proxy_server_request body.
"""
request: Final = _as_mapping(params.get("proxy_server_request"))
return {
**params,
**(_with_redacted_instructions(params) if redact_request_body else params),
**{
key: _redacted_baseline_metadata(value)
for key, value in params.items()
@ -344,8 +441,9 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
redact_streaming_responses=False skips the in-place redaction of the shared streaming
response entries; per-callback redaction hands each opted-out callback its own redacted
copy via redact_streaming_responses_for_custom_logger instead. It also leaves the shared
proxy request body and hidden provider fields as they are, since the spend log reads them.
copy via redacted_model_call_details_for_custom_logger instead. It also leaves the shared
request instructions, proxy request body and hidden provider fields as they are, since the
spend log and the other callbacks read them.
"""
# Redact model_call_details
params: Final = _as_mapping(model_call_details.get("litellm_params"))
@ -358,16 +456,31 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}]
model_call_details["prompt"] = ""
model_call_details["input"] = ""
optional_params: Final = _as_mapping(model_call_details.get("optional_params"))
if redact_streaming_responses:
model_call_details.update(
{
**({"instructions": REDACTED_BY_LITELLM} if model_call_details.get("instructions") is not None else {}),
**({"optional_params": _with_redacted_instructions(optional_params)} if optional_params else {}),
}
)
model_call_details.pop(SERVED_OUTPUT_TEXTS_KEY, None)
standard_logging_object: Final = model_call_details.get("standard_logging_object")
if isinstance(standard_logging_object, Mapping):
model_call_details["standard_logging_object"] = _redact_standard_logging_object(standard_logging_object)
model_call_details["standard_logging_object"] = _redact_standard_logging_object(
standard_logging_object, redact_instructions=redact_streaming_responses
)
redact_vertex_ai_metadata_from_litellm_params(model_call_details)
# Redact streaming response
if redact_streaming_responses and model_call_details.get("stream", False) is True:
for _streaming_key in ("complete_streaming_response", "async_complete_streaming_response"):
_redact_streaming_response(model_call_details.get(_streaming_key))
model_call_details.update(
{
streaming_key: _redact_streaming_response(model_call_details[streaming_key])
for streaming_key in ("complete_streaming_response", "async_complete_streaming_response")
if model_call_details.get(streaming_key) is not None
}
)
# Redact result
if result is not None:
@ -381,10 +494,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
# For async objects, return a simple redacted response without deepcopy
return {"text": REDACTED_BY_LITELLM}
if not (
isinstance(result, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse))
or (isinstance(result, dict) and ("choices" in result or "output" in result))
):
if not _is_redactable_response(result):
return {"text": REDACTED_BY_LITELLM}
_result: Final = copy.deepcopy(result)
@ -399,14 +509,18 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
_redact_model_response_dict_choices(_result["choices"], REDACTED_BY_LITELLM)
redact_vertex_ai_metadata_from_logged_object(_result)
elif isinstance(_result, dict) and "output" in _result:
if isinstance(_result.get("output"), list):
_redact_responses_api_output_dict(_result["output"], REDACTED_BY_LITELLM)
return _redacted_responses_api_response(_result, redact_instructions=redact_streaming_responses)
elif isinstance(_result, litellm.ResponsesAPIResponse):
if hasattr(_result, "output"):
_redact_responses_api_output(_result.output)
# Redact reasoning field in ResponsesAPIResponse
if hasattr(_result, "reasoning") and _result.reasoning is not None:
_result.reasoning = None
return _result.model_copy(
update={"instructions": REDACTED_BY_LITELLM}
if redact_streaming_responses and _result.instructions is not None
else {}
)
elif isinstance(_result, litellm.EmbeddingResponse):
if hasattr(_result, "data") and _result.data is not None:
_result.data = []

117
litellm/litellm_core_utils/streaming_handler.py
View file

@ -10,7 +10,7 @@ import traceback
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Any, Final, NoReturn, Protocol, TypeVar, cast
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeVar, cast
import anyio
import httpx
@ -60,6 +60,9 @@ from .exception_mapping_utils import exception_type, with_deployment_api_base
from .llm_response_utils.get_api_base import get_api_base
from .rules import Rules
if TYPE_CHECKING:
from .litellm_logging import BilledRoundsSpend
# Constants for special delta attribute names
AUDIO_ATTRIBUTE: Final = "audio"
IMAGE_ATTRIBUTE: Final = "images"
@ -259,6 +262,8 @@ def _provider_hidden_params(
class CustomStreamWrapper:
_earlier_rounds_spend: "BilledRoundsSpend | None" = None
@property
def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation
return self._hidden_params
@ -346,6 +351,10 @@ class CustomStreamWrapper:
self.messages = getattr(logging_obj, "messages", None)
self.sent_stream_usage = False
self.send_stream_usage = True if self.check_send_stream_usage(self.stream_options) else False
self.caller_requested_stream_usage: Final = (
self.check_send_stream_usage(getattr(logging_obj, "stream_options", None))
and getattr(logging_obj, "stream_usage_injected", False) is not True
)
self.tool_call = False
# keep track of the returned chunks - used for calculating the input/output tokens for stream options
self.chunks: list[ModelResponseStream] = []
@ -2099,7 +2108,7 @@ class CustomStreamWrapper:
self._handle_stream_fallback_error(error)
if self.sent_last_chunk is True:
complete_streaming_response: Final = self._build_complete_streaming_response()
response = self.model_response_creator()
response = self._new_usage_chunk()
if complete_streaming_response is not None:
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)
@ -2251,27 +2260,10 @@ class CustomStreamWrapper:
)
if _has_usage:
# Store a copy ONLY when usage stripping below will mutate
# the chunk. For non-usage chunks (vast majority), store
# directly to avoid expensive model_copy() per chunk.
self.chunks.append(processed_chunk.model_copy())
# Strip usage from the outgoing chunk so it's not sent twice
# (once in the chunk, once in _hidden_params).
obj_dict = processed_chunk.model_dump()
if "usage" in obj_dict:
del obj_dict["usage"]
processed_chunk = self.model_response_creator(
chunk=obj_dict,
hidden_params=cast( # cast-ok: preserve mapping operations on dynamic storage
Mapping[str, object], getattr(processed_chunk, HIDDEN_PARAMS_ATTR)
),
)
is_empty = is_model_response_stream_empty(
model_response=cast(ModelResponseStream, processed_chunk)
)
if is_empty:
usage_free_chunk = self._record_and_strip_usage_chunk(processed_chunk)
if usage_free_chunk is None:
continue
processed_chunk = usage_free_chunk
else:
# No usage data — safe to store directly without copying
self.chunks.append(processed_chunk)
@ -2302,13 +2294,18 @@ class CustomStreamWrapper:
if processed_chunk is None:
continue
choice = processed_chunk.choices[0]
if isinstance(choice, StreamingChoices):
self._accumulate_streamed_delta(choice.delta)
if processed_chunk.choices:
choice = processed_chunk.choices[0]
if isinstance(choice, StreamingChoices):
self._accumulate_streamed_delta(choice.delta)
self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
# RETURN RESULT
self.chunks.append(processed_chunk)
return processed_chunk
if not self.send_stream_usage or getattr(processed_chunk, "usage", None) is None:
self.chunks.append(processed_chunk)
return processed_chunk
usage_free_chunk = self._record_and_strip_usage_chunk(processed_chunk)
if usage_free_chunk is None:
continue
return usage_free_chunk
except (StopAsyncIteration, StopIteration):
if self._should_fail_on_clean_eof():
await self.aclose()
@ -2318,12 +2315,7 @@ class CustomStreamWrapper:
traceback_exception = traceback.format_exc()
## ADD DEBUG INFORMATION - E.G. LITELLM REQUEST TIMEOUT
traceback_exception += f"\nLiteLLM Default Request Timeout - {litellm.request_timeout}"
if self.logging_obj is not None:
self._record_partial_usage_for_failure()
## LOGGING
self._failure_logging = asyncio.create_task(
self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True)
)
self._dispatch_stream_failure(e, traceback_exception)
await self.aclose()
self._handle_stream_fallback_error(e)
except (httpx.ReadError, httpx.RemoteProtocolError) as e:
@ -2338,7 +2330,7 @@ class CustomStreamWrapper:
async def _finalize_completed_stream(self, cache_hit: bool) -> "ModelResponseStream":
if self.sent_last_chunk is True:
complete_streaming_response: Final = await self._async_build_complete_streaming_response()
response: Final = self.model_response_creator()
response: Final = self._new_usage_chunk()
if complete_streaming_response is not None:
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)
@ -2424,29 +2416,36 @@ class CustomStreamWrapper:
return processed_chunk
def _log_stream_failure_and_raise(self, e: Exception) -> NoReturn:
traceback_exception: Final = traceback.format_exc()
if self.logging_obj is not None:
self._record_partial_usage_for_failure()
## LOGGING
self._failure_logging = asyncio.create_task(
self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True)
)
self._dispatch_stream_failure(e, traceback.format_exc())
self._handle_stream_fallback_error(e)
async def wait_for_failure_logging(self, timeout: float) -> None:
def _dispatch_stream_failure(self, e: Exception, traceback_exception: str) -> None:
self._record_partial_usage_for_failure()
self._failure_logging = asyncio.create_task(
self.logging_obj.dispatch_attempt_failure_handlers(e, traceback_exception)
)
@property
def has_dispatched_failure_logging(self) -> bool:
return self._failure_logging is not None
async def wait_for_failure_logging(self, timeout: float | None) -> None:
"""Let this stream's failure log read its own deployment before a fallback reuses the logging object."""
if self._failure_logging is None:
return
await asyncio.wait((self._failure_logging,), timeout=timeout)
def bill_failure_after_earlier_rounds(self, earlier: "BilledRoundsSpend") -> None:
self._earlier_rounds_spend = earlier
def _record_partial_usage_for_failure(self) -> None:
"""
A stream that breaks mid-flight still billed the provider for the chunks
already delivered. Recover that partial usage from the chunks seen so
far and stash it, with its cost, on the logging object so the failure
handler records the real partial spend instead of zero. A request that
later recovers via a router fallback overwrites this with the combined
success log on the same request id, so this never double counts.
handler records the real partial spend instead of zero. The router
moves it into terminal failure usage before opening the next attempt,
so a recovered attempt logs its own usage.
"""
if self.logging_obj is None or not self.chunks:
return
@ -2465,10 +2464,12 @@ class CustomStreamWrapper:
if self.model:
partial_response.model = self.model
backfill_missing_cache_usage_fields(usage)
self.logging_obj.model_call_details["combined_usage_object"] = usage
self.logging_obj.model_call_details["response_cost"] = (
self.logging_obj.response_cost_calculator(result=partial_response) or 0.0
)
partial_cost: Final = self.logging_obj.response_cost_calculator(result=partial_response) or 0.0
if self._earlier_rounds_spend is None:
self.logging_obj.model_call_details["combined_usage_object"] = usage
self.logging_obj.model_call_details["response_cost"] = partial_cost
return
self.logging_obj.record_failure_after_earlier_rounds(self._earlier_rounds_spend, usage, partial_cost)
except Exception as recover_error:
verbose_logger.debug(
"could not recover partial usage for interrupted stream: %s",
@ -2495,6 +2496,22 @@ class CustomStreamWrapper:
return False
return any(get(field) for field in cls._CONTINUATION_DISQUALIFYING_DELTA_FIELDS)
def _new_usage_chunk(self) -> ModelResponseStream:
usage_chunk: Final = self.model_response_creator()
if self.caller_requested_stream_usage:
usage_chunk.choices = []
return usage_chunk
def _record_and_strip_usage_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream | None:
self.chunks.append(chunk.model_copy())
usage_free_chunk: Final = self.model_response_creator(
chunk={key: value for key, value in chunk.model_dump().items() if key != "usage"},
hidden_params=cast( # cast-ok: preserve mapping operations on dynamic storage
Mapping[str, object], getattr(chunk, HIDDEN_PARAMS_ATTR)
),
)
return None if is_model_response_stream_empty(model_response=usage_free_chunk) else usage_free_chunk
def _accumulate_streamed_delta(self, delta: object) -> None:
# Shared by the sync, async, and non-aiohttp iteration sites so answer
# text and the disqualifying-content latch stay in step across all three.

24
litellm/litellm_core_utils/url_utils.py
View file

@ -23,7 +23,7 @@ import asyncio
import socket
from ipaddress import ip_address, ip_network
from typing import Any, Final, Protocol
from urllib.parse import quote, urlparse, urlunparse
from urllib.parse import quote, urlparse, urlsplit, urlunparse
import httpx
from typing_extensions import ReadOnly, TypedDict
@ -129,6 +129,28 @@ def encode_url_path_segments(value: object, *, field_name: str = "path") -> str:
return "/".join(encode_url_path_segment(segment, field_name=field_name) for segment in value_str.split("/"))
def _strip_trailing_slashes_keeping_the_scheme(api_base: str) -> str:
stripped: Final = api_base.rstrip("/")
parts: Final = urlsplit(stripped)
if parts.scheme and not (parts.netloc or parts.path or parts.query or parts.fragment):
return api_base
if not parts.query or not parts.path.endswith("/"):
return stripped
url_before_query, _, query = stripped.partition("?")
return f"{url_before_query.rstrip('/')}?{query}"
def join_api_base(api_base: str, route: str) -> str:
return f"{_strip_trailing_slashes_keeping_the_scheme(api_base)}/{route.lstrip('/')}"
def ensure_api_route(api_base: str, route: str) -> str:
base: Final = _strip_trailing_slashes_keeping_the_scheme(api_base)
if urlsplit(base).path.endswith(f"/{route.lstrip('/')}"):
return base
return join_api_base(base, route)
def _is_blocked_ip(addr: str) -> bool:
"""Return True for any IP not safe to reach from a user-supplied URL.

3
litellm/llms/aiml/image_generation/transformation.py
View file

@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import ConfigDict, TypeAdapter
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
@ -121,7 +122,7 @@ class AimlImageGenerationConfig(BaseImageGenerationConfig):
complete_url = complete_url.rstrip("/")
# Strip /v1 suffix if present since IMAGE_GENERATION_ENDPOINT already includes v1
complete_url = complete_url.removesuffix("/v1")
complete_url = f"{complete_url}/{self.IMAGE_GENERATION_ENDPOINT}"
complete_url = join_api_base(complete_url, self.IMAGE_GENERATION_ENDPOINT)
return complete_url
def validate_environment(

8
litellm/llms/aiohttp_openai/chat/transformation.py
View file

@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Any, Final
from aiohttp import ClientResponse
from pydantic import ConfigDict, TypeAdapter
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices, ModelResponse
@ -44,12 +45,7 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig):
Ensure - /v1/chat/completions is at the end of the url
"""
if api_base is None:
api_base = "https://api.openai.com"
if not api_base.endswith("/chat/completions"):
api_base += "/chat/completions"
return api_base
return ensure_api_route("https://api.openai.com" if api_base is None else api_base, "chat/completions")
def validate_environment(
self,

12
litellm/llms/anthropic/batches/transformation.py
View file

@ -7,7 +7,7 @@ import httpx
from httpx import Headers, Response
from typing_extensions import ReadOnly, TypedDict
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, ensure_api_route, join_api_base
from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest
@ -106,10 +106,8 @@ class AnthropicBatchesConfig(BaseBatchesConfig):
data: CreateBatchRequest,
) -> str:
"""Get the complete URL for batch creation request."""
api_base = api_base or self.anthropic_model_info.get_api_base(api_base)
if not api_base.endswith("/v1/messages/batches"):
api_base = f"{api_base.rstrip('/')}/v1/messages/batches"
return api_base
api_base = self.anthropic_model_info.resolve_anthropic_api_base(api_base)
return ensure_api_route(api_base, "v1/messages/batches")
def transform_create_batch_request(
self,
@ -158,9 +156,9 @@ class AnthropicBatchesConfig(BaseBatchesConfig):
Returns:
Complete URL for Anthropic batch retrieval: {api_base}/v1/messages/batches/{batch_id}
"""
api_base = api_base or self.anthropic_model_info.get_api_base(api_base)
api_base = self.anthropic_model_info.resolve_anthropic_api_base(api_base)
encoded_batch_id: Final = encode_url_path_segment(batch_id, field_name="batch_id")
return f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_batch_id}"
return join_api_base(api_base, f"v1/messages/batches/{encoded_batch_id}")
def transform_retrieve_batch_request(
self,

72
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -13,7 +13,7 @@ Pattern Overview:
"""
import json
from collections.abc import Mapping, MutableSequence, Sequence
from collections.abc import Iterable, Mapping, MutableSequence, Sequence
from copy import deepcopy
from dataclasses import dataclass
from itertools import chain, repeat
@ -27,8 +27,10 @@ from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.pass_through.adapters.transformation import (
OPENAI_MAX_TOOL_NAME_LENGTH,
LiteLLMAnthropicMessagesAdapter,
is_provider_native_tool_dict,
truncate_tool_name,
)
from litellm.llms.anthropic.pass_through.convertible_blocks import (
keep_non_string_tool_result_texts,
@ -202,6 +204,60 @@ def _is_client_tool_use(block: Mapping[str, object]) -> bool:
)
def _tool_use_name(block: object) -> str | None:
fields: Final = as_str_mapping(block)
if fields is None or fields.get("type") != "tool_use":
return None
return anthropic_tool_name(fields)
def _tool_use_names(message: object) -> tuple[str, ...]:
return tuple(name for name in map(_tool_use_name, list_field(message, "content")) if name is not None)
@dataclass(frozen=True, slots=True)
class _OriginalToolNames:
"""Caller tool names over the guardrail's limit, keyed by the shortened name the guardrail sees."""
declared: Mapping[str, str]
used: Mapping[str, str]
def _restorable_names(names: Iterable[str]) -> Mapping[str, str]:
listed: Final = tuple(names)
taken: Final = frozenset(listed)
shortened: Final = tuple(
(truncate_tool_name(name), name) for name in listed if len(name) > OPENAI_MAX_TOOL_NAME_LENGTH
)
return MappingProxyType({alias: name for alias, name in shortened if alias not in taken})
def _original_tool_names(request: Mapping[str, object]) -> _OriginalToolNames:
return _OriginalToolNames(
declared=_restorable_names(chain.from_iterable(map(anthropic_tool_names, list_field(request, "tools")))),
used=_restorable_names(chain.from_iterable(map(_tool_use_names, list_field(request, "messages")))),
)
def _restore_tool_use_names(messages: Sequence[object], original_names: Mapping[str, str]) -> None:
for message in messages:
for block in list_field(message, "content"):
if (name := _tool_use_name(block)) is not None and name in original_names and isinstance(block, dict):
block["name"] = original_names[name]
def _with_original_name(tool: ChatCompletionToolParam, original_names: Mapping[str, str]) -> ChatCompletionToolParam:
if "function" not in tool:
return tool
function: Final = tool["function"]
fields: Final = as_str_mapping(function)
name: Final = None if fields is None else fields.get("name")
original: Final = original_names.get(name) if isinstance(name, str) else None
if original is None:
return tool
return {**tool, "function": {**function, "name": original}}
def _write_back_system_block(system: object, block_idx: int, response: str) -> None:
if not isinstance(system, list):
return
@ -629,6 +685,9 @@ class AnthropicMessagesHandler(BaseTranslation):
model: Final = data.get("model")
if model:
inputs["model"] = model
original_tool_names: Final = _original_tool_names(
cast(Mapping[str, object], data) # cast-ok: Anthropic request body
)
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=data,
@ -643,7 +702,9 @@ class AnthropicMessagesHandler(BaseTranslation):
anthropic_config: Final = AnthropicConfig()
anthropic_tools: Final[list[AllAnthropicToolsValues]] = []
for tool in guardrailed_tools:
converted_tool, mcp_server = anthropic_config.map_tool_helper(tool)
converted_tool, mcp_server = anthropic_config.map_tool_helper(
_with_original_name(tool, original_tool_names.declared)
)
if converted_tool is not None:
anthropic_tools.append(converted_tool)
# Note: MCP servers are handled separately in the main transformation
@ -676,6 +737,7 @@ class AnthropicMessagesHandler(BaseTranslation):
),
hoisted_system_message=hoisted_system_message,
preserve_system_messages=has_midturn_system_message,
original_tool_names=original_tool_names.used,
)
else:
if guardrailed_texts and len(guardrailed_texts) != len(scanned):
@ -840,6 +902,8 @@ class AnthropicMessagesHandler(BaseTranslation):
structured_messages: list, # mutable-ok: API message payload
hoisted_system_message: object = None,
preserve_system_messages: bool = False,
*,
original_tool_names: Mapping[str, str],
) -> None:
"""Write a guardrail's structured-message rewrite back without losing corrections."""
from litellm.litellm_core_utils.prompt_templates.factory import (
@ -903,6 +967,10 @@ class AnthropicMessagesHandler(BaseTranslation):
for block in content:
if isinstance(block, dict) and block.get("type") == "thinking":
block.pop("cache_control", None)
_restore_tool_use_names(
cast(Sequence[object], converted), # cast-ok: Anthropic message payload
original_tool_names,
)
data["messages"] = keep_non_string_tool_result_texts(converted, data.get("messages"))
@staticmethod

13
litellm/llms/anthropic/common_utils.py
View file

@ -35,6 +35,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
parts_of,
)
from litellm.litellm_core_utils.reasoning_effort_utils import non_numeric_thinking_budget_error
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.wif import (
aget_anthropic_wif_token,
anthropic_base_without_chat_suffix,
@ -468,7 +469,7 @@ def _fetch_anthropic_models_page(
# which a read-only one cannot support
query: Final = f"?after_id={quote(after_id)}" if after_id else ""
response: Final = litellm.module_level_client.get(
url=f"{api_base}/v1/models{query}",
url=f"{join_api_base(api_base, 'v1/models')}{query}",
headers=headers,
follow_redirects=False,
)
@ -1406,7 +1407,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
return {**caller_headers, **anthropic_headers}
@staticmethod
def get_api_base(api_base: str | None = None) -> str | None:
def resolve_anthropic_api_base(api_base: str | None = None) -> str:
from litellm.secret_managers.main import get_secret_str
return (
@ -1416,6 +1417,10 @@ class AnthropicModelInfo(BaseLLMModelInfo):
or "https://api.anthropic.com"
)
@staticmethod
def get_api_base(api_base: str | None = None) -> str | None:
return AnthropicModelInfo.resolve_anthropic_api_base(api_base)
@staticmethod
def get_api_key(api_key: str | None = None) -> str | None:
"""An empty or whitespace-only key counts as unset: it can never authenticate anything, and
@ -1533,14 +1538,14 @@ class AnthropicModelInfo(BaseLLMModelInfo):
api_base: str | None,
litellm_params: Mapping[str, object] | None,
) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override
resolved_api_base: Final = AnthropicModelInfo.get_api_base(api_base)
resolved_api_base: Final = AnthropicModelInfo.resolve_anthropic_api_base(api_base)
auth_header: Final = AnthropicModelInfo.get_auth_header(
api_key,
resolved_api_base,
litellm_params=litellm_params,
allow_workload_identity=config_allows_workload_identity(self),
)
if resolved_api_base is None or auth_header is None:
if auth_header is None:
raise ValueError(
"ANTHROPIC_API_BASE/ANTHROPIC_BASE_URL or ANTHROPIC_API_KEY/ANTHROPIC_AUTH_TOKEN (or workload "
"identity federation via ANTHROPIC_FEDERATION_RULE_ID/ANTHROPIC_ORGANIZATION_ID/"

3
litellm/llms/anthropic/count_tokens/transformation.py
View file

@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
anthropic_system_blocks,
split_leading_system_run,
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers
from litellm.llms.anthropic.wif import resolve_anthropic_base
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
@ -317,7 +318,7 @@ class AnthropicCountTokensConfig:
Returns:
The endpoint URL for the CountTokens API
"""
return resolve_anthropic_base(api_base) + "/v1/messages/count_tokens"
return join_api_base(resolve_anthropic_base(api_base), "v1/messages/count_tokens")
def transform_request_to_count_tokens(
self,

6
litellm/llms/anthropic/files/handler.py
View file

@ -10,7 +10,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.llms.openai import (
FileContentRequest,
@ -74,7 +74,7 @@ class AnthropicFilesHandler:
batch_id = file_id
# Get Anthropic API credentials
api_base = self.anthropic_model_info.get_api_base(api_base)
api_base = self.anthropic_model_info.resolve_anthropic_api_base(api_base)
auth_header: Final = await self.anthropic_model_info.aget_auth_header(
api_key, api_base, litellm_params=litellm_params, allow_workload_identity=True
)
@ -84,7 +84,7 @@ class AnthropicFilesHandler:
# Construct the Anthropic batch results URL
encoded_batch_id: Final = encode_url_path_segment(batch_id, field_name="batch_id")
results_url: Final = f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_batch_id}/results"
results_url: Final = join_api_base(api_base, f"v1/messages/batches/{encoded_batch_id}/results")
# Prepare headers
headers: Final = {

14
litellm/llms/anthropic/files/transformation.py
View file

@ -22,7 +22,7 @@ from openai.types.file_deleted import FileDeleted
from pydantic import ConfigDict, TypeAdapter
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.transformation import (
BaseFilesConfig,
@ -80,7 +80,7 @@ class AnthropicFilesConfig(BaseFilesConfig):
stream: bool | None = None,
) -> str:
api_base = AnthropicModelInfo.get_api_base(api_base) or ANTHROPIC_FILES_API_BASE
return f"{api_base.rstrip('/')}/v1/files"
return join_api_base(api_base, "v1/files")
def get_error_class(
self,
@ -226,7 +226,7 @@ class AnthropicFilesConfig(BaseFilesConfig):
) -> tuple[str, dict]:
api_base: Final = AnthropicModelInfo.get_api_base(litellm_params.get("api_base")) or ANTHROPIC_FILES_API_BASE
encoded_file_id: Final = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}", {}
return join_api_base(api_base, f"v1/files/{encoded_file_id}"), {}
def transform_retrieve_file_response(
self,
@ -245,7 +245,7 @@ class AnthropicFilesConfig(BaseFilesConfig):
) -> tuple[str, dict]:
api_base: Final = AnthropicModelInfo.get_api_base(litellm_params.get("api_base")) or ANTHROPIC_FILES_API_BASE
encoded_file_id: Final = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}", {}
return join_api_base(api_base, f"v1/files/{encoded_file_id}"), {}
def transform_delete_file_response(
self,
@ -264,7 +264,7 @@ class AnthropicFilesConfig(BaseFilesConfig):
litellm_params: dict,
) -> tuple[str, dict]:
api_base: Final = AnthropicModelInfo.get_api_base(litellm_params.get("api_base")) or ANTHROPIC_FILES_API_BASE
url: Final = f"{api_base.rstrip('/')}/v1/files"
url: Final = join_api_base(api_base, "v1/files")
params: Final[dict[str, str]] = {}
if purpose:
params["purpose"] = purpose
@ -299,8 +299,8 @@ class AnthropicFilesConfig(BaseFilesConfig):
api_base: Final = AnthropicModelInfo.get_api_base(litellm_params.get("api_base")) or ANTHROPIC_FILES_API_BASE
encoded_file_id: Final = encode_url_path_segment(file_id, field_name="file_id")
if file_id.startswith(ANTHROPIC_MESSAGE_BATCH_ID_PREFIX):
return f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_file_id}/results", {}
return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}/content", {}
return join_api_base(api_base, f"v1/messages/batches/{encoded_file_id}/results"), {}
return join_api_base(api_base, f"v1/files/{encoded_file_id}/content"), {}
def transform_file_content_response(
self,

26
litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py
View file

@ -16,6 +16,7 @@ from typing import (
Protocol,
cast,
get_args,
runtime_checkable,
)
from pydantic import TypeAdapter
@ -65,6 +66,14 @@ class _ChunkStream(Protocol):
def __aiter__(self) -> "AsyncIterator[ModelResponseStream]": ...
@runtime_checkable
class _FailureLoggingWaiter(Protocol):
@property
def has_dispatched_failure_logging(self) -> bool: ...
async def wait_for_failure_logging(self, timeout: float | None) -> None: ...
def _optional_attr(obj: object, name: str) -> object:
return getattr(obj, name, None)
@ -189,6 +198,14 @@ class _CombinedChunkSplitter:
self._held_calls: Mapping[int, tuple[ModelResponseStream, _ToolCallPart]] = MappingProxyType({})
self._live_call_index: int | None = None
@property
def has_dispatched_failure_logging(self) -> bool:
return isinstance(self._stream, _FailureLoggingWaiter) and self._stream.has_dispatched_failure_logging is True
async def wait_for_failure_logging(self, timeout: float | None) -> None:
if isinstance(self._stream, _FailureLoggingWaiter):
await self._stream.wait_for_failure_logging(timeout=timeout)
@property
def chunks(self) -> "list[ModelResponseStream] | None":
return cast( # cast-ok: chunks is a list of ModelResponseStream on the inner stream
@ -522,7 +539,9 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
# Wrap the upstream stream so chunks that carry both content and a
# finish_reason (fake-streamed providers) are split into two — see
# _CombinedChunkSplitter.
super().__init__(_CombinedChunkSplitter(completion_stream))
combined_chunk_splitter: Final = _CombinedChunkSplitter(completion_stream)
super().__init__(combined_chunk_splitter)
self._combined_chunk_splitter: Final = combined_chunk_splitter
self.model = model
self._message_id: str = f"msg_{uuid.uuid4()}"
if litellm_logging_obj is not None:
@ -1244,7 +1263,10 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
if provider_error is e:
raise
raise provider_error from e
if logging_obj is not None:
stream_has_dispatched_failure_logging: Final = self._combined_chunk_splitter.has_dispatched_failure_logging
if stream_has_dispatched_failure_logging:
await self._combined_chunk_splitter.wait_for_failure_logging(timeout=None)
elif logging_obj is not None:
try:
await logging_obj.dispatch_failure_handlers(
exception=provider_error,

4
litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py
View file

@ -15,7 +15,7 @@ How it works:
"""
import uuid
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
import litellm
@ -48,7 +48,7 @@ class AdvisorOrchestrationHandler(MessagesInterceptor):
def can_handle(
self,
tools: list[dict] | None,
tools: Sequence[Mapping[str, object]] | None,
custom_llm_provider: str | None,
) -> bool:
if not tools:

4
litellm/llms/anthropic/pass_through/messages/interceptors/base.py
View file

@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Mapping, Sequence
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
@ -21,7 +21,7 @@ class MessagesInterceptor(ABC):
@abstractmethod
def can_handle(
self,
tools: list[dict] | None,
tools: Sequence[Mapping[str, object]] | None,
custom_llm_provider: str | None,
) -> bool:
"""Return True if this interceptor should handle the request."""

6
litellm/llms/anthropic/pass_through/messages/transformation.py
View file

@ -9,6 +9,7 @@ from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_b
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.anthropic.pass_through.convertible_blocks import drop_non_string_tool_result_texts
from litellm.llms.anthropic.pass_through.safeguards import proxy_serves_safeguards
from litellm.llms.base_llm.anthropic_messages.transformation import (
@ -287,10 +288,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
litellm_params: dict,
stream: bool | None = None,
) -> str:
api_base = AnthropicModelInfo.get_api_base(api_base) or "https://api.anthropic.com"
if not api_base.endswith("/v1/messages"):
api_base = f"{api_base}/v1/messages"
return api_base
return ensure_api_route(AnthropicModelInfo.get_api_base(api_base) or "https://api.anthropic.com", "v1/messages")
def validate_anthropic_messages_environment(
self,

10
litellm/llms/anthropic/skills/transformation.py
View file

@ -9,7 +9,7 @@ import httpx
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.base_llm.skills.transformation import (
BaseSkillsAPIConfig,
LiteLLMLoggingObj,
@ -75,13 +75,11 @@ class AnthropicSkillsConfig(BaseSkillsAPIConfig):
"""Get complete URL for Anthropic Skills API"""
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
if api_base is None:
api_base = AnthropicModelInfo.get_api_base()
resolved_api_base: Final = AnthropicModelInfo.get_api_base(api_base) or "https://api.anthropic.com"
if skill_id:
encoded_skill_id: Final = encode_url_path_segment(skill_id, field_name="skill_id")
return f"{api_base}/v1/skills/{encoded_skill_id}"
return f"{api_base}/v1/{endpoint}"
return join_api_base(resolved_api_base, f"v1/skills/{encoded_skill_id}")
return join_api_base(resolved_api_base, f"v1/{endpoint}")
def transform_create_skill_request(
self,

3
litellm/llms/anthropic/wif.py
View file

@ -15,6 +15,7 @@ from typing_extensions import assert_never
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.auth.client_credentials import keycloak_assertion_source
from litellm.llms.base_llm.auth.identity_source import (
AnthropicIdentitySourceKind,
@ -324,7 +325,7 @@ def anthropic_internal_issuer_jwks(credential_values: Mapping[str, object]) -> A
def build_anthropic_wif_spec(params: AnthropicWifParams, api_base: str) -> TokenExchangeSpec:
return TokenExchangeSpec(
token_url=api_base.rstrip("/") + ANTHROPIC_TOKEN_EXCHANGE_PATH,
token_url=join_api_base(api_base, ANTHROPIC_TOKEN_EXCHANGE_PATH),
assertion_ref=params.assertion_ref,
assertion_field="assertion",
static_body=MappingProxyType(

7
litellm/llms/apiserpent/search/transformation.py
View file

@ -14,6 +14,7 @@ from urllib.parse import urlencode
import httpx
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.apiserpent.search.defaults import (
DEEP_SEARCH_PATH,
NUM_MAX,
@ -77,13 +78,11 @@ class APISerpentSearchConfig(BaseSearchConfig):
Build the search URL. APISerpent uses GET, so the transformed request is
serialized into the query string. The endpoint path (quick vs deep) is
always applied; an ``api_base`` / ``APISERPENT_API_BASE`` override only
changes the host. The ``endswith`` guard keeps this idempotent, since the
changes the host. ``ensure_api_route`` keeps this idempotent, since the
handler re-invokes this method with the already-resolved URL as api_base.
"""
base = (api_base or get_secret_str("APISERPENT_API_BASE") or APISERPENT_BASE).rstrip("/")
path: Final = DEEP_SEARCH_PATH if self._is_deep_search(optional_params) else QUICK_SEARCH_PATH
if not base.endswith(path):
base = f"{base}{path}"
base: Final = ensure_api_route(api_base or get_secret_str("APISERPENT_API_BASE") or APISERPENT_BASE, path)
if data and isinstance(data, dict) and APISERPENT_PARAMS_KEY in data:
query_string: Final = urlencode(data[APISERPENT_PARAMS_KEY], doseq=True)

4
litellm/llms/azure/audio_transcription/transformation.py
View file

@ -13,6 +13,7 @@ import httpx
from pydantic import ConfigDict, TypeAdapter
from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.audio_transcription.transformation import (
AudioTranscriptionRequestData,
BaseAudioTranscriptionConfig,
@ -110,7 +111,7 @@ class AzureSpeechAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
"language": optional_params.get("language", self.DEFAULT_LANGUAGE),
"format": self._get_azure_response_format(optional_params.get("response_format")),
}
return f"{base_url}{self.STT_ENDPOINT_PATH}?{urlencode(query_params)}"
return f"{join_api_base(base_url, self.STT_ENDPOINT_PATH)}?{urlencode(query_params)}"
def transform_audio_transcription_request(
self,
@ -153,7 +154,6 @@ class AzureSpeechAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
)
def _resolve_stt_base_url(self, api_base: str) -> str:
api_base = api_base.rstrip("/")
parsed_url: Final = urlparse(api_base)
hostname: Final = parsed_url.hostname or ""

3
litellm/llms/azure/common_utils.py
View file

@ -18,6 +18,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.caching.caching import DualCache
from litellm.constants import DEFAULT_MAX_RETRIES
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIAsyncHTTPClient, OpenAIHTTPClient
@ -616,7 +617,7 @@ class BaseAzureLLM(BaseOpenAILLM):
v1_params: Final[_AzureV1ClientParams] = {
"api_key": v1_api_key,
"base_url": f"{api_base}/openai/v1/",
"base_url": f"{ensure_api_route(api_base or '', 'openai/v1')}/",
}
if "timeout" in azure_client_params:
v1_params["timeout"] = azure_client_params["timeout"]

4
litellm/llms/azure/realtime/handler.py
View file

@ -10,6 +10,7 @@ from typing import Final, Protocol, cast
from litellm._logging import verbose_proxy_logger
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.types.realtime import RealtimeQueryParams
from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
@ -136,7 +137,8 @@ class AzureOpenAIRealtime(AzureChatCompletion):
query_parts.append(urlencode({"intent": intent}))
qs: Final = "&".join(query_parts)
return f"{api_base}{path}?{qs}" if qs else f"{api_base}{path}"
endpoint_url: Final = join_api_base(api_base, path)
return f"{endpoint_url}?{qs}" if qs else endpoint_url
def construct_url(
self,

5
litellm/llms/azure/realtime/http_transformation.py
View file

@ -3,6 +3,7 @@
from typing import Final
import litellm
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig
from litellm.secret_managers.main import get_secret_str
@ -39,9 +40,9 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
return _azure_realtime_v1_url(self.get_api_base(api_base), "calls")
def get_transcription_session_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
base: Final = self.get_api_base(api_base).rstrip("/")
base: Final = self.get_api_base(api_base)
version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17"
return f"{base}/openai/realtime/transcription_sessions?api-version={version}"
return f"{join_api_base(base, 'openai/realtime/transcription_sessions')}?api-version={version}"
def get_realtime_calls_headers(self, ephemeral_key: str) -> dict[str, str]:
return {

6
litellm/llms/azure/search/transformation.py
View file

@ -35,6 +35,7 @@ from typing import TYPE_CHECKING, Final, Literal
import httpx
from pydantic import ConfigDict, ValidationError
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.azure.common_utils import cached_azure_ad_token_refresh_provider
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.search.transformation import (
@ -340,10 +341,7 @@ class BingGroundingSearchConfig(BaseSearchConfig):
f"{PROJECT_ENDPOINT_ENV} is not set. Set it to your Microsoft Foundry project "
f"endpoint, e.g. https://<account>.services.ai.azure.com/api/projects/<project>."
)
trimmed: Final = resolved_base.rstrip("/")
if trimmed.endswith(_RESPONSES_PATH):
return trimmed
return f"{trimmed}{_RESPONSES_PATH}"
return ensure_api_route(resolved_base, _RESPONSES_PATH)
def transform_search_request(
self,

11
litellm/llms/azure/text_to_speech/transformation.py
View file

@ -11,6 +11,7 @@ from urllib.parse import urlparse
import httpx
import litellm
from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
from litellm.llms.base_llm.text_to_speech.transformation import (
BaseTextToSpeechConfig,
TextToSpeechRequestData,
@ -329,8 +330,6 @@ class AzureAVATextToSpeechConfig(BaseTextToSpeechConfig):
f"https://{{region}}.{self.TTS_SPEECH_DOMAIN}"
)
# Remove trailing slash and parse URL
api_base = api_base.rstrip("/")
parsed_url: Final = urlparse(api_base)
hostname: Final = parsed_url.hostname or ""
@ -339,14 +338,10 @@ class AzureAVATextToSpeechConfig(BaseTextToSpeechConfig):
region: Final = self._extract_region_from_hostname(hostname=hostname, domain=self.COGNITIVE_SERVICES_DOMAIN)
return self._build_tts_url(region=region)
# Check if it's already a TTS endpoint
if self._is_tts_endpoint(hostname=hostname):
if not api_base.endswith(self.TTS_ENDPOINT_PATH):
return f"{api_base}{self.TTS_ENDPOINT_PATH}"
return api_base
return ensure_api_route(api_base, self.TTS_ENDPOINT_PATH)
# Assume it's a custom endpoint, append the path
return f"{api_base}{self.TTS_ENDPOINT_PATH}"
return join_api_base(api_base, self.TTS_ENDPOINT_PATH)
def _is_cognitive_services_endpoint(self, hostname: str) -> bool:
"""Check if hostname is a Cognitive Services endpoint"""

17
litellm/llms/azure_ai/agents/handler.py
View file

@ -30,7 +30,7 @@ from typing_extensions import ReadOnly
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.hidden_params import set_hidden_param
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.azure_ai.agents.transformation import (
AzureAIAgentsConfig,
AzureAIAgentsError,
@ -130,28 +130,29 @@ class AzureAIAgentsHandler:
# See: https://learn.microsoft.com/en-us/azure/ai-foundry/agents/quickstart
# -------------------------------------------------------------------------
def _build_thread_url(self, api_base: str, api_version: str) -> str:
return f"{api_base}/threads?api-version={api_version}"
return f"{join_api_base(api_base, 'threads')}?api-version={api_version}"
def _build_messages_url(self, api_base: str, thread_id: str, api_version: str) -> str:
encoded_thread_id: Final = encode_url_path_segment(thread_id, field_name="thread_id")
return f"{api_base}/threads/{encoded_thread_id}/messages?api-version={api_version}"
return f"{join_api_base(api_base, f'threads/{encoded_thread_id}/messages')}?api-version={api_version}"
def _build_runs_url(self, api_base: str, thread_id: str, api_version: str) -> str:
encoded_thread_id: Final = encode_url_path_segment(thread_id, field_name="thread_id")
return f"{api_base}/threads/{encoded_thread_id}/runs?api-version={api_version}"
return f"{join_api_base(api_base, f'threads/{encoded_thread_id}/runs')}?api-version={api_version}"
def _build_run_status_url(self, api_base: str, thread_id: str, run_id: str, api_version: str) -> str:
encoded_thread_id: Final = encode_url_path_segment(thread_id, field_name="thread_id")
encoded_run_id: Final = encode_url_path_segment(run_id, field_name="run_id")
return f"{api_base}/threads/{encoded_thread_id}/runs/{encoded_run_id}?api-version={api_version}"
run_status_route: Final = f"threads/{encoded_thread_id}/runs/{encoded_run_id}"
return f"{join_api_base(api_base, run_status_route)}?api-version={api_version}"
def _build_list_messages_url(self, api_base: str, thread_id: str, api_version: str) -> str:
encoded_thread_id: Final = encode_url_path_segment(thread_id, field_name="thread_id")
return f"{api_base}/threads/{encoded_thread_id}/messages?api-version={api_version}"
return f"{join_api_base(api_base, f'threads/{encoded_thread_id}/messages')}?api-version={api_version}"
def _build_create_thread_and_run_url(self, api_base: str, api_version: str) -> str:
"""URL for the create-thread-and-run endpoint (supports streaming)."""
return f"{api_base}/threads/runs?api-version={api_version}"
return f"{join_api_base(api_base, 'threads/runs')}?api-version={api_version}"
# -------------------------------------------------------------------------
# Response Helpers
@ -281,8 +282,6 @@ class AzureAIAgentsHandler:
api_version: Final = optional_params.get("api_version", self.config.DEFAULT_API_VERSION)
agent_id: Final = self.config.get_agent_id(model, optional_params)
thread_id: Final = optional_params.get("thread_id")
api_base = api_base.rstrip("/")
verbose_logger.debug("Azure AI Agents completion - api_base: %s, agent_id: %s", api_base, agent_id)
return headers, api_version, agent_id, thread_id, api_base

18
litellm/llms/azure_ai/anthropic/count_tokens/transformation.py
View file

@ -5,8 +5,10 @@ Extends the base Anthropic CountTokens transformation with Azure authentication.
"""
from typing import Final
from urllib.parse import urlsplit
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountTokensConfig,
)
@ -76,14 +78,8 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
Returns:
The endpoint URL for the CountTokens API
"""
# Azure AI Anthropic endpoint format:
# https://<resource>.services.ai.azure.com/anthropic/v1/messages/count_tokens
api_base = api_base.rstrip("/")
# Ensure the URL has /anthropic path
if not api_base.endswith("/anthropic"):
if "/anthropic" not in api_base:
api_base = f"{api_base}/anthropic"
# Add the count_tokens path
return f"{api_base}/v1/messages/count_tokens"
base: Final = api_base.rstrip("/")
if base.endswith("/v1/messages"):
return join_api_base(base, "count_tokens")
anthropic_base: Final = base if "/anthropic" in urlsplit(base).path else join_api_base(base, "anthropic")
return join_api_base(anthropic_base, "v1/messages/count_tokens")

21
litellm/llms/azure_ai/anthropic/messages_transformation.py
View file

@ -9,6 +9,7 @@ from litellm.llms.anthropic.pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.azure_ai.anthropic.transformation import azure_anthropic_messages_url
from litellm.types.router import GenericLiteLLMParams
@ -96,25 +97,7 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
"Expected format: https://<resource-name>.services.ai.azure.com/anthropic"
)
# Ensure the URL ends with /v1/messages
api_base = api_base.rstrip("/")
if api_base.endswith("/v1/messages") or api_base.endswith("/anthropic/v1/messages"):
# Already correct
pass
else:
# Check if /anthropic is already in the path
if "/anthropic" in api_base:
# /anthropic exists, ensure we end with /anthropic/v1/messages
# Extract the base URL up to and including /anthropic
parts: Final = api_base.split("/anthropic", 1)
api_base = parts[0] + "/anthropic"
else:
# /anthropic not in path, add it
api_base = api_base + "/anthropic"
# Add /v1/messages
api_base = api_base + "/v1/messages"
return api_base
return azure_anthropic_messages_url(api_base)
def _remove_scope_from_cache_control(self, anthropic_messages_request: dict) -> None:
"""

11
litellm/llms/azure_ai/anthropic/transformation.py
View file

@ -4,12 +4,23 @@ Azure Anthropic transformation config - extends AnthropicConfig with Azure authe
from typing import Final
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
def azure_anthropic_messages_url(api_base: str) -> str:
base: Final = api_base.rstrip("/")
if base.endswith("/v1/messages"):
return base
path_start: Final = base.find("/", base.find("://") + len("://")) if "://" in base else 0
anthropic_start: Final = base.find("/anthropic", path_start) if path_start >= 0 else -1
anthropic_root: Final = base[:anthropic_start] if anthropic_start >= 0 else base
return join_api_base(anthropic_root, "anthropic/v1/messages")
def _promote_extra_body_to_optional_params(optional_params: dict) -> None:
"""Promote anthropic-native passthrough keys out of ``extra_body``.

10
litellm/llms/azure_ai/embed/handler.py
View file

@ -5,6 +5,7 @@ from urllib.parse import urlsplit, urlunsplit
from openai import OpenAI
import litellm
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.azure_ai.common_utils import is_foundry_model_inference_base
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -74,13 +75,18 @@ class AzureAIEmbedding(OpenAIChatCompletion):
api_base: str | None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
) -> EmbeddingResponse:
if api_base is None:
raise ValueError(
"api_base is None. Please set AZURE_AI_API_BASE or dynamically via `api_base` param, "
"to make the request."
)
if client is None or not isinstance(client, AsyncHTTPHandler):
client = get_async_httpx_client(
llm_provider=litellm.LlmProviders.AZURE_AI,
params={"timeout": timeout},
)
url: Final = f"{api_base}/images/embeddings"
url: Final = join_api_base(api_base, "images/embeddings")
response: Final = await client.post(
url=url,
@ -123,7 +129,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
if client is None or not isinstance(client, HTTPHandler):
client = HTTPHandler(timeout=timeout, concurrent_limit=1)
url: Final = f"{api_base}/images/embeddings"
url: Final = join_api_base(api_base, "images/embeddings")
response: Final = client.post(
url=url,

4
litellm/llms/azure_ai/image_generation/flux_transformation.py
View file

@ -3,6 +3,7 @@ from types import MappingProxyType
from typing import Final
from litellm.exceptions import BadRequestError, UnsupportedParamsError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.openai.image_generation import GPTImageGenerationConfig
from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
@ -52,7 +53,8 @@ class AzureFoundryFluxImageGenerationConfig(GPTImageGenerationConfig):
return f"{api_base}?api-version={api_version}"
provider_model_path: Final = AzureFoundryFluxImageGenerationConfig.get_flux2_provider_model_path(model)
return f"{api_base}/providers/blackforestlabs/v1/{provider_model_path}?api-version={api_version}"
flux_route: Final = f"providers/blackforestlabs/v1/{provider_model_path}"
return f"{join_api_base(api_base, flux_route)}?api-version={api_version}"
@staticmethod
def is_flux2_model(model: str) -> bool:

9
litellm/llms/azure_ai/image_generation/mai_transformation.py
View file

@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Final
import httpx
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
@ -41,9 +42,9 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
if "/mai/" in path:
prefix, _, _ = path.partition("/images/")
path = f"{prefix}/images/generations"
path = join_api_base(prefix, "images/generations")
else:
path = f"{path}/mai/v1/images/generations"
path = join_api_base(path, "mai/v1/images/generations")
if separator:
return f"{path}?{query}"
@ -63,9 +64,9 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
if "/mai/" in path:
prefix, _, _ = path.partition("/images/")
path = f"{prefix}/images/edits"
path = join_api_base(prefix, "images/edits")
else:
path = f"{path}/mai/v1/images/edits"
path = join_api_base(path, "mai/v1/images/edits")
if separator:
return f"{path}?{query}"

4
litellm/llms/azure_ai/vector_stores/transformation.py
View file

@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Final
import httpx
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.base_llm.vector_store.transformation import (
BaseQueryEmbeddingVectorStoreConfig,
@ -172,7 +173,8 @@ class AzureAIVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig, BaseAzureLLM
litellm_logging_obj.model_call_details["input"] = query_text
litellm_logging_obj.model_call_details["embedding_model"] = litellm_params.get("litellm_embedding_model")
litellm_logging_obj.model_call_details["top_k"] = top_k
return f"{api_base}/indexes/{vector_store_id}/docs/search?api-version=2024-07-01", {
search_route: Final = f"indexes/{vector_store_id}/docs/search"
return f"{join_api_base(api_base, search_route)}?api-version=2024-07-01", {
"search": "*",
"vectorQueries": [{"vector": query_vector, "fields": vector_field, "kind": "vector", "k": top_k}],
"select": "id,content",

3
litellm/llms/base_llm/decisions/transformation.py
View file

@ -12,6 +12,7 @@ from pydantic_core import ErrorDetails
from typing_extensions import assert_never
import litellm
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
from litellm.types.decisions import (
@ -582,7 +583,7 @@ class BaseDecisionsConfig(ABC):
)
def get_complete_url(self, api_base: str, model: str) -> str:
return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}"
return join_api_base(api_base.rstrip("/").removesuffix("/v1"), self.path)
def signs_with_aws(self, api_base: str) -> bool:
return False

3
litellm/llms/base_llm/evals/transformation.py
View file

@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any
import httpx
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.openai_evals import (
CancelEvalResponse,
@ -79,7 +80,7 @@ class BaseEvalsAPIConfig(ABC):
"""
if api_base is None:
raise ValueError("api_base is required")
return f"{api_base}/v1/{endpoint}"
return join_api_base(api_base, f"v1/{endpoint}")
@abstractmethod
def transform_create_eval_request(

3
litellm/llms/base_llm/extraction/__init__.py Normal file
View file

@ -0,0 +1,3 @@
from .transformation import BaseExtractionConfig
__all__ = ["BaseExtractionConfig"]

86
litellm/llms/base_llm/extraction/transformation.py Normal file
View file

@ -0,0 +1,86 @@
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import Final
import httpx
from pydantic import TypeAdapter, ValidationError
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
from litellm.types.extraction import ExtractionRequestBody, ExtractionResponse
_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object)
_RESPONSE_ADAPTER: Final[TypeAdapter[ExtractionResponse]] = TypeAdapter(ExtractionResponse)
_RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"})
class BaseExtractionConfig(ABC):
api_key_env: tuple[str, ...] = ()
api_base_env: tuple[str, ...] = ()
def get_default_api_base(self) -> str | None:
return None
def missing_api_base_message(self, custom_llm_provider: str) -> str:
return f"api_base is required for Extraction provider '{custom_llm_provider}'"
def resolve_api_base(self, api_base: str | None) -> str | None:
return api_base or self._first_secret(self.api_base_env) or self.get_default_api_base()
def resolve_api_key(self, api_key: str | None) -> str | None:
return api_key or self._first_secret(self.api_key_env)
@staticmethod
def _first_secret(names: tuple[str, ...]) -> str | None:
return next((value for value in (get_secret_str(name) for name in names) if value), None)
def canonical_model(self, model: str) -> str:
return model
def request_model(self, model: str) -> str:
return model
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
api_key: str | None,
litellm_params: Mapping[str, object],
) -> Mapping[str, str]:
return {
**{name: value for name, value in headers.items() if name.lower() not in _RESERVED_HEADERS},
**({"Authorization": f"Bearer {api_key}"} if api_key is not None else {}),
"Content-Type": "application/json",
}
@abstractmethod
def get_complete_url(self, api_base: str, model: str) -> str: ...
def transform_extraction_request(self, model: str, request: ExtractionRequestBody) -> Mapping[str, object]:
return {**request.model_dump(mode="json", exclude_unset=True), "model": self.request_model(model)}
def unwrap_response(self, payload: object) -> object:
return payload
def transform_extraction_response(
self,
model: str,
custom_llm_provider: str,
raw_response: httpx.Response,
) -> ExtractionResponse:
payload: Final[object] = _PAYLOAD_ADAPTER.validate_json(raw_response.content)
try:
return _RESPONSE_ADAPTER.validate_python(self.unwrap_response(payload))
except ValidationError as error:
raise BaseLLMException(
status_code=500,
message=f"Extraction provider '{custom_llm_provider}' returned an unexpected response: {error}",
) from error
def get_error_class(
self,
error_message: str,
status_code: int,
headers: Mapping[str, str] | httpx.Headers,
) -> BaseLLMException:
return BaseLLMException(status_code=status_code, message=error_message, headers=httpx.Headers(headers))

22
litellm/llms/base_llm/harness/transformation.py
View file

@ -68,6 +68,16 @@ class HarnessTurnRequest:
cwd: str | None = None
@dataclass(frozen=True)
class HarnessSessionCopy:
"""A saved session under fresh ids: written to `path` in the sandbox, then loaded by `import_command`."""
session_id: str
path: str
data: bytes
import_command: Sequence[str]
@dataclass(frozen=True)
class HarnessTurnResponse:
"""What the runtime produced for one turn, after the process exited."""
@ -128,6 +138,18 @@ class BaseCLIHarnessConfig(BaseHarnessConfig[OptionsT], Generic[OptionsT, Stream
) -> HarnessTurnRequest:
"""argv / env / stdin for one turn. native_session_id is set after the first turn."""
def transform_resume_export_command(self, native_session_id: str) -> Sequence[str] | None:
"""Command that prints a saved session before resuming one this handler did not run.
None resumes the saved session as is. A non-zero exit or a timeout fails the turn, and the next turn
tries again.
"""
return None
def transform_resume_copy(self, exported: str, private_dir: str, workdir: str) -> HarnessSessionCopy | None:
"""The copy to import and continue, from the export command's stdout, or None to continue the original."""
return None
@abstractmethod
def create_stream_state(self) -> StreamStateT:
"""Fresh per-turn parser state."""

8
litellm/llms/base_llm/realtime/http_transformation.py
View file

@ -7,10 +7,11 @@ These are HTTP (not WebSocket) endpoints used by the WebRTC flow:
"""
from abc import ABC, abstractmethod
from typing import Final
import httpx
from litellm.litellm_core_utils.url_utils import join_api_base
class BaseRealtimeHTTPConfig(ABC):
"""
@ -61,7 +62,7 @@ class BaseRealtimeHTTPConfig(ABC):
"""Return the full URL for POST /realtime/transcription_sessions."""
base = (api_base or "").rstrip("/")
base = base.removesuffix("/v1")
return f"{base}/v1/realtime/transcription_sessions"
return join_api_base(base, "v1/realtime/transcription_sessions")
@abstractmethod
def validate_environment(
@ -83,8 +84,7 @@ class BaseRealtimeHTTPConfig(ABC):
def get_realtime_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
"""Return the full URL for POST /realtime/calls (SDP exchange)."""
base: Final = (api_base or "").rstrip("/")
return f"{base}/v1/realtime/calls"
return join_api_base(api_base or "", "v1/realtime/calls")
def get_realtime_calls_headers(self, ephemeral_key: str) -> dict:
"""

3
litellm/llms/base_llm/skills/transformation.py
View file

@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any
import httpx
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic_skills import (
CreateSkillRequest,
@ -71,7 +72,7 @@ class BaseSkillsAPIConfig(ABC):
"""
if api_base is None:
raise ValueError("api_base is required")
return f"{api_base}/v1/{endpoint}"
return join_api_base(api_base, f"v1/{endpoint}")
@abstractmethod
def transform_create_skill_request(

26
litellm/llms/bedrock/batches/handler.py
View file

@ -5,7 +5,10 @@ from typing import TYPE_CHECKING, Final, Literal
from openai.types.batch import BatchRequestCounts
from openai.types.batch import Metadata as OpenAIBatchMetadata
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.llms.bedrock.common_utils import (
bedrock_control_plane_base_url,
configured_bedrock_control_plane_endpoint,
)
from litellm.types.llms.bedrock import AwsAuthParams, AwsSessionTag
from litellm.types.utils import LiteLLMBatch
@ -156,10 +159,14 @@ class BedrockBatchesHandler:
aws_session_tags=aws_session_tags,
)
creds: Final = BedrockBatchesConfig().resolve_credentials(auth_params, region)
control_plane_endpoint: Final = configured_bedrock_control_plane_endpoint(
kwargs.get("aws_bedrock_control_plane_endpoint")
)
client: Final = boto3.client(
"bedrock",
region_name=region,
endpoint_url=control_plane_endpoint,
aws_access_key_id=creds.access_key,
aws_secret_access_key=creds.secret_key,
aws_session_token=creds.token,
@ -171,6 +178,7 @@ class BedrockBatchesHandler:
batch_id=batch_id,
aws_region_name=region,
logging_obj=logging_obj,
aws_bedrock_control_plane_endpoint=control_plane_endpoint,
**auth_params.model_dump(),
)
@ -270,7 +278,7 @@ class BedrockBatchesHandler:
batch_id: str,
aws_region_name: str | None = None,
logging_obj=None,
**kwargs,
**kwargs: object,
) -> "LiteLLMBatch":
"""
Handle ``GetModelInvocationJob`` status check for AWS Bedrock bulk batch
@ -294,7 +302,10 @@ class BedrockBatchesHandler:
``aws_session_token``, ``aws_profile_name``,
``aws_role_name``, ``aws_session_name``,
``aws_web_identity_token``, ``aws_sts_endpoint``,
``aws_external_id``, ``aws_session_tags``). Unknown keys are ignored.
``aws_external_id``, ``aws_session_tags``), plus
``aws_bedrock_control_plane_endpoint`` to send the call to a VPC
endpoint or gateway instead of the regional ``bedrock`` host.
Unknown keys are ignored.
Returns:
``LiteLLMBatch`` shaped like an OpenAI Batch resource.
@ -318,10 +329,14 @@ class BedrockBatchesHandler:
from litellm.llms.bedrock.batches.transformation import BedrockBatchesConfig
creds: Final = BedrockBatchesConfig().resolve_credentials(AwsAuthParams.model_validate(kwargs), region)
control_plane_endpoint: Final = configured_bedrock_control_plane_endpoint(
kwargs.get("aws_bedrock_control_plane_endpoint")
)
client: Final = boto3.client(
"bedrock",
region_name=region,
endpoint_url=control_plane_endpoint,
aws_access_key_id=creds.access_key,
aws_secret_access_key=creds.secret_key,
aws_session_token=creds.token,
@ -334,14 +349,13 @@ class BedrockBatchesHandler:
# `GetModelInvocationJob` accepts either form, but only the bare id
# produces a sensible-looking URL in logs.
url_path_id: Final = _extract_job_id_from_arn(batch_id) or batch_id
control_plane_base: Final = bedrock_control_plane_base_url(control_plane_endpoint, region)
logging_obj.pre_call(
input=batch_id,
api_key="",
additional_args={
"complete_input_dict": {"jobIdentifier": batch_id},
"api_base": (
f"https://bedrock.{region}.{get_aws_dns_suffix(region)}/model-invocation-job/{url_path_id}"
),
"api_base": f"{control_plane_base}/model-invocation-job/{url_path_id}",
},
)

22
litellm/llms/bedrock/batches/transformation.py
View file

@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Final, Literal, cast
from httpx import Headers, Response
from pydantic import TypeAdapter, ValidationError
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix, is_bedrock_arn
from litellm.litellm_core_utils.aws_partition import is_bedrock_arn
from litellm.litellm_core_utils.cloud_storage_security import (
BEDROCK_MANAGED_S3_BATCH_PREFIX,
)
@ -32,6 +32,7 @@ from litellm.types.utils import LiteLLMBatch, LlmProviders, Usage
from ..base_aws_llm import BaseAWSLLM
from ..common_utils import (
CommonBatchFilesUtils,
bedrock_control_plane_base_url,
merge_bedrock_aws_request_params,
resolve_s3_bucket_owner,
resolve_s3_encryption_key_id,
@ -177,11 +178,10 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
request_params: Final = merge_bedrock_aws_request_params(litellm_params, optional_params)
aws_region_name: Final = self._get_aws_region_name(request_params, model)
# Bedrock model invocation job endpoint
# Format: https://bedrock.{region}.{partition dns suffix}/model-invocation-job
bedrock_endpoint: Final = (
f"https://bedrock.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}/model-invocation-job"
control_plane_base: Final = bedrock_control_plane_base_url(
request_params.get("aws_bedrock_control_plane_endpoint"), aws_region_name
)
bedrock_endpoint: Final = f"{control_plane_base}/model-invocation-job"
return bedrock_endpoint
@ -279,9 +279,10 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
# Use common utility for AWS signing
request_params: Final = merge_bedrock_aws_request_params(litellm_params, optional_params)
aws_region_name: Final = self._get_aws_region_name(request_params, model)
endpoint_url: Final = (
f"https://bedrock.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}/model-invocation-job"
control_plane_base: Final = bedrock_control_plane_base_url(
request_params.get("aws_bedrock_control_plane_endpoint"), aws_region_name
)
endpoint_url: Final = f"{control_plane_base}/model-invocation-job"
signed_headers, signed_data = self.common_utils.sign_aws_request(
service_name="bedrock",
data=bedrock_request,
@ -431,12 +432,13 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
import urllib.parse as _ul
encoded_arn: Final = _ul.quote(batch_id, safe="")
endpoint_url: Final = (
f"https://bedrock.{region}.{get_aws_dns_suffix(region)}/model-invocation-job/{encoded_arn}"
request_params: Final = merge_bedrock_aws_request_params(litellm_params, optional_params)
control_plane_base: Final = bedrock_control_plane_base_url(
request_params.get("aws_bedrock_control_plane_endpoint"), region
)
endpoint_url: Final = f"{control_plane_base}/model-invocation-job/{encoded_arn}"
# Use common utility for AWS signing
request_params: Final = merge_bedrock_aws_request_params(litellm_params, optional_params)
signed_headers, _ = self.common_utils.sign_aws_request(
service_name="bedrock",
data={}, # GET request has no body

5
litellm/llms/bedrock/chat/agentcore/transformation.py
View file

@ -18,6 +18,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
)
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.a2a.common_utils import extract_text_from_a2a_response
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
@ -98,14 +99,14 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
# Build the base endpoint URL for AgentCore
# Note: We don't use get_runtime_endpoint as AgentCore has its own endpoint structure
if aws_bedrock_runtime_endpoint:
base_url = aws_bedrock_runtime_endpoint
base_url = f"{aws_bedrock_runtime_endpoint}"
else:
base_url = f"https://bedrock-agentcore.{region}.{get_aws_dns_suffix(region)}"
# Based on boto3 client.invoke_agent_runtime, the path is:
# /runtimes/{URL-ENCODED-ARN}/invocations?qualifier=<value>
encoded_arn: Final = quote(agent_runtime_arn, safe="")
endpoint_url = f"{base_url}/runtimes/{encoded_arn}/invocations"
endpoint_url = join_api_base(base_url, f"runtimes/{encoded_arn}/invocations")
# Add qualifier as query parameter if provided
if "qualifier" in optional_params:

10
litellm/llms/bedrock/chat/chat_completions/transformation.py
View file

@ -33,6 +33,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
inline_remote_image_urls,
inline_remote_media,
)
from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token
from litellm.llms.bedrock.common_utils import (
@ -371,12 +372,9 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
),
aws_region_name=aws_region_name,
)
base: Final = proxy_endpoint_url.rstrip("/")
if base.endswith("/openai/v1/chat/completions"):
return base
if base.endswith("/openai/v1"):
return f"{base}/chat/completions"
return f"{base}/openai/v1/chat/completions"
if proxy_endpoint_url.rstrip("/").endswith("/openai/v1"):
return join_api_base(proxy_endpoint_url, "chat/completions")
return ensure_api_route(proxy_endpoint_url, "openai/v1/chat/completions")
def _params_with_region_from_path(
self, optional_params: dict, model: str | None

9
litellm/llms/bedrock/chat/converse_handler.py
View file

@ -10,6 +10,7 @@ from litellm.anthropic_beta_headers_manager import (
)
from litellm.litellm_core_utils.get_litellm_params import stored_control_options
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm import base_model_iterator
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -363,11 +364,11 @@ class BedrockConverseLLM(BaseAWSLLM):
aws_region_name=aws_region_name,
)
if (stream is not None and stream is True) and not fake_stream:
endpoint_url = f"{endpoint_url}/model/{modelId}/converse-stream"
proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/converse-stream"
endpoint_url = join_api_base(endpoint_url, f"model/{modelId}/converse-stream")
proxy_endpoint_url = join_api_base(proxy_endpoint_url, f"model/{modelId}/converse-stream")
else:
endpoint_url = f"{endpoint_url}/model/{modelId}/converse"
proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/converse"
endpoint_url = join_api_base(endpoint_url, f"model/{modelId}/converse")
proxy_endpoint_url = join_api_base(proxy_endpoint_url, f"model/{modelId}/converse")
## COMPLETION CALL
headers = {"Content-Type": "application/json"}

9
litellm/llms/bedrock/chat/invoke_agent/transformation.py
View file

@ -15,7 +15,7 @@ from litellm._uuid import uuid
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
)
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import BedrockError
@ -101,9 +101,10 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
encoded_agent_alias_id: Final = encode_url_path_segment(agent_alias_id, field_name="agent_alias_id")
encoded_session_id: Final = encode_url_path_segment(session_id, field_name="session_id")
endpoint_url = f"{endpoint_url}/agents/{encoded_agent_id}/agentAliases/{encoded_agent_alias_id}/sessions/{encoded_session_id}/text"
return endpoint_url
return join_api_base(
endpoint_url,
f"agents/{encoded_agent_id}/agentAliases/{encoded_agent_alias_id}/sessions/{encoded_session_id}/text",
)
def sign_request(
self,

10
litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py
View file

@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, Final
import httpx
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.request_metadata import (
@ -97,13 +98,8 @@ class AmazonBedrockOpenAIConfig(OpenAIGPTConfig, BaseAWSLLM):
# Encode model ID for ARNs (e.g., :imported-model/ -> :imported-model%2F)
model_id = CommonUtils.encode_bedrock_runtime_modelid_arn(model_id)
# Build the invoke URL
if stream:
endpoint_url = f"{endpoint_url}/model/{model_id}/invoke-with-response-stream"
else:
endpoint_url = f"{endpoint_url}/model/{model_id}/invoke"
return endpoint_url
invoke_route: Final = "invoke-with-response-stream" if stream else "invoke"
return join_api_base(endpoint_url, f"model/{model_id}/{invoke_route}")
def sign_request(
self,

18
litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py
View file

@ -18,6 +18,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
prompt_factory,
render_custom_prompt_template,
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.bedrock.chat.invoke_handler import make_call, make_sync_call
from litellm.llms.bedrock.common_utils import BedrockError
@ -60,7 +61,7 @@ def _response_header(headers: Mapping[str, str], name: str) -> str | None:
return headers.get(name, None)
def _bedrock_invoke_guardrail_headers(raw_guardrail_config: object) -> "dict[str, str]":
def bedrock_invoke_guardrail_headers(raw_guardrail_config: object) -> "dict[str, str]":
try:
guardrail_config: Final = _GUARDRAIL_CONFIG_VALIDATOR.validate_python(raw_guardrail_config)
except ValidationError as e:
@ -136,20 +137,13 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
aws_bedrock_runtime_endpoint: Final = optional_params.get(
"aws_bedrock_runtime_endpoint", None
) # https://bedrock-runtime.{region_name}.amazonaws.com
endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(
endpoint_url, _ = self.get_runtime_endpoint(
api_base=api_base,
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
aws_region_name=self._get_aws_region_name(optional_params=optional_params, model=model),
)
if (stream is not None and stream is True) and provider != "ai21":
endpoint_url = f"{endpoint_url}/model/{modelId}/invoke-with-response-stream"
proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke-with-response-stream"
else:
endpoint_url = f"{endpoint_url}/model/{modelId}/invoke"
proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke"
return endpoint_url
invoke_route: Final = "invoke-with-response-stream" if stream is True and provider != "ai21" else "invoke"
return join_api_base(endpoint_url, f"model/{modelId}/{invoke_route}")
def sign_request(
self,
@ -440,7 +434,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
guardrail_headers: Final = (
()
if raw_guardrail_config is None
else tuple(_bedrock_invoke_guardrail_headers(raw_guardrail_config).items())
else tuple(bedrock_invoke_guardrail_headers(raw_guardrail_config).items())
)
owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params)
return merge_bedrock_invoke_headers(headers, guardrail_headers, metadata_headers, owned_names)

5
litellm/llms/bedrock/claude_platform/common_utils.py
View file

@ -5,6 +5,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.secret_managers.main import get_secret_str
@ -140,9 +141,7 @@ class BedrockClaudePlatformMixin(BaseAWSLLM):
if api_base is None:
aws_region_name: Final = self._get_required_aws_region_name(optional_params)
api_base = f"https://{CLAUDE_PLATFORM_SERVICE_NAME}.{aws_region_name}.api.aws"
if not api_base.endswith("/v1/messages"):
api_base = f"{api_base.rstrip('/')}/v1/messages"
return api_base
return ensure_api_route(api_base, "v1/messages")
def sign_request(
self,

25
litellm/llms/bedrock/common_utils.py
View file

@ -31,6 +31,7 @@ from litellm import verbose_logger
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -138,6 +139,25 @@ def merge_bedrock_aws_request_params(
return request_params
def configured_bedrock_control_plane_endpoint(endpoint: object) -> str | None:
"""An ``aws_bedrock_control_plane_endpoint`` value without a trailing slash, or None when unset."""
if isinstance(endpoint, str) and endpoint.strip():
return endpoint.strip().rstrip("/")
return None
def bedrock_control_plane_base_url(endpoint: object, aws_region_name: str) -> str:
"""Base URL for Bedrock control-plane calls such as model invocation jobs.
A configured ``aws_bedrock_control_plane_endpoint`` (a VPC endpoint or a compatible gateway)
wins over the regional ``bedrock`` host. ``api_base`` is deliberately not used here, since on a
Bedrock deployment it points at the runtime host, which does not serve these APIs.
"""
return configured_bedrock_control_plane_endpoint(endpoint) or (
f"https://bedrock.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}"
)
def s3_static_key_pair(params: Mapping[str, object]) -> tuple[str, str] | None:
"""The s3_access_key_id / s3_secret_access_key pair when both are set, otherwise None."""
s3_access_key_id: Final = params.get("s3_access_key_id")
@ -1286,10 +1306,7 @@ def build_mantle_messages_url(
"""
override: Final = api_base or aws_bedrock_runtime_endpoint or _mantle_api_base_from_env()
if override:
base: Final = override.rstrip("/")
if base.endswith(MANTLE_MESSAGES_PATH):
return base
return f"{base}{MANTLE_MESSAGES_PATH}"
return ensure_api_route(override, MANTLE_MESSAGES_PATH)
return f"https://bedrock-mantle.{region}.api.aws{MANTLE_MESSAGES_PATH}"

3
litellm/llms/bedrock/count_tokens/mantle_handler.py
View file

@ -7,6 +7,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, AnthropicCountTokensConfig
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing
from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url
@ -43,7 +44,7 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM):
messages_url: Final = build_mantle_messages_url(
api_base=None, aws_bedrock_runtime_endpoint=None, region=aws_region_name
)
return f"{messages_url}{MANTLE_COUNT_TOKENS_SUFFIX}"
return join_api_base(messages_url, MANTLE_COUNT_TOKENS_SUFFIX)
async def handle_count_tokens_request(
self,

5
litellm/llms/bedrock/count_tokens/transformation.py
View file

@ -11,6 +11,7 @@ from typing import Final, Literal
from pydantic import JsonValue
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import get_bedrock_base_model
@ -236,9 +237,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
aws_region_name=aws_region_name,
)
endpoint: Final = f"{base_url}/model/{encoded_model_id}/count-tokens"
return endpoint
return join_api_base(base_url, f"model/{encoded_model_id}/count-tokens")
def transform_bedrock_response_to_anthropic(
self, bedrock_response: Mapping[str, JsonValue]

7
litellm/llms/bedrock/embed/embedding.py
View file

@ -12,6 +12,7 @@ import httpx
import litellm
from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.cohere.embed.handler import embedding as cohere_embedding
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -512,9 +513,9 @@ class BedrockEmbedding(BaseAWSLLM):
aws_region_name=aws_region_name,
)
if has_async_invoke:
endpoint_url = f"{endpoint_url}/async-invoke"
endpoint_url = join_api_base(endpoint_url, "async-invoke")
else:
endpoint_url = f"{endpoint_url}/model/{modelId}/invoke"
endpoint_url = join_api_base(endpoint_url, f"model/{modelId}/invoke")
if batch_data is not None:
if aembedding:
@ -611,7 +612,7 @@ class BedrockEmbedding(BaseAWSLLM):
# Encode the ARN for use in URL path
encoded_arn: Final = quote(invocation_arn, safe="")
status_url: Final = f"{endpoint_url.rstrip('/')}/async-invoke/{encoded_arn}"
status_url: Final = join_api_base(endpoint_url, f"async-invoke/{encoded_arn}")
# Prepare headers for GET request
headers: Final = {"Content-Type": "application/json"}

3
litellm/llms/bedrock/image_edit/handler.py
View file

@ -14,6 +14,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.bedrock.image_edit.amazon_nova_canvas_image_edit_transformation import (
BedrockAmazonNovaCanvasImageEditConfig,
)
@ -226,7 +227,7 @@ class BedrockImageEdit(BaseAWSLLM):
aws_bedrock_runtime_endpoint=boto3_credentials_info.aws_bedrock_runtime_endpoint,
aws_region_name=boto3_credentials_info.aws_region_name,
)
proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke"
proxy_endpoint_url = join_api_base(proxy_endpoint_url, f"model/{modelId}/invoke")
data: Final = self._get_request_body(
model=model,
image=image,

3
litellm/llms/bedrock/image_generation/image_handler.py
View file

@ -8,6 +8,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.bedrock.image_generation.amazon_nova_canvas_transformation import (
AmazonNovaCanvasConfig,
)
@ -248,7 +249,7 @@ class BedrockImageGeneration(BaseAWSLLM):
aws_bedrock_runtime_endpoint=boto3_credentials_info.aws_bedrock_runtime_endpoint,
aws_region_name=boto3_credentials_info.aws_region_name,
)
proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke"
proxy_endpoint_url = join_api_base(proxy_endpoint_url, f"model/{modelId}/invoke")
data: Final = self._get_request_body(
model=model,
prompt=prompt,

38
litellm/llms/bedrock/passthrough/transformation.py
View file

@ -14,6 +14,9 @@ from litellm.constants import AWS_ERROR_TYPE_HEADER
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig, PassthroughStreamCollector
from litellm.llms.bedrock.chat.converse_to_openai import as_json_array, as_json_object
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
bedrock_invoke_guardrail_headers,
)
from litellm.types.llms.openai import ChatCompletionSystemMessage, ChatCompletionUserMessage
from litellm.types.utils import ModelResponseStream
@ -42,6 +45,9 @@ if TYPE_CHECKING:
BEDROCK_STREAMING_ACTIONS: Final = frozenset({"invoke-with-response-stream", "converse-stream"})
_BEDROCK_CONVERSE_ACTIONS: Final = frozenset({"converse", "converse-stream"})
_BEDROCK_INVOKE_ACTIONS: Final = frozenset({"invoke", "invoke-with-response-stream"})
_BEDROCK_INVOKE_GUARDRAIL_HEADERS: Final = frozenset(
{"x-amzn-bedrock-guardrailidentifier", "x-amzn-bedrock-guardrailversion", "x-amzn-bedrock-trace"}
)
def _bedrock_endpoint_action(endpoint: str) -> str:
@ -123,6 +129,31 @@ def _with_invoke_request_metadata(
return merge_bedrock_invoke_headers(dict(headers), (), metadata_headers, owned_names), body
def _converse_guardrail_config(action: str, guardrail_config: object) -> object:
fields: Final = as_json_object(guardrail_config)
if action == "converse-stream" or fields is None:
return guardrail_config
return {key: value for key, value in fields.items() if key != "streamProcessingMode"}
def _with_deployment_guardrail(
action: str,
headers: Mapping[str, str],
body: Mapping[str, object],
guardrail_config: object,
) -> tuple[Mapping[str, str], Mapping[str, object]]:
if guardrail_config is None:
return headers, body
if action in _BEDROCK_CONVERSE_ACTIONS:
return headers, {**body, "guardrailConfig": _converse_guardrail_config(action, guardrail_config)}
if action in _BEDROCK_INVOKE_ACTIONS:
guardrail_headers: Final = tuple(bedrock_invoke_guardrail_headers(guardrail_config).items())
return merge_bedrock_invoke_headers(
dict(headers), (), guardrail_headers, _BEDROCK_INVOKE_GUARDRAIL_HEADERS
), body
return headers, body
def _is_cache_point(entry: object) -> bool:
fields: Final = as_json_object(entry)
return fields is not None and "cachePoint" in fields
@ -453,11 +484,14 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
body=_converse_body_to_send(body) if action in _BEDROCK_CONVERSE_ACTIONS else body,
litellm_params=litellm_params,
)
guarded_headers, guarded_body = _with_deployment_guardrail(
action, signed_headers, signed_body, litellm_params.get("guardrailConfig")
)
return self._sign_request(
service_name="bedrock",
headers=dict(signed_headers),
headers=dict(guarded_headers),
optional_params=optional_params,
request_data=dict(signed_body),
request_data=dict(guarded_body),
api_base=api_base,
model=model,
api_key=self.get_bedrock_bearer_token(optional_params),

11
litellm/llms/bedrock/request_metadata.py
View file

@ -256,12 +256,11 @@ def merge_bedrock_invoke_headers(
"""
Merge the ``X-Amzn-*`` headers the Invoke paths derive from params.
``caller_owned`` (the guardrail headers) defers to a header the caller already set, which is
the long-standing behaviour for those. ``proxy_owned_names`` are dropped from the caller's
headers unconditionally and re-supplied only from ``proxy_owned``, because those names carry
proxy-authenticated identity into an AWS billing record that the caller must not be able to
write. Names are compared case-insensitively so a caller cannot leave a second spelling in
the dict and let the transport pick the winner.
``caller_owned`` defers to a header the caller already set. ``proxy_owned_names`` are dropped
from the caller's headers unconditionally and re-supplied only from ``proxy_owned``, for names
the caller must not be able to write, such as proxy-authenticated identity headed for an AWS
billing record or a deployment's guardrail. Names are compared case-insensitively so a caller
cannot leave a second spelling in the dict and let the transport pick the winner.
"""
if not caller_owned and not proxy_owned and not proxy_owned_names:
return headers

3
litellm/llms/bedrock/rerank/handler.py
View file

@ -6,6 +6,7 @@ from pydantic import ConfigDict, TypeAdapter
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -160,7 +161,7 @@ class BedrockRerankHandler(BaseAWSLLM):
aws_region_name=boto3_credentials_info.aws_region_name,
)
proxy_endpoint_url = proxy_endpoint_url.replace("bedrock-runtime", "bedrock-agent-runtime")
proxy_endpoint_url = f"{proxy_endpoint_url}/rerank"
proxy_endpoint_url = join_api_base(proxy_endpoint_url, "rerank")
body: Final = json.dumps(data).encode("utf-8")
headers = {"Content-Type": "application/json"}

6
litellm/llms/bedrock/responses/transformation.py
View file

@ -46,6 +46,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_convert_url_to_base64,
convert_url_to_base64,
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
@ -223,9 +224,10 @@ class BedrockOpenAIResponsesConfig(BaseAWSLLM, OpenAIResponsesAPIConfig):
litellm_params: dict, # mutable-ok: signature fixed by the BaseResponsesAPIConfig override contract
) -> str:
region: Final = self._get_aws_region_name(optional_params=litellm_params, model=None)
configured_endpoint: Final = litellm_params.get("aws_bedrock_runtime_endpoint")
override: Final = (
api_base
or litellm_params.get("aws_bedrock_runtime_endpoint")
or (f"{configured_endpoint}" if configured_endpoint else None)
or get_secret_str("AWS_BEDROCK_RUNTIME_ENDPOINT")
)
# Partition-aware: bedrock-runtime is amazonaws.com.cn in China, and other
@ -237,7 +239,7 @@ class BedrockOpenAIResponsesConfig(BaseAWSLLM, OpenAIResponsesAPIConfig):
(host[: -len(suffix)] for suffix in BEDROCK_RUNTIME_OPENAI_BASE_SUFFIXES if host.endswith(suffix)),
host,
)
return f"{base}{BEDROCK_RUNTIME_OPENAI_RESPONSES_PATH}"
return join_api_base(base, BEDROCK_RUNTIME_OPENAI_RESPONSES_PATH)
def supports_native_file_search(self) -> bool:
return False

6
litellm/llms/bedrock/vector_stores/transformation.py
View file

@ -6,7 +6,7 @@ from urllib.parse import urlparse
import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import BedrockError
@ -196,7 +196,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
aws_region_name=self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=aws_region_name),
endpoint_type="agent",
)
return f"{endpoint_url}/knowledgebases"
return join_api_base(endpoint_url, "knowledgebases")
def transform_search_vector_store_request(
self,
@ -212,7 +212,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
query = " ".join(query)
encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id")
url: Final = f"{api_base}/{encoded_vector_store_id}/retrieve"
url: Final = join_api_base(api_base, f"{encoded_vector_store_id}/retrieve")
request_body: Final[dict[str, object]] = {
"retrievalQuery": BedrockKBRetrievalQuery(text=query),

3
litellm/llms/bedrock_mantle/messages/transformation.py
View file

@ -2,6 +2,7 @@ from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import MANTLE_MESSAGES_PATH
from litellm.llms.bedrock.messages.mantle_transformation import AmazonMantleMessagesConfig
@ -33,7 +34,7 @@ def build_mantle_native_messages_url(api_base: str | None, litellm_params: Mappi
configured,
)
host: Final = f"https://bedrock-mantle.{region}.api.aws" if MANTLE_HOST_RE.match(stripped) else stripped
return f"{host}{MANTLE_MESSAGES_PATH}"
return join_api_base(host, MANTLE_MESSAGES_PATH)
class BedrockMantleAnthropicMessagesConfig(BedrockMantleAuthMixin, AmazonMantleMessagesConfig):

3
litellm/llms/bedrock_mantle/responses/transformation.py
View file

@ -22,6 +22,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items
from litellm.llms.base_llm.responses.transformation import ReasoningSummaryLevel
@ -100,7 +101,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
if MANTLE_HOST_RE.match(base):
base = f"https://bedrock-mantle.{region}.api.aws"
path: Final = "/openai/v1/responses" if self.use_openai_path else "/v1/responses"
return f"{base}{path}"
return join_api_base(base, path)
def validate_environment(self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None) -> dict:
litellm_params = litellm_params or GenericLiteLLMParams()

Some files were not shown because too many files have changed in this diff Show more