chore: merge main into E2E runtime repairs

This commit is contained in:
Yuneng Jiang 2026-11-02 17:37:02 -08:00
commit fbf2b9cf4f
No known key found for this signature in database
480 changed files with 26245 additions and 7774 deletions

View file

@ -128,6 +128,8 @@ fi
sudo iptables -L integration_only -n -v -x > "$results/egress-guard.txt"
awk '$3 == "REJECT" && $1 > 0 { rejected=1 } END { exit !rejected }' "$results/egress-guard.txt"
export UV_OFFLINE=true
if [ "$mode" = parity ]; then
export INTEGRATION_WORKERS=1
else

View file

@ -4,7 +4,7 @@ on:
pull_request:
paths:
- tests/e2e/claude_code/cron_vm/**
- tests/e2e/claude_code/pr_gate_version_resolver.py
- tests/e2e/cli_harness/npm_version_resolver.py
- .github/workflows/compat-matrix-image.yml
workflow_dispatch:
@ -36,7 +36,7 @@ jobs:
whoami
gh --version
uv --version
version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/claude_code/pr_gate_version_resolver.py)"
version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)"
/opt/litellm/tests/e2e/claude_code/cron_vm/install_claude_code.sh "${version}" /tmp/claude-cli
/tmp/claude-cli/claude --version
'

View file

@ -1,30 +0,0 @@
name: Deploy main to gateway-dev
on:
push:
branches: [main]
permissions: {}
concurrency:
group: deploy-dev-dispatch
cancel-in-progress: true
jobs:
dispatch:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2.2.2
id: token
with:
app-id: ${{ secrets.LITELLM_OPS_DISPATCH_APP_ID }}
private-key: ${{ secrets.LITELLM_OPS_DISPATCH_APP_PRIVATE_KEY }}
owner: BerriAI
repositories: litellm-ops
permission-actions: write
- name: Dispatch the litellm-ops dev deploy
env:
GH_TOKEN: ${{ steps.token.outputs.token }}
run: gh api --method POST repos/BerriAI/litellm-ops/actions/workflows/deploy-dev.yml/dispatches -f ref=main

View file

@ -208,7 +208,7 @@ jobs:
exit 0
fi
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/claude_code/pr_gate_version_resolver.py)"
CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)"
tests/e2e/claude_code/cron_vm/install_claude_code.sh "$CLAUDE_VERSION" "$RUNNER_TEMP/claude-cli"
PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q tests/e2e_harness
uv run --no-sync pytest --tb=short -q --noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures \

View file

@ -1,85 +0,0 @@
name: "Weekly Load Anomaly Check"
on:
schedule:
- cron: "0 12 * * 6"
workflow_dispatch:
permissions:
contents: read
jobs:
weekly-load-anomaly:
if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 45
services:
postgres:
image: postgres:16.6
env:
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U llmproxy"
--health-interval 5s
--health-timeout 5s
--health-retries 10
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
LITELLM_MASTER_KEY: sk-weekly-anomaly-check
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Start the proxy
run: |
nohup uv run --no-sync litellm --config tests/e2e/perf/weekly_anomaly_config.yml --port 4000 > proxy.log 2>&1 &
for _ in $(seq 1 90); do
if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
exit 0
fi
sleep 2
done
echo "proxy never became live"
tail -n 100 proxy.log
exit 1
- name: Run the weekly session anomaly test
env:
E2E_WEEKLY_ANOMALY: "1"
run: |
uv run --no-sync pytest tests/e2e/perf/test_weekly_session_anomaly_e2e.py -v --tb=short -rA
- name: Show proxy log on failure
if: failure()
run: tail -n 300 proxy.log

View file

@ -3,9 +3,11 @@
import asyncio
import base64
import copy
import json
import time
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timedelta
from itertools import chain
from types import MappingProxyType
@ -34,7 +36,7 @@ from typing_extensions import ReadOnly, Unpack
import litellm
from litellm import Router, verbose_logger
from litellm._internal_context import with_service_target
from litellm._internal_context import internal_sub_call, post_response_phase, with_service_target
from litellm._uuid import uuid
from litellm.caching.caching import DualCache
from litellm.constants import (
@ -95,6 +97,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio
request_tags_from_metadata,
)
from litellm.repositories.managed_file_repository import ManagedFileRepository
from litellm.responses.streaming_iterator import spawn_logging_task
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccessIssue]
AllMessageValues,
@ -395,6 +398,134 @@ class _RouterFileCallKwargs(TypedDict, total=False):
custom_llm_provider: ReadOnly[str | None]
_MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str])
_FILE_UPLOAD_ONLY_FIELDS: Final = frozenset({"file", "purpose", "expires_after", "extra_body"})
_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id", "_litellm_internal_model_credentials"})
@dataclass(frozen=True, slots=True)
class _ProviderUpload:
model_id: str
provider_file_id: str
upload_call_id: str | None = None
def _provider_uploads(file_object: OpenAIFileObject) -> tuple[_ProviderUpload, ...]:
file_hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic file metadata
dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
)
model_file_id_mapping: Final = _MODEL_FILE_ID_MAPPING.validate_python(
file_hidden_params.get("model_file_id_mapping") or {}
)
upload_call_id: Final = file_hidden_params.get("litellm_call_id")
return tuple(
_ProviderUpload(
model_id=model_id,
provider_file_id=provider_file_id,
upload_call_id=upload_call_id if isinstance(upload_call_id, str) else None,
)
for model_id, provider_file_id in model_file_id_mapping.items()
)
def _keeps_one_upload_per_target(llm_router: Router, model_id: str) -> bool:
credentials: Final = _provider_file_retrieve_credentials(llm_router=llm_router, model_id=model_id)
return (credentials or {}).get("custom_llm_provider") == LlmProviders.LITELLM_PROXY.value
def _is_kept_upload(uploads: Sequence[_ProviderUpload], index: int, per_target_model_ids: frozenset[str]) -> bool:
upload: Final = uploads[index]
other_uploads: Final = uploads[index + 1 :] if upload.model_id in per_target_model_ids else uploads[:index]
return all(other.model_id != upload.model_id for other in other_uploads)
def _upload_from_trailing_id_fields(decoded_unified_file_id: str) -> _ProviderUpload:
trailing_fields: Final = decoded_unified_file_id.rpartition(";llm_output_file_id,")[2]
provider_file_id, _, model_id = trailing_fields.partition(";llm_output_file_model_id,")
return _ProviderUpload(model_id=model_id, provider_file_id=provider_file_id)
def _carried_upload_missing_from(unified_file_id: str, model_file_ids: Mapping[str, str]) -> _ProviderUpload | None:
decoded: Final = _is_base64_encoded_unified_file_id(unified_file_id)
carried: Final = _upload_from_trailing_id_fields(decoded) if decoded else None
if carried is None or model_file_ids.get(carried.model_id) in (None, carried.provider_file_id):
return None
return carried
async def _delete_provider_file(
llm_router: Router,
model_id: str,
provider_file_id: str,
request_data: Mapping[str, object],
) -> None:
credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
delete_data: Final = {
**{k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS},
**(
{"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
if credentials is not None
else {}
),
}
await llm_router.afile_delete(
model=model_id,
file_id=provider_file_id,
**cast(_RouterFileCallKwargs, delete_data),
)
def _with_own_nested_dicts(request_data: Mapping[str, object]) -> MappingProxyType[str, object]:
return MappingProxyType({k: copy.copy(v) if isinstance(v, dict) else v for k, v in request_data.items()})
async def _delete_redundant_upload(
llm_router: Router,
upload: _ProviderUpload,
delete_request: Mapping[str, object],
parent_call_id: str | None,
) -> None:
try:
with internal_sub_call(parent_call_id):
await _delete_provider_file(
llm_router, upload.model_id, upload.provider_file_id, _with_own_nested_dicts(delete_request)
)
except Exception as error:
verbose_logger.warning(
f"Could not delete the redundant upload provider_file_id={sanitize_for_log(upload.provider_file_id)} "
f"at deployment model_id={sanitize_for_log(upload.model_id)}: "
f"{type(error).__name__} {sanitize_for_log(error)}"
)
async def _delete_redundant_uploads(
llm_router: Router,
uploads: Sequence[_ProviderUpload],
delete_request: Mapping[str, object],
) -> None:
await asyncio.gather(
*(_delete_redundant_upload(llm_router, upload, delete_request, upload.upload_call_id) for upload in uploads)
)
def _start_deleting_redundant_uploads(
llm_router: Router,
uploads: Sequence[_ProviderUpload],
create_file_request: CreateFileRequest,
) -> None:
if not uploads:
return
delete_request: Final = MappingProxyType(
{k: v for k, v in create_file_request.items() if k not in _FILE_UPLOAD_ONLY_FIELDS}
)
with post_response_phase():
spawn_logging_task(
asyncio.get_running_loop(),
_delete_redundant_uploads(llm_router, uploads, delete_request),
task_name="managed file repeat upload cleanup",
)
def _managed_database(prisma_client: _ManagedResourcePrismaClient) -> _ManagedResourceDatabase:
return prisma_client.db
@ -2034,16 +2165,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
target_model_names_list=target_model_names_list,
)
## STORE MODEL MAPPINGS IN DB
model_mappings: Dict[str, str] = {}
for file_object in responses:
file_hidden_params = cast( # cast-ok: preserve mapping operations on dynamic file metadata
dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
)
model_file_id_mapping = file_hidden_params.get("model_file_id_mapping")
if model_file_id_mapping and isinstance(model_file_id_mapping, dict):
model_mappings.update(model_file_id_mapping)
uploads: Final = tuple(chain.from_iterable(map(_provider_uploads, responses)))
per_target_model_ids: Final = frozenset(
model_id
for model_id in {upload.model_id for upload in uploads}
if _keeps_one_upload_per_target(llm_router, model_id)
)
kept_uploads: Final = tuple(
upload for index, upload in enumerate(uploads) if _is_kept_upload(uploads, index, per_target_model_ids)
)
model_mappings: Final = {upload.model_id: upload.provider_file_id for upload in kept_uploads}
await self.store_unified_file_id(
file_id=response.id,
@ -2052,6 +2183,15 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_mappings=model_mappings,
user_api_key_dict=user_api_key_dict,
)
_start_deleting_redundant_uploads(
llm_router,
tuple(
upload
for upload in uploads
if upload not in kept_uploads and upload.model_id not in per_target_model_ids
),
create_file_request,
)
# Emit Prometheus metrics for managed file creation
prom_logger = self._get_prometheus_logger()
@ -2692,23 +2832,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
specific_model_file_id_mapping: Final = model_file_id_mapping.get(file_id)
if not specific_model_file_id_mapping:
return
filtered_data: Final = {
k: v for k, v in data.items() if k not in ("model", "file_id", "_litellm_internal_model_credentials")
}
for model_id, model_file_id in specific_model_file_id_mapping.items():
credentials = llm_router.registry().credentials(model_id, form="with_provider")
delete_data = {
**filtered_data,
**(
{"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
if credentials is not None
else {}
),
}
await llm_router.afile_delete(
model=model_id,
file_id=model_file_id,
**cast(_RouterFileCallKwargs, delete_data),
await _delete_provider_file(llm_router, model_id, model_file_id, data)
unrecorded_upload: Final = _carried_upload_missing_from(file_id, specific_model_file_id_mapping)
delete_call_id: Final = data.get("litellm_call_id")
if unrecorded_upload is not None:
await _delete_redundant_upload(
llm_router, unrecorded_upload, data, delete_call_id if isinstance(delete_call_id, str) else None
)
async def afile_content(

View file

@ -186,6 +186,9 @@ pub struct ModelInfoFields {
pub supports_image_size: Option<bool>,
pub supported_audio_formats: Option<Value>,
pub token_count_framing: Option<i64>,
pub token_count_tool_framing: Option<i64>,
pub token_count_responses_framing: Option<i64>,
pub token_count_responses_tool_framing: Option<i64>,
pub token_count_model: Option<Value>,
pub token_count_tokenizer: Option<Value>,
pub vertex_ai_audio_api: Option<Value>,

View file

@ -104,6 +104,8 @@ attributes! {
/// Cloud regions the model is available in ('global' or region ids).
supported_regions: Vec<String>,
thinking_always_on: bool,
/// Whether the model thinks when the request carries no thinking param.
thinking_on_by_default: bool,
token_count_framing: u64 as NumericCount,
token_count_model: TokenCountModel,
token_count_responses_framing: u64 as NumericCount,
@ -301,6 +303,8 @@ mod tests {
#[case::known_mode("mode", json!("chat"), true)]
#[case::unknown_mode("mode", json!("chatt"), false)]
#[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)]
#[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)]
#[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)]
#[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)]
#[case::reasoning_modes("reasoning_modes", json!(["none", "adaptive"]), true)]
#[case::reasoning_context_not_a_list("reasoning_contexts", json!("reasoning"), false)]

View file

@ -2374,6 +2374,7 @@ if TYPE_CHECKING:
supports_web_search: Callable[..., bool]
supports_tool_search: Callable[..., bool]
supports_url_context: Callable[..., bool]
supports_continuation: Callable[..., bool]
supports_response_schema: Callable[..., bool]
supports_parallel_function_calling: Callable[..., bool]
supports_vision: Callable[..., bool]

View file

@ -33,6 +33,7 @@ UTILS_NAMES: Final = (
"supports_web_search",
"supports_tool_search",
"supports_url_context",
"supports_continuation",
"supports_response_schema",
"supports_parallel_function_calling",
"supports_vision",
@ -488,6 +489,7 @@ _UTILS_IMPORT_MAP: Final = {
"supports_web_search": (".utils", "supports_web_search"),
"supports_tool_search": (".utils", "supports_tool_search"),
"supports_url_context": (".utils", "supports_url_context"),
"supports_continuation": (".utils", "supports_continuation"),
"supports_response_schema": (".utils", "supports_response_schema"),
"supports_parallel_function_calling": (
".utils",

View file

@ -45,13 +45,27 @@ from .redis_semantic_cache import RedisSemanticCache
from .s3_cache import S3Cache
RESPONSE_CACHE_TARGET: Final = "llm_response"
_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"})
_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token", "return_token_ids"})
_DECODED_LITERAL: Final = TypeAdapter(object)
_REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True)
)
def _keyed_provider_params_in_extra_body(kwargs: Mapping[str, object]) -> str:
raw_extra_body: Final = kwargs.get("extra_body")
if not isinstance(raw_extra_body, dict):
return ""
extra_body: Final = cast( # cast-ok: isinstance leaves the key and value types unknown
Mapping[str, object], raw_extra_body
)
return "".join(
f"extra_body.{param}: {extra_body[param]}"
for param in sorted(_ALWAYS_KEYED_PROVIDER_PARAMS)
if extra_body.get(param)
)
@contextmanager
def response_cache_phase(operation: Literal["get", "set"]) -> Generator[None]:
"""The ``cache.get llm_response`` / ``cache.set llm_response`` span a response-cache read or write runs
@ -429,7 +443,10 @@ class Cache:
"" if embeds_thought_signature_in_tool_call_id(kwargs) else "thought_signature_in_tool_call_id: False"
)
hashed_cache_key = Cache._get_hashed_cache_key(
cache_key + reasoning_summary_material + tool_call_id_shape_material
cache_key
+ _keyed_provider_params_in_extra_body(kwargs)
+ reasoning_summary_material
+ tool_call_id_shape_material
)
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
verbose_logger.debug(

View file

@ -673,7 +673,7 @@ class DualCache(BaseCache):
if self.redis_cache is not None:
self.redis_cache.flush_cache()
def delete_cache(self, key):
def delete_cache(self, key: str) -> None:
"""
Delete a key from the cache
"""

View file

@ -699,6 +699,9 @@ STREAM_SSE_DONE_STRING: Final[str] = "[DONE]"
STREAM_SSE_DATA_PREFIX: Final[str] = "data: "
STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping"}\n\n'
STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8")
STREAM_FINISH_CHOICE_FIELDS: Final = ("stop_reason",)
STREAM_CHUNK_CHOICE_FIELDS: Final = ("token_ids",)
VLLM_CHOICE_FIELDS: Final = STREAM_FINISH_CHOICE_FIELDS + STREAM_CHUNK_CHOICE_FIELDS
### SPEND TRACKING ###
DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = get_env_float(
"DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400

View file

@ -1,7 +1,8 @@
import asyncio
import os
import time
from collections.abc import Callable
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from functools import cache
from typing import Final
@ -15,6 +16,7 @@ from litellm.constants import (
AZURE_STORAGE_MSFT_VERSION,
)
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.azure.common_utils import get_azure_ad_token_from_entra_id
from litellm.llms.custom_httpx.http_handler import (
@ -47,6 +49,20 @@ def adls_safe_file_name(payload_id: str | None) -> str:
return f"{name.translate(_ADLS_SAFE_NAME)}.json"
@dataclass(frozen=True, slots=True)
class AzureStorageLogQueueItem:
payload: StandardLoggingPayload
file_name: str
def _queued_log(kwargs: Mapping[str, object], payload: StandardLoggingPayload) -> AzureStorageLogQueueItem:
payload_id: Final = payload.get("id")
return AzureStorageLogQueueItem(
payload=payload,
file_name=adls_safe_file_name(payload_id and logged_event_key(kwargs, payload_id)),
)
@cache
def _cached_credential_chain_token_provider() -> Callable[[], str]:
return get_azure_ad_token_provider(
@ -99,7 +115,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
asyncio.create_task(self.periodic_flush())
self.flush_lock = asyncio.Lock()
self.log_queue: list[StandardLoggingPayload] = []
self.log_queue: list[AzureStorageLogQueueItem] = []
super().__init__(**kwargs, flush_lock=self.flush_lock)
except Exception as e:
verbose_logger.exception(
@ -115,7 +131,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
def azure_storage_blob_endpoint(self) -> str:
return f"https://{self.azure_storage_account_name}.blob.{self.azure_storage_endpoint_suffix}"
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
async def async_log_success_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime):
"""
Async Log success events to Azure Blob Storage
@ -133,12 +149,12 @@ class AzureBlobStorageLogger(CustomBatchLogger):
if standard_logging_payload is None:
raise ValueError("standard_logging_payload is not set")
self.log_queue.append(standard_logging_payload)
self.log_queue.append(_queued_log(kwargs, standard_logging_payload))
except Exception as e:
verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
async def async_log_failure_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime):
"""
Async Log failure events to Azure Blob Storage
@ -156,7 +172,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
if standard_logging_payload is None:
raise ValueError("standard_logging_payload is not set")
self.log_queue.append(standard_logging_payload)
self.log_queue.append(_queued_log(kwargs, standard_logging_payload))
except Exception as e:
verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e)
@ -195,13 +211,15 @@ class AzureBlobStorageLogger(CustomBatchLogger):
len(self.log_queue),
)
for payload in self.log_queue:
await self.async_upload_payload_to_azure_blob_storage(payload=payload)
for item in self.log_queue:
await self.async_upload_payload_to_azure_blob_storage(payload=item.payload, file_name=item.file_name)
except Exception as e:
verbose_logger.exception("AzureBlobStorageLogger Error sending batch API - %s", e)
async def async_upload_payload_to_azure_blob_storage(self, payload: StandardLoggingPayload):
async def async_upload_payload_to_azure_blob_storage(
self, payload: StandardLoggingPayload, file_name: str | None = None
):
"""
Uploads the payload to Azure Blob Storage using a 3-step process:
1. Create file resource
@ -210,14 +228,14 @@ class AzureBlobStorageLogger(CustomBatchLogger):
"""
try:
if self.azure_storage_account_key:
await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload)
await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload, file_name=file_name)
else:
# Get a valid token instead of always requesting a new one
await self.set_valid_azure_ad_token()
async_client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
json_payload: Final = safe_dumps(payload) + "\n" # Add newline for each log entry
payload_bytes: Final = json_payload.encode("utf-8")
filename: Final = adls_safe_file_name(payload.get("id"))
filename: Final = file_name or adls_safe_file_name(payload.get("id"))
base_url = f"{self.azure_storage_dfs_endpoint}/{self.azure_storage_file_system}/{filename}"
# Execute the 3-step upload process
@ -377,7 +395,9 @@ class AzureBlobStorageLogger(CustomBatchLogger):
self._service_client_timeout = self._clock() + _DEFAULT_TTL_FOR_HTTPX_CLIENTS
return self._service_client
async def upload_to_azure_data_lake_with_azure_account_key(self, payload: StandardLoggingPayload):
async def upload_to_azure_data_lake_with_azure_account_key(
self, payload: StandardLoggingPayload, file_name: str | None = None
):
"""
Uploads the payload to Azure Data Lake using the Azure SDK
@ -386,11 +406,11 @@ class AzureBlobStorageLogger(CustomBatchLogger):
try:
today: Final = datetime.now().strftime("%Y-%m-%d")
file_name: Final = adls_safe_file_name(payload.get("id"))
resolved_file_name: Final = file_name or adls_safe_file_name(payload.get("id"))
await self._upload_bytes_with_account_key(
file_path=f"{today}/{file_name}", content=safe_dumps(payload).encode("utf-8")
file_path=f"{today}/{resolved_file_name}", content=safe_dumps(payload).encode("utf-8")
)
verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, file_name)
verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, resolved_file_name)
except Exception as e:
verbose_logger.exception("Error occurred: %s", e)

View file

@ -19,6 +19,7 @@ from litellm._logging import verbose_logger
from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger
from litellm.integrations.clickhouse.context import is_lens_analysis
from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
from litellm.tracing.types import SpendLogPayload, SpendLogRecord
@ -134,7 +135,7 @@ def spend_log_row_from_payload(
usage: Mapping[str, Any] = metadata.get("usage_object") or hidden_params.get("usage_object") or MappingProxyType({})
cache_read_tokens, cache_write_tokens = _cache_tokens(usage)
trace_id, span_id = _find_traceparent(metadata, kwargs)
request_id = str(payload.get("id") or "")
response_id: Final = str(payload.get("id") or "")
redact = litellm.turn_off_message_logging is True
track_end_user: Final = not litellm.disable_end_user_cost_tracking
completion_start_ms = _to_ms(payload.get("completionStartTime"))
@ -157,8 +158,8 @@ def spend_log_row_from_payload(
)
)
return SpendLogRecord(
request_id=request_id,
response_id=strip_cache_hit_suffix(request_id),
request_id=logged_event_key(kwargs, response_id),
response_id=strip_cache_hit_suffix(response_id),
provider_request_id=(
get_provider_request_id(kwargs.get("response_headers"))
or get_provider_request_id(hidden_params.get("additional_headers"))

View file

@ -15,6 +15,7 @@ from litellm.integrations.deepeval.utils import (
to_zod_compatible_iso,
validate_environment,
)
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
# This file includes the custom callbacks for LiteLLM Proxy
@ -109,8 +110,10 @@ class DeepEvalLogger(CustomLogger):
)
else:
output = str(standard_logging_object.get("error_string", ""))
logged_id: Final = standard_logging_object.get("id", uuid.uuid4())
span_id: Final = logged_event_key(kwargs, logged_id) if isinstance(logged_id, str) else logged_id
return BaseApiSpan(
uuid=standard_logging_object.get("id", uuid.uuid4()),
uuid=span_id,
name=("litellm_success_callback" if is_success else "litellm_failure_callback"),
status=(TraceSpanApiStatus.SUCCESS if is_success else TraceSpanApiStatus.ERRORED),
type=SpanApiType.LLM,

View file

@ -9,6 +9,8 @@ from typing import Final, Protocol
import litellm
from litellm._uuid import uuid
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
class _DynamoTable(Protocol):
@ -22,11 +24,13 @@ class _DynamoResource(Protocol):
class DyanmoDBLogger:
# Class variables or attributes
def __init__(self):
def __init__(self, dynamodb: _DynamoResource | None = None):
# Instance variables
import boto3
self.dynamodb: Final[_DynamoResource] = boto3.resource("dynamodb", region_name=os.environ["AWS_REGION_NAME"])
self.dynamodb: Final[_DynamoResource] = dynamodb or boto3.resource(
"dynamodb", region_name=os.environ["AWS_REGION_NAME"]
)
if litellm.dynamodb_table_name is None:
raise ValueError(
"LiteLLM Error, trying to use DynamoDB but not table name passed. Create a table and set `litellm.dynamodb_table_name=<your-table>`"
@ -45,23 +49,23 @@ class DyanmoDBLogger:
_async_log_event = async_log_event
def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
def log_event(self, kwargs: Mapping[str, object], response_obj, start_time, end_time, print_verbose):
try:
print_verbose(f"DynamoDB Logging - Enters logging function for model {kwargs}")
# construct payload to send to DynamoDB
# follows the same params as langfuse.py
litellm_params: Final = kwargs.get("litellm_params", {})
metadata: Final = litellm_params.get("metadata", {}) or {} # if litellm_params['metadata'] == None
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {}
messages: Final = kwargs.get("messages")
optional_params: Final = kwargs.get("optional_params", {})
call_type: Final = kwargs.get("call_type", "litellm.completion")
usage: Final = response_obj["usage"]
id: Final = response_obj.get("id", str(uuid.uuid4()))
response_id: Final = response_obj.get("id", str(uuid.uuid4()))
# Build the initial payload
payload: Final[dict[str, object]] = {
"id": id,
"id": logged_event_key(kwargs, response_id) if isinstance(response_id, str) else response_id,
"call_type": call_type,
"startTime": start_time,
"endTime": end_time,

View file

@ -17,6 +17,7 @@ from litellm.litellm_core_utils.cloud_storage_security import (
sanitize_cloud_object_component,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.gcs_bucket import *
from litellm.types.utils import StandardBatchLineItemsPayload, StandardLoggingPayload
@ -296,30 +297,28 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
else:
await self._send_individual_logs(items_to_process)
def _get_object_name(self, kwargs: dict, logging_payload: StandardLoggingPayload, response_obj: Any) -> str:
def _get_object_name(
self, kwargs: Mapping[str, object], logging_payload: StandardLoggingPayload, response_obj: Any
) -> str:
"""
Get the object name to use for the current payload
"""
current_date: Final = self._get_object_date_from_datetime(datetime.now(timezone.utc))
metadata: Final = as_str_mapping((as_str_mapping(kwargs.get("litellm_params")) or {}).get("metadata")) or {}
gcs_log_id: Final = metadata.get("gcs_log_id")
safe_log_id: Final = sanitize_cloud_object_component(
gcs_log_id if isinstance(gcs_log_id, str) else None, fallback=""
)
if safe_log_id:
return f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
if logging_payload.get("error_str", None) is not None:
object_name = self._generate_failure_object_name(
request_date_str=current_date,
)
else:
object_name = self._generate_success_object_name(
request_date_str=current_date,
response_id=response_obj.get("id", ""),
)
# used for testing
_litellm_params: Final = kwargs.get("litellm_params", None) or {}
_metadata: Final = _litellm_params.get("metadata", None) or {}
if "gcs_log_id" in _metadata:
safe_log_id: Final = sanitize_cloud_object_component(_metadata.get("gcs_log_id"), fallback="")
if safe_log_id:
object_name = f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
return object_name
return self._generate_failure_object_name(request_date_str=current_date)
raw_response_id: Final[object] = response_obj.get("id", "")
response_id: Final = raw_response_id if isinstance(raw_response_id, str) else ""
return self._generate_success_object_name(
request_date_str=current_date,
response_id=logged_event_key(kwargs, response_id),
)
async def get_request_response_payload(
self,

View file

@ -3,12 +3,15 @@
import json
import os
from collections.abc import Mapping
from typing import Final
import httpx
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.llms.custom_httpx.http_handler import (
HTTPHandler,
get_async_httpx_client,
@ -48,7 +51,7 @@ class OpenMeterLogger(CustomLogger):
if len(missing_keys) > 0:
raise Exception(f"Missing keys={missing_keys} in environment.")
def _common_logic(self, kwargs: dict, response_obj):
def _common_logic(self, kwargs: Mapping[str, object], response_obj):
call_id: Final = response_obj.get("id", kwargs.get("litellm_call_id"))
dt: Final = get_utc_datetime().isoformat()
cost: Final = kwargs.get("response_cost", None)
@ -74,8 +77,8 @@ class OpenMeterLogger(CustomLogger):
# If no user provided directly, try to get it from token user_id
if user_param is None:
# Check if user_id is available from the API key metadata
litellm_params: Final = kwargs.get("litellm_params", {})
metadata: Final = litellm_params.get("metadata", {})
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {}
user_api_key_user_id: Final = metadata.get("user_api_key_user_id", None)
if user_api_key_user_id is not None:
@ -89,7 +92,7 @@ class OpenMeterLogger(CustomLogger):
return {
"specversion": "1.0",
"type": os.getenv("OPENMETER_EVENT_TYPE", "litellm_tokens"),
"id": call_id,
"id": logged_event_key(kwargs, call_id) if isinstance(call_id, str) else call_id,
"time": dt,
"subject": subject,
"source": "litellm-proxy",

View file

@ -3036,7 +3036,7 @@ class PrometheusLogger(CustomLogger):
async def async_post_call_failure_hook(
self,
request_data: dict,
request_data: dict[str, object],
original_exception: Exception,
user_api_key_dict: UserAPIKeyAuth,
traceback_str: str | None = None,
@ -3059,10 +3059,14 @@ class PrometheusLogger(CustomLogger):
try:
_tags: Final = StandardLoggingPayloadSetup.get_request_tags(
litellm_params=request_data,
proxy_server_request=request_data.get("proxy_server_request", {}),
proxy_server_request=_logged_fields(request_data.get("proxy_server_request")),
)
_metadata: Final = _logged_fields(request_data.get("metadata"))
model_id: Final = (
_logged_fields(_metadata.get("model_info")).get("id")
or _logged_fields(request_data.get("model_info")).get("id")
or _logged_fields(request_data.get("standard_logging_object")).get("model_id")
)
_metadata: Final = request_data.get("metadata", {}) or {}
model_id: Final = _metadata.get("model_info", {}).get("id") or request_data.get("model_info", {}).get("id")
rate_limit_category, rate_limit_type = self._extract_rate_limit_labels(original_exception)
api_provider: Final = self._extract_api_provider_from_request_data(
request_data

View file

@ -107,6 +107,7 @@ class ExceptionCheckers:
"exceeds the available context size", # llama.cpp/Lemonade
"exceeds the maximum number of tokens allowed", # Gemini
'"code": "context_length_exceeded"',
"prompt is too long",
]
for substring in known_exception_substrings:
if substring in _error_str_lowercase:

View file

@ -26,7 +26,12 @@ from litellm.constants import (
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
NON_INFERENCE_CALL_TYPES,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.initialize_dynamic_callback_params import initialize_standard_callback_dynamic_params
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
BackgroundResponseSettlement,
)
from litellm.types.utils import (
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN,
@ -35,6 +40,8 @@ from litellm.types.utils import (
BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX: Final = "_background_cost"
ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved"
OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved"
ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key"
@ -151,6 +158,26 @@ def is_unbilled_non_inference_call_from_params(
return is_unbilled_non_inference_call(call_type, metadata, response)
def settles_background_response(call_type: str, metadata: Mapping[str, object], kwargs: Mapping[str, object]) -> bool:
return kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES or (
call_type in NON_INFERENCE_CALL_TYPES
and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
)
def logged_event_key(kwargs: Mapping[str, object], event_id: str) -> str:
call_type: Final = kwargs.get("call_type")
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
metadata: Final = (
as_str_mapping(litellm_params.get("litellm_metadata")) or as_str_mapping(litellm_params.get("metadata")) or {}
)
if not event_id or not settles_background_response(
call_type if isinstance(call_type, str) else "", metadata, kwargs
):
return event_id
return f"{event_id}{BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX}"
def sanitize_user_api_key_auth(auth: object) -> object:
"""Copy of the auth object with its budget reservation removed; the cost callback
falls back to reading the reservation from inside the auth object."""

View file

@ -872,6 +872,7 @@ class Logging(LiteLLMLoggingBaseClass):
# The model the proxy shows the client on streamed chunks. The logged streamed response carries it
# once that response is priced, the same way a non-streamed response is logged
self.client_facing_stream_model: str | None = None
self.client_facing_response_id: str | None = None
self.zero_cost_warned: bool = False
self.llm_caching_handler: LLMCachingHandler | None = None
@ -918,6 +919,7 @@ class Logging(LiteLLMLoggingBaseClass):
# post_call guardrails have run; the @client decorator then stores the
# enqueue closure here instead of firing it immediately.
self.defer_async_logging: bool = False
self.drops_stream_token_ids: bool = False
self.enqueue_deferred_logging: Callable[[], None] | None = None
self._async_success_scheduled: bool = False
self._async_success_enqueued: bool = False
@ -3237,6 +3239,23 @@ class Logging(LiteLLMLoggingBaseClass):
return response
return response.model_copy(update={"id": response_id})
def _with_client_facing_response_id(
self,
response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
response_id: Final = self.client_facing_response_id
if response_id is None or getattr(response, "id", None) == response_id:
return response
return response.model_copy(update={"id": response_id})
def _client_facing_streaming_response(
self,
response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
return self._with_client_facing_response_id(
self._with_continued_response_id(self._with_client_facing_stream_model(response))
)
def _success_handler_helper_fn(
self,
result: object = None,
@ -3640,9 +3659,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
self._add_continued_partial_spend(complete_streaming_response)
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
logged_streaming_response: Final = self._with_continued_response_id(
self._with_client_facing_stream_model(complete_streaming_response)
)
logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response)
self.model_call_details["complete_streaming_response"] = logged_streaming_response
self.capture_usage_before_logging(logged_streaming_response, start_time, end_time)
## STANDARDIZED LOGGING PAYLOAD
@ -4239,9 +4256,7 @@ class Logging(LiteLLMLoggingBaseClass):
await self._prepare_baseline_cache_estimate(complete_streaming_response)
logged_streaming_response: Final = self._with_continued_response_id(
self._with_client_facing_stream_model(complete_streaming_response)
)
logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response)
self.model_call_details["async_complete_streaming_response"] = logged_streaming_response
## STANDARDIZED LOGGING PAYLOAD

View file

@ -3,7 +3,8 @@ import json
import re
import time
import traceback
from collections.abc import Generator, Iterator, Mapping, Sequence
from collections.abc import Generator, Iterable, Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import Final, Literal, cast
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
@ -11,7 +12,12 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.constants import (
RESPONSE_FORMAT_TOOL_NAME,
STREAM_CHUNK_CHOICE_FIELDS,
STREAM_FINISH_CHOICE_FIELDS,
VLLM_CHOICE_FIELDS,
)
from litellm.litellm_core_utils.hidden_params import get_hidden_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_reasoning_content,
@ -170,6 +176,28 @@ def _split_assembled_content_for_replay(content: str | None) -> list[str]:
return _REPLAY_CONTENT_SLICE_RE.findall(content)
def _drop_choice_fields(choice: StreamingChoices, fields: Iterable[str]) -> None:
for field in fields:
if hasattr(choice, field):
delattr(choice, field)
def _vllm_choice_fields(choice: Mapping[str, object]) -> Mapping[str, object]:
cached_fields: Final = _choice_provider_specific_fields(choice)
return MappingProxyType(
{
field: choice[field] if field in choice else cached_fields[field]
for field in VLLM_CHOICE_FIELDS
if field in choice or field in cached_fields
}
)
def _choice_provider_specific_fields(choice: Mapping[str, object]) -> Mapping[str, object]:
fields: Final = choice.get("provider_specific_fields")
return _FIELD_MAP.validate_python(fields) if isinstance(fields, dict) else MappingProxyType({})
_REPLAY_REASONING_FIELDS: Final = frozenset({"reasoning_content", "thinking_blocks", "reasoning_items"})
_REPLAY_LEAD_FIELDS: Final = _REPLAY_REASONING_FIELDS | {"role", "provider_specific_fields"}
_REPLAY_TOOL_FIELDS: Final = frozenset({"tool_calls", "function_call"})
@ -220,8 +248,9 @@ def _replay_frame(
choice.finish_reason = finish_reason
if not keeps_choice_metadata:
choice.logprobs = None
if hasattr(choice, "enhancements"):
del choice.enhancements
_drop_choice_fields(choice, ("enhancements", *STREAM_CHUNK_CHOICE_FIELDS))
if finish_reason is None:
_drop_choice_fields(choice, STREAM_FINISH_CHOICE_FIELDS)
return frame
@ -325,7 +354,9 @@ async def convert_to_streaming_response_async(
logprobs = choice.get("logprobs", None)
choice = StreamingChoices(finish_reason=finish_reason, index=idx, delta=delta, logprobs=logprobs)
choice = StreamingChoices(
finish_reason=finish_reason, index=idx, delta=delta, logprobs=logprobs, **_vllm_choice_fields(choice)
)
choice_list.append(choice)
model_response_object.choices = choice_list
@ -388,6 +419,7 @@ def convert_to_streaming_response(
delta=delta,
logprobs=logprobs,
enhancements=enhancements,
**_vllm_choice_fields(choice),
)
choice_list.append(choice)
@ -872,6 +904,7 @@ def convert_to_model_response_object(
logprobs=logprobs,
enhancements=enhancements,
provider_specific_fields=provider_specific_fields,
**_vllm_choice_fields(choice),
)
if provider_sent_logprobs and logprobs is None:
choice.logprobs = None

View file

@ -10,12 +10,18 @@ class ChatTemplateToolSection:
preamble_tokens: int
implicit_system_turn_tokens: int
render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str]
counts_tool_choice: bool = False
renders_strict_only_when_true: bool = False
def _deepseek_v4_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
def _json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
return "\n".join(json.dumps(function, ensure_ascii=False) for function in functions)
def _compact_json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
return "\n".join(json.dumps(function, ensure_ascii=False, separators=(",", ":")) for function in functions)
def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
return "".join(
"\n" + json.dumps({"type": "function", "function": function}, ensure_ascii=False) for function in functions
@ -23,13 +29,29 @@ def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
_TOOL_SECTIONS: Final = (
(
re.compile(r"deepseek-v4(?:[-._]|p)1(?!\d)", re.IGNORECASE),
ChatTemplateToolSection(211, 0, _json_line_tool_schemas),
),
(
re.compile(
r"deepseek-v4(?![.\d]|p\d)|^(?:deepseek/)?deepseek-(?:chat|reasoner|flash)(?::free)?$", re.IGNORECASE
),
ChatTemplateToolSection(222, 0, _deepseek_v4_tool_schemas),
ChatTemplateToolSection(222, 0, _json_line_tool_schemas),
),
(re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)),
(
re.compile(r"grok-4[-.]1(?![.\d])", re.IGNORECASE),
ChatTemplateToolSection(
0, 0, _compact_json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True
),
),
(
re.compile(r"grok-4[-.]3(?![.\d])", re.IGNORECASE),
ChatTemplateToolSection(
0, 0, _json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True
),
),
)

View file

@ -34,6 +34,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
split_leading_system_run,
string_field,
)
from litellm.litellm_core_utils.tool_fields import tools_without_hosted_mcp
from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client, header_value
from litellm.types.files import get_file_extension_from_mime_type
@ -1328,7 +1329,7 @@ def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None:
_get_thought_signature_from_tool = get_thought_signature_from_tool
def _get_dummy_thought_signature() -> str:
def get_dummy_thought_signature() -> str:
"""Generate a dummy thought signature for models that require it.
This is used when transferring conversation history from older models
@ -1415,7 +1416,7 @@ def convert_to_gemini_tool_call_invoke(
# https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/thinking/thought-signatures#parallel_function_calling_example
is_first_function_call = len(_parts_list) == 0
if not thought_signature and is_first_function_call and needs_dummy_signature:
thought_signature = _get_dummy_thought_signature()
thought_signature = get_dummy_thought_signature()
if thought_signature:
part_dict["thoughtSignature"] = thought_signature
@ -1439,7 +1440,7 @@ def convert_to_gemini_tool_call_invoke(
# If no signature found and model is gemini-3, use dummy signature
if not thought_signature and needs_dummy_signature:
thought_signature = _get_dummy_thought_signature()
thought_signature = get_dummy_thought_signature()
if thought_signature:
part_dict_function["thoughtSignature"] = thought_signature
@ -5916,8 +5917,13 @@ def function_call_prompt(
llm_provider: str,
drop_params: bool,
) -> list[dict[str, object]]:
prompt_functions: Final = tools_without_hosted_mcp(
functions, model=model, route=llm_provider, drop_params=drop_params
)
if not prompt_functions:
return messages
header: Final = """To call a function, reply with JSON ONLY in this format {"name": "function_name", "arguments":{"argument_name": "argument_value"}}. Once a function result answers the request, reply to the user in plain text instead of calling a function again. The following functions are available to you:"""
function_prompt: Final = header + "".join(f"\n{function}\n" for function in functions)
function_prompt: Final = header + "".join(f"\n{function}\n" for function in prompt_functions)
if not any(_is_system_message(message) for message in messages):
return [*messages, {"role": "system", "content": function_prompt}]
return [

View file

@ -196,6 +196,9 @@ def _redact_function_call(function_call) -> None:
def _redact_choice_content(choice):
"""Helper to redact content in a choice (message or delta)."""
if isinstance(choice, (litellm.Choices, litellm.utils.StreamingChoices, litellm.utils.TextChoices)):
drop_token_ids(choice)
_redact_provider_specific_fields(choice)
if isinstance(choice, litellm.Choices):
if choice.message.content is not None:
choice.message.content = REDACTED_BY_LITELLM
@ -216,6 +219,9 @@ def _redact_choice_content(choice):
_redact_provider_specific_fields(choice.delta)
_redact_tool_calls(getattr(choice.delta, "tool_calls", None))
_redact_function_call(getattr(choice.delta, "function_call", None))
elif isinstance(choice, litellm.utils.TextChoices):
_redact_field_when_present(choice, "text")
_redact_field_when_present(choice, "reasoning_content")
def _as_mapping(value: object) -> Mapping[str, object] | None:
@ -227,14 +233,27 @@ def _redact_provider_specific_fields(message: object) -> None:
setattr(message, "provider_specific_fields", _redacted_provider_specific_fields(fields)) # noqa: B010 # typed as object
def _drop_provider_specific_token_ids(holder: object) -> None:
fields: Final = _as_mapping(getattr(holder, "provider_specific_fields", None))
if fields is not None and fields.get("token_ids") is not None:
setattr(holder, "provider_specific_fields", {**fields, "token_ids": None}) # noqa: B010 # typed as object
def drop_token_ids(choice: object) -> None:
if getattr(choice, "token_ids", None) is not None:
setattr(choice, "token_ids", None) # noqa: B010 # pydantic extra, not a declared field
for holder in (choice, getattr(choice, "delta", None), getattr(choice, "message", None)):
_drop_provider_specific_token_ids(holder)
def _redact_provider_specific_fields_dict(choice: Mapping[str, object]) -> None:
for message in (choice.get("message"), choice.get("delta")):
for holder in (choice, choice.get("message"), choice.get("delta")):
if (
(readable := _as_mapping(message)) is not None
(readable := _as_mapping(holder)) is not None
and (fields := _as_mapping(readable.get("provider_specific_fields"))) is not None
and isinstance(message, dict)
and isinstance(holder, dict)
):
message["provider_specific_fields"] = _redacted_provider_specific_fields(fields)
holder["provider_specific_fields"] = _redacted_provider_specific_fields(fields)
def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[str, object]:
@ -251,6 +270,7 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[
"compaction_blocks",
"compaction_start",
"compaction_delta",
"token_ids",
)
if key in fields
},
@ -422,6 +442,8 @@ def _redact_model_response_dict_choices(choices, redacted_str: str):
for choice in choices:
if isinstance(choice, dict):
_redact_provider_specific_fields_dict(choice)
if choice.get("token_ids") is not None:
choice["token_ids"] = None
if "message" in choice and isinstance(choice["message"], dict):
if choice["message"].get("content") is not None:
choice["message"]["content"] = redacted_str

View file

@ -1,6 +1,7 @@
from collections.abc import Iterable
from typing import Final
from litellm.constants import VLLM_CHOICE_FIELDS
from litellm.types.utils import ModelResponseStream, StreamingChoices
@ -9,12 +10,19 @@ def is_empty_streaming_choice(choice: StreamingChoices) -> bool:
return False
if getattr(choice, "logprobs", None) is not None:
return False
if not _all_values_none(_vllm_choice_values(choice)):
return False
delta: Final = choice.delta if hasattr(choice, "delta") else None
if delta is None:
return True
return _all_values_none(delta.model_dump().values())
def _vllm_choice_values(choice: StreamingChoices) -> Iterable[object]:
choice_extras: Final = choice.model_extra or {}
return (choice_extras.get(field) for field in VLLM_CHOICE_FIELDS)
def _all_values_none(values: Iterable[object]) -> bool:
return all(value is None for value in values)

View file

@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, Required
import litellm
from litellm._logging import verbose_logger
from litellm.constants import VLLM_CHOICE_FIELDS
from litellm.litellm_core_utils.core_helpers import (
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS,
UPSTREAM_USAGE_COST,
@ -31,6 +32,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict
)
from litellm.litellm_core_utils.redact_messages import drop_token_ids
from litellm.litellm_core_utils.token_counter import usage_tokenizer_model
from litellm.types.llms.openai import (
AllMessageValues,
@ -434,6 +436,8 @@ class ChunkProcessor:
chunks, unreported="tool_calls" if _carries_tool_calls(chunks) else "stop"
),
message={"role": ChunkProcessor._get_role_from_chunks(chunks), "content": ""},
provider_specific_fields=None,
**_streamed_vllm_choice_fields(chain.from_iterable(chunk.get("choices") or () for chunk in chunks)),
)
@staticmethod
@ -504,6 +508,7 @@ class ChunkProcessor:
"index": 0,
"message": {"role": role, "content": ""},
"finish_reason": finish_reason,
**_streamed_vllm_choice_fields(_first_choices(chunks)),
}
]
),
@ -1371,6 +1376,7 @@ def _rebuilt_text_choice(index: int, choices: Sequence[TextChoices]) -> TextChoi
index=index,
logprobs=_merged_logprobs(choices),
finish_reason=next((choice.finish_reason for choice in reversed(choices) if choice.finish_reason), None),
**_streamed_vllm_choice_fields(choices),
)
@ -1504,6 +1510,31 @@ def _field_of(value: object, name: str) -> object:
return getattr(value, name, None)
def _first_choices(chunks: Iterable[object]) -> tuple[object, ...]:
return tuple(next(iter(object_items(_field_of(chunk, "choices"))), None) for chunk in chunks)
def _choice_extras(choice: object) -> Mapping[str, object]:
if isinstance(choice, BaseModel):
return choice.model_extra or {}
return object_mapping(choice) or {}
def _streamed_vllm_choice_fields(choices: Iterable[object]) -> Mapping[str, object]:
extras: Final = tuple(map(_choice_extras, choices))
token_id_runs: Final = tuple(object_items(fields.get("token_ids")) for fields in extras)
stop_reasons: Final = tuple(reason for fields in extras if (reason := fields.get("stop_reason")) is not None)
has_token_ids: Final = any(fields.get("token_ids") is not None for fields in extras)
streamed_values: Final[Mapping[str, object]] = MappingProxyType(
{
"token_ids": list(chain.from_iterable(token_id_runs)) if has_token_ids else None,
"stop_reason": stop_reasons[-1] if stop_reasons else None,
}
)
present_fields: Final = frozenset(chain.from_iterable(fields.keys() for fields in extras))
return MappingProxyType({field: streamed_values[field] for field in VLLM_CHOICE_FIELDS if field in present_fields})
def _delta_reasoning_items(delta: object) -> Sequence[object]:
items: Final = _field_of(delta, "reasoning_items")
return cast(Sequence[object], items) if isinstance(items, list) else () # cast-ok: items forwarded verbatim
@ -1776,6 +1807,19 @@ def stream_chunk_builder(
end_time: object = None,
logging_obj: Optional["Logging"] = None,
count_prompt_tokens: Callable[[], int] | None = None,
) -> ModelResponse | TextCompletionResponse | None:
response: Final = _assembled_stream_response(chunks, messages, logging_obj, count_prompt_tokens)
if response is not None and getattr(logging_obj, "drops_stream_token_ids", False) is True:
for choice in response.choices:
drop_token_ids(choice)
return response
def _assembled_stream_response(
chunks: Sequence[object],
messages: Sequence[AllMessageValues | Message] | None,
logging_obj: Optional["Logging"],
count_prompt_tokens: Callable[[], int] | None,
) -> ModelResponse | TextCompletionResponse | None:
try:
if chunks is None:
@ -1788,7 +1832,7 @@ def stream_chunk_builder(
if not chunks:
return None
processor: Final = ChunkProcessor(chunks, messages)
processor: Final = ChunkProcessor(list(chunks), messages)
chunks = processor.chunks
### BASE-CASE ###

View file

@ -23,6 +23,7 @@ from typing_extensions import NotRequired, TypedDict
import litellm
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.constants import STREAM_CHUNK_CHOICE_FIELDS, STREAM_FINISH_CHOICE_FIELDS
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import (
UPSTREAM_USAGE_COST,
@ -37,10 +38,11 @@ from litellm.litellm_core_utils.hidden_params import (
set_hidden_param,
stored_attribute,
)
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_items
from litellm.litellm_core_utils.model_response_utils import (
is_model_response_stream_empty,
)
from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject
from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject, drop_token_ids
from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
from litellm.types.llms.base import LiteLLMBaseModel
@ -137,6 +139,70 @@ def _prepend_upstream_choice_extras(
choice.__pydantic_fields_set__.update(upstream_extras)
_NO_UPSTREAM_CHOICE_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
_NO_CHOICE_FIELDS_BY_INDEX: Final[Mapping[int, Mapping[str, object]]] = MappingProxyType({})
def _upstream_first_choice(upstream_chunk: object) -> BaseModel | None:
match getattr(upstream_chunk, "choices", None):
case [BaseModel() as first_choice, *_]:
return first_choice
case _:
return None
def _upstream_choice_fields(upstream_chunk: object, fields: Sequence[str]) -> Mapping[str, object]:
first_choice: Final = _upstream_first_choice(upstream_chunk)
if first_choice is None:
return _NO_UPSTREAM_CHOICE_FIELDS
upstream_extras: Final[Mapping[str, object]] = first_choice.model_extra or {}
return MappingProxyType({field: upstream_extras[field] for field in fields if field in upstream_extras})
def _upstream_choice_index(upstream_chunk: object) -> int:
upstream_index: Final[object] = getattr(_upstream_first_choice(upstream_chunk), "index", 0)
return upstream_index if isinstance(upstream_index, int) else 0
def _response_choice_index(response_obj: Mapping[str, object]) -> int:
choice_index: Final = response_obj.get("choice_index")
if isinstance(choice_index, int):
return choice_index
return _upstream_choice_index(response_obj.get("original_chunk"))
def _non_null_upstream_choice_fields(upstream_chunk: object, fields: Sequence[str]) -> Mapping[str, object]:
first_choice: Final = _upstream_first_choice(upstream_chunk)
upstream_extras: Final = first_choice.model_extra if first_choice is not None else None
if not upstream_extras:
return _NO_UPSTREAM_CHOICE_FIELDS
return MappingProxyType(
{field: upstream_extras[field] for field in fields if upstream_extras.get(field) is not None}
)
def _apply_choice_fields(choice: StreamingChoices, fields: Mapping[str, object]) -> None:
for field, value in fields.items():
setattr(choice, field, value)
def _choice_fields_chunk(
model_response: ModelResponseStream, upstream_index: int, fields: Mapping[str, object]
) -> ModelResponseStream:
model_response.choices[0].delta = Delta(content="") # rebind-ok: chunks are built in place here
model_response.choices[0].index = upstream_index # rebind-ok: chunks are built in place here
_apply_choice_fields(model_response.choices[0], fields)
return model_response
def _without_token_ids_under_post_call_rules(chunk: ModelResponseStream | None) -> ModelResponseStream | None:
if chunk is None or not litellm.post_call_rules:
return chunk
for choice in chunk.choices:
drop_token_ids(choice)
return chunk
def _holds_only_scalars(fields: Iterable[tuple[str, object]]) -> bool:
return all(value is None or isinstance(value, (str, int, float, bool)) for _, value in fields)
@ -378,6 +444,8 @@ class CustomStreamWrapper:
self._provider_response_model: str | None = None
self._upstream_provider: str | None = None
self.received_finish_reason: str | None = None
self.received_finish_choice_fields: Mapping[int, Mapping[str, object]] = _NO_CHOICE_FIELDS_BY_INDEX
self.carried_chunk_choice_fields: Mapping[int, Mapping[str, object]] = _NO_CHOICE_FIELDS_BY_INDEX
self.intermittent_finish_reason: str | None = None # finish reasons that show up mid-stream
self.received_any_finish_reason = False # a later chunk can reset intermittent_finish_reason
self.special_tokens = [
@ -823,6 +891,7 @@ class CustomStreamWrapper:
"logprobs": logprobs,
"original_chunk": str_line,
"usage": usage,
"chunk_choice_fields": _non_null_upstream_choice_fields(str_line, STREAM_CHUNK_CHOICE_FIELDS),
}
except Exception as e:
raise e
@ -847,7 +916,7 @@ class CustomStreamWrapper:
except Exception as e:
raise e
def handle_openai_text_completion_chunk(self, chunk):
def handle_openai_text_completion_chunk(self, chunk: object):
try:
text = ""
is_finished = False
@ -860,11 +929,15 @@ class CustomStreamWrapper:
is_finished = True
finish_reason = choices[0].finish_reason
usage = getattr(chunk, "usage", None)
choice_index: Final = _upstream_choice_index(chunk)
return {
"text": text,
"is_finished": is_finished,
"finish_reason": finish_reason,
"usage": usage,
"chunk_choice_fields": _non_null_upstream_choice_fields(chunk, STREAM_CHUNK_CHOICE_FIELDS),
"choice_index": choice_index,
"sibling_finish_reason": finish_reason if choice_index != 0 else None,
}
except Exception as e:
@ -1183,6 +1256,13 @@ class CustomStreamWrapper:
)
is_chunk_non_empty: Final = self.is_chunk_non_empty(completion_obj, model_response, response_obj)
upstream_chunk_choice_fields: Final[Mapping[str, object]] = (
_NO_UPSTREAM_CHOICE_FIELDS
if litellm.post_call_rules
else response_obj.get("chunk_choice_fields", _NO_UPSTREAM_CHOICE_FIELDS)
)
chunk_choice_fields: Final = self._take_carried_chunk_choice_fields(response_obj, upstream_chunk_choice_fields)
sibling_finish_reason: Final[str | None] = response_obj.get("sibling_finish_reason")
if is_chunk_non_empty: # cannot set content of an OpenAI Object to be an empty string
self.raise_on_model_repetition()
@ -1270,16 +1350,40 @@ class CustomStreamWrapper:
if response_obj.get("provider_specific_fields") is not None:
completion_obj["provider_specific_fields"] = response_obj["provider_specific_fields"]
model_response.choices[0].delta = Delta(**completion_obj)
_index: Final[int | None] = completion_obj.get("index")
_index: Final[int | None] = completion_obj.get("index", response_obj.get("choice_index"))
if _index is not None:
model_response.choices[0].index = _index
_apply_choice_fields(model_response.choices[0], chunk_choice_fields)
if sibling_finish_reason is not None:
self._finish_sibling_choice(model_response.choices[0], sibling_finish_reason)
elif response_obj.get("is_finished"):
self._apply_finish_choice_fields(model_response.choices[0])
self._optional_combine_thinking_block_in_choices(model_response=model_response)
return model_response
else:
return
elif sibling_finish_reason is not None:
sibling_chunk: Final = _choice_fields_chunk(
model_response, _response_choice_index(response_obj), chunk_choice_fields
)
self._finish_sibling_choice(sibling_chunk.choices[0], sibling_finish_reason)
return sibling_chunk
elif self.received_finish_reason is not None:
chunk_upstream_index: Final = _response_choice_index(response_obj)
if chunk_choice_fields and self._streams_past_the_pending_finish(
model_response, response_obj, chunk_upstream_index
):
self.carried_chunk_choice_fields = MappingProxyType(
{**self.carried_chunk_choice_fields, chunk_upstream_index: chunk_choice_fields}
)
else:
another_choice_fields_chunk: Final = self._another_choice_fields_chunk(
model_response, response_obj, chunk_choice_fields
)
if another_choice_fields_chunk is not None:
return another_choice_fields_chunk
if self.sent_last_chunk is True:
# Bedrock returns the guardrail trace in the last chunk - we want to return this here
if self.custom_llm_provider == "bedrock" and "trace" in model_response:
@ -1317,12 +1421,20 @@ class CustomStreamWrapper:
model_response.choices[0].finish_reason = self._normalize_tool_call_finish_reason(
map_finish_reason(finish_reason=self.received_finish_reason)
) # ensure consistent output to openai
self._apply_finish_choice_fields(model_response.choices[0])
if (
response_obj.get("is_finished")
and _response_choice_index(response_obj) == model_response.choices[0].index
):
_apply_choice_fields(model_response.choices[0], chunk_choice_fields)
self.sent_last_chunk = True
return model_response
elif self._has_special_delta_content(model_response):
return self._handle_special_delta_content(model_response)
elif chunk_choice_fields:
return _choice_fields_chunk(model_response, _response_choice_index(response_obj), chunk_choice_fields)
else:
if hasattr(model_response, "usage"):
self.chunks.append(model_response)
@ -1545,6 +1657,8 @@ class CustomStreamWrapper:
completion_obj["content"] = response_obj["text"]
print_verbose(f"completion obj content: {completion_obj['content']}")
if response_obj["is_finished"]:
self._record_finish_choice_fields(chunk)
if response_obj["is_finished"] and response_obj["sibling_finish_reason"] is None:
self.received_finish_reason = response_obj["finish_reason"]
if response_obj["usage"] is not None:
_text_completion_usage: Final[Usage] = response_obj["usage"]
@ -1608,6 +1722,7 @@ class CustomStreamWrapper:
self.system_fingerprint = cached_chunk.system_fingerprint
if response_obj["is_finished"]:
self.received_finish_reason = response_obj["finish_reason"]
self._record_finish_choice_fields(cached_chunk)
else: # openai / azure chat model
self.uses_openai_chat_stream = True
if isinstance(chunk, ChatCompletionChunk):
@ -1637,6 +1752,7 @@ class CustomStreamWrapper:
f"{self.custom_llm_provider} raised a streaming error - finish_reason: error, no content string given. Received Chunk={response_obj}"
)
self.received_finish_reason = response_obj["finish_reason"]
self._record_finish_choice_fields(chunk)
if response_obj.get("original_chunk", None) is not None:
if hasattr(response_obj["original_chunk"], "id"):
model_response = self.set_model_id(response_obj["original_chunk"].id, model_response)
@ -1704,7 +1820,7 @@ class CustomStreamWrapper:
completion_obj=completion_obj,
)
if isinstance(dispatch_result, _ProviderChunkEarlyReturn):
return dispatch_result.value
return _without_token_ids_under_post_call_rules(dispatch_result.value)
response_obj = dispatch_result.response_obj
model_response.model = self._served_openrouter_model() or self.model
@ -1794,7 +1910,7 @@ class CustomStreamWrapper:
response_obj=response_obj,
plain_delta_created=plain_delta_created,
)
return result
return _without_token_ids_under_post_call_rules(result)
except StopIteration:
raise StopIteration
@ -1987,6 +2103,75 @@ class CustomStreamWrapper:
return "tool_calls"
return finish_reason
def _record_finish_choice_fields(self, upstream_chunk: object) -> None:
self.received_finish_choice_fields = MappingProxyType(
{
**self.received_finish_choice_fields,
_upstream_choice_index(upstream_chunk): _upstream_choice_fields(
upstream_chunk, STREAM_FINISH_CHOICE_FIELDS
),
}
)
def _apply_finish_choice_fields(self, choice: StreamingChoices) -> None:
_apply_choice_fields(choice, self.received_finish_choice_fields.get(choice.index, _NO_UPSTREAM_CHOICE_FIELDS))
def _finish_sibling_choice(self, choice: StreamingChoices, finish_reason: str) -> None:
choice.finish_reason = map_finish_reason(finish_reason) # rebind-ok: chunks are built in place here
self._apply_finish_choice_fields(choice)
def _streams_past_the_pending_finish(
self, model_response: ModelResponseStream, response_obj: Mapping[str, object], upstream_index: int
) -> bool:
finishing_index: Final = model_response.choices[0].index
return (
not self.sent_last_chunk
and not response_obj.get("is_finished")
and upstream_index != finishing_index
and finishing_index in self.received_finish_choice_fields
)
def _take_carried_chunk_choice_fields(
self, response_obj: Mapping[str, object], chunk_choice_fields: Mapping[str, object]
) -> Mapping[str, object]:
if not self.carried_chunk_choice_fields:
return chunk_choice_fields
upstream_index: Final = _response_choice_index(response_obj)
carried: Final = self.carried_chunk_choice_fields.get(upstream_index)
if carried is None:
return chunk_choice_fields
self.carried_chunk_choice_fields = MappingProxyType(
{index: fields for index, fields in self.carried_chunk_choice_fields.items() if index != upstream_index}
)
return MappingProxyType(
{
field: [*object_items(carried.get(field)), *object_items(chunk_choice_fields.get(field))]
for field in STREAM_CHUNK_CHOICE_FIELDS
if field in carried or field in chunk_choice_fields
}
)
def _another_choice_fields_chunk(
self,
model_response: ModelResponseStream,
response_obj: Mapping[str, object],
chunk_choice_fields: Mapping[str, object],
) -> ModelResponseStream | None:
upstream_index: Final = _response_choice_index(response_obj)
is_finished: Final = bool(response_obj.get("is_finished"))
if is_finished and upstream_index == model_response.choices[0].index and not self.sent_last_chunk:
return None
finish_fields: Final = (
self.received_finish_choice_fields.get(upstream_index, _NO_UPSTREAM_CHOICE_FIELDS)
if is_finished
else _NO_UPSTREAM_CHOICE_FIELDS
)
if not chunk_choice_fields and all(value is None for value in finish_fields.values()):
return None
return _choice_fields_chunk(
model_response, upstream_index, MappingProxyType({**finish_fields, **chunk_choice_fields})
)
def finish_reason_handler(self):
model_response: Final = self.model_response_creator(
hidden_params=None if self.received_model_id is None else {"received_model_id": self.received_model_id}
@ -2005,6 +2190,7 @@ class CustomStreamWrapper:
model_response.choices[0].finish_reason = self._normalize_tool_call_finish_reason(
_finish_reason if _finish_reason is not None else "stop"
)
self._apply_finish_choice_fields(model_response.choices[0])
close_tag: Final = self._close_open_think_block()
if close_tag is not None:
model_response.choices[0].delta.content = close_tag

View file

@ -990,7 +990,11 @@ def _count_extra(
tool_section (Optional[ChatTemplateToolSection]): How the model's chat template renders tools, if known.
"""
reply_primer_tokens: Final = 3
tool_choice_tokens: Final = 0 if tool_section is not None else _count_tool_choice(count_function, tool_choice)
tool_choice_tokens: Final = (
_count_tool_choice(count_function, tool_choice)
if tool_section is None or tool_section.counts_tool_choice
else 0
)
return (
reply_primer_tokens
+ _count_tool_definitions(count_function, tools, includes_system_message, tool_section)
@ -1029,14 +1033,29 @@ def _count_tool_choice(
return 0
def _named_function_definitions(tools: Sequence[object]) -> Iterable[Mapping[str, object]]:
return (function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name"))
def sends_function_tools(tools: Sequence[object] | None) -> bool:
return tools is not None and any(_named_function_definitions(tools))
def _without_false_strict(function: Mapping[str, object]) -> Mapping[str, object]:
return {key: value for key, value in function.items() if key != "strict" or value}
def _count_chat_template_tools(
count_function: TokenCounterFunction,
tools: Sequence[object],
includes_system_message: bool,
tool_section: ChatTemplateToolSection,
) -> int:
functions: Final = tuple(
function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name")
named_functions: Final = tuple(_named_function_definitions(tools))
functions: Final = (
tuple(map(_without_false_strict, named_functions))
if tool_section.renders_strict_only_when_true
else named_functions
)
if not functions:
return 0

View file

@ -1,13 +1,16 @@
from collections.abc import Callable, Mapping
from collections.abc import Callable, Mapping, Sequence
from copy import copy
from dataclasses import dataclass
from functools import reduce
from typing import Annotated, Final, TypeAlias
from typing import Annotated, Final, TypeAlias, TypeVar
from pydantic import BeforeValidator, ConfigDict, Json, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
ROUTES_WITH_TOOLS: Final = frozenset(
{
@ -162,3 +165,145 @@ def resolve_deployment_tool_fields(
if "proxy_server_request" not in kwargs or route not in ROUTES_WITH_TOOLS:
return kwargs
return resolve_tool_fields(kwargs, lambda: (drop_policy(kwargs),))
_ToolT: Final = TypeVar("_ToolT")
HOSTED_MCP_DROP_WARNING: Final = (
"Dropping the hosted MCP tool(s) %s for model=%s on %s (drop_params=True) because this route cannot run a "
"hosted MCP tool."
)
def _hosted_mcp_server_label(tool: object) -> str | None:
match tool:
case {"type": "mcp", "server_label": str() as server_label}:
return server_label
case {"type": "mcp"}:
return "unlabeled"
case _:
return None
def _drop_params_hint(dropped: str) -> str:
return (
f"To drop {dropped} instead, set `litellm.drop_params=True` or for proxy:\n\n"
"`litellm_settings:\n drop_params: true`\n"
)
def _one_log_line(text: str) -> str:
return text.replace("\r", "").replace("\n", "")
def _unrunnable_mcp_error(message: str, model: str, route: str) -> UnsupportedParamsError:
return UnsupportedParamsError(message=message, model=model, llm_provider=route, status_code=400)
def tools_without_hosted_mcp(tools: Sequence[_ToolT], model: str, route: str, drop_params: bool) -> Sequence[_ToolT]:
server_labels: Final = tuple(label for label in map(_hosted_mcp_server_label, tools) if label is not None)
if not server_labels:
return tools
named_servers: Final = ", ".join(f"`{label}`" for label in server_labels)
if not drops_unsupported_params(drop_params):
raise _unrunnable_mcp_error(
f'{model} on {route} cannot run the hosted MCP tool {named_servers} (`type: "mcp"`). To have LiteLLM '
"run the MCP server for this model, register it on the LiteLLM MCP gateway and send "
'`server_url: "litellm_proxy/mcp/<server name>"`. ' + _drop_params_hint("hosted MCP tools"),
model=model,
route=route,
)
verbose_logger.warning(
HOSTED_MCP_DROP_WARNING, _one_log_line(named_servers), _one_log_line(model), _one_log_line(route)
)
return [tool for tool in tools if _hosted_mcp_server_label(tool) is None]
MCP_SERVERS_DROP_WARNING: Final = (
"Dropping mcp_servers %s and their mcp_toolset tools for model=%s on %s (drop_params=True) because LiteLLM "
"cannot pass mcp_servers to this route."
)
ROUTES_WITHOUT_MCP_SERVERS: Final = frozenset(
{"bedrock", "databricks", "deepseek", "gemini", "gigachat", "ollama", "ollama_chat", "vertex_ai", "vertex_ai_beta"}
)
def _mcp_server_name(server: object) -> str:
match server:
case {"name": str() as name}:
return name
case _:
return "unnamed"
def _mcp_server_names(mcp_servers: object) -> tuple[str, ...]:
if not mcp_servers:
return ()
if not isinstance(mcp_servers, list):
return ("unnamed",)
return tuple(map(_mcp_server_name, _TOOLS.validate_python(mcp_servers)))
def _is_mcp_toolset(tool: object) -> bool:
match tool:
case {"type": "mcp_toolset"}:
return True
case _:
return False
def drops_mcp_servers(request_kwargs: Mapping[str, object]) -> bool:
additional_drop_params: Final = request_kwargs.get("additional_drop_params")
return drops_unsupported_params(request_kwargs.get("drop_params")) or (
isinstance(additional_drop_params, list) and "mcp_servers" in additional_drop_params
)
def tools_without_mcp_servers(
mcp_servers: object, tools: Sequence[_ToolT], model: str, route: str, drop_params: bool
) -> Sequence[_ToolT]:
server_names: Final = _mcp_server_names(mcp_servers)
if not server_names:
return tools
named_servers: Final = ", ".join(f"`{name}`" for name in server_names)
if not drops_unsupported_params(drop_params):
raise _unrunnable_mcp_error(
f"LiteLLM cannot pass `mcp_servers` to {model} on {route}, so the MCP server {named_servers} would not "
'run. To have LiteLLM run it for this model, register it on the LiteLLM MCP gateway and send a `type: "mcp"` '
'tool with `server_url: "litellm_proxy/mcp/<server name>"`. ' + _drop_params_hint("`mcp_servers`"),
model=model,
route=route,
)
verbose_logger.warning(
MCP_SERVERS_DROP_WARNING, _one_log_line(named_servers), _one_log_line(model), _one_log_line(route)
)
return [tool for tool in tools if not _is_mcp_toolset(tool)]
_TOOL_SELECTION_PARAMS: Final = frozenset({"tools", "tool_choice", "parallel_tool_calls"})
_ParamsT: Final = TypeVar("_ParamsT", bound=dict[str, object])
def tool_params_without_hosted_mcp(
params: _ParamsT, model: str, route: str, drop_params: bool, runs_hosted_mcp: Callable[[], bool] | None = None
) -> _ParamsT:
tools: Final = params.get("tools")
if not isinstance(tools, list) or not tools:
return params
sent_tools: Final = _TOOLS.validate_python(tools)
if all(_hosted_mcp_server_label(tool) is None for tool in sent_tools) or (
runs_hosted_mcp is not None and runs_hosted_mcp()
):
return params
kept_tools: Final = tools_without_hosted_mcp(tools=sent_tools, model=model, route=route, drop_params=drop_params)
if len(kept_tools) == len(sent_tools):
return params
kept_params: Final = copy(params)
if kept_tools:
kept_params["tools"] = list(kept_tools)
return kept_params
dropped_keys: Final = ("tools",) if params.get("functions") else _TOOL_SELECTION_PARAMS
for key in dropped_keys:
_ = kept_params.pop(key, None)
return kept_params

View file

@ -13,6 +13,7 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm.anthropic_beta_headers_manager import get_unsupported_headers
from litellm.constants import (
ANTHROPIC_BILLING_METADATA_PREFIX,
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS,
@ -56,6 +57,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
split_leading_system_run,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_non_string_reasoning_effort
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
from litellm.llms.base_llm.base_utils import (
ResponseFormatSchema,
as_json_object,
@ -1315,6 +1317,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
*(tool for tool in tools[first_search_tool_index + 1 :] if not _is_openai_tool_search_tool(tool)),
]
def runs_hosted_mcp(self) -> bool:
return ANTHROPIC_BETA_HEADER_VALUES.MCP_CLIENT_2025_04_04.value not in get_unsupported_headers(
self._resolved_provider
)
def _tools_param_with_openai_tool_search_mapped(
self, non_default_params: Mapping[str, object], model: str, drop_params: bool
) -> Mapping[str, object]:
@ -1903,7 +1910,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
model: str,
drop_params: bool,
) -> dict:
is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params=non_default_params)
request_params: Final = tool_params_without_hosted_mcp(
non_default_params,
model=model,
route=self._resolved_provider,
drop_params=drop_params,
runs_hosted_mcp=self.runs_hosted_mcp,
)
is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params=request_params)
# NB: ``map_openai_params`` deliberately does NOT sanitize tool names
# here. Names are the *original* OpenAI names at this stage, and must
@ -1916,7 +1930,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# via ``data = {**optional_params}`` and would 400 with
# ``Extra inputs are not permitted``).
for param, value in non_default_params.items():
for param, value in request_params.items():
if param == "max_tokens" or param == "max_completion_tokens":
optional_params["max_tokens"] = value if isinstance(value, int) else max(1, int(round(value)))
elif param == "tools":
@ -1930,8 +1944,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
optional_params["mcp_servers"] = mcp_servers
elif param == "tool_choice" or param == "parallel_tool_calls":
_tool_choice: AnthropicMessagesToolChoice | None = self._map_tool_choice(
tool_choice=non_default_params.get("tool_choice"),
parallel_tool_use=non_default_params.get("parallel_tool_calls"),
tool_choice=request_params.get("tool_choice"),
parallel_tool_use=request_params.get("parallel_tool_calls"),
)
if _tool_choice is not None:
@ -1957,7 +1971,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if self.uses_native_structured_output_for_schema(
model,
self._extract_json_schema_from_response_format(value),
REQUEST_PARAMS.validate_python(non_default_params),
REQUEST_PARAMS.validate_python(request_params),
):
_output_format = self.map_response_format_to_anthropic_output_format(value)
if _output_format is not None:
@ -1993,7 +2007,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# `thinking={type: enabled, budget_tokens}` interface a
# pre-4.6 model actually supports instead of forwarding a
# shape the model will reject.
max_tokens = non_default_params.get("max_completion_tokens") or non_default_params.get("max_tokens")
max_tokens = request_params.get("max_completion_tokens") or request_params.get("max_tokens")
legacy_thinking = AnthropicConfig._map_reasoning_effort(
reasoning_effort="medium",
model=model,
@ -2087,7 +2101,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
## handle thinking tokens
self.update_optional_params_with_thinking_tokens(
non_default_params=non_default_params, optional_params=optional_params
non_default_params=request_params, optional_params=optional_params
)
return optional_params

View file

@ -15,6 +15,11 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.litellm_core_utils.tool_fields import (
ROUTES_WITHOUT_MCP_SERVERS,
drops_mcp_servers,
tools_without_mcp_servers,
)
from litellm.llms.anthropic.messages.adapters.transformation import (
AnthropicAdapter,
)
@ -610,6 +615,23 @@ class LiteLLMMessagesToCompletionTransformationHandler:
Logging as LiteLLMLoggingObject,
)
provider_kwargs: Final = extra_kwargs or {}
custom_llm_provider: Final = provider_kwargs.get("custom_llm_provider")
target_provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None
drop_params: Final = drops_unsupported_params(provider_kwargs.get("drop_params"))
strips_mcp_servers: Final = target_provider in ROUTES_WITHOUT_MCP_SERVERS
request_tools: Final = (
tools_without_mcp_servers(
provider_kwargs.get("mcp_servers"),
tools or (),
model=model,
route=target_provider,
drop_params=drops_mcp_servers(provider_kwargs),
)
if strips_mcp_servers
else tools
)
request_data: Final[dict[str, object]] = {
"model": model,
"messages": messages,
@ -626,10 +648,10 @@ class LiteLLMMessagesToCompletionTransformationHandler:
request_data["temperature"] = temperature
if thinking:
request_data["thinking"] = thinking
if tool_choice:
if tool_choice and (request_tools or not tools):
request_data["tool_choice"] = tool_choice
if tools:
request_data["tools"] = tools
if request_tools:
request_data["tools"] = request_tools
if top_k is not None:
request_data["top_k"] = top_k
if top_p is not None:
@ -647,16 +669,15 @@ class LiteLLMMessagesToCompletionTransformationHandler:
if "output_config" in extra_kwargs:
request_data["output_config"] = extra_kwargs["output_config"]
custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider")
api_base: Final = extra_kwargs.get("api_base")
(
openai_request,
tool_name_mapping,
) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping(
request_data,
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
custom_llm_provider=target_provider,
api_base=api_base if isinstance(api_base, str) else None,
drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")),
drop_params=drop_params,
)
if openai_request is None:
@ -684,7 +705,11 @@ class LiteLLMMessagesToCompletionTransformationHandler:
# Maintainability: when adding a new Anthropic-only request param to
# ``AnthropicMessagesRequestOptionalParams``, also extend
# ``ANTHROPIC_ONLY_REQUEST_KEYS`` here so it doesn't silently leak.
excluded_keys: Final = ANTHROPIC_ONLY_REQUEST_KEYS | CONVERTED_STREAM_KEYS | {"anthropic_messages"}
excluded_keys: Final = (
ANTHROPIC_ONLY_REQUEST_KEYS
| CONVERTED_STREAM_KEYS
| ({"anthropic_messages", "mcp_servers"} if strips_mcp_servers else {"anthropic_messages"})
)
# NOTE: extra_kwargs was already coerced from None to {} at the top of
# this method (line ~220). It is guaranteed to be a dict here.
for key, value in extra_kwargs.items():

View file

@ -493,10 +493,23 @@ class _CombinedChunkSplitter:
for position, block in enumerate(blocks)
)
@staticmethod
def _without_sibling_finish(chunk: "ModelResponseStream") -> "ModelResponseStream":
choices: Final = _optional_attr_sequence(chunk, "choices")
if (
len(choices) != 1
or _optional_attr(choices[0], "index") in (None, 0)
or _optional_attr(choices[0], "finish_reason") is None
):
return chunk
unfinished: Final = copy.deepcopy(chunk)
unfinished.choices[0].finish_reason = None
return unfinished
def _expand(self, chunk: "ModelResponseStream") -> "tuple[ModelResponseStream, ...]":
split_chunks: Final = tuple(
split_chunk
for combined_chunk in self._split(chunk)
for combined_chunk in self._split(self._without_sibling_finish(chunk))
for split_chunk in self._split_by_payload_kind(combined_chunk) # comprehension-ok: flattens nested splits
)
block_chunks: Final = tuple(itertools.chain.from_iterable(map(self._split_thinking_blocks, split_chunks)))

View file

@ -24,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
prompt_cache_breakpoint_with_readable_mode,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.tool_fields import drops_mcp_servers, tools_without_mcp_servers
from litellm.responses.utils import auto_reasoning_summary
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
from litellm.types.llms.anthropic import (
@ -272,6 +273,14 @@ def _build_responses_kwargs(
"""
forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs)
reject_malformed_tool_result_text(messages, model, forwarded_kwargs)
target_provider: Final = forwarded_kwargs.get("custom_llm_provider")
request_tools: Final = tools_without_mcp_servers(
forwarded_kwargs.get("mcp_servers"),
tools or (),
model=model,
route=target_provider if isinstance(target_provider, str) else "this route",
drop_params=drops_mcp_servers(forwarded_kwargs),
)
block_rewrite: Final = _prompt_cache_block_rewrite(model, forwarded_kwargs)
request_messages: Final[AnthropicRequestMessages] = (
[
@ -304,10 +313,10 @@ def _build_responses_kwargs(
request_data["temperature"] = temperature
if thinking:
request_data["thinking"] = thinking
if tool_choice:
if tool_choice and (request_tools or not tools):
request_data["tool_choice"] = tool_choice
if tools:
request_data["tools"] = tools
if request_tools:
request_data["tools"] = list(request_tools)
if top_p is not None:
request_data["top_p"] = top_p
if output_format:
@ -347,7 +356,7 @@ def _build_responses_kwargs(
responses_kwargs["stream"] = True
# Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.)
excluded: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages"}
excluded: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages", "mcp_servers"}
for key, value in forwarded_kwargs.items():
if key == "litellm_logging_obj" and value is not None:
from litellm.litellm_core_utils.litellm_logging import (

View file

@ -164,7 +164,7 @@ class BaseGoogleGenAIGenerateContentConfig(ABC):
tools: ToolConfigDict | None,
generate_content_config_dict: dict,
system_instruction: object | None = None,
) -> dict:
) -> dict[str, object]:
"""
Transform the request parameters for the generate content API.

View file

@ -52,6 +52,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
parts_of,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
from litellm.llms.anthropic.chat.transformation import (
DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING,
DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING,
@ -1068,20 +1069,23 @@ class AmazonConverseConfig(BaseConfig):
drop_params: bool,
model_info: Mapping[str, object] | None = None,
) -> dict:
is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params)
request_params: Final = tool_params_without_hosted_mcp(
non_default_params, model=model, route="bedrock", drop_params=drop_params
)
is_thinking_enabled: Final = self.is_thinking_enabled(request_params)
base_model: Final = BedrockModelInfo.get_base_model(model)
drop_thinking_param: Final = self._is_deepseek_model(model=model, base_model=base_model)
drop_reasoning_effort_param: Final = self._model_rejects_reasoning_effort_param(
model=model, base_model=base_model
)
for param, value in non_default_params.items():
for param, value in request_params.items():
if param == "response_format" and isinstance(value, dict):
optional_params = self._translate_response_format_param(
value=value,
model=model,
optional_params=optional_params,
non_default_params=non_default_params,
non_default_params=request_params,
is_thinking_enabled=is_thinking_enabled,
model_info=model_info,
)
@ -1135,7 +1139,7 @@ class AmazonConverseConfig(BaseConfig):
),
),
model=model,
non_default_params=non_default_params,
non_default_params=request_params,
optional_params=optional_params,
)
if param == "tool_choice":
@ -1160,7 +1164,7 @@ class AmazonConverseConfig(BaseConfig):
and value.get("type") == "adaptive"
and not AnthropicConfig.is_adaptive_thinking_model(model, "bedrock")
):
max_tokens = non_default_params.get("max_completion_tokens") or non_default_params.get("max_tokens")
max_tokens = request_params.get("max_completion_tokens") or request_params.get("max_tokens")
legacy_thinking = AnthropicConfig.map_reasoning_effort(
reasoning_effort="medium",
model=model,
@ -1230,7 +1234,7 @@ class AmazonConverseConfig(BaseConfig):
# Nova 2 handles token budgeting differently through reasoningConfig
if "gpt-oss" not in model and not self._is_nova_2_model(model):
self.update_optional_params_with_thinking_tokens(
non_default_params=non_default_params, optional_params=optional_params
non_default_params=request_params, optional_params=optional_params
)
final_is_thinking_enabled: Final = self.is_thinking_enabled(optional_params)

View file

@ -30,6 +30,9 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig):
def should_strip_billing_metadata(self) -> bool:
return True
def runs_hosted_mcp(self) -> bool:
return True
def validate_environment(
self,
headers: dict,

View file

@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, Final
import httpx
import litellm
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.types.llms.cohere import CohereV2ChatResponse
@ -248,6 +249,9 @@ class CohereV2ChatConfig(OpenAIGPTConfig):
prompt_tokens: Final = token_usage.get("input_tokens", 0)
completion_tokens: Final = token_usage.get("output_tokens", 0)
model_response.choices[0].finish_reason = map_finish_reason( # rebind-ok: ModelResponse is filled in place
cohere_v2_chat_response.get("finish_reason", "COMPLETE")
)
model_response.created = int(time.time())
model_response.model = model
usage: Final = Usage(

View file

@ -455,6 +455,20 @@ def _google_genai_streaming_hidden_params(
}
def _log_generate_content_request(
logging_obj: LiteLLMLoggingObj,
contents: object,
api_base: str,
headers: Mapping[str, object],
body: Mapping[str, object],
) -> None:
logging_obj.pre_call(
input=contents,
api_key="",
additional_args={"complete_input_dict": body, "api_base": api_base, "headers": headers},
)
@lru_cache(maxsize=None)
def _responses_api_optional_request_param_names() -> frozenset[str]:
return frozenset(get_type_hints(ResponsesAPIOptionalRequestParams).keys())
@ -12443,6 +12457,10 @@ class BaseLLMHTTPHandler:
from litellm.google_genai.streaming_iterator import (
GoogleGenAIGenerateContentStreamingIterator,
)
from litellm.llms.vertex_ai.gemini.rejected_thought_signature_retry import (
post_with_accepted_thought_signatures_sync,
with_placeholder_native_thought_signatures,
)
if _is_async:
return self.async_generate_content_handler(
@ -12494,27 +12512,19 @@ class BaseLLMHTTPHandler:
if extra_body:
data.update(extra_body)
## LOGGING
logging_obj.pre_call(
input=contents,
api_key="",
additional_args={
"complete_input_dict": data,
"api_base": api_base,
"headers": headers,
},
)
log_request: Final = partial(_log_generate_content_request, logging_obj, contents, api_base, headers)
log_request(data)
def post(body: Mapping[str, object]) -> httpx.Response:
return sync_httpx_client.post(
url=api_base, headers=headers, json=dict(body), timeout=timeout, stream=stream
)
try:
sent_body, response = post_with_accepted_thought_signatures_sync(
data, with_placeholder_native_thought_signatures, None, post, log_request
)
if stream:
response = sync_httpx_client.post(
url=api_base,
headers=headers,
json=data,
timeout=timeout,
stream=True,
)
# Return streaming iterator
return GoogleGenAIGenerateContentStreamingIterator(
response=response,
model=model,
@ -12522,7 +12532,7 @@ class BaseLLMHTTPHandler:
generate_content_provider_config=generate_content_provider_config,
litellm_metadata=litellm_metadata or {},
custom_llm_provider=custom_llm_provider,
request_body=data,
request_body=dict(sent_body),
hidden_params=_google_genai_streaming_hidden_params(
api_base=api_base,
litellm_params=litellm_params,
@ -12530,13 +12540,6 @@ class BaseLLMHTTPHandler:
response_headers=response.headers,
),
)
else:
response = sync_httpx_client.post(
url=api_base,
headers=headers,
json=data,
timeout=timeout,
)
except Exception as e:
raise self._handle_error(
e=e,
@ -12574,6 +12577,10 @@ class BaseLLMHTTPHandler:
from litellm.google_genai.streaming_iterator import (
AsyncGoogleGenAIGenerateContentStreamingIterator,
)
from litellm.llms.vertex_ai.gemini.rejected_thought_signature_retry import (
post_with_accepted_thought_signatures_async,
with_placeholder_native_thought_signatures,
)
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
@ -12609,27 +12616,19 @@ class BaseLLMHTTPHandler:
if extra_body:
data.update(extra_body)
## LOGGING
logging_obj.pre_call(
input=contents,
api_key="",
additional_args={
"complete_input_dict": data,
"api_base": api_base,
"headers": headers,
},
)
log_request: Final = partial(_log_generate_content_request, logging_obj, contents, api_base, headers)
log_request(data)
async def post(body: Mapping[str, object]) -> httpx.Response:
return await async_httpx_client.post(
url=api_base, headers=headers, json=dict(body), timeout=timeout, stream=stream
)
try:
sent_body, response = await post_with_accepted_thought_signatures_async(
data, with_placeholder_native_thought_signatures, None, post, log_request
)
if stream:
response = await async_httpx_client.post(
url=api_base,
headers=headers,
json=data,
timeout=timeout,
stream=True,
)
# Return async streaming iterator
return AsyncGoogleGenAIGenerateContentStreamingIterator(
response=response,
model=model,
@ -12637,7 +12636,7 @@ class BaseLLMHTTPHandler:
generate_content_provider_config=generate_content_provider_config,
litellm_metadata=litellm_metadata or {},
custom_llm_provider=custom_llm_provider,
request_body=data,
request_body=dict(sent_body),
hidden_params=_google_genai_streaming_hidden_params(
api_base=api_base,
litellm_params=litellm_params,
@ -12645,13 +12644,6 @@ class BaseLLMHTTPHandler:
response_headers=response.headers,
),
)
else:
response = await async_httpx_client.post(
url=api_base,
headers=headers,
json=data,
timeout=timeout,
)
except Exception as e:
raise self._handle_error(
e=e,

View file

@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
strip_name_from_message,
strip_prefill_prefix_field,
)
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.base_utils import (
@ -467,7 +468,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
use_ai_gateway: Final = self.uses_ai_gateway(model)
return join_api_base(self._get_api_base(api_base, use_ai_gateway=use_ai_gateway), "chat/completions")
def get_supported_openai_params(self, model: str | None = None) -> list:
def get_supported_openai_params(self, model: str | None = None) -> list[str]:
return [
"stream",
"stop",
@ -573,8 +574,13 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
drop_params: bool,
replace_max_completion_tokens_with_max_tokens: bool = True,
) -> dict:
is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params)
mapped_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
request_params: Final = (
tool_params_without_hosted_mcp(non_default_params, model=model, route="databricks", drop_params=drop_params)
if "claude" in model
else non_default_params
)
is_thinking_enabled: Final = self.is_thinking_enabled(request_params)
mapped_params: Final = super().map_openai_params(request_params, optional_params, model, drop_params)
if "claude" in model:
AnthropicConfig.translate_legacy_thinking_for_adaptive_model(
model=model, optional_params=mapped_params, custom_llm_provider="databricks", drop_params=drop_params
@ -583,16 +589,16 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
mapped_params["tools"] = self._map_openai_to_dbrx_tool(
model=model, tools=mapped_params["tools"], drop_params=drop_params
)
if "max_completion_tokens" in non_default_params and replace_max_completion_tokens_with_max_tokens:
mapped_params["max_tokens"] = non_default_params[
if "max_completion_tokens" in request_params and replace_max_completion_tokens_with_max_tokens:
mapped_params["max_tokens"] = request_params[
"max_completion_tokens"
] # most openai-compatible providers support 'max_tokens' not 'max_completion_tokens'
mapped_params.pop("max_completion_tokens", None)
if "response_format" in non_default_params and "claude" in model:
if "response_format" in request_params and "claude" in model:
_tool: Final = self.map_response_format_to_databricks_tool(
model,
non_default_params["response_format"],
request_params["response_format"],
mapped_params,
is_thinking_enabled,
)
@ -610,8 +616,8 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
"response_format", None
) # unsupported for claude models - if json_schema -> convert to tool call
if "reasoning_effort" in non_default_params and self._uses_anthropic_thinking_param(model):
reasoning_effort_value: Final = non_default_params.get("reasoning_effort")
if "reasoning_effort" in request_params and self._uses_anthropic_thinking_param(model):
reasoning_effort_value: Final = request_params.get("reasoning_effort")
mapped_thinking: Final = AnthropicConfig._map_reasoning_effort(
reasoning_effort=reasoning_effort_value,
model=model,
@ -637,7 +643,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
optional_params.pop("reasoning_effort", None)
## handle thinking tokens
self.update_optional_params_with_thinking_tokens(
non_default_params=non_default_params, optional_params=mapped_params
non_default_params=request_params, optional_params=mapped_params
)
gated_thinking: Final = strip_beta_gated_thinking(
mapped_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # mapped_params is the base config's bare dict

View file

@ -12,6 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_search_results_text,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
@ -33,17 +34,15 @@ def _warn_on_dropped_budget(thinking: object, model: str) -> None:
class DeepSeekChatConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> list:
def get_supported_openai_params(self, model: str) -> list[str]:
"""
DeepSeek reasoner models support thinking parameter.
"""
params: Final = super().get_supported_openai_params(model)
params.extend(["thinking", "reasoning_effort"])
return params
return [*super().get_supported_openai_params(model), "thinking", "reasoning_effort"]
def map_openai_params(
self,
non_default_params: dict,
non_default_params: dict[str, object],
optional_params: dict,
model: str,
drop_params: bool,
@ -59,7 +58,12 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
Reference: https://api-docs.deepseek.com/guides/thinking_mode
"""
# Let parent handle standard params first
optional_params = super().map_openai_params(non_default_params, optional_params, model, drop_params)
optional_params = super().map_openai_params(
tool_params_without_hosted_mcp(non_default_params, model=model, route="deepseek", drop_params=drop_params),
optional_params,
model,
drop_params,
)
# Pop thinking/reasoning_effort from optional_params first (parent may have added them)
# Then re-add only if valid for DeepSeek
@ -265,6 +269,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
tools: Final = optional_params.get("tools")
if not isinstance(tools, list) or not tools:
return optional_params
sent_tools: Final[tuple[object, ...]] = tuple(tools)
def _is_function_tool(tool: object) -> bool:
return isinstance(tool, dict) and tool.get("type") == "function"
@ -289,14 +294,14 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
name: Final = function.get("name")
return isinstance(name, str) and name in function_tool_names
function_tools: Final = [tool for tool in tools if _is_function_tool(tool)]
if len(function_tools) == len(tools):
function_tools: Final = [tool for tool in sent_tools if _is_function_tool(tool)]
if len(function_tools) == len(sent_tools):
return optional_params
dropped_types: Final = sorted(
{
str(tool.get("type")) if isinstance(tool, dict) else type(tool).__name__
for tool in tools
for tool in sent_tools
if not _is_function_tool(tool)
}
)

View file

@ -1,5 +1,5 @@
from collections.abc import Mapping
from typing import Final, cast
from typing import ClassVar, Final, cast
import litellm
from litellm.litellm_core_utils.prompt_templates.factory import (
@ -49,6 +49,8 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
Note: Please make sure to modify the default parameters as required for your use case.
"""
_route: ClassVar[str] = "gemini"
temperature: float | None = None
max_output_tokens: int | None = None
top_p: float | None = None

View file

@ -17,6 +17,7 @@ import httpx
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.gigachat.utils import convert_usage, get_api_base
@ -169,7 +170,10 @@ class GigaChatConfig(BaseConfig):
drop_params: bool,
) -> dict: # mutable-ok: base class contract returns dict
"""Map OpenAI parameters to GigaChat parameters."""
for param, value in non_default_params.items():
request_params: Final = tool_params_without_hosted_mcp(
dict(non_default_params), model=model, route="gigachat", drop_params=drop_params
)
for param, value in request_params.items():
if param == "stream":
optional_params["stream"] = value
elif param == "temperature":

View file

@ -4,7 +4,7 @@ from collections.abc import AsyncIterator, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final, cast
from httpx._models import Headers, Response
from pydantic import BaseModel
from pydantic import BaseModel, TypeAdapter
import litellm
from litellm._uuid import uuid
@ -15,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_reasoning_content,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@ -40,6 +41,8 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
_REQUEST_PARAMS: Final = TypeAdapter(dict[str, object])
class OllamaChatConfig(BaseConfig):
"""
@ -152,7 +155,13 @@ class OllamaChatConfig(BaseConfig):
model: str,
drop_params: bool,
) -> dict:
for param, value in non_default_params.items():
request_params: Final = tool_params_without_hosted_mcp(
_REQUEST_PARAMS.validate_python(non_default_params),
model=model,
route="ollama_chat",
drop_params=drop_params,
)
for param, value in request_params.items():
if param == "max_tokens" or param == "max_completion_tokens":
optional_params["num_predict"] = value
if param == "stream":

View file

@ -1,5 +1,5 @@
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Final
from typing import TYPE_CHECKING, ClassVar, Final
import httpx
from pydantic import TypeAdapter
@ -159,6 +159,7 @@ class OpenCodeGoMessagesChatConfig(_GoReportedCostMixin, AnthropicConfig):
class OpenCodeGoGeminiChatConfig(_GoReportedCostMixin, GoogleAIStudioGeminiConfig):
go_usage_key: str = "usageMetadata"
_route: ClassVar[str] = "opencode_go"
@property
def supports_stream_param_in_request_body(self) -> bool:

View file

@ -1,4 +1,4 @@
from collections.abc import Awaitable, Callable
from collections.abc import Awaitable, Callable, Mapping, Sequence
from copy import deepcopy
from functools import partial
from typing import Final, TypeVar
@ -17,16 +17,27 @@ Built: Final = TypeVar("Built")
Sent: Final = TypeVar("Sent")
def cache_rebuild(
def _build_from_copies(
build: Callable[..., Built], messages: Sequence[AllMessageValues], optional_params: Mapping[str, object]
) -> Built:
return build(messages=deepcopy(list(messages)), optional_params=independent_snapshot(dict(optional_params)))
def cache_rebuilds(
build: Callable[..., Built], messages: list[AllMessageValues], optional_params: dict
) -> Callable[[], Built] | None:
"""Copies the inputs before the first transform, which pops keys from ``optional_params`` and ``messages``."""
) -> tuple[Callable[[], Built], Callable[[], Built]] | tuple[None, None]:
"""Returns one build with a fresh cache and one with the history inline, both over a snapshot of the inputs
taken before the first transform, which pops keys from ``optional_params`` and ``messages``. Each call
copies the snapshot again, since every transform consumes the inputs it is given."""
if not any(map(is_cached_message, messages)):
return None
return partial(build, messages=deepcopy(messages), optional_params=independent_snapshot(optional_params))
return None, None
from_snapshot: Final = partial(
_build_from_copies, messages=deepcopy(messages), optional_params=independent_snapshot(optional_params)
)
return partial(from_snapshot, build), partial(from_snapshot, partial(build, context_caching=False))
def _status_and_message(error: httpx.HTTPStatusError | BaseLLMException) -> tuple[int, str]:
def status_and_message(error: httpx.HTTPStatusError | BaseLLMException) -> tuple[int, str]:
match error:
case httpx.HTTPStatusError():
return error.response.status_code, error.response.text
@ -35,7 +46,7 @@ def _status_and_message(error: httpx.HTTPStatusError | BaseLLMException) -> tupl
def _cache_rejected(body: RequestBody, error: httpx.HTTPStatusError | BaseLLMException) -> bool:
return body.get("cachedContent") is not None and names_rejected_cache(*_status_and_message(error))
return body.get("cachedContent") is not None and names_rejected_cache(*status_and_message(error))
async def post_with_fresh_cache_async(

View file

@ -425,6 +425,7 @@ class ContextCachingEndpoints(VertexBase):
vertex_auth_header: str | None,
extra_headers: dict | None = None,
cached_content: str | None = None,
context_caching: bool = True,
) -> tuple[list[AllMessageValues], dict, str | None]:
"""
Receives
@ -436,7 +437,7 @@ class ContextCachingEndpoints(VertexBase):
Follows - https://ai.google.dev/api/caching#request-body
"""
if cached_content is not None:
if cached_content is not None or not context_caching:
return messages, optional_params, cached_content
cached_messages, non_cached_messages = separate_cached_messages(messages=messages)
@ -573,6 +574,7 @@ class ContextCachingEndpoints(VertexBase):
vertex_auth_header: str | None,
extra_headers: dict | None = None,
cached_content: str | None = None,
context_caching: bool = True,
) -> tuple[list[AllMessageValues], dict, str | None]:
"""
Receives
@ -584,7 +586,7 @@ class ContextCachingEndpoints(VertexBase):
Follows - https://ai.google.dev/api/caching#request-body
"""
if cached_content is not None:
if cached_content is not None or not context_caching:
return messages, optional_params, cached_content
cached_messages, non_cached_messages = separate_cached_messages(messages=messages)

View file

@ -0,0 +1,235 @@
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from typing import Final, TypeVar
import httpx
from pydantic import TypeAdapter, ValidationError
from litellm import verbose_logger
from litellm.litellm_core_utils.prompt_templates.factory import get_dummy_thought_signature
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.vertex_ai import ContentType, PartType, RequestBody
from ..context_caching.rejected_cache_retry import status_and_message
Body: Final = TypeVar("Body", bound=Mapping[str, object])
Sent: Final = TypeVar("Sent")
_REJECTED_SIGNATURE_MARKER: Final = "thought signature"
_SIGNATURE_KEYS: Final = frozenset({"thoughtSignature", "thought_signature"})
_ENTRIES: Final = TypeAdapter(list[dict[str, object]])
def names_rejected_thought_signature(status_code: int, message: str) -> bool:
"""A signature only validates on the backend that minted it. AI Studio answers a replayed Vertex signature
with ``400 Corrupted thought signature.`` and Vertex answers an AI Studio one with ``400 Invalid thought
signature.``"""
return status_code == 400 and _REJECTED_SIGNATURE_MARKER in message.lower()
def _is_rejected_signature(error: httpx.HTTPStatusError | BaseLLMException) -> bool:
return names_rejected_thought_signature(*status_and_message(error))
def _entries(value: object) -> Sequence[Mapping[str, object]]:
try:
return _ENTRIES.validate_python(value)
except ValidationError:
return []
def _foreign_signatures(body: Mapping[str, object], placeholder: str) -> Iterator[object]:
for content in _entries(body.get("contents")):
for part in _entries(content.get("parts")):
yield from (
value for key, value in part.items() if key in _SIGNATURE_KEYS and value and value != placeholder
)
def _foreign_signature_count(body: Mapping[str, object]) -> int:
return sum(1 for _ in _foreign_signatures(body, get_dummy_thought_signature()))
def _part_with_placeholder(part: PartType, placeholder: str) -> PartType:
if not part.get("thoughtSignature"):
return part
return {**part, "thoughtSignature": placeholder}
def _content_with_placeholders(content: ContentType, placeholder: str) -> ContentType:
return {**content, "parts": [_part_with_placeholder(part, placeholder) for part in content["parts"]]}
def with_placeholder_thought_signatures(body: RequestBody) -> RequestBody | None:
"""Swaps every thought signature in the history for Google's documented skip-validator placeholder, or
returns ``None`` when the body carries nothing the backend could have rejected."""
placeholder: Final = get_dummy_thought_signature()
if next(_foreign_signatures(body, placeholder), None) is None:
return None
return {**body, "contents": [_content_with_placeholders(content, placeholder) for content in body["contents"]]}
def _native_part_with_placeholder(part: Mapping[str, object], placeholder: str) -> Mapping[str, object]:
return {key: placeholder if key in _SIGNATURE_KEYS and value else value for key, value in part.items()}
def _native_content_with_placeholders(content: Mapping[str, object], placeholder: str) -> Mapping[str, object]:
parts: Final = _entries(content.get("parts"))
if not parts:
return content
return {**content, "parts": [_native_part_with_placeholder(part, placeholder) for part in parts]}
def with_placeholder_native_thought_signatures(body: Mapping[str, object]) -> Mapping[str, object] | None:
"""The same swap on a ``generateContent`` body as the client sent it, whose parts may spell the key
``thoughtSignature`` or ``thought_signature``."""
placeholder: Final = get_dummy_thought_signature()
if next(_foreign_signatures(body, placeholder), None) is None:
return None
return {
**body,
"contents": [
_native_content_with_placeholders(content, placeholder) for content in _entries(body.get("contents"))
],
}
async def _inline_history_async(body: Body, uncached: Callable[[], Awaitable[Body]] | None) -> Body | None:
if uncached is None or body.get("cachedContent") is None:
return None
inline: Final = await uncached()
return inline if _foreign_signature_count(inline) > _foreign_signature_count(body) else None
def _inline_history_sync(body: Body, uncached: Callable[[], Body] | None) -> Body | None:
if uncached is None or body.get("cachedContent") is None:
return None
inline: Final = uncached()
return inline if _foreign_signature_count(inline) > _foreign_signature_count(body) else None
def _resend_body(signed: Body, placeholders: Callable[[Body], Body | None]) -> Body | None:
resend_body: Final = placeholders(signed)
if resend_body is not None:
verbose_logger.warning(
"Gemini rejected a replayed thought signature, resending with the skip_thought_signature_validator"
" placeholder in place of the replayed signatures"
)
return resend_body
async def build_with_accepted_thought_signatures_async(
build: Callable[[], Awaitable[Body]],
placeholders: Callable[[Body], Body | None],
uncached: Callable[[], Awaitable[Body]] | None,
) -> Body:
"""Vertex validates the signatures in a cache's contents when it creates the cache, so a rejected one there
sends the history inline with the placeholder instead."""
try:
return await build()
except BaseLLMException as error:
if uncached is None or not _is_rejected_signature(error):
raise
inline_body: Final = _resend_body(await uncached(), placeholders)
if inline_body is None:
raise
return inline_body
def build_with_accepted_thought_signatures_sync(
build: Callable[[], Body],
placeholders: Callable[[Body], Body | None],
uncached: Callable[[], Body] | None,
) -> Body:
try:
return build()
except BaseLLMException as error:
if uncached is None or not _is_rejected_signature(error):
raise
inline_body: Final = _resend_body(uncached(), placeholders)
if inline_body is None:
raise
return inline_body
async def _post_inline_async(
body: Body,
placeholders: Callable[[Body], Body | None],
uncached: Callable[[], Awaitable[Body]] | None,
post: Callable[[Body], Awaitable[Sent]],
log_request: Callable[[Body], None],
rejection: httpx.HTTPStatusError | BaseLLMException,
) -> tuple[Body, Sent]:
inline: Final = await _inline_history_async(body, uncached)
resend_body: Final = None if inline is None else _resend_body(inline, placeholders)
if resend_body is None:
raise rejection
log_request(resend_body)
return resend_body, await post(resend_body)
def _post_inline_sync(
body: Body,
placeholders: Callable[[Body], Body | None],
uncached: Callable[[], Body] | None,
post: Callable[[Body], Sent],
log_request: Callable[[Body], None],
rejection: httpx.HTTPStatusError | BaseLLMException,
) -> tuple[Body, Sent]:
inline: Final = _inline_history_sync(body, uncached)
resend_body: Final = None if inline is None else _resend_body(inline, placeholders)
if resend_body is None:
raise rejection
log_request(resend_body)
return resend_body, post(resend_body)
async def post_with_accepted_thought_signatures_async(
body: Body,
placeholders: Callable[[Body], Body | None],
uncached: Callable[[], Awaitable[Body]] | None,
post: Callable[[Body], Awaitable[Sent]],
log_request: Callable[[Body], None],
) -> tuple[Body, Sent]:
"""Innermost retry around the raw post, so a cache rebuild (which re-transforms the messages and brings
the rejected signatures back) is itself wrapped by it. The resend keeps the cache and swaps the signatures
the body carries, and sends the history inline only when that is rejected too and the cache holds one.
Returns the body the backend accepted."""
try:
return body, await post(body)
except (httpx.HTTPStatusError, BaseLLMException) as error:
if not _is_rejected_signature(error):
raise
resend_body: Final = _resend_body(body, placeholders)
if resend_body is None:
return await _post_inline_async(body, placeholders, uncached, post, log_request, error)
log_request(resend_body)
try:
return resend_body, await post(resend_body)
except (httpx.HTTPStatusError, BaseLLMException) as error:
if not _is_rejected_signature(error):
raise
return await _post_inline_async(body, placeholders, uncached, post, log_request, error)
def post_with_accepted_thought_signatures_sync(
body: Body,
placeholders: Callable[[Body], Body | None],
uncached: Callable[[], Body] | None,
post: Callable[[Body], Sent],
log_request: Callable[[Body], None],
) -> tuple[Body, Sent]:
try:
return body, post(body)
except (httpx.HTTPStatusError, BaseLLMException) as error:
if not _is_rejected_signature(error):
raise
resend_body: Final = _resend_body(body, placeholders)
if resend_body is None:
return _post_inline_sync(body, placeholders, uncached, post, log_request, error)
log_request(resend_body)
try:
return resend_body, post(resend_body)
except (httpx.HTTPStatusError, BaseLLMException) as error:
if not _is_rejected_signature(error):
raise
return _post_inline_sync(body, placeholders, uncached, post, log_request, error)

View file

@ -1256,6 +1256,7 @@ def sync_transform_request_body(
vertex_project: str | None,
vertex_location: str | None,
vertex_auth_header: str | None,
context_caching: bool = True,
) -> RequestBody:
from ..context_caching.vertex_ai_context_caching import ContextCachingEndpoints
@ -1275,6 +1276,7 @@ def sync_transform_request_body(
timeout=timeout,
extra_headers=extra_headers,
cached_content=optional_params.pop("cached_content", None),
context_caching=context_caching,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
vertex_project=vertex_project,
@ -1390,6 +1392,7 @@ async def async_transform_request_body(
vertex_project: str | None,
vertex_location: str | None,
vertex_auth_header: str | None,
context_caching: bool = True,
) -> RequestBody:
from ..context_caching.vertex_ai_context_caching import ContextCachingEndpoints
@ -1409,6 +1412,7 @@ async def async_transform_request_body(
timeout=timeout,
extra_headers=extra_headers,
cached_content=optional_params.pop("cached_content", None),
context_caching=context_caching,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
vertex_project=vertex_project,

View file

@ -9,7 +9,7 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence
from copy import deepcopy
from functools import lru_cache, partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, Protocol, Union, cast, get_args
import httpx
from pydantic import JsonValue, TypeAdapter
@ -34,6 +34,7 @@ from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAcc
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
from litellm.litellm_core_utils.prompt_templates.factory import encode_tool_call_id_with_signature
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -103,7 +104,7 @@ from ..common_utils import (
)
from ..context_caching.known_cached_contents import known_cached_contents
from ..context_caching.rejected_cache_retry import (
cache_rebuild,
cache_rebuilds,
post_with_fresh_cache_async,
post_with_fresh_cache_sync,
)
@ -134,6 +135,13 @@ from .long_decode import (
with_initial_prompt_tokens,
with_spans_shifted,
)
from .rejected_thought_signature_retry import (
build_with_accepted_thought_signatures_async,
build_with_accepted_thought_signatures_sync,
post_with_accepted_thought_signatures_async,
post_with_accepted_thought_signatures_sync,
with_placeholder_thought_signatures,
)
from .transformation import (
async_transform_request_body,
gemini_convert_messages_with_history,
@ -614,6 +622,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
Note: Please make sure to modify the default parameters as required for your use case.
"""
_route: ClassVar[str] = "vertex_ai"
temperature: float | None = None
max_output_tokens: int | None = None
top_p: float | None = None
@ -1667,9 +1677,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
drop_params: bool,
allow_integer_enums: bool = True,
) -> dict:
self._apply_include_server_side_tool_invocations(non_default_params, optional_params)
request_params: Final = tool_params_without_hosted_mcp(
non_default_params, model=model, route=self._route, drop_params=drop_params
)
self._apply_include_server_side_tool_invocations(request_params, optional_params)
supports_sampling_params: Final = self.supports_sampling_params(model)
for param, value in non_default_params.items():
for param, value in request_params.items():
if param in ("temperature", "top_p", "top_k") and not supports_sampling_params:
continue
if param == "temperature":
@ -1733,7 +1746,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
if _tool_choice_value is not None:
optional_params["tool_choice"] = _tool_choice_value
elif param == "parallel_tool_calls":
tools_list = non_default_params.get("tools", non_default_params.get("functions"))
tools_list = request_params.get("tools", request_params.get("functions"))
num_tools = len(tools_list) if isinstance(tools_list, list) else 0
# Gemini does not support parallel_tool_calls=False with multiple
# tools. Drop the param instead of failing — Responses API clients
@ -3494,23 +3507,17 @@ class VertexLLM(VertexBase):
litellm_params=litellm_params,
)
rebuild: Final = cache_rebuild(
partial(
async_transform_request_body,
**data,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_auth_header=auth_header,
),
messages,
optional_params,
)
request_body: Final = await async_transform_request_body(
transform: Final = partial(
async_transform_request_body,
**data,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_auth_header=auth_header,
)
rebuild, uncached = cache_rebuilds(transform, messages, optional_params)
request_body: Final = await build_with_accepted_thought_signatures_async(
transform, with_placeholder_thought_signatures, uncached
)
## LOGGING
log_request: Final = partial(_log_generate_request, logging_obj, messages, api_base, headers)
@ -3539,7 +3546,14 @@ class VertexLLM(VertexBase):
async def post(body: RequestBody) -> ModelResponseIterator:
return await open_stream(client=client, data=json.dumps(body), request_body=body)
return (await post_with_fresh_cache_async(request_body, rebuild, post, log_request))[1]
accepted: Final = partial(
post_with_accepted_thought_signatures_async,
placeholders=with_placeholder_thought_signatures,
uncached=uncached,
post=post,
log_request=log_request,
)
return (await post_with_fresh_cache_async(request_body, rebuild, accepted, log_request))[1][1]
streaming_response: Final = CustomStreamWrapper(
completion_stream=None,
@ -3606,23 +3620,17 @@ class VertexLLM(VertexBase):
litellm_params=litellm_params,
)
rebuild: Final = cache_rebuild(
partial(
async_transform_request_body,
**data,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_auth_header=auth_header,
),
messages,
optional_params,
)
request_body: Final = await async_transform_request_body(
transform: Final = partial(
async_transform_request_body,
**data,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_auth_header=auth_header,
)
rebuild, uncached = cache_rebuilds(transform, messages, optional_params)
request_body: Final = await build_with_accepted_thought_signatures_async(
transform, with_placeholder_thought_signatures, uncached
)
_async_client_params: Final = {}
if timeout:
@ -3663,7 +3671,14 @@ class VertexLLM(VertexBase):
)
return response
sent_body, first_response = await post_with_fresh_cache_async(request_body, rebuild, post, log_request)
accepted: Final = partial(
post_with_accepted_thought_signatures_async,
placeholders=with_placeholder_thought_signatures,
uncached=uncached,
post=post,
log_request=log_request,
)
_, (sent_body, first_response) = await post_with_fresh_cache_async(request_body, rebuild, accepted, log_request)
return _transform_long_decode_hops(
model=model,
hops=await collect_hops_async(parse_body(first_response), sent_body, post),
@ -3795,23 +3810,17 @@ class VertexLLM(VertexBase):
)
## TRANSFORMATION ##
rebuild: Final = cache_rebuild(
partial(
sync_transform_request_body,
**transform_request_params,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_auth_header=auth_header,
),
messages,
optional_params,
)
data: Final = sync_transform_request_body(
transform: Final = partial(
sync_transform_request_body,
**transform_request_params,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_auth_header=auth_header,
)
rebuild, uncached = cache_rebuilds(transform, messages, optional_params)
data: Final = build_with_accepted_thought_signatures_sync(
transform, with_placeholder_thought_signatures, uncached
)
## LOGGING
log_request: Final = partial(_log_generate_request, logging_obj, messages, url, headers)
@ -3839,7 +3848,14 @@ class VertexLLM(VertexBase):
def post(body: RequestBody) -> ModelResponseIterator:
return open_stream(client=client, data=json.dumps(body), request_body=body)
return post_with_fresh_cache_sync(data, rebuild, post, log_request)[1]
accepted: Final = partial(
post_with_accepted_thought_signatures_sync,
placeholders=with_placeholder_thought_signatures,
uncached=uncached,
post=post,
log_request=log_request,
)
return post_with_fresh_cache_sync(data, rebuild, accepted, log_request)[1][1]
streaming_response: Final = CustomStreamWrapper(
completion_stream=None,
@ -3883,7 +3899,14 @@ class VertexLLM(VertexBase):
)
return response
sent_body, first_response = post_with_fresh_cache_sync(data, rebuild, post, log_request)
accepted: Final = partial(
post_with_accepted_thought_signatures_sync,
placeholders=with_placeholder_thought_signatures,
uncached=uncached,
post=post,
log_request=log_request,
)
_, (sent_body, first_response) = post_with_fresh_cache_sync(data, rebuild, accepted, log_request)
return _transform_long_decode_hops(
model=model,
hops=collect_hops_sync(parse_body(first_response), sent_body, post),

View file

@ -13478,6 +13478,10 @@
"input_cost_per_token": 2e-07,
"output_cost_per_token": 5e-07,
"litellm_provider": "azure_ai",
"token_count_framing": 149,
"token_count_tool_framing": 302,
"token_count_responses_framing": 145,
"token_count_responses_tool_framing": 100,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"mode": "chat",
@ -13831,7 +13835,7 @@
"input_cost_per_token_flex": 3.7e-07,
"input_cost_per_token_priority": 1.295e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
@ -13845,7 +13849,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/ap-northeast-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -14002,7 +14006,7 @@
"input_cost_per_token_flex": 3.7e-07,
"input_cost_per_token_priority": 1.295e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
@ -14016,7 +14020,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/ap-south-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -14140,12 +14144,12 @@
"input_cost_per_token": 7.4e-07,
"input_cost_per_token_flex": 3.7e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
"output_cost_per_token_flex": 1.11e-06,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html",
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md",
"supports_audio_input": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
@ -14250,7 +14254,7 @@
"input_cost_per_token_flex": 3.7e-07,
"input_cost_per_token_priority": 1.295e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
@ -14264,7 +14268,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/eu-north-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -14786,7 +14790,7 @@
"input_cost_per_token_flex": 3.7e-07,
"input_cost_per_token_priority": 1.295e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
@ -14800,7 +14804,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/sa-east-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -15035,7 +15039,7 @@
"input_cost_per_token_flex": 3.1e-07,
"input_cost_per_token_priority": 1.085e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
@ -15049,7 +15053,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/us-east-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -15156,7 +15160,7 @@
"input_cost_per_token_flex": 3.1e-07,
"input_cost_per_token_priority": 1.085e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
@ -15170,7 +15174,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/us-east-2/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -15552,6 +15556,7 @@
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md"
},
"bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 2.65e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 8000,
@ -15562,6 +15567,7 @@
"supports_pdf_input": true
},
"bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 8000,
@ -15708,7 +15714,7 @@
"input_cost_per_token_flex": 3.1e-07,
"input_cost_per_token_priority": 1.085e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
@ -15722,7 +15728,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/us-west-2/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -23519,12 +23525,12 @@
]
},
"deepgram/streaming/nova-3": {
"input_cost_per_second": 8e-05,
"input_cost_per_second": 0.00012833,
"litellm_provider": "deepgram",
"metadata": {
"calculation": "$0.0048/60 seconds = $0.00008000 per second",
"calculation": "$0.0077/60 seconds = $0.00012833 per second",
"note": "Nova-3 monolingual streaming, pay as you go",
"original_pricing_per_minute": 0.0048
"original_pricing_per_minute": 0.0077
},
"mode": "audio_transcription",
"output_cost_per_second": 0.0,
@ -23533,13 +23539,28 @@
"/v1/listen"
]
},
"deepgram/streaming/nova-3-multilingual": {
"input_cost_per_second": 9.667e-05,
"deepgram/streaming/nova-3-medical": {
"input_cost_per_second": 0.00012833,
"litellm_provider": "deepgram",
"metadata": {
"calculation": "$0.0058/60 seconds = $0.00009667 per second",
"calculation": "$0.0077/60 seconds = $0.00012833 per second",
"note": "Nova-3 Medical streaming, launch price from Deepgram's 2025-03-03 announcement. deepgram.com/pricing does not list Nova-3 Medical, and this matches its current Nova-3 monolingual streaming pay as you go rate",
"original_pricing_per_minute": 0.0077
},
"mode": "audio_transcription",
"output_cost_per_second": 0.0,
"source": "https://www.businesswire.com/news/home/20250303470439/en",
"supported_endpoints": [
"/v1/listen"
]
},
"deepgram/streaming/nova-3-multilingual": {
"input_cost_per_second": 0.00015333,
"litellm_provider": "deepgram",
"metadata": {
"calculation": "$0.0092/60 seconds = $0.00015333 per second",
"note": "Nova-3 multilingual (language=multi) streaming, pay as you go",
"original_pricing_per_minute": 0.0058
"original_pricing_per_minute": 0.0092
},
"mode": "audio_transcription",
"output_cost_per_second": 0.0,
@ -24519,7 +24540,7 @@
"input_cost_per_token_flex": 3.1e-07,
"input_cost_per_token_priority": 1.085e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 164000,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
@ -24530,7 +24551,7 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json",
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md",
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
@ -54457,6 +54478,8 @@
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"token_count_framing": 185,
"token_count_tool_framing": 214,
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"mode": "chat",
@ -62640,7 +62663,9 @@
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"token_count_framing": 23,
"token_count_tool_framing": 12
},
"deepseek/deepseek-v4-flash-vision-exp": {
"cache_creation_input_token_cost": 0.0,
@ -62746,7 +62771,9 @@
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": false
"supports_vision": false,
"token_count_framing": 76,
"token_count_tool_framing": 75
},
"tencent/deepseek-v4-pro": {
"cache_creation_input_token_cost": 0.0,
@ -64032,6 +64059,7 @@
"input_cost_per_token": 3e-07,
"input_cost_per_token_priority": 3.75e-07,
"litellm_provider": "fireworks_ai",
"token_count_framing": 23,
"max_input_tokens": 1048576,
"max_output_tokens": 393216,
"mode": "chat",
@ -64133,6 +64161,7 @@
"input_cost_per_token": 3e-07,
"input_cost_per_token_priority": 3.75e-07,
"litellm_provider": "fireworks_ai",
"token_count_framing": 23,
"max_input_tokens": 1048576,
"max_output_tokens": 393216,
"mode": "chat",
@ -76801,6 +76830,7 @@
"supports_audio_input": true,
"supports_audio_output": false,
"supports_computer_use": true,
"supports_continuation": true,
"supports_function_calling": true,
"supports_minimal_reasoning_effort": false,
"supports_native_streaming": true,
@ -76860,6 +76890,7 @@
],
"supports_audio_input": true,
"supports_computer_use": true,
"supports_continuation": true,
"supports_function_calling": true,
"supports_minimal_reasoning_effort": false,
"supports_native_streaming": true,
@ -76917,6 +76948,7 @@
"text"
],
"supports_audio_input": true,
"supports_continuation": true,
"supports_function_calling": true,
"supports_minimal_reasoning_effort": false,
"supports_native_streaming": true,

View file

@ -125,6 +125,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
NO_TEAM_MEMBERSHIP_SENTINEL,
TAG_REGISTRY_OVERFLOW_SENTINEL,
UserApiKeyCache,
budget_windows_absent_cache_key,
end_user_cache_key,
end_user_restricted_registry_cache_key,
get_management_object_ttl,
@ -3471,11 +3472,22 @@ async def delete_cache_key_object(
)
await publish_auth_cache_invalidation(cache_key=key)
await evict_budget_windows_absent_marker(
entity_type=Litellm_EntityType.KEY, entity_id=hashed_token, user_api_key_cache=user_api_key_cache
)
_delete_cache_key_object: Final = delete_cache_key_object
async def evict_budget_windows_absent_marker(
entity_type: Litellm_EntityType, entity_id: str, user_api_key_cache: UserApiKeyCache
) -> None:
marker: Final = budget_windows_absent_cache_key(entity_type.value, entity_id)
evict_local(cache_keys=(marker,), user_api_key_cache=user_api_key_cache)
await publish_auth_cache_invalidation(cache_key=marker)
async def delete_cache_key_objects(
hashed_tokens: Sequence[str],
user_api_key_cache: UserApiKeyCache,

View file

@ -770,9 +770,10 @@ def codex_model_catalog(
A model Codex does not know gets the fallback
entry, with the same base instructions Codex itself uses so the agent never
runs without a system prompt, and offers the reasoning efforts `groups` gives its model group
that the installed Codex, at `client_version`, can parse. Apart from aliases, the catalog is
the one `/v1/models?client_version=...` serves, built from the installed Codex's own entries
instead of the proxy's vendored ones.
that the installed Codex, at `client_version`, can parse. Apart from aliases, and the window the
deployments' `model_info.max_input_tokens` gives a model Codex knows, the catalog is the one
`/v1/models?client_version=...` serves, built from the installed Codex's own entries instead of
the proxy's vendored ones.
"""
rows: Final = tuple(_codex_row(m, upstreams.get(m.id), groups) for m in models)
body: Final = codex_models_response_json(

View file

@ -183,7 +183,7 @@ def evict_local(cache_keys: Sequence[str], user_api_key_cache: "UserApiKeyCache"
def _drop_from_memory(cache_key: str, user_api_key_cache: "UserApiKeyCache") -> None:
user_api_key_cache.in_memory_cache_for(cache_key).delete_cache(cache_key)
user_api_key_cache.delete_from_memory(cache_key)
@with_service_target(AUTH_OBJECTS_TARGET)

View file

@ -7,8 +7,10 @@ never the OpenAI list shape. This module builds that body from the rows ``/v1/mo
lists: a model Codex knows keeps Codex's own entry for it, any other model gets the fallback entry
Codex itself uses for an unknown slug, and ``model_info.service_tiers`` becomes the entry's
``service_tiers``, which Codex turns into slash commands (``/ultrafast``) that send ``service_tier``
upstream. A fallback entry offers the model group's ``supported_reasoning_efforts`` as its reasoning
levels, trimmed to the ones the requesting Codex release can parse.
upstream. A ``model_info.max_input_tokens`` the operator sets replaces a known model's
``context_window``, up to the most Codex lets its own ``model_context_window`` raise that model to.
A fallback entry offers the model group's ``supported_reasoning_efforts`` as its reasoning levels,
trimmed to the ones the requesting Codex release can parse.
The stock entries are Codex 0.159.3's bundled catalog, vendored unchanged from
https://github.com/openai/codex/blob/rust-v0.159.3/codex-rs/models-manager/models.json (Apache-2.0)
@ -158,6 +160,7 @@ class CodexStockModel(LiteLLMBaseModel):
supports_search_tool: bool = False
supported_reasoning_levels: tuple[CodexReasoningLevel, ...] = ()
context_window: int | None = None
max_context_window: int | None = None
class CodexStockCatalog(LiteLLMBaseModel):
@ -173,8 +176,9 @@ class CodexCatalogRow:
"""What the catalog needs to know about one listed model: its public id, the listing's mode and
input limit, the upstream model behind it, the reasoning effort its deployments pin, and the
`model_info` values the operator set; `service_tiers` holds one raw `model_info.service_tiers`
value per deployment behind the id, None where a deployment sets none. `supported_reasoning_efforts`
is the model group's, None when unknown."""
value per deployment behind the id, None where a deployment sets none, and
`configured_max_input_tokens` one `model_info.max_input_tokens` per deployment the same way.
`supported_reasoning_efforts` is the model group's, None when unknown."""
id: str
mode: str | None = None
@ -185,6 +189,7 @@ class CodexCatalogRow:
service_tiers: tuple[object, ...] = ()
supported_reasoning_efforts: tuple[str, ...] | None = None
search_tool_support: tuple[bool | None, ...] = ()
configured_max_input_tokens: tuple[int | None, ...] = ()
@cache
@ -333,6 +338,25 @@ def _alias_context_window(row: CodexCatalogRow, stock_model: CodexStockModel) ->
return MappingProxyType({"context_window": min(stock_model.context_window, row.max_input_tokens)})
def _configured_window(row: CodexCatalogRow, stock_model: CodexStockModel) -> Mapping[str, int]:
"""The `context_window` the operator's `model_info.max_input_tokens` gives a known model's entry,
or nothing, which keeps `stock_model`'s own.
A request to the row can route to any of its deployments, so the window is the smallest one of them
takes: its `max_input_tokens` where the operator set a positive one, Codex's own window where not. A
limit set on only some deployments can therefore lower the window but never raise it. The window never
goes past `stock_model`'s `max_context_window`, the most Codex lets its own `model_context_window`
setting raise that model to."""
per_deployment: Final = row.configured_max_input_tokens
declared: Final = tuple(limit for limit in per_deployment if limit is not None and limit > 0)
unset_window: Final = () if len(declared) == len(per_deployment) else (stock_model.context_window,)
if not declared or None in unset_window:
return MappingProxyType({})
smallest: Final = min(limit for limit in (*declared, *unset_window) if limit is not None)
ceiling: Final = stock_model.max_context_window
return MappingProxyType({"context_window": smallest if ceiling is None else min(smallest, ceiling)})
def _alias_reasoning_level(row: CodexCatalogRow, stock_model: CodexStockModel) -> str | None:
efforts: Final = frozenset(level.effort for level in stock_model.supported_reasoning_levels)
return row.reasoning_effort if row.reasoning_effort in efforts else None
@ -398,6 +422,7 @@ def _entry(
"upgrade": upgrade,
**({} if search_tool is None else {"supports_search_tool": search_tool}),
**(_deployment_defaults(row, stock_model) if is_alias and alias_defaults == "deployment" else {}),
**_configured_window(row, stock_model),
}
if configured is None:
return stock_model.model_copy(update=listing)
@ -473,10 +498,12 @@ def codex_models_response_json(
byte limit silently and keeps its bundled one, so the body holds only the entries that fit,
the ones offering a service tier taken first and the rest in listing order, an entry too large
for the bytes left passed over, every kept entry at its listing position; `byte_limit=None`
keeps every entry. A row listed under another name than the Codex model its `upstream_model`
names (an alias) takes that model's entry; `alias_defaults="deployment"` drops the defaults
Codex picks for that model in favor of what the alias's deployments pin, while `"codex"` keeps
them, as Codex would use them for the model's own name. `client_version` is the requesting Codex's version, which decides the reasoning
keeps every entry. A row whose deployments set `model_info.max_input_tokens` gets the smallest
window any of them takes as its known model's `context_window`, capped at that model's
`max_context_window`. A row listed under another name than the Codex model its `upstream_model` names (an alias) takes that model's
entry; `alias_defaults="deployment"` drops the defaults Codex picks for that model in favor of
what the alias's deployments pin, while `"codex"` keeps them, as Codex would use them for the
model's own name. `client_version` is the requesting Codex's version, which decides the reasoning
levels a fallback entry may offer; None counts as a version too old for custom levels.
"""
known: Final = bundled_codex_models() if stock is None else stock
@ -519,6 +546,9 @@ def _catalog_row(
search_tool_support=(
registry.resolve(lookup_id, kind="search_tool_support", team_id=team_id) if registry is not None else ()
),
configured_max_input_tokens=(
registry.group(lookup_id, field="max_input_tokens", team_id=team_id) if registry is not None else ()
),
)
@ -532,9 +562,9 @@ def codex_catalog_rows(
"""`rows` joined with the router's configured metadata, looked up by each entry's internal id so
team-scoped rows resolve the way the Anthropic listing's display names do, a `model_group_alias`
reading its target's deployments under its own name; `team_id` is the requesting key's team, so the
upstream model that picks Codex's stock entry and the tiers are read only off the deployments its
requests can route to, and the reasoning efforts are the ones `/model_group/info` reports for the
entry. Display names are read under `label_entries`' ids when given."""
upstream model that picks Codex's stock entry, the tiers, and the configured input limits are read
only off the deployments its requests can route to, and the reasoning efforts are the ones
`/model_group/info` reports for the entry. Display names are read under `label_entries`' ids when given."""
lookup_ids: Final = MappingProxyType(dict(entries))
label_ids: Final = lookup_ids if label_entries is None else MappingProxyType(dict(label_entries))
return tuple(

View file

@ -56,6 +56,22 @@ class TokenCountSettings:
framing: int | None
count_model: TokenCountModel | None
tokenizer: TokenCountTokenizer | None = None
tool_framing: int | None = None
responses_framing: int | None = None
responses_tool_framing: int | None = None
@property
def frames_counts(self) -> bool:
return any(
framing is not None
for framing in (self.framing, self.tool_framing, self.responses_framing, self.responses_tool_framing)
)
def hidden_prompt_tokens(self, carries_tools: bool, responses_api: bool) -> int:
chat: Final = self.tool_framing if carries_tools else None
responses: Final = self.responses_tool_framing if carries_tools else self.responses_framing
narrower: Final = responses if responses_api else chat
return (narrower if narrower is not None else self.framing) or 0
@dataclass(frozen=True, slots=True)
@ -105,6 +121,15 @@ def _framing(value: object) -> int | None:
return None
def _first_framing(source: Mapping[str, object], keys: tuple[str, ...]) -> int | None:
return next((framing for framing in map(_framing, map(source.get, keys)) if framing is not None), None)
def _configured_framing(configured: Mapping[str, object], default: Mapping[str, object], *keys: str) -> int | None:
operator_sets_one: Final = _first_framing(configured, (*keys, "token_count_framing")) is not None
return _first_framing(configured if operator_sets_one else default, keys)
def _non_blank(value: object) -> str | None:
return value if isinstance(value, str) and value.strip() else None
@ -175,13 +200,21 @@ def _tokenizer(value: object) -> TokenCountTokenizer | None:
def resolve_token_count_settings(
configured: Mapping[str, object], default: Mapping[str, object], deployment_region: Callable[[], str]
) -> TokenCountSettings:
framing: Final = _framing(configured.get("token_count_framing"))
return TokenCountSettings(
framing=framing if framing is not None else _framing(default.get("token_count_framing")),
framing=_configured_framing(configured, default, "token_count_framing"),
count_model=_resolved_count_model(
configured.get("token_count_model"), default.get("token_count_model"), deployment_region
),
tokenizer=_tokenizer(default.get("token_count_tokenizer")),
tool_framing=_configured_framing(configured, default, "token_count_tool_framing"),
responses_framing=_configured_framing(configured, default, "token_count_responses_framing"),
responses_tool_framing=_configured_framing(
configured,
default,
"token_count_responses_tool_framing",
"token_count_responses_framing",
"token_count_tool_framing",
),
)

View file

@ -24,6 +24,7 @@ T = TypeVar("T", bound=BaseModel)
from litellm.constants import AUTH_OBJECTS_TARGET as AUTH_OBJECTS_TARGET # noqa: E402, PLC0414 # public re-export
_HASHED_TOKEN_CACHE_KEY: Final = re.compile(r"[0-9a-f]{64}")
_BUDGET_WINDOWS_ABSENT_PREFIX: Final = "budget_windows_absent:"
def is_user_key_cache_key(key: str) -> bool:
@ -70,6 +71,7 @@ class UserApiKeyCache(DualCache):
default_in_memory_ttl: float | None = None,
default_redis_ttl: float | None = None,
key_object_in_memory_cache: InMemoryCache | None = None,
budget_windows_absent_in_memory_cache: InMemoryCache | None = None,
) -> None:
super().__init__(
in_memory_cache=in_memory_cache,
@ -83,19 +85,33 @@ class UserApiKeyCache(DualCache):
default_in_memory_ttl=default_in_memory_ttl,
default_redis_ttl=default_redis_ttl,
)
self.budget_windows_absent_cache: Final = DualCache(
in_memory_cache=budget_windows_absent_in_memory_cache
or InMemoryCache(default_ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL),
default_in_memory_ttl=default_in_memory_ttl,
)
def in_memory_cache_for(self, key: str) -> InMemoryCache:
return self.key_object_cache.in_memory_cache if is_user_key_cache_key(key) else self.in_memory_cache
def delete_from_memory(self, key: str) -> None:
self.in_memory_cache_for(key).delete_cache(key)
if key.startswith(_BUDGET_WINDOWS_ABSENT_PREFIX):
self.budget_windows_absent_cache.delete_cache(key)
def update_cache_ttl(self, default_in_memory_ttl: float | None, default_redis_ttl: float | None) -> None:
super().update_cache_ttl(default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl)
self.key_object_cache.update_cache_ttl(
default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl
)
self.budget_windows_absent_cache.update_cache_ttl(
default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=None
)
def update_in_memory_max_size(self, max_size: int | None) -> None:
super().update_in_memory_max_size(max_size)
self.key_object_cache.update_in_memory_max_size(max_size)
self.budget_windows_absent_cache.update_in_memory_max_size(max_size)
def attach_redis_cache(
self, redis_cache: RedisCache | None = None, *, default_redis_ttl: float | None = None
@ -272,6 +288,7 @@ class UserApiKeyCache(DualCache):
def flush_cache(self) -> None:
super().flush_cache()
self.key_object_cache.in_memory_cache.flush_cache()
self.budget_windows_absent_cache.flush_cache()
async def async_set_cache_pipeline(
self, cache_list: Sequence[tuple[str, object]], local_only: bool = False, **kwargs: object
@ -400,6 +417,11 @@ def team_membership_reservation_cache_key(user_id: str, team_id: str) -> str:
NO_TEAM_MEMBERSHIP_SENTINEL: Final = "__no_team_membership__"
def budget_windows_absent_cache_key(entity_type: str, entity_id: str) -> str:
"""Cache key marking a key or team that holds no budget windows, so the spend fallback skips its row read."""
return f"{_BUDGET_WINDOWS_ABSENT_PREFIX}{entity_type}:{entity_id}"
def get_management_object_ttl(cache: DualCache) -> float:
"""
In-memory TTL for management-object cache writes (keys, teams, users, budgets, ...).

View file

@ -42,10 +42,12 @@ _EMPTY_ROWS: Final[Mapping[DbRow, Mapping[str, JsonValue]]] = MappingProxyType({
class SettingsStore(MutableMapping[str, JsonValue]):
def __init__(self, section: Section) -> None:
def __init__(self, section: Section, *, managed_db_keys: frozenset[str] | None = None) -> None:
self._section: Final = section
self._managed_db_keys: Final = managed_db_keys
self._yaml_values: Mapping[str, JsonValue] = _EMPTY_VALUES
self._database_rows: Mapping[DbRow, Mapping[str, JsonValue]] = _EMPTY_ROWS
self._unmanaged_db_keys: frozenset[str] = frozenset()
self._runtime_values: Mapping[str, JsonValue] = _EMPTY_VALUES
self._deleted_runtime_keys: frozenset[str] = frozenset()
@ -77,15 +79,24 @@ class SettingsStore(MutableMapping[str, JsonValue]):
return self.owned_by_config(key) and self._db_value_is_shadowed(key)
def apply_db_row(self, row: DbRow, db_row: Mapping[str, JsonValue]) -> None:
applied_row: Final = MappingProxyType({key: db_row[key] for key in db_row if self._db_manages(row, key)})
previous_row: Final = self._database_rows.get(row, _EMPTY_VALUES)
changed: Final = frozenset(
key
for key in (*previous_row, *db_row)
if previous_row.get(key, ABSENT) != db_row.get(key, ABSENT) # pyright: ignore[reportUnknownArgumentType] # JsonValue vs Absent compare
for key in (*previous_row, *applied_row)
if previous_row.get(key, ABSENT) != applied_row.get(key, ABSENT) # pyright: ignore[reportUnknownArgumentType] # JsonValue vs Absent compare
)
self._database_rows = MappingProxyType({**self._database_rows, row: MappingProxyType(dict(db_row))})
self._database_rows = MappingProxyType({**self._database_rows, row: applied_row})
if row == self._section:
self._unmanaged_db_keys = frozenset(key for key in db_row if key not in applied_row)
self._clear_runtime_keys(changed)
def unmanaged_db_keys(self) -> tuple[str, ...]:
return tuple(sorted(self._unmanaged_db_keys))
def _db_manages(self, row: DbRow, key: str) -> bool:
return row != self._section or self._managed_db_keys is None or key in self._managed_db_keys
def resolved(self) -> Mapping[str, JsonValue]:
return MappingProxyType(dict(self))

View file

@ -107,9 +107,20 @@ class MCPSecurityGuardrail(CustomGuardrail):
from litellm.proxy.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
from litellm.proxy.proxy_server import prisma_client
async with global_mcp_server_manager.catalog.operation():
registry: Final = global_mcp_server_manager.get_registry()
registered_names: Final = set(registry.keys())
return frozenset(requested_servers - registered_names)
not_servers: Final = frozenset(
name
for name in requested_servers
if global_mcp_server_manager.get_mcp_server_by_identifier(name) is None
)
if prisma_client is None:
return not_servers
return frozenset(
[
name
for name in not_servers
if await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name) is None
]
)

View file

@ -7,22 +7,25 @@ instead of writing immediately on each request.
from collections.abc import AsyncGenerator, Callable, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final, TypeVar
from fastapi import HTTPException
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy._types import LitellmUserRoles
from litellm.proxy.common_utils.encrypt_decrypt_utils import (
decrypt_value_helper,
encrypt_value_helper,
)
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import (
BaseLiteLLMOpenAIResponseObject,
ResponsesAPIResponse,
)
from litellm.types.responses.main import DeleteResponseResult
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypesLiteral, LLMResponseTypes, SpecialEnums
if TYPE_CHECKING:
@ -67,6 +70,8 @@ def _proxy_signing_key() -> str | None:
_RESPONSE_PAYLOAD_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_ResponseObjectT = TypeVar("_ResponseObjectT", bound=BaseLiteLLMOpenAIResponseObject)
_AddressedResultT = TypeVar("_AddressedResultT", bound=ResponsesAPIResponse | DeleteResponseResult)
def _response_payload(response_obj: object) -> Mapping[str, object] | None:
@ -77,9 +82,9 @@ def _response_payload(response_obj: object) -> Mapping[str, object] | None:
def _rewrite_advertised_id(
event: BaseLiteLLMOpenAIResponseObject,
event: _ResponseObjectT,
rewrite: Callable[[str], str],
) -> BaseLiteLLMOpenAIResponseObject:
) -> _ResponseObjectT:
event_id: Final = getattr(event, "id", None)
if isinstance(event_id, str) and event_id.startswith("resp_"):
setattr(event, "id", rewrite(event_id))
@ -116,6 +121,35 @@ def _streams_response_ids(request_route: str | None) -> bool:
return canonical in _RESPONSES_API_ROUTES or (parent in _RESPONSES_API_ROUTES and response_id != "")
_NO_HELD_IDS: Final[Mapping[str, str]] = MappingProxyType({})
def _upstream_response_id(response_id: str) -> str:
return ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(response_id)
def _names_the_same_response(response_id: str | None, original_response_id: str) -> bool:
return response_id is not None and _upstream_response_id(response_id) == _upstream_response_id(original_response_id)
def _held_id_naming(response_id: str | None, held_ids: Mapping[str, str]) -> str | None:
return next(
(
held_id
for original_response_id, held_id in held_ids.items()
if _names_the_same_response(response_id, original_response_id)
),
None,
)
def _id_sent_back(response: ResponsesAPIResponse | DeleteResponseResult, held_ids: Mapping[str, str]) -> str | None:
held_id: Final = _held_id_naming(response.id, held_ids)
if held_id is not None:
return held_id
return response.id if isinstance(response, DeleteResponseResult) else None
class ResponsesIDSecurity(CustomLogger):
def __init__(
self,
@ -275,10 +309,18 @@ class ResponsesIDSecurity(CustomLogger):
def _encrypt_response_id(
self,
response: BaseLiteLLMOpenAIResponseObject,
response: _ResponseObjectT,
user_api_key_dict: "UserAPIKeyAuth",
request_cache: dict[str, str] | None = None,
) -> BaseLiteLLMOpenAIResponseObject:
) -> _ResponseObjectT:
encrypt: Final = self._response_id_encrypter(user_api_key_dict, request_cache)
return response if encrypt is None else _rewrite_advertised_id(response, encrypt)
def _response_id_encrypter(
self,
user_api_key_dict: "UserAPIKeyAuth",
request_cache: dict[str, str] | None,
) -> Callable[[str], str] | None:
signing_key: Final = self._get_signing_key()
if signing_key is None:
verbose_proxy_logger.debug(
@ -287,7 +329,7 @@ class ResponsesIDSecurity(CustomLogger):
"Skipping response ID encryption. "
"See: https://docs.litellm.ai/docs/proxy/prod#5-set-litellm-salt-key"
)
return response
return None
def encrypt(original_id: str) -> str:
cached: Final = request_cache.get(original_id) if request_cache is not None else None
@ -304,47 +346,73 @@ class ResponsesIDSecurity(CustomLogger):
request_cache[original_id] = encrypted_id
return encrypted_id
return _rewrite_advertised_id(response, encrypt)
return encrypt
def _streamed_response_id_advertiser(
self,
user_api_key_dict: "UserAPIKeyAuth",
request_data: Mapping[str, object],
) -> Callable[[str], str] | None:
if not _streams_response_ids(user_api_key_dict.request_route):
return None
if self._general_settings_reader().get("disable_responses_id_security", False):
return None
encrypt: Final = self._response_id_encrypter(user_api_key_dict, request_cache={})
if encrypt is None:
return None
logging_obj: Final = request_data.get("litellm_logging_obj")
held_ids: Final = self._ids_the_caller_holds(request_data.get(ADDRESSED_RESPONSE_ID_FIELD))
def advertise(original_id: str) -> str:
logged_id: Final = encrypt(original_id)
if isinstance(logging_obj, LiteLLMLoggingObj):
logging_obj.client_facing_response_id = logged_id
return _held_id_naming(original_id, held_ids) or logged_id
return advertise
def _ids_the_caller_holds(self, addressed_id: object) -> Mapping[str, str]:
if not isinstance(addressed_id, str) or not self._is_encrypted_response_id(addressed_id):
return _NO_HELD_IDS
original_response_id, _, _ = self._decrypt_response_id(addressed_id)
return MappingProxyType({original_response_id: addressed_id})
def _advertised_response(
self,
response: _AddressedResultT,
user_api_key_dict: "UserAPIKeyAuth",
request_data: Mapping[str, object],
) -> _AddressedResultT:
sent_back_id: Final = _id_sent_back(
response, self._ids_the_caller_holds(request_data.get(ADDRESSED_RESPONSE_ID_FIELD))
)
logged_response: Final = self._encrypt_response_id(response, user_api_key_dict, request_cache=None)
return logged_response if sent_back_id is None else logged_response.model_copy(update={"id": sent_back_id})
async def async_post_call_success_hook(
self,
data: dict,
data: dict[str, object],
user_api_key_dict: "UserAPIKeyAuth",
response: LLMResponseTypes,
) -> LLMResponseTypes:
"""
Queue response IDs for batch processing instead of writing directly to DB.
This method adds response IDs to an in-memory queue, which are then
batch-processed by the DBSpendUpdateWriter during regular database update cycles.
"""
general_settings: Final = self._general_settings_reader()
if general_settings.get("disable_responses_id_security", False):
return response
if isinstance(response, ResponsesAPIResponse):
response = cast(
ResponsesAPIResponse,
self._encrypt_response_id(response, user_api_key_dict, request_cache=None),
)
if isinstance(response, ResponsesAPIResponse | DeleteResponseResult):
return self._advertised_response(response, user_api_key_dict, data)
return response
async def async_post_call_streaming_iterator_hook(
self, user_api_key_dict: "UserAPIKeyAuth", response: Any, request_data: dict
self, user_api_key_dict: "UserAPIKeyAuth", response: Any, request_data: dict[str, object]
) -> AsyncGenerator[BaseLiteLLMOpenAIResponseObject, None]:
general_settings: Final = self._general_settings_reader()
# Create a request-scoped cache for consistent encryption across streaming chunks.
request_encryption_cache: Final[dict[str, str]] = {}
advertise: Final = self._streamed_response_id_advertiser(user_api_key_dict, request_data)
async for chunk in response:
if (
isinstance(chunk, BaseLiteLLMOpenAIResponseObject)
and _streams_response_ids(user_api_key_dict.request_route)
and not general_settings.get("disable_responses_id_security", False)
):
chunk = self._encrypt_response_id(chunk, user_api_key_dict, request_encryption_cache)
yield chunk
yield (
_rewrite_advertised_id(chunk, advertise)
if advertise is not None and isinstance(chunk, BaseLiteLLMOpenAIResponseObject)
else chunk
)
def decrypted_response_id(response_id: str) -> str:

View file

@ -37,6 +37,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
from litellm.proxy.route_llm_request import route_request
from litellm.types.images.main import ImageEditRequestParams
from litellm.types.llms.openai import ChatCompletionUserMessage
from litellm.types.utils import CallTypes
router: Final = APIRouter()
_IMAGE_GENERATION_MODEL_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object)
@ -221,7 +222,10 @@ async def image_generation(
return response
except Exception as e:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=data,
dispatched_call_type=CallTypes.aimage_generation.value,
)
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, HTTPException):

View file

@ -109,6 +109,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
can_org_access_model,
delete_cache_key_objects,
delete_cache_team_object,
evict_budget_windows_absent_marker,
get_jwt_key_mapping_cache_keys_for_tokens,
get_org_object,
get_team_membership,
@ -612,6 +613,9 @@ async def refresh_cached_team(
await publish_auth_cache_invalidation(cache_key=f"team_id:{team_row.team_id}")
if team_table.team_alias is not None:
await publish_auth_cache_invalidation(cache_key=f"team_alias:{team_table.team_alias}")
await evict_budget_windows_absent_marker(
entity_type=Litellm_EntityType.TEAM, entity_id=team_row.team_id, user_api_key_cache=user_api_key_cache
)
_refresh_cached_team: Final = refresh_cached_team

View file

@ -21,7 +21,7 @@ import httpx
from fastapi import FastAPI, HTTPException
from pydantic import ConfigDict, TypeAdapter, ValidationError
from starlette.requests import Request as StarletteRequest
from starlette.responses import JSONResponse
from starlette.responses import JSONResponse, Response
from starlette.routing import Route
from starlette.types import Message, Receive, Scope, Send
@ -2489,6 +2489,9 @@ if MCP_AVAILABLE:
},
)(scope, receive, send)
return
if scope.get("method") == "OPTIONS":
await Response(status_code=204, headers={"Allow": "GET, POST, DELETE, OPTIONS"})(scope, receive, send)
return
path: Final = _scope_path(scope)
peek: Final = _LazyPeekedBody(receive) if scope.get("method") == "POST" else None
if peek is not None:

View file

@ -26,6 +26,7 @@ from collections.abc import (
Collection,
Container,
Iterable,
Iterator,
Mapping,
MutableMapping,
Sequence,
@ -90,6 +91,7 @@ from litellm.constants import (
LITELLM_UI_SESSION_DURATION,
OPENAI_LIVE_SESSION_START_TIMEOUT_SECONDS,
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
VLLM_CHOICE_FIELDS,
)
from litellm.litellm_core_utils.access_windows import access_windows_config_error
from litellm.litellm_core_utils.add_retry_fallback_headers import (
@ -107,6 +109,7 @@ from litellm.litellm_core_utils.litellm_logging import (
initialize_sentry,
)
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER, LoggingWorker
from litellm.litellm_core_utils.prompt_templates.chat_template_tools import chat_template_tool_section
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
from litellm.litellm_core_utils.routing_groups import parse_routing_groups
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@ -115,6 +118,7 @@ from litellm.litellm_core_utils.token_counter import (
UnsupportedContentTypeError,
deployment_openai_model,
offload_token_count,
sends_function_tools,
)
from litellm.llms.bedrock.count_tokens.kimi_k3 import (
HUGGING_FACE_SOURCE,
@ -203,6 +207,7 @@ from litellm.rust_bridge.catalog import Target
from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.utils import (
PRICING_OVERRIDES_KEY,
CallTypes,
CountContextManagement,
LlmProviders,
ModelResponse,
@ -585,6 +590,7 @@ from litellm.proxy.common_utils.token_count_log import (
)
from litellm.proxy.common_utils.user_api_key_cache import (
UserApiKeyCache,
budget_windows_absent_cache_key,
end_user_cache_key,
get_management_object_ttl,
model_access_group_cache_key,
@ -1007,6 +1013,7 @@ from litellm.proxy.video_endpoints.endpoints import router as video_router
from litellm.repositories.base_repository import SupportsModelDump
from litellm.repositories.credentials_repository import CredentialsRepository
from litellm.repositories.prisma_protocols import TableActions
from litellm.responses.main import provider_stores_the_conversation
from litellm.responses.streaming_iterator import drain_logging_tasks
from litellm.router import (
AssistantsTypedDict,
@ -3632,6 +3639,52 @@ def _cached_budget_limits(cached: object) -> object:
return _UNTYPED_VALUE.validate_python(json.loads(stored)) if isinstance(stored, str) else stored
def _has_budget_windows(budget_limits: object) -> bool:
return isinstance(budget_limits, list) and budget_limits != []
def _budget_windows_absent_marker(absent_key: str) -> object:
return _UNTYPED_VALUE.validate_python(user_api_key_cache.budget_windows_absent_cache.get_cache(key=absent_key))
def _finish_pending_read(absent_key: str, pending_read: str, remember_absent: bool) -> None:
if _budget_windows_absent_marker(absent_key) != pending_read:
return
if remember_absent:
user_api_key_cache.budget_windows_absent_cache.set_cache(key=absent_key, value=True)
return
user_api_key_cache.budget_windows_absent_cache.delete_cache(absent_key)
async def _stored_budget_limits(entity_type: Litellm_EntityType, entity_id: str) -> object:
"""Read only: caching the row here could race a delete and put a just-deleted key back into the auth cache."""
if prisma_client is None:
return None
absent_key: Final = budget_windows_absent_cache_key(entity_type.value, entity_id)
if _budget_windows_absent_marker(absent_key) is True:
return None
pending_read: Final = f"reading:{secrets.token_hex(8)}"
user_api_key_cache.budget_windows_absent_cache.set_cache(key=absent_key, value=pending_read)
try:
row: Final[SupportsModelDump | None] = (
await VerificationTokenRepository(prisma_client).table.find_unique(where={"token": entity_id})
if entity_type == Litellm_EntityType.KEY
else await TeamRepository(prisma_client).table.find_unique(where={"team_id": entity_id})
)
except Exception as e: # noqa: BLE001 # spend tracking must never fail the cost callback
verbose_proxy_logger.warning(
"Unable to read budget windows for %s=%s; this request's window spend is not counted: %s",
entity_type.value,
entity_id,
e,
)
_finish_pending_read(absent_key, pending_read, remember_absent=False)
return None
budget_limits: Final[object] = None if row is None else _cached_budget_limits(row)
_finish_pending_read(absent_key, pending_read, remember_absent=not _has_budget_windows(budget_limits))
return budget_limits
async def _increment_spend_counters_batched(
token: str | None,
team_id: str | None,
@ -3662,7 +3715,68 @@ async def _increment_spend_counters_batched(
cost: Final[float] = response_cost
async def _key_scope(key_token: str) -> tuple[PendingSpendIncrement | BaseException, ...]:
async def _window_increment(
entity_type: Litellm_EntityType, entity_id: str, window: object
) -> PendingSpendIncrement | None:
duration: Final = (
window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None)
)
window_reset_at: Final = (
window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None)
)
window_counter: Final = f"spend:{entity_type.value}:{entity_id}:window:{duration}"
window_start: Final = get_budget_window_start(window)
pending_window: Final = (
await _prepare_window_spend_counter_increment(
counter_key=window_counter,
entity_type=entity_type.value.capitalize(),
entity_id=entity_id,
window_duration=duration,
window_start=window_start,
increment=cost,
)
if window_counter not in reserved_counter_keys
else None
)
await _enqueue_window_spend_row_update(
entity_type=entity_type,
entity_id=entity_id,
reset_at=window_reset_at,
window_duration=duration,
window_start=window_start,
increment=cost,
request_started_at=request_started_at,
)
return pending_window
async def _window_increments(
entity_type: Litellm_EntityType, entity_id: str, budget_limits: object
) -> tuple[PendingSpendIncrement | BaseException, ...]:
if not isinstance(budget_limits, list):
return ()
window_pending: Final = await asyncio.gather(
*(_window_increment(entity_type, entity_id, window) for window in budget_limits), return_exceptions=True
)
return tuple(item for item in window_pending if item is not None)
async def _cached_window_increments(
entity_type: Litellm_EntityType, entity_id: str, cache_key: str
) -> tuple[_SpendScopeItem, ...]:
cached: Final[object] = _UNTYPED_VALUE.validate_python(await user_api_key_cache.async_get_cache(key=cache_key))
if cached is None:
return (_UncachedBudgetWindows(entity_type=entity_type, entity_id=entity_id),)
return await _window_increments(entity_type, entity_id, _cached_budget_limits(cached))
async def _stored_window_increments(
uncached: _UncachedBudgetWindows,
) -> tuple[PendingSpendIncrement | BaseException, ...]:
return await _window_increments(
uncached.entity_type,
uncached.entity_id,
await _stored_budget_limits(uncached.entity_type, uncached.entity_id),
)
async def _key_scope(key_token: str) -> tuple[_SpendScopeItem, ...]:
# key_token arrives pre-hashed from metadata["user_api_key"] (auth flow
# hashes raw "sk-..." keys before they reach the callback). The
# startswith("sk-") check is a safety net matching update_cache —
@ -3685,52 +3799,9 @@ async def _increment_spend_counters_batched(
)
)
async def _key_window_increment(window: object) -> PendingSpendIncrement | None:
duration = (
window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None)
)
key_window_reset_at = (
window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None)
)
key_window_counter: Final = f"spend:key:{hashed_token}:window:{duration}"
key_window_start = get_budget_window_start(window)
pending_window: Final = (
await _prepare_window_spend_counter_increment(
counter_key=key_window_counter,
entity_type="Key",
entity_id=hashed_token,
window_duration=duration,
window_start=key_window_start,
increment=cost,
)
if key_window_counter not in reserved_counter_keys
else None
)
await _enqueue_window_spend_row_update(
entity_type=Litellm_EntityType.KEY,
entity_id=hashed_token,
reset_at=key_window_reset_at,
window_duration=duration,
window_start=key_window_start,
increment=cost,
request_started_at=request_started_at,
)
return pending_window
return key_pending + await _cached_window_increments(Litellm_EntityType.KEY, hashed_token, hashed_token)
key_obj: Final[object] = _UNTYPED_VALUE.validate_python(
await user_api_key_cache.async_get_cache(key=hashed_token)
)
if key_obj is None:
return key_pending
key_budget_limits: Final = _cached_budget_limits(key_obj)
if not isinstance(key_budget_limits, list):
return key_pending
window_pending: Final = await asyncio.gather(
*(_key_window_increment(window) for window in key_budget_limits), return_exceptions=True
)
return key_pending + tuple(item for item in window_pending if item is not None)
async def _team_scope(scope_team_id: str) -> tuple[PendingSpendIncrement | BaseException, ...]:
async def _team_scope(scope_team_id: str) -> tuple[_SpendScopeItem, ...]:
team_counter_key: Final = f"spend:team:{scope_team_id}"
team_pending: Final[tuple[PendingSpendIncrement, ...]] = (
()
@ -3744,50 +3815,9 @@ async def _increment_spend_counters_batched(
)
)
async def _team_window_increment(window: object) -> PendingSpendIncrement | None:
duration = (
window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None)
)
team_window_reset_at = (
window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None)
)
team_window_counter: Final = f"spend:team:{scope_team_id}:window:{duration}"
team_window_start = get_budget_window_start(window)
pending_window: Final = (
await _prepare_window_spend_counter_increment(
counter_key=team_window_counter,
entity_type="Team",
entity_id=scope_team_id,
window_duration=duration,
window_start=team_window_start,
increment=cost,
)
if team_window_counter not in reserved_counter_keys
else None
)
await _enqueue_window_spend_row_update(
entity_type=Litellm_EntityType.TEAM,
entity_id=scope_team_id,
reset_at=team_window_reset_at,
window_duration=duration,
window_start=team_window_start,
increment=cost,
request_started_at=request_started_at,
)
return pending_window
team_obj: Final[object] = _UNTYPED_VALUE.validate_python(
await user_api_key_cache.async_get_cache(key=f"team_id:{scope_team_id}")
return team_pending + await _cached_window_increments(
Litellm_EntityType.TEAM, scope_team_id, f"team_id:{scope_team_id}"
)
if team_obj is None:
return team_pending
team_budget_limits: Final = _cached_budget_limits(team_obj)
if not isinstance(team_budget_limits, list):
return team_pending
window_pending: Final = await asyncio.gather(
*(_team_window_increment(window) for window in team_budget_limits), return_exceptions=True
)
return team_pending + tuple(item for item in window_pending if item is not None)
async def _team_member_scope(
scope_user_id: str, scope_team_id: str
@ -3858,22 +3888,30 @@ async def _increment_spend_counters_batched(
# return_exceptions so a failing scope does not leave its siblings running
# as orphaned tasks that race the caller's reservation-counter invalidation;
# all scopes settle, then the first error propagates as before.
scope_results: Final = await asyncio.gather(*scope_coros, return_exceptions=True)
scope_errors: Final = tuple(
item
for scope in scope_results
for item in (scope if isinstance(scope, tuple) else (scope,))
if isinstance(item, BaseException)
scope_items: Final = tuple(_flatten_spend_scopes(await asyncio.gather(*scope_coros, return_exceptions=True)))
uncached_windows: Final = tuple(item for item in scope_items if isinstance(item, _UncachedBudgetWindows))
await _apply_spend_counter_increments(
pending=reservation_update.pending
+ tuple(item for item in scope_items if isinstance(item, PendingSpendIncrement))
)
pending: Final = tuple(
item
for scope in scope_results
if not isinstance(scope, BaseException)
for item in scope
if not isinstance(item, BaseException)
stored_window_items: Final = (
tuple(
_flatten_spend_scopes(
await asyncio.gather(
*(_stored_window_increments(uncached) for uncached in uncached_windows), return_exceptions=True
)
)
)
if uncached_windows
else ()
)
await _apply_spend_counter_increments(pending=reservation_update.pending + pending)
stored_window_pending: Final = tuple(
item for item in stored_window_items if isinstance(item, PendingSpendIncrement)
)
if stored_window_pending:
await _apply_spend_counter_increments(pending=stored_window_pending)
stamp_budget_reservation_actual_cost(budget_reservation=budget_reservation, actual_cost=response_cost)
scope_errors: Final = tuple(item for item in scope_items + stored_window_items if isinstance(item, BaseException))
if scope_errors:
raise scope_errors[0]
@ -3881,6 +3919,25 @@ async def _increment_spend_counters_batched(
budget_reservation["finalized"] = True
@dataclass(frozen=True, slots=True)
class _UncachedBudgetWindows:
entity_type: Litellm_EntityType
entity_id: str
_SpendScopeItem: TypeAlias = PendingSpendIncrement | BaseException | _UncachedBudgetWindows
def _flatten_spend_scopes(
scope_results: Sequence[tuple[_SpendScopeItem, ...] | BaseException],
) -> Iterator[_SpendScopeItem]:
for scope in scope_results:
if isinstance(scope, BaseException):
yield scope
continue
yield from scope
@dataclass(frozen=True, slots=True)
class _ReservationCounterUpdate:
"""The reserved counters the direct increment must skip, and the adjustments that settle them on the actual
@ -5661,11 +5718,18 @@ def _with_config_file_pass_through_endpoints(
def _reload_settings_store(section: Section, store: SettingsStore, section_config: object) -> None:
serving_pass_throughs: Final = store.get("pass_through_endpoints")
store.load_yaml(_as_settings_mapping(section_config))
store.apply_db_row(section, _EMPTY_SETTINGS_MAPPING)
if is_resource_list(section, "pass_through_endpoints") and serving_pass_throughs is not None:
store["pass_through_endpoints"] = serving_pass_throughs
def _general_settings_row_is_readable(row: _ConfigParamValueRow | None) -> bool:
return row is None or row.param_value is None or isinstance(row.param_value, Mapping)
def _db_managed_general_settings_keys() -> frozenset[str]:
return frozenset((*ConfigGeneralSettings.model_fields, *_CLEANUP_SCHEDULE_KEYS, "allowed_ips", "email_settings"))
def _bind_general_settings_store(settings: SettingsStore) -> None:
global general_settings
general_settings = settings # pyright: ignore[reportAssignmentType] # legacy global accepts mappings
@ -5816,7 +5880,9 @@ class ProxyConfig:
# whether an existing request predates the prices it just fetched, and re-serving one
# costs a single fetch where skipping one leaves it priced wrong indefinitely
self.model_cost_map_applied_revision: int = 0
self.settings: Final[SettingsStore] = SettingsStore("general_settings")
self.settings: Final[SettingsStore] = SettingsStore(
"general_settings", managed_db_keys=_db_managed_general_settings_keys()
)
self.router_settings: Final[SettingsStore] = SettingsStore("router_settings")
self.litellm_settings: Final[SettingsStore] = SettingsStore("litellm_settings")
self.environment_variables: Final[SettingsStore] = SettingsStore("environment_variables")
@ -5827,6 +5893,8 @@ class ProxyConfig:
self._db_environment_keys: frozenset[str] = frozenset()
self._sso_environment_removals: frozenset[str] = frozenset()
self._warned_shadowed_keys: frozenset[tuple[Section, str]] = frozenset()
self._warned_unmanaged_keys: frozenset[str] = frozenset()
self._warned_unreadable_db_general_settings: bool = False
self._db_config_callback_entries: Mapping[tuple[str, str], _DbCallbackRegistration] = MappingProxyType({})
self._settings_stores: Final[Mapping[Section, SettingsStore]] = MappingProxyType(
{
@ -6472,6 +6540,9 @@ class ProxyConfig:
prisma_client=prisma_client,
store_model_in_db=store_model_in_db,
)
else:
for section, store in self._settings_stores.items():
store.apply_db_row(section, _EMPTY_SETTINGS_MAPPING)
config = self._config_with_resolved_settings(config)
@ -8350,15 +8421,20 @@ class ProxyConfig:
general_settings.get("maximum_spend_logs_retention_interval", "1d"),
)
async def _update_general_settings(self, db_general_settings: Mapping[str, SettingsJsonValue] | None) -> None:
async def _update_general_settings(self, db_general_settings: Mapping[str, SettingsJsonValue] | str | None) -> None:
global general_settings
if db_general_settings is None:
return
if not isinstance(db_general_settings, Mapping):
self._warn_about_unreadable_db_general_settings()
return
self._warned_unreadable_db_general_settings = False
if not isinstance(general_settings, SettingsStore):
self.settings.load_yaml(_as_settings_mapping(general_settings))
cache_size_was_db: Final = self.settings.source("user_api_key_cache_max_size") == "db"
previous_cleanup_schedule: Final = self._resolved_cleanup_schedule()
self.settings.apply_db_row("general_settings", db_general_settings)
self._warn_about_unmanaged_db_settings()
_bind_general_settings_store(self.settings)
await self._apply_general_settings_side_effects(
db_general_settings,
@ -8550,12 +8626,24 @@ class ProxyConfig:
environment_read: Final = next(self._environment_read_numbers)
sections: Final = tuple(self._settings_stores)
responses: Final = await asyncio.gather(*(get_config_param(prisma_client, section) for section in sections))
general_settings_response: Final = responses[sections.index("general_settings")]
general_settings_value: Final[Mapping[str, SettingsJsonValue]] = (
self._prepared_db_settings_values("general_settings", general_settings_response.param_value)
if general_settings_response is not None and general_settings_response.param_value is not None
else _EMPTY_SETTINGS_MAPPING
)
self._settings_stores["general_settings"].apply_db_row("general_settings", general_settings_value)
response: _ConfigParamValueRow | None
for section, response in zip(sections, responses):
if section == "general_settings":
continue
if response is None:
self._settings_stores[section].apply_db_row(section, _EMPTY_SETTINGS_MAPPING)
continue
param_value: JsonValue = getattr(response, "param_value", None)
if param_value is None:
self._settings_stores[section].apply_db_row(section, _EMPTY_SETTINGS_MAPPING)
continue
verbose_proxy_logger.debug(
@ -8566,6 +8654,7 @@ class ProxyConfig:
if section == "litellm_settings":
self._apply_litellm_settings_db_values(self._prepared_db_settings_values(section, param_value))
elif section == "environment_variables":
self._settings_stores[section].apply_db_row(section, _EMPTY_SETTINGS_MAPPING)
await self._apply_db_environment_variables_row(
prisma_client, response, environment_before_db, environment_read
)
@ -8576,6 +8665,7 @@ class ProxyConfig:
)
self._warn_about_shadowed_db_settings()
self._warn_about_unmanaged_db_settings()
return self._config_with_resolved_settings(config)
async def _apply_db_environment_variables_row(
@ -8634,6 +8724,30 @@ class ProxyConfig:
)
self._warned_shadowed_keys = shadowed
def _warn_about_unmanaged_db_settings(self) -> None:
unmanaged: Final = frozenset(
key for key in self.settings.unmanaged_db_keys() if key not in _GENERAL_SETTINGS_UI_LITELLM_FIELDS
)
for key in sorted(unmanaged - self._warned_unmanaged_keys):
verbose_proxy_logger.warning(
"general_settings.%s is stored in the database, but the proxy does not apply that stored value, so it "
"is ignored. %s",
key,
"The config file's value applies. Remove the stored copy with POST /config/field/delete."
if self.settings.owned_by_config(key)
else "Set it in the config file to use it, or remove it with POST /config/field/delete.",
)
self._warned_unmanaged_keys = unmanaged
def _warn_about_unreadable_db_general_settings(self) -> None:
if self._warned_unreadable_db_general_settings:
return
verbose_proxy_logger.warning(
"general_settings is stored in the database as something other than a JSON object, so the proxy "
"ignores every stored general setting."
)
self._warned_unreadable_db_general_settings = True
def _prepared_db_settings_values(self, section: Section, value: object) -> Mapping[str, SettingsJsonValue]:
scrubbed: Final = _scrub_db_overlay_remote_module_loads(section=section, db_value=value)
return _as_settings_mapping(scrubbed)
@ -8801,7 +8915,10 @@ class ProxyConfig:
)
# initialize vector stores, guardrails, etc. table in db
await self._init_non_llm_objects_in_db(prisma_client=prisma_client)
await self._init_non_llm_objects_in_db(
prisma_client=prisma_client,
refresh_pass_through_endpoints=_general_settings_row_is_readable(db_general_settings),
)
except Exception as e:
verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - %s", e)
@ -8870,7 +8987,9 @@ class ProxyConfig:
except Exception as e: # noqa: BLE001 # best-effort: a failing stop must not break proxy shutdown
verbose_proxy_logger.error("Error stopping auth cache invalidation subscriber: %s", e)
async def _init_non_llm_objects_in_db(self, prisma_client: PrismaClient):
async def _init_non_llm_objects_in_db(
self, prisma_client: PrismaClient, *, refresh_pass_through_endpoints: bool = True
):
"""
Use this to read non-llm objects from the db and initialize them
@ -8894,7 +9013,7 @@ class ProxyConfig:
if self._should_load_db_object(object_type="agents"):
await self._init_agents_in_db(prisma_client=prisma_client)
if self._should_load_db_object(object_type="pass_through_endpoints"):
if refresh_pass_through_endpoints and self._should_load_db_object(object_type="pass_through_endpoints"):
await self._init_pass_through_endpoints_in_db()
if self._should_load_db_object(object_type="prompts"):
@ -10223,6 +10342,11 @@ def _restamp_nested_response_model_in_sse_line(line: bytes, target_model: str) -
return _SSE_DATA_FIELD + b" " + serialized_event.encode() + line_ending
def _vllm_choice_fields(choice: BaseModel) -> Mapping[str, object]:
choice_extras: Final = choice.model_extra or {}
return MappingProxyType({field: choice_extras[field] for field in VLLM_CHOICE_FIELDS if field in choice_extras})
def _json_encodes(value: object) -> bool:
try:
to_jsonable_python(value)
@ -10278,11 +10402,52 @@ def _serialize_chat_chunk(chunk: ModelResponseStream) -> bytes:
def _serialize_streaming_chunk(chunk: BaseModel) -> bytes:
choices: Final = _streaming_chunk_choices(chunk)
null_choice_fields: Final = tuple(_null_vllm_choice_fields(choice) for choice in choices)
if any(null_choice_fields):
serialized_chunk: Final = _serialize_keeping_null_choice_fields(chunk, choices, null_choice_fields)
if serialized_chunk is not None:
return serialized_chunk
if isinstance(chunk, ModelResponseStream):
return _serialize_chat_chunk(chunk)
return chunk.model_dump_json(exclude_none=True, exclude_unset=True).encode()
def _serialize_keeping_null_choice_fields(
chunk: BaseModel, choices: Sequence[BaseModel], null_choice_fields: Sequence[Mapping[str, None]]
) -> bytes | None:
try:
return json.dumps(
{
**_dump_set_fields(chunk, exclude=frozenset({"choices"})),
"choices": [
{**_dump_set_fields(choice), **nulls}
for choice, nulls in zip(choices, null_choice_fields, strict=True)
],
},
separators=(",", ":"),
ensure_ascii=False,
).encode()
except ValueError:
return None
def _streaming_chunk_choices(chunk: BaseModel) -> Sequence[BaseModel]:
match chunk:
case ModelResponseStream() | TextCompletionResponse():
return chunk.choices
case _:
return ()
def _null_vllm_choice_fields(choice: BaseModel) -> Mapping[str, None]:
return MappingProxyType({field: None for field, value in _vllm_choice_fields(choice).items() if value is None})
def _dump_set_fields(model: BaseModel, exclude: AbstractSet[str] = frozenset()) -> Mapping[str, object]:
return model.model_dump(mode="json", exclude_none=True, exclude_unset=True, exclude=set(exclude))
async def _apply_streaming_chunk_hooks(
*,
chunk: Any,
@ -13396,7 +13561,10 @@ async def moderations(
return response
except Exception as e:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=data,
dispatched_call_type=CallTypes.amoderation.value,
)
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, ProxyException):
@ -13541,6 +13709,7 @@ async def audio_speech(
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=data,
dispatched_call_type=CallTypes.aspeech.value,
)
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, ProxyException):
@ -13692,7 +13861,10 @@ async def audio_transcriptions(
return response
except Exception as e:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=data,
dispatched_call_type=CallTypes.atranscription.value,
)
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, HTTPException):
@ -14790,6 +14962,7 @@ from litellm.repositories.table_repositories import (
)
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
from litellm.responses.litellm_completion_transformation.count_request import (
ChatCountBridgeError,
chat_count_request,
@ -15037,7 +15210,7 @@ async def _provider_token_count(
framing_can_stand_in: Final = (count_model is None or not count_model.configured) and (
failure.cannot_count_model or (pinned_count_model is not None and failure.denied_in_region)
)
if not (framing_can_stand_in and settings.framing is not None and local_estimate_allowed):
if not (framing_can_stand_in and settings.frames_counts and local_estimate_allowed):
raise failure.to_proxy_exception()
if pinned_count_model is not None:
verbose_proxy_logger.warning(
@ -15237,6 +15410,7 @@ async def _tokenizer_or_provider_error(
async def _local_token_estimate(
model: str,
template_model: str,
prompt: str | None,
messages: Sequence[AllMessageValues] | None,
tools: list[ChatCompletionToolParam] | None,
@ -15249,7 +15423,7 @@ async def _local_token_estimate(
else await _tokenizer_or_provider_error(model, custom_tokenizer, provider_failure)
)
count: Final = offload_token_count(litellm.token_counter)(
model=model,
model=template_model,
text=prompt,
messages=messages,
tools=tools,
@ -15315,10 +15489,15 @@ def _deployment_aws_region(deployment: _TokenCountDeployment) -> str:
)
def _deployment_base_model(deployment: _TokenCountDeployment) -> str | None:
return _non_empty_str(deployment.get("model_info", {}).get("base_model")) or _non_empty_str(
deployment.get("litellm_params", {}).get("base_model")
)
def _token_count_settings_model(deployment: _TokenCountDeployment, request_model: str) -> tuple[str, str | None]:
model_info: Final = deployment.get("model_info", {})
params: Final = deployment.get("litellm_params", {})
base_model: Final = _non_empty_str(model_info.get("base_model")) or _non_empty_str(params.get("base_model"))
base_model: Final = _deployment_base_model(deployment)
if base_model is not None:
return base_model, None
model: Final = _non_empty_str(params.get("model"))
@ -15470,12 +15649,15 @@ def _responses_routing_kwargs(responses_request: Mapping[str, object] | None) ->
return {"previous_response_id": previous_response_id} if isinstance(previous_response_id, str) else {}
def _inference_params(
litellm_params: Mapping[str, object], responses_request: Mapping[str, object]
) -> Mapping[str, object]:
return {**litellm_params, **_router_default_litellm_params(llm_router), **responses_request}
def _count_drop_policy(deployment: Mapping[str, object] | None, responses_request: Mapping[str, object]) -> DropPolicy:
litellm_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {}
router_defaults: Final[Mapping[str, object]] = (
{} if llm_router is None else _router_default_litellm_params(llm_router)
)
return drop_policy({**litellm_params, **router_defaults, **responses_request})
return drop_policy(_inference_params(litellm_params, responses_request))
def _native_responses_count(
@ -15498,8 +15680,54 @@ def _native_responses_count(
)
def _deployment_api_base(deployment: _TokenCountDeployment) -> str | None:
return _non_empty_str(deployment.get("litellm_params", {}).get("api_base"))
def _deployment_api_base(
deployment: _TokenCountDeployment, responses_request: Mapping[str, object] | None = None
) -> str | None:
return _non_empty_str(
_inference_params(deployment.get("litellm_params", {}), responses_request or {}).get("api_base")
)
def _deployment_provider(
deployment: _TokenCountDeployment, request_model: str, responses_request: Mapping[str, object] | None
) -> str | None:
params: Final = deployment.get("litellm_params", {})
try:
return litellm.get_llm_provider(
model=_non_empty_str(params.get("model")) or request_model,
custom_llm_provider=_non_empty_str(params.get("custom_llm_provider")),
api_base=_deployment_api_base(deployment, responses_request),
)[1]
except litellm.BadRequestError:
return None
def _serves_native_responses(
deployment: _TokenCountDeployment, request_model: str, responses_request: Mapping[str, object]
) -> bool:
params: Final = deployment.get("litellm_params", {})
inference: Final = _inference_params(params, responses_request)
try:
return provider_stores_the_conversation(
_non_empty_str(params.get("model")) or request_model,
_non_empty_str(params.get("custom_llm_provider")),
{
"use_chat_completions_api": bool(inference.get("use_chat_completions_api")),
"model_info": dict(deployment.get("model_info", {})),
"api_base": _non_empty_str(inference.get("api_base")),
},
)
except litellm.BadRequestError:
return False
_AZURE_PROVIDERS: Final = frozenset((litellm.LlmProviders.AZURE.value, litellm.LlmProviders.AZURE_AI.value))
def _with_strict_by_default(tools: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]:
return tuple(
{**tool, "strict": True} if tool.get("type") == "function" and "strict" not in tool else tool for tool in tools
)
def _system_message(system: object) -> ChatCompletionSystemMessage | None:
@ -15704,7 +15932,7 @@ async def _count_request_tokens(
else await chat_count_request(
model_to_use,
custom_llm_provider,
_deployment_api_base(deployment_view),
_deployment_api_base(deployment_view, routed_request),
routed_request,
_count_drop_policy(routed_deployment, routed_request),
)
@ -15802,10 +16030,31 @@ async def _count_request_tokens(
counted_messages: Final = (
typed_messages if typed_messages is None or system_message is None else (system_message, *typed_messages)
)
counted_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats
native_responses: Final = responses_request is not None and _serves_native_responses(
deployment_view, request.model, responses_request
)
sent_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats
list[ChatCompletionToolParam] | None, tools if counted_messages is not None else None
)
framing: Final = settings.framing if counted_messages is not None and settings.framing is not None else 0
openai_model: Final = deployment_openai_model(deployment_view)
template_model: Final = openai_model or _deployment_base_model(deployment_view) or model_to_use
tool_section: Final = chat_template_tool_section(template_model)
counted_tools: Final = (
cast(list[ChatCompletionToolParam], list(_with_strict_by_default(sent_tools))) # cast-ok: strict added
if sent_tools is not None
and native_responses
and _deployment_provider(deployment_view, request.model, responses_request) in _AZURE_PROVIDERS
and tool_section is not None
and tool_section.renders_strict_only_when_true
else sent_tools
)
framing: Final = (
0
if counted_messages is None
else settings.hidden_prompt_tokens(
carries_tools=sends_function_tools(counted_tools), responses_api=native_responses
)
)
kimi_k3_messages: Final = counted_messages if settings.tokenizer == "kimi_k3" else None
exact: Final = (
await _kimi_k3_count(
@ -15829,7 +16078,8 @@ async def _count_request_tokens(
count_source="local_estimate",
)
estimate: Final = await _local_token_estimate(
model=deployment_openai_model(deployment_view) or model_to_use,
model=openai_model or model_to_use,
template_model=template_model,
prompt=prompt if counted_messages is None else None,
messages=counted_messages,
tools=counted_tools,
@ -20931,53 +21181,56 @@ async def delete_config_general_settings(
if data.field_name in _GENERAL_SETTINGS_UI_LITELLM_FIELDS:
return await _reset_general_settings_ui_litellm_field(data.field_name, user_api_key_dict)
if data.field_name not in ConfigGeneralSettings.model_fields:
config_table: Final[TableActions[_ConfigParamValueRow]] = _config_param_table(prisma_client)
stored_row: Final = await config_table.find_first(where={"param_name": "general_settings"})
stored: Final = (
_SETTINGS_MAPPING.validate_python(stored_row.param_value)
if stored_row is not None and isinstance(stored_row.param_value, Mapping)
else None
)
if not _general_settings_row_is_readable(stored_row):
raise HTTPException(
status_code=400,
detail={
"error": "general_settings is stored in the database as something other than a JSON object, so no "
"field can be removed from it"
},
)
if data.field_name not in ConfigGeneralSettings.model_fields and (stored is None or data.field_name not in stored):
raise HTTPException(
status_code=400,
detail={"error": f"Invalid field={data.field_name} passed in."},
)
proxy_config.reject_config_owned_deletes(section_name="general_settings", keys=(data.field_name,))
if data.field_name in _db_managed_general_settings_keys():
proxy_config.reject_config_owned_deletes(section_name="general_settings", keys=(data.field_name,))
## get general settings from db
db_general_settings: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
where={"param_name": "general_settings"}
)
### pop the value
if db_general_settings is None or db_general_settings.param_value is None:
if stored is None:
raise HTTPException(
status_code=400,
detail={"error": f"Field name={data.field_name} not in config"},
)
else:
general_settings = dict(db_general_settings.param_value)
before_general_settings: Final = copy.deepcopy(general_settings)
## update db
general_settings.pop(data.field_name, None)
remaining: Final = {key: value for key, value in stored.items() if key != data.field_name}
response: Final = await ConfigRepository(prisma_client).table.upsert(
where={"param_name": "general_settings"},
data={
"create": {
"param_name": "general_settings",
"param_value": json.dumps(general_settings),
"param_value": json.dumps(remaining),
},
"update": {"param_value": json.dumps(general_settings)},
"update": {"param_value": json.dumps(remaining)},
},
)
await invalidate_config_param("general_settings")
proxy_config.settings.apply_db_row("general_settings", general_settings)
proxy_config.settings.apply_db_row("general_settings", remaining)
if is_resource_list("general_settings", data.field_name):
stored_endpoints: Final = general_settings.get("pass_through_endpoints")
stored_endpoints: Final = remaining.get("pass_through_endpoints")
await proxy_config.serve_pass_through_endpoints(stored_endpoints if isinstance(stored_endpoints, list) else ())
asyncio.create_task(
create_config_audit_log(
"general_settings", "deleted", before_general_settings, general_settings, user_api_key_dict
)
create_config_audit_log("general_settings", "deleted", dict(stored), remaining, user_api_key_dict)
)
return response

View file

@ -20,6 +20,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
openai_error_param,
openai_error_type,
)
from litellm.types.utils import CallTypes
router: Final = APIRouter()
@ -114,7 +115,10 @@ async def rerank(
return response
except Exception as e:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=data,
dispatched_call_type=CallTypes.arerank.value,
)
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, HTTPException):

View file

@ -20,7 +20,6 @@ from litellm.constants import (
CALLER_SESSION_ID_METADATA_KEY,
CLI_SESSION_KEY_PREFIX,
EMPTY_MAPPING,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
KUBERNETES_POD_ROUTING_KEY,
LITELLM_PROXY_MASTER_KEY_ALIAS,
LITELLM_TRUNCATED_PAYLOAD_FIELD,
@ -30,7 +29,6 @@ from litellm.constants import (
MAX_SPEND_LOG_MODEL_NAME_LENGTH,
MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB,
MCP_SPEND_LOG_MODEL_PREFIX,
NON_INFERENCE_CALL_TYPES,
REDACTED_BY_LITELM_STRING,
SESSION_ID_OMITTED_METADATA_KEY,
UNKNOWN_MODEL_SPEND_LOG_MODEL,
@ -53,7 +51,11 @@ from litellm.litellm_core_utils.core_helpers import (
)
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
from litellm.litellm_core_utils.hidden_params import served_from_cache
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call
from litellm.litellm_core_utils.internal_call_metadata import (
BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX,
is_unbilled_non_inference_call,
settles_background_response,
)
from litellm.litellm_core_utils.litellm_logging import (
StandardLoggingPayloadSetup,
UsageObservation,
@ -79,13 +81,8 @@ from litellm.responses.litellm_completion_transformation.compaction_items import
summary_only_compaction_content,
with_compaction_summary,
)
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
BackgroundResponseSettlement,
)
from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag
from litellm.types.utils import (
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
PROMPT_CARRYING_GUARDRAIL_FIELDS,
AzureSpillover,
CallTypes,
@ -347,24 +344,21 @@ def _get_spend_logs_metadata(
BATCH_COST_REQUEST_ID_SUFFIX: Final = "_batch_cost"
BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX: Final = "_background_cost"
def _cost_row_request_id_suffix(
call_type: str, metadata: Mapping[str, object], settles_background_response: bool
) -> str:
def _cost_row_request_id_suffix(call_type: str, metadata: Mapping[str, object], kwargs: Mapping[str, object]) -> str:
if call_type == CallTypes.aretrieve_batch.value:
return BATCH_COST_REQUEST_ID_SUFFIX
if settles_background_response or (
call_type in NON_INFERENCE_CALL_TYPES
and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
):
if settles_background_response(call_type, metadata, kwargs):
return BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX
return ""
def get_spend_logs_id(
call_type: str, response_obj: dict, kwargs: Mapping[str, object], metadata: Mapping[str, object] = EMPTY_MAPPING
call_type: str,
response_obj: Mapping[str, object],
kwargs: Mapping[str, object],
metadata: Mapping[str, object] = EMPTY_MAPPING,
) -> str | None:
standard_logging_payload = kwargs.get("standard_logging_object")
candidate_ids: Final = (
@ -377,10 +371,7 @@ def get_spend_logs_id(
)
if resolved_id is None:
return None
settles_background_response: Final = (
kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES
)
return f"{resolved_id}{_cost_row_request_id_suffix(call_type, metadata, settles_background_response)}"
return f"{resolved_id}{_cost_row_request_id_suffix(call_type, metadata, kwargs)}"
_MISSING_ATTRIBUTE: Final = object()

View file

@ -68,6 +68,7 @@ from litellm.constants import (
SPEND_LOG_QUEUE_MAX_BYTES,
SPEND_LOG_WRITE_BATCH_MAX_BYTES,
SPEND_LOG_WRITE_BATCH_MAX_ROWS,
VLLM_CHOICE_FIELDS,
)
from litellm.cost_calculator import BaseTokenUsageProcessor
from litellm.litellm_core_utils.bug_report import (
@ -170,6 +171,7 @@ from litellm.litellm_core_utils.litellm_logging import (
deployment_pricing_model_info,
)
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.litellm_core_utils.redact_messages import drop_token_ids
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
@ -179,6 +181,7 @@ from litellm.litellm_core_utils.served_output_texts import (
)
from litellm.litellm_core_utils.token_counter import offload_token_count, usage_tokenizer_model
from litellm.llms import load_guardrail_translation_mappings
from litellm.llms.base_llm.guardrail_translation.utils import stream_item_field, stream_item_items
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.proxy._types import (
AlertType,
@ -707,6 +710,56 @@ def _pipeline_step_streams_live(step: "PipelineStep") -> bool:
return not unified_guardrail.resolve_streaming_flag(callback, "streaming_buffer_until_moderated", True)
def _drop_stream_chunk_token_ids(chunk: object) -> None:
for choice in stream_item_items(chunk, "choices"):
drop_token_ids(choice)
def _drop_built_stream_token_ids(request_data: Mapping[str, object]) -> None:
logging_obj: Final = request_data.get("litellm_logging_obj")
if isinstance(logging_obj, Logging):
logging_obj.drops_stream_token_ids = True
def _drop_token_ids_ahead_of_chunk_check(
callback: CustomLogger, chunk: object, request_data: Mapping[str, object]
) -> None:
if _overrides_hook(callback, "async_post_call_streaming_hook"):
_drop_stream_chunk_token_ids(chunk)
_drop_built_stream_token_ids(request_data)
_PLAIN_STREAM_FIELDS: Final = TypeAdapter(dict[str, object])
_EMPTY_STREAM_VALUES: Final[tuple[object, ...]] = (None, "", [], {})
def _holds_stream_payload(value: object) -> bool:
if isinstance(value, dict):
nested: Final = _PLAIN_STREAM_FIELDS.validate_python(value).values()
return any(item not in _EMPTY_STREAM_VALUES for item in nested)
return value not in _EMPTY_STREAM_VALUES
def _choice_carries_only_token_ids(choice: object) -> bool:
if stream_item_field(choice, "token_ids") is None or not isinstance(choice, BaseModel):
return False
fields: Final = _PLAIN_STREAM_FIELDS.validate_python(choice.model_dump())
return not any(_holds_stream_payload(value) for key, value in fields.items() if key not in ("index", "token_ids"))
def _is_token_id_only_chunk(chunk: object) -> bool:
choices: Final = stream_item_items(chunk, "choices")
return bool(choices) and all(_choice_carries_only_token_ids(choice) for choice in choices)
async def _without_token_id_only_chunks(response: AsyncIterable[object]) -> AsyncGenerator[object, None]:
async for chunk in response:
if _is_token_id_only_chunk(chunk):
_drop_stream_chunk_token_ids(chunk)
continue
yield chunk
def _pipelines_stream_live(pipelines: Sequence[tuple[str, "GuardrailPipeline"]]) -> bool:
return all(_pipeline_step_streams_live(step) for step in _pipeline_steps(pipelines))
@ -1707,7 +1760,10 @@ def _proxy_only_error_log_fields(
def _failure_fields_to_lift(
request_data: Mapping[str, object], provider_billed: bool = False, dispatched_call_type: str | None = None
request_data: Mapping[str, object],
provider_billed: bool = False,
dispatched_call_type: str | None = None,
route_call_type: str | None = None,
) -> Mapping[str, object]:
"""Failure-path callbacks run after ``litellm_logging_obj`` is popped from
request_data (it is not serialisable), so the caller merges these fields
@ -1717,7 +1773,8 @@ def _failure_fields_to_lift(
attribution on failed-request spend logs."""
_logging_obj: Final = request_data.get("litellm_logging_obj")
if _logging_obj is None:
return _EMPTY_LIFT
return _EMPTY_LIFT if dispatched_call_type is None else MappingProxyType({"call_type": dispatched_call_type})
_estimate_call_type: Final = dispatched_call_type or route_call_type
_model_call_details: Final = getattr(_logging_obj, "model_call_details", {})
_first_handoff: Final = _model_call_details.get("first_api_call_start_time")
_litellm_params: Final = getattr(_logging_obj, "litellm_params", None)
@ -1732,10 +1789,10 @@ def _failure_fields_to_lift(
provider_billed=_billed,
deployment_id=_deployment_id,
base_model=_base_model,
dispatched_call_type=dispatched_call_type,
dispatched_call_type=_estimate_call_type,
)
_prompt_estimate: Final = (
_estimated_prompt_usage(_model_call_details, request_data, dispatched_call_type)
_estimated_prompt_usage(_model_call_details, request_data, _estimate_call_type)
if _billed and _first_handoff is not None and not _model_call_details.get("failed_attempts_usage")
else None
)
@ -1755,7 +1812,7 @@ def _failure_fields_to_lift(
_billed,
_deployment_id,
_base_model,
dispatched_call_type,
_estimate_call_type,
)
if Logging.usage_capture_callback is not None
and (_billed or _model_call_details.get("combined_usage_object") is not None)
@ -4051,13 +4108,15 @@ class ProxyLogging:
route=route,
original_exception=original_exception,
error_type=error_type,
dispatched_call_type=dispatched_call_type,
)
request_data.update(
await offload_token_count(_failure_fields_to_lift)(
request_data,
provider_billed=_provider_billed_the_failure(original_exception),
dispatched_call_type=dispatched_call_type or _call_type_for_route(user_api_key_dict.request_route),
dispatched_call_type=dispatched_call_type,
route_call_type=_call_type_for_route(user_api_key_dict.request_route),
)
)
@ -4146,6 +4205,7 @@ class ProxyLogging:
route: str | None = None,
original_exception: Exception | None = None,
error_type: ProxyErrorTypes | None = None,
dispatched_call_type: str | None = None,
):
"""
Handle logging for proxy only errors by calling `litellm_logging_obj.async_failure_handler`
@ -4245,7 +4305,7 @@ class ProxyLogging:
litellm_logging_obj.model_call_details["input"] = input
body_shape_call_type = CallTypes.aembedding.value
resolved_call_type: Final = _call_type_for_route(route) or (
None if set_up_by_request else body_shape_call_type
None if set_up_by_request else dispatched_call_type or body_shape_call_type
)
if resolved_call_type is not None and litellm_logging_obj.call_type != CallTypes.pass_through.value:
litellm_logging_obj.call_type = resolved_call_type
@ -4710,6 +4770,7 @@ class ProxyLogging:
else:
_callback = callback
if _callback is not None and isinstance(_callback, CustomLogger):
_drop_token_ids_ahead_of_chunk_check(_callback, response, data)
if str_so_far is not None:
complete_response = str_so_far + response_str
else:
@ -4731,7 +4792,7 @@ class ProxyLogging:
self,
response,
user_api_key_dict: UserAPIKeyAuth,
request_data: dict,
request_data: dict[str, object],
):
"""
Allow user to modify outgoing streaming data -> Given a whole response iterator.
@ -4765,20 +4826,26 @@ class ProxyLogging:
# Merge model-level guardrails before checking which guardrails to run
request_data = check_and_merge_model_level_guardrails(data=request_data, llm_router=llm_router)
current_response = response
stream_needs_translation: Final = ProxyLogging._stream_requires_guardrail_translation(user_api_key_dict)
pipeline_gated_names: Final = _pipeline_step_guardrail_names(post_call_pipelines)
running_overrides: Final = tuple(
(callback, kind)
for callback, kind in caps.iterator_overrides
if not isinstance(callback, CustomGuardrail)
or (
callback.guardrail_name not in pipeline_gated_names
and callback.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is True
)
)
drops_token_ids: Final = bool(post_call_pipelines) or any(
isinstance(callback, CustomGuardrail) for callback, _ in running_overrides
)
if drops_token_ids:
_drop_built_stream_token_ids(request_data)
current_response = _without_token_id_only_chunks(response) if drops_token_ids else response
guarded_layers: Final[list[AsyncGenerator[object, None]]] = [] # mutable-ok: closed on disconnect
for resolved_callback, kind in caps.iterator_overrides:
if isinstance(resolved_callback, CustomGuardrail):
if resolved_callback.guardrail_name in pipeline_gated_names:
continue
if (
resolved_callback.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call)
is not True
):
continue
for resolved_callback, kind in running_overrides:
effective_kind = (
"apply_guardrail"
if (
@ -4836,6 +4903,8 @@ class ProxyLogging:
try:
async for chunk in current_response:
served_chunks.append(chunk)
if drops_token_ids:
_drop_stream_chunk_token_ids(served_chunks[-1])
yield chunk
except (GeneratorExit, asyncio.CancelledError):
await ProxyLogging._close_guarded_layers(guarded_layers)
@ -10203,7 +10272,7 @@ _PRESERVED_NONE_FIELDS: Final[list[tuple[str, str]]] = [
("message", "refusal"),
("delta", "content"), # null in streaming chunks
]
_PRESERVED_NONE_CHOICE_FIELDS: Final = ("logprobs",)
_PRESERVED_NONE_CHOICE_FIELDS: Final = ("logprobs", *VLLM_CHOICE_FIELDS)
_PRESERVED_NONE_RESPONSE_FIELDS: Final = ("system_fingerprint",)

View file

@ -1,14 +1,16 @@
import asyncio
import json
from collections.abc import Mapping, Sequence
from collections.abc import Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from itertools import chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, cast
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER
from litellm.litellm_core_utils.cold_storage_handler import ColdStorageHandler
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.responses.litellm_completion_transformation.client_tool_search import (
bridge_client_tool_search,
client_tool_search_loaded_tools,
@ -69,6 +71,90 @@ def _is_a_conversation_turn(spend_log: "SpendLogsPayload") -> bool:
return spend_log.get("call_type") not in _MCP_TOOL_CALL_TYPES and not _is_a_failed_turn(spend_log)
_RESPONSE_READ_CALL_TYPES: Final = frozenset(("aget_responses", "acancel_responses"))
def _output_item_id(item: object) -> str | None:
match item:
case {"id": str() as item_id}:
return item_id
case _:
return None
def _is_a_response_read(spend_log: "SpendLogsPayload") -> bool:
return "call_type" in spend_log and spend_log["call_type"] in _RESPONSE_READ_CALL_TYPES
def _stored_response(spend_log: "SpendLogsPayload") -> object:
return spend_log.get("response")
def _output_items(spend_log: "SpendLogsPayload") -> Iterator[tuple[str | None, object]]:
output: Final = _stored_responses_api_output(_stored_response(spend_log)) or []
return zip(map(_output_item_id, output), output)
def _identified_output_items(spend_log: "SpendLogsPayload") -> Iterator[tuple[str, object]]:
return ((item_id, item) for item_id, item in _output_items(spend_log) if item_id is not None)
def _output_item_ids(spend_log: "SpendLogsPayload") -> frozenset[str]:
return frozenset(item_id for item_id, _ in _identified_output_items(spend_log))
def _first_index_holding_each_item(indexed_turns: Iterable[tuple[int, "SpendLogsPayload"]]) -> Mapping[str, int]:
return dict(
chain.from_iterable(
((item_id, index) for item_id in _output_item_ids(turn)) for index, turn in reversed(tuple(indexed_turns))
)
)
def _turn_replaying_each_item(turns: Sequence["SpendLogsPayload"]) -> Mapping[str, int]:
indexed_turns: Final = tuple(enumerate(turns))
first_read_holding: Final = _first_index_holding_each_item(
(index, turn) for index, turn in indexed_turns if _is_a_response_read(turn)
)
first_other_turn_holding: Final = _first_index_holding_each_item(
(index, turn) for index, turn in indexed_turns if not _is_a_response_read(turn)
)
return MappingProxyType({**first_read_holding, **first_other_turn_holding})
def _latest_read_copy_of_each_item(turns: Sequence["SpendLogsPayload"]) -> Mapping[str, object]:
return MappingProxyType(
dict(chain.from_iterable(_identified_output_items(turn) for turn in turns if _is_a_response_read(turn)))
)
def _with_only_new_output(
index: int, turn: "SpendLogsPayload", turn_replaying: Mapping[str, int], latest_copy: Mapping[str, object]
) -> "SpendLogsPayload | None":
if not _is_a_response_read(turn):
return turn
stored_response: Final = as_str_mapping(_stored_response(turn))
new_output: Final = [
item if item_id is None else latest_copy[item_id]
for item_id, item in _output_items(turn)
if item_id is None or turn_replaying.get(item_id) == index
]
if stored_response is None or not new_output:
return None
replayed_turn: Final[SpendLogsPayload] = {**turn, "response": {**stored_response, "output": new_output}}
return replayed_turn
def _conversation_turns(spend_logs: Sequence["SpendLogsPayload"]) -> tuple["SpendLogsPayload", ...]:
turns: Final = tuple(filter(_is_a_conversation_turn, spend_logs))
turn_replaying: Final = _turn_replaying_each_item(turns)
latest_copy: Final = _latest_read_copy_of_each_item(turns)
replayed: Final = (
_with_only_new_output(index, turn, turn_replaying, latest_copy) for index, turn in enumerate(turns)
)
return tuple(turn for turn in replayed if turn is not None)
_MCP_AUTO_EXECUTE_OUTPUT_TYPES: Final = frozenset(("mcp_tools_fetched", "tool_execution_results"))
@ -101,7 +187,7 @@ class StoredTurns:
def _compaction_point(
index: int, spend_log: "SpendLogsPayload", proxy_server_request: Mapping[str, object] | None
) -> _ReplayStart | None:
if response_holds_compaction_block(spend_log.get("response")):
if response_holds_compaction_block(_stored_response(spend_log)):
return _ReplayStart(index=index, at_output=True)
if proxy_server_request is not None and holds_compaction_summary(proxy_server_request.get("input")):
return _ReplayStart(index=index, at_output=False)
@ -271,12 +357,10 @@ class ResponsesSessionHandler:
previous_response_id: str,
client_response_ids: tuple[str, ...] = (),
) -> StoredTurns:
spend_logs: Final = tuple(
spend_log
for spend_log in await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id(
spend_logs: Final = _conversation_turns(
await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id(
previous_response_id, client_response_ids
)
if _is_a_conversation_turn(spend_log)
)
return StoredTurns(
spend_logs=spend_logs,
@ -378,7 +462,7 @@ class ResponsesSessionHandler:
############################################################
# Add Output messages for this Spend Log
############################################################
_response_output: Final[object] = spend_log.get("response", "{}")
_response_output: Final = _stored_response(spend_log)
if (responses_api_output := _stored_responses_api_output(_response_output)) is not None:
chat_completion_message_history.extend(
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(

View file

@ -155,6 +155,7 @@ GroupField: TypeAlias = Literal[
"display_name",
"mode",
"service_tiers",
"max_input_tokens",
"token_limits",
"listing_info",
]
@ -855,6 +856,16 @@ class RouterRegistry:
team_id: str | None = None,
) -> tuple[object, ...]: ...
@overload
def group(
self,
name: str,
*,
field: Literal["max_input_tokens"],
include_hidden: bool = False,
team_id: str | None = None,
) -> tuple[int | None, ...]: ...
@overload
def group(
self,
@ -882,9 +893,7 @@ class RouterRegistry:
field: GroupField = "info",
include_hidden: bool = False,
team_id: str | None = None,
) -> (
"ModelGroupInfo | DeploymentModelListingInfo | tuple[int | None, int | None] | tuple[object, ...] | str | None"
):
) -> "ModelGroupInfo | DeploymentModelListingInfo | tuple[int | None, ...] | tuple[object, ...] | str | None":
"""
What the router knows about a model group. ``field``:
@ -894,6 +903,7 @@ class RouterRegistry:
(were ``get_configured_display_name``, ``get_configured_mode``,
``get_configured_token_limits``, ``get_model_listing_info``)
- ``service_tiers``: (was ``get_configured_service_tiers``)
- ``max_input_tokens``: each routable deployment's own ``model_info.max_input_tokens``
"""
r: Final = self._router
if field == "info":
@ -906,6 +916,8 @@ class RouterRegistry:
return r._core_get_configured_mode(name)
if field == "service_tiers":
return r._core_get_configured_service_tiers(name, team_id=team_id)
if field == "max_input_tokens":
return r._core_get_configured_max_input_tokens(name, team_id=team_id)
if field == "token_limits":
return r._core_get_configured_token_limits(name)
if field == "listing_info":

View file

@ -141,6 +141,7 @@ from litellm.litellm_core_utils.get_llm_provider_logic import (
is_registered_custom_provider,
resolve_model_and_provider_for_metadata,
)
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.get_retry_from_policy import (
get_num_retries_from_retry_policy as _get_num_retries_from_retry_policy,
)
@ -475,6 +476,7 @@ from litellm.types.router import (
from litellm.types.services import ServiceTypes
from litellm.types.utils import (
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
COST_MAP_LOOKUP_KEY,
PROMPT_QUOTING_ROUTING_DECISION_FIELDS,
CustomPricingLiteLLMParams,
GenericBudgetConfigType,
@ -486,6 +488,7 @@ from litellm.types.utils import (
StandardLoggingRoutingDecision,
Usage,
all_litellm_params,
echoed_cost_map_fields,
is_server_derived_pricing_key,
shared_backend_model_info,
)
@ -772,6 +775,17 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj
return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {})
def _operator_max_input_tokens(deployment: Mapping[str, object]) -> int | None:
model_info: Final = _configured_model_info(deployment)
catalog_key: Final = model_info.get(COST_MAP_LOOKUP_KEY)
catalog_entry: Final = (
GetModelCostMap.loaded_model_cost_map().get(catalog_key) if isinstance(catalog_key, str) else None
)
if catalog_entry is not None and "max_input_tokens" in echoed_cost_map_fields(model_info, catalog_entry):
return None
return coerce_token_limit(model_info.get("max_input_tokens"))
def _search_tool_support(deployment: DeploymentTypedDict) -> bool | None:
from litellm.responses.litellm_completion_transformation.tool_search import chat_config_builds_anthropic_request
@ -12693,6 +12707,24 @@ class PythonCore(DeprecatedRouterMembers):
for deployment in self._routable_deployments(model_name, team_id)
)
def _core_get_configured_max_input_tokens(
self, model_name: str, team_id: str | None = None
) -> tuple[int | None, ...]:
"""
Return the max_input_tokens each deployment a request from team_id to
model_name can route to (see _routable_deployments) sets in its own
model_info, in model_list order, None for a deployment that sets none
or a malformed one. Unlike get_model_listing_info, neither the cost map
nor a discovered limit is read, so only an operator's own value counts,
and a value a saved /model/info response copied from the cost map is
skipped too.
Returns an empty tuple for wildcard-expanded or unknown names.
"""
return tuple(
_operator_max_input_tokens(deployment) for deployment in self._routable_deployments(model_name, team_id)
)
def _core_get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]:
"""
Whether each deployment a request from team_id to model_name can route
@ -13215,6 +13247,7 @@ class PythonCore(DeprecatedRouterMembers):
"providers": [llm_provider],
"description": model_group_description,
**model_info,
"supports_continuation": model_info.get("supports_continuation") is True,
"supports_fast_mode": True,
"supported_reasoning_efforts": None,
"supported_endpoints": None,
@ -13282,6 +13315,8 @@ class PythonCore(DeprecatedRouterMembers):
and model_info["supports_url_context"] is True
):
model_group_info.supports_url_context = True
if model_info.get("supports_continuation") is True:
model_group_info.supports_continuation = True
if model_info.get("supports_reasoning", None) is not None and model_info["supports_reasoning"] is True:
model_group_info.supports_reasoning = True

View file

@ -1,5 +1,6 @@
from typing import Final
from litellm.constants import VLLM_CHOICE_FIELDS
from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact
from litellm.types.utils import Choices, ModelResponse, ModelResponseStream, TextChoices, TextCompletionResponse
@ -37,16 +38,19 @@ class TextCompletionStreamWrapper:
response["object"] = "text_completion"
response["created"] = chunk.get("created", None)
response["model"] = chunk.get("model", None)
text_choices: Final = TextChoices()
if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices
raise Exception
if chunk["choices"]:
delta: Final = chunk["choices"][0]["delta"]
chat_choice: Final = chunk["choices"][0]
text_choices: Final = TextChoices(
**{field: chat_choice[field] for field in VLLM_CHOICE_FIELDS if field in chat_choice}
)
delta: Final = chat_choice["delta"]
text_choices["text"] = delta["content"]
text_choices["reasoning_content"] = delta.get("reasoning_content")
text_choices["index"] = chunk["choices"][0]["index"]
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
text_choices["index"] = chat_choice["index"]
text_choices["finish_reason"] = chat_choice["finish_reason"]
response["choices"] = [text_choices]
else:
response["choices"] = []

View file

@ -849,6 +849,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum):
ADVISOR_TOOL_2026_03_01 = "advisor-tool-2026-03-01"
PER_TURN_CONTROL_2026_07_01 = "per-turn-control-2026-07-01"
DANGEROUS_TOOL_USE_2026_09_03 = "dangerous-tool-use-2026-09-03"
MCP_CLIENT_2025_04_04 = "mcp-client-2025-04-04"
# Tool search beta header constant (for Anthropic direct API and Microsoft Foundry)

View file

@ -1061,6 +1061,7 @@ class ModelGroupInfo(LiteLLMBaseModel):
supports_vision: bool = Field(default=False)
supports_web_search: bool = Field(default=False)
supports_url_context: bool = Field(default=False)
supports_continuation: bool = Field(default=False)
supports_reasoning: bool = Field(default=False)
supports_function_calling: bool = Field(default=False)
supports_fast_mode: bool = Field(default=False)

View file

@ -227,6 +227,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
supports_tool_search: bool | None
supports_mid_conversation_system: bool | None
supports_url_context: bool | None
supports_continuation: ReadOnly[bool | None]
supports_none_reasoning_effort: bool | None
supports_minimal_reasoning_effort: bool | None
supports_low_reasoning_effort: bool | None
@ -246,6 +247,9 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
bedrock_converse_supports_strict_tools: bool | None
supports_regex_lookaround: ReadOnly[bool | None]
token_count_framing: ReadOnly[int | None]
token_count_tool_framing: ReadOnly[int | None]
token_count_responses_framing: ReadOnly[int | None]
token_count_responses_tool_framing: ReadOnly[int | None]
token_count_model: ReadOnly[TokenCountModelInfo | None]
token_count_tokenizer: ReadOnly[TokenCountTokenizer | None]
@ -4231,12 +4235,21 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset(
"supports_assistant_prefill",
"supports_bedrock_native_structured_output",
"supports_pdf_input",
"supports_continuation",
"reasoning_modes",
"reasoning_contexts",
}
)
DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset({"token_count_framing", "token_count_model"})
DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset(
{
"token_count_framing",
"token_count_tool_framing",
"token_count_responses_framing",
"token_count_responses_tool_framing",
"token_count_model",
}
)
SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = (
frozenset(ModelInfoBase.__required_keys__ | ModelInfoBase.__optional_keys__)

View file

@ -98,7 +98,11 @@ from litellm.litellm_core_utils.model_retirement import is_hidden_as_retired
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
from litellm.litellm_core_utils.tokenizer import strip_special_tokens
from litellm.litellm_core_utils.tool_fields import MalformedField, resolve_deployment_tool_fields
from litellm.litellm_core_utils.tool_fields import (
MalformedField,
resolve_deployment_tool_fields,
tools_without_hosted_mcp,
)
from litellm.rust_bridge import tokenizer as tokenizer_dispatch
from litellm.rust_bridge.catalog import decision
from litellm.rust_bridge.configuration import Decision
@ -3183,6 +3187,14 @@ def supports_url_context(model: str, custom_llm_provider: str | None = None) ->
)
def supports_continuation(model: str, custom_llm_provider: str | None = None) -> bool:
return supports_factory(
model=model,
custom_llm_provider=custom_llm_provider,
key="supports_continuation",
)
def supports_native_streaming(model: str, custom_llm_provider: str | None) -> bool:
"""
Check if the given model supports native streaming and return a boolean value.
@ -5186,7 +5198,12 @@ def remove_sensitive_keys_from_dict(d: dict) -> dict:
return d
def pre_process_optional_params(passed_params: dict, non_default_params: dict, custom_llm_provider: str) -> dict:
_TOOL_LIST: Final = TypeAdapter(list[object])
def pre_process_optional_params(
passed_params: dict, non_default_params: dict, custom_llm_provider: str, model: str, drop_params: bool | None
) -> dict:
"""For .completion(), preprocess optional params"""
optional_params: dict = {}
@ -5243,14 +5260,20 @@ def pre_process_optional_params(passed_params: dict, non_default_params: dict, c
and custom_llm_provider not in litellm.openai_compatible_providers
):
if custom_llm_provider == "ollama":
if "tools" in non_default_params:
sent_tools: Final = _TOOL_LIST.validate_python(non_default_params.pop("tools"))
non_default_params.pop("tool_choice", None) # causes ollama requests to hang
prompt_tools: Final = tools_without_hosted_mcp(
sent_tools, model=model, route=custom_llm_provider, drop_params=bool(drop_params)
)
if sent_tools and not prompt_tools:
return optional_params
optional_params["functions_unsupported_model"] = prompt_tools
elif "functions" in non_default_params:
optional_params["functions_unsupported_model"] = non_default_params.pop("functions")
# ollama actually supports json output
optional_params["format"] = "json"
litellm.add_function_to_prompt = True # so that main.py adds the function call to the prompt
if "tools" in non_default_params:
optional_params["functions_unsupported_model"] = non_default_params.pop("tools")
non_default_params.pop("tool_choice", None) # causes ollama requests to hang
elif "functions" in non_default_params:
optional_params["functions_unsupported_model"] = non_default_params.pop("functions")
elif litellm.add_function_to_prompt: # if user opts to add it to prompt instead
optional_params["functions_unsupported_model"] = non_default_params.pop(
"tools", non_default_params.pop("functions", None)
@ -5385,6 +5408,8 @@ def get_optional_params(
passed_params=passed_params,
non_default_params=non_default_params,
custom_llm_provider=custom_llm_provider,
model=model,
drop_params=drop_params,
)
def _check_valid_arg(supported_params: list[str]):
@ -7360,6 +7385,7 @@ def get_model_info_helper(
),
supports_web_search=_model_info.get("supports_web_search", None),
supports_url_context=_model_info.get("supports_url_context", None),
supports_continuation=_model_info.get("supports_continuation", None),
supports_reasoning=_model_info.get("supports_reasoning", None),
supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None),
supports_legacy_thinking=_model_info.get("supports_legacy_thinking", None),
@ -7398,6 +7424,9 @@ def get_model_info_helper(
supports_image_size=_model_info.get("supports_image_size", None),
supported_audio_formats=_model_info.get("supported_audio_formats", None),
token_count_framing=_model_info.get("token_count_framing", None),
token_count_tool_framing=_model_info.get("token_count_tool_framing", None),
token_count_responses_framing=_model_info.get("token_count_responses_framing", None),
token_count_responses_tool_framing=_model_info.get("token_count_responses_tool_framing", None),
token_count_model=_model_info.get("token_count_model", None),
token_count_tokenizer=_model_info.get("token_count_tokenizer", None),
vertex_ai_audio_api=_model_info.get("vertex_ai_audio_api", None),
@ -7516,6 +7545,7 @@ def get_model_info(
supports_pdf_input: Optional[bool]
supports_web_search: Optional[bool]
supports_url_context: Optional[bool]
supports_continuation: Optional[bool]
supports_reasoning: Optional[bool]
Raises:
Exception: If the model is not mapped yet.

View file

@ -13478,6 +13478,10 @@
"input_cost_per_token": 2e-07,
"output_cost_per_token": 5e-07,
"litellm_provider": "azure_ai",
"token_count_framing": 149,
"token_count_tool_framing": 302,
"token_count_responses_framing": 145,
"token_count_responses_tool_framing": 100,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"mode": "chat",
@ -13831,7 +13835,7 @@
"input_cost_per_token_flex": 3.7e-07,
"input_cost_per_token_priority": 1.295e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
@ -13845,7 +13849,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/ap-northeast-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -14002,7 +14006,7 @@
"input_cost_per_token_flex": 3.7e-07,
"input_cost_per_token_priority": 1.295e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
@ -14016,7 +14020,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/ap-south-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -14140,12 +14144,12 @@
"input_cost_per_token": 7.4e-07,
"input_cost_per_token_flex": 3.7e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
"output_cost_per_token_flex": 1.11e-06,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html",
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md",
"supports_audio_input": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
@ -14250,7 +14254,7 @@
"input_cost_per_token_flex": 3.7e-07,
"input_cost_per_token_priority": 1.295e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
@ -14264,7 +14268,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/eu-north-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -14786,7 +14790,7 @@
"input_cost_per_token_flex": 3.7e-07,
"input_cost_per_token_priority": 1.295e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
@ -14800,7 +14804,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/sa-east-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -15035,7 +15039,7 @@
"input_cost_per_token_flex": 3.1e-07,
"input_cost_per_token_priority": 1.085e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
@ -15049,7 +15053,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/us-east-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -15156,7 +15160,7 @@
"input_cost_per_token_flex": 3.1e-07,
"input_cost_per_token_priority": 1.085e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
@ -15170,7 +15174,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/us-east-2/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -15552,6 +15556,7 @@
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md"
},
"bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 2.65e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 8000,
@ -15562,6 +15567,7 @@
"supports_pdf_input": true
},
"bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 8000,
@ -15708,7 +15714,7 @@
"input_cost_per_token_flex": 3.1e-07,
"input_cost_per_token_priority": 1.085e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
@ -15722,7 +15728,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/us-west-2/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -23519,12 +23525,12 @@
]
},
"deepgram/streaming/nova-3": {
"input_cost_per_second": 8e-05,
"input_cost_per_second": 0.00012833,
"litellm_provider": "deepgram",
"metadata": {
"calculation": "$0.0048/60 seconds = $0.00008000 per second",
"calculation": "$0.0077/60 seconds = $0.00012833 per second",
"note": "Nova-3 monolingual streaming, pay as you go",
"original_pricing_per_minute": 0.0048
"original_pricing_per_minute": 0.0077
},
"mode": "audio_transcription",
"output_cost_per_second": 0.0,
@ -23533,13 +23539,28 @@
"/v1/listen"
]
},
"deepgram/streaming/nova-3-multilingual": {
"input_cost_per_second": 9.667e-05,
"deepgram/streaming/nova-3-medical": {
"input_cost_per_second": 0.00012833,
"litellm_provider": "deepgram",
"metadata": {
"calculation": "$0.0058/60 seconds = $0.00009667 per second",
"calculation": "$0.0077/60 seconds = $0.00012833 per second",
"note": "Nova-3 Medical streaming, launch price from Deepgram's 2025-03-03 announcement. deepgram.com/pricing does not list Nova-3 Medical, and this matches its current Nova-3 monolingual streaming pay as you go rate",
"original_pricing_per_minute": 0.0077
},
"mode": "audio_transcription",
"output_cost_per_second": 0.0,
"source": "https://www.businesswire.com/news/home/20250303470439/en",
"supported_endpoints": [
"/v1/listen"
]
},
"deepgram/streaming/nova-3-multilingual": {
"input_cost_per_second": 0.00015333,
"litellm_provider": "deepgram",
"metadata": {
"calculation": "$0.0092/60 seconds = $0.00015333 per second",
"note": "Nova-3 multilingual (language=multi) streaming, pay as you go",
"original_pricing_per_minute": 0.0058
"original_pricing_per_minute": 0.0092
},
"mode": "audio_transcription",
"output_cost_per_second": 0.0,
@ -24519,7 +24540,7 @@
"input_cost_per_token_flex": 3.1e-07,
"input_cost_per_token_priority": 1.085e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 164000,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
@ -24530,7 +24551,7 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json",
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md",
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
@ -54457,6 +54478,8 @@
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"token_count_framing": 185,
"token_count_tool_framing": 214,
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"mode": "chat",
@ -62640,7 +62663,9 @@
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"token_count_framing": 23,
"token_count_tool_framing": 12
},
"deepseek/deepseek-v4-flash-vision-exp": {
"cache_creation_input_token_cost": 0.0,
@ -62746,7 +62771,9 @@
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": false
"supports_vision": false,
"token_count_framing": 76,
"token_count_tool_framing": 75
},
"tencent/deepseek-v4-pro": {
"cache_creation_input_token_cost": 0.0,
@ -64032,6 +64059,7 @@
"input_cost_per_token": 3e-07,
"input_cost_per_token_priority": 3.75e-07,
"litellm_provider": "fireworks_ai",
"token_count_framing": 23,
"max_input_tokens": 1048576,
"max_output_tokens": 393216,
"mode": "chat",
@ -64133,6 +64161,7 @@
"input_cost_per_token": 3e-07,
"input_cost_per_token_priority": 3.75e-07,
"litellm_provider": "fireworks_ai",
"token_count_framing": 23,
"max_input_tokens": 1048576,
"max_output_tokens": 393216,
"mode": "chat",
@ -76801,6 +76830,7 @@
"supports_audio_input": true,
"supports_audio_output": false,
"supports_computer_use": true,
"supports_continuation": true,
"supports_function_calling": true,
"supports_minimal_reasoning_effort": false,
"supports_native_streaming": true,
@ -76860,6 +76890,7 @@
],
"supports_audio_input": true,
"supports_computer_use": true,
"supports_continuation": true,
"supports_function_calling": true,
"supports_minimal_reasoning_effort": false,
"supports_native_streaming": true,
@ -76917,6 +76948,7 @@
"text"
],
"supports_audio_input": true,
"supports_continuation": true,
"supports_function_calling": true,
"supports_minimal_reasoning_effort": false,
"supports_native_streaming": true,

View file

@ -26,7 +26,7 @@ dependencies = [
"tiktoken>=0.12.0,<1.0; python_version == '3.14'",
"tiktoken>=0.14.0,<1.0; python_version >= '3.15'",
"tokenizers>=0.21.0,<1.0",
"huggingface-hub>=0.34.0,<2.0",
"huggingface-hub>=0.34.0,<3.0",
"click>=8.0.0,<9.0",
"jinja2>=3.1.6,<4.0",
"aiohttp>=3.14.2,<4.0",

View file

@ -3,7 +3,7 @@ raw HTTP client imports (requests, urllib.request, httpx, aiohttp, http.client)
banned in suite code, and tests/e2e_harness, which tests that harness, is held to the
same ban. Importing requests' exception types for catching is fine anywhere; a small
allowlist grandfathers the files that legitimately make raw calls (the transport
itself, the root conftest liveness probe, the claude_code version resolver's constant
itself, the root conftest liveness probe, the cli_harness npm version resolver's constant
registry URL fetch, and the mcp OAuth client, whose httpx client is the object the
official mcp SDK's streamable_http_client requires and so cannot go through the sync
requests transport). Referenced by tests/e2e/AGENTS.md."""
@ -24,7 +24,7 @@ BANNED_MODULES = ("requests", "urllib.request", "http.client", "httpx", "aiohttp
ALLOWED_RAW_CLIENT_FILES = {
"e2e/e2e_http.py": ("requests",),
"e2e/conftest.py": ("requests",),
"e2e/claude_code/pr_gate_version_resolver.py": ("urllib.request",),
"e2e/cli_harness/npm_version_resolver.py": ("urllib.request",),
}
EXCEPTION_ONLY_NAMES = frozenset({"RequestException", "ConnectionError", "Timeout", "HTTPError"})

File diff suppressed because one or more lines are too long

View file

@ -16,23 +16,18 @@ from __future__ import annotations
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Union
from typing import Any, Callable, Dict, Final, List, Mapping, Optional, Sequence, Tuple, Union
from cli_harness.lockdown import isolated_home, locked_down_env
from cli_harness.rate_limiter import RateBucket, RateLimiter, get_default_limiter
from e2e_metadata import step
from claude_code.rate_limiter import (
RateLimiter,
get_default_limiter,
infer_provider,
)
CLAUDE_CLI_DEFAULT = "claude"
# 120s is fine for a single isolated CLI call against an unloaded
# upstream, but the matrix run launches up to 75 concurrent calls and
@ -77,61 +72,36 @@ Anthropic / Bedrock / Vertex) to reset. Both knobs are env-tunable so
a matrix run can trade wall time for resilience without code edits;
retries=0 disables the behavior entirely."""
# Env vars the `claude` Node CLI legitimately needs to function:
# locating its own binary + node, basic locale/terminal plumbing.
# Deliberately excludes every credential-bearing var that the
# surrounding CI job sets for the proxy (ANTHROPIC_API_KEY, AWS_*,
# AZURE_*, VERTEXAI_CREDENTIALS, GITHUB_TOKEN, OPENAI_API_KEY,
# DATABASE_URL, ...). The CLI talks to the proxy via the explicit
# ANTHROPIC_BASE_URL/ANTHROPIC_AUTH_TOKEN we set below — it has no
# business reading the proxy's upstream credentials, and a
# compromised CLI release shouldn't be able to exfiltrate them out
# of the CI environment.
#
# `HOME` is intentionally NOT in this list: see `_make_isolated_home`
# below. We give the CLI a fresh empty per-invocation HOME so a
# compromised claude package or a model-directed `Read` tool call
# can't reach files like `~/.config/gh/hosts.yml`, `~/.ssh/id_rsa`,
# or `~/.bash_history` on the cron VM (and on the CircleCI executor
# the same isolation prevents accidentally exposing checkout-adjacent
# files even though the runner home is ephemeral there).
_CLI_ENV_ALLOWLIST: tuple = (
"PATH",
"USER",
"LOGNAME",
"SHELL",
"TERM",
"TMPDIR",
"LANG",
"LC_ALL",
"LC_CTYPE",
"NODE_PATH",
"NVM_DIR",
"NVM_BIN",
)
def infer_provider(model: str) -> RateBucket:
"""Map a model alias to its compat-matrix rate bucket.
def _make_isolated_home() -> str:
"""Create a fresh empty HOME directory for a single `claude` subprocess.
The CLI needs *a* writable HOME (it caches per-session state under
`$HOME/.claude/projects/<sha>/`), but it has no legitimate need
for the *user's* HOME. Handing it the real one means a compromised
`@anthropic-ai/claude-code` release, or a model-directed `Read`
tool call during a PDF/vision cell, can read host files like
`~/.config/gh/hosts.yml` (GitHub CLI host token), `~/.ssh/`,
`~/.bash_history`, or any other dotfile under the runtime user's
home. On the cron VM the runtime user is a real interactive
account (`mateo`) with a populated home directory, so this is a
real exfiltration surface.
The directory is created under `tempfile.gettempdir()` (which is
`/tmp` on Linux; under systemd's `PrivateTmp=true` that's a
per-service tmpfs that the service user can't otherwise reach).
Caller is responsible for `shutil.rmtree`-ing it after the
subprocess exits.
The matrix column layout is fixed; aliases registered in the proxy
encode the provider via a suffix (`-bedrock-converse`,
`-bedrock-invoke`, `-azure`, `-vertex`, `-openai`, `-azure-openai`,
`-bedrock-mantle`) or its absence (Anthropic). Order matters:
`-azure-openai` also ends with `-openai`, and the bedrock suffixes
all contain `bedrock`, so the more-specific suffixes are tested
first.
"""
return tempfile.mkdtemp(prefix="claude-cli-home-")
if not model:
raise ValueError("model must be a non-empty string")
lower: Final = model.lower()
if lower.endswith("-azure-openai"):
return RateBucket.AZURE_OPENAI
if lower.endswith("-openai"):
return RateBucket.OPENAI
if lower.endswith("-bedrock-mantle"):
return RateBucket.BEDROCK_MANTLE
if lower.endswith("-bedrock-converse"):
return RateBucket.BEDROCK_CONVERSE
if lower.endswith("-bedrock-invoke"):
return RateBucket.BEDROCK_INVOKE
if lower.endswith("-azure"):
return RateBucket.AZURE
if lower.endswith("-vertex"):
return RateBucket.VERTEX_AI
return RateBucket.ANTHROPIC
_FIXED_CLI_USER_ID = "0" * 64
@ -176,7 +146,7 @@ def _stable_cli_state() -> Tuple[str, str]:
Pinning both makes the bodies repeatable across builds.
This narrows what survives rather than widening it: HOME stays fresh and
empty per invocation, so the isolation `_make_isolated_home` describes is
empty per invocation, so the isolation `cli_harness.lockdown` describes is
unchanged, and the CLI's own state no longer outlives the pod either. The
working directory is deliberately not the checkout, so a model-directed
`Read` sees an empty directory instead of the repository.
@ -291,38 +261,26 @@ def run_claude(
cmd.append("--")
cmd.append(prompt)
# Build a minimal env for the CLI subprocess: only the allowlisted
# process-runtime vars from os.environ, plus the explicit proxy
# creds, plus any caller-supplied overrides. See _CLI_ENV_ALLOWLIST
# above for the security rationale.
env: Dict[str, str] = {
key: os.environ[key] for key in _CLI_ENV_ALLOWLIST if key in os.environ
}
env["ANTHROPIC_BASE_URL"] = base_url
env["ANTHROPIC_AUTH_TOKEN"] = api_key
env["DISABLE_AUTOUPDATER"] = "1"
# Hand the CLI a fresh empty HOME so a compromised claude package
# or a model-directed Read tool call can't see the runtime user's
# real dotfiles. Created here, removed in the `finally` below
# regardless of how the subprocess exits.
isolated_home = _make_isolated_home()
env["HOME"] = isolated_home
config_dir, workspace = _stable_cli_state()
env["CLAUDE_CONFIG_DIR"] = config_dir
if extra_env:
env.update(extra_env)
# Throttle by provider *before* launching the CLI. Doing this here
# (rather than per-test) means every code path that lands on
# `run_claude` is rate-limited automatically — including
# `run_claude_models_parallel`, which is the hot path during the
# full matrix run.
limiter = rate_limiter if rate_limiter is not None else get_default_limiter()
provider = infer_provider(model)
limiter.acquire(provider)
run_fn = runner or subprocess.run
try:
with isolated_home("claude-cli-home-") as home:
config_dir, workspace = _stable_cli_state()
env = locked_down_env(
home=home,
overrides={
"ANTHROPIC_BASE_URL": base_url,
"ANTHROPIC_AUTH_TOKEN": api_key,
"DISABLE_AUTOUPDATER": "1",
"CLAUDE_CONFIG_DIR": config_dir,
**(extra_env or {}),
},
)
limiter.acquire(infer_provider(model))
try:
completed = run_fn(
cmd,
@ -340,15 +298,6 @@ def run_claude(
) from exc
except subprocess.TimeoutExpired as exc:
raise ClaudeCLIError(f"claude CLI timed out after {timeout}s") from exc
finally:
# Best-effort cleanup. If the subprocess wrote a `.claude/`
# session dir under the isolated HOME, we remove it here so
# parallel matrix runs don't accumulate per-call tmpdirs.
# `ignore_errors=True` because rmtree races with any
# not-yet-reaped child (SIGTERM'd `claude` on a host-side
# timeout) are benign — the next matrix run starts from a
# fresh tmpdir anyway.
shutil.rmtree(isolated_home, ignore_errors=True)
events = _parse_stream_json(completed.stdout or "")
text = _extract_assistant_text(events)

View file

@ -52,7 +52,7 @@ than as a GitHub Action or on a dedicated VM. Trade-offs:
the stable venv doesn't install) is never loaded. The tag's own
`tests/e2e/` is deliberately not used.
3. **Resolves and installs the Claude Code CLI under test**:
`pr_gate_version_resolver.py` (run on the venv, the image has no
`tests/e2e/cli_harness/npm_version_resolver.py` (run on the venv, the image has no
Python of its own) picks the newest `@anthropic-ai/claude-code` npm
release published at least 3 days ago, the same buffer the PR gate
uses, unless `CLAUDE_CODE_VERSION` pins one, and

View file

@ -8,7 +8,7 @@
# Releases API.
# 2. Update a long-lived worktree at $WORKTREE to that tag and `uv sync` it.
# 3. Resolve the Claude Code CLI version under test (the newest npm
# release at least 3 days old, via pr_gate_version_resolver.py, or
# release at least 3 days old, via cli_harness/npm_version_resolver.py, or
# $CLAUDE_CODE_VERSION when set) and download that release's
# linux-x64 binary into the run's scratch dir, checksum-verified
# against the vendor's release manifest (install_claude_code.sh).
@ -329,7 +329,7 @@ PROXY_CONFIG="${WORKTREE}/tests/e2e/claude_code/test_config.yaml"
if [[ -z "${CLAUDE_CODE_VERSION}" ]]; then
CLAUDE_CODE_VERSION="$(
cd "${WORKTREE}" \
&& "${WORKTREE_UV}" run --no-sync python "${POPULATOR_DIR}/../pr_gate_version_resolver.py"
&& "${WORKTREE_UV}" run --no-sync python "${POPULATOR_DIR}/../../cli_harness/npm_version_resolver.py" @anthropic-ai/claude-code
)" || die "could not resolve the Claude Code version to test"
log "resolved claude code: ${CLAUDE_CODE_VERSION}"
else

View file

@ -31,8 +31,7 @@ from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING
from pydantic import BaseModel
from cli_harness.rate_limiter import RateLimiter, get_default_limiter
from e2e_http import (
NetworkError,
RateLimitedError,
@ -59,12 +58,9 @@ from models import (
JsonSchemaProperty,
ToolInputSchema,
)
from pydantic import BaseModel
from claude_code.rate_limiter import (
RateLimiter,
get_default_limiter,
infer_provider,
)
from claude_code.cli_driver import infer_provider
if TYPE_CHECKING:
from proxy_client import ProxyClient

View file

@ -1,155 +1,16 @@
"""Claude Code PR-Gate Version Resolver.
Resolves the `@anthropic-ai/claude-code` npm version that the PR-gate CI
job installs. Selects the newest version (by publish timestamp) whose
publish timestamp is at least 3 days old. The 3-day window is a security
review buffer — see PRD #26476, "Version resolvers".
Two surfaces:
- ``resolve_pr_gate_version(...)`` — the importable function. Accepts
pre-fetched npm metadata (for unit tests) or a custom ``fetcher``
callable. The default fetcher hits the public npm registry.
- ``python -m claude_code.pr_gate_version_resolver`` — prints the
resolved version string to stdout, suitable for piping into a shell
``$(...)`` substitution inside the CircleCI job.
The CLI form is what CircleCI runs at job start; engineers reading the
job log can see the selected version on a single line above the
``npm install -g`` step (acceptance criterion: "the selected Claude
Code version is logged in the CI output").
"""
"""The path the Buildkite runner images run (`python3 tests/e2e/claude_code/pr_gate_version_resolver.py`) to pick the
Claude Code release they install. They fall back to @latest when this file is missing, so keep it until they call
`cli_harness/npm_version_resolver.py` themselves."""
from __future__ import annotations
import json
import subprocess
import sys
import urllib.request
from datetime import datetime, timedelta, timezone
from typing import Callable, Mapping, Optional
PACKAGE_NAME = "@anthropic-ai/claude-code"
NPM_REGISTRY_URL = "https://registry.npmjs.org/{package}"
DEFAULT_MIN_AGE = timedelta(days=3)
DEFAULT_FETCH_TIMEOUT_SECONDS = 30
# npm's `time` map mixes per-version timestamps with these meta keys.
_TIME_META_KEYS = frozenset({"created", "modified"})
class NoEligibleVersionError(RuntimeError):
"""Raised when no version in the npm metadata satisfies the min-age cutoff."""
def _parse_npm_timestamp(value: str) -> datetime:
"""Parse the ISO-8601 timestamps npm emits (always UTC, may use ``Z``)."""
if value.endswith("Z"):
value = value[:-1] + "+00:00"
parsed = datetime.fromisoformat(value)
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed
def _default_fetcher(package_name: str) -> dict:
"""Fetch the npm packument for ``package_name`` over HTTPS.
Uses urllib (stdlib) so this module has no extra dependencies in the
CI environment. Returns the raw JSON dict.
"""
# urllib.parse.quote would encode the leading '@' / '/' which the
# npm registry expects literally; do a minimal hand-roll instead.
url = NPM_REGISTRY_URL.format(package=package_name.replace("/", "%2F"))
req = urllib.request.Request(url, headers={"Accept": "application/json"})
with urllib.request.urlopen( # noqa: S310 — registry URL is constant
req, timeout=DEFAULT_FETCH_TIMEOUT_SECONDS
) as response:
body = response.read().decode("utf-8")
return json.loads(body)
def resolve_pr_gate_version(
*,
metadata: Optional[Mapping] = None,
fetcher: Optional[Callable[[str], Mapping]] = None,
as_of: Optional[datetime] = None,
min_age: timedelta = DEFAULT_MIN_AGE,
package_name: str = PACKAGE_NAME,
) -> str:
"""Return the newest npm version of ``package_name`` published >= ``min_age`` ago.
"Newest" means newest by **publish time**, not semver string order —
if a patch lands on an older major after a newer release, the
patched line is the eligible one. A version npm has unpublished keeps
its ``time`` entry but drops out of ``versions``, so only versions
still present in ``versions`` are candidates.
Args:
metadata: Pre-fetched npm packument (skips the HTTP call). Useful
for unit tests.
fetcher: Callable taking a package name and returning the
packument. Defaults to a stdlib HTTPS fetcher.
as_of: The clock used to decide whether a version is "old
enough". Defaults to ``datetime.now(timezone.utc)``.
min_age: Minimum publish age. Defaults to 3 days.
package_name: Defaults to ``@anthropic-ai/claude-code``.
Raises:
NoEligibleVersionError: when no version in the registry meets
the age cutoff.
"""
if metadata is None:
fetch = fetcher or _default_fetcher
metadata = fetch(package_name)
times = metadata.get("time") or {}
versions = metadata.get("versions") or {}
if as_of is None:
as_of = datetime.now(timezone.utc)
cutoff = as_of - min_age
eligible: list[tuple[datetime, str]] = []
for version, raw_ts in times.items():
if version in _TIME_META_KEYS:
continue
if version not in versions:
continue
if not isinstance(raw_ts, str):
continue
if "-" in version:
continue
published = _parse_npm_timestamp(raw_ts)
if published <= cutoff:
eligible.append((published, version))
if not eligible:
raise NoEligibleVersionError(
f"no version of {package_name} is at least {min_age} old "
f"as of {as_of.isoformat()}"
)
eligible.sort(key=lambda pair: pair[0], reverse=True)
return eligible[0][1]
def _main(argv: list[str]) -> int:
"""Print the resolved version to stdout. Exit code 0 on success.
Stderr carries the human-readable announcement so the version can be
captured cleanly with ``$(python -m ...)`` in shell.
"""
try:
version = resolve_pr_gate_version()
except Exception as exc: # noqa: BLE001 — CLI surface, want everything
print(f"pr_gate_version_resolver: {exc}", file=sys.stderr) # noqa: T201
return 1
print( # noqa: T201
f"pr_gate_version_resolver: selected {PACKAGE_NAME}@{version}",
file=sys.stderr,
)
print(version) # noqa: T201
return 0
from pathlib import Path
from typing import Final
_RESOLVER: Final = Path(__file__).resolve().parent.parent / "cli_harness" / "npm_version_resolver.py"
_PACKAGE: Final = "@anthropic-ai/claude-code"
if __name__ == "__main__":
raise SystemExit(_main(sys.argv[1:]))
raise SystemExit(subprocess.run((sys.executable, "-I", str(_RESOLVER), _PACKAGE), check=False).returncode)

View file

@ -1,378 +0,0 @@
"""Cross-process token-bucket rate limiter for the Claude Code compat suite.
The compat matrix runs 75 live `claude` CLI invocations (25 cells × 3
Claude tiers per cell). When pytest-xdist fans these out across worker
processes, each worker would maintain its own in-memory rate limiter
and the *aggregate* request rate hitting any one upstream provider
would be `workers × per-worker rate` — exactly the situation that
trips Anthropic / Azure / Bedrock / Vertex 429s in the middle of a
matrix run and silently flips green cells red.
The fix is a **shared, cross-process** token bucket per provider,
backed by a small JSON state file and an OS-level `flock`. Each
`run_claude` invocation acquires one token (sleeping if the bucket is
empty) before launching the CLI; refills happen lazily based on wall
time, so workers can be killed and restarted without losing or
double-spending budget.
Configuration is driven entirely by environment variables so a
binary-search workflow can shift per-provider rates without code
edits:
LITELLM_COMPAT_RATE_ANTHROPIC (req/s, default 5.0)
LITELLM_COMPAT_RATE_AZURE (req/s, default 5.0)
LITELLM_COMPAT_RATE_VERTEX_AI (req/s, default 5.0)
LITELLM_COMPAT_RATE_BEDROCK_CONVERSE (req/s, default 5.0)
LITELLM_COMPAT_RATE_BEDROCK_INVOKE (req/s, default 5.0)
LITELLM_COMPAT_RATE_OPENAI (req/s, default 5.0)
LITELLM_COMPAT_RATE_AZURE_OPENAI (req/s, default 5.0)
LITELLM_COMPAT_RATE_BEDROCK_MANTLE (req/s, default 5.0)
LITELLM_COMPAT_RATE_BURST (per-bucket burst override;
default = rate)
LITELLM_COMPAT_RATE_STATE_DIR (state file directory;
default = $TMPDIR/litellm-claude-compat-ratelimit)
A rate of 0 (or any non-positive value) disables throttling for that
provider — useful when you trust the upstream to handle the burst or
when running unit-test-shaped workloads that never actually hit the
network.
The provider id is inferred from the model id by `infer_provider`,
mirroring the matrix's column layout (`anthropic`, `azure`,
`vertex_ai`, `bedrock_converse`, `bedrock_invoke`, `openai`,
`azure_openai`, `bedrock_mantle`). The `vertex_ai_gpt` matrix column
has no bucket: its cells are static not_applicable stubs that never
reach the network.
"""
from __future__ import annotations
import contextlib
import json
import math
import os
import tempfile
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Iterator, Mapping, Optional
# `fcntl` is POSIX-only; the suite is Linux/macOS only, so we don't
# attempt a Windows fallback. Importing at module load fails fast on
# the (currently non-existent) Windows runner so we don't silently
# degrade to no-locking behavior.
import fcntl
PROVIDER_ANTHROPIC = "anthropic"
PROVIDER_AZURE = "azure"
PROVIDER_VERTEX_AI = "vertex_ai"
PROVIDER_BEDROCK_CONVERSE = "bedrock_converse"
PROVIDER_BEDROCK_INVOKE = "bedrock_invoke"
PROVIDER_OPENAI = "openai"
PROVIDER_AZURE_OPENAI = "azure_openai"
PROVIDER_BEDROCK_MANTLE = "bedrock_mantle"
ALL_PROVIDERS = (
PROVIDER_ANTHROPIC,
PROVIDER_AZURE,
PROVIDER_VERTEX_AI,
PROVIDER_BEDROCK_CONVERSE,
PROVIDER_BEDROCK_INVOKE,
PROVIDER_OPENAI,
PROVIDER_AZURE_OPENAI,
PROVIDER_BEDROCK_MANTLE,
)
DEFAULT_RATE = 5.0 # req/s per provider, conservative starting point
RATE_ENV_PREFIX = "LITELLM_COMPAT_RATE_"
BURST_ENV = "LITELLM_COMPAT_RATE_BURST"
STATE_DIR_ENV = "LITELLM_COMPAT_RATE_STATE_DIR"
DEFAULT_STATE_DIR_NAME = "litellm-claude-compat-ratelimit"
def infer_provider(model: str) -> str:
"""Map a model alias to its compat-matrix provider id.
The matrix column layout is fixed; aliases registered in the proxy
encode the provider via a suffix (`-bedrock-converse`,
`-bedrock-invoke`, `-azure`, `-vertex`, `-openai`, `-azure-openai`,
`-bedrock-mantle`) or its absence (Anthropic). Order matters:
`-azure-openai` also ends with `-openai`, and the bedrock suffixes
all contain `bedrock`, so the more-specific suffixes are tested
first.
"""
if not model:
raise ValueError("model must be a non-empty string")
lower = model.lower()
if lower.endswith("-azure-openai"):
return PROVIDER_AZURE_OPENAI
if lower.endswith("-openai"):
return PROVIDER_OPENAI
if lower.endswith("-bedrock-mantle"):
return PROVIDER_BEDROCK_MANTLE
if lower.endswith("-bedrock-converse"):
return PROVIDER_BEDROCK_CONVERSE
if lower.endswith("-bedrock-invoke"):
return PROVIDER_BEDROCK_INVOKE
if lower.endswith("-azure"):
return PROVIDER_AZURE
if lower.endswith("-vertex"):
return PROVIDER_VERTEX_AI
return PROVIDER_ANTHROPIC
@dataclass(frozen=True)
class ProviderConfig:
"""Static config snapshot for one provider's bucket.
Captured up front (rather than re-read per acquire) so the limiter's
behavior in a single process is stable even if env vars are mutated
mid-run. A fresh `RateLimiter` picks up env changes on construction.
"""
rate_per_sec: float
burst: float
@property
def enabled(self) -> bool:
return self.rate_per_sec > 0 and self.burst > 0
def load_config(
env: Optional[Mapping[str, str]] = None,
) -> Dict[str, ProviderConfig]:
"""Build a {provider: ProviderConfig} from env, applying defaults.
Parsing failures fall back to the default rate rather than
crashing the test session — a typo in `LITELLM_COMPAT_RATE_AZURE`
should not silently disable throttling, but it also shouldn't
abort 75 live tests with a `ValueError` ten minutes in.
"""
src = env if env is not None else os.environ
def _as_float(value: Optional[str], default: float) -> float:
if value is None or value == "":
return default
try:
return float(value)
except (TypeError, ValueError):
return default
burst_override = _as_float(src.get(BURST_ENV), -1.0)
out: Dict[str, ProviderConfig] = {}
for provider in ALL_PROVIDERS:
env_key = RATE_ENV_PREFIX + provider.upper()
rate = _as_float(src.get(env_key), DEFAULT_RATE)
burst = burst_override if burst_override > 0 else max(rate, 1.0)
out[provider] = ProviderConfig(rate_per_sec=rate, burst=burst)
return out
def _state_dir(env: Optional[Mapping[str, str]] = None) -> Path:
"""Resolve the directory holding per-provider state files.
A user-supplied `LITELLM_COMPAT_RATE_STATE_DIR` wins for tests and
container environments where `$TMPDIR` may be ephemeral or shared
in surprising ways. The directory is created lazily with
`parents=True, exist_ok=True` so first-run setup needs no
fixture wiring.
"""
src = env if env is not None else os.environ
explicit = src.get(STATE_DIR_ENV)
if explicit:
return Path(explicit)
return Path(tempfile.gettempdir()) / DEFAULT_STATE_DIR_NAME
class RateLimiter:
"""Cross-process token bucket per provider.
Each `acquire(provider)` call:
1. Opens (creating if needed) `<state_dir>/<provider>.json`.
2. Holds an exclusive `flock` while reading + updating the
{tokens, last_refill} state.
3. Refills tokens based on `now - last_refill`, capped at burst.
4. If tokens >= 1, subtracts one and returns immediately.
5. Otherwise, computes the wall-time delay needed to earn a
single token at the configured rate, releases the lock, and
sleeps. After sleeping it retries — staying under the lock
while sleeping would serialize all workers behind whichever
one held it longest.
This bounds the *aggregate* req/s seen by the upstream, regardless
of how many xdist workers, threads, or processes are concurrently
running tests against the same provider.
A `_clock` / `_sleep` injection seam keeps the unit tests fast and
deterministic; production callers should never override either.
"""
def __init__(
self,
config: Optional[Mapping[str, ProviderConfig]] = None,
state_dir: Optional[Path] = None,
clock: Optional[callable] = None,
sleep: Optional[callable] = None,
) -> None:
self._config = dict(config) if config is not None else load_config()
self._state_dir = Path(state_dir) if state_dir is not None else _state_dir()
self._clock = clock or time.monotonic
self._sleep = sleep or time.sleep
# `_state_dir.mkdir` once on construction is fine; concurrent
# workers all racing to create the same directory is benign.
self._state_dir.mkdir(parents=True, exist_ok=True)
def acquire(self, provider: str) -> float:
"""Block until one token is available for `provider`.
Returns the cumulative wall-time spent waiting (0.0 when the
bucket had budget and we returned immediately). Callers can
log this to attribute slow cells to throttling vs. upstream
latency — same role `DriverResult.duration_ms` plays for the
actual CLI invocation.
"""
cfg = self._config.get(provider)
if cfg is None or not cfg.enabled:
return 0.0
path = self._state_path(provider)
total_waited = 0.0
while True:
now = self._clock()
sleep_for = self._try_consume(path, cfg, now)
if sleep_for <= 0:
return total_waited
self._sleep(sleep_for)
total_waited += sleep_for
def _state_path(self, provider: str) -> Path:
return self._state_dir / f"{provider}.json"
def _try_consume(self, path: Path, cfg: ProviderConfig, now: float) -> float:
"""Atomically refill and try to take one token.
Returns 0.0 if a token was consumed, or a positive sleep
duration (seconds) if the caller must wait before retrying.
We hold an exclusive `flock` only across the read-modify-write
of the JSON state — never across a `sleep` — so workers don't
serialize while one of them is parked.
"""
# `os.open` + `os.O_CREAT | os.O_RDWR` gives us a fd we can
# both lock and read/write through. Opening with `"a+"` then
# seeking is equivalent but uglier; this version is closer to
# the canonical flock recipe.
fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o600)
try:
fcntl.flock(fd, fcntl.LOCK_EX)
try:
tokens, last_refill = self._read_state(fd, cfg, now)
tokens, last_refill = self._refill(tokens, last_refill, now, cfg)
if tokens >= 1.0:
tokens -= 1.0
self._write_state(fd, tokens, last_refill)
return 0.0
# Not enough budget. Persist the refilled state so a
# subsequent caller doesn't have to redo the math, then
# release the lock and tell the caller how long to
# sleep before retrying.
self._write_state(fd, tokens, last_refill)
deficit = 1.0 - tokens
# `deficit / rate` seconds will earn exactly enough
# for one token. Add a tiny safety margin so we don't
# wake up nanoseconds early and spin.
return (deficit / cfg.rate_per_sec) + 1e-3
finally:
fcntl.flock(fd, fcntl.LOCK_UN)
finally:
os.close(fd)
@staticmethod
def _read_state(fd: int, cfg: ProviderConfig, now: float) -> tuple:
"""Read {tokens, last_refill} from `fd`, defaulting to a full
bucket on a missing/empty/corrupt file.
New files start full so the very first request never waits;
corrupt files are treated like new files because the
alternative — refusing to run — is worse than briefly
over-spending one bucket's worth of budget.
"""
os.lseek(fd, 0, os.SEEK_SET)
raw = os.read(fd, 4096).decode("utf-8")
if not raw.strip():
return cfg.burst, now
try:
obj = json.loads(raw)
tokens = float(obj.get("tokens", cfg.burst))
last_refill = float(obj.get("last_refill", now))
return tokens, last_refill
except (ValueError, TypeError):
return cfg.burst, now
@staticmethod
def _refill(
tokens: float, last_refill: float, now: float, cfg: ProviderConfig
) -> tuple:
"""Apply elapsed time to the bucket, capped at burst.
Negative elapsed (clock went backward, e.g. across host
sleep/resume or a manually-tweaked monotonic mock) is clamped
to zero so we never *remove* tokens.
"""
elapsed = max(0.0, now - last_refill)
tokens = min(cfg.burst, tokens + elapsed * cfg.rate_per_sec)
return tokens, now
@staticmethod
def _write_state(fd: int, tokens: float, last_refill: float) -> None:
payload = json.dumps({"tokens": tokens, "last_refill": last_refill}).encode(
"utf-8"
)
os.lseek(fd, 0, os.SEEK_SET)
os.ftruncate(fd, 0)
os.write(fd, payload)
# A single process-wide instance is all we need: provider-keyed state
# is stored in files, so multiple `RateLimiter` instances would just
# duplicate the in-process bookkeeping. We expose a getter rather than
# the instance directly so unit tests can install a custom limiter
# scoped to a tmp directory without monkeypatching globals.
_default: Optional[RateLimiter] = None
def get_default_limiter() -> RateLimiter:
global _default
if _default is None:
_default = RateLimiter()
return _default
def reset_default_limiter() -> None:
"""Drop the cached default limiter; the next `get_default_limiter`
call rebuilds it from the current environment.
Useful between unit tests that patch env vars: without this they'd
keep reading the stale config snapshot from the first call.
"""
global _default
_default = None
@contextlib.contextmanager
def use_limiter(limiter: RateLimiter) -> Iterator[RateLimiter]:
"""Temporarily install `limiter` as the process default.
The driver's `run_claude` calls `get_default_limiter()`; tests that
want a controlled tmp-dir-backed limiter use this contextmanager
to swap one in without touching env vars or the on-disk default
state.
"""
global _default
previous = _default
_default = limiter
try:
yield limiter
finally:
_default = previous

View file

@ -1,7 +1,7 @@
#!/usr/bin/env bash
# Run the Claude Code compat matrix end-to-end against a live LiteLLM
# proxy, with per-provider rate limits applied via the cross-process
# token bucket in `tests/e2e/claude_code/rate_limiter.py`.
# token bucket in `tests/e2e/cli_harness/rate_limiter.py`.
#
# Designed for binary-searching the ideal X / Y / Z req/s per provider:
# 1. Pick an initial rate (e.g. 5/s for everyone).

View file

View file

@ -0,0 +1,49 @@
"""What a coding-agent CLI under test may see of the host.
The CI job holds the proxy's upstream credentials (provider keys, AWS, Azure, the database URL, GitHub tokens) in its
environment, and the cron VM runs as a real user with a populated home. A compromised CLI release, or a model told to
read a file, must reach neither, so the CLI gets only the allowlisted runtime variables plus what its suite passes in,
and a fresh empty HOME that is deleted when it exits.
"""
from __future__ import annotations
import os
import shutil
import tempfile
from collections.abc import Generator, Mapping
from contextlib import contextmanager
from pathlib import Path
from types import MappingProxyType
from typing import Final
CLI_ENV_ALLOWLIST: Final = (
"PATH",
"USER",
"LOGNAME",
"SHELL",
"TERM",
"TMPDIR",
"LANG",
"LC_ALL",
"LC_CTYPE",
"NODE_PATH",
"NVM_DIR",
"NVM_BIN",
)
@contextmanager
def isolated_home(prefix: str) -> Generator[Path]:
home: Final = Path(tempfile.mkdtemp(prefix=prefix))
try:
yield home
finally:
shutil.rmtree(home, ignore_errors=True)
def locked_down_env(
*, home: Path, overrides: Mapping[str, str], source: Mapping[str, str] = os.environ
) -> Mapping[str, str]:
allowed: Final = {name: source[name] for name in CLI_ENV_ALLOWLIST if name in source}
return MappingProxyType({**allowed, "HOME": str(home), **overrides})

View file

@ -0,0 +1,92 @@
"""The npm release of a coding-agent CLI that CI installs: the newest one published at least three days ago.
The three days are a security review buffer (PRD #26476, "Version resolvers"), so a release that gets pulled or
patched within days never drives a run. Newest means by publish time, not semver order.
`python tests/e2e/cli_harness/npm_version_resolver.py <npm package>` prints the version on stdout for a shell
`$(...)`. It is stdlib only because the cron image runs it without the project's dependencies.
"""
from __future__ import annotations
import json
import sys
import urllib.parse
import urllib.request
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from types import MappingProxyType
from typing import BinaryIO, Final, cast
MIN_AGE: Final = timedelta(days=3)
_REGISTRY_URL: Final = "https://registry.npmjs.org/{package}"
_FETCH_TIMEOUT_SECONDS: Final = 30
@dataclass(frozen=True, slots=True)
class Packument:
published_at: Mapping[str, datetime]
versions: frozenset[str]
@classmethod
def from_json(cls, raw: object) -> Packument:
document: Final = _as_mapping(raw)
times: Final = _as_mapping(document.get("time"))
return cls(
published_at=MappingProxyType({version: _parse_utc(stamp) for version, stamp in _string_values(times)}),
versions=frozenset(_as_mapping(document.get("versions"))),
)
def newest_version_at_least(packument: Packument, *, min_age: timedelta, as_of: datetime) -> str | None:
cutoff: Final = as_of - min_age
eligible: Final = (
(published, version)
for version, published in packument.published_at.items()
if version in packument.versions and "-" not in version and published <= cutoff
)
newest: Final = max(eligible, default=None)
return None if newest is None else newest[1]
def fetch_packument(package: str) -> Packument:
url: Final = _REGISTRY_URL.format(package=urllib.parse.quote(package, safe="@"))
request: Final = urllib.request.Request(url, headers={"Accept": "application/json"})
with cast(BinaryIO, urllib.request.urlopen(request, timeout=_FETCH_TIMEOUT_SECONDS)) as response:
return Packument.from_json(cast(object, json.loads(response.read())))
def _as_mapping(raw: object) -> Mapping[str, object]:
if not isinstance(raw, dict):
return {}
return {str(key): value for key, value in cast(dict[object, object], raw).items()}
def _string_values(raw: Mapping[str, object]) -> tuple[tuple[str, str], ...]:
return tuple((key, value) for key, value in raw.items() if isinstance(value, str))
def _parse_utc(stamp: str) -> datetime:
parsed: Final = datetime.fromisoformat(stamp.replace("Z", "+00:00"))
return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc)
def _main(argv: list[str]) -> int:
if len(argv) != 1:
print("usage: npm_version_resolver.py <npm package>", file=sys.stderr) # noqa: T201 # CLI surface
return 2
package: Final = argv[0]
version: Final = newest_version_at_least(
fetch_packument(package), min_age=MIN_AGE, as_of=datetime.now(timezone.utc)
)
if version is None:
print(f"npm_version_resolver: no release of {package} is at least {MIN_AGE} old", file=sys.stderr) # noqa: T201 # CLI surface
return 1
print(f"npm_version_resolver: selected {package}@{version}", file=sys.stderr) # noqa: T201 # CLI surface
print(version) # noqa: T201 # CLI surface
return 0
if __name__ == "__main__":
raise SystemExit(_main(sys.argv[1:]))

View file

@ -0,0 +1,151 @@
"""A token bucket per upstream, shared by every process on the host, so parallel CLI suites and xdist workers
together stay under one request rate per provider instead of one rate each.
Each bucket is a small JSON file read and written under an exclusive `flock`, in `LITELLM_COMPAT_RATE_STATE_DIR`
(default: a fixed folder under `$TMPDIR`). `LITELLM_COMPAT_RATE_<BUCKET>` sets a bucket's req/s (default 5, 0 or less
turns it off) and `LITELLM_COMPAT_RATE_BURST` overrides every bucket's burst (default: its rate).
"""
from __future__ import annotations
import fcntl
import os
import tempfile
import time
from collections.abc import Callable, Iterator, Mapping
from dataclasses import dataclass, field
from enum import StrEnum
from functools import cache
from pathlib import Path
from types import MappingProxyType
from typing import Final
from pydantic import BaseModel, ConfigDict, ValidationError
DEFAULT_RATE: Final = 5.0
RATE_ENV_PREFIX: Final = "LITELLM_COMPAT_RATE_"
BURST_ENV: Final = "LITELLM_COMPAT_RATE_BURST"
STATE_DIR_ENV: Final = "LITELLM_COMPAT_RATE_STATE_DIR"
DEFAULT_STATE_DIR_NAME: Final = "litellm-claude-compat-ratelimit"
_WAKE_MARGIN_SECONDS: Final = 1e-3
class RateBucket(StrEnum):
ANTHROPIC = "anthropic"
AZURE = "azure"
VERTEX_AI = "vertex_ai"
BEDROCK_CONVERSE = "bedrock_converse"
BEDROCK_INVOKE = "bedrock_invoke"
OPENAI = "openai"
AZURE_OPENAI = "azure_openai"
BEDROCK_MANTLE = "bedrock_mantle"
@dataclass(frozen=True, slots=True)
class ProviderConfig:
rate_per_sec: float
burst: float
@property
def enabled(self) -> bool:
return self.rate_per_sec > 0 and self.burst > 0
class _BucketState(BaseModel):
model_config = ConfigDict(frozen=True)
tokens: float
last_refill: float
def load_config(env: Mapping[str, str] = os.environ) -> Mapping[RateBucket, ProviderConfig]:
burst_override: Final = _as_float(env.get(BURST_ENV), -1.0)
return MappingProxyType(
{
bucket: _bucket_config(_as_float(env.get(_rate_env(bucket)), DEFAULT_RATE), burst_override)
for bucket in RateBucket
}
)
def default_state_dir(env: Mapping[str, str] = os.environ) -> Path:
explicit: Final = env.get(STATE_DIR_ENV)
return Path(explicit) if explicit else Path(tempfile.gettempdir()) / DEFAULT_STATE_DIR_NAME
@dataclass(frozen=True, slots=True)
class RateLimiter:
config: Mapping[RateBucket, ProviderConfig] = field(default_factory=load_config)
state_dir: Path = field(default_factory=default_state_dir)
clock: Callable[[], float] = time.monotonic
sleep: Callable[[float], None] = time.sleep
def __post_init__(self) -> None:
self.state_dir.mkdir(parents=True, exist_ok=True)
def acquire(self, bucket: RateBucket) -> float:
cfg: Final = self.config.get(bucket)
if cfg is None or not cfg.enabled:
return 0.0
return sum(self._waits(self.state_dir / f"{bucket.value}.json", cfg))
def _waits(self, path: Path, cfg: ProviderConfig) -> Iterator[float]:
while (wait := _try_consume(path, cfg, self.clock())) > 0:
self.sleep(wait)
yield wait
@cache
def get_default_limiter() -> RateLimiter:
return RateLimiter()
def _rate_env(bucket: RateBucket) -> str:
return RATE_ENV_PREFIX + bucket.value.upper()
def _bucket_config(rate: float, burst_override: float) -> ProviderConfig:
return ProviderConfig(rate_per_sec=rate, burst=burst_override if burst_override > 0 else max(rate, 1.0))
def _as_float(value: str | None, default: float) -> float:
if not value:
return default
try:
return float(value)
except ValueError:
return default
def _try_consume(path: Path, cfg: ProviderConfig, now: float) -> float:
fd: Final = os.open(path, os.O_CREAT | os.O_RDWR, 0o600)
try:
fcntl.flock(fd, fcntl.LOCK_EX)
state: Final = _refilled(_read_state(fd, cfg, now), now, cfg)
if state.tokens >= 1.0:
_write_state(fd, _BucketState(tokens=state.tokens - 1.0, last_refill=state.last_refill))
return 0.0
_write_state(fd, state)
return (1.0 - state.tokens) / cfg.rate_per_sec + _WAKE_MARGIN_SECONDS
finally:
os.close(fd)
def _read_state(fd: int, cfg: ProviderConfig, now: float) -> _BucketState:
os.lseek(fd, 0, os.SEEK_SET)
raw: Final = os.read(fd, 4096)
try:
return _BucketState.model_validate_json(raw)
except ValidationError:
return _BucketState(tokens=cfg.burst, last_refill=now)
def _refilled(state: _BucketState, now: float, cfg: ProviderConfig) -> _BucketState:
elapsed: Final = max(0.0, now - state.last_refill)
return _BucketState(tokens=min(cfg.burst, state.tokens + elapsed * cfg.rate_per_sec), last_refill=now)
def _write_state(fd: int, state: _BucketState) -> None:
os.lseek(fd, 0, os.SEEK_SET)
os.ftruncate(fd, 0)
os.write(fd, state.model_dump_json().encode("utf-8"))

View file

View file

@ -0,0 +1,42 @@
from __future__ import annotations
from collections.abc import Iterator
from dataclasses import dataclass
from typing import Final
import pytest
from cli_harness.rate_limiter import get_default_limiter
from codex.harness import CodexHarness, codex_harness
from codex.setups import Deployment, ModelSetup
from e2e_config import PROXY_BASE_URL, unique_marker
from lifecycle import ResourceManager
from proxy_client import ProxyClient
@dataclass(frozen=True, slots=True)
class _Client:
proxy: ProxyClient
@pytest.fixture(scope="session")
def client(proxy: ProxyClient) -> _Client:
return _Client(proxy)
@pytest.fixture
def setup(deployment: Deployment, proxy: ProxyClient, resources: ResourceManager) -> ModelSetup:
model: Final = f"e2e-codex-{unique_marker()}"
model_id: Final = proxy.create_model(model, deployment.litellm_params, provider_live=True)
resources.defer(lambda: proxy.delete_model(model_id))
return ModelSetup(model=model, rate_bucket=deployment.rate_bucket)
@pytest.fixture
def codex(setup: ModelSetup, resources: ResourceManager) -> Iterator[CodexHarness]:
with codex_harness(
gateway=PROXY_BASE_URL,
api_key=resources.key(models=[setup.model], user_id=None),
rate_bucket=setup.rate_bucket,
limiter=get_default_limiter(),
) as harness:
yield harness

View file

@ -0,0 +1,86 @@
from __future__ import annotations
import subprocess
import tempfile
from collections.abc import Generator, Mapping
from contextlib import contextmanager
from dataclasses import dataclass, field
from pathlib import Path
from typing import Final
from cli_harness.lockdown import isolated_home, locked_down_env
from cli_harness.rate_limiter import RateBucket, RateLimiter
from codex.run import CodexRun, codex_run
from e2e_metadata import step
CODEX_CLI: Final = "codex"
EXEC_TIMEOUT_SECONDS: Final = 600.0
_API_KEY_ENV: Final = "LITELLM_API_KEY"
_CONFIG_TOML: Final = """\
model_provider = "litellm"
approval_policy = "never"
sandbox_mode = "workspace-write"
check_for_update_on_startup = false
[analytics]
enabled = false
[model_providers.litellm]
name = "LiteLLM"
base_url = "{gateway}/v1"
env_key = "{api_key_env}"
wire_api = "responses"
model_catalog_url = "{gateway}/v1/models"
"""
@dataclass(frozen=True, slots=True)
class CodexHarness:
home: Path
api_key: str = field(repr=False)
rate_bucket: RateBucket
limiter: RateLimiter
cli: str = CODEX_CLI
timeout_seconds: float = EXEC_TIMEOUT_SECONDS
@property
def codex_home(self) -> Path:
return self.home / ".codex"
@step("Create a workspace folder holding the test's files")
def new_workspace(self, files: Mapping[str, str]) -> Path:
workspace: Final = Path(tempfile.mkdtemp(prefix="workspace-", dir=self.home))
for name, content in files.items():
_ = (workspace / name).write_text(content)
return workspace
@step('Run Codex on {model} through the proxy with the prompt "{prompt}"')
def exec(self, prompt: str, *, model: str, cwd: Path) -> CodexRun:
_ = self.limiter.acquire(self.rate_bucket)
completed: Final = subprocess.run(
(self.cli, "exec", "--json", "--skip-git-repo-check", "--model", model, "--", prompt),
cwd=cwd,
env=locked_down_env(
home=self.home, overrides={"CODEX_HOME": str(self.codex_home), _API_KEY_ENV: self.api_key}
),
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=self.timeout_seconds,
check=False,
)
return codex_run(exit_code=completed.returncode, stdout=completed.stdout, stderr=completed.stderr)
@step("Give Codex a fresh empty home whose config sends every request to the proxy")
@contextmanager
def codex_harness(
*, gateway: str, api_key: str, rate_bucket: RateBucket, limiter: RateLimiter
) -> Generator[CodexHarness]:
with isolated_home("codex-cli-home-") as home:
harness: Final = CodexHarness(home=home, api_key=api_key, rate_bucket=rate_bucket, limiter=limiter)
harness.codex_home.mkdir()
_ = (harness.codex_home / "config.toml").write_text(
_CONFIG_TOML.format(gateway=gateway, api_key_env=_API_KEY_ENV)
)
yield harness

187
tests/e2e/codex/run.py Normal file
View file

@ -0,0 +1,187 @@
"""What one `codex exec --json` run did, read from the JSONL events it printed.
The event shapes are Codex's own (`codex-rs/exec/src/exec_events.rs`). We take the latest codex release > 3 days old, so an event or item
type this module does not model is skipped, while one it does model is validated strictly and fails loudly when
Codex changes it.
"""
from __future__ import annotations
import re
from collections.abc import Callable, Iterator, Mapping
from dataclasses import dataclass
from itertools import chain
from pathlib import Path
from typing import Annotated, Final, Literal, cast
from pydantic import BaseModel, ConfigDict, Discriminator, Tag, TypeAdapter
_HTTP_STATUS: Final = re.compile(r"\bstatus:? (\d{3})\b")
class _Frozen(BaseModel):
model_config = ConfigDict(frozen=True)
class TokenUsage(_Frozen):
input_tokens: int
cached_input_tokens: int
cache_write_input_tokens: int = 0
output_tokens: int
reasoning_output_tokens: int
class _AgentMessage(_Frozen):
type: Literal["agent_message"]
text: str
class _CommandExecution(_Frozen):
type: Literal["command_execution"]
command: str
aggregated_output: str
exit_code: int | None
class _FileUpdate(_Frozen):
path: str
kind: Literal["add", "delete", "update"]
class _FileChange(_Frozen):
type: Literal["file_change"]
changes: tuple[_FileUpdate, ...]
status: Literal["in_progress", "completed", "failed"]
class _ErrorItem(_Frozen):
type: Literal["error"]
message: str
class _Unmodelled(_Frozen):
type: str
class _Message(_Frozen):
message: str
def _known_or_unmodelled(known: frozenset[str]) -> Callable[[object], str]:
def tag(raw: object) -> str:
kind: Final = cast("Mapping[str, object]", raw).get("type") if isinstance(raw, Mapping) else None
return kind if isinstance(kind, str) and kind in known else "unmodelled"
return tag
_Item = Annotated[
Annotated[_AgentMessage, Tag("agent_message")]
| Annotated[_CommandExecution, Tag("command_execution")]
| Annotated[_FileChange, Tag("file_change")]
| Annotated[_ErrorItem, Tag("error")]
| Annotated[_Unmodelled, Tag("unmodelled")],
Discriminator(_known_or_unmodelled(frozenset({"agent_message", "command_execution", "file_change", "error"}))),
]
class _ThreadStarted(_Frozen):
type: Literal["thread.started"]
thread_id: str
class _ItemCompleted(_Frozen):
type: Literal["item.completed"]
item: _Item
class _TurnCompleted(_Frozen):
type: Literal["turn.completed"]
usage: TokenUsage
class _TurnFailed(_Frozen):
type: Literal["turn.failed"]
error: _Message
class _StreamError(_Frozen):
type: Literal["error"]
message: str
_Event = Annotated[
Annotated[_ThreadStarted, Tag("thread.started")]
| Annotated[_ItemCompleted, Tag("item.completed")]
| Annotated[_TurnCompleted, Tag("turn.completed")]
| Annotated[_TurnFailed, Tag("turn.failed")]
| Annotated[_StreamError, Tag("error")]
| Annotated[_Unmodelled, Tag("unmodelled")],
Discriminator(
_known_or_unmodelled(frozenset({"thread.started", "item.completed", "turn.completed", "turn.failed", "error"}))
),
]
_EVENT: Final = TypeAdapter[_Event](_Event)
@dataclass(frozen=True, slots=True)
class Command:
command: str
output: str
exit_code: int | None
@dataclass(frozen=True, slots=True)
class CodexError:
message: str
status_code: int | None
@dataclass(frozen=True, slots=True)
class CodexRun:
exit_code: int
thread_id: str | None
commands: tuple[Command, ...]
patched_files: tuple[Path, ...]
answer: str
usage: TokenUsage | None
errors: tuple[CodexError, ...]
stderr: str
def codex_run(*, exit_code: int, stdout: str, stderr: str) -> CodexRun:
events: Final = tuple(_EVENT.validate_json(line) for line in stdout.splitlines() if line.strip())
items: Final = tuple(event.item for event in events if isinstance(event, _ItemCompleted))
patches: Final = tuple(item for item in items if isinstance(item, _FileChange) and item.status == "completed")
answers: Final = tuple(item.text for item in items if isinstance(item, _AgentMessage))
usages: Final = tuple(event.usage for event in events if isinstance(event, _TurnCompleted))
return CodexRun(
exit_code=exit_code,
thread_id=next((event.thread_id for event in events if isinstance(event, _ThreadStarted)), None),
commands=tuple(
Command(command=item.command, output=item.aggregated_output, exit_code=item.exit_code)
for item in items
if isinstance(item, _CommandExecution)
),
patched_files=tuple(Path(change.path) for change in chain.from_iterable(patch.changes for patch in patches)),
answer=answers[-1] if answers else "",
usage=usages[-1] if usages else None,
errors=tuple(_errors(events)),
stderr=stderr,
)
def _errors(events: tuple[_Event, ...]) -> Iterator[CodexError]:
for event in events:
match event:
case _TurnFailed(error=_Message(message=message)) | _StreamError(message=message):
yield _codex_error(message)
case _ItemCompleted(item=_ErrorItem(message=message)):
yield _codex_error(message)
case _:
pass
def _codex_error(message: str) -> CodexError:
status: Final = _HTTP_STATUS.search(message)
return CodexError(message=message, status_code=int(status.group(1)) if status else None)

84
tests/e2e/codex/setups.py Normal file
View file

@ -0,0 +1,84 @@
"""The credentials are `os.environ/` references the gateway resolves, the same names the Claude Code matrix's
`test_config.yaml` uses, and Bedrock rides the gateway's default AWS identity."""
from __future__ import annotations
from dataclasses import dataclass
from typing import Final
import pytest
from cli_harness.rate_limiter import RateBucket
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from models import LiteLLMParamsBody
GPT_ON_BEDROCK_MANTLE: Final = "bedrock_mantle/openai.gpt-6.1-sol"
GPT_ON_BEDROCK_RUNTIME: Final = "bedrock/us.openai.gpt-6.1-sol"
GPT_ON_AZURE_OPENAI: Final = "azure/gpt-6.1-sol"
GPT_ON_OPENAI: Final = "openai/gpt-6.1-sol"
CLAUDE_ON_BEDROCK: Final = "bedrock/us.anthropic.claude-sonnet-5-5"
_BEDROCK_REGION: Final = "us-east-1"
@dataclass(frozen=True, slots=True)
class Deployment:
litellm_params: LiteLLMParamsBody
rate_bucket: RateBucket
@dataclass(frozen=True, slots=True)
class ModelSetup:
model: str
rate_bucket: RateBucket
def _drives(provider: Provider, model: str) -> pytest.MarkDecorator:
return meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(provider,),
models=(model,),
mode=Mode.STREAM,
)
)
SETUPS: Final = (
pytest.param(
Deployment(
LiteLLMParamsBody(model=GPT_ON_BEDROCK_MANTLE, aws_region_name=_BEDROCK_REGION), RateBucket.BEDROCK_MANTLE
),
id="bedrock_mantle",
marks=_drives(Provider.BEDROCK_MANTLE, GPT_ON_BEDROCK_MANTLE),
),
pytest.param(
Deployment(
LiteLLMParamsBody(model=GPT_ON_BEDROCK_RUNTIME, aws_region_name=_BEDROCK_REGION),
RateBucket.BEDROCK_CONVERSE,
),
id="bedrock_runtime",
marks=_drives(Provider.BEDROCK, GPT_ON_BEDROCK_RUNTIME),
),
pytest.param(
Deployment(
LiteLLMParamsBody(
model=GPT_ON_AZURE_OPENAI, api_base="os.environ/AZURE_API_BASE", api_key="os.environ/AZURE_API_KEY"
),
RateBucket.AZURE_OPENAI,
),
id="azure_openai",
marks=_drives(Provider.AZURE, GPT_ON_AZURE_OPENAI),
),
pytest.param(
Deployment(LiteLLMParamsBody(model=GPT_ON_OPENAI, api_key="os.environ/OPENAI_API_KEY"), RateBucket.OPENAI),
id="openai",
marks=_drives(Provider.OPENAI, GPT_ON_OPENAI),
),
pytest.param(
Deployment(
LiteLLMParamsBody(model=CLAUDE_ON_BEDROCK, aws_region_name=_BEDROCK_REGION), RateBucket.BEDROCK_CONVERSE
),
id="claude_bedrock",
marks=_drives(Provider.BEDROCK, CLAUDE_ON_BEDROCK),
),
)

View file

@ -0,0 +1,23 @@
from __future__ import annotations
from typing import Final
import pytest
from codex.harness import CodexHarness
from codex.setups import SETUPS, ModelSetup
from e2e_config import unique_marker
pytestmark = pytest.mark.e2e
@pytest.mark.parametrize("deployment", SETUPS)
class TestCodexCommandRoundTrip:
def test_answers_with_what_a_command_printed(self, codex: CodexHarness, setup: ModelSetup) -> None:
marker: Final = unique_marker()
workspace: Final = codex.new_workspace(files={"secret.txt": marker})
run: Final = codex.exec(
"Run `cat secret.txt`, then reply with exactly what it printed.", model=setup.model, cwd=workspace
)
assert run.exit_code == 0, run
assert any(marker in command.output for command in run.commands), run
assert marker in run.answer, run

View file

@ -69,9 +69,6 @@ def _retrieve_response(proxy: ProxyClient, key: str, response_id: str) -> Respon
class TestResponsesRetrieve:
@pytest.mark.skip(
reason="stage red: product gap (LIT-5446), retrieve returns a different id than the stored response (non-idempotent response-id re-encryption)"
)
@pytest.mark.covers("llm.responses.openai.basic.nonstream.works")
@meta(
Subject(

View file

@ -2,7 +2,7 @@
Tests of the harness under `tests/e2e/` (the transport, the clients, the fixture bundle and replay edge, the stack lock, the IdP launcher, the coverage collector, the JUnit properties and the Claude Code driver), not of the product. They live outside `tests/e2e/` because the Buildkite e2e run copies that folder into the runner image and runs every test in it, so a harness test in there counts as a product test in the nightly numbers. Nothing here needs a proxy, provider keys or the network
The layout mirrors `tests/e2e/`: `test_e2e_http.py` covers `tests/e2e/e2e_http.py`, `logging/test_datadog_reader.py` covers `tests/e2e/logging/datadog_reader.py`, and `claude_code/` covers the driver, builder, probe and version resolver. Put a new harness test under the folder that mirrors the suite folder whose module it covers
The layout mirrors `tests/e2e/`: `test_e2e_http.py` covers `tests/e2e/e2e_http.py`, `logging/test_datadog_reader.py` covers `tests/e2e/logging/datadog_reader.py`, `claude_code/` covers the driver, builder and probe, `cli_harness/` covers the version resolver, the CLI env lockdown and the rate limiter, and `codex/` covers how the Codex driver reads the CLI's JSONL events. Put a new harness test under the folder that mirrors the suite folder whose module it covers
Run them from the repo root. `e2e_config` reads `LITELLM_MASTER_KEY` at import and any value will do, the CI lane sets a dummy:

Some files were not shown because too many files have changed in this diff Show more