mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
chore: merge main into E2E runtime repairs
This commit is contained in:
commit
fbf2b9cf4f
480 changed files with 26245 additions and 7774 deletions
|
|
@ -128,6 +128,8 @@ fi
|
|||
sudo iptables -L integration_only -n -v -x > "$results/egress-guard.txt"
|
||||
awk '$3 == "REJECT" && $1 > 0 { rejected=1 } END { exit !rejected }' "$results/egress-guard.txt"
|
||||
|
||||
export UV_OFFLINE=true
|
||||
|
||||
if [ "$mode" = parity ]; then
|
||||
export INTEGRATION_WORKERS=1
|
||||
else
|
||||
|
|
|
|||
4
.github/workflows/compat-matrix-image.yml
vendored
4
.github/workflows/compat-matrix-image.yml
vendored
|
|
@ -4,7 +4,7 @@ on:
|
|||
pull_request:
|
||||
paths:
|
||||
- tests/e2e/claude_code/cron_vm/**
|
||||
- tests/e2e/claude_code/pr_gate_version_resolver.py
|
||||
- tests/e2e/cli_harness/npm_version_resolver.py
|
||||
- .github/workflows/compat-matrix-image.yml
|
||||
workflow_dispatch:
|
||||
|
||||
|
|
@ -36,7 +36,7 @@ jobs:
|
|||
whoami
|
||||
gh --version
|
||||
uv --version
|
||||
version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/claude_code/pr_gate_version_resolver.py)"
|
||||
version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)"
|
||||
/opt/litellm/tests/e2e/claude_code/cron_vm/install_claude_code.sh "${version}" /tmp/claude-cli
|
||||
/tmp/claude-cli/claude --version
|
||||
'
|
||||
|
|
|
|||
30
.github/workflows/deploy-dev-on-main.yml
vendored
30
.github/workflows/deploy-dev-on-main.yml
vendored
|
|
@ -1,30 +0,0 @@
|
|||
name: Deploy main to gateway-dev
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
permissions: {}
|
||||
|
||||
concurrency:
|
||||
group: deploy-dev-dispatch
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
dispatch:
|
||||
if: github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2.2.2
|
||||
id: token
|
||||
with:
|
||||
app-id: ${{ secrets.LITELLM_OPS_DISPATCH_APP_ID }}
|
||||
private-key: ${{ secrets.LITELLM_OPS_DISPATCH_APP_PRIVATE_KEY }}
|
||||
owner: BerriAI
|
||||
repositories: litellm-ops
|
||||
permission-actions: write
|
||||
- name: Dispatch the litellm-ops dev deploy
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.token.outputs.token }}
|
||||
run: gh api --method POST repos/BerriAI/litellm-ops/actions/workflows/deploy-dev.yml/dispatches -f ref=main
|
||||
2
.github/workflows/test-linting.yml
vendored
2
.github/workflows/test-linting.yml
vendored
|
|
@ -208,7 +208,7 @@ jobs:
|
|||
exit 0
|
||||
fi
|
||||
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
|
||||
CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/claude_code/pr_gate_version_resolver.py)"
|
||||
CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)"
|
||||
tests/e2e/claude_code/cron_vm/install_claude_code.sh "$CLAUDE_VERSION" "$RUNNER_TEMP/claude-cli"
|
||||
PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q tests/e2e_harness
|
||||
uv run --no-sync pytest --tb=short -q --noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures \
|
||||
|
|
|
|||
85
.github/workflows/weekly_load_anomaly.yml
vendored
85
.github/workflows/weekly_load_anomaly.yml
vendored
|
|
@ -1,85 +0,0 @@
|
|||
name: "Weekly Load Anomaly Check"
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 12 * * 6"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
weekly-load-anomaly:
|
||||
if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.6
|
||||
env:
|
||||
POSTGRES_USER: llmproxy
|
||||
POSTGRES_PASSWORD: dbpassword9090
|
||||
POSTGRES_DB: litellm
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U llmproxy"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
env:
|
||||
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
|
||||
LITELLM_MASTER_KEY: sk-weekly-anomaly-check
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache the Rust build
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
|
||||
|
||||
- name: Cache Prisma binaries
|
||||
uses: ./.github/actions/cache-prisma-binaries
|
||||
|
||||
- name: Generate Prisma client
|
||||
run: |
|
||||
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
||||
|
||||
- name: Start the proxy
|
||||
run: |
|
||||
nohup uv run --no-sync litellm --config tests/e2e/perf/weekly_anomaly_config.yml --port 4000 > proxy.log 2>&1 &
|
||||
for _ in $(seq 1 90); do
|
||||
if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "proxy never became live"
|
||||
tail -n 100 proxy.log
|
||||
exit 1
|
||||
|
||||
- name: Run the weekly session anomaly test
|
||||
env:
|
||||
E2E_WEEKLY_ANOMALY: "1"
|
||||
run: |
|
||||
uv run --no-sync pytest tests/e2e/perf/test_weekly_session_anomaly_e2e.py -v --tb=short -rA
|
||||
|
||||
- name: Show proxy log on failure
|
||||
if: failure()
|
||||
run: tail -n 300 proxy.log
|
||||
|
|
@ -3,9 +3,11 @@
|
|||
|
||||
import asyncio
|
||||
import base64
|
||||
import copy
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
|
|
@ -34,7 +36,7 @@ from typing_extensions import ReadOnly, Unpack
|
|||
|
||||
import litellm
|
||||
from litellm import Router, verbose_logger
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._internal_context import internal_sub_call, post_response_phase, with_service_target
|
||||
from litellm._uuid import uuid
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.constants import (
|
||||
|
|
@ -95,6 +97,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio
|
|||
request_tags_from_metadata,
|
||||
)
|
||||
from litellm.repositories.managed_file_repository import ManagedFileRepository
|
||||
from litellm.responses.streaming_iterator import spawn_logging_task
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccessIssue]
|
||||
AllMessageValues,
|
||||
|
|
@ -395,6 +398,134 @@ class _RouterFileCallKwargs(TypedDict, total=False):
|
|||
custom_llm_provider: ReadOnly[str | None]
|
||||
|
||||
|
||||
_MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str])
|
||||
_FILE_UPLOAD_ONLY_FIELDS: Final = frozenset({"file", "purpose", "expires_after", "extra_body"})
|
||||
_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id", "_litellm_internal_model_credentials"})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ProviderUpload:
|
||||
model_id: str
|
||||
provider_file_id: str
|
||||
upload_call_id: str | None = None
|
||||
|
||||
|
||||
def _provider_uploads(file_object: OpenAIFileObject) -> tuple[_ProviderUpload, ...]:
|
||||
file_hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic file metadata
|
||||
dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
|
||||
)
|
||||
model_file_id_mapping: Final = _MODEL_FILE_ID_MAPPING.validate_python(
|
||||
file_hidden_params.get("model_file_id_mapping") or {}
|
||||
)
|
||||
upload_call_id: Final = file_hidden_params.get("litellm_call_id")
|
||||
return tuple(
|
||||
_ProviderUpload(
|
||||
model_id=model_id,
|
||||
provider_file_id=provider_file_id,
|
||||
upload_call_id=upload_call_id if isinstance(upload_call_id, str) else None,
|
||||
)
|
||||
for model_id, provider_file_id in model_file_id_mapping.items()
|
||||
)
|
||||
|
||||
|
||||
def _keeps_one_upload_per_target(llm_router: Router, model_id: str) -> bool:
|
||||
credentials: Final = _provider_file_retrieve_credentials(llm_router=llm_router, model_id=model_id)
|
||||
return (credentials or {}).get("custom_llm_provider") == LlmProviders.LITELLM_PROXY.value
|
||||
|
||||
|
||||
def _is_kept_upload(uploads: Sequence[_ProviderUpload], index: int, per_target_model_ids: frozenset[str]) -> bool:
|
||||
upload: Final = uploads[index]
|
||||
other_uploads: Final = uploads[index + 1 :] if upload.model_id in per_target_model_ids else uploads[:index]
|
||||
return all(other.model_id != upload.model_id for other in other_uploads)
|
||||
|
||||
|
||||
def _upload_from_trailing_id_fields(decoded_unified_file_id: str) -> _ProviderUpload:
|
||||
trailing_fields: Final = decoded_unified_file_id.rpartition(";llm_output_file_id,")[2]
|
||||
provider_file_id, _, model_id = trailing_fields.partition(";llm_output_file_model_id,")
|
||||
return _ProviderUpload(model_id=model_id, provider_file_id=provider_file_id)
|
||||
|
||||
|
||||
def _carried_upload_missing_from(unified_file_id: str, model_file_ids: Mapping[str, str]) -> _ProviderUpload | None:
|
||||
decoded: Final = _is_base64_encoded_unified_file_id(unified_file_id)
|
||||
carried: Final = _upload_from_trailing_id_fields(decoded) if decoded else None
|
||||
if carried is None or model_file_ids.get(carried.model_id) in (None, carried.provider_file_id):
|
||||
return None
|
||||
return carried
|
||||
|
||||
|
||||
async def _delete_provider_file(
|
||||
llm_router: Router,
|
||||
model_id: str,
|
||||
provider_file_id: str,
|
||||
request_data: Mapping[str, object],
|
||||
) -> None:
|
||||
credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
|
||||
delete_data: Final = {
|
||||
**{k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS},
|
||||
**(
|
||||
{"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
|
||||
if credentials is not None
|
||||
else {}
|
||||
),
|
||||
}
|
||||
await llm_router.afile_delete(
|
||||
model=model_id,
|
||||
file_id=provider_file_id,
|
||||
**cast(_RouterFileCallKwargs, delete_data),
|
||||
)
|
||||
|
||||
|
||||
def _with_own_nested_dicts(request_data: Mapping[str, object]) -> MappingProxyType[str, object]:
|
||||
return MappingProxyType({k: copy.copy(v) if isinstance(v, dict) else v for k, v in request_data.items()})
|
||||
|
||||
|
||||
async def _delete_redundant_upload(
|
||||
llm_router: Router,
|
||||
upload: _ProviderUpload,
|
||||
delete_request: Mapping[str, object],
|
||||
parent_call_id: str | None,
|
||||
) -> None:
|
||||
try:
|
||||
with internal_sub_call(parent_call_id):
|
||||
await _delete_provider_file(
|
||||
llm_router, upload.model_id, upload.provider_file_id, _with_own_nested_dicts(delete_request)
|
||||
)
|
||||
except Exception as error:
|
||||
verbose_logger.warning(
|
||||
f"Could not delete the redundant upload provider_file_id={sanitize_for_log(upload.provider_file_id)} "
|
||||
f"at deployment model_id={sanitize_for_log(upload.model_id)}: "
|
||||
f"{type(error).__name__} {sanitize_for_log(error)}"
|
||||
)
|
||||
|
||||
|
||||
async def _delete_redundant_uploads(
|
||||
llm_router: Router,
|
||||
uploads: Sequence[_ProviderUpload],
|
||||
delete_request: Mapping[str, object],
|
||||
) -> None:
|
||||
await asyncio.gather(
|
||||
*(_delete_redundant_upload(llm_router, upload, delete_request, upload.upload_call_id) for upload in uploads)
|
||||
)
|
||||
|
||||
|
||||
def _start_deleting_redundant_uploads(
|
||||
llm_router: Router,
|
||||
uploads: Sequence[_ProviderUpload],
|
||||
create_file_request: CreateFileRequest,
|
||||
) -> None:
|
||||
if not uploads:
|
||||
return
|
||||
delete_request: Final = MappingProxyType(
|
||||
{k: v for k, v in create_file_request.items() if k not in _FILE_UPLOAD_ONLY_FIELDS}
|
||||
)
|
||||
with post_response_phase():
|
||||
spawn_logging_task(
|
||||
asyncio.get_running_loop(),
|
||||
_delete_redundant_uploads(llm_router, uploads, delete_request),
|
||||
task_name="managed file repeat upload cleanup",
|
||||
)
|
||||
|
||||
|
||||
def _managed_database(prisma_client: _ManagedResourcePrismaClient) -> _ManagedResourceDatabase:
|
||||
return prisma_client.db
|
||||
|
||||
|
|
@ -2034,16 +2165,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
target_model_names_list=target_model_names_list,
|
||||
)
|
||||
|
||||
## STORE MODEL MAPPINGS IN DB
|
||||
model_mappings: Dict[str, str] = {}
|
||||
|
||||
for file_object in responses:
|
||||
file_hidden_params = cast( # cast-ok: preserve mapping operations on dynamic file metadata
|
||||
dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
|
||||
)
|
||||
model_file_id_mapping = file_hidden_params.get("model_file_id_mapping")
|
||||
if model_file_id_mapping and isinstance(model_file_id_mapping, dict):
|
||||
model_mappings.update(model_file_id_mapping)
|
||||
uploads: Final = tuple(chain.from_iterable(map(_provider_uploads, responses)))
|
||||
per_target_model_ids: Final = frozenset(
|
||||
model_id
|
||||
for model_id in {upload.model_id for upload in uploads}
|
||||
if _keeps_one_upload_per_target(llm_router, model_id)
|
||||
)
|
||||
kept_uploads: Final = tuple(
|
||||
upload for index, upload in enumerate(uploads) if _is_kept_upload(uploads, index, per_target_model_ids)
|
||||
)
|
||||
model_mappings: Final = {upload.model_id: upload.provider_file_id for upload in kept_uploads}
|
||||
|
||||
await self.store_unified_file_id(
|
||||
file_id=response.id,
|
||||
|
|
@ -2052,6 +2183,15 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
model_mappings=model_mappings,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
_start_deleting_redundant_uploads(
|
||||
llm_router,
|
||||
tuple(
|
||||
upload
|
||||
for upload in uploads
|
||||
if upload not in kept_uploads and upload.model_id not in per_target_model_ids
|
||||
),
|
||||
create_file_request,
|
||||
)
|
||||
|
||||
# Emit Prometheus metrics for managed file creation
|
||||
prom_logger = self._get_prometheus_logger()
|
||||
|
|
@ -2692,23 +2832,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
specific_model_file_id_mapping: Final = model_file_id_mapping.get(file_id)
|
||||
if not specific_model_file_id_mapping:
|
||||
return
|
||||
filtered_data: Final = {
|
||||
k: v for k, v in data.items() if k not in ("model", "file_id", "_litellm_internal_model_credentials")
|
||||
}
|
||||
for model_id, model_file_id in specific_model_file_id_mapping.items():
|
||||
credentials = llm_router.registry().credentials(model_id, form="with_provider")
|
||||
delete_data = {
|
||||
**filtered_data,
|
||||
**(
|
||||
{"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
|
||||
if credentials is not None
|
||||
else {}
|
||||
),
|
||||
}
|
||||
await llm_router.afile_delete(
|
||||
model=model_id,
|
||||
file_id=model_file_id,
|
||||
**cast(_RouterFileCallKwargs, delete_data),
|
||||
await _delete_provider_file(llm_router, model_id, model_file_id, data)
|
||||
unrecorded_upload: Final = _carried_upload_missing_from(file_id, specific_model_file_id_mapping)
|
||||
delete_call_id: Final = data.get("litellm_call_id")
|
||||
if unrecorded_upload is not None:
|
||||
await _delete_redundant_upload(
|
||||
llm_router, unrecorded_upload, data, delete_call_id if isinstance(delete_call_id, str) else None
|
||||
)
|
||||
|
||||
async def afile_content(
|
||||
|
|
|
|||
|
|
@ -186,6 +186,9 @@ pub struct ModelInfoFields {
|
|||
pub supports_image_size: Option<bool>,
|
||||
pub supported_audio_formats: Option<Value>,
|
||||
pub token_count_framing: Option<i64>,
|
||||
pub token_count_tool_framing: Option<i64>,
|
||||
pub token_count_responses_framing: Option<i64>,
|
||||
pub token_count_responses_tool_framing: Option<i64>,
|
||||
pub token_count_model: Option<Value>,
|
||||
pub token_count_tokenizer: Option<Value>,
|
||||
pub vertex_ai_audio_api: Option<Value>,
|
||||
|
|
|
|||
|
|
@ -104,6 +104,8 @@ attributes! {
|
|||
/// Cloud regions the model is available in ('global' or region ids).
|
||||
supported_regions: Vec<String>,
|
||||
thinking_always_on: bool,
|
||||
/// Whether the model thinks when the request carries no thinking param.
|
||||
thinking_on_by_default: bool,
|
||||
token_count_framing: u64 as NumericCount,
|
||||
token_count_model: TokenCountModel,
|
||||
token_count_responses_framing: u64 as NumericCount,
|
||||
|
|
@ -301,6 +303,8 @@ mod tests {
|
|||
#[case::known_mode("mode", json!("chat"), true)]
|
||||
#[case::unknown_mode("mode", json!("chatt"), false)]
|
||||
#[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)]
|
||||
#[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)]
|
||||
#[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)]
|
||||
#[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)]
|
||||
#[case::reasoning_modes("reasoning_modes", json!(["none", "adaptive"]), true)]
|
||||
#[case::reasoning_context_not_a_list("reasoning_contexts", json!("reasoning"), false)]
|
||||
|
|
|
|||
|
|
@ -2374,6 +2374,7 @@ if TYPE_CHECKING:
|
|||
supports_web_search: Callable[..., bool]
|
||||
supports_tool_search: Callable[..., bool]
|
||||
supports_url_context: Callable[..., bool]
|
||||
supports_continuation: Callable[..., bool]
|
||||
supports_response_schema: Callable[..., bool]
|
||||
supports_parallel_function_calling: Callable[..., bool]
|
||||
supports_vision: Callable[..., bool]
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ UTILS_NAMES: Final = (
|
|||
"supports_web_search",
|
||||
"supports_tool_search",
|
||||
"supports_url_context",
|
||||
"supports_continuation",
|
||||
"supports_response_schema",
|
||||
"supports_parallel_function_calling",
|
||||
"supports_vision",
|
||||
|
|
@ -488,6 +489,7 @@ _UTILS_IMPORT_MAP: Final = {
|
|||
"supports_web_search": (".utils", "supports_web_search"),
|
||||
"supports_tool_search": (".utils", "supports_tool_search"),
|
||||
"supports_url_context": (".utils", "supports_url_context"),
|
||||
"supports_continuation": (".utils", "supports_continuation"),
|
||||
"supports_response_schema": (".utils", "supports_response_schema"),
|
||||
"supports_parallel_function_calling": (
|
||||
".utils",
|
||||
|
|
|
|||
|
|
@ -45,13 +45,27 @@ from .redis_semantic_cache import RedisSemanticCache
|
|||
from .s3_cache import S3Cache
|
||||
|
||||
RESPONSE_CACHE_TARGET: Final = "llm_response"
|
||||
_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"})
|
||||
_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token", "return_token_ids"})
|
||||
_DECODED_LITERAL: Final = TypeAdapter(object)
|
||||
_REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
|
||||
list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True)
|
||||
)
|
||||
|
||||
|
||||
def _keyed_provider_params_in_extra_body(kwargs: Mapping[str, object]) -> str:
|
||||
raw_extra_body: Final = kwargs.get("extra_body")
|
||||
if not isinstance(raw_extra_body, dict):
|
||||
return ""
|
||||
extra_body: Final = cast( # cast-ok: isinstance leaves the key and value types unknown
|
||||
Mapping[str, object], raw_extra_body
|
||||
)
|
||||
return "".join(
|
||||
f"extra_body.{param}: {extra_body[param]}"
|
||||
for param in sorted(_ALWAYS_KEYED_PROVIDER_PARAMS)
|
||||
if extra_body.get(param)
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def response_cache_phase(operation: Literal["get", "set"]) -> Generator[None]:
|
||||
"""The ``cache.get llm_response`` / ``cache.set llm_response`` span a response-cache read or write runs
|
||||
|
|
@ -429,7 +443,10 @@ class Cache:
|
|||
"" if embeds_thought_signature_in_tool_call_id(kwargs) else "thought_signature_in_tool_call_id: False"
|
||||
)
|
||||
hashed_cache_key = Cache._get_hashed_cache_key(
|
||||
cache_key + reasoning_summary_material + tool_call_id_shape_material
|
||||
cache_key
|
||||
+ _keyed_provider_params_in_extra_body(kwargs)
|
||||
+ reasoning_summary_material
|
||||
+ tool_call_id_shape_material
|
||||
)
|
||||
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
|
||||
verbose_logger.debug(
|
||||
|
|
|
|||
|
|
@ -673,7 +673,7 @@ class DualCache(BaseCache):
|
|||
if self.redis_cache is not None:
|
||||
self.redis_cache.flush_cache()
|
||||
|
||||
def delete_cache(self, key):
|
||||
def delete_cache(self, key: str) -> None:
|
||||
"""
|
||||
Delete a key from the cache
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -699,6 +699,9 @@ STREAM_SSE_DONE_STRING: Final[str] = "[DONE]"
|
|||
STREAM_SSE_DATA_PREFIX: Final[str] = "data: "
|
||||
STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping"}\n\n'
|
||||
STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8")
|
||||
STREAM_FINISH_CHOICE_FIELDS: Final = ("stop_reason",)
|
||||
STREAM_CHUNK_CHOICE_FIELDS: Final = ("token_ids",)
|
||||
VLLM_CHOICE_FIELDS: Final = STREAM_FINISH_CHOICE_FIELDS + STREAM_CHUNK_CHOICE_FIELDS
|
||||
### SPEND TRACKING ###
|
||||
DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = get_env_float(
|
||||
"DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import asyncio
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from functools import cache
|
||||
from typing import Final
|
||||
|
|
@ -15,6 +16,7 @@ from litellm.constants import (
|
|||
AZURE_STORAGE_MSFT_VERSION,
|
||||
)
|
||||
from litellm.integrations.custom_batch_logger import CustomBatchLogger
|
||||
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.llms.azure.common_utils import get_azure_ad_token_from_entra_id
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
|
|
@ -47,6 +49,20 @@ def adls_safe_file_name(payload_id: str | None) -> str:
|
|||
return f"{name.translate(_ADLS_SAFE_NAME)}.json"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AzureStorageLogQueueItem:
|
||||
payload: StandardLoggingPayload
|
||||
file_name: str
|
||||
|
||||
|
||||
def _queued_log(kwargs: Mapping[str, object], payload: StandardLoggingPayload) -> AzureStorageLogQueueItem:
|
||||
payload_id: Final = payload.get("id")
|
||||
return AzureStorageLogQueueItem(
|
||||
payload=payload,
|
||||
file_name=adls_safe_file_name(payload_id and logged_event_key(kwargs, payload_id)),
|
||||
)
|
||||
|
||||
|
||||
@cache
|
||||
def _cached_credential_chain_token_provider() -> Callable[[], str]:
|
||||
return get_azure_ad_token_provider(
|
||||
|
|
@ -99,7 +115,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
|
|||
|
||||
asyncio.create_task(self.periodic_flush())
|
||||
self.flush_lock = asyncio.Lock()
|
||||
self.log_queue: list[StandardLoggingPayload] = []
|
||||
self.log_queue: list[AzureStorageLogQueueItem] = []
|
||||
super().__init__(**kwargs, flush_lock=self.flush_lock)
|
||||
except Exception as e:
|
||||
verbose_logger.exception(
|
||||
|
|
@ -115,7 +131,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
|
|||
def azure_storage_blob_endpoint(self) -> str:
|
||||
return f"https://{self.azure_storage_account_name}.blob.{self.azure_storage_endpoint_suffix}"
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
async def async_log_success_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime):
|
||||
"""
|
||||
Async Log success events to Azure Blob Storage
|
||||
|
||||
|
|
@ -133,12 +149,12 @@ class AzureBlobStorageLogger(CustomBatchLogger):
|
|||
if standard_logging_payload is None:
|
||||
raise ValueError("standard_logging_payload is not set")
|
||||
|
||||
self.log_queue.append(standard_logging_payload)
|
||||
self.log_queue.append(_queued_log(kwargs, standard_logging_payload))
|
||||
|
||||
except Exception as e:
|
||||
verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e)
|
||||
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
async def async_log_failure_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime):
|
||||
"""
|
||||
Async Log failure events to Azure Blob Storage
|
||||
|
||||
|
|
@ -156,7 +172,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
|
|||
if standard_logging_payload is None:
|
||||
raise ValueError("standard_logging_payload is not set")
|
||||
|
||||
self.log_queue.append(standard_logging_payload)
|
||||
self.log_queue.append(_queued_log(kwargs, standard_logging_payload))
|
||||
except Exception as e:
|
||||
verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e)
|
||||
|
||||
|
|
@ -195,13 +211,15 @@ class AzureBlobStorageLogger(CustomBatchLogger):
|
|||
len(self.log_queue),
|
||||
)
|
||||
|
||||
for payload in self.log_queue:
|
||||
await self.async_upload_payload_to_azure_blob_storage(payload=payload)
|
||||
for item in self.log_queue:
|
||||
await self.async_upload_payload_to_azure_blob_storage(payload=item.payload, file_name=item.file_name)
|
||||
|
||||
except Exception as e:
|
||||
verbose_logger.exception("AzureBlobStorageLogger Error sending batch API - %s", e)
|
||||
|
||||
async def async_upload_payload_to_azure_blob_storage(self, payload: StandardLoggingPayload):
|
||||
async def async_upload_payload_to_azure_blob_storage(
|
||||
self, payload: StandardLoggingPayload, file_name: str | None = None
|
||||
):
|
||||
"""
|
||||
Uploads the payload to Azure Blob Storage using a 3-step process:
|
||||
1. Create file resource
|
||||
|
|
@ -210,14 +228,14 @@ class AzureBlobStorageLogger(CustomBatchLogger):
|
|||
"""
|
||||
try:
|
||||
if self.azure_storage_account_key:
|
||||
await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload)
|
||||
await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload, file_name=file_name)
|
||||
else:
|
||||
# Get a valid token instead of always requesting a new one
|
||||
await self.set_valid_azure_ad_token()
|
||||
async_client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
|
||||
json_payload: Final = safe_dumps(payload) + "\n" # Add newline for each log entry
|
||||
payload_bytes: Final = json_payload.encode("utf-8")
|
||||
filename: Final = adls_safe_file_name(payload.get("id"))
|
||||
filename: Final = file_name or adls_safe_file_name(payload.get("id"))
|
||||
base_url = f"{self.azure_storage_dfs_endpoint}/{self.azure_storage_file_system}/{filename}"
|
||||
|
||||
# Execute the 3-step upload process
|
||||
|
|
@ -377,7 +395,9 @@ class AzureBlobStorageLogger(CustomBatchLogger):
|
|||
self._service_client_timeout = self._clock() + _DEFAULT_TTL_FOR_HTTPX_CLIENTS
|
||||
return self._service_client
|
||||
|
||||
async def upload_to_azure_data_lake_with_azure_account_key(self, payload: StandardLoggingPayload):
|
||||
async def upload_to_azure_data_lake_with_azure_account_key(
|
||||
self, payload: StandardLoggingPayload, file_name: str | None = None
|
||||
):
|
||||
"""
|
||||
Uploads the payload to Azure Data Lake using the Azure SDK
|
||||
|
||||
|
|
@ -386,11 +406,11 @@ class AzureBlobStorageLogger(CustomBatchLogger):
|
|||
|
||||
try:
|
||||
today: Final = datetime.now().strftime("%Y-%m-%d")
|
||||
file_name: Final = adls_safe_file_name(payload.get("id"))
|
||||
resolved_file_name: Final = file_name or adls_safe_file_name(payload.get("id"))
|
||||
await self._upload_bytes_with_account_key(
|
||||
file_path=f"{today}/{file_name}", content=safe_dumps(payload).encode("utf-8")
|
||||
file_path=f"{today}/{resolved_file_name}", content=safe_dumps(payload).encode("utf-8")
|
||||
)
|
||||
verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, file_name)
|
||||
verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, resolved_file_name)
|
||||
|
||||
except Exception as e:
|
||||
verbose_logger.exception("Error occurred: %s", e)
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger
|
||||
from litellm.integrations.clickhouse.context import is_lens_analysis
|
||||
from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE
|
||||
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
|
||||
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
|
||||
from litellm.tracing.types import SpendLogPayload, SpendLogRecord
|
||||
|
|
@ -134,7 +135,7 @@ def spend_log_row_from_payload(
|
|||
usage: Mapping[str, Any] = metadata.get("usage_object") or hidden_params.get("usage_object") or MappingProxyType({})
|
||||
cache_read_tokens, cache_write_tokens = _cache_tokens(usage)
|
||||
trace_id, span_id = _find_traceparent(metadata, kwargs)
|
||||
request_id = str(payload.get("id") or "")
|
||||
response_id: Final = str(payload.get("id") or "")
|
||||
redact = litellm.turn_off_message_logging is True
|
||||
track_end_user: Final = not litellm.disable_end_user_cost_tracking
|
||||
completion_start_ms = _to_ms(payload.get("completionStartTime"))
|
||||
|
|
@ -157,8 +158,8 @@ def spend_log_row_from_payload(
|
|||
)
|
||||
)
|
||||
return SpendLogRecord(
|
||||
request_id=request_id,
|
||||
response_id=strip_cache_hit_suffix(request_id),
|
||||
request_id=logged_event_key(kwargs, response_id),
|
||||
response_id=strip_cache_hit_suffix(response_id),
|
||||
provider_request_id=(
|
||||
get_provider_request_id(kwargs.get("response_headers"))
|
||||
or get_provider_request_id(hidden_params.get("additional_headers"))
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from litellm.integrations.deepeval.utils import (
|
|||
to_zod_compatible_iso,
|
||||
validate_environment,
|
||||
)
|
||||
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
|
||||
|
||||
|
||||
# This file includes the custom callbacks for LiteLLM Proxy
|
||||
|
|
@ -109,8 +110,10 @@ class DeepEvalLogger(CustomLogger):
|
|||
)
|
||||
else:
|
||||
output = str(standard_logging_object.get("error_string", ""))
|
||||
logged_id: Final = standard_logging_object.get("id", uuid.uuid4())
|
||||
span_id: Final = logged_event_key(kwargs, logged_id) if isinstance(logged_id, str) else logged_id
|
||||
return BaseApiSpan(
|
||||
uuid=standard_logging_object.get("id", uuid.uuid4()),
|
||||
uuid=span_id,
|
||||
name=("litellm_success_callback" if is_success else "litellm_failure_callback"),
|
||||
status=(TraceSpanApiStatus.SUCCESS if is_success else TraceSpanApiStatus.ERRORED),
|
||||
type=SpanApiType.LLM,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ from typing import Final, Protocol
|
|||
|
||||
import litellm
|
||||
from litellm._uuid import uuid
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
|
||||
|
||||
|
||||
class _DynamoTable(Protocol):
|
||||
|
|
@ -22,11 +24,13 @@ class _DynamoResource(Protocol):
|
|||
class DyanmoDBLogger:
|
||||
# Class variables or attributes
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, dynamodb: _DynamoResource | None = None):
|
||||
# Instance variables
|
||||
import boto3
|
||||
|
||||
self.dynamodb: Final[_DynamoResource] = boto3.resource("dynamodb", region_name=os.environ["AWS_REGION_NAME"])
|
||||
self.dynamodb: Final[_DynamoResource] = dynamodb or boto3.resource(
|
||||
"dynamodb", region_name=os.environ["AWS_REGION_NAME"]
|
||||
)
|
||||
if litellm.dynamodb_table_name is None:
|
||||
raise ValueError(
|
||||
"LiteLLM Error, trying to use DynamoDB but not table name passed. Create a table and set `litellm.dynamodb_table_name=<your-table>`"
|
||||
|
|
@ -45,23 +49,23 @@ class DyanmoDBLogger:
|
|||
|
||||
_async_log_event = async_log_event
|
||||
|
||||
def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
|
||||
def log_event(self, kwargs: Mapping[str, object], response_obj, start_time, end_time, print_verbose):
|
||||
try:
|
||||
print_verbose(f"DynamoDB Logging - Enters logging function for model {kwargs}")
|
||||
|
||||
# construct payload to send to DynamoDB
|
||||
# follows the same params as langfuse.py
|
||||
litellm_params: Final = kwargs.get("litellm_params", {})
|
||||
metadata: Final = litellm_params.get("metadata", {}) or {} # if litellm_params['metadata'] == None
|
||||
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
|
||||
metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {}
|
||||
messages: Final = kwargs.get("messages")
|
||||
optional_params: Final = kwargs.get("optional_params", {})
|
||||
call_type: Final = kwargs.get("call_type", "litellm.completion")
|
||||
usage: Final = response_obj["usage"]
|
||||
id: Final = response_obj.get("id", str(uuid.uuid4()))
|
||||
response_id: Final = response_obj.get("id", str(uuid.uuid4()))
|
||||
|
||||
# Build the initial payload
|
||||
payload: Final[dict[str, object]] = {
|
||||
"id": id,
|
||||
"id": logged_event_key(kwargs, response_id) if isinstance(response_id, str) else response_id,
|
||||
"call_type": call_type,
|
||||
"startTime": start_time,
|
||||
"endTime": end_time,
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.litellm_core_utils.cloud_storage_security import (
|
|||
sanitize_cloud_object_component,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
|
||||
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
|
||||
from litellm.types.integrations.gcs_bucket import *
|
||||
from litellm.types.utils import StandardBatchLineItemsPayload, StandardLoggingPayload
|
||||
|
|
@ -296,30 +297,28 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
|
|||
else:
|
||||
await self._send_individual_logs(items_to_process)
|
||||
|
||||
def _get_object_name(self, kwargs: dict, logging_payload: StandardLoggingPayload, response_obj: Any) -> str:
|
||||
def _get_object_name(
|
||||
self, kwargs: Mapping[str, object], logging_payload: StandardLoggingPayload, response_obj: Any
|
||||
) -> str:
|
||||
"""
|
||||
Get the object name to use for the current payload
|
||||
"""
|
||||
current_date: Final = self._get_object_date_from_datetime(datetime.now(timezone.utc))
|
||||
metadata: Final = as_str_mapping((as_str_mapping(kwargs.get("litellm_params")) or {}).get("metadata")) or {}
|
||||
gcs_log_id: Final = metadata.get("gcs_log_id")
|
||||
safe_log_id: Final = sanitize_cloud_object_component(
|
||||
gcs_log_id if isinstance(gcs_log_id, str) else None, fallback=""
|
||||
)
|
||||
if safe_log_id:
|
||||
return f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
|
||||
if logging_payload.get("error_str", None) is not None:
|
||||
object_name = self._generate_failure_object_name(
|
||||
request_date_str=current_date,
|
||||
)
|
||||
else:
|
||||
object_name = self._generate_success_object_name(
|
||||
request_date_str=current_date,
|
||||
response_id=response_obj.get("id", ""),
|
||||
)
|
||||
|
||||
# used for testing
|
||||
_litellm_params: Final = kwargs.get("litellm_params", None) or {}
|
||||
_metadata: Final = _litellm_params.get("metadata", None) or {}
|
||||
if "gcs_log_id" in _metadata:
|
||||
safe_log_id: Final = sanitize_cloud_object_component(_metadata.get("gcs_log_id"), fallback="")
|
||||
if safe_log_id:
|
||||
object_name = f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
|
||||
|
||||
return object_name
|
||||
return self._generate_failure_object_name(request_date_str=current_date)
|
||||
raw_response_id: Final[object] = response_obj.get("id", "")
|
||||
response_id: Final = raw_response_id if isinstance(raw_response_id, str) else ""
|
||||
return self._generate_success_object_name(
|
||||
request_date_str=current_date,
|
||||
response_id=logged_event_key(kwargs, response_id),
|
||||
)
|
||||
|
||||
async def get_request_response_payload(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -3,12 +3,15 @@
|
|||
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
HTTPHandler,
|
||||
get_async_httpx_client,
|
||||
|
|
@ -48,7 +51,7 @@ class OpenMeterLogger(CustomLogger):
|
|||
if len(missing_keys) > 0:
|
||||
raise Exception(f"Missing keys={missing_keys} in environment.")
|
||||
|
||||
def _common_logic(self, kwargs: dict, response_obj):
|
||||
def _common_logic(self, kwargs: Mapping[str, object], response_obj):
|
||||
call_id: Final = response_obj.get("id", kwargs.get("litellm_call_id"))
|
||||
dt: Final = get_utc_datetime().isoformat()
|
||||
cost: Final = kwargs.get("response_cost", None)
|
||||
|
|
@ -74,8 +77,8 @@ class OpenMeterLogger(CustomLogger):
|
|||
# If no user provided directly, try to get it from token user_id
|
||||
if user_param is None:
|
||||
# Check if user_id is available from the API key metadata
|
||||
litellm_params: Final = kwargs.get("litellm_params", {})
|
||||
metadata: Final = litellm_params.get("metadata", {})
|
||||
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
|
||||
metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {}
|
||||
user_api_key_user_id: Final = metadata.get("user_api_key_user_id", None)
|
||||
|
||||
if user_api_key_user_id is not None:
|
||||
|
|
@ -89,7 +92,7 @@ class OpenMeterLogger(CustomLogger):
|
|||
return {
|
||||
"specversion": "1.0",
|
||||
"type": os.getenv("OPENMETER_EVENT_TYPE", "litellm_tokens"),
|
||||
"id": call_id,
|
||||
"id": logged_event_key(kwargs, call_id) if isinstance(call_id, str) else call_id,
|
||||
"time": dt,
|
||||
"subject": subject,
|
||||
"source": "litellm-proxy",
|
||||
|
|
|
|||
|
|
@ -3036,7 +3036,7 @@ class PrometheusLogger(CustomLogger):
|
|||
|
||||
async def async_post_call_failure_hook(
|
||||
self,
|
||||
request_data: dict,
|
||||
request_data: dict[str, object],
|
||||
original_exception: Exception,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
traceback_str: str | None = None,
|
||||
|
|
@ -3059,10 +3059,14 @@ class PrometheusLogger(CustomLogger):
|
|||
try:
|
||||
_tags: Final = StandardLoggingPayloadSetup.get_request_tags(
|
||||
litellm_params=request_data,
|
||||
proxy_server_request=request_data.get("proxy_server_request", {}),
|
||||
proxy_server_request=_logged_fields(request_data.get("proxy_server_request")),
|
||||
)
|
||||
_metadata: Final = _logged_fields(request_data.get("metadata"))
|
||||
model_id: Final = (
|
||||
_logged_fields(_metadata.get("model_info")).get("id")
|
||||
or _logged_fields(request_data.get("model_info")).get("id")
|
||||
or _logged_fields(request_data.get("standard_logging_object")).get("model_id")
|
||||
)
|
||||
_metadata: Final = request_data.get("metadata", {}) or {}
|
||||
model_id: Final = _metadata.get("model_info", {}).get("id") or request_data.get("model_info", {}).get("id")
|
||||
rate_limit_category, rate_limit_type = self._extract_rate_limit_labels(original_exception)
|
||||
api_provider: Final = self._extract_api_provider_from_request_data(
|
||||
request_data
|
||||
|
|
|
|||
|
|
@ -107,6 +107,7 @@ class ExceptionCheckers:
|
|||
"exceeds the available context size", # llama.cpp/Lemonade
|
||||
"exceeds the maximum number of tokens allowed", # Gemini
|
||||
'"code": "context_length_exceeded"',
|
||||
"prompt is too long",
|
||||
]
|
||||
for substring in known_exception_substrings:
|
||||
if substring in _error_str_lowercase:
|
||||
|
|
|
|||
|
|
@ -26,7 +26,12 @@ from litellm.constants import (
|
|||
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
|
||||
NON_INFERENCE_CALL_TYPES,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import initialize_standard_callback_dynamic_params
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
||||
SETTLES_BACKGROUND_RESPONSE_KWARG,
|
||||
BackgroundResponseSettlement,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
|
||||
BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN,
|
||||
|
|
@ -35,6 +40,8 @@ from litellm.types.utils import (
|
|||
|
||||
BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
|
||||
|
||||
BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX: Final = "_background_cost"
|
||||
|
||||
ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved"
|
||||
OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved"
|
||||
ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key"
|
||||
|
|
@ -151,6 +158,26 @@ def is_unbilled_non_inference_call_from_params(
|
|||
return is_unbilled_non_inference_call(call_type, metadata, response)
|
||||
|
||||
|
||||
def settles_background_response(call_type: str, metadata: Mapping[str, object], kwargs: Mapping[str, object]) -> bool:
|
||||
return kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES or (
|
||||
call_type in NON_INFERENCE_CALL_TYPES
|
||||
and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
|
||||
)
|
||||
|
||||
|
||||
def logged_event_key(kwargs: Mapping[str, object], event_id: str) -> str:
|
||||
call_type: Final = kwargs.get("call_type")
|
||||
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
|
||||
metadata: Final = (
|
||||
as_str_mapping(litellm_params.get("litellm_metadata")) or as_str_mapping(litellm_params.get("metadata")) or {}
|
||||
)
|
||||
if not event_id or not settles_background_response(
|
||||
call_type if isinstance(call_type, str) else "", metadata, kwargs
|
||||
):
|
||||
return event_id
|
||||
return f"{event_id}{BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX}"
|
||||
|
||||
|
||||
def sanitize_user_api_key_auth(auth: object) -> object:
|
||||
"""Copy of the auth object with its budget reservation removed; the cost callback
|
||||
falls back to reading the reservation from inside the auth object."""
|
||||
|
|
|
|||
|
|
@ -872,6 +872,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
# The model the proxy shows the client on streamed chunks. The logged streamed response carries it
|
||||
# once that response is priced, the same way a non-streamed response is logged
|
||||
self.client_facing_stream_model: str | None = None
|
||||
self.client_facing_response_id: str | None = None
|
||||
self.zero_cost_warned: bool = False
|
||||
self.llm_caching_handler: LLMCachingHandler | None = None
|
||||
|
||||
|
|
@ -918,6 +919,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
# post_call guardrails have run; the @client decorator then stores the
|
||||
# enqueue closure here instead of firing it immediately.
|
||||
self.defer_async_logging: bool = False
|
||||
self.drops_stream_token_ids: bool = False
|
||||
self.enqueue_deferred_logging: Callable[[], None] | None = None
|
||||
self._async_success_scheduled: bool = False
|
||||
self._async_success_enqueued: bool = False
|
||||
|
|
@ -3237,6 +3239,23 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
return response
|
||||
return response.model_copy(update={"id": response_id})
|
||||
|
||||
def _with_client_facing_response_id(
|
||||
self,
|
||||
response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
|
||||
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
|
||||
response_id: Final = self.client_facing_response_id
|
||||
if response_id is None or getattr(response, "id", None) == response_id:
|
||||
return response
|
||||
return response.model_copy(update={"id": response_id})
|
||||
|
||||
def _client_facing_streaming_response(
|
||||
self,
|
||||
response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
|
||||
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
|
||||
return self._with_client_facing_response_id(
|
||||
self._with_continued_response_id(self._with_client_facing_stream_model(response))
|
||||
)
|
||||
|
||||
def _success_handler_helper_fn(
|
||||
self,
|
||||
result: object = None,
|
||||
|
|
@ -3640,9 +3659,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
self._add_continued_partial_spend(complete_streaming_response)
|
||||
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
|
||||
logged_streaming_response: Final = self._with_continued_response_id(
|
||||
self._with_client_facing_stream_model(complete_streaming_response)
|
||||
)
|
||||
logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response)
|
||||
self.model_call_details["complete_streaming_response"] = logged_streaming_response
|
||||
self.capture_usage_before_logging(logged_streaming_response, start_time, end_time)
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
|
|
@ -4239,9 +4256,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
await self._prepare_baseline_cache_estimate(complete_streaming_response)
|
||||
|
||||
logged_streaming_response: Final = self._with_continued_response_id(
|
||||
self._with_client_facing_stream_model(complete_streaming_response)
|
||||
)
|
||||
logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response)
|
||||
self.model_call_details["async_complete_streaming_response"] = logged_streaming_response
|
||||
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@ import json
|
|||
import re
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import Generator, Iterator, Mapping, Sequence
|
||||
from collections.abc import Generator, Iterable, Iterator, Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, cast
|
||||
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
|
||||
|
|
@ -11,7 +12,12 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
|
||||
from litellm.constants import (
|
||||
RESPONSE_FORMAT_TOOL_NAME,
|
||||
STREAM_CHUNK_CHOICE_FIELDS,
|
||||
STREAM_FINISH_CHOICE_FIELDS,
|
||||
VLLM_CHOICE_FIELDS,
|
||||
)
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
extract_reasoning_content,
|
||||
|
|
@ -170,6 +176,28 @@ def _split_assembled_content_for_replay(content: str | None) -> list[str]:
|
|||
return _REPLAY_CONTENT_SLICE_RE.findall(content)
|
||||
|
||||
|
||||
def _drop_choice_fields(choice: StreamingChoices, fields: Iterable[str]) -> None:
|
||||
for field in fields:
|
||||
if hasattr(choice, field):
|
||||
delattr(choice, field)
|
||||
|
||||
|
||||
def _vllm_choice_fields(choice: Mapping[str, object]) -> Mapping[str, object]:
|
||||
cached_fields: Final = _choice_provider_specific_fields(choice)
|
||||
return MappingProxyType(
|
||||
{
|
||||
field: choice[field] if field in choice else cached_fields[field]
|
||||
for field in VLLM_CHOICE_FIELDS
|
||||
if field in choice or field in cached_fields
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _choice_provider_specific_fields(choice: Mapping[str, object]) -> Mapping[str, object]:
|
||||
fields: Final = choice.get("provider_specific_fields")
|
||||
return _FIELD_MAP.validate_python(fields) if isinstance(fields, dict) else MappingProxyType({})
|
||||
|
||||
|
||||
_REPLAY_REASONING_FIELDS: Final = frozenset({"reasoning_content", "thinking_blocks", "reasoning_items"})
|
||||
_REPLAY_LEAD_FIELDS: Final = _REPLAY_REASONING_FIELDS | {"role", "provider_specific_fields"}
|
||||
_REPLAY_TOOL_FIELDS: Final = frozenset({"tool_calls", "function_call"})
|
||||
|
|
@ -220,8 +248,9 @@ def _replay_frame(
|
|||
choice.finish_reason = finish_reason
|
||||
if not keeps_choice_metadata:
|
||||
choice.logprobs = None
|
||||
if hasattr(choice, "enhancements"):
|
||||
del choice.enhancements
|
||||
_drop_choice_fields(choice, ("enhancements", *STREAM_CHUNK_CHOICE_FIELDS))
|
||||
if finish_reason is None:
|
||||
_drop_choice_fields(choice, STREAM_FINISH_CHOICE_FIELDS)
|
||||
return frame
|
||||
|
||||
|
||||
|
|
@ -325,7 +354,9 @@ async def convert_to_streaming_response_async(
|
|||
|
||||
logprobs = choice.get("logprobs", None)
|
||||
|
||||
choice = StreamingChoices(finish_reason=finish_reason, index=idx, delta=delta, logprobs=logprobs)
|
||||
choice = StreamingChoices(
|
||||
finish_reason=finish_reason, index=idx, delta=delta, logprobs=logprobs, **_vllm_choice_fields(choice)
|
||||
)
|
||||
choice_list.append(choice)
|
||||
|
||||
model_response_object.choices = choice_list
|
||||
|
|
@ -388,6 +419,7 @@ def convert_to_streaming_response(
|
|||
delta=delta,
|
||||
logprobs=logprobs,
|
||||
enhancements=enhancements,
|
||||
**_vllm_choice_fields(choice),
|
||||
)
|
||||
|
||||
choice_list.append(choice)
|
||||
|
|
@ -872,6 +904,7 @@ def convert_to_model_response_object(
|
|||
logprobs=logprobs,
|
||||
enhancements=enhancements,
|
||||
provider_specific_fields=provider_specific_fields,
|
||||
**_vllm_choice_fields(choice),
|
||||
)
|
||||
if provider_sent_logprobs and logprobs is None:
|
||||
choice.logprobs = None
|
||||
|
|
|
|||
|
|
@ -10,12 +10,18 @@ class ChatTemplateToolSection:
|
|||
preamble_tokens: int
|
||||
implicit_system_turn_tokens: int
|
||||
render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str]
|
||||
counts_tool_choice: bool = False
|
||||
renders_strict_only_when_true: bool = False
|
||||
|
||||
|
||||
def _deepseek_v4_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
|
||||
def _json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
|
||||
return "\n".join(json.dumps(function, ensure_ascii=False) for function in functions)
|
||||
|
||||
|
||||
def _compact_json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
|
||||
return "\n".join(json.dumps(function, ensure_ascii=False, separators=(",", ":")) for function in functions)
|
||||
|
||||
|
||||
def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
|
||||
return "".join(
|
||||
"\n" + json.dumps({"type": "function", "function": function}, ensure_ascii=False) for function in functions
|
||||
|
|
@ -23,13 +29,29 @@ def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
|
|||
|
||||
|
||||
_TOOL_SECTIONS: Final = (
|
||||
(
|
||||
re.compile(r"deepseek-v4(?:[-._]|p)1(?!\d)", re.IGNORECASE),
|
||||
ChatTemplateToolSection(211, 0, _json_line_tool_schemas),
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"deepseek-v4(?![.\d]|p\d)|^(?:deepseek/)?deepseek-(?:chat|reasoner|flash)(?::free)?$", re.IGNORECASE
|
||||
),
|
||||
ChatTemplateToolSection(222, 0, _deepseek_v4_tool_schemas),
|
||||
ChatTemplateToolSection(222, 0, _json_line_tool_schemas),
|
||||
),
|
||||
(re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)),
|
||||
(
|
||||
re.compile(r"grok-4[-.]1(?![.\d])", re.IGNORECASE),
|
||||
ChatTemplateToolSection(
|
||||
0, 0, _compact_json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True
|
||||
),
|
||||
),
|
||||
(
|
||||
re.compile(r"grok-4[-.]3(?![.\d])", re.IGNORECASE),
|
||||
ChatTemplateToolSection(
|
||||
0, 0, _json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
|
|||
split_leading_system_run,
|
||||
string_field,
|
||||
)
|
||||
from litellm.litellm_core_utils.tool_fields import tools_without_hosted_mcp
|
||||
from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client, header_value
|
||||
from litellm.types.files import get_file_extension_from_mime_type
|
||||
|
|
@ -1328,7 +1329,7 @@ def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None:
|
|||
_get_thought_signature_from_tool = get_thought_signature_from_tool
|
||||
|
||||
|
||||
def _get_dummy_thought_signature() -> str:
|
||||
def get_dummy_thought_signature() -> str:
|
||||
"""Generate a dummy thought signature for models that require it.
|
||||
|
||||
This is used when transferring conversation history from older models
|
||||
|
|
@ -1415,7 +1416,7 @@ def convert_to_gemini_tool_call_invoke(
|
|||
# https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/thinking/thought-signatures#parallel_function_calling_example
|
||||
is_first_function_call = len(_parts_list) == 0
|
||||
if not thought_signature and is_first_function_call and needs_dummy_signature:
|
||||
thought_signature = _get_dummy_thought_signature()
|
||||
thought_signature = get_dummy_thought_signature()
|
||||
if thought_signature:
|
||||
part_dict["thoughtSignature"] = thought_signature
|
||||
|
||||
|
|
@ -1439,7 +1440,7 @@ def convert_to_gemini_tool_call_invoke(
|
|||
|
||||
# If no signature found and model is gemini-3, use dummy signature
|
||||
if not thought_signature and needs_dummy_signature:
|
||||
thought_signature = _get_dummy_thought_signature()
|
||||
thought_signature = get_dummy_thought_signature()
|
||||
|
||||
if thought_signature:
|
||||
part_dict_function["thoughtSignature"] = thought_signature
|
||||
|
|
@ -5916,8 +5917,13 @@ def function_call_prompt(
|
|||
llm_provider: str,
|
||||
drop_params: bool,
|
||||
) -> list[dict[str, object]]:
|
||||
prompt_functions: Final = tools_without_hosted_mcp(
|
||||
functions, model=model, route=llm_provider, drop_params=drop_params
|
||||
)
|
||||
if not prompt_functions:
|
||||
return messages
|
||||
header: Final = """To call a function, reply with JSON ONLY in this format {"name": "function_name", "arguments":{"argument_name": "argument_value"}}. Once a function result answers the request, reply to the user in plain text instead of calling a function again. The following functions are available to you:"""
|
||||
function_prompt: Final = header + "".join(f"\n{function}\n" for function in functions)
|
||||
function_prompt: Final = header + "".join(f"\n{function}\n" for function in prompt_functions)
|
||||
if not any(_is_system_message(message) for message in messages):
|
||||
return [*messages, {"role": "system", "content": function_prompt}]
|
||||
return [
|
||||
|
|
|
|||
|
|
@ -196,6 +196,9 @@ def _redact_function_call(function_call) -> None:
|
|||
|
||||
def _redact_choice_content(choice):
|
||||
"""Helper to redact content in a choice (message or delta)."""
|
||||
if isinstance(choice, (litellm.Choices, litellm.utils.StreamingChoices, litellm.utils.TextChoices)):
|
||||
drop_token_ids(choice)
|
||||
_redact_provider_specific_fields(choice)
|
||||
if isinstance(choice, litellm.Choices):
|
||||
if choice.message.content is not None:
|
||||
choice.message.content = REDACTED_BY_LITELLM
|
||||
|
|
@ -216,6 +219,9 @@ def _redact_choice_content(choice):
|
|||
_redact_provider_specific_fields(choice.delta)
|
||||
_redact_tool_calls(getattr(choice.delta, "tool_calls", None))
|
||||
_redact_function_call(getattr(choice.delta, "function_call", None))
|
||||
elif isinstance(choice, litellm.utils.TextChoices):
|
||||
_redact_field_when_present(choice, "text")
|
||||
_redact_field_when_present(choice, "reasoning_content")
|
||||
|
||||
|
||||
def _as_mapping(value: object) -> Mapping[str, object] | None:
|
||||
|
|
@ -227,14 +233,27 @@ def _redact_provider_specific_fields(message: object) -> None:
|
|||
setattr(message, "provider_specific_fields", _redacted_provider_specific_fields(fields)) # noqa: B010 # typed as object
|
||||
|
||||
|
||||
def _drop_provider_specific_token_ids(holder: object) -> None:
|
||||
fields: Final = _as_mapping(getattr(holder, "provider_specific_fields", None))
|
||||
if fields is not None and fields.get("token_ids") is not None:
|
||||
setattr(holder, "provider_specific_fields", {**fields, "token_ids": None}) # noqa: B010 # typed as object
|
||||
|
||||
|
||||
def drop_token_ids(choice: object) -> None:
|
||||
if getattr(choice, "token_ids", None) is not None:
|
||||
setattr(choice, "token_ids", None) # noqa: B010 # pydantic extra, not a declared field
|
||||
for holder in (choice, getattr(choice, "delta", None), getattr(choice, "message", None)):
|
||||
_drop_provider_specific_token_ids(holder)
|
||||
|
||||
|
||||
def _redact_provider_specific_fields_dict(choice: Mapping[str, object]) -> None:
|
||||
for message in (choice.get("message"), choice.get("delta")):
|
||||
for holder in (choice, choice.get("message"), choice.get("delta")):
|
||||
if (
|
||||
(readable := _as_mapping(message)) is not None
|
||||
(readable := _as_mapping(holder)) is not None
|
||||
and (fields := _as_mapping(readable.get("provider_specific_fields"))) is not None
|
||||
and isinstance(message, dict)
|
||||
and isinstance(holder, dict)
|
||||
):
|
||||
message["provider_specific_fields"] = _redacted_provider_specific_fields(fields)
|
||||
holder["provider_specific_fields"] = _redacted_provider_specific_fields(fields)
|
||||
|
||||
|
||||
def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[str, object]:
|
||||
|
|
@ -251,6 +270,7 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[
|
|||
"compaction_blocks",
|
||||
"compaction_start",
|
||||
"compaction_delta",
|
||||
"token_ids",
|
||||
)
|
||||
if key in fields
|
||||
},
|
||||
|
|
@ -422,6 +442,8 @@ def _redact_model_response_dict_choices(choices, redacted_str: str):
|
|||
for choice in choices:
|
||||
if isinstance(choice, dict):
|
||||
_redact_provider_specific_fields_dict(choice)
|
||||
if choice.get("token_ids") is not None:
|
||||
choice["token_ids"] = None
|
||||
if "message" in choice and isinstance(choice["message"], dict):
|
||||
if choice["message"].get("content") is not None:
|
||||
choice["message"]["content"] = redacted_str
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from collections.abc import Iterable
|
||||
from typing import Final
|
||||
|
||||
from litellm.constants import VLLM_CHOICE_FIELDS
|
||||
from litellm.types.utils import ModelResponseStream, StreamingChoices
|
||||
|
||||
|
||||
|
|
@ -9,12 +10,19 @@ def is_empty_streaming_choice(choice: StreamingChoices) -> bool:
|
|||
return False
|
||||
if getattr(choice, "logprobs", None) is not None:
|
||||
return False
|
||||
if not _all_values_none(_vllm_choice_values(choice)):
|
||||
return False
|
||||
delta: Final = choice.delta if hasattr(choice, "delta") else None
|
||||
if delta is None:
|
||||
return True
|
||||
return _all_values_none(delta.model_dump().values())
|
||||
|
||||
|
||||
def _vllm_choice_values(choice: StreamingChoices) -> Iterable[object]:
|
||||
choice_extras: Final = choice.model_extra or {}
|
||||
return (choice_extras.get(field) for field in VLLM_CHOICE_FIELDS)
|
||||
|
||||
|
||||
def _all_values_none(values: Iterable[object]) -> bool:
|
||||
return all(value is None for value in values)
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, Required
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import VLLM_CHOICE_FIELDS
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS,
|
||||
UPSTREAM_USAGE_COST,
|
||||
|
|
@ -31,6 +32,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
|
|||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict
|
||||
)
|
||||
from litellm.litellm_core_utils.redact_messages import drop_token_ids
|
||||
from litellm.litellm_core_utils.token_counter import usage_tokenizer_model
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
|
|
@ -434,6 +436,8 @@ class ChunkProcessor:
|
|||
chunks, unreported="tool_calls" if _carries_tool_calls(chunks) else "stop"
|
||||
),
|
||||
message={"role": ChunkProcessor._get_role_from_chunks(chunks), "content": ""},
|
||||
provider_specific_fields=None,
|
||||
**_streamed_vllm_choice_fields(chain.from_iterable(chunk.get("choices") or () for chunk in chunks)),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -504,6 +508,7 @@ class ChunkProcessor:
|
|||
"index": 0,
|
||||
"message": {"role": role, "content": ""},
|
||||
"finish_reason": finish_reason,
|
||||
**_streamed_vllm_choice_fields(_first_choices(chunks)),
|
||||
}
|
||||
]
|
||||
),
|
||||
|
|
@ -1371,6 +1376,7 @@ def _rebuilt_text_choice(index: int, choices: Sequence[TextChoices]) -> TextChoi
|
|||
index=index,
|
||||
logprobs=_merged_logprobs(choices),
|
||||
finish_reason=next((choice.finish_reason for choice in reversed(choices) if choice.finish_reason), None),
|
||||
**_streamed_vllm_choice_fields(choices),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1504,6 +1510,31 @@ def _field_of(value: object, name: str) -> object:
|
|||
return getattr(value, name, None)
|
||||
|
||||
|
||||
def _first_choices(chunks: Iterable[object]) -> tuple[object, ...]:
|
||||
return tuple(next(iter(object_items(_field_of(chunk, "choices"))), None) for chunk in chunks)
|
||||
|
||||
|
||||
def _choice_extras(choice: object) -> Mapping[str, object]:
|
||||
if isinstance(choice, BaseModel):
|
||||
return choice.model_extra or {}
|
||||
return object_mapping(choice) or {}
|
||||
|
||||
|
||||
def _streamed_vllm_choice_fields(choices: Iterable[object]) -> Mapping[str, object]:
|
||||
extras: Final = tuple(map(_choice_extras, choices))
|
||||
token_id_runs: Final = tuple(object_items(fields.get("token_ids")) for fields in extras)
|
||||
stop_reasons: Final = tuple(reason for fields in extras if (reason := fields.get("stop_reason")) is not None)
|
||||
has_token_ids: Final = any(fields.get("token_ids") is not None for fields in extras)
|
||||
streamed_values: Final[Mapping[str, object]] = MappingProxyType(
|
||||
{
|
||||
"token_ids": list(chain.from_iterable(token_id_runs)) if has_token_ids else None,
|
||||
"stop_reason": stop_reasons[-1] if stop_reasons else None,
|
||||
}
|
||||
)
|
||||
present_fields: Final = frozenset(chain.from_iterable(fields.keys() for fields in extras))
|
||||
return MappingProxyType({field: streamed_values[field] for field in VLLM_CHOICE_FIELDS if field in present_fields})
|
||||
|
||||
|
||||
def _delta_reasoning_items(delta: object) -> Sequence[object]:
|
||||
items: Final = _field_of(delta, "reasoning_items")
|
||||
return cast(Sequence[object], items) if isinstance(items, list) else () # cast-ok: items forwarded verbatim
|
||||
|
|
@ -1776,6 +1807,19 @@ def stream_chunk_builder(
|
|||
end_time: object = None,
|
||||
logging_obj: Optional["Logging"] = None,
|
||||
count_prompt_tokens: Callable[[], int] | None = None,
|
||||
) -> ModelResponse | TextCompletionResponse | None:
|
||||
response: Final = _assembled_stream_response(chunks, messages, logging_obj, count_prompt_tokens)
|
||||
if response is not None and getattr(logging_obj, "drops_stream_token_ids", False) is True:
|
||||
for choice in response.choices:
|
||||
drop_token_ids(choice)
|
||||
return response
|
||||
|
||||
|
||||
def _assembled_stream_response(
|
||||
chunks: Sequence[object],
|
||||
messages: Sequence[AllMessageValues | Message] | None,
|
||||
logging_obj: Optional["Logging"],
|
||||
count_prompt_tokens: Callable[[], int] | None,
|
||||
) -> ModelResponse | TextCompletionResponse | None:
|
||||
try:
|
||||
if chunks is None:
|
||||
|
|
@ -1788,7 +1832,7 @@ def stream_chunk_builder(
|
|||
if not chunks:
|
||||
return None
|
||||
|
||||
processor: Final = ChunkProcessor(chunks, messages)
|
||||
processor: Final = ChunkProcessor(list(chunks), messages)
|
||||
chunks = processor.chunks
|
||||
|
||||
### BASE-CASE ###
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from typing_extensions import NotRequired, TypedDict
|
|||
import litellm
|
||||
from litellm import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.constants import STREAM_CHUNK_CHOICE_FIELDS, STREAM_FINISH_CHOICE_FIELDS
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
UPSTREAM_USAGE_COST,
|
||||
|
|
@ -37,10 +38,11 @@ from litellm.litellm_core_utils.hidden_params import (
|
|||
set_hidden_param,
|
||||
stored_attribute,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_items
|
||||
from litellm.litellm_core_utils.model_response_utils import (
|
||||
is_model_response_stream_empty,
|
||||
)
|
||||
from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject
|
||||
from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject, drop_token_ids
|
||||
from litellm.litellm_core_utils.thread_pool_executor import executor
|
||||
from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
|
@ -137,6 +139,70 @@ def _prepend_upstream_choice_extras(
|
|||
choice.__pydantic_fields_set__.update(upstream_extras)
|
||||
|
||||
|
||||
_NO_UPSTREAM_CHOICE_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_NO_CHOICE_FIELDS_BY_INDEX: Final[Mapping[int, Mapping[str, object]]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _upstream_first_choice(upstream_chunk: object) -> BaseModel | None:
|
||||
match getattr(upstream_chunk, "choices", None):
|
||||
case [BaseModel() as first_choice, *_]:
|
||||
return first_choice
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _upstream_choice_fields(upstream_chunk: object, fields: Sequence[str]) -> Mapping[str, object]:
|
||||
first_choice: Final = _upstream_first_choice(upstream_chunk)
|
||||
if first_choice is None:
|
||||
return _NO_UPSTREAM_CHOICE_FIELDS
|
||||
upstream_extras: Final[Mapping[str, object]] = first_choice.model_extra or {}
|
||||
return MappingProxyType({field: upstream_extras[field] for field in fields if field in upstream_extras})
|
||||
|
||||
|
||||
def _upstream_choice_index(upstream_chunk: object) -> int:
|
||||
upstream_index: Final[object] = getattr(_upstream_first_choice(upstream_chunk), "index", 0)
|
||||
return upstream_index if isinstance(upstream_index, int) else 0
|
||||
|
||||
|
||||
def _response_choice_index(response_obj: Mapping[str, object]) -> int:
|
||||
choice_index: Final = response_obj.get("choice_index")
|
||||
if isinstance(choice_index, int):
|
||||
return choice_index
|
||||
return _upstream_choice_index(response_obj.get("original_chunk"))
|
||||
|
||||
|
||||
def _non_null_upstream_choice_fields(upstream_chunk: object, fields: Sequence[str]) -> Mapping[str, object]:
|
||||
first_choice: Final = _upstream_first_choice(upstream_chunk)
|
||||
upstream_extras: Final = first_choice.model_extra if first_choice is not None else None
|
||||
if not upstream_extras:
|
||||
return _NO_UPSTREAM_CHOICE_FIELDS
|
||||
return MappingProxyType(
|
||||
{field: upstream_extras[field] for field in fields if upstream_extras.get(field) is not None}
|
||||
)
|
||||
|
||||
|
||||
def _apply_choice_fields(choice: StreamingChoices, fields: Mapping[str, object]) -> None:
|
||||
for field, value in fields.items():
|
||||
setattr(choice, field, value)
|
||||
|
||||
|
||||
def _choice_fields_chunk(
|
||||
model_response: ModelResponseStream, upstream_index: int, fields: Mapping[str, object]
|
||||
) -> ModelResponseStream:
|
||||
model_response.choices[0].delta = Delta(content="") # rebind-ok: chunks are built in place here
|
||||
model_response.choices[0].index = upstream_index # rebind-ok: chunks are built in place here
|
||||
_apply_choice_fields(model_response.choices[0], fields)
|
||||
return model_response
|
||||
|
||||
|
||||
def _without_token_ids_under_post_call_rules(chunk: ModelResponseStream | None) -> ModelResponseStream | None:
|
||||
if chunk is None or not litellm.post_call_rules:
|
||||
return chunk
|
||||
for choice in chunk.choices:
|
||||
drop_token_ids(choice)
|
||||
return chunk
|
||||
|
||||
|
||||
def _holds_only_scalars(fields: Iterable[tuple[str, object]]) -> bool:
|
||||
return all(value is None or isinstance(value, (str, int, float, bool)) for _, value in fields)
|
||||
|
||||
|
|
@ -378,6 +444,8 @@ class CustomStreamWrapper:
|
|||
self._provider_response_model: str | None = None
|
||||
self._upstream_provider: str | None = None
|
||||
self.received_finish_reason: str | None = None
|
||||
self.received_finish_choice_fields: Mapping[int, Mapping[str, object]] = _NO_CHOICE_FIELDS_BY_INDEX
|
||||
self.carried_chunk_choice_fields: Mapping[int, Mapping[str, object]] = _NO_CHOICE_FIELDS_BY_INDEX
|
||||
self.intermittent_finish_reason: str | None = None # finish reasons that show up mid-stream
|
||||
self.received_any_finish_reason = False # a later chunk can reset intermittent_finish_reason
|
||||
self.special_tokens = [
|
||||
|
|
@ -823,6 +891,7 @@ class CustomStreamWrapper:
|
|||
"logprobs": logprobs,
|
||||
"original_chunk": str_line,
|
||||
"usage": usage,
|
||||
"chunk_choice_fields": _non_null_upstream_choice_fields(str_line, STREAM_CHUNK_CHOICE_FIELDS),
|
||||
}
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
|
@ -847,7 +916,7 @@ class CustomStreamWrapper:
|
|||
except Exception as e:
|
||||
raise e
|
||||
|
||||
def handle_openai_text_completion_chunk(self, chunk):
|
||||
def handle_openai_text_completion_chunk(self, chunk: object):
|
||||
try:
|
||||
text = ""
|
||||
is_finished = False
|
||||
|
|
@ -860,11 +929,15 @@ class CustomStreamWrapper:
|
|||
is_finished = True
|
||||
finish_reason = choices[0].finish_reason
|
||||
usage = getattr(chunk, "usage", None)
|
||||
choice_index: Final = _upstream_choice_index(chunk)
|
||||
return {
|
||||
"text": text,
|
||||
"is_finished": is_finished,
|
||||
"finish_reason": finish_reason,
|
||||
"usage": usage,
|
||||
"chunk_choice_fields": _non_null_upstream_choice_fields(chunk, STREAM_CHUNK_CHOICE_FIELDS),
|
||||
"choice_index": choice_index,
|
||||
"sibling_finish_reason": finish_reason if choice_index != 0 else None,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
|
|
@ -1183,6 +1256,13 @@ class CustomStreamWrapper:
|
|||
)
|
||||
|
||||
is_chunk_non_empty: Final = self.is_chunk_non_empty(completion_obj, model_response, response_obj)
|
||||
upstream_chunk_choice_fields: Final[Mapping[str, object]] = (
|
||||
_NO_UPSTREAM_CHOICE_FIELDS
|
||||
if litellm.post_call_rules
|
||||
else response_obj.get("chunk_choice_fields", _NO_UPSTREAM_CHOICE_FIELDS)
|
||||
)
|
||||
chunk_choice_fields: Final = self._take_carried_chunk_choice_fields(response_obj, upstream_chunk_choice_fields)
|
||||
sibling_finish_reason: Final[str | None] = response_obj.get("sibling_finish_reason")
|
||||
|
||||
if is_chunk_non_empty: # cannot set content of an OpenAI Object to be an empty string
|
||||
self.raise_on_model_repetition()
|
||||
|
|
@ -1270,16 +1350,40 @@ class CustomStreamWrapper:
|
|||
if response_obj.get("provider_specific_fields") is not None:
|
||||
completion_obj["provider_specific_fields"] = response_obj["provider_specific_fields"]
|
||||
model_response.choices[0].delta = Delta(**completion_obj)
|
||||
_index: Final[int | None] = completion_obj.get("index")
|
||||
_index: Final[int | None] = completion_obj.get("index", response_obj.get("choice_index"))
|
||||
if _index is not None:
|
||||
model_response.choices[0].index = _index
|
||||
|
||||
_apply_choice_fields(model_response.choices[0], chunk_choice_fields)
|
||||
if sibling_finish_reason is not None:
|
||||
self._finish_sibling_choice(model_response.choices[0], sibling_finish_reason)
|
||||
elif response_obj.get("is_finished"):
|
||||
self._apply_finish_choice_fields(model_response.choices[0])
|
||||
self._optional_combine_thinking_block_in_choices(model_response=model_response)
|
||||
|
||||
return model_response
|
||||
else:
|
||||
return
|
||||
elif sibling_finish_reason is not None:
|
||||
sibling_chunk: Final = _choice_fields_chunk(
|
||||
model_response, _response_choice_index(response_obj), chunk_choice_fields
|
||||
)
|
||||
self._finish_sibling_choice(sibling_chunk.choices[0], sibling_finish_reason)
|
||||
return sibling_chunk
|
||||
elif self.received_finish_reason is not None:
|
||||
chunk_upstream_index: Final = _response_choice_index(response_obj)
|
||||
if chunk_choice_fields and self._streams_past_the_pending_finish(
|
||||
model_response, response_obj, chunk_upstream_index
|
||||
):
|
||||
self.carried_chunk_choice_fields = MappingProxyType(
|
||||
{**self.carried_chunk_choice_fields, chunk_upstream_index: chunk_choice_fields}
|
||||
)
|
||||
else:
|
||||
another_choice_fields_chunk: Final = self._another_choice_fields_chunk(
|
||||
model_response, response_obj, chunk_choice_fields
|
||||
)
|
||||
if another_choice_fields_chunk is not None:
|
||||
return another_choice_fields_chunk
|
||||
if self.sent_last_chunk is True:
|
||||
# Bedrock returns the guardrail trace in the last chunk - we want to return this here
|
||||
if self.custom_llm_provider == "bedrock" and "trace" in model_response:
|
||||
|
|
@ -1317,12 +1421,20 @@ class CustomStreamWrapper:
|
|||
model_response.choices[0].finish_reason = self._normalize_tool_call_finish_reason(
|
||||
map_finish_reason(finish_reason=self.received_finish_reason)
|
||||
) # ensure consistent output to openai
|
||||
self._apply_finish_choice_fields(model_response.choices[0])
|
||||
if (
|
||||
response_obj.get("is_finished")
|
||||
and _response_choice_index(response_obj) == model_response.choices[0].index
|
||||
):
|
||||
_apply_choice_fields(model_response.choices[0], chunk_choice_fields)
|
||||
|
||||
self.sent_last_chunk = True
|
||||
|
||||
return model_response
|
||||
elif self._has_special_delta_content(model_response):
|
||||
return self._handle_special_delta_content(model_response)
|
||||
elif chunk_choice_fields:
|
||||
return _choice_fields_chunk(model_response, _response_choice_index(response_obj), chunk_choice_fields)
|
||||
else:
|
||||
if hasattr(model_response, "usage"):
|
||||
self.chunks.append(model_response)
|
||||
|
|
@ -1545,6 +1657,8 @@ class CustomStreamWrapper:
|
|||
completion_obj["content"] = response_obj["text"]
|
||||
print_verbose(f"completion obj content: {completion_obj['content']}")
|
||||
if response_obj["is_finished"]:
|
||||
self._record_finish_choice_fields(chunk)
|
||||
if response_obj["is_finished"] and response_obj["sibling_finish_reason"] is None:
|
||||
self.received_finish_reason = response_obj["finish_reason"]
|
||||
if response_obj["usage"] is not None:
|
||||
_text_completion_usage: Final[Usage] = response_obj["usage"]
|
||||
|
|
@ -1608,6 +1722,7 @@ class CustomStreamWrapper:
|
|||
self.system_fingerprint = cached_chunk.system_fingerprint
|
||||
if response_obj["is_finished"]:
|
||||
self.received_finish_reason = response_obj["finish_reason"]
|
||||
self._record_finish_choice_fields(cached_chunk)
|
||||
else: # openai / azure chat model
|
||||
self.uses_openai_chat_stream = True
|
||||
if isinstance(chunk, ChatCompletionChunk):
|
||||
|
|
@ -1637,6 +1752,7 @@ class CustomStreamWrapper:
|
|||
f"{self.custom_llm_provider} raised a streaming error - finish_reason: error, no content string given. Received Chunk={response_obj}"
|
||||
)
|
||||
self.received_finish_reason = response_obj["finish_reason"]
|
||||
self._record_finish_choice_fields(chunk)
|
||||
if response_obj.get("original_chunk", None) is not None:
|
||||
if hasattr(response_obj["original_chunk"], "id"):
|
||||
model_response = self.set_model_id(response_obj["original_chunk"].id, model_response)
|
||||
|
|
@ -1704,7 +1820,7 @@ class CustomStreamWrapper:
|
|||
completion_obj=completion_obj,
|
||||
)
|
||||
if isinstance(dispatch_result, _ProviderChunkEarlyReturn):
|
||||
return dispatch_result.value
|
||||
return _without_token_ids_under_post_call_rules(dispatch_result.value)
|
||||
response_obj = dispatch_result.response_obj
|
||||
|
||||
model_response.model = self._served_openrouter_model() or self.model
|
||||
|
|
@ -1794,7 +1910,7 @@ class CustomStreamWrapper:
|
|||
response_obj=response_obj,
|
||||
plain_delta_created=plain_delta_created,
|
||||
)
|
||||
return result
|
||||
return _without_token_ids_under_post_call_rules(result)
|
||||
|
||||
except StopIteration:
|
||||
raise StopIteration
|
||||
|
|
@ -1987,6 +2103,75 @@ class CustomStreamWrapper:
|
|||
return "tool_calls"
|
||||
return finish_reason
|
||||
|
||||
def _record_finish_choice_fields(self, upstream_chunk: object) -> None:
|
||||
self.received_finish_choice_fields = MappingProxyType(
|
||||
{
|
||||
**self.received_finish_choice_fields,
|
||||
_upstream_choice_index(upstream_chunk): _upstream_choice_fields(
|
||||
upstream_chunk, STREAM_FINISH_CHOICE_FIELDS
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
def _apply_finish_choice_fields(self, choice: StreamingChoices) -> None:
|
||||
_apply_choice_fields(choice, self.received_finish_choice_fields.get(choice.index, _NO_UPSTREAM_CHOICE_FIELDS))
|
||||
|
||||
def _finish_sibling_choice(self, choice: StreamingChoices, finish_reason: str) -> None:
|
||||
choice.finish_reason = map_finish_reason(finish_reason) # rebind-ok: chunks are built in place here
|
||||
self._apply_finish_choice_fields(choice)
|
||||
|
||||
def _streams_past_the_pending_finish(
|
||||
self, model_response: ModelResponseStream, response_obj: Mapping[str, object], upstream_index: int
|
||||
) -> bool:
|
||||
finishing_index: Final = model_response.choices[0].index
|
||||
return (
|
||||
not self.sent_last_chunk
|
||||
and not response_obj.get("is_finished")
|
||||
and upstream_index != finishing_index
|
||||
and finishing_index in self.received_finish_choice_fields
|
||||
)
|
||||
|
||||
def _take_carried_chunk_choice_fields(
|
||||
self, response_obj: Mapping[str, object], chunk_choice_fields: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
if not self.carried_chunk_choice_fields:
|
||||
return chunk_choice_fields
|
||||
upstream_index: Final = _response_choice_index(response_obj)
|
||||
carried: Final = self.carried_chunk_choice_fields.get(upstream_index)
|
||||
if carried is None:
|
||||
return chunk_choice_fields
|
||||
self.carried_chunk_choice_fields = MappingProxyType(
|
||||
{index: fields for index, fields in self.carried_chunk_choice_fields.items() if index != upstream_index}
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
field: [*object_items(carried.get(field)), *object_items(chunk_choice_fields.get(field))]
|
||||
for field in STREAM_CHUNK_CHOICE_FIELDS
|
||||
if field in carried or field in chunk_choice_fields
|
||||
}
|
||||
)
|
||||
|
||||
def _another_choice_fields_chunk(
|
||||
self,
|
||||
model_response: ModelResponseStream,
|
||||
response_obj: Mapping[str, object],
|
||||
chunk_choice_fields: Mapping[str, object],
|
||||
) -> ModelResponseStream | None:
|
||||
upstream_index: Final = _response_choice_index(response_obj)
|
||||
is_finished: Final = bool(response_obj.get("is_finished"))
|
||||
if is_finished and upstream_index == model_response.choices[0].index and not self.sent_last_chunk:
|
||||
return None
|
||||
finish_fields: Final = (
|
||||
self.received_finish_choice_fields.get(upstream_index, _NO_UPSTREAM_CHOICE_FIELDS)
|
||||
if is_finished
|
||||
else _NO_UPSTREAM_CHOICE_FIELDS
|
||||
)
|
||||
if not chunk_choice_fields and all(value is None for value in finish_fields.values()):
|
||||
return None
|
||||
return _choice_fields_chunk(
|
||||
model_response, upstream_index, MappingProxyType({**finish_fields, **chunk_choice_fields})
|
||||
)
|
||||
|
||||
def finish_reason_handler(self):
|
||||
model_response: Final = self.model_response_creator(
|
||||
hidden_params=None if self.received_model_id is None else {"received_model_id": self.received_model_id}
|
||||
|
|
@ -2005,6 +2190,7 @@ class CustomStreamWrapper:
|
|||
model_response.choices[0].finish_reason = self._normalize_tool_call_finish_reason(
|
||||
_finish_reason if _finish_reason is not None else "stop"
|
||||
)
|
||||
self._apply_finish_choice_fields(model_response.choices[0])
|
||||
close_tag: Final = self._close_open_think_block()
|
||||
if close_tag is not None:
|
||||
model_response.choices[0].delta.content = close_tag
|
||||
|
|
|
|||
|
|
@ -990,7 +990,11 @@ def _count_extra(
|
|||
tool_section (Optional[ChatTemplateToolSection]): How the model's chat template renders tools, if known.
|
||||
"""
|
||||
reply_primer_tokens: Final = 3
|
||||
tool_choice_tokens: Final = 0 if tool_section is not None else _count_tool_choice(count_function, tool_choice)
|
||||
tool_choice_tokens: Final = (
|
||||
_count_tool_choice(count_function, tool_choice)
|
||||
if tool_section is None or tool_section.counts_tool_choice
|
||||
else 0
|
||||
)
|
||||
return (
|
||||
reply_primer_tokens
|
||||
+ _count_tool_definitions(count_function, tools, includes_system_message, tool_section)
|
||||
|
|
@ -1029,14 +1033,29 @@ def _count_tool_choice(
|
|||
return 0
|
||||
|
||||
|
||||
def _named_function_definitions(tools: Sequence[object]) -> Iterable[Mapping[str, object]]:
|
||||
return (function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name"))
|
||||
|
||||
|
||||
def sends_function_tools(tools: Sequence[object] | None) -> bool:
|
||||
return tools is not None and any(_named_function_definitions(tools))
|
||||
|
||||
|
||||
def _without_false_strict(function: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {key: value for key, value in function.items() if key != "strict" or value}
|
||||
|
||||
|
||||
def _count_chat_template_tools(
|
||||
count_function: TokenCounterFunction,
|
||||
tools: Sequence[object],
|
||||
includes_system_message: bool,
|
||||
tool_section: ChatTemplateToolSection,
|
||||
) -> int:
|
||||
functions: Final = tuple(
|
||||
function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name")
|
||||
named_functions: Final = tuple(_named_function_definitions(tools))
|
||||
functions: Final = (
|
||||
tuple(map(_without_false_strict, named_functions))
|
||||
if tool_section.renders_strict_only_when_true
|
||||
else named_functions
|
||||
)
|
||||
if not functions:
|
||||
return 0
|
||||
|
|
|
|||
|
|
@ -1,13 +1,16 @@
|
|||
from collections.abc import Callable, Mapping
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from copy import copy
|
||||
from dataclasses import dataclass
|
||||
from functools import reduce
|
||||
from typing import Annotated, Final, TypeAlias
|
||||
from typing import Annotated, Final, TypeAlias, TypeVar
|
||||
|
||||
from pydantic import BeforeValidator, ConfigDict, Json, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
|
||||
|
||||
ROUTES_WITH_TOOLS: Final = frozenset(
|
||||
{
|
||||
|
|
@ -162,3 +165,145 @@ def resolve_deployment_tool_fields(
|
|||
if "proxy_server_request" not in kwargs or route not in ROUTES_WITH_TOOLS:
|
||||
return kwargs
|
||||
return resolve_tool_fields(kwargs, lambda: (drop_policy(kwargs),))
|
||||
|
||||
|
||||
_ToolT: Final = TypeVar("_ToolT")
|
||||
|
||||
HOSTED_MCP_DROP_WARNING: Final = (
|
||||
"Dropping the hosted MCP tool(s) %s for model=%s on %s (drop_params=True) because this route cannot run a "
|
||||
"hosted MCP tool."
|
||||
)
|
||||
|
||||
|
||||
def _hosted_mcp_server_label(tool: object) -> str | None:
|
||||
match tool:
|
||||
case {"type": "mcp", "server_label": str() as server_label}:
|
||||
return server_label
|
||||
case {"type": "mcp"}:
|
||||
return "unlabeled"
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _drop_params_hint(dropped: str) -> str:
|
||||
return (
|
||||
f"To drop {dropped} instead, set `litellm.drop_params=True` or for proxy:\n\n"
|
||||
"`litellm_settings:\n drop_params: true`\n"
|
||||
)
|
||||
|
||||
|
||||
def _one_log_line(text: str) -> str:
|
||||
return text.replace("\r", "").replace("\n", "")
|
||||
|
||||
|
||||
def _unrunnable_mcp_error(message: str, model: str, route: str) -> UnsupportedParamsError:
|
||||
return UnsupportedParamsError(message=message, model=model, llm_provider=route, status_code=400)
|
||||
|
||||
|
||||
def tools_without_hosted_mcp(tools: Sequence[_ToolT], model: str, route: str, drop_params: bool) -> Sequence[_ToolT]:
|
||||
server_labels: Final = tuple(label for label in map(_hosted_mcp_server_label, tools) if label is not None)
|
||||
if not server_labels:
|
||||
return tools
|
||||
named_servers: Final = ", ".join(f"`{label}`" for label in server_labels)
|
||||
if not drops_unsupported_params(drop_params):
|
||||
raise _unrunnable_mcp_error(
|
||||
f'{model} on {route} cannot run the hosted MCP tool {named_servers} (`type: "mcp"`). To have LiteLLM '
|
||||
"run the MCP server for this model, register it on the LiteLLM MCP gateway and send "
|
||||
'`server_url: "litellm_proxy/mcp/<server name>"`. ' + _drop_params_hint("hosted MCP tools"),
|
||||
model=model,
|
||||
route=route,
|
||||
)
|
||||
verbose_logger.warning(
|
||||
HOSTED_MCP_DROP_WARNING, _one_log_line(named_servers), _one_log_line(model), _one_log_line(route)
|
||||
)
|
||||
return [tool for tool in tools if _hosted_mcp_server_label(tool) is None]
|
||||
|
||||
|
||||
MCP_SERVERS_DROP_WARNING: Final = (
|
||||
"Dropping mcp_servers %s and their mcp_toolset tools for model=%s on %s (drop_params=True) because LiteLLM "
|
||||
"cannot pass mcp_servers to this route."
|
||||
)
|
||||
|
||||
ROUTES_WITHOUT_MCP_SERVERS: Final = frozenset(
|
||||
{"bedrock", "databricks", "deepseek", "gemini", "gigachat", "ollama", "ollama_chat", "vertex_ai", "vertex_ai_beta"}
|
||||
)
|
||||
|
||||
|
||||
def _mcp_server_name(server: object) -> str:
|
||||
match server:
|
||||
case {"name": str() as name}:
|
||||
return name
|
||||
case _:
|
||||
return "unnamed"
|
||||
|
||||
|
||||
def _mcp_server_names(mcp_servers: object) -> tuple[str, ...]:
|
||||
if not mcp_servers:
|
||||
return ()
|
||||
if not isinstance(mcp_servers, list):
|
||||
return ("unnamed",)
|
||||
return tuple(map(_mcp_server_name, _TOOLS.validate_python(mcp_servers)))
|
||||
|
||||
|
||||
def _is_mcp_toolset(tool: object) -> bool:
|
||||
match tool:
|
||||
case {"type": "mcp_toolset"}:
|
||||
return True
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def drops_mcp_servers(request_kwargs: Mapping[str, object]) -> bool:
|
||||
additional_drop_params: Final = request_kwargs.get("additional_drop_params")
|
||||
return drops_unsupported_params(request_kwargs.get("drop_params")) or (
|
||||
isinstance(additional_drop_params, list) and "mcp_servers" in additional_drop_params
|
||||
)
|
||||
|
||||
|
||||
def tools_without_mcp_servers(
|
||||
mcp_servers: object, tools: Sequence[_ToolT], model: str, route: str, drop_params: bool
|
||||
) -> Sequence[_ToolT]:
|
||||
server_names: Final = _mcp_server_names(mcp_servers)
|
||||
if not server_names:
|
||||
return tools
|
||||
named_servers: Final = ", ".join(f"`{name}`" for name in server_names)
|
||||
if not drops_unsupported_params(drop_params):
|
||||
raise _unrunnable_mcp_error(
|
||||
f"LiteLLM cannot pass `mcp_servers` to {model} on {route}, so the MCP server {named_servers} would not "
|
||||
'run. To have LiteLLM run it for this model, register it on the LiteLLM MCP gateway and send a `type: "mcp"` '
|
||||
'tool with `server_url: "litellm_proxy/mcp/<server name>"`. ' + _drop_params_hint("`mcp_servers`"),
|
||||
model=model,
|
||||
route=route,
|
||||
)
|
||||
verbose_logger.warning(
|
||||
MCP_SERVERS_DROP_WARNING, _one_log_line(named_servers), _one_log_line(model), _one_log_line(route)
|
||||
)
|
||||
return [tool for tool in tools if not _is_mcp_toolset(tool)]
|
||||
|
||||
|
||||
_TOOL_SELECTION_PARAMS: Final = frozenset({"tools", "tool_choice", "parallel_tool_calls"})
|
||||
_ParamsT: Final = TypeVar("_ParamsT", bound=dict[str, object])
|
||||
|
||||
|
||||
def tool_params_without_hosted_mcp(
|
||||
params: _ParamsT, model: str, route: str, drop_params: bool, runs_hosted_mcp: Callable[[], bool] | None = None
|
||||
) -> _ParamsT:
|
||||
tools: Final = params.get("tools")
|
||||
if not isinstance(tools, list) or not tools:
|
||||
return params
|
||||
sent_tools: Final = _TOOLS.validate_python(tools)
|
||||
if all(_hosted_mcp_server_label(tool) is None for tool in sent_tools) or (
|
||||
runs_hosted_mcp is not None and runs_hosted_mcp()
|
||||
):
|
||||
return params
|
||||
kept_tools: Final = tools_without_hosted_mcp(tools=sent_tools, model=model, route=route, drop_params=drop_params)
|
||||
if len(kept_tools) == len(sent_tools):
|
||||
return params
|
||||
kept_params: Final = copy(params)
|
||||
if kept_tools:
|
||||
kept_params["tools"] = list(kept_tools)
|
||||
return kept_params
|
||||
dropped_keys: Final = ("tools",) if params.get("functions") else _TOOL_SELECTION_PARAMS
|
||||
for key in dropped_keys:
|
||||
_ = kept_params.pop(key, None)
|
||||
return kept_params
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
|||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm.anthropic_beta_headers_manager import get_unsupported_headers
|
||||
from litellm.constants import (
|
||||
ANTHROPIC_BILLING_METADATA_PREFIX,
|
||||
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS,
|
||||
|
|
@ -56,6 +57,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
|
|||
split_leading_system_run,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_non_string_reasoning_effort
|
||||
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
|
||||
from litellm.llms.base_llm.base_utils import (
|
||||
ResponseFormatSchema,
|
||||
as_json_object,
|
||||
|
|
@ -1315,6 +1317,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
*(tool for tool in tools[first_search_tool_index + 1 :] if not _is_openai_tool_search_tool(tool)),
|
||||
]
|
||||
|
||||
def runs_hosted_mcp(self) -> bool:
|
||||
return ANTHROPIC_BETA_HEADER_VALUES.MCP_CLIENT_2025_04_04.value not in get_unsupported_headers(
|
||||
self._resolved_provider
|
||||
)
|
||||
|
||||
def _tools_param_with_openai_tool_search_mapped(
|
||||
self, non_default_params: Mapping[str, object], model: str, drop_params: bool
|
||||
) -> Mapping[str, object]:
|
||||
|
|
@ -1903,7 +1910,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params=non_default_params)
|
||||
request_params: Final = tool_params_without_hosted_mcp(
|
||||
non_default_params,
|
||||
model=model,
|
||||
route=self._resolved_provider,
|
||||
drop_params=drop_params,
|
||||
runs_hosted_mcp=self.runs_hosted_mcp,
|
||||
)
|
||||
is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params=request_params)
|
||||
|
||||
# NB: ``map_openai_params`` deliberately does NOT sanitize tool names
|
||||
# here. Names are the *original* OpenAI names at this stage, and must
|
||||
|
|
@ -1916,7 +1930,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
# via ``data = {**optional_params}`` and would 400 with
|
||||
# ``Extra inputs are not permitted``).
|
||||
|
||||
for param, value in non_default_params.items():
|
||||
for param, value in request_params.items():
|
||||
if param == "max_tokens" or param == "max_completion_tokens":
|
||||
optional_params["max_tokens"] = value if isinstance(value, int) else max(1, int(round(value)))
|
||||
elif param == "tools":
|
||||
|
|
@ -1930,8 +1944,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
optional_params["mcp_servers"] = mcp_servers
|
||||
elif param == "tool_choice" or param == "parallel_tool_calls":
|
||||
_tool_choice: AnthropicMessagesToolChoice | None = self._map_tool_choice(
|
||||
tool_choice=non_default_params.get("tool_choice"),
|
||||
parallel_tool_use=non_default_params.get("parallel_tool_calls"),
|
||||
tool_choice=request_params.get("tool_choice"),
|
||||
parallel_tool_use=request_params.get("parallel_tool_calls"),
|
||||
)
|
||||
|
||||
if _tool_choice is not None:
|
||||
|
|
@ -1957,7 +1971,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
if self.uses_native_structured_output_for_schema(
|
||||
model,
|
||||
self._extract_json_schema_from_response_format(value),
|
||||
REQUEST_PARAMS.validate_python(non_default_params),
|
||||
REQUEST_PARAMS.validate_python(request_params),
|
||||
):
|
||||
_output_format = self.map_response_format_to_anthropic_output_format(value)
|
||||
if _output_format is not None:
|
||||
|
|
@ -1993,7 +2007,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
# `thinking={type: enabled, budget_tokens}` interface a
|
||||
# pre-4.6 model actually supports instead of forwarding a
|
||||
# shape the model will reject.
|
||||
max_tokens = non_default_params.get("max_completion_tokens") or non_default_params.get("max_tokens")
|
||||
max_tokens = request_params.get("max_completion_tokens") or request_params.get("max_tokens")
|
||||
legacy_thinking = AnthropicConfig._map_reasoning_effort(
|
||||
reasoning_effort="medium",
|
||||
model=model,
|
||||
|
|
@ -2087,7 +2101,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
|
||||
## handle thinking tokens
|
||||
self.update_optional_params_with_thinking_tokens(
|
||||
non_default_params=non_default_params, optional_params=optional_params
|
||||
non_default_params=request_params, optional_params=optional_params
|
||||
)
|
||||
|
||||
return optional_params
|
||||
|
|
|
|||
|
|
@ -15,6 +15,11 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
|
||||
from litellm.litellm_core_utils.tool_fields import (
|
||||
ROUTES_WITHOUT_MCP_SERVERS,
|
||||
drops_mcp_servers,
|
||||
tools_without_mcp_servers,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import (
|
||||
AnthropicAdapter,
|
||||
)
|
||||
|
|
@ -610,6 +615,23 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
Logging as LiteLLMLoggingObject,
|
||||
)
|
||||
|
||||
provider_kwargs: Final = extra_kwargs or {}
|
||||
custom_llm_provider: Final = provider_kwargs.get("custom_llm_provider")
|
||||
target_provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None
|
||||
drop_params: Final = drops_unsupported_params(provider_kwargs.get("drop_params"))
|
||||
strips_mcp_servers: Final = target_provider in ROUTES_WITHOUT_MCP_SERVERS
|
||||
request_tools: Final = (
|
||||
tools_without_mcp_servers(
|
||||
provider_kwargs.get("mcp_servers"),
|
||||
tools or (),
|
||||
model=model,
|
||||
route=target_provider,
|
||||
drop_params=drops_mcp_servers(provider_kwargs),
|
||||
)
|
||||
if strips_mcp_servers
|
||||
else tools
|
||||
)
|
||||
|
||||
request_data: Final[dict[str, object]] = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
|
|
@ -626,10 +648,10 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
request_data["temperature"] = temperature
|
||||
if thinking:
|
||||
request_data["thinking"] = thinking
|
||||
if tool_choice:
|
||||
if tool_choice and (request_tools or not tools):
|
||||
request_data["tool_choice"] = tool_choice
|
||||
if tools:
|
||||
request_data["tools"] = tools
|
||||
if request_tools:
|
||||
request_data["tools"] = request_tools
|
||||
if top_k is not None:
|
||||
request_data["top_k"] = top_k
|
||||
if top_p is not None:
|
||||
|
|
@ -647,16 +669,15 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
if "output_config" in extra_kwargs:
|
||||
request_data["output_config"] = extra_kwargs["output_config"]
|
||||
|
||||
custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider")
|
||||
api_base: Final = extra_kwargs.get("api_base")
|
||||
(
|
||||
openai_request,
|
||||
tool_name_mapping,
|
||||
) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping(
|
||||
request_data,
|
||||
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
|
||||
custom_llm_provider=target_provider,
|
||||
api_base=api_base if isinstance(api_base, str) else None,
|
||||
drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")),
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
||||
if openai_request is None:
|
||||
|
|
@ -684,7 +705,11 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
# Maintainability: when adding a new Anthropic-only request param to
|
||||
# ``AnthropicMessagesRequestOptionalParams``, also extend
|
||||
# ``ANTHROPIC_ONLY_REQUEST_KEYS`` here so it doesn't silently leak.
|
||||
excluded_keys: Final = ANTHROPIC_ONLY_REQUEST_KEYS | CONVERTED_STREAM_KEYS | {"anthropic_messages"}
|
||||
excluded_keys: Final = (
|
||||
ANTHROPIC_ONLY_REQUEST_KEYS
|
||||
| CONVERTED_STREAM_KEYS
|
||||
| ({"anthropic_messages", "mcp_servers"} if strips_mcp_servers else {"anthropic_messages"})
|
||||
)
|
||||
# NOTE: extra_kwargs was already coerced from None to {} at the top of
|
||||
# this method (line ~220). It is guaranteed to be a dict here.
|
||||
for key, value in extra_kwargs.items():
|
||||
|
|
|
|||
|
|
@ -493,10 +493,23 @@ class _CombinedChunkSplitter:
|
|||
for position, block in enumerate(blocks)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _without_sibling_finish(chunk: "ModelResponseStream") -> "ModelResponseStream":
|
||||
choices: Final = _optional_attr_sequence(chunk, "choices")
|
||||
if (
|
||||
len(choices) != 1
|
||||
or _optional_attr(choices[0], "index") in (None, 0)
|
||||
or _optional_attr(choices[0], "finish_reason") is None
|
||||
):
|
||||
return chunk
|
||||
unfinished: Final = copy.deepcopy(chunk)
|
||||
unfinished.choices[0].finish_reason = None
|
||||
return unfinished
|
||||
|
||||
def _expand(self, chunk: "ModelResponseStream") -> "tuple[ModelResponseStream, ...]":
|
||||
split_chunks: Final = tuple(
|
||||
split_chunk
|
||||
for combined_chunk in self._split(chunk)
|
||||
for combined_chunk in self._split(self._without_sibling_finish(chunk))
|
||||
for split_chunk in self._split_by_payload_kind(combined_chunk) # comprehension-ok: flattens nested splits
|
||||
)
|
||||
block_chunks: Final = tuple(itertools.chain.from_iterable(map(self._split_thinking_blocks, split_chunks)))
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
prompt_cache_breakpoint_with_readable_mode,
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.litellm_core_utils.tool_fields import drops_mcp_servers, tools_without_mcp_servers
|
||||
from litellm.responses.utils import auto_reasoning_summary
|
||||
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
|
||||
from litellm.types.llms.anthropic import (
|
||||
|
|
@ -272,6 +273,14 @@ def _build_responses_kwargs(
|
|||
"""
|
||||
forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs)
|
||||
reject_malformed_tool_result_text(messages, model, forwarded_kwargs)
|
||||
target_provider: Final = forwarded_kwargs.get("custom_llm_provider")
|
||||
request_tools: Final = tools_without_mcp_servers(
|
||||
forwarded_kwargs.get("mcp_servers"),
|
||||
tools or (),
|
||||
model=model,
|
||||
route=target_provider if isinstance(target_provider, str) else "this route",
|
||||
drop_params=drops_mcp_servers(forwarded_kwargs),
|
||||
)
|
||||
block_rewrite: Final = _prompt_cache_block_rewrite(model, forwarded_kwargs)
|
||||
request_messages: Final[AnthropicRequestMessages] = (
|
||||
[
|
||||
|
|
@ -304,10 +313,10 @@ def _build_responses_kwargs(
|
|||
request_data["temperature"] = temperature
|
||||
if thinking:
|
||||
request_data["thinking"] = thinking
|
||||
if tool_choice:
|
||||
if tool_choice and (request_tools or not tools):
|
||||
request_data["tool_choice"] = tool_choice
|
||||
if tools:
|
||||
request_data["tools"] = tools
|
||||
if request_tools:
|
||||
request_data["tools"] = list(request_tools)
|
||||
if top_p is not None:
|
||||
request_data["top_p"] = top_p
|
||||
if output_format:
|
||||
|
|
@ -347,7 +356,7 @@ def _build_responses_kwargs(
|
|||
responses_kwargs["stream"] = True
|
||||
|
||||
# Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.)
|
||||
excluded: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages"}
|
||||
excluded: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages", "mcp_servers"}
|
||||
for key, value in forwarded_kwargs.items():
|
||||
if key == "litellm_logging_obj" and value is not None:
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
|
|
|
|||
|
|
@ -164,7 +164,7 @@ class BaseGoogleGenAIGenerateContentConfig(ABC):
|
|||
tools: ToolConfigDict | None,
|
||||
generate_content_config_dict: dict,
|
||||
system_instruction: object | None = None,
|
||||
) -> dict:
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
Transform the request parameters for the generate content API.
|
||||
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
|
|||
parts_of,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
|
||||
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
|
||||
from litellm.llms.anthropic.chat.transformation import (
|
||||
DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING,
|
||||
DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING,
|
||||
|
|
@ -1068,20 +1069,23 @@ class AmazonConverseConfig(BaseConfig):
|
|||
drop_params: bool,
|
||||
model_info: Mapping[str, object] | None = None,
|
||||
) -> dict:
|
||||
is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params)
|
||||
request_params: Final = tool_params_without_hosted_mcp(
|
||||
non_default_params, model=model, route="bedrock", drop_params=drop_params
|
||||
)
|
||||
is_thinking_enabled: Final = self.is_thinking_enabled(request_params)
|
||||
base_model: Final = BedrockModelInfo.get_base_model(model)
|
||||
drop_thinking_param: Final = self._is_deepseek_model(model=model, base_model=base_model)
|
||||
drop_reasoning_effort_param: Final = self._model_rejects_reasoning_effort_param(
|
||||
model=model, base_model=base_model
|
||||
)
|
||||
|
||||
for param, value in non_default_params.items():
|
||||
for param, value in request_params.items():
|
||||
if param == "response_format" and isinstance(value, dict):
|
||||
optional_params = self._translate_response_format_param(
|
||||
value=value,
|
||||
model=model,
|
||||
optional_params=optional_params,
|
||||
non_default_params=non_default_params,
|
||||
non_default_params=request_params,
|
||||
is_thinking_enabled=is_thinking_enabled,
|
||||
model_info=model_info,
|
||||
)
|
||||
|
|
@ -1135,7 +1139,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
),
|
||||
),
|
||||
model=model,
|
||||
non_default_params=non_default_params,
|
||||
non_default_params=request_params,
|
||||
optional_params=optional_params,
|
||||
)
|
||||
if param == "tool_choice":
|
||||
|
|
@ -1160,7 +1164,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
and value.get("type") == "adaptive"
|
||||
and not AnthropicConfig.is_adaptive_thinking_model(model, "bedrock")
|
||||
):
|
||||
max_tokens = non_default_params.get("max_completion_tokens") or non_default_params.get("max_tokens")
|
||||
max_tokens = request_params.get("max_completion_tokens") or request_params.get("max_tokens")
|
||||
legacy_thinking = AnthropicConfig.map_reasoning_effort(
|
||||
reasoning_effort="medium",
|
||||
model=model,
|
||||
|
|
@ -1230,7 +1234,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
# Nova 2 handles token budgeting differently through reasoningConfig
|
||||
if "gpt-oss" not in model and not self._is_nova_2_model(model):
|
||||
self.update_optional_params_with_thinking_tokens(
|
||||
non_default_params=non_default_params, optional_params=optional_params
|
||||
non_default_params=request_params, optional_params=optional_params
|
||||
)
|
||||
|
||||
final_is_thinking_enabled: Final = self.is_thinking_enabled(optional_params)
|
||||
|
|
|
|||
|
|
@ -30,6 +30,9 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig):
|
|||
def should_strip_billing_metadata(self) -> bool:
|
||||
return True
|
||||
|
||||
def runs_hosted_mcp(self) -> bool:
|
||||
return True
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, Final
|
|||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.types.llms.cohere import CohereV2ChatResponse
|
||||
|
|
@ -248,6 +249,9 @@ class CohereV2ChatConfig(OpenAIGPTConfig):
|
|||
prompt_tokens: Final = token_usage.get("input_tokens", 0)
|
||||
completion_tokens: Final = token_usage.get("output_tokens", 0)
|
||||
|
||||
model_response.choices[0].finish_reason = map_finish_reason( # rebind-ok: ModelResponse is filled in place
|
||||
cohere_v2_chat_response.get("finish_reason", "COMPLETE")
|
||||
)
|
||||
model_response.created = int(time.time())
|
||||
model_response.model = model
|
||||
usage: Final = Usage(
|
||||
|
|
|
|||
|
|
@ -455,6 +455,20 @@ def _google_genai_streaming_hidden_params(
|
|||
}
|
||||
|
||||
|
||||
def _log_generate_content_request(
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
contents: object,
|
||||
api_base: str,
|
||||
headers: Mapping[str, object],
|
||||
body: Mapping[str, object],
|
||||
) -> None:
|
||||
logging_obj.pre_call(
|
||||
input=contents,
|
||||
api_key="",
|
||||
additional_args={"complete_input_dict": body, "api_base": api_base, "headers": headers},
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _responses_api_optional_request_param_names() -> frozenset[str]:
|
||||
return frozenset(get_type_hints(ResponsesAPIOptionalRequestParams).keys())
|
||||
|
|
@ -12443,6 +12457,10 @@ class BaseLLMHTTPHandler:
|
|||
from litellm.google_genai.streaming_iterator import (
|
||||
GoogleGenAIGenerateContentStreamingIterator,
|
||||
)
|
||||
from litellm.llms.vertex_ai.gemini.rejected_thought_signature_retry import (
|
||||
post_with_accepted_thought_signatures_sync,
|
||||
with_placeholder_native_thought_signatures,
|
||||
)
|
||||
|
||||
if _is_async:
|
||||
return self.async_generate_content_handler(
|
||||
|
|
@ -12494,27 +12512,19 @@ class BaseLLMHTTPHandler:
|
|||
if extra_body:
|
||||
data.update(extra_body)
|
||||
|
||||
## LOGGING
|
||||
logging_obj.pre_call(
|
||||
input=contents,
|
||||
api_key="",
|
||||
additional_args={
|
||||
"complete_input_dict": data,
|
||||
"api_base": api_base,
|
||||
"headers": headers,
|
||||
},
|
||||
)
|
||||
log_request: Final = partial(_log_generate_content_request, logging_obj, contents, api_base, headers)
|
||||
log_request(data)
|
||||
|
||||
def post(body: Mapping[str, object]) -> httpx.Response:
|
||||
return sync_httpx_client.post(
|
||||
url=api_base, headers=headers, json=dict(body), timeout=timeout, stream=stream
|
||||
)
|
||||
|
||||
try:
|
||||
sent_body, response = post_with_accepted_thought_signatures_sync(
|
||||
data, with_placeholder_native_thought_signatures, None, post, log_request
|
||||
)
|
||||
if stream:
|
||||
response = sync_httpx_client.post(
|
||||
url=api_base,
|
||||
headers=headers,
|
||||
json=data,
|
||||
timeout=timeout,
|
||||
stream=True,
|
||||
)
|
||||
# Return streaming iterator
|
||||
return GoogleGenAIGenerateContentStreamingIterator(
|
||||
response=response,
|
||||
model=model,
|
||||
|
|
@ -12522,7 +12532,7 @@ class BaseLLMHTTPHandler:
|
|||
generate_content_provider_config=generate_content_provider_config,
|
||||
litellm_metadata=litellm_metadata or {},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
request_body=data,
|
||||
request_body=dict(sent_body),
|
||||
hidden_params=_google_genai_streaming_hidden_params(
|
||||
api_base=api_base,
|
||||
litellm_params=litellm_params,
|
||||
|
|
@ -12530,13 +12540,6 @@ class BaseLLMHTTPHandler:
|
|||
response_headers=response.headers,
|
||||
),
|
||||
)
|
||||
else:
|
||||
response = sync_httpx_client.post(
|
||||
url=api_base,
|
||||
headers=headers,
|
||||
json=data,
|
||||
timeout=timeout,
|
||||
)
|
||||
except Exception as e:
|
||||
raise self._handle_error(
|
||||
e=e,
|
||||
|
|
@ -12574,6 +12577,10 @@ class BaseLLMHTTPHandler:
|
|||
from litellm.google_genai.streaming_iterator import (
|
||||
AsyncGoogleGenAIGenerateContentStreamingIterator,
|
||||
)
|
||||
from litellm.llms.vertex_ai.gemini.rejected_thought_signature_retry import (
|
||||
post_with_accepted_thought_signatures_async,
|
||||
with_placeholder_native_thought_signatures,
|
||||
)
|
||||
|
||||
if client is None or not isinstance(client, AsyncHTTPHandler):
|
||||
async_httpx_client = get_async_httpx_client(
|
||||
|
|
@ -12609,27 +12616,19 @@ class BaseLLMHTTPHandler:
|
|||
if extra_body:
|
||||
data.update(extra_body)
|
||||
|
||||
## LOGGING
|
||||
logging_obj.pre_call(
|
||||
input=contents,
|
||||
api_key="",
|
||||
additional_args={
|
||||
"complete_input_dict": data,
|
||||
"api_base": api_base,
|
||||
"headers": headers,
|
||||
},
|
||||
)
|
||||
log_request: Final = partial(_log_generate_content_request, logging_obj, contents, api_base, headers)
|
||||
log_request(data)
|
||||
|
||||
async def post(body: Mapping[str, object]) -> httpx.Response:
|
||||
return await async_httpx_client.post(
|
||||
url=api_base, headers=headers, json=dict(body), timeout=timeout, stream=stream
|
||||
)
|
||||
|
||||
try:
|
||||
sent_body, response = await post_with_accepted_thought_signatures_async(
|
||||
data, with_placeholder_native_thought_signatures, None, post, log_request
|
||||
)
|
||||
if stream:
|
||||
response = await async_httpx_client.post(
|
||||
url=api_base,
|
||||
headers=headers,
|
||||
json=data,
|
||||
timeout=timeout,
|
||||
stream=True,
|
||||
)
|
||||
# Return async streaming iterator
|
||||
return AsyncGoogleGenAIGenerateContentStreamingIterator(
|
||||
response=response,
|
||||
model=model,
|
||||
|
|
@ -12637,7 +12636,7 @@ class BaseLLMHTTPHandler:
|
|||
generate_content_provider_config=generate_content_provider_config,
|
||||
litellm_metadata=litellm_metadata or {},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
request_body=data,
|
||||
request_body=dict(sent_body),
|
||||
hidden_params=_google_genai_streaming_hidden_params(
|
||||
api_base=api_base,
|
||||
litellm_params=litellm_params,
|
||||
|
|
@ -12645,13 +12644,6 @@ class BaseLLMHTTPHandler:
|
|||
response_headers=response.headers,
|
||||
),
|
||||
)
|
||||
else:
|
||||
response = await async_httpx_client.post(
|
||||
url=api_base,
|
||||
headers=headers,
|
||||
json=data,
|
||||
timeout=timeout,
|
||||
)
|
||||
except Exception as e:
|
||||
raise self._handle_error(
|
||||
e=e,
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
strip_name_from_message,
|
||||
strip_prefill_prefix_field,
|
||||
)
|
||||
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
from litellm.llms.base_llm.base_utils import (
|
||||
|
|
@ -467,7 +468,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
|
|||
use_ai_gateway: Final = self.uses_ai_gateway(model)
|
||||
return join_api_base(self._get_api_base(api_base, use_ai_gateway=use_ai_gateway), "chat/completions")
|
||||
|
||||
def get_supported_openai_params(self, model: str | None = None) -> list:
|
||||
def get_supported_openai_params(self, model: str | None = None) -> list[str]:
|
||||
return [
|
||||
"stream",
|
||||
"stop",
|
||||
|
|
@ -573,8 +574,13 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
|
|||
drop_params: bool,
|
||||
replace_max_completion_tokens_with_max_tokens: bool = True,
|
||||
) -> dict:
|
||||
is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params)
|
||||
mapped_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
|
||||
request_params: Final = (
|
||||
tool_params_without_hosted_mcp(non_default_params, model=model, route="databricks", drop_params=drop_params)
|
||||
if "claude" in model
|
||||
else non_default_params
|
||||
)
|
||||
is_thinking_enabled: Final = self.is_thinking_enabled(request_params)
|
||||
mapped_params: Final = super().map_openai_params(request_params, optional_params, model, drop_params)
|
||||
if "claude" in model:
|
||||
AnthropicConfig.translate_legacy_thinking_for_adaptive_model(
|
||||
model=model, optional_params=mapped_params, custom_llm_provider="databricks", drop_params=drop_params
|
||||
|
|
@ -583,16 +589,16 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
|
|||
mapped_params["tools"] = self._map_openai_to_dbrx_tool(
|
||||
model=model, tools=mapped_params["tools"], drop_params=drop_params
|
||||
)
|
||||
if "max_completion_tokens" in non_default_params and replace_max_completion_tokens_with_max_tokens:
|
||||
mapped_params["max_tokens"] = non_default_params[
|
||||
if "max_completion_tokens" in request_params and replace_max_completion_tokens_with_max_tokens:
|
||||
mapped_params["max_tokens"] = request_params[
|
||||
"max_completion_tokens"
|
||||
] # most openai-compatible providers support 'max_tokens' not 'max_completion_tokens'
|
||||
mapped_params.pop("max_completion_tokens", None)
|
||||
|
||||
if "response_format" in non_default_params and "claude" in model:
|
||||
if "response_format" in request_params and "claude" in model:
|
||||
_tool: Final = self.map_response_format_to_databricks_tool(
|
||||
model,
|
||||
non_default_params["response_format"],
|
||||
request_params["response_format"],
|
||||
mapped_params,
|
||||
is_thinking_enabled,
|
||||
)
|
||||
|
|
@ -610,8 +616,8 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
|
|||
"response_format", None
|
||||
) # unsupported for claude models - if json_schema -> convert to tool call
|
||||
|
||||
if "reasoning_effort" in non_default_params and self._uses_anthropic_thinking_param(model):
|
||||
reasoning_effort_value: Final = non_default_params.get("reasoning_effort")
|
||||
if "reasoning_effort" in request_params and self._uses_anthropic_thinking_param(model):
|
||||
reasoning_effort_value: Final = request_params.get("reasoning_effort")
|
||||
mapped_thinking: Final = AnthropicConfig._map_reasoning_effort(
|
||||
reasoning_effort=reasoning_effort_value,
|
||||
model=model,
|
||||
|
|
@ -637,7 +643,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
|
|||
optional_params.pop("reasoning_effort", None)
|
||||
## handle thinking tokens
|
||||
self.update_optional_params_with_thinking_tokens(
|
||||
non_default_params=non_default_params, optional_params=mapped_params
|
||||
non_default_params=request_params, optional_params=mapped_params
|
||||
)
|
||||
gated_thinking: Final = strip_beta_gated_thinking(
|
||||
mapped_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # mapped_params is the base config's bare dict
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
extract_search_results_text,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
|
||||
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
|
@ -33,17 +34,15 @@ def _warn_on_dropped_budget(thinking: object, model: str) -> None:
|
|||
|
||||
|
||||
class DeepSeekChatConfig(OpenAIGPTConfig):
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
"""
|
||||
DeepSeek reasoner models support thinking parameter.
|
||||
"""
|
||||
params: Final = super().get_supported_openai_params(model)
|
||||
params.extend(["thinking", "reasoning_effort"])
|
||||
return params
|
||||
return [*super().get_supported_openai_params(model), "thinking", "reasoning_effort"]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
non_default_params: dict[str, object],
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
|
|
@ -59,7 +58,12 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
|
|||
Reference: https://api-docs.deepseek.com/guides/thinking_mode
|
||||
"""
|
||||
# Let parent handle standard params first
|
||||
optional_params = super().map_openai_params(non_default_params, optional_params, model, drop_params)
|
||||
optional_params = super().map_openai_params(
|
||||
tool_params_without_hosted_mcp(non_default_params, model=model, route="deepseek", drop_params=drop_params),
|
||||
optional_params,
|
||||
model,
|
||||
drop_params,
|
||||
)
|
||||
|
||||
# Pop thinking/reasoning_effort from optional_params first (parent may have added them)
|
||||
# Then re-add only if valid for DeepSeek
|
||||
|
|
@ -265,6 +269,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
|
|||
tools: Final = optional_params.get("tools")
|
||||
if not isinstance(tools, list) or not tools:
|
||||
return optional_params
|
||||
sent_tools: Final[tuple[object, ...]] = tuple(tools)
|
||||
|
||||
def _is_function_tool(tool: object) -> bool:
|
||||
return isinstance(tool, dict) and tool.get("type") == "function"
|
||||
|
|
@ -289,14 +294,14 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
|
|||
name: Final = function.get("name")
|
||||
return isinstance(name, str) and name in function_tool_names
|
||||
|
||||
function_tools: Final = [tool for tool in tools if _is_function_tool(tool)]
|
||||
if len(function_tools) == len(tools):
|
||||
function_tools: Final = [tool for tool in sent_tools if _is_function_tool(tool)]
|
||||
if len(function_tools) == len(sent_tools):
|
||||
return optional_params
|
||||
|
||||
dropped_types: Final = sorted(
|
||||
{
|
||||
str(tool.get("type")) if isinstance(tool, dict) else type(tool).__name__
|
||||
for tool in tools
|
||||
for tool in sent_tools
|
||||
if not _is_function_tool(tool)
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final, cast
|
||||
from typing import ClassVar, Final, cast
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
|
|
@ -49,6 +49,8 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
|
|||
Note: Please make sure to modify the default parameters as required for your use case.
|
||||
"""
|
||||
|
||||
_route: ClassVar[str] = "gemini"
|
||||
|
||||
temperature: float | None = None
|
||||
max_output_tokens: int | None = None
|
||||
top_p: float | None = None
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import httpx
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.llms.gigachat.utils import convert_usage, get_api_base
|
||||
|
|
@ -169,7 +170,10 @@ class GigaChatConfig(BaseConfig):
|
|||
drop_params: bool,
|
||||
) -> dict: # mutable-ok: base class contract returns dict
|
||||
"""Map OpenAI parameters to GigaChat parameters."""
|
||||
for param, value in non_default_params.items():
|
||||
request_params: Final = tool_params_without_hosted_mcp(
|
||||
dict(non_default_params), model=model, route="gigachat", drop_params=drop_params
|
||||
)
|
||||
for param, value in request_params.items():
|
||||
if param == "stream":
|
||||
optional_params["stream"] = value
|
||||
elif param == "temperature":
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from collections.abc import AsyncIterator, Iterator, Mapping
|
|||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
|
||||
from httpx._models import Headers, Response
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._uuid import uuid
|
||||
|
|
@ -15,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
extract_reasoning_content,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
|
||||
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
|
|
@ -40,6 +41,8 @@ if TYPE_CHECKING:
|
|||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
_REQUEST_PARAMS: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
class OllamaChatConfig(BaseConfig):
|
||||
"""
|
||||
|
|
@ -152,7 +155,13 @@ class OllamaChatConfig(BaseConfig):
|
|||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
for param, value in non_default_params.items():
|
||||
request_params: Final = tool_params_without_hosted_mcp(
|
||||
_REQUEST_PARAMS.validate_python(non_default_params),
|
||||
model=model,
|
||||
route="ollama_chat",
|
||||
drop_params=drop_params,
|
||||
)
|
||||
for param, value in request_params.items():
|
||||
if param == "max_tokens" or param == "max_completion_tokens":
|
||||
optional_params["num_predict"] = value
|
||||
if param == "stream":
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Final
|
||||
from typing import TYPE_CHECKING, ClassVar, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
|
@ -159,6 +159,7 @@ class OpenCodeGoMessagesChatConfig(_GoReportedCostMixin, AnthropicConfig):
|
|||
|
||||
class OpenCodeGoGeminiChatConfig(_GoReportedCostMixin, GoogleAIStudioGeminiConfig):
|
||||
go_usage_key: str = "usageMetadata"
|
||||
_route: ClassVar[str] = "opencode_go"
|
||||
|
||||
@property
|
||||
def supports_stream_param_in_request_body(self) -> bool:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from collections.abc import Awaitable, Callable
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from copy import deepcopy
|
||||
from functools import partial
|
||||
from typing import Final, TypeVar
|
||||
|
|
@ -17,16 +17,27 @@ Built: Final = TypeVar("Built")
|
|||
Sent: Final = TypeVar("Sent")
|
||||
|
||||
|
||||
def cache_rebuild(
|
||||
def _build_from_copies(
|
||||
build: Callable[..., Built], messages: Sequence[AllMessageValues], optional_params: Mapping[str, object]
|
||||
) -> Built:
|
||||
return build(messages=deepcopy(list(messages)), optional_params=independent_snapshot(dict(optional_params)))
|
||||
|
||||
|
||||
def cache_rebuilds(
|
||||
build: Callable[..., Built], messages: list[AllMessageValues], optional_params: dict
|
||||
) -> Callable[[], Built] | None:
|
||||
"""Copies the inputs before the first transform, which pops keys from ``optional_params`` and ``messages``."""
|
||||
) -> tuple[Callable[[], Built], Callable[[], Built]] | tuple[None, None]:
|
||||
"""Returns one build with a fresh cache and one with the history inline, both over a snapshot of the inputs
|
||||
taken before the first transform, which pops keys from ``optional_params`` and ``messages``. Each call
|
||||
copies the snapshot again, since every transform consumes the inputs it is given."""
|
||||
if not any(map(is_cached_message, messages)):
|
||||
return None
|
||||
return partial(build, messages=deepcopy(messages), optional_params=independent_snapshot(optional_params))
|
||||
return None, None
|
||||
from_snapshot: Final = partial(
|
||||
_build_from_copies, messages=deepcopy(messages), optional_params=independent_snapshot(optional_params)
|
||||
)
|
||||
return partial(from_snapshot, build), partial(from_snapshot, partial(build, context_caching=False))
|
||||
|
||||
|
||||
def _status_and_message(error: httpx.HTTPStatusError | BaseLLMException) -> tuple[int, str]:
|
||||
def status_and_message(error: httpx.HTTPStatusError | BaseLLMException) -> tuple[int, str]:
|
||||
match error:
|
||||
case httpx.HTTPStatusError():
|
||||
return error.response.status_code, error.response.text
|
||||
|
|
@ -35,7 +46,7 @@ def _status_and_message(error: httpx.HTTPStatusError | BaseLLMException) -> tupl
|
|||
|
||||
|
||||
def _cache_rejected(body: RequestBody, error: httpx.HTTPStatusError | BaseLLMException) -> bool:
|
||||
return body.get("cachedContent") is not None and names_rejected_cache(*_status_and_message(error))
|
||||
return body.get("cachedContent") is not None and names_rejected_cache(*status_and_message(error))
|
||||
|
||||
|
||||
async def post_with_fresh_cache_async(
|
||||
|
|
|
|||
|
|
@ -425,6 +425,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_auth_header: str | None,
|
||||
extra_headers: dict | None = None,
|
||||
cached_content: str | None = None,
|
||||
context_caching: bool = True,
|
||||
) -> tuple[list[AllMessageValues], dict, str | None]:
|
||||
"""
|
||||
Receives
|
||||
|
|
@ -436,7 +437,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
|
||||
Follows - https://ai.google.dev/api/caching#request-body
|
||||
"""
|
||||
if cached_content is not None:
|
||||
if cached_content is not None or not context_caching:
|
||||
return messages, optional_params, cached_content
|
||||
|
||||
cached_messages, non_cached_messages = separate_cached_messages(messages=messages)
|
||||
|
|
@ -573,6 +574,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_auth_header: str | None,
|
||||
extra_headers: dict | None = None,
|
||||
cached_content: str | None = None,
|
||||
context_caching: bool = True,
|
||||
) -> tuple[list[AllMessageValues], dict, str | None]:
|
||||
"""
|
||||
Receives
|
||||
|
|
@ -584,7 +586,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
|
||||
Follows - https://ai.google.dev/api/caching#request-body
|
||||
"""
|
||||
if cached_content is not None:
|
||||
if cached_content is not None or not context_caching:
|
||||
return messages, optional_params, cached_content
|
||||
|
||||
cached_messages, non_cached_messages = separate_cached_messages(messages=messages)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,235 @@
|
|||
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from typing import Final, TypeVar
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm import verbose_logger
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import get_dummy_thought_signature
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.llms.vertex_ai import ContentType, PartType, RequestBody
|
||||
|
||||
from ..context_caching.rejected_cache_retry import status_and_message
|
||||
|
||||
Body: Final = TypeVar("Body", bound=Mapping[str, object])
|
||||
Sent: Final = TypeVar("Sent")
|
||||
|
||||
_REJECTED_SIGNATURE_MARKER: Final = "thought signature"
|
||||
_SIGNATURE_KEYS: Final = frozenset({"thoughtSignature", "thought_signature"})
|
||||
_ENTRIES: Final = TypeAdapter(list[dict[str, object]])
|
||||
|
||||
|
||||
def names_rejected_thought_signature(status_code: int, message: str) -> bool:
|
||||
"""A signature only validates on the backend that minted it. AI Studio answers a replayed Vertex signature
|
||||
with ``400 Corrupted thought signature.`` and Vertex answers an AI Studio one with ``400 Invalid thought
|
||||
signature.``"""
|
||||
return status_code == 400 and _REJECTED_SIGNATURE_MARKER in message.lower()
|
||||
|
||||
|
||||
def _is_rejected_signature(error: httpx.HTTPStatusError | BaseLLMException) -> bool:
|
||||
return names_rejected_thought_signature(*status_and_message(error))
|
||||
|
||||
|
||||
def _entries(value: object) -> Sequence[Mapping[str, object]]:
|
||||
try:
|
||||
return _ENTRIES.validate_python(value)
|
||||
except ValidationError:
|
||||
return []
|
||||
|
||||
|
||||
def _foreign_signatures(body: Mapping[str, object], placeholder: str) -> Iterator[object]:
|
||||
for content in _entries(body.get("contents")):
|
||||
for part in _entries(content.get("parts")):
|
||||
yield from (
|
||||
value for key, value in part.items() if key in _SIGNATURE_KEYS and value and value != placeholder
|
||||
)
|
||||
|
||||
|
||||
def _foreign_signature_count(body: Mapping[str, object]) -> int:
|
||||
return sum(1 for _ in _foreign_signatures(body, get_dummy_thought_signature()))
|
||||
|
||||
|
||||
def _part_with_placeholder(part: PartType, placeholder: str) -> PartType:
|
||||
if not part.get("thoughtSignature"):
|
||||
return part
|
||||
return {**part, "thoughtSignature": placeholder}
|
||||
|
||||
|
||||
def _content_with_placeholders(content: ContentType, placeholder: str) -> ContentType:
|
||||
return {**content, "parts": [_part_with_placeholder(part, placeholder) for part in content["parts"]]}
|
||||
|
||||
|
||||
def with_placeholder_thought_signatures(body: RequestBody) -> RequestBody | None:
|
||||
"""Swaps every thought signature in the history for Google's documented skip-validator placeholder, or
|
||||
returns ``None`` when the body carries nothing the backend could have rejected."""
|
||||
placeholder: Final = get_dummy_thought_signature()
|
||||
if next(_foreign_signatures(body, placeholder), None) is None:
|
||||
return None
|
||||
return {**body, "contents": [_content_with_placeholders(content, placeholder) for content in body["contents"]]}
|
||||
|
||||
|
||||
def _native_part_with_placeholder(part: Mapping[str, object], placeholder: str) -> Mapping[str, object]:
|
||||
return {key: placeholder if key in _SIGNATURE_KEYS and value else value for key, value in part.items()}
|
||||
|
||||
|
||||
def _native_content_with_placeholders(content: Mapping[str, object], placeholder: str) -> Mapping[str, object]:
|
||||
parts: Final = _entries(content.get("parts"))
|
||||
if not parts:
|
||||
return content
|
||||
return {**content, "parts": [_native_part_with_placeholder(part, placeholder) for part in parts]}
|
||||
|
||||
|
||||
def with_placeholder_native_thought_signatures(body: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
"""The same swap on a ``generateContent`` body as the client sent it, whose parts may spell the key
|
||||
``thoughtSignature`` or ``thought_signature``."""
|
||||
placeholder: Final = get_dummy_thought_signature()
|
||||
if next(_foreign_signatures(body, placeholder), None) is None:
|
||||
return None
|
||||
return {
|
||||
**body,
|
||||
"contents": [
|
||||
_native_content_with_placeholders(content, placeholder) for content in _entries(body.get("contents"))
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
async def _inline_history_async(body: Body, uncached: Callable[[], Awaitable[Body]] | None) -> Body | None:
|
||||
if uncached is None or body.get("cachedContent") is None:
|
||||
return None
|
||||
inline: Final = await uncached()
|
||||
return inline if _foreign_signature_count(inline) > _foreign_signature_count(body) else None
|
||||
|
||||
|
||||
def _inline_history_sync(body: Body, uncached: Callable[[], Body] | None) -> Body | None:
|
||||
if uncached is None or body.get("cachedContent") is None:
|
||||
return None
|
||||
inline: Final = uncached()
|
||||
return inline if _foreign_signature_count(inline) > _foreign_signature_count(body) else None
|
||||
|
||||
|
||||
def _resend_body(signed: Body, placeholders: Callable[[Body], Body | None]) -> Body | None:
|
||||
resend_body: Final = placeholders(signed)
|
||||
if resend_body is not None:
|
||||
verbose_logger.warning(
|
||||
"Gemini rejected a replayed thought signature, resending with the skip_thought_signature_validator"
|
||||
" placeholder in place of the replayed signatures"
|
||||
)
|
||||
return resend_body
|
||||
|
||||
|
||||
async def build_with_accepted_thought_signatures_async(
|
||||
build: Callable[[], Awaitable[Body]],
|
||||
placeholders: Callable[[Body], Body | None],
|
||||
uncached: Callable[[], Awaitable[Body]] | None,
|
||||
) -> Body:
|
||||
"""Vertex validates the signatures in a cache's contents when it creates the cache, so a rejected one there
|
||||
sends the history inline with the placeholder instead."""
|
||||
try:
|
||||
return await build()
|
||||
except BaseLLMException as error:
|
||||
if uncached is None or not _is_rejected_signature(error):
|
||||
raise
|
||||
inline_body: Final = _resend_body(await uncached(), placeholders)
|
||||
if inline_body is None:
|
||||
raise
|
||||
return inline_body
|
||||
|
||||
|
||||
def build_with_accepted_thought_signatures_sync(
|
||||
build: Callable[[], Body],
|
||||
placeholders: Callable[[Body], Body | None],
|
||||
uncached: Callable[[], Body] | None,
|
||||
) -> Body:
|
||||
try:
|
||||
return build()
|
||||
except BaseLLMException as error:
|
||||
if uncached is None or not _is_rejected_signature(error):
|
||||
raise
|
||||
inline_body: Final = _resend_body(uncached(), placeholders)
|
||||
if inline_body is None:
|
||||
raise
|
||||
return inline_body
|
||||
|
||||
|
||||
async def _post_inline_async(
|
||||
body: Body,
|
||||
placeholders: Callable[[Body], Body | None],
|
||||
uncached: Callable[[], Awaitable[Body]] | None,
|
||||
post: Callable[[Body], Awaitable[Sent]],
|
||||
log_request: Callable[[Body], None],
|
||||
rejection: httpx.HTTPStatusError | BaseLLMException,
|
||||
) -> tuple[Body, Sent]:
|
||||
inline: Final = await _inline_history_async(body, uncached)
|
||||
resend_body: Final = None if inline is None else _resend_body(inline, placeholders)
|
||||
if resend_body is None:
|
||||
raise rejection
|
||||
log_request(resend_body)
|
||||
return resend_body, await post(resend_body)
|
||||
|
||||
|
||||
def _post_inline_sync(
|
||||
body: Body,
|
||||
placeholders: Callable[[Body], Body | None],
|
||||
uncached: Callable[[], Body] | None,
|
||||
post: Callable[[Body], Sent],
|
||||
log_request: Callable[[Body], None],
|
||||
rejection: httpx.HTTPStatusError | BaseLLMException,
|
||||
) -> tuple[Body, Sent]:
|
||||
inline: Final = _inline_history_sync(body, uncached)
|
||||
resend_body: Final = None if inline is None else _resend_body(inline, placeholders)
|
||||
if resend_body is None:
|
||||
raise rejection
|
||||
log_request(resend_body)
|
||||
return resend_body, post(resend_body)
|
||||
|
||||
|
||||
async def post_with_accepted_thought_signatures_async(
|
||||
body: Body,
|
||||
placeholders: Callable[[Body], Body | None],
|
||||
uncached: Callable[[], Awaitable[Body]] | None,
|
||||
post: Callable[[Body], Awaitable[Sent]],
|
||||
log_request: Callable[[Body], None],
|
||||
) -> tuple[Body, Sent]:
|
||||
"""Innermost retry around the raw post, so a cache rebuild (which re-transforms the messages and brings
|
||||
the rejected signatures back) is itself wrapped by it. The resend keeps the cache and swaps the signatures
|
||||
the body carries, and sends the history inline only when that is rejected too and the cache holds one.
|
||||
Returns the body the backend accepted."""
|
||||
try:
|
||||
return body, await post(body)
|
||||
except (httpx.HTTPStatusError, BaseLLMException) as error:
|
||||
if not _is_rejected_signature(error):
|
||||
raise
|
||||
resend_body: Final = _resend_body(body, placeholders)
|
||||
if resend_body is None:
|
||||
return await _post_inline_async(body, placeholders, uncached, post, log_request, error)
|
||||
log_request(resend_body)
|
||||
try:
|
||||
return resend_body, await post(resend_body)
|
||||
except (httpx.HTTPStatusError, BaseLLMException) as error:
|
||||
if not _is_rejected_signature(error):
|
||||
raise
|
||||
return await _post_inline_async(body, placeholders, uncached, post, log_request, error)
|
||||
|
||||
|
||||
def post_with_accepted_thought_signatures_sync(
|
||||
body: Body,
|
||||
placeholders: Callable[[Body], Body | None],
|
||||
uncached: Callable[[], Body] | None,
|
||||
post: Callable[[Body], Sent],
|
||||
log_request: Callable[[Body], None],
|
||||
) -> tuple[Body, Sent]:
|
||||
try:
|
||||
return body, post(body)
|
||||
except (httpx.HTTPStatusError, BaseLLMException) as error:
|
||||
if not _is_rejected_signature(error):
|
||||
raise
|
||||
resend_body: Final = _resend_body(body, placeholders)
|
||||
if resend_body is None:
|
||||
return _post_inline_sync(body, placeholders, uncached, post, log_request, error)
|
||||
log_request(resend_body)
|
||||
try:
|
||||
return resend_body, post(resend_body)
|
||||
except (httpx.HTTPStatusError, BaseLLMException) as error:
|
||||
if not _is_rejected_signature(error):
|
||||
raise
|
||||
return _post_inline_sync(body, placeholders, uncached, post, log_request, error)
|
||||
|
|
@ -1256,6 +1256,7 @@ def sync_transform_request_body(
|
|||
vertex_project: str | None,
|
||||
vertex_location: str | None,
|
||||
vertex_auth_header: str | None,
|
||||
context_caching: bool = True,
|
||||
) -> RequestBody:
|
||||
from ..context_caching.vertex_ai_context_caching import ContextCachingEndpoints
|
||||
|
||||
|
|
@ -1275,6 +1276,7 @@ def sync_transform_request_body(
|
|||
timeout=timeout,
|
||||
extra_headers=extra_headers,
|
||||
cached_content=optional_params.pop("cached_content", None),
|
||||
context_caching=context_caching,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
vertex_project=vertex_project,
|
||||
|
|
@ -1390,6 +1392,7 @@ async def async_transform_request_body(
|
|||
vertex_project: str | None,
|
||||
vertex_location: str | None,
|
||||
vertex_auth_header: str | None,
|
||||
context_caching: bool = True,
|
||||
) -> RequestBody:
|
||||
from ..context_caching.vertex_ai_context_caching import ContextCachingEndpoints
|
||||
|
||||
|
|
@ -1409,6 +1412,7 @@ async def async_transform_request_body(
|
|||
timeout=timeout,
|
||||
extra_headers=extra_headers,
|
||||
cached_content=optional_params.pop("cached_content", None),
|
||||
context_caching=context_caching,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
vertex_project=vertex_project,
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence
|
|||
from copy import deepcopy
|
||||
from functools import lru_cache, partial
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, Protocol, Union, cast, get_args
|
||||
|
||||
import httpx
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
|
@ -34,6 +34,7 @@ from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAcc
|
|||
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import encode_tool_call_id_with_signature
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
|
||||
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
|
|
@ -103,7 +104,7 @@ from ..common_utils import (
|
|||
)
|
||||
from ..context_caching.known_cached_contents import known_cached_contents
|
||||
from ..context_caching.rejected_cache_retry import (
|
||||
cache_rebuild,
|
||||
cache_rebuilds,
|
||||
post_with_fresh_cache_async,
|
||||
post_with_fresh_cache_sync,
|
||||
)
|
||||
|
|
@ -134,6 +135,13 @@ from .long_decode import (
|
|||
with_initial_prompt_tokens,
|
||||
with_spans_shifted,
|
||||
)
|
||||
from .rejected_thought_signature_retry import (
|
||||
build_with_accepted_thought_signatures_async,
|
||||
build_with_accepted_thought_signatures_sync,
|
||||
post_with_accepted_thought_signatures_async,
|
||||
post_with_accepted_thought_signatures_sync,
|
||||
with_placeholder_thought_signatures,
|
||||
)
|
||||
from .transformation import (
|
||||
async_transform_request_body,
|
||||
gemini_convert_messages_with_history,
|
||||
|
|
@ -614,6 +622,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
Note: Please make sure to modify the default parameters as required for your use case.
|
||||
"""
|
||||
|
||||
_route: ClassVar[str] = "vertex_ai"
|
||||
|
||||
temperature: float | None = None
|
||||
max_output_tokens: int | None = None
|
||||
top_p: float | None = None
|
||||
|
|
@ -1667,9 +1677,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
drop_params: bool,
|
||||
allow_integer_enums: bool = True,
|
||||
) -> dict:
|
||||
self._apply_include_server_side_tool_invocations(non_default_params, optional_params)
|
||||
request_params: Final = tool_params_without_hosted_mcp(
|
||||
non_default_params, model=model, route=self._route, drop_params=drop_params
|
||||
)
|
||||
self._apply_include_server_side_tool_invocations(request_params, optional_params)
|
||||
supports_sampling_params: Final = self.supports_sampling_params(model)
|
||||
for param, value in non_default_params.items():
|
||||
for param, value in request_params.items():
|
||||
if param in ("temperature", "top_p", "top_k") and not supports_sampling_params:
|
||||
continue
|
||||
if param == "temperature":
|
||||
|
|
@ -1733,7 +1746,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
if _tool_choice_value is not None:
|
||||
optional_params["tool_choice"] = _tool_choice_value
|
||||
elif param == "parallel_tool_calls":
|
||||
tools_list = non_default_params.get("tools", non_default_params.get("functions"))
|
||||
tools_list = request_params.get("tools", request_params.get("functions"))
|
||||
num_tools = len(tools_list) if isinstance(tools_list, list) else 0
|
||||
# Gemini does not support parallel_tool_calls=False with multiple
|
||||
# tools. Drop the param instead of failing — Responses API clients
|
||||
|
|
@ -3494,23 +3507,17 @@ class VertexLLM(VertexBase):
|
|||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
rebuild: Final = cache_rebuild(
|
||||
partial(
|
||||
async_transform_request_body,
|
||||
**data,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=auth_header,
|
||||
),
|
||||
messages,
|
||||
optional_params,
|
||||
)
|
||||
request_body: Final = await async_transform_request_body(
|
||||
transform: Final = partial(
|
||||
async_transform_request_body,
|
||||
**data,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=auth_header,
|
||||
)
|
||||
rebuild, uncached = cache_rebuilds(transform, messages, optional_params)
|
||||
request_body: Final = await build_with_accepted_thought_signatures_async(
|
||||
transform, with_placeholder_thought_signatures, uncached
|
||||
)
|
||||
|
||||
## LOGGING
|
||||
log_request: Final = partial(_log_generate_request, logging_obj, messages, api_base, headers)
|
||||
|
|
@ -3539,7 +3546,14 @@ class VertexLLM(VertexBase):
|
|||
async def post(body: RequestBody) -> ModelResponseIterator:
|
||||
return await open_stream(client=client, data=json.dumps(body), request_body=body)
|
||||
|
||||
return (await post_with_fresh_cache_async(request_body, rebuild, post, log_request))[1]
|
||||
accepted: Final = partial(
|
||||
post_with_accepted_thought_signatures_async,
|
||||
placeholders=with_placeholder_thought_signatures,
|
||||
uncached=uncached,
|
||||
post=post,
|
||||
log_request=log_request,
|
||||
)
|
||||
return (await post_with_fresh_cache_async(request_body, rebuild, accepted, log_request))[1][1]
|
||||
|
||||
streaming_response: Final = CustomStreamWrapper(
|
||||
completion_stream=None,
|
||||
|
|
@ -3606,23 +3620,17 @@ class VertexLLM(VertexBase):
|
|||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
rebuild: Final = cache_rebuild(
|
||||
partial(
|
||||
async_transform_request_body,
|
||||
**data,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=auth_header,
|
||||
),
|
||||
messages,
|
||||
optional_params,
|
||||
)
|
||||
request_body: Final = await async_transform_request_body(
|
||||
transform: Final = partial(
|
||||
async_transform_request_body,
|
||||
**data,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=auth_header,
|
||||
)
|
||||
rebuild, uncached = cache_rebuilds(transform, messages, optional_params)
|
||||
request_body: Final = await build_with_accepted_thought_signatures_async(
|
||||
transform, with_placeholder_thought_signatures, uncached
|
||||
)
|
||||
|
||||
_async_client_params: Final = {}
|
||||
if timeout:
|
||||
|
|
@ -3663,7 +3671,14 @@ class VertexLLM(VertexBase):
|
|||
)
|
||||
return response
|
||||
|
||||
sent_body, first_response = await post_with_fresh_cache_async(request_body, rebuild, post, log_request)
|
||||
accepted: Final = partial(
|
||||
post_with_accepted_thought_signatures_async,
|
||||
placeholders=with_placeholder_thought_signatures,
|
||||
uncached=uncached,
|
||||
post=post,
|
||||
log_request=log_request,
|
||||
)
|
||||
_, (sent_body, first_response) = await post_with_fresh_cache_async(request_body, rebuild, accepted, log_request)
|
||||
return _transform_long_decode_hops(
|
||||
model=model,
|
||||
hops=await collect_hops_async(parse_body(first_response), sent_body, post),
|
||||
|
|
@ -3795,23 +3810,17 @@ class VertexLLM(VertexBase):
|
|||
)
|
||||
|
||||
## TRANSFORMATION ##
|
||||
rebuild: Final = cache_rebuild(
|
||||
partial(
|
||||
sync_transform_request_body,
|
||||
**transform_request_params,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=auth_header,
|
||||
),
|
||||
messages,
|
||||
optional_params,
|
||||
)
|
||||
data: Final = sync_transform_request_body(
|
||||
transform: Final = partial(
|
||||
sync_transform_request_body,
|
||||
**transform_request_params,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=auth_header,
|
||||
)
|
||||
rebuild, uncached = cache_rebuilds(transform, messages, optional_params)
|
||||
data: Final = build_with_accepted_thought_signatures_sync(
|
||||
transform, with_placeholder_thought_signatures, uncached
|
||||
)
|
||||
|
||||
## LOGGING
|
||||
log_request: Final = partial(_log_generate_request, logging_obj, messages, url, headers)
|
||||
|
|
@ -3839,7 +3848,14 @@ class VertexLLM(VertexBase):
|
|||
def post(body: RequestBody) -> ModelResponseIterator:
|
||||
return open_stream(client=client, data=json.dumps(body), request_body=body)
|
||||
|
||||
return post_with_fresh_cache_sync(data, rebuild, post, log_request)[1]
|
||||
accepted: Final = partial(
|
||||
post_with_accepted_thought_signatures_sync,
|
||||
placeholders=with_placeholder_thought_signatures,
|
||||
uncached=uncached,
|
||||
post=post,
|
||||
log_request=log_request,
|
||||
)
|
||||
return post_with_fresh_cache_sync(data, rebuild, accepted, log_request)[1][1]
|
||||
|
||||
streaming_response: Final = CustomStreamWrapper(
|
||||
completion_stream=None,
|
||||
|
|
@ -3883,7 +3899,14 @@ class VertexLLM(VertexBase):
|
|||
)
|
||||
return response
|
||||
|
||||
sent_body, first_response = post_with_fresh_cache_sync(data, rebuild, post, log_request)
|
||||
accepted: Final = partial(
|
||||
post_with_accepted_thought_signatures_sync,
|
||||
placeholders=with_placeholder_thought_signatures,
|
||||
uncached=uncached,
|
||||
post=post,
|
||||
log_request=log_request,
|
||||
)
|
||||
_, (sent_body, first_response) = post_with_fresh_cache_sync(data, rebuild, accepted, log_request)
|
||||
return _transform_long_decode_hops(
|
||||
model=model,
|
||||
hops=collect_hops_sync(parse_body(first_response), sent_body, post),
|
||||
|
|
|
|||
|
|
@ -13478,6 +13478,10 @@
|
|||
"input_cost_per_token": 2e-07,
|
||||
"output_cost_per_token": 5e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"token_count_framing": 149,
|
||||
"token_count_tool_framing": 302,
|
||||
"token_count_responses_framing": 145,
|
||||
"token_count_responses_tool_framing": 100,
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
|
|
@ -13831,7 +13835,7 @@
|
|||
"input_cost_per_token_flex": 3.7e-07,
|
||||
"input_cost_per_token_priority": 1.295e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 163840,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.22e-06,
|
||||
|
|
@ -13845,7 +13849,7 @@
|
|||
"supports_audio_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": false,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
|
||||
},
|
||||
"bedrock/ap-northeast-1/minimax.minimax-m2.1": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
|
|
@ -14002,7 +14006,7 @@
|
|||
"input_cost_per_token_flex": 3.7e-07,
|
||||
"input_cost_per_token_priority": 1.295e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 163840,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.22e-06,
|
||||
|
|
@ -14016,7 +14020,7 @@
|
|||
"supports_audio_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": false,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
|
||||
},
|
||||
"bedrock/ap-south-1/minimax.minimax-m2.1": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
|
|
@ -14140,12 +14144,12 @@
|
|||
"input_cost_per_token": 7.4e-07,
|
||||
"input_cost_per_token_flex": 3.7e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 163840,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.22e-06,
|
||||
"output_cost_per_token_flex": 1.11e-06,
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html",
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md",
|
||||
"supports_audio_input": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -14250,7 +14254,7 @@
|
|||
"input_cost_per_token_flex": 3.7e-07,
|
||||
"input_cost_per_token_priority": 1.295e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 163840,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.22e-06,
|
||||
|
|
@ -14264,7 +14268,7 @@
|
|||
"supports_audio_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": false,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
|
||||
},
|
||||
"bedrock/eu-north-1/minimax.minimax-m2.1": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
|
|
@ -14786,7 +14790,7 @@
|
|||
"input_cost_per_token_flex": 3.7e-07,
|
||||
"input_cost_per_token_priority": 1.295e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 163840,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.22e-06,
|
||||
|
|
@ -14800,7 +14804,7 @@
|
|||
"supports_audio_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": false,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
|
||||
},
|
||||
"bedrock/sa-east-1/minimax.minimax-m2.1": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
|
|
@ -15035,7 +15039,7 @@
|
|||
"input_cost_per_token_flex": 3.1e-07,
|
||||
"input_cost_per_token_priority": 1.085e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 163840,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.85e-06,
|
||||
|
|
@ -15049,7 +15053,7 @@
|
|||
"supports_audio_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": false,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
|
||||
},
|
||||
"bedrock/us-east-1/minimax.minimax-m2.1": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
|
|
@ -15156,7 +15160,7 @@
|
|||
"input_cost_per_token_flex": 3.1e-07,
|
||||
"input_cost_per_token_priority": 1.085e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 163840,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.85e-06,
|
||||
|
|
@ -15170,7 +15174,7 @@
|
|||
"supports_audio_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": false,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
|
||||
},
|
||||
"bedrock/us-east-2/minimax.minimax-m2.1": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
|
|
@ -15552,6 +15556,7 @@
|
|||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md"
|
||||
},
|
||||
"bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 2.65e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 8000,
|
||||
|
|
@ -15562,6 +15567,7 @@
|
|||
"supports_pdf_input": true
|
||||
},
|
||||
"bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 8000,
|
||||
|
|
@ -15708,7 +15714,7 @@
|
|||
"input_cost_per_token_flex": 3.1e-07,
|
||||
"input_cost_per_token_priority": 1.085e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 163840,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.85e-06,
|
||||
|
|
@ -15722,7 +15728,7 @@
|
|||
"supports_audio_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": false,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
|
||||
},
|
||||
"bedrock/us-west-2/minimax.minimax-m2.1": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
|
|
@ -23519,12 +23525,12 @@
|
|||
]
|
||||
},
|
||||
"deepgram/streaming/nova-3": {
|
||||
"input_cost_per_second": 8e-05,
|
||||
"input_cost_per_second": 0.00012833,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0048/60 seconds = $0.00008000 per second",
|
||||
"calculation": "$0.0077/60 seconds = $0.00012833 per second",
|
||||
"note": "Nova-3 monolingual streaming, pay as you go",
|
||||
"original_pricing_per_minute": 0.0048
|
||||
"original_pricing_per_minute": 0.0077
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
|
|
@ -23533,13 +23539,28 @@
|
|||
"/v1/listen"
|
||||
]
|
||||
},
|
||||
"deepgram/streaming/nova-3-multilingual": {
|
||||
"input_cost_per_second": 9.667e-05,
|
||||
"deepgram/streaming/nova-3-medical": {
|
||||
"input_cost_per_second": 0.00012833,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0058/60 seconds = $0.00009667 per second",
|
||||
"calculation": "$0.0077/60 seconds = $0.00012833 per second",
|
||||
"note": "Nova-3 Medical streaming, launch price from Deepgram's 2025-03-03 announcement. deepgram.com/pricing does not list Nova-3 Medical, and this matches its current Nova-3 monolingual streaming pay as you go rate",
|
||||
"original_pricing_per_minute": 0.0077
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
"source": "https://www.businesswire.com/news/home/20250303470439/en",
|
||||
"supported_endpoints": [
|
||||
"/v1/listen"
|
||||
]
|
||||
},
|
||||
"deepgram/streaming/nova-3-multilingual": {
|
||||
"input_cost_per_second": 0.00015333,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0092/60 seconds = $0.00015333 per second",
|
||||
"note": "Nova-3 multilingual (language=multi) streaming, pay as you go",
|
||||
"original_pricing_per_minute": 0.0058
|
||||
"original_pricing_per_minute": 0.0092
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
|
|
@ -24519,7 +24540,7 @@
|
|||
"input_cost_per_token_flex": 3.1e-07,
|
||||
"input_cost_per_token_priority": 1.085e-06,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 164000,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.85e-06,
|
||||
|
|
@ -24530,7 +24551,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json",
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md",
|
||||
"supports_audio_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": false
|
||||
|
|
@ -54457,6 +54478,8 @@
|
|||
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
|
||||
"input_cost_per_token_batches": 1e-06,
|
||||
"litellm_provider": "xai",
|
||||
"token_count_framing": 185,
|
||||
"token_count_tool_framing": 214,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 1000000,
|
||||
"mode": "chat",
|
||||
|
|
@ -62640,7 +62663,9 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"token_count_framing": 23,
|
||||
"token_count_tool_framing": 12
|
||||
},
|
||||
"deepseek/deepseek-v4-flash-vision-exp": {
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
|
|
@ -62746,7 +62771,9 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
"supports_vision": false,
|
||||
"token_count_framing": 76,
|
||||
"token_count_tool_framing": 75
|
||||
},
|
||||
"tencent/deepseek-v4-pro": {
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
|
|
@ -64032,6 +64059,7 @@
|
|||
"input_cost_per_token": 3e-07,
|
||||
"input_cost_per_token_priority": 3.75e-07,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"token_count_framing": 23,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 393216,
|
||||
"mode": "chat",
|
||||
|
|
@ -64133,6 +64161,7 @@
|
|||
"input_cost_per_token": 3e-07,
|
||||
"input_cost_per_token_priority": 3.75e-07,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"token_count_framing": 23,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 393216,
|
||||
"mode": "chat",
|
||||
|
|
@ -76801,6 +76830,7 @@
|
|||
"supports_audio_input": true,
|
||||
"supports_audio_output": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_continuation": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
|
|
@ -76860,6 +76890,7 @@
|
|||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_computer_use": true,
|
||||
"supports_continuation": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
|
|
@ -76917,6 +76948,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_continuation": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
|
|
|
|||
|
|
@ -125,6 +125,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
|
|||
NO_TEAM_MEMBERSHIP_SENTINEL,
|
||||
TAG_REGISTRY_OVERFLOW_SENTINEL,
|
||||
UserApiKeyCache,
|
||||
budget_windows_absent_cache_key,
|
||||
end_user_cache_key,
|
||||
end_user_restricted_registry_cache_key,
|
||||
get_management_object_ttl,
|
||||
|
|
@ -3471,11 +3472,22 @@ async def delete_cache_key_object(
|
|||
)
|
||||
|
||||
await publish_auth_cache_invalidation(cache_key=key)
|
||||
await evict_budget_windows_absent_marker(
|
||||
entity_type=Litellm_EntityType.KEY, entity_id=hashed_token, user_api_key_cache=user_api_key_cache
|
||||
)
|
||||
|
||||
|
||||
_delete_cache_key_object: Final = delete_cache_key_object
|
||||
|
||||
|
||||
async def evict_budget_windows_absent_marker(
|
||||
entity_type: Litellm_EntityType, entity_id: str, user_api_key_cache: UserApiKeyCache
|
||||
) -> None:
|
||||
marker: Final = budget_windows_absent_cache_key(entity_type.value, entity_id)
|
||||
evict_local(cache_keys=(marker,), user_api_key_cache=user_api_key_cache)
|
||||
await publish_auth_cache_invalidation(cache_key=marker)
|
||||
|
||||
|
||||
async def delete_cache_key_objects(
|
||||
hashed_tokens: Sequence[str],
|
||||
user_api_key_cache: UserApiKeyCache,
|
||||
|
|
|
|||
|
|
@ -770,9 +770,10 @@ def codex_model_catalog(
|
|||
A model Codex does not know gets the fallback
|
||||
entry, with the same base instructions Codex itself uses so the agent never
|
||||
runs without a system prompt, and offers the reasoning efforts `groups` gives its model group
|
||||
that the installed Codex, at `client_version`, can parse. Apart from aliases, the catalog is
|
||||
the one `/v1/models?client_version=...` serves, built from the installed Codex's own entries
|
||||
instead of the proxy's vendored ones.
|
||||
that the installed Codex, at `client_version`, can parse. Apart from aliases, and the window the
|
||||
deployments' `model_info.max_input_tokens` gives a model Codex knows, the catalog is the one
|
||||
`/v1/models?client_version=...` serves, built from the installed Codex's own entries instead of
|
||||
the proxy's vendored ones.
|
||||
"""
|
||||
rows: Final = tuple(_codex_row(m, upstreams.get(m.id), groups) for m in models)
|
||||
body: Final = codex_models_response_json(
|
||||
|
|
|
|||
|
|
@ -183,7 +183,7 @@ def evict_local(cache_keys: Sequence[str], user_api_key_cache: "UserApiKeyCache"
|
|||
|
||||
|
||||
def _drop_from_memory(cache_key: str, user_api_key_cache: "UserApiKeyCache") -> None:
|
||||
user_api_key_cache.in_memory_cache_for(cache_key).delete_cache(cache_key)
|
||||
user_api_key_cache.delete_from_memory(cache_key)
|
||||
|
||||
|
||||
@with_service_target(AUTH_OBJECTS_TARGET)
|
||||
|
|
|
|||
|
|
@ -7,8 +7,10 @@ never the OpenAI list shape. This module builds that body from the rows ``/v1/mo
|
|||
lists: a model Codex knows keeps Codex's own entry for it, any other model gets the fallback entry
|
||||
Codex itself uses for an unknown slug, and ``model_info.service_tiers`` becomes the entry's
|
||||
``service_tiers``, which Codex turns into slash commands (``/ultrafast``) that send ``service_tier``
|
||||
upstream. A fallback entry offers the model group's ``supported_reasoning_efforts`` as its reasoning
|
||||
levels, trimmed to the ones the requesting Codex release can parse.
|
||||
upstream. A ``model_info.max_input_tokens`` the operator sets replaces a known model's
|
||||
``context_window``, up to the most Codex lets its own ``model_context_window`` raise that model to.
|
||||
A fallback entry offers the model group's ``supported_reasoning_efforts`` as its reasoning levels,
|
||||
trimmed to the ones the requesting Codex release can parse.
|
||||
|
||||
The stock entries are Codex 0.159.3's bundled catalog, vendored unchanged from
|
||||
https://github.com/openai/codex/blob/rust-v0.159.3/codex-rs/models-manager/models.json (Apache-2.0)
|
||||
|
|
@ -158,6 +160,7 @@ class CodexStockModel(LiteLLMBaseModel):
|
|||
supports_search_tool: bool = False
|
||||
supported_reasoning_levels: tuple[CodexReasoningLevel, ...] = ()
|
||||
context_window: int | None = None
|
||||
max_context_window: int | None = None
|
||||
|
||||
|
||||
class CodexStockCatalog(LiteLLMBaseModel):
|
||||
|
|
@ -173,8 +176,9 @@ class CodexCatalogRow:
|
|||
"""What the catalog needs to know about one listed model: its public id, the listing's mode and
|
||||
input limit, the upstream model behind it, the reasoning effort its deployments pin, and the
|
||||
`model_info` values the operator set; `service_tiers` holds one raw `model_info.service_tiers`
|
||||
value per deployment behind the id, None where a deployment sets none. `supported_reasoning_efforts`
|
||||
is the model group's, None when unknown."""
|
||||
value per deployment behind the id, None where a deployment sets none, and
|
||||
`configured_max_input_tokens` one `model_info.max_input_tokens` per deployment the same way.
|
||||
`supported_reasoning_efforts` is the model group's, None when unknown."""
|
||||
|
||||
id: str
|
||||
mode: str | None = None
|
||||
|
|
@ -185,6 +189,7 @@ class CodexCatalogRow:
|
|||
service_tiers: tuple[object, ...] = ()
|
||||
supported_reasoning_efforts: tuple[str, ...] | None = None
|
||||
search_tool_support: tuple[bool | None, ...] = ()
|
||||
configured_max_input_tokens: tuple[int | None, ...] = ()
|
||||
|
||||
|
||||
@cache
|
||||
|
|
@ -333,6 +338,25 @@ def _alias_context_window(row: CodexCatalogRow, stock_model: CodexStockModel) ->
|
|||
return MappingProxyType({"context_window": min(stock_model.context_window, row.max_input_tokens)})
|
||||
|
||||
|
||||
def _configured_window(row: CodexCatalogRow, stock_model: CodexStockModel) -> Mapping[str, int]:
|
||||
"""The `context_window` the operator's `model_info.max_input_tokens` gives a known model's entry,
|
||||
or nothing, which keeps `stock_model`'s own.
|
||||
|
||||
A request to the row can route to any of its deployments, so the window is the smallest one of them
|
||||
takes: its `max_input_tokens` where the operator set a positive one, Codex's own window where not. A
|
||||
limit set on only some deployments can therefore lower the window but never raise it. The window never
|
||||
goes past `stock_model`'s `max_context_window`, the most Codex lets its own `model_context_window`
|
||||
setting raise that model to."""
|
||||
per_deployment: Final = row.configured_max_input_tokens
|
||||
declared: Final = tuple(limit for limit in per_deployment if limit is not None and limit > 0)
|
||||
unset_window: Final = () if len(declared) == len(per_deployment) else (stock_model.context_window,)
|
||||
if not declared or None in unset_window:
|
||||
return MappingProxyType({})
|
||||
smallest: Final = min(limit for limit in (*declared, *unset_window) if limit is not None)
|
||||
ceiling: Final = stock_model.max_context_window
|
||||
return MappingProxyType({"context_window": smallest if ceiling is None else min(smallest, ceiling)})
|
||||
|
||||
|
||||
def _alias_reasoning_level(row: CodexCatalogRow, stock_model: CodexStockModel) -> str | None:
|
||||
efforts: Final = frozenset(level.effort for level in stock_model.supported_reasoning_levels)
|
||||
return row.reasoning_effort if row.reasoning_effort in efforts else None
|
||||
|
|
@ -398,6 +422,7 @@ def _entry(
|
|||
"upgrade": upgrade,
|
||||
**({} if search_tool is None else {"supports_search_tool": search_tool}),
|
||||
**(_deployment_defaults(row, stock_model) if is_alias and alias_defaults == "deployment" else {}),
|
||||
**_configured_window(row, stock_model),
|
||||
}
|
||||
if configured is None:
|
||||
return stock_model.model_copy(update=listing)
|
||||
|
|
@ -473,10 +498,12 @@ def codex_models_response_json(
|
|||
byte limit silently and keeps its bundled one, so the body holds only the entries that fit,
|
||||
the ones offering a service tier taken first and the rest in listing order, an entry too large
|
||||
for the bytes left passed over, every kept entry at its listing position; `byte_limit=None`
|
||||
keeps every entry. A row listed under another name than the Codex model its `upstream_model`
|
||||
names (an alias) takes that model's entry; `alias_defaults="deployment"` drops the defaults
|
||||
Codex picks for that model in favor of what the alias's deployments pin, while `"codex"` keeps
|
||||
them, as Codex would use them for the model's own name. `client_version` is the requesting Codex's version, which decides the reasoning
|
||||
keeps every entry. A row whose deployments set `model_info.max_input_tokens` gets the smallest
|
||||
window any of them takes as its known model's `context_window`, capped at that model's
|
||||
`max_context_window`. A row listed under another name than the Codex model its `upstream_model` names (an alias) takes that model's
|
||||
entry; `alias_defaults="deployment"` drops the defaults Codex picks for that model in favor of
|
||||
what the alias's deployments pin, while `"codex"` keeps them, as Codex would use them for the
|
||||
model's own name. `client_version` is the requesting Codex's version, which decides the reasoning
|
||||
levels a fallback entry may offer; None counts as a version too old for custom levels.
|
||||
"""
|
||||
known: Final = bundled_codex_models() if stock is None else stock
|
||||
|
|
@ -519,6 +546,9 @@ def _catalog_row(
|
|||
search_tool_support=(
|
||||
registry.resolve(lookup_id, kind="search_tool_support", team_id=team_id) if registry is not None else ()
|
||||
),
|
||||
configured_max_input_tokens=(
|
||||
registry.group(lookup_id, field="max_input_tokens", team_id=team_id) if registry is not None else ()
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -532,9 +562,9 @@ def codex_catalog_rows(
|
|||
"""`rows` joined with the router's configured metadata, looked up by each entry's internal id so
|
||||
team-scoped rows resolve the way the Anthropic listing's display names do, a `model_group_alias`
|
||||
reading its target's deployments under its own name; `team_id` is the requesting key's team, so the
|
||||
upstream model that picks Codex's stock entry and the tiers are read only off the deployments its
|
||||
requests can route to, and the reasoning efforts are the ones `/model_group/info` reports for the
|
||||
entry. Display names are read under `label_entries`' ids when given."""
|
||||
upstream model that picks Codex's stock entry, the tiers, and the configured input limits are read
|
||||
only off the deployments its requests can route to, and the reasoning efforts are the ones
|
||||
`/model_group/info` reports for the entry. Display names are read under `label_entries`' ids when given."""
|
||||
lookup_ids: Final = MappingProxyType(dict(entries))
|
||||
label_ids: Final = lookup_ids if label_entries is None else MappingProxyType(dict(label_entries))
|
||||
return tuple(
|
||||
|
|
|
|||
|
|
@ -56,6 +56,22 @@ class TokenCountSettings:
|
|||
framing: int | None
|
||||
count_model: TokenCountModel | None
|
||||
tokenizer: TokenCountTokenizer | None = None
|
||||
tool_framing: int | None = None
|
||||
responses_framing: int | None = None
|
||||
responses_tool_framing: int | None = None
|
||||
|
||||
@property
|
||||
def frames_counts(self) -> bool:
|
||||
return any(
|
||||
framing is not None
|
||||
for framing in (self.framing, self.tool_framing, self.responses_framing, self.responses_tool_framing)
|
||||
)
|
||||
|
||||
def hidden_prompt_tokens(self, carries_tools: bool, responses_api: bool) -> int:
|
||||
chat: Final = self.tool_framing if carries_tools else None
|
||||
responses: Final = self.responses_tool_framing if carries_tools else self.responses_framing
|
||||
narrower: Final = responses if responses_api else chat
|
||||
return (narrower if narrower is not None else self.framing) or 0
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -105,6 +121,15 @@ def _framing(value: object) -> int | None:
|
|||
return None
|
||||
|
||||
|
||||
def _first_framing(source: Mapping[str, object], keys: tuple[str, ...]) -> int | None:
|
||||
return next((framing for framing in map(_framing, map(source.get, keys)) if framing is not None), None)
|
||||
|
||||
|
||||
def _configured_framing(configured: Mapping[str, object], default: Mapping[str, object], *keys: str) -> int | None:
|
||||
operator_sets_one: Final = _first_framing(configured, (*keys, "token_count_framing")) is not None
|
||||
return _first_framing(configured if operator_sets_one else default, keys)
|
||||
|
||||
|
||||
def _non_blank(value: object) -> str | None:
|
||||
return value if isinstance(value, str) and value.strip() else None
|
||||
|
||||
|
|
@ -175,13 +200,21 @@ def _tokenizer(value: object) -> TokenCountTokenizer | None:
|
|||
def resolve_token_count_settings(
|
||||
configured: Mapping[str, object], default: Mapping[str, object], deployment_region: Callable[[], str]
|
||||
) -> TokenCountSettings:
|
||||
framing: Final = _framing(configured.get("token_count_framing"))
|
||||
return TokenCountSettings(
|
||||
framing=framing if framing is not None else _framing(default.get("token_count_framing")),
|
||||
framing=_configured_framing(configured, default, "token_count_framing"),
|
||||
count_model=_resolved_count_model(
|
||||
configured.get("token_count_model"), default.get("token_count_model"), deployment_region
|
||||
),
|
||||
tokenizer=_tokenizer(default.get("token_count_tokenizer")),
|
||||
tool_framing=_configured_framing(configured, default, "token_count_tool_framing"),
|
||||
responses_framing=_configured_framing(configured, default, "token_count_responses_framing"),
|
||||
responses_tool_framing=_configured_framing(
|
||||
configured,
|
||||
default,
|
||||
"token_count_responses_tool_framing",
|
||||
"token_count_responses_framing",
|
||||
"token_count_tool_framing",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ T = TypeVar("T", bound=BaseModel)
|
|||
from litellm.constants import AUTH_OBJECTS_TARGET as AUTH_OBJECTS_TARGET # noqa: E402, PLC0414 # public re-export
|
||||
|
||||
_HASHED_TOKEN_CACHE_KEY: Final = re.compile(r"[0-9a-f]{64}")
|
||||
_BUDGET_WINDOWS_ABSENT_PREFIX: Final = "budget_windows_absent:"
|
||||
|
||||
|
||||
def is_user_key_cache_key(key: str) -> bool:
|
||||
|
|
@ -70,6 +71,7 @@ class UserApiKeyCache(DualCache):
|
|||
default_in_memory_ttl: float | None = None,
|
||||
default_redis_ttl: float | None = None,
|
||||
key_object_in_memory_cache: InMemoryCache | None = None,
|
||||
budget_windows_absent_in_memory_cache: InMemoryCache | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
in_memory_cache=in_memory_cache,
|
||||
|
|
@ -83,19 +85,33 @@ class UserApiKeyCache(DualCache):
|
|||
default_in_memory_ttl=default_in_memory_ttl,
|
||||
default_redis_ttl=default_redis_ttl,
|
||||
)
|
||||
self.budget_windows_absent_cache: Final = DualCache(
|
||||
in_memory_cache=budget_windows_absent_in_memory_cache
|
||||
or InMemoryCache(default_ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL),
|
||||
default_in_memory_ttl=default_in_memory_ttl,
|
||||
)
|
||||
|
||||
def in_memory_cache_for(self, key: str) -> InMemoryCache:
|
||||
return self.key_object_cache.in_memory_cache if is_user_key_cache_key(key) else self.in_memory_cache
|
||||
|
||||
def delete_from_memory(self, key: str) -> None:
|
||||
self.in_memory_cache_for(key).delete_cache(key)
|
||||
if key.startswith(_BUDGET_WINDOWS_ABSENT_PREFIX):
|
||||
self.budget_windows_absent_cache.delete_cache(key)
|
||||
|
||||
def update_cache_ttl(self, default_in_memory_ttl: float | None, default_redis_ttl: float | None) -> None:
|
||||
super().update_cache_ttl(default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl)
|
||||
self.key_object_cache.update_cache_ttl(
|
||||
default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl
|
||||
)
|
||||
self.budget_windows_absent_cache.update_cache_ttl(
|
||||
default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=None
|
||||
)
|
||||
|
||||
def update_in_memory_max_size(self, max_size: int | None) -> None:
|
||||
super().update_in_memory_max_size(max_size)
|
||||
self.key_object_cache.update_in_memory_max_size(max_size)
|
||||
self.budget_windows_absent_cache.update_in_memory_max_size(max_size)
|
||||
|
||||
def attach_redis_cache(
|
||||
self, redis_cache: RedisCache | None = None, *, default_redis_ttl: float | None = None
|
||||
|
|
@ -272,6 +288,7 @@ class UserApiKeyCache(DualCache):
|
|||
def flush_cache(self) -> None:
|
||||
super().flush_cache()
|
||||
self.key_object_cache.in_memory_cache.flush_cache()
|
||||
self.budget_windows_absent_cache.flush_cache()
|
||||
|
||||
async def async_set_cache_pipeline(
|
||||
self, cache_list: Sequence[tuple[str, object]], local_only: bool = False, **kwargs: object
|
||||
|
|
@ -400,6 +417,11 @@ def team_membership_reservation_cache_key(user_id: str, team_id: str) -> str:
|
|||
NO_TEAM_MEMBERSHIP_SENTINEL: Final = "__no_team_membership__"
|
||||
|
||||
|
||||
def budget_windows_absent_cache_key(entity_type: str, entity_id: str) -> str:
|
||||
"""Cache key marking a key or team that holds no budget windows, so the spend fallback skips its row read."""
|
||||
return f"{_BUDGET_WINDOWS_ABSENT_PREFIX}{entity_type}:{entity_id}"
|
||||
|
||||
|
||||
def get_management_object_ttl(cache: DualCache) -> float:
|
||||
"""
|
||||
In-memory TTL for management-object cache writes (keys, teams, users, budgets, ...).
|
||||
|
|
|
|||
|
|
@ -42,10 +42,12 @@ _EMPTY_ROWS: Final[Mapping[DbRow, Mapping[str, JsonValue]]] = MappingProxyType({
|
|||
|
||||
|
||||
class SettingsStore(MutableMapping[str, JsonValue]):
|
||||
def __init__(self, section: Section) -> None:
|
||||
def __init__(self, section: Section, *, managed_db_keys: frozenset[str] | None = None) -> None:
|
||||
self._section: Final = section
|
||||
self._managed_db_keys: Final = managed_db_keys
|
||||
self._yaml_values: Mapping[str, JsonValue] = _EMPTY_VALUES
|
||||
self._database_rows: Mapping[DbRow, Mapping[str, JsonValue]] = _EMPTY_ROWS
|
||||
self._unmanaged_db_keys: frozenset[str] = frozenset()
|
||||
self._runtime_values: Mapping[str, JsonValue] = _EMPTY_VALUES
|
||||
self._deleted_runtime_keys: frozenset[str] = frozenset()
|
||||
|
||||
|
|
@ -77,15 +79,24 @@ class SettingsStore(MutableMapping[str, JsonValue]):
|
|||
return self.owned_by_config(key) and self._db_value_is_shadowed(key)
|
||||
|
||||
def apply_db_row(self, row: DbRow, db_row: Mapping[str, JsonValue]) -> None:
|
||||
applied_row: Final = MappingProxyType({key: db_row[key] for key in db_row if self._db_manages(row, key)})
|
||||
previous_row: Final = self._database_rows.get(row, _EMPTY_VALUES)
|
||||
changed: Final = frozenset(
|
||||
key
|
||||
for key in (*previous_row, *db_row)
|
||||
if previous_row.get(key, ABSENT) != db_row.get(key, ABSENT) # pyright: ignore[reportUnknownArgumentType] # JsonValue vs Absent compare
|
||||
for key in (*previous_row, *applied_row)
|
||||
if previous_row.get(key, ABSENT) != applied_row.get(key, ABSENT) # pyright: ignore[reportUnknownArgumentType] # JsonValue vs Absent compare
|
||||
)
|
||||
self._database_rows = MappingProxyType({**self._database_rows, row: MappingProxyType(dict(db_row))})
|
||||
self._database_rows = MappingProxyType({**self._database_rows, row: applied_row})
|
||||
if row == self._section:
|
||||
self._unmanaged_db_keys = frozenset(key for key in db_row if key not in applied_row)
|
||||
self._clear_runtime_keys(changed)
|
||||
|
||||
def unmanaged_db_keys(self) -> tuple[str, ...]:
|
||||
return tuple(sorted(self._unmanaged_db_keys))
|
||||
|
||||
def _db_manages(self, row: DbRow, key: str) -> bool:
|
||||
return row != self._section or self._managed_db_keys is None or key in self._managed_db_keys
|
||||
|
||||
def resolved(self) -> Mapping[str, JsonValue]:
|
||||
return MappingProxyType(dict(self))
|
||||
|
||||
|
|
|
|||
|
|
@ -107,9 +107,20 @@ class MCPSecurityGuardrail(CustomGuardrail):
|
|||
from litellm.proxy.mcp_server.mcp_server_manager import (
|
||||
global_mcp_server_manager,
|
||||
)
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
async with global_mcp_server_manager.catalog.operation():
|
||||
registry: Final = global_mcp_server_manager.get_registry()
|
||||
registered_names: Final = set(registry.keys())
|
||||
|
||||
return frozenset(requested_servers - registered_names)
|
||||
not_servers: Final = frozenset(
|
||||
name
|
||||
for name in requested_servers
|
||||
if global_mcp_server_manager.get_mcp_server_by_identifier(name) is None
|
||||
)
|
||||
if prisma_client is None:
|
||||
return not_servers
|
||||
return frozenset(
|
||||
[
|
||||
name
|
||||
for name in not_servers
|
||||
if await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name) is None
|
||||
]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,22 +7,25 @@ instead of writing immediately on each request.
|
|||
|
||||
from collections.abc import AsyncGenerator, Callable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, TypeVar
|
||||
|
||||
from fastapi import HTTPException
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import (
|
||||
decrypt_value_helper,
|
||||
encrypt_value_helper,
|
||||
)
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.types.llms.openai import (
|
||||
BaseLiteLLMOpenAIResponseObject,
|
||||
ResponsesAPIResponse,
|
||||
)
|
||||
from litellm.types.responses.main import DeleteResponseResult
|
||||
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypesLiteral, LLMResponseTypes, SpecialEnums
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -67,6 +70,8 @@ def _proxy_signing_key() -> str | None:
|
|||
|
||||
|
||||
_RESPONSE_PAYLOAD_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_ResponseObjectT = TypeVar("_ResponseObjectT", bound=BaseLiteLLMOpenAIResponseObject)
|
||||
_AddressedResultT = TypeVar("_AddressedResultT", bound=ResponsesAPIResponse | DeleteResponseResult)
|
||||
|
||||
|
||||
def _response_payload(response_obj: object) -> Mapping[str, object] | None:
|
||||
|
|
@ -77,9 +82,9 @@ def _response_payload(response_obj: object) -> Mapping[str, object] | None:
|
|||
|
||||
|
||||
def _rewrite_advertised_id(
|
||||
event: BaseLiteLLMOpenAIResponseObject,
|
||||
event: _ResponseObjectT,
|
||||
rewrite: Callable[[str], str],
|
||||
) -> BaseLiteLLMOpenAIResponseObject:
|
||||
) -> _ResponseObjectT:
|
||||
event_id: Final = getattr(event, "id", None)
|
||||
if isinstance(event_id, str) and event_id.startswith("resp_"):
|
||||
setattr(event, "id", rewrite(event_id))
|
||||
|
|
@ -116,6 +121,35 @@ def _streams_response_ids(request_route: str | None) -> bool:
|
|||
return canonical in _RESPONSES_API_ROUTES or (parent in _RESPONSES_API_ROUTES and response_id != "")
|
||||
|
||||
|
||||
_NO_HELD_IDS: Final[Mapping[str, str]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _upstream_response_id(response_id: str) -> str:
|
||||
return ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(response_id)
|
||||
|
||||
|
||||
def _names_the_same_response(response_id: str | None, original_response_id: str) -> bool:
|
||||
return response_id is not None and _upstream_response_id(response_id) == _upstream_response_id(original_response_id)
|
||||
|
||||
|
||||
def _held_id_naming(response_id: str | None, held_ids: Mapping[str, str]) -> str | None:
|
||||
return next(
|
||||
(
|
||||
held_id
|
||||
for original_response_id, held_id in held_ids.items()
|
||||
if _names_the_same_response(response_id, original_response_id)
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _id_sent_back(response: ResponsesAPIResponse | DeleteResponseResult, held_ids: Mapping[str, str]) -> str | None:
|
||||
held_id: Final = _held_id_naming(response.id, held_ids)
|
||||
if held_id is not None:
|
||||
return held_id
|
||||
return response.id if isinstance(response, DeleteResponseResult) else None
|
||||
|
||||
|
||||
class ResponsesIDSecurity(CustomLogger):
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -275,10 +309,18 @@ class ResponsesIDSecurity(CustomLogger):
|
|||
|
||||
def _encrypt_response_id(
|
||||
self,
|
||||
response: BaseLiteLLMOpenAIResponseObject,
|
||||
response: _ResponseObjectT,
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
request_cache: dict[str, str] | None = None,
|
||||
) -> BaseLiteLLMOpenAIResponseObject:
|
||||
) -> _ResponseObjectT:
|
||||
encrypt: Final = self._response_id_encrypter(user_api_key_dict, request_cache)
|
||||
return response if encrypt is None else _rewrite_advertised_id(response, encrypt)
|
||||
|
||||
def _response_id_encrypter(
|
||||
self,
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
request_cache: dict[str, str] | None,
|
||||
) -> Callable[[str], str] | None:
|
||||
signing_key: Final = self._get_signing_key()
|
||||
if signing_key is None:
|
||||
verbose_proxy_logger.debug(
|
||||
|
|
@ -287,7 +329,7 @@ class ResponsesIDSecurity(CustomLogger):
|
|||
"Skipping response ID encryption. "
|
||||
"See: https://docs.litellm.ai/docs/proxy/prod#5-set-litellm-salt-key"
|
||||
)
|
||||
return response
|
||||
return None
|
||||
|
||||
def encrypt(original_id: str) -> str:
|
||||
cached: Final = request_cache.get(original_id) if request_cache is not None else None
|
||||
|
|
@ -304,47 +346,73 @@ class ResponsesIDSecurity(CustomLogger):
|
|||
request_cache[original_id] = encrypted_id
|
||||
return encrypted_id
|
||||
|
||||
return _rewrite_advertised_id(response, encrypt)
|
||||
return encrypt
|
||||
|
||||
def _streamed_response_id_advertiser(
|
||||
self,
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
request_data: Mapping[str, object],
|
||||
) -> Callable[[str], str] | None:
|
||||
if not _streams_response_ids(user_api_key_dict.request_route):
|
||||
return None
|
||||
if self._general_settings_reader().get("disable_responses_id_security", False):
|
||||
return None
|
||||
encrypt: Final = self._response_id_encrypter(user_api_key_dict, request_cache={})
|
||||
if encrypt is None:
|
||||
return None
|
||||
logging_obj: Final = request_data.get("litellm_logging_obj")
|
||||
held_ids: Final = self._ids_the_caller_holds(request_data.get(ADDRESSED_RESPONSE_ID_FIELD))
|
||||
|
||||
def advertise(original_id: str) -> str:
|
||||
logged_id: Final = encrypt(original_id)
|
||||
if isinstance(logging_obj, LiteLLMLoggingObj):
|
||||
logging_obj.client_facing_response_id = logged_id
|
||||
return _held_id_naming(original_id, held_ids) or logged_id
|
||||
|
||||
return advertise
|
||||
|
||||
def _ids_the_caller_holds(self, addressed_id: object) -> Mapping[str, str]:
|
||||
if not isinstance(addressed_id, str) or not self._is_encrypted_response_id(addressed_id):
|
||||
return _NO_HELD_IDS
|
||||
original_response_id, _, _ = self._decrypt_response_id(addressed_id)
|
||||
return MappingProxyType({original_response_id: addressed_id})
|
||||
|
||||
def _advertised_response(
|
||||
self,
|
||||
response: _AddressedResultT,
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
request_data: Mapping[str, object],
|
||||
) -> _AddressedResultT:
|
||||
sent_back_id: Final = _id_sent_back(
|
||||
response, self._ids_the_caller_holds(request_data.get(ADDRESSED_RESPONSE_ID_FIELD))
|
||||
)
|
||||
logged_response: Final = self._encrypt_response_id(response, user_api_key_dict, request_cache=None)
|
||||
return logged_response if sent_back_id is None else logged_response.model_copy(update={"id": sent_back_id})
|
||||
|
||||
async def async_post_call_success_hook(
|
||||
self,
|
||||
data: dict,
|
||||
data: dict[str, object],
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
response: LLMResponseTypes,
|
||||
) -> LLMResponseTypes:
|
||||
"""
|
||||
Queue response IDs for batch processing instead of writing directly to DB.
|
||||
|
||||
This method adds response IDs to an in-memory queue, which are then
|
||||
batch-processed by the DBSpendUpdateWriter during regular database update cycles.
|
||||
"""
|
||||
general_settings: Final = self._general_settings_reader()
|
||||
|
||||
if general_settings.get("disable_responses_id_security", False):
|
||||
return response
|
||||
if isinstance(response, ResponsesAPIResponse):
|
||||
response = cast(
|
||||
ResponsesAPIResponse,
|
||||
self._encrypt_response_id(response, user_api_key_dict, request_cache=None),
|
||||
)
|
||||
if isinstance(response, ResponsesAPIResponse | DeleteResponseResult):
|
||||
return self._advertised_response(response, user_api_key_dict, data)
|
||||
return response
|
||||
|
||||
async def async_post_call_streaming_iterator_hook(
|
||||
self, user_api_key_dict: "UserAPIKeyAuth", response: Any, request_data: dict
|
||||
self, user_api_key_dict: "UserAPIKeyAuth", response: Any, request_data: dict[str, object]
|
||||
) -> AsyncGenerator[BaseLiteLLMOpenAIResponseObject, None]:
|
||||
general_settings: Final = self._general_settings_reader()
|
||||
|
||||
# Create a request-scoped cache for consistent encryption across streaming chunks.
|
||||
request_encryption_cache: Final[dict[str, str]] = {}
|
||||
|
||||
advertise: Final = self._streamed_response_id_advertiser(user_api_key_dict, request_data)
|
||||
async for chunk in response:
|
||||
if (
|
||||
isinstance(chunk, BaseLiteLLMOpenAIResponseObject)
|
||||
and _streams_response_ids(user_api_key_dict.request_route)
|
||||
and not general_settings.get("disable_responses_id_security", False)
|
||||
):
|
||||
chunk = self._encrypt_response_id(chunk, user_api_key_dict, request_encryption_cache)
|
||||
yield chunk
|
||||
yield (
|
||||
_rewrite_advertised_id(chunk, advertise)
|
||||
if advertise is not None and isinstance(chunk, BaseLiteLLMOpenAIResponseObject)
|
||||
else chunk
|
||||
)
|
||||
|
||||
|
||||
def decrypted_response_id(response_id: str) -> str:
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
from litellm.proxy.route_llm_request import route_request
|
||||
from litellm.types.images.main import ImageEditRequestParams
|
||||
from litellm.types.llms.openai import ChatCompletionUserMessage
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
router: Final = APIRouter()
|
||||
_IMAGE_GENERATION_MODEL_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object)
|
||||
|
|
@ -221,7 +222,10 @@ async def image_generation(
|
|||
return response
|
||||
except Exception as e:
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=e,
|
||||
request_data=data,
|
||||
dispatched_call_type=CallTypes.aimage_generation.value,
|
||||
)
|
||||
log_llm_api_exception(e, litellm_call_id)
|
||||
if isinstance(e, HTTPException):
|
||||
|
|
|
|||
|
|
@ -109,6 +109,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
|
|||
can_org_access_model,
|
||||
delete_cache_key_objects,
|
||||
delete_cache_team_object,
|
||||
evict_budget_windows_absent_marker,
|
||||
get_jwt_key_mapping_cache_keys_for_tokens,
|
||||
get_org_object,
|
||||
get_team_membership,
|
||||
|
|
@ -612,6 +613,9 @@ async def refresh_cached_team(
|
|||
await publish_auth_cache_invalidation(cache_key=f"team_id:{team_row.team_id}")
|
||||
if team_table.team_alias is not None:
|
||||
await publish_auth_cache_invalidation(cache_key=f"team_alias:{team_table.team_alias}")
|
||||
await evict_budget_windows_absent_marker(
|
||||
entity_type=Litellm_EntityType.TEAM, entity_id=team_row.team_id, user_api_key_cache=user_api_key_cache
|
||||
)
|
||||
|
||||
|
||||
_refresh_cached_team: Final = refresh_cached_team
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import httpx
|
|||
from fastapi import FastAPI, HTTPException
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError
|
||||
from starlette.requests import Request as StarletteRequest
|
||||
from starlette.responses import JSONResponse
|
||||
from starlette.responses import JSONResponse, Response
|
||||
from starlette.routing import Route
|
||||
from starlette.types import Message, Receive, Scope, Send
|
||||
|
||||
|
|
@ -2489,6 +2489,9 @@ if MCP_AVAILABLE:
|
|||
},
|
||||
)(scope, receive, send)
|
||||
return
|
||||
if scope.get("method") == "OPTIONS":
|
||||
await Response(status_code=204, headers={"Allow": "GET, POST, DELETE, OPTIONS"})(scope, receive, send)
|
||||
return
|
||||
path: Final = _scope_path(scope)
|
||||
peek: Final = _LazyPeekedBody(receive) if scope.get("method") == "POST" else None
|
||||
if peek is not None:
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from collections.abc import (
|
|||
Collection,
|
||||
Container,
|
||||
Iterable,
|
||||
Iterator,
|
||||
Mapping,
|
||||
MutableMapping,
|
||||
Sequence,
|
||||
|
|
@ -90,6 +91,7 @@ from litellm.constants import (
|
|||
LITELLM_UI_SESSION_DURATION,
|
||||
OPENAI_LIVE_SESSION_START_TIMEOUT_SECONDS,
|
||||
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
|
||||
VLLM_CHOICE_FIELDS,
|
||||
)
|
||||
from litellm.litellm_core_utils.access_windows import access_windows_config_error
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
||||
|
|
@ -107,6 +109,7 @@ from litellm.litellm_core_utils.litellm_logging import (
|
|||
initialize_sentry,
|
||||
)
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER, LoggingWorker
|
||||
from litellm.litellm_core_utils.prompt_templates.chat_template_tools import chat_template_tool_section
|
||||
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
|
||||
from litellm.litellm_core_utils.routing_groups import parse_routing_groups
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
|
|
@ -115,6 +118,7 @@ from litellm.litellm_core_utils.token_counter import (
|
|||
UnsupportedContentTypeError,
|
||||
deployment_openai_model,
|
||||
offload_token_count,
|
||||
sends_function_tools,
|
||||
)
|
||||
from litellm.llms.bedrock.count_tokens.kimi_k3 import (
|
||||
HUGGING_FACE_SOURCE,
|
||||
|
|
@ -203,6 +207,7 @@ from litellm.rust_bridge.catalog import Target
|
|||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
from litellm.types.utils import (
|
||||
PRICING_OVERRIDES_KEY,
|
||||
CallTypes,
|
||||
CountContextManagement,
|
||||
LlmProviders,
|
||||
ModelResponse,
|
||||
|
|
@ -585,6 +590,7 @@ from litellm.proxy.common_utils.token_count_log import (
|
|||
)
|
||||
from litellm.proxy.common_utils.user_api_key_cache import (
|
||||
UserApiKeyCache,
|
||||
budget_windows_absent_cache_key,
|
||||
end_user_cache_key,
|
||||
get_management_object_ttl,
|
||||
model_access_group_cache_key,
|
||||
|
|
@ -1007,6 +1013,7 @@ from litellm.proxy.video_endpoints.endpoints import router as video_router
|
|||
from litellm.repositories.base_repository import SupportsModelDump
|
||||
from litellm.repositories.credentials_repository import CredentialsRepository
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.responses.main import provider_stores_the_conversation
|
||||
from litellm.responses.streaming_iterator import drain_logging_tasks
|
||||
from litellm.router import (
|
||||
AssistantsTypedDict,
|
||||
|
|
@ -3632,6 +3639,52 @@ def _cached_budget_limits(cached: object) -> object:
|
|||
return _UNTYPED_VALUE.validate_python(json.loads(stored)) if isinstance(stored, str) else stored
|
||||
|
||||
|
||||
def _has_budget_windows(budget_limits: object) -> bool:
|
||||
return isinstance(budget_limits, list) and budget_limits != []
|
||||
|
||||
|
||||
def _budget_windows_absent_marker(absent_key: str) -> object:
|
||||
return _UNTYPED_VALUE.validate_python(user_api_key_cache.budget_windows_absent_cache.get_cache(key=absent_key))
|
||||
|
||||
|
||||
def _finish_pending_read(absent_key: str, pending_read: str, remember_absent: bool) -> None:
|
||||
if _budget_windows_absent_marker(absent_key) != pending_read:
|
||||
return
|
||||
if remember_absent:
|
||||
user_api_key_cache.budget_windows_absent_cache.set_cache(key=absent_key, value=True)
|
||||
return
|
||||
user_api_key_cache.budget_windows_absent_cache.delete_cache(absent_key)
|
||||
|
||||
|
||||
async def _stored_budget_limits(entity_type: Litellm_EntityType, entity_id: str) -> object:
|
||||
"""Read only: caching the row here could race a delete and put a just-deleted key back into the auth cache."""
|
||||
if prisma_client is None:
|
||||
return None
|
||||
absent_key: Final = budget_windows_absent_cache_key(entity_type.value, entity_id)
|
||||
if _budget_windows_absent_marker(absent_key) is True:
|
||||
return None
|
||||
pending_read: Final = f"reading:{secrets.token_hex(8)}"
|
||||
user_api_key_cache.budget_windows_absent_cache.set_cache(key=absent_key, value=pending_read)
|
||||
try:
|
||||
row: Final[SupportsModelDump | None] = (
|
||||
await VerificationTokenRepository(prisma_client).table.find_unique(where={"token": entity_id})
|
||||
if entity_type == Litellm_EntityType.KEY
|
||||
else await TeamRepository(prisma_client).table.find_unique(where={"team_id": entity_id})
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # spend tracking must never fail the cost callback
|
||||
verbose_proxy_logger.warning(
|
||||
"Unable to read budget windows for %s=%s; this request's window spend is not counted: %s",
|
||||
entity_type.value,
|
||||
entity_id,
|
||||
e,
|
||||
)
|
||||
_finish_pending_read(absent_key, pending_read, remember_absent=False)
|
||||
return None
|
||||
budget_limits: Final[object] = None if row is None else _cached_budget_limits(row)
|
||||
_finish_pending_read(absent_key, pending_read, remember_absent=not _has_budget_windows(budget_limits))
|
||||
return budget_limits
|
||||
|
||||
|
||||
async def _increment_spend_counters_batched(
|
||||
token: str | None,
|
||||
team_id: str | None,
|
||||
|
|
@ -3662,7 +3715,68 @@ async def _increment_spend_counters_batched(
|
|||
|
||||
cost: Final[float] = response_cost
|
||||
|
||||
async def _key_scope(key_token: str) -> tuple[PendingSpendIncrement | BaseException, ...]:
|
||||
async def _window_increment(
|
||||
entity_type: Litellm_EntityType, entity_id: str, window: object
|
||||
) -> PendingSpendIncrement | None:
|
||||
duration: Final = (
|
||||
window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None)
|
||||
)
|
||||
window_reset_at: Final = (
|
||||
window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None)
|
||||
)
|
||||
window_counter: Final = f"spend:{entity_type.value}:{entity_id}:window:{duration}"
|
||||
window_start: Final = get_budget_window_start(window)
|
||||
pending_window: Final = (
|
||||
await _prepare_window_spend_counter_increment(
|
||||
counter_key=window_counter,
|
||||
entity_type=entity_type.value.capitalize(),
|
||||
entity_id=entity_id,
|
||||
window_duration=duration,
|
||||
window_start=window_start,
|
||||
increment=cost,
|
||||
)
|
||||
if window_counter not in reserved_counter_keys
|
||||
else None
|
||||
)
|
||||
await _enqueue_window_spend_row_update(
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
reset_at=window_reset_at,
|
||||
window_duration=duration,
|
||||
window_start=window_start,
|
||||
increment=cost,
|
||||
request_started_at=request_started_at,
|
||||
)
|
||||
return pending_window
|
||||
|
||||
async def _window_increments(
|
||||
entity_type: Litellm_EntityType, entity_id: str, budget_limits: object
|
||||
) -> tuple[PendingSpendIncrement | BaseException, ...]:
|
||||
if not isinstance(budget_limits, list):
|
||||
return ()
|
||||
window_pending: Final = await asyncio.gather(
|
||||
*(_window_increment(entity_type, entity_id, window) for window in budget_limits), return_exceptions=True
|
||||
)
|
||||
return tuple(item for item in window_pending if item is not None)
|
||||
|
||||
async def _cached_window_increments(
|
||||
entity_type: Litellm_EntityType, entity_id: str, cache_key: str
|
||||
) -> tuple[_SpendScopeItem, ...]:
|
||||
cached: Final[object] = _UNTYPED_VALUE.validate_python(await user_api_key_cache.async_get_cache(key=cache_key))
|
||||
if cached is None:
|
||||
return (_UncachedBudgetWindows(entity_type=entity_type, entity_id=entity_id),)
|
||||
return await _window_increments(entity_type, entity_id, _cached_budget_limits(cached))
|
||||
|
||||
async def _stored_window_increments(
|
||||
uncached: _UncachedBudgetWindows,
|
||||
) -> tuple[PendingSpendIncrement | BaseException, ...]:
|
||||
return await _window_increments(
|
||||
uncached.entity_type,
|
||||
uncached.entity_id,
|
||||
await _stored_budget_limits(uncached.entity_type, uncached.entity_id),
|
||||
)
|
||||
|
||||
async def _key_scope(key_token: str) -> tuple[_SpendScopeItem, ...]:
|
||||
# key_token arrives pre-hashed from metadata["user_api_key"] (auth flow
|
||||
# hashes raw "sk-..." keys before they reach the callback). The
|
||||
# startswith("sk-") check is a safety net matching update_cache —
|
||||
|
|
@ -3685,52 +3799,9 @@ async def _increment_spend_counters_batched(
|
|||
)
|
||||
)
|
||||
|
||||
async def _key_window_increment(window: object) -> PendingSpendIncrement | None:
|
||||
duration = (
|
||||
window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None)
|
||||
)
|
||||
key_window_reset_at = (
|
||||
window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None)
|
||||
)
|
||||
key_window_counter: Final = f"spend:key:{hashed_token}:window:{duration}"
|
||||
key_window_start = get_budget_window_start(window)
|
||||
pending_window: Final = (
|
||||
await _prepare_window_spend_counter_increment(
|
||||
counter_key=key_window_counter,
|
||||
entity_type="Key",
|
||||
entity_id=hashed_token,
|
||||
window_duration=duration,
|
||||
window_start=key_window_start,
|
||||
increment=cost,
|
||||
)
|
||||
if key_window_counter not in reserved_counter_keys
|
||||
else None
|
||||
)
|
||||
await _enqueue_window_spend_row_update(
|
||||
entity_type=Litellm_EntityType.KEY,
|
||||
entity_id=hashed_token,
|
||||
reset_at=key_window_reset_at,
|
||||
window_duration=duration,
|
||||
window_start=key_window_start,
|
||||
increment=cost,
|
||||
request_started_at=request_started_at,
|
||||
)
|
||||
return pending_window
|
||||
return key_pending + await _cached_window_increments(Litellm_EntityType.KEY, hashed_token, hashed_token)
|
||||
|
||||
key_obj: Final[object] = _UNTYPED_VALUE.validate_python(
|
||||
await user_api_key_cache.async_get_cache(key=hashed_token)
|
||||
)
|
||||
if key_obj is None:
|
||||
return key_pending
|
||||
key_budget_limits: Final = _cached_budget_limits(key_obj)
|
||||
if not isinstance(key_budget_limits, list):
|
||||
return key_pending
|
||||
window_pending: Final = await asyncio.gather(
|
||||
*(_key_window_increment(window) for window in key_budget_limits), return_exceptions=True
|
||||
)
|
||||
return key_pending + tuple(item for item in window_pending if item is not None)
|
||||
|
||||
async def _team_scope(scope_team_id: str) -> tuple[PendingSpendIncrement | BaseException, ...]:
|
||||
async def _team_scope(scope_team_id: str) -> tuple[_SpendScopeItem, ...]:
|
||||
team_counter_key: Final = f"spend:team:{scope_team_id}"
|
||||
team_pending: Final[tuple[PendingSpendIncrement, ...]] = (
|
||||
()
|
||||
|
|
@ -3744,50 +3815,9 @@ async def _increment_spend_counters_batched(
|
|||
)
|
||||
)
|
||||
|
||||
async def _team_window_increment(window: object) -> PendingSpendIncrement | None:
|
||||
duration = (
|
||||
window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None)
|
||||
)
|
||||
team_window_reset_at = (
|
||||
window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None)
|
||||
)
|
||||
team_window_counter: Final = f"spend:team:{scope_team_id}:window:{duration}"
|
||||
team_window_start = get_budget_window_start(window)
|
||||
pending_window: Final = (
|
||||
await _prepare_window_spend_counter_increment(
|
||||
counter_key=team_window_counter,
|
||||
entity_type="Team",
|
||||
entity_id=scope_team_id,
|
||||
window_duration=duration,
|
||||
window_start=team_window_start,
|
||||
increment=cost,
|
||||
)
|
||||
if team_window_counter not in reserved_counter_keys
|
||||
else None
|
||||
)
|
||||
await _enqueue_window_spend_row_update(
|
||||
entity_type=Litellm_EntityType.TEAM,
|
||||
entity_id=scope_team_id,
|
||||
reset_at=team_window_reset_at,
|
||||
window_duration=duration,
|
||||
window_start=team_window_start,
|
||||
increment=cost,
|
||||
request_started_at=request_started_at,
|
||||
)
|
||||
return pending_window
|
||||
|
||||
team_obj: Final[object] = _UNTYPED_VALUE.validate_python(
|
||||
await user_api_key_cache.async_get_cache(key=f"team_id:{scope_team_id}")
|
||||
return team_pending + await _cached_window_increments(
|
||||
Litellm_EntityType.TEAM, scope_team_id, f"team_id:{scope_team_id}"
|
||||
)
|
||||
if team_obj is None:
|
||||
return team_pending
|
||||
team_budget_limits: Final = _cached_budget_limits(team_obj)
|
||||
if not isinstance(team_budget_limits, list):
|
||||
return team_pending
|
||||
window_pending: Final = await asyncio.gather(
|
||||
*(_team_window_increment(window) for window in team_budget_limits), return_exceptions=True
|
||||
)
|
||||
return team_pending + tuple(item for item in window_pending if item is not None)
|
||||
|
||||
async def _team_member_scope(
|
||||
scope_user_id: str, scope_team_id: str
|
||||
|
|
@ -3858,22 +3888,30 @@ async def _increment_spend_counters_batched(
|
|||
# return_exceptions so a failing scope does not leave its siblings running
|
||||
# as orphaned tasks that race the caller's reservation-counter invalidation;
|
||||
# all scopes settle, then the first error propagates as before.
|
||||
scope_results: Final = await asyncio.gather(*scope_coros, return_exceptions=True)
|
||||
scope_errors: Final = tuple(
|
||||
item
|
||||
for scope in scope_results
|
||||
for item in (scope if isinstance(scope, tuple) else (scope,))
|
||||
if isinstance(item, BaseException)
|
||||
scope_items: Final = tuple(_flatten_spend_scopes(await asyncio.gather(*scope_coros, return_exceptions=True)))
|
||||
uncached_windows: Final = tuple(item for item in scope_items if isinstance(item, _UncachedBudgetWindows))
|
||||
await _apply_spend_counter_increments(
|
||||
pending=reservation_update.pending
|
||||
+ tuple(item for item in scope_items if isinstance(item, PendingSpendIncrement))
|
||||
)
|
||||
pending: Final = tuple(
|
||||
item
|
||||
for scope in scope_results
|
||||
if not isinstance(scope, BaseException)
|
||||
for item in scope
|
||||
if not isinstance(item, BaseException)
|
||||
stored_window_items: Final = (
|
||||
tuple(
|
||||
_flatten_spend_scopes(
|
||||
await asyncio.gather(
|
||||
*(_stored_window_increments(uncached) for uncached in uncached_windows), return_exceptions=True
|
||||
)
|
||||
)
|
||||
)
|
||||
if uncached_windows
|
||||
else ()
|
||||
)
|
||||
await _apply_spend_counter_increments(pending=reservation_update.pending + pending)
|
||||
stored_window_pending: Final = tuple(
|
||||
item for item in stored_window_items if isinstance(item, PendingSpendIncrement)
|
||||
)
|
||||
if stored_window_pending:
|
||||
await _apply_spend_counter_increments(pending=stored_window_pending)
|
||||
stamp_budget_reservation_actual_cost(budget_reservation=budget_reservation, actual_cost=response_cost)
|
||||
scope_errors: Final = tuple(item for item in scope_items + stored_window_items if isinstance(item, BaseException))
|
||||
if scope_errors:
|
||||
raise scope_errors[0]
|
||||
|
||||
|
|
@ -3881,6 +3919,25 @@ async def _increment_spend_counters_batched(
|
|||
budget_reservation["finalized"] = True
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _UncachedBudgetWindows:
|
||||
entity_type: Litellm_EntityType
|
||||
entity_id: str
|
||||
|
||||
|
||||
_SpendScopeItem: TypeAlias = PendingSpendIncrement | BaseException | _UncachedBudgetWindows
|
||||
|
||||
|
||||
def _flatten_spend_scopes(
|
||||
scope_results: Sequence[tuple[_SpendScopeItem, ...] | BaseException],
|
||||
) -> Iterator[_SpendScopeItem]:
|
||||
for scope in scope_results:
|
||||
if isinstance(scope, BaseException):
|
||||
yield scope
|
||||
continue
|
||||
yield from scope
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ReservationCounterUpdate:
|
||||
"""The reserved counters the direct increment must skip, and the adjustments that settle them on the actual
|
||||
|
|
@ -5661,11 +5718,18 @@ def _with_config_file_pass_through_endpoints(
|
|||
def _reload_settings_store(section: Section, store: SettingsStore, section_config: object) -> None:
|
||||
serving_pass_throughs: Final = store.get("pass_through_endpoints")
|
||||
store.load_yaml(_as_settings_mapping(section_config))
|
||||
store.apply_db_row(section, _EMPTY_SETTINGS_MAPPING)
|
||||
if is_resource_list(section, "pass_through_endpoints") and serving_pass_throughs is not None:
|
||||
store["pass_through_endpoints"] = serving_pass_throughs
|
||||
|
||||
|
||||
def _general_settings_row_is_readable(row: _ConfigParamValueRow | None) -> bool:
|
||||
return row is None or row.param_value is None or isinstance(row.param_value, Mapping)
|
||||
|
||||
|
||||
def _db_managed_general_settings_keys() -> frozenset[str]:
|
||||
return frozenset((*ConfigGeneralSettings.model_fields, *_CLEANUP_SCHEDULE_KEYS, "allowed_ips", "email_settings"))
|
||||
|
||||
|
||||
def _bind_general_settings_store(settings: SettingsStore) -> None:
|
||||
global general_settings
|
||||
general_settings = settings # pyright: ignore[reportAssignmentType] # legacy global accepts mappings
|
||||
|
|
@ -5816,7 +5880,9 @@ class ProxyConfig:
|
|||
# whether an existing request predates the prices it just fetched, and re-serving one
|
||||
# costs a single fetch where skipping one leaves it priced wrong indefinitely
|
||||
self.model_cost_map_applied_revision: int = 0
|
||||
self.settings: Final[SettingsStore] = SettingsStore("general_settings")
|
||||
self.settings: Final[SettingsStore] = SettingsStore(
|
||||
"general_settings", managed_db_keys=_db_managed_general_settings_keys()
|
||||
)
|
||||
self.router_settings: Final[SettingsStore] = SettingsStore("router_settings")
|
||||
self.litellm_settings: Final[SettingsStore] = SettingsStore("litellm_settings")
|
||||
self.environment_variables: Final[SettingsStore] = SettingsStore("environment_variables")
|
||||
|
|
@ -5827,6 +5893,8 @@ class ProxyConfig:
|
|||
self._db_environment_keys: frozenset[str] = frozenset()
|
||||
self._sso_environment_removals: frozenset[str] = frozenset()
|
||||
self._warned_shadowed_keys: frozenset[tuple[Section, str]] = frozenset()
|
||||
self._warned_unmanaged_keys: frozenset[str] = frozenset()
|
||||
self._warned_unreadable_db_general_settings: bool = False
|
||||
self._db_config_callback_entries: Mapping[tuple[str, str], _DbCallbackRegistration] = MappingProxyType({})
|
||||
self._settings_stores: Final[Mapping[Section, SettingsStore]] = MappingProxyType(
|
||||
{
|
||||
|
|
@ -6472,6 +6540,9 @@ class ProxyConfig:
|
|||
prisma_client=prisma_client,
|
||||
store_model_in_db=store_model_in_db,
|
||||
)
|
||||
else:
|
||||
for section, store in self._settings_stores.items():
|
||||
store.apply_db_row(section, _EMPTY_SETTINGS_MAPPING)
|
||||
|
||||
config = self._config_with_resolved_settings(config)
|
||||
|
||||
|
|
@ -8350,15 +8421,20 @@ class ProxyConfig:
|
|||
general_settings.get("maximum_spend_logs_retention_interval", "1d"),
|
||||
)
|
||||
|
||||
async def _update_general_settings(self, db_general_settings: Mapping[str, SettingsJsonValue] | None) -> None:
|
||||
async def _update_general_settings(self, db_general_settings: Mapping[str, SettingsJsonValue] | str | None) -> None:
|
||||
global general_settings
|
||||
if db_general_settings is None:
|
||||
return
|
||||
if not isinstance(db_general_settings, Mapping):
|
||||
self._warn_about_unreadable_db_general_settings()
|
||||
return
|
||||
self._warned_unreadable_db_general_settings = False
|
||||
if not isinstance(general_settings, SettingsStore):
|
||||
self.settings.load_yaml(_as_settings_mapping(general_settings))
|
||||
cache_size_was_db: Final = self.settings.source("user_api_key_cache_max_size") == "db"
|
||||
previous_cleanup_schedule: Final = self._resolved_cleanup_schedule()
|
||||
self.settings.apply_db_row("general_settings", db_general_settings)
|
||||
self._warn_about_unmanaged_db_settings()
|
||||
_bind_general_settings_store(self.settings)
|
||||
await self._apply_general_settings_side_effects(
|
||||
db_general_settings,
|
||||
|
|
@ -8550,12 +8626,24 @@ class ProxyConfig:
|
|||
environment_read: Final = next(self._environment_read_numbers)
|
||||
sections: Final = tuple(self._settings_stores)
|
||||
responses: Final = await asyncio.gather(*(get_config_param(prisma_client, section) for section in sections))
|
||||
general_settings_response: Final = responses[sections.index("general_settings")]
|
||||
general_settings_value: Final[Mapping[str, SettingsJsonValue]] = (
|
||||
self._prepared_db_settings_values("general_settings", general_settings_response.param_value)
|
||||
if general_settings_response is not None and general_settings_response.param_value is not None
|
||||
else _EMPTY_SETTINGS_MAPPING
|
||||
)
|
||||
self._settings_stores["general_settings"].apply_db_row("general_settings", general_settings_value)
|
||||
|
||||
response: _ConfigParamValueRow | None
|
||||
for section, response in zip(sections, responses):
|
||||
if section == "general_settings":
|
||||
continue
|
||||
if response is None:
|
||||
self._settings_stores[section].apply_db_row(section, _EMPTY_SETTINGS_MAPPING)
|
||||
continue
|
||||
param_value: JsonValue = getattr(response, "param_value", None)
|
||||
if param_value is None:
|
||||
self._settings_stores[section].apply_db_row(section, _EMPTY_SETTINGS_MAPPING)
|
||||
continue
|
||||
|
||||
verbose_proxy_logger.debug(
|
||||
|
|
@ -8566,6 +8654,7 @@ class ProxyConfig:
|
|||
if section == "litellm_settings":
|
||||
self._apply_litellm_settings_db_values(self._prepared_db_settings_values(section, param_value))
|
||||
elif section == "environment_variables":
|
||||
self._settings_stores[section].apply_db_row(section, _EMPTY_SETTINGS_MAPPING)
|
||||
await self._apply_db_environment_variables_row(
|
||||
prisma_client, response, environment_before_db, environment_read
|
||||
)
|
||||
|
|
@ -8576,6 +8665,7 @@ class ProxyConfig:
|
|||
)
|
||||
|
||||
self._warn_about_shadowed_db_settings()
|
||||
self._warn_about_unmanaged_db_settings()
|
||||
return self._config_with_resolved_settings(config)
|
||||
|
||||
async def _apply_db_environment_variables_row(
|
||||
|
|
@ -8634,6 +8724,30 @@ class ProxyConfig:
|
|||
)
|
||||
self._warned_shadowed_keys = shadowed
|
||||
|
||||
def _warn_about_unmanaged_db_settings(self) -> None:
|
||||
unmanaged: Final = frozenset(
|
||||
key for key in self.settings.unmanaged_db_keys() if key not in _GENERAL_SETTINGS_UI_LITELLM_FIELDS
|
||||
)
|
||||
for key in sorted(unmanaged - self._warned_unmanaged_keys):
|
||||
verbose_proxy_logger.warning(
|
||||
"general_settings.%s is stored in the database, but the proxy does not apply that stored value, so it "
|
||||
"is ignored. %s",
|
||||
key,
|
||||
"The config file's value applies. Remove the stored copy with POST /config/field/delete."
|
||||
if self.settings.owned_by_config(key)
|
||||
else "Set it in the config file to use it, or remove it with POST /config/field/delete.",
|
||||
)
|
||||
self._warned_unmanaged_keys = unmanaged
|
||||
|
||||
def _warn_about_unreadable_db_general_settings(self) -> None:
|
||||
if self._warned_unreadable_db_general_settings:
|
||||
return
|
||||
verbose_proxy_logger.warning(
|
||||
"general_settings is stored in the database as something other than a JSON object, so the proxy "
|
||||
"ignores every stored general setting."
|
||||
)
|
||||
self._warned_unreadable_db_general_settings = True
|
||||
|
||||
def _prepared_db_settings_values(self, section: Section, value: object) -> Mapping[str, SettingsJsonValue]:
|
||||
scrubbed: Final = _scrub_db_overlay_remote_module_loads(section=section, db_value=value)
|
||||
return _as_settings_mapping(scrubbed)
|
||||
|
|
@ -8801,7 +8915,10 @@ class ProxyConfig:
|
|||
)
|
||||
|
||||
# initialize vector stores, guardrails, etc. table in db
|
||||
await self._init_non_llm_objects_in_db(prisma_client=prisma_client)
|
||||
await self._init_non_llm_objects_in_db(
|
||||
prisma_client=prisma_client,
|
||||
refresh_pass_through_endpoints=_general_settings_row_is_readable(db_general_settings),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - %s", e)
|
||||
|
|
@ -8870,7 +8987,9 @@ class ProxyConfig:
|
|||
except Exception as e: # noqa: BLE001 # best-effort: a failing stop must not break proxy shutdown
|
||||
verbose_proxy_logger.error("Error stopping auth cache invalidation subscriber: %s", e)
|
||||
|
||||
async def _init_non_llm_objects_in_db(self, prisma_client: PrismaClient):
|
||||
async def _init_non_llm_objects_in_db(
|
||||
self, prisma_client: PrismaClient, *, refresh_pass_through_endpoints: bool = True
|
||||
):
|
||||
"""
|
||||
Use this to read non-llm objects from the db and initialize them
|
||||
|
||||
|
|
@ -8894,7 +9013,7 @@ class ProxyConfig:
|
|||
if self._should_load_db_object(object_type="agents"):
|
||||
await self._init_agents_in_db(prisma_client=prisma_client)
|
||||
|
||||
if self._should_load_db_object(object_type="pass_through_endpoints"):
|
||||
if refresh_pass_through_endpoints and self._should_load_db_object(object_type="pass_through_endpoints"):
|
||||
await self._init_pass_through_endpoints_in_db()
|
||||
|
||||
if self._should_load_db_object(object_type="prompts"):
|
||||
|
|
@ -10223,6 +10342,11 @@ def _restamp_nested_response_model_in_sse_line(line: bytes, target_model: str) -
|
|||
return _SSE_DATA_FIELD + b" " + serialized_event.encode() + line_ending
|
||||
|
||||
|
||||
def _vllm_choice_fields(choice: BaseModel) -> Mapping[str, object]:
|
||||
choice_extras: Final = choice.model_extra or {}
|
||||
return MappingProxyType({field: choice_extras[field] for field in VLLM_CHOICE_FIELDS if field in choice_extras})
|
||||
|
||||
|
||||
def _json_encodes(value: object) -> bool:
|
||||
try:
|
||||
to_jsonable_python(value)
|
||||
|
|
@ -10278,11 +10402,52 @@ def _serialize_chat_chunk(chunk: ModelResponseStream) -> bytes:
|
|||
|
||||
|
||||
def _serialize_streaming_chunk(chunk: BaseModel) -> bytes:
|
||||
choices: Final = _streaming_chunk_choices(chunk)
|
||||
null_choice_fields: Final = tuple(_null_vllm_choice_fields(choice) for choice in choices)
|
||||
if any(null_choice_fields):
|
||||
serialized_chunk: Final = _serialize_keeping_null_choice_fields(chunk, choices, null_choice_fields)
|
||||
if serialized_chunk is not None:
|
||||
return serialized_chunk
|
||||
if isinstance(chunk, ModelResponseStream):
|
||||
return _serialize_chat_chunk(chunk)
|
||||
return chunk.model_dump_json(exclude_none=True, exclude_unset=True).encode()
|
||||
|
||||
|
||||
def _serialize_keeping_null_choice_fields(
|
||||
chunk: BaseModel, choices: Sequence[BaseModel], null_choice_fields: Sequence[Mapping[str, None]]
|
||||
) -> bytes | None:
|
||||
try:
|
||||
return json.dumps(
|
||||
{
|
||||
**_dump_set_fields(chunk, exclude=frozenset({"choices"})),
|
||||
"choices": [
|
||||
{**_dump_set_fields(choice), **nulls}
|
||||
for choice, nulls in zip(choices, null_choice_fields, strict=True)
|
||||
],
|
||||
},
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=False,
|
||||
).encode()
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _streaming_chunk_choices(chunk: BaseModel) -> Sequence[BaseModel]:
|
||||
match chunk:
|
||||
case ModelResponseStream() | TextCompletionResponse():
|
||||
return chunk.choices
|
||||
case _:
|
||||
return ()
|
||||
|
||||
|
||||
def _null_vllm_choice_fields(choice: BaseModel) -> Mapping[str, None]:
|
||||
return MappingProxyType({field: None for field, value in _vllm_choice_fields(choice).items() if value is None})
|
||||
|
||||
|
||||
def _dump_set_fields(model: BaseModel, exclude: AbstractSet[str] = frozenset()) -> Mapping[str, object]:
|
||||
return model.model_dump(mode="json", exclude_none=True, exclude_unset=True, exclude=set(exclude))
|
||||
|
||||
|
||||
async def _apply_streaming_chunk_hooks(
|
||||
*,
|
||||
chunk: Any,
|
||||
|
|
@ -13396,7 +13561,10 @@ async def moderations(
|
|||
return response
|
||||
except Exception as e:
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=e,
|
||||
request_data=data,
|
||||
dispatched_call_type=CallTypes.amoderation.value,
|
||||
)
|
||||
log_llm_api_exception(e, litellm_call_id)
|
||||
if isinstance(e, ProxyException):
|
||||
|
|
@ -13541,6 +13709,7 @@ async def audio_speech(
|
|||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=e,
|
||||
request_data=data,
|
||||
dispatched_call_type=CallTypes.aspeech.value,
|
||||
)
|
||||
log_llm_api_exception(e, litellm_call_id)
|
||||
if isinstance(e, ProxyException):
|
||||
|
|
@ -13692,7 +13861,10 @@ async def audio_transcriptions(
|
|||
return response
|
||||
except Exception as e:
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=e,
|
||||
request_data=data,
|
||||
dispatched_call_type=CallTypes.atranscription.value,
|
||||
)
|
||||
log_llm_api_exception(e, litellm_call_id)
|
||||
if isinstance(e, HTTPException):
|
||||
|
|
@ -14790,6 +14962,7 @@ from litellm.repositories.table_repositories import (
|
|||
)
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
from litellm.repositories.verification_token_repository import VerificationTokenRepository
|
||||
from litellm.responses.litellm_completion_transformation.count_request import (
|
||||
ChatCountBridgeError,
|
||||
chat_count_request,
|
||||
|
|
@ -15037,7 +15210,7 @@ async def _provider_token_count(
|
|||
framing_can_stand_in: Final = (count_model is None or not count_model.configured) and (
|
||||
failure.cannot_count_model or (pinned_count_model is not None and failure.denied_in_region)
|
||||
)
|
||||
if not (framing_can_stand_in and settings.framing is not None and local_estimate_allowed):
|
||||
if not (framing_can_stand_in and settings.frames_counts and local_estimate_allowed):
|
||||
raise failure.to_proxy_exception()
|
||||
if pinned_count_model is not None:
|
||||
verbose_proxy_logger.warning(
|
||||
|
|
@ -15237,6 +15410,7 @@ async def _tokenizer_or_provider_error(
|
|||
|
||||
async def _local_token_estimate(
|
||||
model: str,
|
||||
template_model: str,
|
||||
prompt: str | None,
|
||||
messages: Sequence[AllMessageValues] | None,
|
||||
tools: list[ChatCompletionToolParam] | None,
|
||||
|
|
@ -15249,7 +15423,7 @@ async def _local_token_estimate(
|
|||
else await _tokenizer_or_provider_error(model, custom_tokenizer, provider_failure)
|
||||
)
|
||||
count: Final = offload_token_count(litellm.token_counter)(
|
||||
model=model,
|
||||
model=template_model,
|
||||
text=prompt,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
|
|
@ -15315,10 +15489,15 @@ def _deployment_aws_region(deployment: _TokenCountDeployment) -> str:
|
|||
)
|
||||
|
||||
|
||||
def _deployment_base_model(deployment: _TokenCountDeployment) -> str | None:
|
||||
return _non_empty_str(deployment.get("model_info", {}).get("base_model")) or _non_empty_str(
|
||||
deployment.get("litellm_params", {}).get("base_model")
|
||||
)
|
||||
|
||||
|
||||
def _token_count_settings_model(deployment: _TokenCountDeployment, request_model: str) -> tuple[str, str | None]:
|
||||
model_info: Final = deployment.get("model_info", {})
|
||||
params: Final = deployment.get("litellm_params", {})
|
||||
base_model: Final = _non_empty_str(model_info.get("base_model")) or _non_empty_str(params.get("base_model"))
|
||||
base_model: Final = _deployment_base_model(deployment)
|
||||
if base_model is not None:
|
||||
return base_model, None
|
||||
model: Final = _non_empty_str(params.get("model"))
|
||||
|
|
@ -15470,12 +15649,15 @@ def _responses_routing_kwargs(responses_request: Mapping[str, object] | None) ->
|
|||
return {"previous_response_id": previous_response_id} if isinstance(previous_response_id, str) else {}
|
||||
|
||||
|
||||
def _inference_params(
|
||||
litellm_params: Mapping[str, object], responses_request: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
return {**litellm_params, **_router_default_litellm_params(llm_router), **responses_request}
|
||||
|
||||
|
||||
def _count_drop_policy(deployment: Mapping[str, object] | None, responses_request: Mapping[str, object]) -> DropPolicy:
|
||||
litellm_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {}
|
||||
router_defaults: Final[Mapping[str, object]] = (
|
||||
{} if llm_router is None else _router_default_litellm_params(llm_router)
|
||||
)
|
||||
return drop_policy({**litellm_params, **router_defaults, **responses_request})
|
||||
return drop_policy(_inference_params(litellm_params, responses_request))
|
||||
|
||||
|
||||
def _native_responses_count(
|
||||
|
|
@ -15498,8 +15680,54 @@ def _native_responses_count(
|
|||
)
|
||||
|
||||
|
||||
def _deployment_api_base(deployment: _TokenCountDeployment) -> str | None:
|
||||
return _non_empty_str(deployment.get("litellm_params", {}).get("api_base"))
|
||||
def _deployment_api_base(
|
||||
deployment: _TokenCountDeployment, responses_request: Mapping[str, object] | None = None
|
||||
) -> str | None:
|
||||
return _non_empty_str(
|
||||
_inference_params(deployment.get("litellm_params", {}), responses_request or {}).get("api_base")
|
||||
)
|
||||
|
||||
|
||||
def _deployment_provider(
|
||||
deployment: _TokenCountDeployment, request_model: str, responses_request: Mapping[str, object] | None
|
||||
) -> str | None:
|
||||
params: Final = deployment.get("litellm_params", {})
|
||||
try:
|
||||
return litellm.get_llm_provider(
|
||||
model=_non_empty_str(params.get("model")) or request_model,
|
||||
custom_llm_provider=_non_empty_str(params.get("custom_llm_provider")),
|
||||
api_base=_deployment_api_base(deployment, responses_request),
|
||||
)[1]
|
||||
except litellm.BadRequestError:
|
||||
return None
|
||||
|
||||
|
||||
def _serves_native_responses(
|
||||
deployment: _TokenCountDeployment, request_model: str, responses_request: Mapping[str, object]
|
||||
) -> bool:
|
||||
params: Final = deployment.get("litellm_params", {})
|
||||
inference: Final = _inference_params(params, responses_request)
|
||||
try:
|
||||
return provider_stores_the_conversation(
|
||||
_non_empty_str(params.get("model")) or request_model,
|
||||
_non_empty_str(params.get("custom_llm_provider")),
|
||||
{
|
||||
"use_chat_completions_api": bool(inference.get("use_chat_completions_api")),
|
||||
"model_info": dict(deployment.get("model_info", {})),
|
||||
"api_base": _non_empty_str(inference.get("api_base")),
|
||||
},
|
||||
)
|
||||
except litellm.BadRequestError:
|
||||
return False
|
||||
|
||||
|
||||
_AZURE_PROVIDERS: Final = frozenset((litellm.LlmProviders.AZURE.value, litellm.LlmProviders.AZURE_AI.value))
|
||||
|
||||
|
||||
def _with_strict_by_default(tools: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]:
|
||||
return tuple(
|
||||
{**tool, "strict": True} if tool.get("type") == "function" and "strict" not in tool else tool for tool in tools
|
||||
)
|
||||
|
||||
|
||||
def _system_message(system: object) -> ChatCompletionSystemMessage | None:
|
||||
|
|
@ -15704,7 +15932,7 @@ async def _count_request_tokens(
|
|||
else await chat_count_request(
|
||||
model_to_use,
|
||||
custom_llm_provider,
|
||||
_deployment_api_base(deployment_view),
|
||||
_deployment_api_base(deployment_view, routed_request),
|
||||
routed_request,
|
||||
_count_drop_policy(routed_deployment, routed_request),
|
||||
)
|
||||
|
|
@ -15802,10 +16030,31 @@ async def _count_request_tokens(
|
|||
counted_messages: Final = (
|
||||
typed_messages if typed_messages is None or system_message is None else (system_message, *typed_messages)
|
||||
)
|
||||
counted_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats
|
||||
native_responses: Final = responses_request is not None and _serves_native_responses(
|
||||
deployment_view, request.model, responses_request
|
||||
)
|
||||
sent_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats
|
||||
list[ChatCompletionToolParam] | None, tools if counted_messages is not None else None
|
||||
)
|
||||
framing: Final = settings.framing if counted_messages is not None and settings.framing is not None else 0
|
||||
openai_model: Final = deployment_openai_model(deployment_view)
|
||||
template_model: Final = openai_model or _deployment_base_model(deployment_view) or model_to_use
|
||||
tool_section: Final = chat_template_tool_section(template_model)
|
||||
counted_tools: Final = (
|
||||
cast(list[ChatCompletionToolParam], list(_with_strict_by_default(sent_tools))) # cast-ok: strict added
|
||||
if sent_tools is not None
|
||||
and native_responses
|
||||
and _deployment_provider(deployment_view, request.model, responses_request) in _AZURE_PROVIDERS
|
||||
and tool_section is not None
|
||||
and tool_section.renders_strict_only_when_true
|
||||
else sent_tools
|
||||
)
|
||||
framing: Final = (
|
||||
0
|
||||
if counted_messages is None
|
||||
else settings.hidden_prompt_tokens(
|
||||
carries_tools=sends_function_tools(counted_tools), responses_api=native_responses
|
||||
)
|
||||
)
|
||||
kimi_k3_messages: Final = counted_messages if settings.tokenizer == "kimi_k3" else None
|
||||
exact: Final = (
|
||||
await _kimi_k3_count(
|
||||
|
|
@ -15829,7 +16078,8 @@ async def _count_request_tokens(
|
|||
count_source="local_estimate",
|
||||
)
|
||||
estimate: Final = await _local_token_estimate(
|
||||
model=deployment_openai_model(deployment_view) or model_to_use,
|
||||
model=openai_model or model_to_use,
|
||||
template_model=template_model,
|
||||
prompt=prompt if counted_messages is None else None,
|
||||
messages=counted_messages,
|
||||
tools=counted_tools,
|
||||
|
|
@ -20931,53 +21181,56 @@ async def delete_config_general_settings(
|
|||
if data.field_name in _GENERAL_SETTINGS_UI_LITELLM_FIELDS:
|
||||
return await _reset_general_settings_ui_litellm_field(data.field_name, user_api_key_dict)
|
||||
|
||||
if data.field_name not in ConfigGeneralSettings.model_fields:
|
||||
config_table: Final[TableActions[_ConfigParamValueRow]] = _config_param_table(prisma_client)
|
||||
stored_row: Final = await config_table.find_first(where={"param_name": "general_settings"})
|
||||
stored: Final = (
|
||||
_SETTINGS_MAPPING.validate_python(stored_row.param_value)
|
||||
if stored_row is not None and isinstance(stored_row.param_value, Mapping)
|
||||
else None
|
||||
)
|
||||
|
||||
if not _general_settings_row_is_readable(stored_row):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"error": "general_settings is stored in the database as something other than a JSON object, so no "
|
||||
"field can be removed from it"
|
||||
},
|
||||
)
|
||||
|
||||
if data.field_name not in ConfigGeneralSettings.model_fields and (stored is None or data.field_name not in stored):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"error": f"Invalid field={data.field_name} passed in."},
|
||||
)
|
||||
|
||||
proxy_config.reject_config_owned_deletes(section_name="general_settings", keys=(data.field_name,))
|
||||
if data.field_name in _db_managed_general_settings_keys():
|
||||
proxy_config.reject_config_owned_deletes(section_name="general_settings", keys=(data.field_name,))
|
||||
|
||||
## get general settings from db
|
||||
db_general_settings: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
|
||||
where={"param_name": "general_settings"}
|
||||
)
|
||||
### pop the value
|
||||
|
||||
if db_general_settings is None or db_general_settings.param_value is None:
|
||||
if stored is None:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"error": f"Field name={data.field_name} not in config"},
|
||||
)
|
||||
else:
|
||||
general_settings = dict(db_general_settings.param_value)
|
||||
|
||||
before_general_settings: Final = copy.deepcopy(general_settings)
|
||||
|
||||
## update db
|
||||
|
||||
general_settings.pop(data.field_name, None)
|
||||
|
||||
remaining: Final = {key: value for key, value in stored.items() if key != data.field_name}
|
||||
response: Final = await ConfigRepository(prisma_client).table.upsert(
|
||||
where={"param_name": "general_settings"},
|
||||
data={
|
||||
"create": {
|
||||
"param_name": "general_settings",
|
||||
"param_value": json.dumps(general_settings),
|
||||
"param_value": json.dumps(remaining),
|
||||
},
|
||||
"update": {"param_value": json.dumps(general_settings)},
|
||||
"update": {"param_value": json.dumps(remaining)},
|
||||
},
|
||||
)
|
||||
await invalidate_config_param("general_settings")
|
||||
proxy_config.settings.apply_db_row("general_settings", general_settings)
|
||||
proxy_config.settings.apply_db_row("general_settings", remaining)
|
||||
if is_resource_list("general_settings", data.field_name):
|
||||
stored_endpoints: Final = general_settings.get("pass_through_endpoints")
|
||||
stored_endpoints: Final = remaining.get("pass_through_endpoints")
|
||||
await proxy_config.serve_pass_through_endpoints(stored_endpoints if isinstance(stored_endpoints, list) else ())
|
||||
asyncio.create_task(
|
||||
create_config_audit_log(
|
||||
"general_settings", "deleted", before_general_settings, general_settings, user_api_key_dict
|
||||
)
|
||||
create_config_audit_log("general_settings", "deleted", dict(stored), remaining, user_api_key_dict)
|
||||
)
|
||||
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
openai_error_param,
|
||||
openai_error_type,
|
||||
)
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
router: Final = APIRouter()
|
||||
|
||||
|
|
@ -114,7 +115,10 @@ async def rerank(
|
|||
return response
|
||||
except Exception as e:
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=e,
|
||||
request_data=data,
|
||||
dispatched_call_type=CallTypes.arerank.value,
|
||||
)
|
||||
log_llm_api_exception(e, litellm_call_id)
|
||||
if isinstance(e, HTTPException):
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ from litellm.constants import (
|
|||
CALLER_SESSION_ID_METADATA_KEY,
|
||||
CLI_SESSION_KEY_PREFIX,
|
||||
EMPTY_MAPPING,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
KUBERNETES_POD_ROUTING_KEY,
|
||||
LITELLM_PROXY_MASTER_KEY_ALIAS,
|
||||
LITELLM_TRUNCATED_PAYLOAD_FIELD,
|
||||
|
|
@ -30,7 +29,6 @@ from litellm.constants import (
|
|||
MAX_SPEND_LOG_MODEL_NAME_LENGTH,
|
||||
MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB,
|
||||
MCP_SPEND_LOG_MODEL_PREFIX,
|
||||
NON_INFERENCE_CALL_TYPES,
|
||||
REDACTED_BY_LITELM_STRING,
|
||||
SESSION_ID_OMITTED_METADATA_KEY,
|
||||
UNKNOWN_MODEL_SPEND_LOG_MODEL,
|
||||
|
|
@ -53,7 +51,11 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
)
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
from litellm.litellm_core_utils.hidden_params import served_from_cache
|
||||
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call
|
||||
from litellm.litellm_core_utils.internal_call_metadata import (
|
||||
BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX,
|
||||
is_unbilled_non_inference_call,
|
||||
settles_background_response,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
StandardLoggingPayloadSetup,
|
||||
UsageObservation,
|
||||
|
|
@ -79,13 +81,8 @@ from litellm.responses.litellm_completion_transformation.compaction_items import
|
|||
summary_only_compaction_content,
|
||||
with_compaction_summary,
|
||||
)
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
||||
SETTLES_BACKGROUND_RESPONSE_KWARG,
|
||||
BackgroundResponseSettlement,
|
||||
)
|
||||
from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag
|
||||
from litellm.types.utils import (
|
||||
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
|
||||
PROMPT_CARRYING_GUARDRAIL_FIELDS,
|
||||
AzureSpillover,
|
||||
CallTypes,
|
||||
|
|
@ -347,24 +344,21 @@ def _get_spend_logs_metadata(
|
|||
|
||||
|
||||
BATCH_COST_REQUEST_ID_SUFFIX: Final = "_batch_cost"
|
||||
BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX: Final = "_background_cost"
|
||||
|
||||
|
||||
def _cost_row_request_id_suffix(
|
||||
call_type: str, metadata: Mapping[str, object], settles_background_response: bool
|
||||
) -> str:
|
||||
def _cost_row_request_id_suffix(call_type: str, metadata: Mapping[str, object], kwargs: Mapping[str, object]) -> str:
|
||||
if call_type == CallTypes.aretrieve_batch.value:
|
||||
return BATCH_COST_REQUEST_ID_SUFFIX
|
||||
if settles_background_response or (
|
||||
call_type in NON_INFERENCE_CALL_TYPES
|
||||
and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
|
||||
):
|
||||
if settles_background_response(call_type, metadata, kwargs):
|
||||
return BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX
|
||||
return ""
|
||||
|
||||
|
||||
def get_spend_logs_id(
|
||||
call_type: str, response_obj: dict, kwargs: Mapping[str, object], metadata: Mapping[str, object] = EMPTY_MAPPING
|
||||
call_type: str,
|
||||
response_obj: Mapping[str, object],
|
||||
kwargs: Mapping[str, object],
|
||||
metadata: Mapping[str, object] = EMPTY_MAPPING,
|
||||
) -> str | None:
|
||||
standard_logging_payload = kwargs.get("standard_logging_object")
|
||||
candidate_ids: Final = (
|
||||
|
|
@ -377,10 +371,7 @@ def get_spend_logs_id(
|
|||
)
|
||||
if resolved_id is None:
|
||||
return None
|
||||
settles_background_response: Final = (
|
||||
kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES
|
||||
)
|
||||
return f"{resolved_id}{_cost_row_request_id_suffix(call_type, metadata, settles_background_response)}"
|
||||
return f"{resolved_id}{_cost_row_request_id_suffix(call_type, metadata, kwargs)}"
|
||||
|
||||
|
||||
_MISSING_ATTRIBUTE: Final = object()
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ from litellm.constants import (
|
|||
SPEND_LOG_QUEUE_MAX_BYTES,
|
||||
SPEND_LOG_WRITE_BATCH_MAX_BYTES,
|
||||
SPEND_LOG_WRITE_BATCH_MAX_ROWS,
|
||||
VLLM_CHOICE_FIELDS,
|
||||
)
|
||||
from litellm.cost_calculator import BaseTokenUsageProcessor
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
|
|
@ -170,6 +171,7 @@ from litellm.litellm_core_utils.litellm_logging import (
|
|||
deployment_pricing_model_info,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.redact_messages import drop_token_ids
|
||||
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
|
|
@ -179,6 +181,7 @@ from litellm.litellm_core_utils.served_output_texts import (
|
|||
)
|
||||
from litellm.litellm_core_utils.token_counter import offload_token_count, usage_tokenizer_model
|
||||
from litellm.llms import load_guardrail_translation_mappings
|
||||
from litellm.llms.base_llm.guardrail_translation.utils import stream_item_field, stream_item_items
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.proxy._types import (
|
||||
AlertType,
|
||||
|
|
@ -707,6 +710,56 @@ def _pipeline_step_streams_live(step: "PipelineStep") -> bool:
|
|||
return not unified_guardrail.resolve_streaming_flag(callback, "streaming_buffer_until_moderated", True)
|
||||
|
||||
|
||||
def _drop_stream_chunk_token_ids(chunk: object) -> None:
|
||||
for choice in stream_item_items(chunk, "choices"):
|
||||
drop_token_ids(choice)
|
||||
|
||||
|
||||
def _drop_built_stream_token_ids(request_data: Mapping[str, object]) -> None:
|
||||
logging_obj: Final = request_data.get("litellm_logging_obj")
|
||||
if isinstance(logging_obj, Logging):
|
||||
logging_obj.drops_stream_token_ids = True
|
||||
|
||||
|
||||
def _drop_token_ids_ahead_of_chunk_check(
|
||||
callback: CustomLogger, chunk: object, request_data: Mapping[str, object]
|
||||
) -> None:
|
||||
if _overrides_hook(callback, "async_post_call_streaming_hook"):
|
||||
_drop_stream_chunk_token_ids(chunk)
|
||||
_drop_built_stream_token_ids(request_data)
|
||||
|
||||
|
||||
_PLAIN_STREAM_FIELDS: Final = TypeAdapter(dict[str, object])
|
||||
_EMPTY_STREAM_VALUES: Final[tuple[object, ...]] = (None, "", [], {})
|
||||
|
||||
|
||||
def _holds_stream_payload(value: object) -> bool:
|
||||
if isinstance(value, dict):
|
||||
nested: Final = _PLAIN_STREAM_FIELDS.validate_python(value).values()
|
||||
return any(item not in _EMPTY_STREAM_VALUES for item in nested)
|
||||
return value not in _EMPTY_STREAM_VALUES
|
||||
|
||||
|
||||
def _choice_carries_only_token_ids(choice: object) -> bool:
|
||||
if stream_item_field(choice, "token_ids") is None or not isinstance(choice, BaseModel):
|
||||
return False
|
||||
fields: Final = _PLAIN_STREAM_FIELDS.validate_python(choice.model_dump())
|
||||
return not any(_holds_stream_payload(value) for key, value in fields.items() if key not in ("index", "token_ids"))
|
||||
|
||||
|
||||
def _is_token_id_only_chunk(chunk: object) -> bool:
|
||||
choices: Final = stream_item_items(chunk, "choices")
|
||||
return bool(choices) and all(_choice_carries_only_token_ids(choice) for choice in choices)
|
||||
|
||||
|
||||
async def _without_token_id_only_chunks(response: AsyncIterable[object]) -> AsyncGenerator[object, None]:
|
||||
async for chunk in response:
|
||||
if _is_token_id_only_chunk(chunk):
|
||||
_drop_stream_chunk_token_ids(chunk)
|
||||
continue
|
||||
yield chunk
|
||||
|
||||
|
||||
def _pipelines_stream_live(pipelines: Sequence[tuple[str, "GuardrailPipeline"]]) -> bool:
|
||||
return all(_pipeline_step_streams_live(step) for step in _pipeline_steps(pipelines))
|
||||
|
||||
|
|
@ -1707,7 +1760,10 @@ def _proxy_only_error_log_fields(
|
|||
|
||||
|
||||
def _failure_fields_to_lift(
|
||||
request_data: Mapping[str, object], provider_billed: bool = False, dispatched_call_type: str | None = None
|
||||
request_data: Mapping[str, object],
|
||||
provider_billed: bool = False,
|
||||
dispatched_call_type: str | None = None,
|
||||
route_call_type: str | None = None,
|
||||
) -> Mapping[str, object]:
|
||||
"""Failure-path callbacks run after ``litellm_logging_obj`` is popped from
|
||||
request_data (it is not serialisable), so the caller merges these fields
|
||||
|
|
@ -1717,7 +1773,8 @@ def _failure_fields_to_lift(
|
|||
attribution on failed-request spend logs."""
|
||||
_logging_obj: Final = request_data.get("litellm_logging_obj")
|
||||
if _logging_obj is None:
|
||||
return _EMPTY_LIFT
|
||||
return _EMPTY_LIFT if dispatched_call_type is None else MappingProxyType({"call_type": dispatched_call_type})
|
||||
_estimate_call_type: Final = dispatched_call_type or route_call_type
|
||||
_model_call_details: Final = getattr(_logging_obj, "model_call_details", {})
|
||||
_first_handoff: Final = _model_call_details.get("first_api_call_start_time")
|
||||
_litellm_params: Final = getattr(_logging_obj, "litellm_params", None)
|
||||
|
|
@ -1732,10 +1789,10 @@ def _failure_fields_to_lift(
|
|||
provider_billed=_billed,
|
||||
deployment_id=_deployment_id,
|
||||
base_model=_base_model,
|
||||
dispatched_call_type=dispatched_call_type,
|
||||
dispatched_call_type=_estimate_call_type,
|
||||
)
|
||||
_prompt_estimate: Final = (
|
||||
_estimated_prompt_usage(_model_call_details, request_data, dispatched_call_type)
|
||||
_estimated_prompt_usage(_model_call_details, request_data, _estimate_call_type)
|
||||
if _billed and _first_handoff is not None and not _model_call_details.get("failed_attempts_usage")
|
||||
else None
|
||||
)
|
||||
|
|
@ -1755,7 +1812,7 @@ def _failure_fields_to_lift(
|
|||
_billed,
|
||||
_deployment_id,
|
||||
_base_model,
|
||||
dispatched_call_type,
|
||||
_estimate_call_type,
|
||||
)
|
||||
if Logging.usage_capture_callback is not None
|
||||
and (_billed or _model_call_details.get("combined_usage_object") is not None)
|
||||
|
|
@ -4051,13 +4108,15 @@ class ProxyLogging:
|
|||
route=route,
|
||||
original_exception=original_exception,
|
||||
error_type=error_type,
|
||||
dispatched_call_type=dispatched_call_type,
|
||||
)
|
||||
|
||||
request_data.update(
|
||||
await offload_token_count(_failure_fields_to_lift)(
|
||||
request_data,
|
||||
provider_billed=_provider_billed_the_failure(original_exception),
|
||||
dispatched_call_type=dispatched_call_type or _call_type_for_route(user_api_key_dict.request_route),
|
||||
dispatched_call_type=dispatched_call_type,
|
||||
route_call_type=_call_type_for_route(user_api_key_dict.request_route),
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -4146,6 +4205,7 @@ class ProxyLogging:
|
|||
route: str | None = None,
|
||||
original_exception: Exception | None = None,
|
||||
error_type: ProxyErrorTypes | None = None,
|
||||
dispatched_call_type: str | None = None,
|
||||
):
|
||||
"""
|
||||
Handle logging for proxy only errors by calling `litellm_logging_obj.async_failure_handler`
|
||||
|
|
@ -4245,7 +4305,7 @@ class ProxyLogging:
|
|||
litellm_logging_obj.model_call_details["input"] = input
|
||||
body_shape_call_type = CallTypes.aembedding.value
|
||||
resolved_call_type: Final = _call_type_for_route(route) or (
|
||||
None if set_up_by_request else body_shape_call_type
|
||||
None if set_up_by_request else dispatched_call_type or body_shape_call_type
|
||||
)
|
||||
if resolved_call_type is not None and litellm_logging_obj.call_type != CallTypes.pass_through.value:
|
||||
litellm_logging_obj.call_type = resolved_call_type
|
||||
|
|
@ -4710,6 +4770,7 @@ class ProxyLogging:
|
|||
else:
|
||||
_callback = callback
|
||||
if _callback is not None and isinstance(_callback, CustomLogger):
|
||||
_drop_token_ids_ahead_of_chunk_check(_callback, response, data)
|
||||
if str_so_far is not None:
|
||||
complete_response = str_so_far + response_str
|
||||
else:
|
||||
|
|
@ -4731,7 +4792,7 @@ class ProxyLogging:
|
|||
self,
|
||||
response,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
request_data: dict,
|
||||
request_data: dict[str, object],
|
||||
):
|
||||
"""
|
||||
Allow user to modify outgoing streaming data -> Given a whole response iterator.
|
||||
|
|
@ -4765,20 +4826,26 @@ class ProxyLogging:
|
|||
# Merge model-level guardrails before checking which guardrails to run
|
||||
request_data = check_and_merge_model_level_guardrails(data=request_data, llm_router=llm_router)
|
||||
|
||||
current_response = response
|
||||
stream_needs_translation: Final = ProxyLogging._stream_requires_guardrail_translation(user_api_key_dict)
|
||||
|
||||
pipeline_gated_names: Final = _pipeline_step_guardrail_names(post_call_pipelines)
|
||||
running_overrides: Final = tuple(
|
||||
(callback, kind)
|
||||
for callback, kind in caps.iterator_overrides
|
||||
if not isinstance(callback, CustomGuardrail)
|
||||
or (
|
||||
callback.guardrail_name not in pipeline_gated_names
|
||||
and callback.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is True
|
||||
)
|
||||
)
|
||||
drops_token_ids: Final = bool(post_call_pipelines) or any(
|
||||
isinstance(callback, CustomGuardrail) for callback, _ in running_overrides
|
||||
)
|
||||
if drops_token_ids:
|
||||
_drop_built_stream_token_ids(request_data)
|
||||
current_response = _without_token_id_only_chunks(response) if drops_token_ids else response
|
||||
guarded_layers: Final[list[AsyncGenerator[object, None]]] = [] # mutable-ok: closed on disconnect
|
||||
for resolved_callback, kind in caps.iterator_overrides:
|
||||
if isinstance(resolved_callback, CustomGuardrail):
|
||||
if resolved_callback.guardrail_name in pipeline_gated_names:
|
||||
continue
|
||||
if (
|
||||
resolved_callback.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call)
|
||||
is not True
|
||||
):
|
||||
continue
|
||||
for resolved_callback, kind in running_overrides:
|
||||
effective_kind = (
|
||||
"apply_guardrail"
|
||||
if (
|
||||
|
|
@ -4836,6 +4903,8 @@ class ProxyLogging:
|
|||
try:
|
||||
async for chunk in current_response:
|
||||
served_chunks.append(chunk)
|
||||
if drops_token_ids:
|
||||
_drop_stream_chunk_token_ids(served_chunks[-1])
|
||||
yield chunk
|
||||
except (GeneratorExit, asyncio.CancelledError):
|
||||
await ProxyLogging._close_guarded_layers(guarded_layers)
|
||||
|
|
@ -10203,7 +10272,7 @@ _PRESERVED_NONE_FIELDS: Final[list[tuple[str, str]]] = [
|
|||
("message", "refusal"),
|
||||
("delta", "content"), # null in streaming chunks
|
||||
]
|
||||
_PRESERVED_NONE_CHOICE_FIELDS: Final = ("logprobs",)
|
||||
_PRESERVED_NONE_CHOICE_FIELDS: Final = ("logprobs", *VLLM_CHOICE_FIELDS)
|
||||
_PRESERVED_NONE_RESPONSE_FIELDS: Final = ("system_fingerprint",)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,16 @@
|
|||
import asyncio
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER
|
||||
from litellm.litellm_core_utils.cold_storage_handler import ColdStorageHandler
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.responses.litellm_completion_transformation.client_tool_search import (
|
||||
bridge_client_tool_search,
|
||||
client_tool_search_loaded_tools,
|
||||
|
|
@ -69,6 +71,90 @@ def _is_a_conversation_turn(spend_log: "SpendLogsPayload") -> bool:
|
|||
return spend_log.get("call_type") not in _MCP_TOOL_CALL_TYPES and not _is_a_failed_turn(spend_log)
|
||||
|
||||
|
||||
_RESPONSE_READ_CALL_TYPES: Final = frozenset(("aget_responses", "acancel_responses"))
|
||||
|
||||
|
||||
def _output_item_id(item: object) -> str | None:
|
||||
match item:
|
||||
case {"id": str() as item_id}:
|
||||
return item_id
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _is_a_response_read(spend_log: "SpendLogsPayload") -> bool:
|
||||
return "call_type" in spend_log and spend_log["call_type"] in _RESPONSE_READ_CALL_TYPES
|
||||
|
||||
|
||||
def _stored_response(spend_log: "SpendLogsPayload") -> object:
|
||||
return spend_log.get("response")
|
||||
|
||||
|
||||
def _output_items(spend_log: "SpendLogsPayload") -> Iterator[tuple[str | None, object]]:
|
||||
output: Final = _stored_responses_api_output(_stored_response(spend_log)) or []
|
||||
return zip(map(_output_item_id, output), output)
|
||||
|
||||
|
||||
def _identified_output_items(spend_log: "SpendLogsPayload") -> Iterator[tuple[str, object]]:
|
||||
return ((item_id, item) for item_id, item in _output_items(spend_log) if item_id is not None)
|
||||
|
||||
|
||||
def _output_item_ids(spend_log: "SpendLogsPayload") -> frozenset[str]:
|
||||
return frozenset(item_id for item_id, _ in _identified_output_items(spend_log))
|
||||
|
||||
|
||||
def _first_index_holding_each_item(indexed_turns: Iterable[tuple[int, "SpendLogsPayload"]]) -> Mapping[str, int]:
|
||||
return dict(
|
||||
chain.from_iterable(
|
||||
((item_id, index) for item_id in _output_item_ids(turn)) for index, turn in reversed(tuple(indexed_turns))
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _turn_replaying_each_item(turns: Sequence["SpendLogsPayload"]) -> Mapping[str, int]:
|
||||
indexed_turns: Final = tuple(enumerate(turns))
|
||||
first_read_holding: Final = _first_index_holding_each_item(
|
||||
(index, turn) for index, turn in indexed_turns if _is_a_response_read(turn)
|
||||
)
|
||||
first_other_turn_holding: Final = _first_index_holding_each_item(
|
||||
(index, turn) for index, turn in indexed_turns if not _is_a_response_read(turn)
|
||||
)
|
||||
return MappingProxyType({**first_read_holding, **first_other_turn_holding})
|
||||
|
||||
|
||||
def _latest_read_copy_of_each_item(turns: Sequence["SpendLogsPayload"]) -> Mapping[str, object]:
|
||||
return MappingProxyType(
|
||||
dict(chain.from_iterable(_identified_output_items(turn) for turn in turns if _is_a_response_read(turn)))
|
||||
)
|
||||
|
||||
|
||||
def _with_only_new_output(
|
||||
index: int, turn: "SpendLogsPayload", turn_replaying: Mapping[str, int], latest_copy: Mapping[str, object]
|
||||
) -> "SpendLogsPayload | None":
|
||||
if not _is_a_response_read(turn):
|
||||
return turn
|
||||
stored_response: Final = as_str_mapping(_stored_response(turn))
|
||||
new_output: Final = [
|
||||
item if item_id is None else latest_copy[item_id]
|
||||
for item_id, item in _output_items(turn)
|
||||
if item_id is None or turn_replaying.get(item_id) == index
|
||||
]
|
||||
if stored_response is None or not new_output:
|
||||
return None
|
||||
replayed_turn: Final[SpendLogsPayload] = {**turn, "response": {**stored_response, "output": new_output}}
|
||||
return replayed_turn
|
||||
|
||||
|
||||
def _conversation_turns(spend_logs: Sequence["SpendLogsPayload"]) -> tuple["SpendLogsPayload", ...]:
|
||||
turns: Final = tuple(filter(_is_a_conversation_turn, spend_logs))
|
||||
turn_replaying: Final = _turn_replaying_each_item(turns)
|
||||
latest_copy: Final = _latest_read_copy_of_each_item(turns)
|
||||
replayed: Final = (
|
||||
_with_only_new_output(index, turn, turn_replaying, latest_copy) for index, turn in enumerate(turns)
|
||||
)
|
||||
return tuple(turn for turn in replayed if turn is not None)
|
||||
|
||||
|
||||
_MCP_AUTO_EXECUTE_OUTPUT_TYPES: Final = frozenset(("mcp_tools_fetched", "tool_execution_results"))
|
||||
|
||||
|
||||
|
|
@ -101,7 +187,7 @@ class StoredTurns:
|
|||
def _compaction_point(
|
||||
index: int, spend_log: "SpendLogsPayload", proxy_server_request: Mapping[str, object] | None
|
||||
) -> _ReplayStart | None:
|
||||
if response_holds_compaction_block(spend_log.get("response")):
|
||||
if response_holds_compaction_block(_stored_response(spend_log)):
|
||||
return _ReplayStart(index=index, at_output=True)
|
||||
if proxy_server_request is not None and holds_compaction_summary(proxy_server_request.get("input")):
|
||||
return _ReplayStart(index=index, at_output=False)
|
||||
|
|
@ -271,12 +357,10 @@ class ResponsesSessionHandler:
|
|||
previous_response_id: str,
|
||||
client_response_ids: tuple[str, ...] = (),
|
||||
) -> StoredTurns:
|
||||
spend_logs: Final = tuple(
|
||||
spend_log
|
||||
for spend_log in await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id(
|
||||
spend_logs: Final = _conversation_turns(
|
||||
await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id(
|
||||
previous_response_id, client_response_ids
|
||||
)
|
||||
if _is_a_conversation_turn(spend_log)
|
||||
)
|
||||
return StoredTurns(
|
||||
spend_logs=spend_logs,
|
||||
|
|
@ -378,7 +462,7 @@ class ResponsesSessionHandler:
|
|||
############################################################
|
||||
# Add Output messages for this Spend Log
|
||||
############################################################
|
||||
_response_output: Final[object] = spend_log.get("response", "{}")
|
||||
_response_output: Final = _stored_response(spend_log)
|
||||
if (responses_api_output := _stored_responses_api_output(_response_output)) is not None:
|
||||
chat_completion_message_history.extend(
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
|
|
|
|||
|
|
@ -155,6 +155,7 @@ GroupField: TypeAlias = Literal[
|
|||
"display_name",
|
||||
"mode",
|
||||
"service_tiers",
|
||||
"max_input_tokens",
|
||||
"token_limits",
|
||||
"listing_info",
|
||||
]
|
||||
|
|
@ -855,6 +856,16 @@ class RouterRegistry:
|
|||
team_id: str | None = None,
|
||||
) -> tuple[object, ...]: ...
|
||||
|
||||
@overload
|
||||
def group(
|
||||
self,
|
||||
name: str,
|
||||
*,
|
||||
field: Literal["max_input_tokens"],
|
||||
include_hidden: bool = False,
|
||||
team_id: str | None = None,
|
||||
) -> tuple[int | None, ...]: ...
|
||||
|
||||
@overload
|
||||
def group(
|
||||
self,
|
||||
|
|
@ -882,9 +893,7 @@ class RouterRegistry:
|
|||
field: GroupField = "info",
|
||||
include_hidden: bool = False,
|
||||
team_id: str | None = None,
|
||||
) -> (
|
||||
"ModelGroupInfo | DeploymentModelListingInfo | tuple[int | None, int | None] | tuple[object, ...] | str | None"
|
||||
):
|
||||
) -> "ModelGroupInfo | DeploymentModelListingInfo | tuple[int | None, ...] | tuple[object, ...] | str | None":
|
||||
"""
|
||||
What the router knows about a model group. ``field``:
|
||||
|
||||
|
|
@ -894,6 +903,7 @@ class RouterRegistry:
|
|||
(were ``get_configured_display_name``, ``get_configured_mode``,
|
||||
``get_configured_token_limits``, ``get_model_listing_info``)
|
||||
- ``service_tiers``: (was ``get_configured_service_tiers``)
|
||||
- ``max_input_tokens``: each routable deployment's own ``model_info.max_input_tokens``
|
||||
"""
|
||||
r: Final = self._router
|
||||
if field == "info":
|
||||
|
|
@ -906,6 +916,8 @@ class RouterRegistry:
|
|||
return r._core_get_configured_mode(name)
|
||||
if field == "service_tiers":
|
||||
return r._core_get_configured_service_tiers(name, team_id=team_id)
|
||||
if field == "max_input_tokens":
|
||||
return r._core_get_configured_max_input_tokens(name, team_id=team_id)
|
||||
if field == "token_limits":
|
||||
return r._core_get_configured_token_limits(name)
|
||||
if field == "listing_info":
|
||||
|
|
|
|||
|
|
@ -141,6 +141,7 @@ from litellm.litellm_core_utils.get_llm_provider_logic import (
|
|||
is_registered_custom_provider,
|
||||
resolve_model_and_provider_for_metadata,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.get_retry_from_policy import (
|
||||
get_num_retries_from_retry_policy as _get_num_retries_from_retry_policy,
|
||||
)
|
||||
|
|
@ -475,6 +476,7 @@ from litellm.types.router import (
|
|||
from litellm.types.services import ServiceTypes
|
||||
from litellm.types.utils import (
|
||||
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
|
||||
COST_MAP_LOOKUP_KEY,
|
||||
PROMPT_QUOTING_ROUTING_DECISION_FIELDS,
|
||||
CustomPricingLiteLLMParams,
|
||||
GenericBudgetConfigType,
|
||||
|
|
@ -486,6 +488,7 @@ from litellm.types.utils import (
|
|||
StandardLoggingRoutingDecision,
|
||||
Usage,
|
||||
all_litellm_params,
|
||||
echoed_cost_map_fields,
|
||||
is_server_derived_pricing_key,
|
||||
shared_backend_model_info,
|
||||
)
|
||||
|
|
@ -772,6 +775,17 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj
|
|||
return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {})
|
||||
|
||||
|
||||
def _operator_max_input_tokens(deployment: Mapping[str, object]) -> int | None:
|
||||
model_info: Final = _configured_model_info(deployment)
|
||||
catalog_key: Final = model_info.get(COST_MAP_LOOKUP_KEY)
|
||||
catalog_entry: Final = (
|
||||
GetModelCostMap.loaded_model_cost_map().get(catalog_key) if isinstance(catalog_key, str) else None
|
||||
)
|
||||
if catalog_entry is not None and "max_input_tokens" in echoed_cost_map_fields(model_info, catalog_entry):
|
||||
return None
|
||||
return coerce_token_limit(model_info.get("max_input_tokens"))
|
||||
|
||||
|
||||
def _search_tool_support(deployment: DeploymentTypedDict) -> bool | None:
|
||||
from litellm.responses.litellm_completion_transformation.tool_search import chat_config_builds_anthropic_request
|
||||
|
||||
|
|
@ -12693,6 +12707,24 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
for deployment in self._routable_deployments(model_name, team_id)
|
||||
)
|
||||
|
||||
def _core_get_configured_max_input_tokens(
|
||||
self, model_name: str, team_id: str | None = None
|
||||
) -> tuple[int | None, ...]:
|
||||
"""
|
||||
Return the max_input_tokens each deployment a request from team_id to
|
||||
model_name can route to (see _routable_deployments) sets in its own
|
||||
model_info, in model_list order, None for a deployment that sets none
|
||||
or a malformed one. Unlike get_model_listing_info, neither the cost map
|
||||
nor a discovered limit is read, so only an operator's own value counts,
|
||||
and a value a saved /model/info response copied from the cost map is
|
||||
skipped too.
|
||||
|
||||
Returns an empty tuple for wildcard-expanded or unknown names.
|
||||
"""
|
||||
return tuple(
|
||||
_operator_max_input_tokens(deployment) for deployment in self._routable_deployments(model_name, team_id)
|
||||
)
|
||||
|
||||
def _core_get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]:
|
||||
"""
|
||||
Whether each deployment a request from team_id to model_name can route
|
||||
|
|
@ -13215,6 +13247,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
"providers": [llm_provider],
|
||||
"description": model_group_description,
|
||||
**model_info,
|
||||
"supports_continuation": model_info.get("supports_continuation") is True,
|
||||
"supports_fast_mode": True,
|
||||
"supported_reasoning_efforts": None,
|
||||
"supported_endpoints": None,
|
||||
|
|
@ -13282,6 +13315,8 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
and model_info["supports_url_context"] is True
|
||||
):
|
||||
model_group_info.supports_url_context = True
|
||||
if model_info.get("supports_continuation") is True:
|
||||
model_group_info.supports_continuation = True
|
||||
|
||||
if model_info.get("supports_reasoning", None) is not None and model_info["supports_reasoning"] is True:
|
||||
model_group_info.supports_reasoning = True
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.constants import VLLM_CHOICE_FIELDS
|
||||
from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact
|
||||
from litellm.types.utils import Choices, ModelResponse, ModelResponseStream, TextChoices, TextCompletionResponse
|
||||
|
||||
|
|
@ -37,16 +38,19 @@ class TextCompletionStreamWrapper:
|
|||
response["object"] = "text_completion"
|
||||
response["created"] = chunk.get("created", None)
|
||||
response["model"] = chunk.get("model", None)
|
||||
text_choices: Final = TextChoices()
|
||||
if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices
|
||||
raise Exception
|
||||
|
||||
if chunk["choices"]:
|
||||
delta: Final = chunk["choices"][0]["delta"]
|
||||
chat_choice: Final = chunk["choices"][0]
|
||||
text_choices: Final = TextChoices(
|
||||
**{field: chat_choice[field] for field in VLLM_CHOICE_FIELDS if field in chat_choice}
|
||||
)
|
||||
delta: Final = chat_choice["delta"]
|
||||
text_choices["text"] = delta["content"]
|
||||
text_choices["reasoning_content"] = delta.get("reasoning_content")
|
||||
text_choices["index"] = chunk["choices"][0]["index"]
|
||||
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
|
||||
text_choices["index"] = chat_choice["index"]
|
||||
text_choices["finish_reason"] = chat_choice["finish_reason"]
|
||||
response["choices"] = [text_choices]
|
||||
else:
|
||||
response["choices"] = []
|
||||
|
|
|
|||
|
|
@ -849,6 +849,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum):
|
|||
ADVISOR_TOOL_2026_03_01 = "advisor-tool-2026-03-01"
|
||||
PER_TURN_CONTROL_2026_07_01 = "per-turn-control-2026-07-01"
|
||||
DANGEROUS_TOOL_USE_2026_09_03 = "dangerous-tool-use-2026-09-03"
|
||||
MCP_CLIENT_2025_04_04 = "mcp-client-2025-04-04"
|
||||
|
||||
|
||||
# Tool search beta header constant (for Anthropic direct API and Microsoft Foundry)
|
||||
|
|
|
|||
|
|
@ -1061,6 +1061,7 @@ class ModelGroupInfo(LiteLLMBaseModel):
|
|||
supports_vision: bool = Field(default=False)
|
||||
supports_web_search: bool = Field(default=False)
|
||||
supports_url_context: bool = Field(default=False)
|
||||
supports_continuation: bool = Field(default=False)
|
||||
supports_reasoning: bool = Field(default=False)
|
||||
supports_function_calling: bool = Field(default=False)
|
||||
supports_fast_mode: bool = Field(default=False)
|
||||
|
|
|
|||
|
|
@ -227,6 +227,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
|
|||
supports_tool_search: bool | None
|
||||
supports_mid_conversation_system: bool | None
|
||||
supports_url_context: bool | None
|
||||
supports_continuation: ReadOnly[bool | None]
|
||||
supports_none_reasoning_effort: bool | None
|
||||
supports_minimal_reasoning_effort: bool | None
|
||||
supports_low_reasoning_effort: bool | None
|
||||
|
|
@ -246,6 +247,9 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
|
|||
bedrock_converse_supports_strict_tools: bool | None
|
||||
supports_regex_lookaround: ReadOnly[bool | None]
|
||||
token_count_framing: ReadOnly[int | None]
|
||||
token_count_tool_framing: ReadOnly[int | None]
|
||||
token_count_responses_framing: ReadOnly[int | None]
|
||||
token_count_responses_tool_framing: ReadOnly[int | None]
|
||||
token_count_model: ReadOnly[TokenCountModelInfo | None]
|
||||
token_count_tokenizer: ReadOnly[TokenCountTokenizer | None]
|
||||
|
||||
|
|
@ -4231,12 +4235,21 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset(
|
|||
"supports_assistant_prefill",
|
||||
"supports_bedrock_native_structured_output",
|
||||
"supports_pdf_input",
|
||||
"supports_continuation",
|
||||
"reasoning_modes",
|
||||
"reasoning_contexts",
|
||||
}
|
||||
)
|
||||
|
||||
DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset({"token_count_framing", "token_count_model"})
|
||||
DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
"token_count_framing",
|
||||
"token_count_tool_framing",
|
||||
"token_count_responses_framing",
|
||||
"token_count_responses_tool_framing",
|
||||
"token_count_model",
|
||||
}
|
||||
)
|
||||
|
||||
SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = (
|
||||
frozenset(ModelInfoBase.__required_keys__ | ModelInfoBase.__optional_keys__)
|
||||
|
|
|
|||
|
|
@ -98,7 +98,11 @@ from litellm.litellm_core_utils.model_retirement import is_hidden_as_retired
|
|||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
|
||||
from litellm.litellm_core_utils.tokenizer import strip_special_tokens
|
||||
from litellm.litellm_core_utils.tool_fields import MalformedField, resolve_deployment_tool_fields
|
||||
from litellm.litellm_core_utils.tool_fields import (
|
||||
MalformedField,
|
||||
resolve_deployment_tool_fields,
|
||||
tools_without_hosted_mcp,
|
||||
)
|
||||
from litellm.rust_bridge import tokenizer as tokenizer_dispatch
|
||||
from litellm.rust_bridge.catalog import decision
|
||||
from litellm.rust_bridge.configuration import Decision
|
||||
|
|
@ -3183,6 +3187,14 @@ def supports_url_context(model: str, custom_llm_provider: str | None = None) ->
|
|||
)
|
||||
|
||||
|
||||
def supports_continuation(model: str, custom_llm_provider: str | None = None) -> bool:
|
||||
return supports_factory(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
key="supports_continuation",
|
||||
)
|
||||
|
||||
|
||||
def supports_native_streaming(model: str, custom_llm_provider: str | None) -> bool:
|
||||
"""
|
||||
Check if the given model supports native streaming and return a boolean value.
|
||||
|
|
@ -5186,7 +5198,12 @@ def remove_sensitive_keys_from_dict(d: dict) -> dict:
|
|||
return d
|
||||
|
||||
|
||||
def pre_process_optional_params(passed_params: dict, non_default_params: dict, custom_llm_provider: str) -> dict:
|
||||
_TOOL_LIST: Final = TypeAdapter(list[object])
|
||||
|
||||
|
||||
def pre_process_optional_params(
|
||||
passed_params: dict, non_default_params: dict, custom_llm_provider: str, model: str, drop_params: bool | None
|
||||
) -> dict:
|
||||
"""For .completion(), preprocess optional params"""
|
||||
optional_params: dict = {}
|
||||
|
||||
|
|
@ -5243,14 +5260,20 @@ def pre_process_optional_params(passed_params: dict, non_default_params: dict, c
|
|||
and custom_llm_provider not in litellm.openai_compatible_providers
|
||||
):
|
||||
if custom_llm_provider == "ollama":
|
||||
if "tools" in non_default_params:
|
||||
sent_tools: Final = _TOOL_LIST.validate_python(non_default_params.pop("tools"))
|
||||
non_default_params.pop("tool_choice", None) # causes ollama requests to hang
|
||||
prompt_tools: Final = tools_without_hosted_mcp(
|
||||
sent_tools, model=model, route=custom_llm_provider, drop_params=bool(drop_params)
|
||||
)
|
||||
if sent_tools and not prompt_tools:
|
||||
return optional_params
|
||||
optional_params["functions_unsupported_model"] = prompt_tools
|
||||
elif "functions" in non_default_params:
|
||||
optional_params["functions_unsupported_model"] = non_default_params.pop("functions")
|
||||
# ollama actually supports json output
|
||||
optional_params["format"] = "json"
|
||||
litellm.add_function_to_prompt = True # so that main.py adds the function call to the prompt
|
||||
if "tools" in non_default_params:
|
||||
optional_params["functions_unsupported_model"] = non_default_params.pop("tools")
|
||||
non_default_params.pop("tool_choice", None) # causes ollama requests to hang
|
||||
elif "functions" in non_default_params:
|
||||
optional_params["functions_unsupported_model"] = non_default_params.pop("functions")
|
||||
elif litellm.add_function_to_prompt: # if user opts to add it to prompt instead
|
||||
optional_params["functions_unsupported_model"] = non_default_params.pop(
|
||||
"tools", non_default_params.pop("functions", None)
|
||||
|
|
@ -5385,6 +5408,8 @@ def get_optional_params(
|
|||
passed_params=passed_params,
|
||||
non_default_params=non_default_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model=model,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
||||
def _check_valid_arg(supported_params: list[str]):
|
||||
|
|
@ -7360,6 +7385,7 @@ def get_model_info_helper(
|
|||
),
|
||||
supports_web_search=_model_info.get("supports_web_search", None),
|
||||
supports_url_context=_model_info.get("supports_url_context", None),
|
||||
supports_continuation=_model_info.get("supports_continuation", None),
|
||||
supports_reasoning=_model_info.get("supports_reasoning", None),
|
||||
supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None),
|
||||
supports_legacy_thinking=_model_info.get("supports_legacy_thinking", None),
|
||||
|
|
@ -7398,6 +7424,9 @@ def get_model_info_helper(
|
|||
supports_image_size=_model_info.get("supports_image_size", None),
|
||||
supported_audio_formats=_model_info.get("supported_audio_formats", None),
|
||||
token_count_framing=_model_info.get("token_count_framing", None),
|
||||
token_count_tool_framing=_model_info.get("token_count_tool_framing", None),
|
||||
token_count_responses_framing=_model_info.get("token_count_responses_framing", None),
|
||||
token_count_responses_tool_framing=_model_info.get("token_count_responses_tool_framing", None),
|
||||
token_count_model=_model_info.get("token_count_model", None),
|
||||
token_count_tokenizer=_model_info.get("token_count_tokenizer", None),
|
||||
vertex_ai_audio_api=_model_info.get("vertex_ai_audio_api", None),
|
||||
|
|
@ -7516,6 +7545,7 @@ def get_model_info(
|
|||
supports_pdf_input: Optional[bool]
|
||||
supports_web_search: Optional[bool]
|
||||
supports_url_context: Optional[bool]
|
||||
supports_continuation: Optional[bool]
|
||||
supports_reasoning: Optional[bool]
|
||||
Raises:
|
||||
Exception: If the model is not mapped yet.
|
||||
|
|
|
|||
|
|
@ -13478,6 +13478,10 @@
|
|||
"input_cost_per_token": 2e-07,
|
||||
"output_cost_per_token": 5e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"token_count_framing": 149,
|
||||
"token_count_tool_framing": 302,
|
||||
"token_count_responses_framing": 145,
|
||||
"token_count_responses_tool_framing": 100,
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
|
|
@ -13831,7 +13835,7 @@
|
|||
"input_cost_per_token_flex": 3.7e-07,
|
||||
"input_cost_per_token_priority": 1.295e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 163840,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.22e-06,
|
||||
|
|
@ -13845,7 +13849,7 @@
|
|||
"supports_audio_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": false,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
|
||||
},
|
||||
"bedrock/ap-northeast-1/minimax.minimax-m2.1": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
|
|
@ -14002,7 +14006,7 @@
|
|||
"input_cost_per_token_flex": 3.7e-07,
|
||||
"input_cost_per_token_priority": 1.295e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 163840,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.22e-06,
|
||||
|
|
@ -14016,7 +14020,7 @@
|
|||
"supports_audio_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": false,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
|
||||
},
|
||||
"bedrock/ap-south-1/minimax.minimax-m2.1": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
|
|
@ -14140,12 +14144,12 @@
|
|||
"input_cost_per_token": 7.4e-07,
|
||||
"input_cost_per_token_flex": 3.7e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 163840,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.22e-06,
|
||||
"output_cost_per_token_flex": 1.11e-06,
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html",
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md",
|
||||
"supports_audio_input": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_native_structured_output": true,
|
||||
|
|
@ -14250,7 +14254,7 @@
|
|||
"input_cost_per_token_flex": 3.7e-07,
|
||||
"input_cost_per_token_priority": 1.295e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 163840,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.22e-06,
|
||||
|
|
@ -14264,7 +14268,7 @@
|
|||
"supports_audio_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": false,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
|
||||
},
|
||||
"bedrock/eu-north-1/minimax.minimax-m2.1": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
|
|
@ -14786,7 +14790,7 @@
|
|||
"input_cost_per_token_flex": 3.7e-07,
|
||||
"input_cost_per_token_priority": 1.295e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 163840,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.22e-06,
|
||||
|
|
@ -14800,7 +14804,7 @@
|
|||
"supports_audio_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": false,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
|
||||
},
|
||||
"bedrock/sa-east-1/minimax.minimax-m2.1": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
|
|
@ -15035,7 +15039,7 @@
|
|||
"input_cost_per_token_flex": 3.1e-07,
|
||||
"input_cost_per_token_priority": 1.085e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 163840,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.85e-06,
|
||||
|
|
@ -15049,7 +15053,7 @@
|
|||
"supports_audio_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": false,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
|
||||
},
|
||||
"bedrock/us-east-1/minimax.minimax-m2.1": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
|
|
@ -15156,7 +15160,7 @@
|
|||
"input_cost_per_token_flex": 3.1e-07,
|
||||
"input_cost_per_token_priority": 1.085e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 163840,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.85e-06,
|
||||
|
|
@ -15170,7 +15174,7 @@
|
|||
"supports_audio_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": false,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
|
||||
},
|
||||
"bedrock/us-east-2/minimax.minimax-m2.1": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
|
|
@ -15552,6 +15556,7 @@
|
|||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md"
|
||||
},
|
||||
"bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 2.65e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 8000,
|
||||
|
|
@ -15562,6 +15567,7 @@
|
|||
"supports_pdf_input": true
|
||||
},
|
||||
"bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 8000,
|
||||
|
|
@ -15708,7 +15714,7 @@
|
|||
"input_cost_per_token_flex": 3.1e-07,
|
||||
"input_cost_per_token_priority": 1.085e-06,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 163840,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.85e-06,
|
||||
|
|
@ -15722,7 +15728,7 @@
|
|||
"supports_audio_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": false,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
|
||||
},
|
||||
"bedrock/us-west-2/minimax.minimax-m2.1": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
|
|
@ -23519,12 +23525,12 @@
|
|||
]
|
||||
},
|
||||
"deepgram/streaming/nova-3": {
|
||||
"input_cost_per_second": 8e-05,
|
||||
"input_cost_per_second": 0.00012833,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0048/60 seconds = $0.00008000 per second",
|
||||
"calculation": "$0.0077/60 seconds = $0.00012833 per second",
|
||||
"note": "Nova-3 monolingual streaming, pay as you go",
|
||||
"original_pricing_per_minute": 0.0048
|
||||
"original_pricing_per_minute": 0.0077
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
|
|
@ -23533,13 +23539,28 @@
|
|||
"/v1/listen"
|
||||
]
|
||||
},
|
||||
"deepgram/streaming/nova-3-multilingual": {
|
||||
"input_cost_per_second": 9.667e-05,
|
||||
"deepgram/streaming/nova-3-medical": {
|
||||
"input_cost_per_second": 0.00012833,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0058/60 seconds = $0.00009667 per second",
|
||||
"calculation": "$0.0077/60 seconds = $0.00012833 per second",
|
||||
"note": "Nova-3 Medical streaming, launch price from Deepgram's 2025-03-03 announcement. deepgram.com/pricing does not list Nova-3 Medical, and this matches its current Nova-3 monolingual streaming pay as you go rate",
|
||||
"original_pricing_per_minute": 0.0077
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
"source": "https://www.businesswire.com/news/home/20250303470439/en",
|
||||
"supported_endpoints": [
|
||||
"/v1/listen"
|
||||
]
|
||||
},
|
||||
"deepgram/streaming/nova-3-multilingual": {
|
||||
"input_cost_per_second": 0.00015333,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0092/60 seconds = $0.00015333 per second",
|
||||
"note": "Nova-3 multilingual (language=multi) streaming, pay as you go",
|
||||
"original_pricing_per_minute": 0.0058
|
||||
"original_pricing_per_minute": 0.0092
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
|
|
@ -24519,7 +24540,7 @@
|
|||
"input_cost_per_token_flex": 3.1e-07,
|
||||
"input_cost_per_token_priority": 1.085e-06,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 164000,
|
||||
"max_input_tokens": 156000,
|
||||
"max_output_tokens": 8000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.85e-06,
|
||||
|
|
@ -24530,7 +24551,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json",
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md",
|
||||
"supports_audio_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": false
|
||||
|
|
@ -54457,6 +54478,8 @@
|
|||
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
|
||||
"input_cost_per_token_batches": 1e-06,
|
||||
"litellm_provider": "xai",
|
||||
"token_count_framing": 185,
|
||||
"token_count_tool_framing": 214,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 1000000,
|
||||
"mode": "chat",
|
||||
|
|
@ -62640,7 +62663,9 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"token_count_framing": 23,
|
||||
"token_count_tool_framing": 12
|
||||
},
|
||||
"deepseek/deepseek-v4-flash-vision-exp": {
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
|
|
@ -62746,7 +62771,9 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
"supports_vision": false,
|
||||
"token_count_framing": 76,
|
||||
"token_count_tool_framing": 75
|
||||
},
|
||||
"tencent/deepseek-v4-pro": {
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
|
|
@ -64032,6 +64059,7 @@
|
|||
"input_cost_per_token": 3e-07,
|
||||
"input_cost_per_token_priority": 3.75e-07,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"token_count_framing": 23,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 393216,
|
||||
"mode": "chat",
|
||||
|
|
@ -64133,6 +64161,7 @@
|
|||
"input_cost_per_token": 3e-07,
|
||||
"input_cost_per_token_priority": 3.75e-07,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"token_count_framing": 23,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 393216,
|
||||
"mode": "chat",
|
||||
|
|
@ -76801,6 +76830,7 @@
|
|||
"supports_audio_input": true,
|
||||
"supports_audio_output": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_continuation": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
|
|
@ -76860,6 +76890,7 @@
|
|||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_computer_use": true,
|
||||
"supports_continuation": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
|
|
@ -76917,6 +76948,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_continuation": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ dependencies = [
|
|||
"tiktoken>=0.12.0,<1.0; python_version == '3.14'",
|
||||
"tiktoken>=0.14.0,<1.0; python_version >= '3.15'",
|
||||
"tokenizers>=0.21.0,<1.0",
|
||||
"huggingface-hub>=0.34.0,<2.0",
|
||||
"huggingface-hub>=0.34.0,<3.0",
|
||||
"click>=8.0.0,<9.0",
|
||||
"jinja2>=3.1.6,<4.0",
|
||||
"aiohttp>=3.14.2,<4.0",
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ raw HTTP client imports (requests, urllib.request, httpx, aiohttp, http.client)
|
|||
banned in suite code, and tests/e2e_harness, which tests that harness, is held to the
|
||||
same ban. Importing requests' exception types for catching is fine anywhere; a small
|
||||
allowlist grandfathers the files that legitimately make raw calls (the transport
|
||||
itself, the root conftest liveness probe, the claude_code version resolver's constant
|
||||
itself, the root conftest liveness probe, the cli_harness npm version resolver's constant
|
||||
registry URL fetch, and the mcp OAuth client, whose httpx client is the object the
|
||||
official mcp SDK's streamable_http_client requires and so cannot go through the sync
|
||||
requests transport). Referenced by tests/e2e/AGENTS.md."""
|
||||
|
|
@ -24,7 +24,7 @@ BANNED_MODULES = ("requests", "urllib.request", "http.client", "httpx", "aiohttp
|
|||
ALLOWED_RAW_CLIENT_FILES = {
|
||||
"e2e/e2e_http.py": ("requests",),
|
||||
"e2e/conftest.py": ("requests",),
|
||||
"e2e/claude_code/pr_gate_version_resolver.py": ("urllib.request",),
|
||||
"e2e/cli_harness/npm_version_resolver.py": ("urllib.request",),
|
||||
}
|
||||
|
||||
EXCEPTION_ONLY_NAMES = frozenset({"RequestException", "ConnectionError", "Timeout", "HTTPError"})
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -16,23 +16,18 @@ from __future__ import annotations
|
|||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Union
|
||||
from typing import Any, Callable, Dict, Final, List, Mapping, Optional, Sequence, Tuple, Union
|
||||
|
||||
from cli_harness.lockdown import isolated_home, locked_down_env
|
||||
from cli_harness.rate_limiter import RateBucket, RateLimiter, get_default_limiter
|
||||
from e2e_metadata import step
|
||||
|
||||
from claude_code.rate_limiter import (
|
||||
RateLimiter,
|
||||
get_default_limiter,
|
||||
infer_provider,
|
||||
)
|
||||
|
||||
CLAUDE_CLI_DEFAULT = "claude"
|
||||
# 120s is fine for a single isolated CLI call against an unloaded
|
||||
# upstream, but the matrix run launches up to 75 concurrent calls and
|
||||
|
|
@ -77,61 +72,36 @@ Anthropic / Bedrock / Vertex) to reset. Both knobs are env-tunable so
|
|||
a matrix run can trade wall time for resilience without code edits;
|
||||
retries=0 disables the behavior entirely."""
|
||||
|
||||
# Env vars the `claude` Node CLI legitimately needs to function:
|
||||
# locating its own binary + node, basic locale/terminal plumbing.
|
||||
# Deliberately excludes every credential-bearing var that the
|
||||
# surrounding CI job sets for the proxy (ANTHROPIC_API_KEY, AWS_*,
|
||||
# AZURE_*, VERTEXAI_CREDENTIALS, GITHUB_TOKEN, OPENAI_API_KEY,
|
||||
# DATABASE_URL, ...). The CLI talks to the proxy via the explicit
|
||||
# ANTHROPIC_BASE_URL/ANTHROPIC_AUTH_TOKEN we set below — it has no
|
||||
# business reading the proxy's upstream credentials, and a
|
||||
# compromised CLI release shouldn't be able to exfiltrate them out
|
||||
# of the CI environment.
|
||||
#
|
||||
# `HOME` is intentionally NOT in this list: see `_make_isolated_home`
|
||||
# below. We give the CLI a fresh empty per-invocation HOME so a
|
||||
# compromised claude package or a model-directed `Read` tool call
|
||||
# can't reach files like `~/.config/gh/hosts.yml`, `~/.ssh/id_rsa`,
|
||||
# or `~/.bash_history` on the cron VM (and on the CircleCI executor
|
||||
# the same isolation prevents accidentally exposing checkout-adjacent
|
||||
# files even though the runner home is ephemeral there).
|
||||
_CLI_ENV_ALLOWLIST: tuple = (
|
||||
"PATH",
|
||||
"USER",
|
||||
"LOGNAME",
|
||||
"SHELL",
|
||||
"TERM",
|
||||
"TMPDIR",
|
||||
"LANG",
|
||||
"LC_ALL",
|
||||
"LC_CTYPE",
|
||||
"NODE_PATH",
|
||||
"NVM_DIR",
|
||||
"NVM_BIN",
|
||||
)
|
||||
|
||||
def infer_provider(model: str) -> RateBucket:
|
||||
"""Map a model alias to its compat-matrix rate bucket.
|
||||
|
||||
def _make_isolated_home() -> str:
|
||||
"""Create a fresh empty HOME directory for a single `claude` subprocess.
|
||||
|
||||
The CLI needs *a* writable HOME (it caches per-session state under
|
||||
`$HOME/.claude/projects/<sha>/`), but it has no legitimate need
|
||||
for the *user's* HOME. Handing it the real one means a compromised
|
||||
`@anthropic-ai/claude-code` release, or a model-directed `Read`
|
||||
tool call during a PDF/vision cell, can read host files like
|
||||
`~/.config/gh/hosts.yml` (GitHub CLI host token), `~/.ssh/`,
|
||||
`~/.bash_history`, or any other dotfile under the runtime user's
|
||||
home. On the cron VM the runtime user is a real interactive
|
||||
account (`mateo`) with a populated home directory, so this is a
|
||||
real exfiltration surface.
|
||||
|
||||
The directory is created under `tempfile.gettempdir()` (which is
|
||||
`/tmp` on Linux; under systemd's `PrivateTmp=true` that's a
|
||||
per-service tmpfs that the service user can't otherwise reach).
|
||||
Caller is responsible for `shutil.rmtree`-ing it after the
|
||||
subprocess exits.
|
||||
The matrix column layout is fixed; aliases registered in the proxy
|
||||
encode the provider via a suffix (`-bedrock-converse`,
|
||||
`-bedrock-invoke`, `-azure`, `-vertex`, `-openai`, `-azure-openai`,
|
||||
`-bedrock-mantle`) or its absence (Anthropic). Order matters:
|
||||
`-azure-openai` also ends with `-openai`, and the bedrock suffixes
|
||||
all contain `bedrock`, so the more-specific suffixes are tested
|
||||
first.
|
||||
"""
|
||||
return tempfile.mkdtemp(prefix="claude-cli-home-")
|
||||
if not model:
|
||||
raise ValueError("model must be a non-empty string")
|
||||
lower: Final = model.lower()
|
||||
if lower.endswith("-azure-openai"):
|
||||
return RateBucket.AZURE_OPENAI
|
||||
if lower.endswith("-openai"):
|
||||
return RateBucket.OPENAI
|
||||
if lower.endswith("-bedrock-mantle"):
|
||||
return RateBucket.BEDROCK_MANTLE
|
||||
if lower.endswith("-bedrock-converse"):
|
||||
return RateBucket.BEDROCK_CONVERSE
|
||||
if lower.endswith("-bedrock-invoke"):
|
||||
return RateBucket.BEDROCK_INVOKE
|
||||
if lower.endswith("-azure"):
|
||||
return RateBucket.AZURE
|
||||
if lower.endswith("-vertex"):
|
||||
return RateBucket.VERTEX_AI
|
||||
return RateBucket.ANTHROPIC
|
||||
|
||||
|
||||
_FIXED_CLI_USER_ID = "0" * 64
|
||||
|
|
@ -176,7 +146,7 @@ def _stable_cli_state() -> Tuple[str, str]:
|
|||
Pinning both makes the bodies repeatable across builds.
|
||||
|
||||
This narrows what survives rather than widening it: HOME stays fresh and
|
||||
empty per invocation, so the isolation `_make_isolated_home` describes is
|
||||
empty per invocation, so the isolation `cli_harness.lockdown` describes is
|
||||
unchanged, and the CLI's own state no longer outlives the pod either. The
|
||||
working directory is deliberately not the checkout, so a model-directed
|
||||
`Read` sees an empty directory instead of the repository.
|
||||
|
|
@ -291,38 +261,26 @@ def run_claude(
|
|||
cmd.append("--")
|
||||
cmd.append(prompt)
|
||||
|
||||
# Build a minimal env for the CLI subprocess: only the allowlisted
|
||||
# process-runtime vars from os.environ, plus the explicit proxy
|
||||
# creds, plus any caller-supplied overrides. See _CLI_ENV_ALLOWLIST
|
||||
# above for the security rationale.
|
||||
env: Dict[str, str] = {
|
||||
key: os.environ[key] for key in _CLI_ENV_ALLOWLIST if key in os.environ
|
||||
}
|
||||
env["ANTHROPIC_BASE_URL"] = base_url
|
||||
env["ANTHROPIC_AUTH_TOKEN"] = api_key
|
||||
env["DISABLE_AUTOUPDATER"] = "1"
|
||||
# Hand the CLI a fresh empty HOME so a compromised claude package
|
||||
# or a model-directed Read tool call can't see the runtime user's
|
||||
# real dotfiles. Created here, removed in the `finally` below
|
||||
# regardless of how the subprocess exits.
|
||||
isolated_home = _make_isolated_home()
|
||||
env["HOME"] = isolated_home
|
||||
config_dir, workspace = _stable_cli_state()
|
||||
env["CLAUDE_CONFIG_DIR"] = config_dir
|
||||
if extra_env:
|
||||
env.update(extra_env)
|
||||
|
||||
# Throttle by provider *before* launching the CLI. Doing this here
|
||||
# (rather than per-test) means every code path that lands on
|
||||
# `run_claude` is rate-limited automatically — including
|
||||
# `run_claude_models_parallel`, which is the hot path during the
|
||||
# full matrix run.
|
||||
limiter = rate_limiter if rate_limiter is not None else get_default_limiter()
|
||||
provider = infer_provider(model)
|
||||
limiter.acquire(provider)
|
||||
|
||||
run_fn = runner or subprocess.run
|
||||
try:
|
||||
with isolated_home("claude-cli-home-") as home:
|
||||
config_dir, workspace = _stable_cli_state()
|
||||
env = locked_down_env(
|
||||
home=home,
|
||||
overrides={
|
||||
"ANTHROPIC_BASE_URL": base_url,
|
||||
"ANTHROPIC_AUTH_TOKEN": api_key,
|
||||
"DISABLE_AUTOUPDATER": "1",
|
||||
"CLAUDE_CONFIG_DIR": config_dir,
|
||||
**(extra_env or {}),
|
||||
},
|
||||
)
|
||||
limiter.acquire(infer_provider(model))
|
||||
try:
|
||||
completed = run_fn(
|
||||
cmd,
|
||||
|
|
@ -340,15 +298,6 @@ def run_claude(
|
|||
) from exc
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
raise ClaudeCLIError(f"claude CLI timed out after {timeout}s") from exc
|
||||
finally:
|
||||
# Best-effort cleanup. If the subprocess wrote a `.claude/`
|
||||
# session dir under the isolated HOME, we remove it here so
|
||||
# parallel matrix runs don't accumulate per-call tmpdirs.
|
||||
# `ignore_errors=True` because rmtree races with any
|
||||
# not-yet-reaped child (SIGTERM'd `claude` on a host-side
|
||||
# timeout) are benign — the next matrix run starts from a
|
||||
# fresh tmpdir anyway.
|
||||
shutil.rmtree(isolated_home, ignore_errors=True)
|
||||
|
||||
events = _parse_stream_json(completed.stdout or "")
|
||||
text = _extract_assistant_text(events)
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ than as a GitHub Action or on a dedicated VM. Trade-offs:
|
|||
the stable venv doesn't install) is never loaded. The tag's own
|
||||
`tests/e2e/` is deliberately not used.
|
||||
3. **Resolves and installs the Claude Code CLI under test**:
|
||||
`pr_gate_version_resolver.py` (run on the venv, the image has no
|
||||
`tests/e2e/cli_harness/npm_version_resolver.py` (run on the venv, the image has no
|
||||
Python of its own) picks the newest `@anthropic-ai/claude-code` npm
|
||||
release published at least 3 days ago, the same buffer the PR gate
|
||||
uses, unless `CLAUDE_CODE_VERSION` pins one, and
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
# Releases API.
|
||||
# 2. Update a long-lived worktree at $WORKTREE to that tag and `uv sync` it.
|
||||
# 3. Resolve the Claude Code CLI version under test (the newest npm
|
||||
# release at least 3 days old, via pr_gate_version_resolver.py, or
|
||||
# release at least 3 days old, via cli_harness/npm_version_resolver.py, or
|
||||
# $CLAUDE_CODE_VERSION when set) and download that release's
|
||||
# linux-x64 binary into the run's scratch dir, checksum-verified
|
||||
# against the vendor's release manifest (install_claude_code.sh).
|
||||
|
|
@ -329,7 +329,7 @@ PROXY_CONFIG="${WORKTREE}/tests/e2e/claude_code/test_config.yaml"
|
|||
if [[ -z "${CLAUDE_CODE_VERSION}" ]]; then
|
||||
CLAUDE_CODE_VERSION="$(
|
||||
cd "${WORKTREE}" \
|
||||
&& "${WORKTREE_UV}" run --no-sync python "${POPULATOR_DIR}/../pr_gate_version_resolver.py"
|
||||
&& "${WORKTREE_UV}" run --no-sync python "${POPULATOR_DIR}/../../cli_harness/npm_version_resolver.py" @anthropic-ai/claude-code
|
||||
)" || die "could not resolve the Claude Code version to test"
|
||||
log "resolved claude code: ${CLAUDE_CODE_VERSION}"
|
||||
else
|
||||
|
|
|
|||
|
|
@ -31,8 +31,7 @@ from __future__ import annotations
|
|||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from cli_harness.rate_limiter import RateLimiter, get_default_limiter
|
||||
from e2e_http import (
|
||||
NetworkError,
|
||||
RateLimitedError,
|
||||
|
|
@ -59,12 +58,9 @@ from models import (
|
|||
JsonSchemaProperty,
|
||||
ToolInputSchema,
|
||||
)
|
||||
from pydantic import BaseModel
|
||||
|
||||
from claude_code.rate_limiter import (
|
||||
RateLimiter,
|
||||
get_default_limiter,
|
||||
infer_provider,
|
||||
)
|
||||
from claude_code.cli_driver import infer_provider
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from proxy_client import ProxyClient
|
||||
|
|
|
|||
|
|
@ -1,155 +1,16 @@
|
|||
"""Claude Code PR-Gate Version Resolver.
|
||||
|
||||
Resolves the `@anthropic-ai/claude-code` npm version that the PR-gate CI
|
||||
job installs. Selects the newest version (by publish timestamp) whose
|
||||
publish timestamp is at least 3 days old. The 3-day window is a security
|
||||
review buffer — see PRD #26476, "Version resolvers".
|
||||
|
||||
Two surfaces:
|
||||
|
||||
- ``resolve_pr_gate_version(...)`` — the importable function. Accepts
|
||||
pre-fetched npm metadata (for unit tests) or a custom ``fetcher``
|
||||
callable. The default fetcher hits the public npm registry.
|
||||
- ``python -m claude_code.pr_gate_version_resolver`` — prints the
|
||||
resolved version string to stdout, suitable for piping into a shell
|
||||
``$(...)`` substitution inside the CircleCI job.
|
||||
|
||||
The CLI form is what CircleCI runs at job start; engineers reading the
|
||||
job log can see the selected version on a single line above the
|
||||
``npm install -g`` step (acceptance criterion: "the selected Claude
|
||||
Code version is logged in the CI output").
|
||||
"""
|
||||
"""The path the Buildkite runner images run (`python3 tests/e2e/claude_code/pr_gate_version_resolver.py`) to pick the
|
||||
Claude Code release they install. They fall back to @latest when this file is missing, so keep it until they call
|
||||
`cli_harness/npm_version_resolver.py` themselves."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Callable, Mapping, Optional
|
||||
|
||||
PACKAGE_NAME = "@anthropic-ai/claude-code"
|
||||
NPM_REGISTRY_URL = "https://registry.npmjs.org/{package}"
|
||||
DEFAULT_MIN_AGE = timedelta(days=3)
|
||||
DEFAULT_FETCH_TIMEOUT_SECONDS = 30
|
||||
|
||||
# npm's `time` map mixes per-version timestamps with these meta keys.
|
||||
_TIME_META_KEYS = frozenset({"created", "modified"})
|
||||
|
||||
|
||||
class NoEligibleVersionError(RuntimeError):
|
||||
"""Raised when no version in the npm metadata satisfies the min-age cutoff."""
|
||||
|
||||
|
||||
def _parse_npm_timestamp(value: str) -> datetime:
|
||||
"""Parse the ISO-8601 timestamps npm emits (always UTC, may use ``Z``)."""
|
||||
if value.endswith("Z"):
|
||||
value = value[:-1] + "+00:00"
|
||||
parsed = datetime.fromisoformat(value)
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed
|
||||
|
||||
|
||||
def _default_fetcher(package_name: str) -> dict:
|
||||
"""Fetch the npm packument for ``package_name`` over HTTPS.
|
||||
|
||||
Uses urllib (stdlib) so this module has no extra dependencies in the
|
||||
CI environment. Returns the raw JSON dict.
|
||||
"""
|
||||
# urllib.parse.quote would encode the leading '@' / '/' which the
|
||||
# npm registry expects literally; do a minimal hand-roll instead.
|
||||
url = NPM_REGISTRY_URL.format(package=package_name.replace("/", "%2F"))
|
||||
req = urllib.request.Request(url, headers={"Accept": "application/json"})
|
||||
with urllib.request.urlopen( # noqa: S310 — registry URL is constant
|
||||
req, timeout=DEFAULT_FETCH_TIMEOUT_SECONDS
|
||||
) as response:
|
||||
body = response.read().decode("utf-8")
|
||||
return json.loads(body)
|
||||
|
||||
|
||||
def resolve_pr_gate_version(
|
||||
*,
|
||||
metadata: Optional[Mapping] = None,
|
||||
fetcher: Optional[Callable[[str], Mapping]] = None,
|
||||
as_of: Optional[datetime] = None,
|
||||
min_age: timedelta = DEFAULT_MIN_AGE,
|
||||
package_name: str = PACKAGE_NAME,
|
||||
) -> str:
|
||||
"""Return the newest npm version of ``package_name`` published >= ``min_age`` ago.
|
||||
|
||||
"Newest" means newest by **publish time**, not semver string order —
|
||||
if a patch lands on an older major after a newer release, the
|
||||
patched line is the eligible one. A version npm has unpublished keeps
|
||||
its ``time`` entry but drops out of ``versions``, so only versions
|
||||
still present in ``versions`` are candidates.
|
||||
|
||||
Args:
|
||||
metadata: Pre-fetched npm packument (skips the HTTP call). Useful
|
||||
for unit tests.
|
||||
fetcher: Callable taking a package name and returning the
|
||||
packument. Defaults to a stdlib HTTPS fetcher.
|
||||
as_of: The clock used to decide whether a version is "old
|
||||
enough". Defaults to ``datetime.now(timezone.utc)``.
|
||||
min_age: Minimum publish age. Defaults to 3 days.
|
||||
package_name: Defaults to ``@anthropic-ai/claude-code``.
|
||||
|
||||
Raises:
|
||||
NoEligibleVersionError: when no version in the registry meets
|
||||
the age cutoff.
|
||||
"""
|
||||
if metadata is None:
|
||||
fetch = fetcher or _default_fetcher
|
||||
metadata = fetch(package_name)
|
||||
|
||||
times = metadata.get("time") or {}
|
||||
versions = metadata.get("versions") or {}
|
||||
if as_of is None:
|
||||
as_of = datetime.now(timezone.utc)
|
||||
cutoff = as_of - min_age
|
||||
|
||||
eligible: list[tuple[datetime, str]] = []
|
||||
for version, raw_ts in times.items():
|
||||
if version in _TIME_META_KEYS:
|
||||
continue
|
||||
if version not in versions:
|
||||
continue
|
||||
if not isinstance(raw_ts, str):
|
||||
continue
|
||||
if "-" in version:
|
||||
continue
|
||||
published = _parse_npm_timestamp(raw_ts)
|
||||
if published <= cutoff:
|
||||
eligible.append((published, version))
|
||||
|
||||
if not eligible:
|
||||
raise NoEligibleVersionError(
|
||||
f"no version of {package_name} is at least {min_age} old "
|
||||
f"as of {as_of.isoformat()}"
|
||||
)
|
||||
|
||||
eligible.sort(key=lambda pair: pair[0], reverse=True)
|
||||
return eligible[0][1]
|
||||
|
||||
|
||||
def _main(argv: list[str]) -> int:
|
||||
"""Print the resolved version to stdout. Exit code 0 on success.
|
||||
|
||||
Stderr carries the human-readable announcement so the version can be
|
||||
captured cleanly with ``$(python -m ...)`` in shell.
|
||||
"""
|
||||
try:
|
||||
version = resolve_pr_gate_version()
|
||||
except Exception as exc: # noqa: BLE001 — CLI surface, want everything
|
||||
print(f"pr_gate_version_resolver: {exc}", file=sys.stderr) # noqa: T201
|
||||
return 1
|
||||
print( # noqa: T201
|
||||
f"pr_gate_version_resolver: selected {PACKAGE_NAME}@{version}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(version) # noqa: T201
|
||||
return 0
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
_RESOLVER: Final = Path(__file__).resolve().parent.parent / "cli_harness" / "npm_version_resolver.py"
|
||||
_PACKAGE: Final = "@anthropic-ai/claude-code"
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(_main(sys.argv[1:]))
|
||||
raise SystemExit(subprocess.run((sys.executable, "-I", str(_RESOLVER), _PACKAGE), check=False).returncode)
|
||||
|
|
|
|||
|
|
@ -1,378 +0,0 @@
|
|||
"""Cross-process token-bucket rate limiter for the Claude Code compat suite.
|
||||
|
||||
The compat matrix runs 75 live `claude` CLI invocations (25 cells × 3
|
||||
Claude tiers per cell). When pytest-xdist fans these out across worker
|
||||
processes, each worker would maintain its own in-memory rate limiter
|
||||
and the *aggregate* request rate hitting any one upstream provider
|
||||
would be `workers × per-worker rate` — exactly the situation that
|
||||
trips Anthropic / Azure / Bedrock / Vertex 429s in the middle of a
|
||||
matrix run and silently flips green cells red.
|
||||
|
||||
The fix is a **shared, cross-process** token bucket per provider,
|
||||
backed by a small JSON state file and an OS-level `flock`. Each
|
||||
`run_claude` invocation acquires one token (sleeping if the bucket is
|
||||
empty) before launching the CLI; refills happen lazily based on wall
|
||||
time, so workers can be killed and restarted without losing or
|
||||
double-spending budget.
|
||||
|
||||
Configuration is driven entirely by environment variables so a
|
||||
binary-search workflow can shift per-provider rates without code
|
||||
edits:
|
||||
|
||||
LITELLM_COMPAT_RATE_ANTHROPIC (req/s, default 5.0)
|
||||
LITELLM_COMPAT_RATE_AZURE (req/s, default 5.0)
|
||||
LITELLM_COMPAT_RATE_VERTEX_AI (req/s, default 5.0)
|
||||
LITELLM_COMPAT_RATE_BEDROCK_CONVERSE (req/s, default 5.0)
|
||||
LITELLM_COMPAT_RATE_BEDROCK_INVOKE (req/s, default 5.0)
|
||||
LITELLM_COMPAT_RATE_OPENAI (req/s, default 5.0)
|
||||
LITELLM_COMPAT_RATE_AZURE_OPENAI (req/s, default 5.0)
|
||||
LITELLM_COMPAT_RATE_BEDROCK_MANTLE (req/s, default 5.0)
|
||||
LITELLM_COMPAT_RATE_BURST (per-bucket burst override;
|
||||
default = rate)
|
||||
LITELLM_COMPAT_RATE_STATE_DIR (state file directory;
|
||||
default = $TMPDIR/litellm-claude-compat-ratelimit)
|
||||
|
||||
A rate of 0 (or any non-positive value) disables throttling for that
|
||||
provider — useful when you trust the upstream to handle the burst or
|
||||
when running unit-test-shaped workloads that never actually hit the
|
||||
network.
|
||||
|
||||
The provider id is inferred from the model id by `infer_provider`,
|
||||
mirroring the matrix's column layout (`anthropic`, `azure`,
|
||||
`vertex_ai`, `bedrock_converse`, `bedrock_invoke`, `openai`,
|
||||
`azure_openai`, `bedrock_mantle`). The `vertex_ai_gpt` matrix column
|
||||
has no bucket: its cells are static not_applicable stubs that never
|
||||
reach the network.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterator, Mapping, Optional
|
||||
|
||||
# `fcntl` is POSIX-only; the suite is Linux/macOS only, so we don't
|
||||
# attempt a Windows fallback. Importing at module load fails fast on
|
||||
# the (currently non-existent) Windows runner so we don't silently
|
||||
# degrade to no-locking behavior.
|
||||
import fcntl
|
||||
|
||||
PROVIDER_ANTHROPIC = "anthropic"
|
||||
PROVIDER_AZURE = "azure"
|
||||
PROVIDER_VERTEX_AI = "vertex_ai"
|
||||
PROVIDER_BEDROCK_CONVERSE = "bedrock_converse"
|
||||
PROVIDER_BEDROCK_INVOKE = "bedrock_invoke"
|
||||
PROVIDER_OPENAI = "openai"
|
||||
PROVIDER_AZURE_OPENAI = "azure_openai"
|
||||
PROVIDER_BEDROCK_MANTLE = "bedrock_mantle"
|
||||
|
||||
ALL_PROVIDERS = (
|
||||
PROVIDER_ANTHROPIC,
|
||||
PROVIDER_AZURE,
|
||||
PROVIDER_VERTEX_AI,
|
||||
PROVIDER_BEDROCK_CONVERSE,
|
||||
PROVIDER_BEDROCK_INVOKE,
|
||||
PROVIDER_OPENAI,
|
||||
PROVIDER_AZURE_OPENAI,
|
||||
PROVIDER_BEDROCK_MANTLE,
|
||||
)
|
||||
|
||||
DEFAULT_RATE = 5.0 # req/s per provider, conservative starting point
|
||||
RATE_ENV_PREFIX = "LITELLM_COMPAT_RATE_"
|
||||
BURST_ENV = "LITELLM_COMPAT_RATE_BURST"
|
||||
STATE_DIR_ENV = "LITELLM_COMPAT_RATE_STATE_DIR"
|
||||
DEFAULT_STATE_DIR_NAME = "litellm-claude-compat-ratelimit"
|
||||
|
||||
|
||||
def infer_provider(model: str) -> str:
|
||||
"""Map a model alias to its compat-matrix provider id.
|
||||
|
||||
The matrix column layout is fixed; aliases registered in the proxy
|
||||
encode the provider via a suffix (`-bedrock-converse`,
|
||||
`-bedrock-invoke`, `-azure`, `-vertex`, `-openai`, `-azure-openai`,
|
||||
`-bedrock-mantle`) or its absence (Anthropic). Order matters:
|
||||
`-azure-openai` also ends with `-openai`, and the bedrock suffixes
|
||||
all contain `bedrock`, so the more-specific suffixes are tested
|
||||
first.
|
||||
"""
|
||||
if not model:
|
||||
raise ValueError("model must be a non-empty string")
|
||||
lower = model.lower()
|
||||
if lower.endswith("-azure-openai"):
|
||||
return PROVIDER_AZURE_OPENAI
|
||||
if lower.endswith("-openai"):
|
||||
return PROVIDER_OPENAI
|
||||
if lower.endswith("-bedrock-mantle"):
|
||||
return PROVIDER_BEDROCK_MANTLE
|
||||
if lower.endswith("-bedrock-converse"):
|
||||
return PROVIDER_BEDROCK_CONVERSE
|
||||
if lower.endswith("-bedrock-invoke"):
|
||||
return PROVIDER_BEDROCK_INVOKE
|
||||
if lower.endswith("-azure"):
|
||||
return PROVIDER_AZURE
|
||||
if lower.endswith("-vertex"):
|
||||
return PROVIDER_VERTEX_AI
|
||||
return PROVIDER_ANTHROPIC
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProviderConfig:
|
||||
"""Static config snapshot for one provider's bucket.
|
||||
|
||||
Captured up front (rather than re-read per acquire) so the limiter's
|
||||
behavior in a single process is stable even if env vars are mutated
|
||||
mid-run. A fresh `RateLimiter` picks up env changes on construction.
|
||||
"""
|
||||
|
||||
rate_per_sec: float
|
||||
burst: float
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self.rate_per_sec > 0 and self.burst > 0
|
||||
|
||||
|
||||
def load_config(
|
||||
env: Optional[Mapping[str, str]] = None,
|
||||
) -> Dict[str, ProviderConfig]:
|
||||
"""Build a {provider: ProviderConfig} from env, applying defaults.
|
||||
|
||||
Parsing failures fall back to the default rate rather than
|
||||
crashing the test session — a typo in `LITELLM_COMPAT_RATE_AZURE`
|
||||
should not silently disable throttling, but it also shouldn't
|
||||
abort 75 live tests with a `ValueError` ten minutes in.
|
||||
"""
|
||||
src = env if env is not None else os.environ
|
||||
|
||||
def _as_float(value: Optional[str], default: float) -> float:
|
||||
if value is None or value == "":
|
||||
return default
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
burst_override = _as_float(src.get(BURST_ENV), -1.0)
|
||||
|
||||
out: Dict[str, ProviderConfig] = {}
|
||||
for provider in ALL_PROVIDERS:
|
||||
env_key = RATE_ENV_PREFIX + provider.upper()
|
||||
rate = _as_float(src.get(env_key), DEFAULT_RATE)
|
||||
burst = burst_override if burst_override > 0 else max(rate, 1.0)
|
||||
out[provider] = ProviderConfig(rate_per_sec=rate, burst=burst)
|
||||
return out
|
||||
|
||||
|
||||
def _state_dir(env: Optional[Mapping[str, str]] = None) -> Path:
|
||||
"""Resolve the directory holding per-provider state files.
|
||||
|
||||
A user-supplied `LITELLM_COMPAT_RATE_STATE_DIR` wins for tests and
|
||||
container environments where `$TMPDIR` may be ephemeral or shared
|
||||
in surprising ways. The directory is created lazily with
|
||||
`parents=True, exist_ok=True` so first-run setup needs no
|
||||
fixture wiring.
|
||||
"""
|
||||
src = env if env is not None else os.environ
|
||||
explicit = src.get(STATE_DIR_ENV)
|
||||
if explicit:
|
||||
return Path(explicit)
|
||||
return Path(tempfile.gettempdir()) / DEFAULT_STATE_DIR_NAME
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
"""Cross-process token bucket per provider.
|
||||
|
||||
Each `acquire(provider)` call:
|
||||
1. Opens (creating if needed) `<state_dir>/<provider>.json`.
|
||||
2. Holds an exclusive `flock` while reading + updating the
|
||||
{tokens, last_refill} state.
|
||||
3. Refills tokens based on `now - last_refill`, capped at burst.
|
||||
4. If tokens >= 1, subtracts one and returns immediately.
|
||||
5. Otherwise, computes the wall-time delay needed to earn a
|
||||
single token at the configured rate, releases the lock, and
|
||||
sleeps. After sleeping it retries — staying under the lock
|
||||
while sleeping would serialize all workers behind whichever
|
||||
one held it longest.
|
||||
|
||||
This bounds the *aggregate* req/s seen by the upstream, regardless
|
||||
of how many xdist workers, threads, or processes are concurrently
|
||||
running tests against the same provider.
|
||||
|
||||
A `_clock` / `_sleep` injection seam keeps the unit tests fast and
|
||||
deterministic; production callers should never override either.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Optional[Mapping[str, ProviderConfig]] = None,
|
||||
state_dir: Optional[Path] = None,
|
||||
clock: Optional[callable] = None,
|
||||
sleep: Optional[callable] = None,
|
||||
) -> None:
|
||||
self._config = dict(config) if config is not None else load_config()
|
||||
self._state_dir = Path(state_dir) if state_dir is not None else _state_dir()
|
||||
self._clock = clock or time.monotonic
|
||||
self._sleep = sleep or time.sleep
|
||||
# `_state_dir.mkdir` once on construction is fine; concurrent
|
||||
# workers all racing to create the same directory is benign.
|
||||
self._state_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def acquire(self, provider: str) -> float:
|
||||
"""Block until one token is available for `provider`.
|
||||
|
||||
Returns the cumulative wall-time spent waiting (0.0 when the
|
||||
bucket had budget and we returned immediately). Callers can
|
||||
log this to attribute slow cells to throttling vs. upstream
|
||||
latency — same role `DriverResult.duration_ms` plays for the
|
||||
actual CLI invocation.
|
||||
"""
|
||||
cfg = self._config.get(provider)
|
||||
if cfg is None or not cfg.enabled:
|
||||
return 0.0
|
||||
|
||||
path = self._state_path(provider)
|
||||
total_waited = 0.0
|
||||
while True:
|
||||
now = self._clock()
|
||||
sleep_for = self._try_consume(path, cfg, now)
|
||||
if sleep_for <= 0:
|
||||
return total_waited
|
||||
self._sleep(sleep_for)
|
||||
total_waited += sleep_for
|
||||
|
||||
def _state_path(self, provider: str) -> Path:
|
||||
return self._state_dir / f"{provider}.json"
|
||||
|
||||
def _try_consume(self, path: Path, cfg: ProviderConfig, now: float) -> float:
|
||||
"""Atomically refill and try to take one token.
|
||||
|
||||
Returns 0.0 if a token was consumed, or a positive sleep
|
||||
duration (seconds) if the caller must wait before retrying.
|
||||
|
||||
We hold an exclusive `flock` only across the read-modify-write
|
||||
of the JSON state — never across a `sleep` — so workers don't
|
||||
serialize while one of them is parked.
|
||||
"""
|
||||
# `os.open` + `os.O_CREAT | os.O_RDWR` gives us a fd we can
|
||||
# both lock and read/write through. Opening with `"a+"` then
|
||||
# seeking is equivalent but uglier; this version is closer to
|
||||
# the canonical flock recipe.
|
||||
fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o600)
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||
try:
|
||||
tokens, last_refill = self._read_state(fd, cfg, now)
|
||||
tokens, last_refill = self._refill(tokens, last_refill, now, cfg)
|
||||
if tokens >= 1.0:
|
||||
tokens -= 1.0
|
||||
self._write_state(fd, tokens, last_refill)
|
||||
return 0.0
|
||||
# Not enough budget. Persist the refilled state so a
|
||||
# subsequent caller doesn't have to redo the math, then
|
||||
# release the lock and tell the caller how long to
|
||||
# sleep before retrying.
|
||||
self._write_state(fd, tokens, last_refill)
|
||||
deficit = 1.0 - tokens
|
||||
# `deficit / rate` seconds will earn exactly enough
|
||||
# for one token. Add a tiny safety margin so we don't
|
||||
# wake up nanoseconds early and spin.
|
||||
return (deficit / cfg.rate_per_sec) + 1e-3
|
||||
finally:
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
@staticmethod
|
||||
def _read_state(fd: int, cfg: ProviderConfig, now: float) -> tuple:
|
||||
"""Read {tokens, last_refill} from `fd`, defaulting to a full
|
||||
bucket on a missing/empty/corrupt file.
|
||||
|
||||
New files start full so the very first request never waits;
|
||||
corrupt files are treated like new files because the
|
||||
alternative — refusing to run — is worse than briefly
|
||||
over-spending one bucket's worth of budget.
|
||||
"""
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
raw = os.read(fd, 4096).decode("utf-8")
|
||||
if not raw.strip():
|
||||
return cfg.burst, now
|
||||
try:
|
||||
obj = json.loads(raw)
|
||||
tokens = float(obj.get("tokens", cfg.burst))
|
||||
last_refill = float(obj.get("last_refill", now))
|
||||
return tokens, last_refill
|
||||
except (ValueError, TypeError):
|
||||
return cfg.burst, now
|
||||
|
||||
@staticmethod
|
||||
def _refill(
|
||||
tokens: float, last_refill: float, now: float, cfg: ProviderConfig
|
||||
) -> tuple:
|
||||
"""Apply elapsed time to the bucket, capped at burst.
|
||||
|
||||
Negative elapsed (clock went backward, e.g. across host
|
||||
sleep/resume or a manually-tweaked monotonic mock) is clamped
|
||||
to zero so we never *remove* tokens.
|
||||
"""
|
||||
elapsed = max(0.0, now - last_refill)
|
||||
tokens = min(cfg.burst, tokens + elapsed * cfg.rate_per_sec)
|
||||
return tokens, now
|
||||
|
||||
@staticmethod
|
||||
def _write_state(fd: int, tokens: float, last_refill: float) -> None:
|
||||
payload = json.dumps({"tokens": tokens, "last_refill": last_refill}).encode(
|
||||
"utf-8"
|
||||
)
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
os.ftruncate(fd, 0)
|
||||
os.write(fd, payload)
|
||||
|
||||
|
||||
# A single process-wide instance is all we need: provider-keyed state
|
||||
# is stored in files, so multiple `RateLimiter` instances would just
|
||||
# duplicate the in-process bookkeeping. We expose a getter rather than
|
||||
# the instance directly so unit tests can install a custom limiter
|
||||
# scoped to a tmp directory without monkeypatching globals.
|
||||
_default: Optional[RateLimiter] = None
|
||||
|
||||
|
||||
def get_default_limiter() -> RateLimiter:
|
||||
global _default
|
||||
if _default is None:
|
||||
_default = RateLimiter()
|
||||
return _default
|
||||
|
||||
|
||||
def reset_default_limiter() -> None:
|
||||
"""Drop the cached default limiter; the next `get_default_limiter`
|
||||
call rebuilds it from the current environment.
|
||||
|
||||
Useful between unit tests that patch env vars: without this they'd
|
||||
keep reading the stale config snapshot from the first call.
|
||||
"""
|
||||
global _default
|
||||
_default = None
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def use_limiter(limiter: RateLimiter) -> Iterator[RateLimiter]:
|
||||
"""Temporarily install `limiter` as the process default.
|
||||
|
||||
The driver's `run_claude` calls `get_default_limiter()`; tests that
|
||||
want a controlled tmp-dir-backed limiter use this contextmanager
|
||||
to swap one in without touching env vars or the on-disk default
|
||||
state.
|
||||
"""
|
||||
global _default
|
||||
previous = _default
|
||||
_default = limiter
|
||||
try:
|
||||
yield limiter
|
||||
finally:
|
||||
_default = previous
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env bash
|
||||
# Run the Claude Code compat matrix end-to-end against a live LiteLLM
|
||||
# proxy, with per-provider rate limits applied via the cross-process
|
||||
# token bucket in `tests/e2e/claude_code/rate_limiter.py`.
|
||||
# token bucket in `tests/e2e/cli_harness/rate_limiter.py`.
|
||||
#
|
||||
# Designed for binary-searching the ideal X / Y / Z req/s per provider:
|
||||
# 1. Pick an initial rate (e.g. 5/s for everyone).
|
||||
|
|
|
|||
0
tests/e2e/cli_harness/__init__.py
Normal file
0
tests/e2e/cli_harness/__init__.py
Normal file
49
tests/e2e/cli_harness/lockdown.py
Normal file
49
tests/e2e/cli_harness/lockdown.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
"""What a coding-agent CLI under test may see of the host.
|
||||
|
||||
The CI job holds the proxy's upstream credentials (provider keys, AWS, Azure, the database URL, GitHub tokens) in its
|
||||
environment, and the cron VM runs as a real user with a populated home. A compromised CLI release, or a model told to
|
||||
read a file, must reach neither, so the CLI gets only the allowlisted runtime variables plus what its suite passes in,
|
||||
and a fresh empty HOME that is deleted when it exits.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from collections.abc import Generator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
CLI_ENV_ALLOWLIST: Final = (
|
||||
"PATH",
|
||||
"USER",
|
||||
"LOGNAME",
|
||||
"SHELL",
|
||||
"TERM",
|
||||
"TMPDIR",
|
||||
"LANG",
|
||||
"LC_ALL",
|
||||
"LC_CTYPE",
|
||||
"NODE_PATH",
|
||||
"NVM_DIR",
|
||||
"NVM_BIN",
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def isolated_home(prefix: str) -> Generator[Path]:
|
||||
home: Final = Path(tempfile.mkdtemp(prefix=prefix))
|
||||
try:
|
||||
yield home
|
||||
finally:
|
||||
shutil.rmtree(home, ignore_errors=True)
|
||||
|
||||
|
||||
def locked_down_env(
|
||||
*, home: Path, overrides: Mapping[str, str], source: Mapping[str, str] = os.environ
|
||||
) -> Mapping[str, str]:
|
||||
allowed: Final = {name: source[name] for name in CLI_ENV_ALLOWLIST if name in source}
|
||||
return MappingProxyType({**allowed, "HOME": str(home), **overrides})
|
||||
92
tests/e2e/cli_harness/npm_version_resolver.py
Normal file
92
tests/e2e/cli_harness/npm_version_resolver.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
"""The npm release of a coding-agent CLI that CI installs: the newest one published at least three days ago.
|
||||
|
||||
The three days are a security review buffer (PRD #26476, "Version resolvers"), so a release that gets pulled or
|
||||
patched within days never drives a run. Newest means by publish time, not semver order.
|
||||
|
||||
`python tests/e2e/cli_harness/npm_version_resolver.py <npm package>` prints the version on stdout for a shell
|
||||
`$(...)`. It is stdlib only because the cron image runs it without the project's dependencies.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import BinaryIO, Final, cast
|
||||
|
||||
MIN_AGE: Final = timedelta(days=3)
|
||||
_REGISTRY_URL: Final = "https://registry.npmjs.org/{package}"
|
||||
_FETCH_TIMEOUT_SECONDS: Final = 30
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Packument:
|
||||
published_at: Mapping[str, datetime]
|
||||
versions: frozenset[str]
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, raw: object) -> Packument:
|
||||
document: Final = _as_mapping(raw)
|
||||
times: Final = _as_mapping(document.get("time"))
|
||||
return cls(
|
||||
published_at=MappingProxyType({version: _parse_utc(stamp) for version, stamp in _string_values(times)}),
|
||||
versions=frozenset(_as_mapping(document.get("versions"))),
|
||||
)
|
||||
|
||||
|
||||
def newest_version_at_least(packument: Packument, *, min_age: timedelta, as_of: datetime) -> str | None:
|
||||
cutoff: Final = as_of - min_age
|
||||
eligible: Final = (
|
||||
(published, version)
|
||||
for version, published in packument.published_at.items()
|
||||
if version in packument.versions and "-" not in version and published <= cutoff
|
||||
)
|
||||
newest: Final = max(eligible, default=None)
|
||||
return None if newest is None else newest[1]
|
||||
|
||||
|
||||
def fetch_packument(package: str) -> Packument:
|
||||
url: Final = _REGISTRY_URL.format(package=urllib.parse.quote(package, safe="@"))
|
||||
request: Final = urllib.request.Request(url, headers={"Accept": "application/json"})
|
||||
with cast(BinaryIO, urllib.request.urlopen(request, timeout=_FETCH_TIMEOUT_SECONDS)) as response:
|
||||
return Packument.from_json(cast(object, json.loads(response.read())))
|
||||
|
||||
|
||||
def _as_mapping(raw: object) -> Mapping[str, object]:
|
||||
if not isinstance(raw, dict):
|
||||
return {}
|
||||
return {str(key): value for key, value in cast(dict[object, object], raw).items()}
|
||||
|
||||
|
||||
def _string_values(raw: Mapping[str, object]) -> tuple[tuple[str, str], ...]:
|
||||
return tuple((key, value) for key, value in raw.items() if isinstance(value, str))
|
||||
|
||||
|
||||
def _parse_utc(stamp: str) -> datetime:
|
||||
parsed: Final = datetime.fromisoformat(stamp.replace("Z", "+00:00"))
|
||||
return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _main(argv: list[str]) -> int:
|
||||
if len(argv) != 1:
|
||||
print("usage: npm_version_resolver.py <npm package>", file=sys.stderr) # noqa: T201 # CLI surface
|
||||
return 2
|
||||
package: Final = argv[0]
|
||||
version: Final = newest_version_at_least(
|
||||
fetch_packument(package), min_age=MIN_AGE, as_of=datetime.now(timezone.utc)
|
||||
)
|
||||
if version is None:
|
||||
print(f"npm_version_resolver: no release of {package} is at least {MIN_AGE} old", file=sys.stderr) # noqa: T201 # CLI surface
|
||||
return 1
|
||||
print(f"npm_version_resolver: selected {package}@{version}", file=sys.stderr) # noqa: T201 # CLI surface
|
||||
print(version) # noqa: T201 # CLI surface
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(_main(sys.argv[1:]))
|
||||
151
tests/e2e/cli_harness/rate_limiter.py
Normal file
151
tests/e2e/cli_harness/rate_limiter.py
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
"""A token bucket per upstream, shared by every process on the host, so parallel CLI suites and xdist workers
|
||||
together stay under one request rate per provider instead of one rate each.
|
||||
|
||||
Each bucket is a small JSON file read and written under an exclusive `flock`, in `LITELLM_COMPAT_RATE_STATE_DIR`
|
||||
(default: a fixed folder under `$TMPDIR`). `LITELLM_COMPAT_RATE_<BUCKET>` sets a bucket's req/s (default 5, 0 or less
|
||||
turns it off) and `LITELLM_COMPAT_RATE_BURST` overrides every bucket's burst (default: its rate).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from enum import StrEnum
|
||||
from functools import cache
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, ValidationError
|
||||
|
||||
DEFAULT_RATE: Final = 5.0
|
||||
RATE_ENV_PREFIX: Final = "LITELLM_COMPAT_RATE_"
|
||||
BURST_ENV: Final = "LITELLM_COMPAT_RATE_BURST"
|
||||
STATE_DIR_ENV: Final = "LITELLM_COMPAT_RATE_STATE_DIR"
|
||||
DEFAULT_STATE_DIR_NAME: Final = "litellm-claude-compat-ratelimit"
|
||||
_WAKE_MARGIN_SECONDS: Final = 1e-3
|
||||
|
||||
|
||||
class RateBucket(StrEnum):
|
||||
ANTHROPIC = "anthropic"
|
||||
AZURE = "azure"
|
||||
VERTEX_AI = "vertex_ai"
|
||||
BEDROCK_CONVERSE = "bedrock_converse"
|
||||
BEDROCK_INVOKE = "bedrock_invoke"
|
||||
OPENAI = "openai"
|
||||
AZURE_OPENAI = "azure_openai"
|
||||
BEDROCK_MANTLE = "bedrock_mantle"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProviderConfig:
|
||||
rate_per_sec: float
|
||||
burst: float
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self.rate_per_sec > 0 and self.burst > 0
|
||||
|
||||
|
||||
class _BucketState(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
tokens: float
|
||||
last_refill: float
|
||||
|
||||
|
||||
def load_config(env: Mapping[str, str] = os.environ) -> Mapping[RateBucket, ProviderConfig]:
|
||||
burst_override: Final = _as_float(env.get(BURST_ENV), -1.0)
|
||||
return MappingProxyType(
|
||||
{
|
||||
bucket: _bucket_config(_as_float(env.get(_rate_env(bucket)), DEFAULT_RATE), burst_override)
|
||||
for bucket in RateBucket
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def default_state_dir(env: Mapping[str, str] = os.environ) -> Path:
|
||||
explicit: Final = env.get(STATE_DIR_ENV)
|
||||
return Path(explicit) if explicit else Path(tempfile.gettempdir()) / DEFAULT_STATE_DIR_NAME
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RateLimiter:
|
||||
config: Mapping[RateBucket, ProviderConfig] = field(default_factory=load_config)
|
||||
state_dir: Path = field(default_factory=default_state_dir)
|
||||
clock: Callable[[], float] = time.monotonic
|
||||
sleep: Callable[[float], None] = time.sleep
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.state_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def acquire(self, bucket: RateBucket) -> float:
|
||||
cfg: Final = self.config.get(bucket)
|
||||
if cfg is None or not cfg.enabled:
|
||||
return 0.0
|
||||
return sum(self._waits(self.state_dir / f"{bucket.value}.json", cfg))
|
||||
|
||||
def _waits(self, path: Path, cfg: ProviderConfig) -> Iterator[float]:
|
||||
while (wait := _try_consume(path, cfg, self.clock())) > 0:
|
||||
self.sleep(wait)
|
||||
yield wait
|
||||
|
||||
|
||||
@cache
|
||||
def get_default_limiter() -> RateLimiter:
|
||||
return RateLimiter()
|
||||
|
||||
|
||||
def _rate_env(bucket: RateBucket) -> str:
|
||||
return RATE_ENV_PREFIX + bucket.value.upper()
|
||||
|
||||
|
||||
def _bucket_config(rate: float, burst_override: float) -> ProviderConfig:
|
||||
return ProviderConfig(rate_per_sec=rate, burst=burst_override if burst_override > 0 else max(rate, 1.0))
|
||||
|
||||
|
||||
def _as_float(value: str | None, default: float) -> float:
|
||||
if not value:
|
||||
return default
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def _try_consume(path: Path, cfg: ProviderConfig, now: float) -> float:
|
||||
fd: Final = os.open(path, os.O_CREAT | os.O_RDWR, 0o600)
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||
state: Final = _refilled(_read_state(fd, cfg, now), now, cfg)
|
||||
if state.tokens >= 1.0:
|
||||
_write_state(fd, _BucketState(tokens=state.tokens - 1.0, last_refill=state.last_refill))
|
||||
return 0.0
|
||||
_write_state(fd, state)
|
||||
return (1.0 - state.tokens) / cfg.rate_per_sec + _WAKE_MARGIN_SECONDS
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _read_state(fd: int, cfg: ProviderConfig, now: float) -> _BucketState:
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
raw: Final = os.read(fd, 4096)
|
||||
try:
|
||||
return _BucketState.model_validate_json(raw)
|
||||
except ValidationError:
|
||||
return _BucketState(tokens=cfg.burst, last_refill=now)
|
||||
|
||||
|
||||
def _refilled(state: _BucketState, now: float, cfg: ProviderConfig) -> _BucketState:
|
||||
elapsed: Final = max(0.0, now - state.last_refill)
|
||||
return _BucketState(tokens=min(cfg.burst, state.tokens + elapsed * cfg.rate_per_sec), last_refill=now)
|
||||
|
||||
|
||||
def _write_state(fd: int, state: _BucketState) -> None:
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
os.ftruncate(fd, 0)
|
||||
os.write(fd, state.model_dump_json().encode("utf-8"))
|
||||
0
tests/e2e/codex/__init__.py
Normal file
0
tests/e2e/codex/__init__.py
Normal file
42
tests/e2e/codex/conftest.py
Normal file
42
tests/e2e/codex/conftest.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from cli_harness.rate_limiter import get_default_limiter
|
||||
from codex.harness import CodexHarness, codex_harness
|
||||
from codex.setups import Deployment, ModelSetup
|
||||
from e2e_config import PROXY_BASE_URL, unique_marker
|
||||
from lifecycle import ResourceManager
|
||||
from proxy_client import ProxyClient
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Client:
|
||||
proxy: ProxyClient
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def client(proxy: ProxyClient) -> _Client:
|
||||
return _Client(proxy)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def setup(deployment: Deployment, proxy: ProxyClient, resources: ResourceManager) -> ModelSetup:
|
||||
model: Final = f"e2e-codex-{unique_marker()}"
|
||||
model_id: Final = proxy.create_model(model, deployment.litellm_params, provider_live=True)
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
return ModelSetup(model=model, rate_bucket=deployment.rate_bucket)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def codex(setup: ModelSetup, resources: ResourceManager) -> Iterator[CodexHarness]:
|
||||
with codex_harness(
|
||||
gateway=PROXY_BASE_URL,
|
||||
api_key=resources.key(models=[setup.model], user_id=None),
|
||||
rate_bucket=setup.rate_bucket,
|
||||
limiter=get_default_limiter(),
|
||||
) as harness:
|
||||
yield harness
|
||||
86
tests/e2e/codex/harness.py
Normal file
86
tests/e2e/codex/harness.py
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import tempfile
|
||||
from collections.abc import Generator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from cli_harness.lockdown import isolated_home, locked_down_env
|
||||
from cli_harness.rate_limiter import RateBucket, RateLimiter
|
||||
from codex.run import CodexRun, codex_run
|
||||
from e2e_metadata import step
|
||||
|
||||
CODEX_CLI: Final = "codex"
|
||||
EXEC_TIMEOUT_SECONDS: Final = 600.0
|
||||
_API_KEY_ENV: Final = "LITELLM_API_KEY"
|
||||
_CONFIG_TOML: Final = """\
|
||||
model_provider = "litellm"
|
||||
approval_policy = "never"
|
||||
sandbox_mode = "workspace-write"
|
||||
check_for_update_on_startup = false
|
||||
|
||||
[analytics]
|
||||
enabled = false
|
||||
|
||||
[model_providers.litellm]
|
||||
name = "LiteLLM"
|
||||
base_url = "{gateway}/v1"
|
||||
env_key = "{api_key_env}"
|
||||
wire_api = "responses"
|
||||
model_catalog_url = "{gateway}/v1/models"
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CodexHarness:
|
||||
home: Path
|
||||
api_key: str = field(repr=False)
|
||||
rate_bucket: RateBucket
|
||||
limiter: RateLimiter
|
||||
cli: str = CODEX_CLI
|
||||
timeout_seconds: float = EXEC_TIMEOUT_SECONDS
|
||||
|
||||
@property
|
||||
def codex_home(self) -> Path:
|
||||
return self.home / ".codex"
|
||||
|
||||
@step("Create a workspace folder holding the test's files")
|
||||
def new_workspace(self, files: Mapping[str, str]) -> Path:
|
||||
workspace: Final = Path(tempfile.mkdtemp(prefix="workspace-", dir=self.home))
|
||||
for name, content in files.items():
|
||||
_ = (workspace / name).write_text(content)
|
||||
return workspace
|
||||
|
||||
@step('Run Codex on {model} through the proxy with the prompt "{prompt}"')
|
||||
def exec(self, prompt: str, *, model: str, cwd: Path) -> CodexRun:
|
||||
_ = self.limiter.acquire(self.rate_bucket)
|
||||
completed: Final = subprocess.run(
|
||||
(self.cli, "exec", "--json", "--skip-git-repo-check", "--model", model, "--", prompt),
|
||||
cwd=cwd,
|
||||
env=locked_down_env(
|
||||
home=self.home, overrides={"CODEX_HOME": str(self.codex_home), _API_KEY_ENV: self.api_key}
|
||||
),
|
||||
stdin=subprocess.DEVNULL,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=self.timeout_seconds,
|
||||
check=False,
|
||||
)
|
||||
return codex_run(exit_code=completed.returncode, stdout=completed.stdout, stderr=completed.stderr)
|
||||
|
||||
|
||||
@step("Give Codex a fresh empty home whose config sends every request to the proxy")
|
||||
@contextmanager
|
||||
def codex_harness(
|
||||
*, gateway: str, api_key: str, rate_bucket: RateBucket, limiter: RateLimiter
|
||||
) -> Generator[CodexHarness]:
|
||||
with isolated_home("codex-cli-home-") as home:
|
||||
harness: Final = CodexHarness(home=home, api_key=api_key, rate_bucket=rate_bucket, limiter=limiter)
|
||||
harness.codex_home.mkdir()
|
||||
_ = (harness.codex_home / "config.toml").write_text(
|
||||
_CONFIG_TOML.format(gateway=gateway, api_key_env=_API_KEY_ENV)
|
||||
)
|
||||
yield harness
|
||||
187
tests/e2e/codex/run.py
Normal file
187
tests/e2e/codex/run.py
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
"""What one `codex exec --json` run did, read from the JSONL events it printed.
|
||||
|
||||
The event shapes are Codex's own (`codex-rs/exec/src/exec_events.rs`). We take the latest codex release > 3 days old, so an event or item
|
||||
type this module does not model is skipped, while one it does model is validated strictly and fails loudly when
|
||||
Codex changes it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Final, Literal, cast
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Discriminator, Tag, TypeAdapter
|
||||
|
||||
_HTTP_STATUS: Final = re.compile(r"\bstatus:? (\d{3})\b")
|
||||
|
||||
|
||||
class _Frozen(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
|
||||
class TokenUsage(_Frozen):
|
||||
input_tokens: int
|
||||
cached_input_tokens: int
|
||||
cache_write_input_tokens: int = 0
|
||||
output_tokens: int
|
||||
reasoning_output_tokens: int
|
||||
|
||||
|
||||
class _AgentMessage(_Frozen):
|
||||
type: Literal["agent_message"]
|
||||
text: str
|
||||
|
||||
|
||||
class _CommandExecution(_Frozen):
|
||||
type: Literal["command_execution"]
|
||||
command: str
|
||||
aggregated_output: str
|
||||
exit_code: int | None
|
||||
|
||||
|
||||
class _FileUpdate(_Frozen):
|
||||
path: str
|
||||
kind: Literal["add", "delete", "update"]
|
||||
|
||||
|
||||
class _FileChange(_Frozen):
|
||||
type: Literal["file_change"]
|
||||
changes: tuple[_FileUpdate, ...]
|
||||
status: Literal["in_progress", "completed", "failed"]
|
||||
|
||||
|
||||
class _ErrorItem(_Frozen):
|
||||
type: Literal["error"]
|
||||
message: str
|
||||
|
||||
|
||||
class _Unmodelled(_Frozen):
|
||||
type: str
|
||||
|
||||
|
||||
class _Message(_Frozen):
|
||||
message: str
|
||||
|
||||
|
||||
def _known_or_unmodelled(known: frozenset[str]) -> Callable[[object], str]:
|
||||
def tag(raw: object) -> str:
|
||||
kind: Final = cast("Mapping[str, object]", raw).get("type") if isinstance(raw, Mapping) else None
|
||||
return kind if isinstance(kind, str) and kind in known else "unmodelled"
|
||||
|
||||
return tag
|
||||
|
||||
|
||||
_Item = Annotated[
|
||||
Annotated[_AgentMessage, Tag("agent_message")]
|
||||
| Annotated[_CommandExecution, Tag("command_execution")]
|
||||
| Annotated[_FileChange, Tag("file_change")]
|
||||
| Annotated[_ErrorItem, Tag("error")]
|
||||
| Annotated[_Unmodelled, Tag("unmodelled")],
|
||||
Discriminator(_known_or_unmodelled(frozenset({"agent_message", "command_execution", "file_change", "error"}))),
|
||||
]
|
||||
|
||||
|
||||
class _ThreadStarted(_Frozen):
|
||||
type: Literal["thread.started"]
|
||||
thread_id: str
|
||||
|
||||
|
||||
class _ItemCompleted(_Frozen):
|
||||
type: Literal["item.completed"]
|
||||
item: _Item
|
||||
|
||||
|
||||
class _TurnCompleted(_Frozen):
|
||||
type: Literal["turn.completed"]
|
||||
usage: TokenUsage
|
||||
|
||||
|
||||
class _TurnFailed(_Frozen):
|
||||
type: Literal["turn.failed"]
|
||||
error: _Message
|
||||
|
||||
|
||||
class _StreamError(_Frozen):
|
||||
type: Literal["error"]
|
||||
message: str
|
||||
|
||||
|
||||
_Event = Annotated[
|
||||
Annotated[_ThreadStarted, Tag("thread.started")]
|
||||
| Annotated[_ItemCompleted, Tag("item.completed")]
|
||||
| Annotated[_TurnCompleted, Tag("turn.completed")]
|
||||
| Annotated[_TurnFailed, Tag("turn.failed")]
|
||||
| Annotated[_StreamError, Tag("error")]
|
||||
| Annotated[_Unmodelled, Tag("unmodelled")],
|
||||
Discriminator(
|
||||
_known_or_unmodelled(frozenset({"thread.started", "item.completed", "turn.completed", "turn.failed", "error"}))
|
||||
),
|
||||
]
|
||||
_EVENT: Final = TypeAdapter[_Event](_Event)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Command:
|
||||
command: str
|
||||
output: str
|
||||
exit_code: int | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CodexError:
|
||||
message: str
|
||||
status_code: int | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CodexRun:
|
||||
exit_code: int
|
||||
thread_id: str | None
|
||||
commands: tuple[Command, ...]
|
||||
patched_files: tuple[Path, ...]
|
||||
answer: str
|
||||
usage: TokenUsage | None
|
||||
errors: tuple[CodexError, ...]
|
||||
stderr: str
|
||||
|
||||
|
||||
def codex_run(*, exit_code: int, stdout: str, stderr: str) -> CodexRun:
|
||||
events: Final = tuple(_EVENT.validate_json(line) for line in stdout.splitlines() if line.strip())
|
||||
items: Final = tuple(event.item for event in events if isinstance(event, _ItemCompleted))
|
||||
patches: Final = tuple(item for item in items if isinstance(item, _FileChange) and item.status == "completed")
|
||||
answers: Final = tuple(item.text for item in items if isinstance(item, _AgentMessage))
|
||||
usages: Final = tuple(event.usage for event in events if isinstance(event, _TurnCompleted))
|
||||
return CodexRun(
|
||||
exit_code=exit_code,
|
||||
thread_id=next((event.thread_id for event in events if isinstance(event, _ThreadStarted)), None),
|
||||
commands=tuple(
|
||||
Command(command=item.command, output=item.aggregated_output, exit_code=item.exit_code)
|
||||
for item in items
|
||||
if isinstance(item, _CommandExecution)
|
||||
),
|
||||
patched_files=tuple(Path(change.path) for change in chain.from_iterable(patch.changes for patch in patches)),
|
||||
answer=answers[-1] if answers else "",
|
||||
usage=usages[-1] if usages else None,
|
||||
errors=tuple(_errors(events)),
|
||||
stderr=stderr,
|
||||
)
|
||||
|
||||
|
||||
def _errors(events: tuple[_Event, ...]) -> Iterator[CodexError]:
|
||||
for event in events:
|
||||
match event:
|
||||
case _TurnFailed(error=_Message(message=message)) | _StreamError(message=message):
|
||||
yield _codex_error(message)
|
||||
case _ItemCompleted(item=_ErrorItem(message=message)):
|
||||
yield _codex_error(message)
|
||||
case _:
|
||||
pass
|
||||
|
||||
|
||||
def _codex_error(message: str) -> CodexError:
|
||||
status: Final = _HTTP_STATUS.search(message)
|
||||
return CodexError(message=message, status_code=int(status.group(1)) if status else None)
|
||||
84
tests/e2e/codex/setups.py
Normal file
84
tests/e2e/codex/setups.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
"""The credentials are `os.environ/` references the gateway resolves, the same names the Claude Code matrix's
|
||||
`test_config.yaml` uses, and Bedrock rides the gateway's default AWS identity."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from cli_harness.rate_limiter import RateBucket
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from models import LiteLLMParamsBody
|
||||
|
||||
GPT_ON_BEDROCK_MANTLE: Final = "bedrock_mantle/openai.gpt-6.1-sol"
|
||||
GPT_ON_BEDROCK_RUNTIME: Final = "bedrock/us.openai.gpt-6.1-sol"
|
||||
GPT_ON_AZURE_OPENAI: Final = "azure/gpt-6.1-sol"
|
||||
GPT_ON_OPENAI: Final = "openai/gpt-6.1-sol"
|
||||
CLAUDE_ON_BEDROCK: Final = "bedrock/us.anthropic.claude-sonnet-5-5"
|
||||
_BEDROCK_REGION: Final = "us-east-1"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Deployment:
|
||||
litellm_params: LiteLLMParamsBody
|
||||
rate_bucket: RateBucket
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModelSetup:
|
||||
model: str
|
||||
rate_bucket: RateBucket
|
||||
|
||||
|
||||
def _drives(provider: Provider, model: str) -> pytest.MarkDecorator:
|
||||
return meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.RESPONSES,
|
||||
providers=(provider,),
|
||||
models=(model,),
|
||||
mode=Mode.STREAM,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
SETUPS: Final = (
|
||||
pytest.param(
|
||||
Deployment(
|
||||
LiteLLMParamsBody(model=GPT_ON_BEDROCK_MANTLE, aws_region_name=_BEDROCK_REGION), RateBucket.BEDROCK_MANTLE
|
||||
),
|
||||
id="bedrock_mantle",
|
||||
marks=_drives(Provider.BEDROCK_MANTLE, GPT_ON_BEDROCK_MANTLE),
|
||||
),
|
||||
pytest.param(
|
||||
Deployment(
|
||||
LiteLLMParamsBody(model=GPT_ON_BEDROCK_RUNTIME, aws_region_name=_BEDROCK_REGION),
|
||||
RateBucket.BEDROCK_CONVERSE,
|
||||
),
|
||||
id="bedrock_runtime",
|
||||
marks=_drives(Provider.BEDROCK, GPT_ON_BEDROCK_RUNTIME),
|
||||
),
|
||||
pytest.param(
|
||||
Deployment(
|
||||
LiteLLMParamsBody(
|
||||
model=GPT_ON_AZURE_OPENAI, api_base="os.environ/AZURE_API_BASE", api_key="os.environ/AZURE_API_KEY"
|
||||
),
|
||||
RateBucket.AZURE_OPENAI,
|
||||
),
|
||||
id="azure_openai",
|
||||
marks=_drives(Provider.AZURE, GPT_ON_AZURE_OPENAI),
|
||||
),
|
||||
pytest.param(
|
||||
Deployment(LiteLLMParamsBody(model=GPT_ON_OPENAI, api_key="os.environ/OPENAI_API_KEY"), RateBucket.OPENAI),
|
||||
id="openai",
|
||||
marks=_drives(Provider.OPENAI, GPT_ON_OPENAI),
|
||||
),
|
||||
pytest.param(
|
||||
Deployment(
|
||||
LiteLLMParamsBody(model=CLAUDE_ON_BEDROCK, aws_region_name=_BEDROCK_REGION), RateBucket.BEDROCK_CONVERSE
|
||||
),
|
||||
id="claude_bedrock",
|
||||
marks=_drives(Provider.BEDROCK, CLAUDE_ON_BEDROCK),
|
||||
),
|
||||
)
|
||||
23
tests/e2e/codex/test_command_round_trip_e2e.py
Normal file
23
tests/e2e/codex/test_command_round_trip_e2e.py
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from codex.harness import CodexHarness
|
||||
from codex.setups import SETUPS, ModelSetup
|
||||
from e2e_config import unique_marker
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deployment", SETUPS)
|
||||
class TestCodexCommandRoundTrip:
|
||||
def test_answers_with_what_a_command_printed(self, codex: CodexHarness, setup: ModelSetup) -> None:
|
||||
marker: Final = unique_marker()
|
||||
workspace: Final = codex.new_workspace(files={"secret.txt": marker})
|
||||
run: Final = codex.exec(
|
||||
"Run `cat secret.txt`, then reply with exactly what it printed.", model=setup.model, cwd=workspace
|
||||
)
|
||||
assert run.exit_code == 0, run
|
||||
assert any(marker in command.output for command in run.commands), run
|
||||
assert marker in run.answer, run
|
||||
|
|
@ -69,9 +69,6 @@ def _retrieve_response(proxy: ProxyClient, key: str, response_id: str) -> Respon
|
|||
|
||||
|
||||
class TestResponsesRetrieve:
|
||||
@pytest.mark.skip(
|
||||
reason="stage red: product gap (LIT-5446), retrieve returns a different id than the stored response (non-idempotent response-id re-encryption)"
|
||||
)
|
||||
@pytest.mark.covers("llm.responses.openai.basic.nonstream.works")
|
||||
@meta(
|
||||
Subject(
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
Tests of the harness under `tests/e2e/` (the transport, the clients, the fixture bundle and replay edge, the stack lock, the IdP launcher, the coverage collector, the JUnit properties and the Claude Code driver), not of the product. They live outside `tests/e2e/` because the Buildkite e2e run copies that folder into the runner image and runs every test in it, so a harness test in there counts as a product test in the nightly numbers. Nothing here needs a proxy, provider keys or the network
|
||||
|
||||
The layout mirrors `tests/e2e/`: `test_e2e_http.py` covers `tests/e2e/e2e_http.py`, `logging/test_datadog_reader.py` covers `tests/e2e/logging/datadog_reader.py`, and `claude_code/` covers the driver, builder, probe and version resolver. Put a new harness test under the folder that mirrors the suite folder whose module it covers
|
||||
The layout mirrors `tests/e2e/`: `test_e2e_http.py` covers `tests/e2e/e2e_http.py`, `logging/test_datadog_reader.py` covers `tests/e2e/logging/datadog_reader.py`, `claude_code/` covers the driver, builder and probe, `cli_harness/` covers the version resolver, the CLI env lockdown and the rate limiter, and `codex/` covers how the Codex driver reads the CLI's JSONL events. Put a new harness test under the folder that mirrors the suite folder whose module it covers
|
||||
|
||||
Run them from the repo root. `e2e_config` reads `LITELLM_MASTER_KEY` at import and any value will do, the CI lane sets a dummy:
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue