mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge remote-tracking branch 'origin/main' into HEAD
# Conflicts: # litellm/types/router.py
This commit is contained in:
commit
90b154da10
179 changed files with 12521 additions and 1174 deletions
7
.circleci/AGENTS.md
Normal file
7
.circleci/AGENTS.md
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# .circleci
|
||||
|
||||
The `integration` workflow on `main` runs from a scheduled pipeline in the CircleCI project settings, cron `9 0-8,14-23 * * *` (UTC) on `main` with no pipeline parameters. It has no `triggers:` block in this file because a workflow with one runs only on its schedule, so the API pipelines that set `run_migration_tests` or `routing_parity_base` would stop running it. If main stops getting integration runs, check that the schedule still exists and recreate it with these values
|
||||
|
||||
Jobs clone with the `checkout_over_https` command, never the built-in `checkout` step, because CircleCI falls back to the project's SSH checkout key without its GitHub App access, GitHub rejects that key, and the public repo needs no key. It fetches the job's commit with full history and no file contents (`--filter=blob:none`), like the built-in step did, so `scripts/path_filter.sh` can find a merge base
|
||||
|
||||
A step that checks out files from another commit fetches that commit with `--no-filter`, as `routing_parity` does, because the clone has no file contents and git would otherwise download each missing file in its own request, which took 210 to 896 seconds for the routing parity base
|
||||
|
|
@ -21,19 +21,30 @@ orbs:
|
|||
win: circleci/windows@5.0 # Add Windows orb
|
||||
|
||||
commands:
|
||||
checkout_migration_source:
|
||||
checkout_over_https:
|
||||
parameters:
|
||||
revision:
|
||||
type: string
|
||||
default: ""
|
||||
shell:
|
||||
type: string
|
||||
default: /bin/bash -eo pipefail
|
||||
steps:
|
||||
- run:
|
||||
name: Select the requested migration test revision
|
||||
name: Check out the public repository over HTTPS
|
||||
shell: << parameters.shell >>
|
||||
environment:
|
||||
MIGRATION_SOURCE_SHA: << pipeline.parameters.migration_source_sha >>
|
||||
REQUESTED_REVISION: << parameters.revision >>
|
||||
command: |
|
||||
revision="${MIGRATION_SOURCE_SHA:-$CIRCLE_SHA1}"
|
||||
revision="${REQUESTED_REVISION:-$CIRCLE_SHA1}"
|
||||
[[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1
|
||||
git init
|
||||
git remote add origin https://github.com/BerriAI/litellm.git
|
||||
git fetch --depth 1 origin "$revision"
|
||||
git checkout --detach FETCH_HEAD
|
||||
for attempt in 1 2 3; do
|
||||
git fetch --filter=blob:none origin "$revision" && git checkout --detach FETCH_HEAD && break
|
||||
[[ $attempt -lt 3 ]] || exit 1
|
||||
sleep 5
|
||||
done
|
||||
skip_if_unrelated_changes:
|
||||
parameters:
|
||||
category:
|
||||
|
|
@ -347,7 +358,7 @@ commands:
|
|||
- when:
|
||||
condition: << parameters.checkout >>
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- install_rust
|
||||
|
|
@ -377,7 +388,8 @@ jobs:
|
|||
CARGO_HTTP_MULTIPLEXING: "false"
|
||||
CARGO_NET_RETRY: "5"
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https:
|
||||
shell: bash.exe
|
||||
- run:
|
||||
name: Skip job when no windows-release-relevant files changed
|
||||
shell: bash.exe
|
||||
|
|
@ -451,7 +463,7 @@ jobs:
|
|||
password: ${DOCKERHUB_PASSWORD}
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- install_rust
|
||||
|
|
@ -482,7 +494,7 @@ jobs:
|
|||
working_directory: ~/project
|
||||
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- skip_if_unrelated_changes
|
||||
- attach_workspace:
|
||||
at: ~/project
|
||||
|
|
@ -559,7 +571,7 @@ jobs:
|
|||
resource_class: medium
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- run:
|
||||
name: Generate LiteLLM master key
|
||||
command: |
|
||||
|
|
@ -659,7 +671,7 @@ jobs:
|
|||
resource_class: large
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- run:
|
||||
name: Generate LiteLLM master key
|
||||
command: |
|
||||
|
|
@ -740,7 +752,7 @@ jobs:
|
|||
resource_class: large
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- skip_if_unrelated_changes:
|
||||
category: client
|
||||
- setup_google_dns
|
||||
|
|
@ -794,7 +806,7 @@ jobs:
|
|||
# the rest of the suite: nothing else performs a logout.
|
||||
PROXY_LOGOUT_URL: "https://www.example.com"
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- run:
|
||||
name: Generate LiteLLM master key
|
||||
command: |
|
||||
|
|
@ -940,7 +952,7 @@ jobs:
|
|||
# and by migration.serverRootPath.config.ts, which refuses to run without it.
|
||||
SERVER_ROOT_PATH: "/litellm"
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- run:
|
||||
name: Generate LiteLLM master key
|
||||
command: |
|
||||
|
|
@ -1066,11 +1078,12 @@ jobs:
|
|||
- when:
|
||||
condition: << parameters.migration_qualification >>
|
||||
steps:
|
||||
- checkout_migration_source
|
||||
- checkout_over_https:
|
||||
revision: << pipeline.parameters.migration_source_sha >>
|
||||
- unless:
|
||||
condition: << parameters.migration_qualification >>
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- skip_if_unrelated_changes
|
||||
|
||||
- run:
|
||||
|
|
@ -1122,7 +1135,8 @@ jobs:
|
|||
MIGRATION_TEST_OUTPUT: /tmp/migration-results
|
||||
PYTHONPATH: tests/e2e
|
||||
steps:
|
||||
- checkout_migration_source
|
||||
- checkout_over_https:
|
||||
revision: << pipeline.parameters.migration_source_sha >>
|
||||
- run:
|
||||
name: Generate LiteLLM master key
|
||||
command: |
|
||||
|
|
@ -1197,7 +1211,7 @@ jobs:
|
|||
resource_class: medium
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- skip_if_unrelated_changes
|
||||
- attach_workspace:
|
||||
at: ~/project
|
||||
|
|
@ -1371,7 +1385,7 @@ jobs:
|
|||
environment:
|
||||
UV_PYTHON_DOWNLOADS: never
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- install_uv
|
||||
- run:
|
||||
name: Install the locked shard verifier and coverage dependencies
|
||||
|
|
@ -1419,7 +1433,7 @@ jobs:
|
|||
ROUTING_PARITY_BASE: << pipeline.parameters.routing_parity_base >>
|
||||
command: |
|
||||
[[ "$ROUTING_PARITY_BASE" =~ ^[0-9a-f]{40}$ ]] || exit 1
|
||||
git fetch --depth 1 origin "$ROUTING_PARITY_BASE"
|
||||
git fetch --depth 1 --no-filter origin "$ROUTING_PARITY_BASE"
|
||||
git rm -r -f --quiet litellm enterprise litellm-proxy-extras
|
||||
git checkout "$ROUTING_PARITY_BASE" -- litellm enterprise litellm-proxy-extras
|
||||
git reset --quiet
|
||||
|
|
@ -1494,7 +1508,7 @@ jobs:
|
|||
environment:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- skip_if_unrelated_changes
|
||||
- setup_litellm_test_deps:
|
||||
checkout: false
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ const FINISH_REASONS: &[(&str, &str)] = &[
|
|||
("max_tokens", "length"),
|
||||
("refusal", "content_filter"),
|
||||
("compaction", "length"),
|
||||
("pause_turn", "length"),
|
||||
("model_context_window_exceeded", "length"),
|
||||
("guardrail_intervened", "content_filter"),
|
||||
("content_filtered", "content_filter"),
|
||||
|
|
@ -75,6 +76,7 @@ mod tests {
|
|||
#[case::end_turn("end_turn", "stop")]
|
||||
#[case::stop_sequence("stop_sequence", "stop")]
|
||||
#[case::max_tokens("max_tokens", "length")]
|
||||
#[case::pause_turn("pause_turn", "length")]
|
||||
#[case::model_context_window_exceeded("model_context_window_exceeded", "length")]
|
||||
#[case::refusal("refusal", "content_filter")]
|
||||
#[case::guardrail_intervened("guardrail_intervened", "content_filter")]
|
||||
|
|
|
|||
|
|
@ -25,7 +25,11 @@ from litellm._internal_context import current_service_target, service_target
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import CACHED_STREAMING_CHUNK_DELAY
|
||||
from litellm.integrations.otel.runtime import phase_span
|
||||
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper, peek_reasoning_summary_aliases
|
||||
from litellm.litellm_core_utils.model_param_helper import (
|
||||
ModelParamHelper,
|
||||
embeds_thought_signature_in_tool_call_id,
|
||||
peek_reasoning_summary_aliases,
|
||||
)
|
||||
from litellm.types.caching import *
|
||||
from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
|
||||
|
||||
|
|
@ -421,7 +425,12 @@ class Cache:
|
|||
reasoning_summary_material: Final = (
|
||||
"" if reasoning_summary is None else f"reasoning_summary: {reasoning_summary}"
|
||||
)
|
||||
hashed_cache_key = Cache._get_hashed_cache_key(cache_key + reasoning_summary_material)
|
||||
tool_call_id_shape_material: Final = (
|
||||
"" if embeds_thought_signature_in_tool_call_id(kwargs) else "thought_signature_in_tool_call_id: False"
|
||||
)
|
||||
hashed_cache_key = Cache._get_hashed_cache_key(
|
||||
cache_key + reasoning_summary_material + tool_call_id_shape_material
|
||||
)
|
||||
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
|
||||
verbose_logger.debug(
|
||||
"\nCreated cache key: %s (source material length: %d)",
|
||||
|
|
|
|||
|
|
@ -177,6 +177,8 @@ MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0
|
|||
# Set to 0 to disable truncation.
|
||||
MAX_BASE64_LENGTH_FOR_LOGGING: Final = get_env_int("MAX_BASE64_LENGTH_FOR_LOGGING", 64)
|
||||
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: Final = 256 * 1024
|
||||
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS: Final = 256
|
||||
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS: Final = 8 * 1024 * 1024
|
||||
REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
|
||||
# in-memory stand-in handed to provider converters for redacted arguments; never stored
|
||||
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"
|
||||
|
|
@ -634,6 +636,8 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60
|
|||
BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)
|
||||
# Anthropic's Messages API rejects thinking.budget_tokens < 1024.
|
||||
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024
|
||||
ANTHROPIC_MAX_USER_ID_LENGTH: Final = 512
|
||||
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH: Final = 255
|
||||
ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:"
|
||||
REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
|
||||
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000)
|
||||
|
|
@ -1163,6 +1167,7 @@ openai_compatible_providers: Final[list[str]] = [
|
|||
"cortecs",
|
||||
"scx-ai",
|
||||
"prism",
|
||||
"hostinger_router",
|
||||
"sail",
|
||||
"reka",
|
||||
"streamlake",
|
||||
|
|
@ -1686,6 +1691,7 @@ BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
|
|||
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
|
||||
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
|
||||
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY: Final = "responses_websocket_call_id"
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
|
||||
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
|
||||
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
|
||||
|
|
|
|||
|
|
@ -3319,7 +3319,7 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
|
|||
)
|
||||
|
||||
|
||||
_RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete"})
|
||||
_RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete", "response.failed"})
|
||||
|
||||
|
||||
class _ResponsesWsEventResponse(LiteLLMBaseModel):
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@ from collections.abc import Mapping, Sequence
|
|||
from dataclasses import dataclass, field, replace
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeAlias
|
||||
from typing import Annotated, Final, Literal, TypeAlias
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, BeforeValidator, ConfigDict, TypeAdapter, ValidationError
|
||||
from pydantic_core import to_jsonable_python
|
||||
from typing_extensions import assert_never
|
||||
|
||||
|
|
@ -23,6 +23,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
|||
from litellm.llms.base_llm.decisions.transformation import (
|
||||
BaseDecisionsConfig,
|
||||
DecisionsResponseValidationError,
|
||||
catalog_mode,
|
||||
image_media_type,
|
||||
image_param,
|
||||
input_images,
|
||||
|
|
@ -61,7 +62,7 @@ from litellm.types.decisions import (
|
|||
UnsupportedDecisionsRequest,
|
||||
)
|
||||
from litellm.types.llms.openai import ChatCompletionUserMessage
|
||||
from litellm.types.utils import LlmProviders, ProviderSpecificHeader
|
||||
from litellm.types.utils import LlmProviders, ProviderSpecificHeader, is_decisions_model_mode
|
||||
from litellm.utils import ProviderConfigManager, get_model_cost_key
|
||||
|
||||
SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
|
||||
|
|
@ -257,19 +258,105 @@ def _request_safety_identifier(
|
|||
return None
|
||||
|
||||
|
||||
def _fields_without_unsupported_media(
|
||||
fields: RequestFields,
|
||||
def _bool_or_none(value: object) -> bool | None:
|
||||
return value if isinstance(value, bool) else None
|
||||
|
||||
|
||||
def _str_or_none(value: object) -> str | None:
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
_DeploymentFlag: TypeAlias = Annotated[bool | None, BeforeValidator(_bool_or_none)]
|
||||
|
||||
|
||||
class _DeploymentCapabilities(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, from_attributes=True)
|
||||
|
||||
mode: Annotated[str | None, BeforeValidator(_str_or_none)] = None
|
||||
supports_box_questions: _DeploymentFlag = None
|
||||
supports_audio_input: _DeploymentFlag = None
|
||||
supports_video_input: _DeploymentFlag = None
|
||||
|
||||
def decisions_media_flag(self, flag: bool | None, model: str, custom_llm_provider: str) -> bool | None:
|
||||
if flag is None:
|
||||
return None
|
||||
mode: Final = self.mode if self.mode is not None else catalog_mode(model, custom_llm_provider)
|
||||
return flag if mode is None or is_decisions_model_mode(mode) else None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ResolvedSupport:
|
||||
box_questions: bool
|
||||
audio: bool
|
||||
videos: bool
|
||||
|
||||
|
||||
def _takes_audio(
|
||||
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
|
||||
) -> bool:
|
||||
return provider_config.supports_audio_input(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider),
|
||||
)
|
||||
|
||||
|
||||
def _takes_videos(
|
||||
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
|
||||
) -> bool:
|
||||
return provider_config.supports_video_input(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider),
|
||||
)
|
||||
|
||||
|
||||
def deployment_refuses_media(
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
model_info: Mapping[str, object],
|
||||
call_kwargs: Mapping[str, object],
|
||||
*,
|
||||
audio: bool,
|
||||
videos: bool,
|
||||
) -> bool:
|
||||
if _drops_params(call_kwargs):
|
||||
return False
|
||||
provider_config: Final = get_decisions_provider_config(model, custom_llm_provider)
|
||||
canonical_model: Final = provider_config.canonical_model(model)
|
||||
deployment: Final = _DeploymentCapabilities.model_validate(model_info)
|
||||
return (audio and not _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) or (
|
||||
videos and not _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment)
|
||||
)
|
||||
|
||||
|
||||
def _resolved_support(
|
||||
provider_config: BaseDecisionsConfig,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
headers: Mapping[str, str],
|
||||
kwargs: Mapping[str, object],
|
||||
fields: RequestFields,
|
||||
) -> _ResolvedSupport:
|
||||
deployment: Final = _DeploymentCapabilities.model_validate(kwargs.get("model_info") or {})
|
||||
return _ResolvedSupport(
|
||||
box_questions=provider_config.supports_box_questions(
|
||||
model, custom_llm_provider, headers, deployment.supports_box_questions
|
||||
),
|
||||
audio=bool(fields.audio) and _takes_audio(provider_config, model, custom_llm_provider, deployment),
|
||||
videos=bool(fields.videos) and _takes_videos(provider_config, model, custom_llm_provider, deployment),
|
||||
)
|
||||
|
||||
|
||||
def _fields_without_unsupported_media(
|
||||
fields: RequestFields, support: _ResolvedSupport, kwargs: Mapping[str, object]
|
||||
) -> RequestFields:
|
||||
if not _drops_params(kwargs):
|
||||
return fields
|
||||
return replace(
|
||||
fields,
|
||||
audio=fields.audio if provider_config.supports_audio_input(model, custom_llm_provider) else None,
|
||||
videos=fields.videos if provider_config.supports_video_input(model, custom_llm_provider) else None,
|
||||
audio=fields.audio if support.audio else None,
|
||||
videos=fields.videos if support.videos else None,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -292,10 +379,8 @@ class _MalformedImages:
|
|||
def _unsupported_params(
|
||||
ir_request: DecisionsIRRequest,
|
||||
provider_config: BaseDecisionsConfig,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
headers: Mapping[str, str],
|
||||
supports_box_questions: bool,
|
||||
support: _ResolvedSupport,
|
||||
) -> tuple[str, ...]:
|
||||
refuses_safety_identifier: Final = (
|
||||
ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier
|
||||
|
|
@ -304,13 +389,9 @@ def _unsupported_params(
|
|||
None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input)
|
||||
)
|
||||
refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers)
|
||||
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not supports_box_questions
|
||||
refuses_audio: Final = bool(ir_request.audio) and not provider_config.supports_audio_input(
|
||||
model, custom_llm_provider
|
||||
)
|
||||
refuses_videos: Final = bool(ir_request.videos) and not provider_config.supports_video_input(
|
||||
model, custom_llm_provider
|
||||
)
|
||||
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not support.box_questions
|
||||
refuses_audio: Final = bool(ir_request.audio) and not support.audio
|
||||
refuses_videos: Final = bool(ir_request.videos) and not support.videos
|
||||
return (
|
||||
*(("safety_identifier",) if refuses_safety_identifier else ()),
|
||||
*(() if refused_image_param is None else (refused_image_param,)),
|
||||
|
|
@ -333,7 +414,7 @@ def _without_unsupported_params(
|
|||
ir_request: DecisionsIRRequest,
|
||||
provider_config: BaseDecisionsConfig,
|
||||
headers: Mapping[str, str],
|
||||
supports_box_questions: bool,
|
||||
support: _ResolvedSupport,
|
||||
) -> DecisionsIRRequest:
|
||||
return replace(
|
||||
ir_request,
|
||||
|
|
@ -345,9 +426,7 @@ def _without_unsupported_params(
|
|||
safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None,
|
||||
assets=ir_request.assets if provider_config.supports_assets(headers) else MappingProxyType({}),
|
||||
questions=(
|
||||
ir_request.questions
|
||||
if supports_box_questions
|
||||
else tuple(filter(_is_not_box_question, ir_request.questions))
|
||||
ir_request.questions if support.box_questions else tuple(filter(_is_not_box_question, ir_request.questions))
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -368,39 +447,20 @@ def _without_malformed_images(
|
|||
return well_formed
|
||||
|
||||
|
||||
class _DeploymentBoxQuestions(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, from_attributes=True)
|
||||
|
||||
supports_box_questions: StrictBool | None = None
|
||||
|
||||
|
||||
def _deployment_supports_box_questions(kwargs: Mapping[str, object]) -> bool | None:
|
||||
try:
|
||||
return _DeploymentBoxQuestions.model_validate(kwargs.get("model_info") or {}).supports_box_questions
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _provider_ir_request(
|
||||
ir_request: DecisionsIRRequest,
|
||||
*,
|
||||
provider_config: BaseDecisionsConfig,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
headers: Mapping[str, str],
|
||||
support: _ResolvedSupport,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> DecisionsIRRequest | _UnsupportedParams | _NothingLeftAfterDrop:
|
||||
supports_box_questions: Final = provider_config.supports_box_questions(
|
||||
model, custom_llm_provider, headers, _deployment_supports_box_questions(kwargs)
|
||||
)
|
||||
unsupported: Final = _unsupported_params(
|
||||
ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions
|
||||
)
|
||||
unsupported: Final = _unsupported_params(ir_request, provider_config, headers, support)
|
||||
if not unsupported:
|
||||
return ir_request
|
||||
if not _drops_params(kwargs):
|
||||
return _UnsupportedParams(names=unsupported)
|
||||
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, supports_box_questions)
|
||||
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, support)
|
||||
if _leaves_no_input(supported.input):
|
||||
return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on")
|
||||
if not supported.questions:
|
||||
|
|
@ -479,10 +539,11 @@ def prepare_call(
|
|||
raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
|
||||
try:
|
||||
headers: Final = decisions_outbound_headers(extra_headers, provider, kwargs)
|
||||
support: Final = _resolved_support(provider_config, canonical_model, provider, headers, kwargs, fields)
|
||||
request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs)
|
||||
request: Final = _fields_without_unsupported_media(
|
||||
fields, provider_config, canonical_model, provider, kwargs
|
||||
).validate(safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs))
|
||||
request: Final = _fields_without_unsupported_media(fields, support, kwargs).validate(
|
||||
safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs)
|
||||
)
|
||||
except ValidationError as error:
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Invalid Decisions request: {error}",
|
||||
|
|
@ -523,12 +584,7 @@ def prepare_call(
|
|||
llm_provider=provider,
|
||||
)
|
||||
ir_request: Final = _provider_ir_request(
|
||||
caller_ir_request,
|
||||
provider_config=provider_config,
|
||||
model=canonical_model,
|
||||
custom_llm_provider=provider,
|
||||
headers=headers,
|
||||
kwargs=kwargs,
|
||||
caller_ir_request, provider_config=provider_config, headers=headers, support=support, kwargs=kwargs
|
||||
)
|
||||
if isinstance(ir_request, _UnsupportedParams):
|
||||
raise litellm.UnsupportedParamsError(
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import itertools
|
|||
import os
|
||||
import re
|
||||
from collections.abc import Callable, Container, Iterable, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
|
@ -102,6 +102,192 @@ def _validated_object_list(value: object) -> list[object] | None:
|
|||
return None
|
||||
|
||||
|
||||
_MessageT: Final = TypeVar("_MessageT")
|
||||
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
|
||||
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
|
||||
|
||||
|
||||
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
|
||||
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
|
||||
return tuple(
|
||||
itertools.chain.from_iterable(
|
||||
_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _block_carries_cache_control(block: object) -> bool:
|
||||
return isinstance(block, dict) and _CONTENT_PART_ADAPTER.validate_python(block).get("cache_control") is not None
|
||||
|
||||
|
||||
_ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT: Final = frozenset({"tool", "function"})
|
||||
|
||||
|
||||
def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool:
|
||||
if fields.get("cache_control") is None:
|
||||
return False
|
||||
return (
|
||||
not isinstance(fields.get("content"), list)
|
||||
or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
|
||||
)
|
||||
|
||||
|
||||
def cache_control_count(message: object) -> int:
|
||||
if not isinstance(message, dict):
|
||||
return 0
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
|
||||
return int(_message_mark_reaches_provider(fields)) + block_marks
|
||||
|
||||
|
||||
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
|
||||
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
|
||||
if "cache_control" not in fields:
|
||||
return others
|
||||
cache_control: Final = rewrite(fields["cache_control"])
|
||||
return others if cache_control is None else {**others, "cache_control": cache_control}
|
||||
|
||||
|
||||
def block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
|
||||
if not isinstance(block, dict):
|
||||
return block
|
||||
return _fields_with_cache_control(_CONTENT_PART_ADAPTER.validate_python(block), rewrite)
|
||||
|
||||
|
||||
def with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
|
||||
if not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
blocks_rewritten: Final = {
|
||||
key: (
|
||||
[block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
|
||||
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
|
||||
else value
|
||||
)
|
||||
for key, value in fields.items()
|
||||
}
|
||||
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
|
||||
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
|
||||
)
|
||||
|
||||
|
||||
def dropped_cache_control(_: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _first_marked_index(value: object) -> int | None:
|
||||
if not isinstance(value, list):
|
||||
return None
|
||||
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
|
||||
return next((index for index, block in enumerate(blocks) if _block_carries_cache_control(block)), None)
|
||||
|
||||
|
||||
_MarkSlot: TypeAlias = tuple[str, int]
|
||||
_MESSAGE_MARK_SLOT: Final[_MarkSlot] = ("cache_control", -1)
|
||||
|
||||
|
||||
def _block_mark_slots(fields: Mapping[str, object], key: str) -> tuple[_MarkSlot, ...]:
|
||||
value: Final = fields.get(key)
|
||||
if not isinstance(value, list):
|
||||
return ()
|
||||
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
|
||||
return tuple((key, index) for index, block in enumerate(blocks) if _block_carries_cache_control(block))
|
||||
|
||||
|
||||
def _mark_slots_in_wire_order(fields: Mapping[str, object]) -> tuple[_MarkSlot, ...]:
|
||||
content_slots: Final = _block_mark_slots(fields, "content")
|
||||
tool_call_slots: Final = _block_mark_slots(fields, "tool_calls")
|
||||
if not _message_mark_reaches_provider(fields):
|
||||
return content_slots + tool_call_slots
|
||||
if isinstance(fields.get("content"), list):
|
||||
return content_slots + (_MESSAGE_MARK_SLOT,) + tool_call_slots
|
||||
return (_MESSAGE_MARK_SLOT,) + content_slots + tool_call_slots
|
||||
|
||||
|
||||
def _blocks_without_marks(key: str, value: object, dropped_slots: frozenset[_MarkSlot]) -> object:
|
||||
if key not in _MARK_CARRYING_MESSAGE_FIELDS or not isinstance(value, list):
|
||||
return value
|
||||
return [
|
||||
block_with_cache_control(block, dropped_cache_control) if (key, index) in dropped_slots else block
|
||||
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(value))
|
||||
]
|
||||
|
||||
|
||||
def _without_oldest_marks(message: _MessageT, dropped: int) -> _MessageT:
|
||||
if dropped == 0 or not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
dropped_slots: Final = frozenset(_mark_slots_in_wire_order(fields)[:dropped])
|
||||
blocks_unmarked: Final = {key: _blocks_without_marks(key, value, dropped_slots) for key, value in fields.items()}
|
||||
return cast( # cast-ok: the same message with its oldest wire-order marks dropped
|
||||
_MessageT,
|
||||
_fields_with_cache_control(blocks_unmarked, dropped_cache_control)
|
||||
if _MESSAGE_MARK_SLOT in dropped_slots
|
||||
else blocks_unmarked,
|
||||
)
|
||||
|
||||
|
||||
def tool_message_with_single_cache_mark(message: _MessageT) -> _MessageT:
|
||||
if not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
content: Final = fields.get("content")
|
||||
if fields.get("role") != "tool" or not isinstance(content, list):
|
||||
return message
|
||||
kept_index: Final = None if fields.get("cache_control") is not None else _first_marked_index(content)
|
||||
return cast( # cast-ok: the same tool message keeping only the mark its single cache point reads
|
||||
_MessageT,
|
||||
{
|
||||
**fields,
|
||||
"content": [
|
||||
block if index == kept_index else block_with_cache_control(block, dropped_cache_control)
|
||||
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(content))
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
_BudgetSpend: TypeAlias = Callable[[tuple[int, int], int], tuple[int, int]]
|
||||
|
||||
|
||||
def _newest_marks_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
|
||||
remaining, _ = state
|
||||
kept: Final = min(marks, remaining)
|
||||
return remaining - kept, marks - kept
|
||||
|
||||
|
||||
def _whole_message_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
|
||||
remaining, _ = state
|
||||
if marks > remaining:
|
||||
return remaining, marks
|
||||
return remaining - marks, 0
|
||||
|
||||
|
||||
def _messages_within_budget(
|
||||
messages: Sequence[_MessageT], reserved_marks: int, spend: _BudgetSpend
|
||||
) -> Sequence[_MessageT]:
|
||||
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
|
||||
newest_first: Final = tuple(reversed(messages))
|
||||
budget_after_each: Final = itertools.accumulate(
|
||||
(cache_control_count(message) for message in newest_first), spend, initial=(budget, 0)
|
||||
)
|
||||
dropped_counts: Final = tuple(dropped for _, dropped in itertools.islice(budget_after_each, 1, None))
|
||||
kept_newest_first: Final = tuple(
|
||||
_without_oldest_marks(message, dropped) for message, dropped in zip(newest_first, dropped_counts)
|
||||
)
|
||||
return tuple(reversed(kept_newest_first))
|
||||
|
||||
|
||||
def newest_marks_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
|
||||
return _messages_within_budget(messages, reserved_marks, _newest_marks_kept)
|
||||
|
||||
|
||||
def whole_messages_within_cache_control_limit(
|
||||
messages: Sequence[_MessageT], reserved_marks: int
|
||||
) -> Sequence[_MessageT]:
|
||||
return _messages_within_budget(messages, reserved_marks, _whole_message_kept)
|
||||
|
||||
|
||||
def _part_with_cache_control_renamed(part: object) -> object:
|
||||
if not _accepts_prompt_cache_breakpoint(part):
|
||||
return part
|
||||
|
|
@ -1439,3 +1625,25 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
llm_router=None,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def messages_within_cache_control_cap(
|
||||
messages: Sequence[_MessageT], tools: object, cache_control: object = None, request_kwargs: object = None
|
||||
) -> Sequence[_MessageT]:
|
||||
"""Under drop_params, drop the oldest message hints the provider's cache_control cap has no room for.
|
||||
|
||||
Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
|
||||
"""
|
||||
reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
|
||||
_validated_object_list(tools), cache_control, request_kwargs
|
||||
)
|
||||
kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
|
||||
dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
|
||||
if dropped > 0:
|
||||
verbose_logger.warning(
|
||||
"Dropped %d cache_control hint(s) from the oldest messages because drop_params is on and the "
|
||||
"provider accepts at most %d cache_control blocks per request",
|
||||
dropped,
|
||||
MAX_CACHE_CONTROL_BLOCKS,
|
||||
)
|
||||
return kept
|
||||
|
|
|
|||
|
|
@ -311,6 +311,7 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
|
|||
"tool_use": "tool_calls",
|
||||
"refusal": "content_filter",
|
||||
"compaction": "length",
|
||||
"pause_turn": "length",
|
||||
"model_context_window_exceeded": "length",
|
||||
# Cohere
|
||||
"COMPLETE": "stop",
|
||||
|
|
|
|||
|
|
@ -84,6 +84,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
|
|||
"github_copilot_host",
|
||||
"github_copilot_user_session",
|
||||
"fireworks_forward_user_id",
|
||||
"thought_signature_in_tool_call_id",
|
||||
"responses_api_version",
|
||||
"allowed_openai_params",
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
|
||||
|
|
@ -99,6 +100,8 @@ OPTIONAL_KWARGS_KEYS: Final = (
|
|||
# Backward-compatible alias for existing imports/tests.
|
||||
_OPTIONAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS
|
||||
|
||||
SUPPLEMENTAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS | {"provider_specific_header", "extra_headers"}
|
||||
|
||||
_CONTROL_OPTIONS: Final = TypeAdapter(ControlOptions)
|
||||
_CONTROL_OPTION_NAMES: Final = tuple(field.name for field in fields(ControlOptions))
|
||||
_MAX_SHOWN_INT_BITS: Final = 64
|
||||
|
|
|
|||
|
|
@ -208,11 +208,14 @@ class StandardBuiltInToolCostTracking:
|
|||
usage=usage, response_object=response_object
|
||||
)
|
||||
|
||||
response_model: Final = _output_item_field(response_object, "model")
|
||||
if model_info is not None and resolved_usage is not None and resolved_provider is not None:
|
||||
result: Final = get_cost_for_web_search_request(
|
||||
custom_llm_provider=resolved_provider,
|
||||
usage=resolved_usage,
|
||||
model_info=model_info,
|
||||
response_model=response_model if isinstance(response_model, str) else None,
|
||||
lookup_model_info=StandardBuiltInToolCostTracking._safe_get_model_info,
|
||||
)
|
||||
if result is not None:
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -3,13 +3,15 @@ import functools
|
|||
import inspect
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast
|
||||
|
||||
from litellm._logging import format_base64_size, verbose_logger
|
||||
from litellm.constants import (
|
||||
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS,
|
||||
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS,
|
||||
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS,
|
||||
MAX_BASE64_LENGTH_FOR_LOGGING,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
|
||||
|
|
@ -44,6 +46,8 @@ Helper utils used for logging callbacks
|
|||
# Regex matching data-URI base64 content: "data:<mime>;base64,<payload>"
|
||||
# Captures: group(1)=mime_type, group(2)=base64_payload
|
||||
_DATA_URI_RE: Final = re.compile(r"data:([^;,\s]{1,255});base64,([A-Za-z0-9+/=]+)")
|
||||
_DATA_URI_PREFIX: Final = "data:"
|
||||
_DATA_URI_BASE64_MARKER: Final = ";base64,"
|
||||
|
||||
# Maximum nesting depth for _truncate_base64_in_value to guard against
|
||||
# pathological payloads. OpenAI message format is typically 3-4 levels deep.
|
||||
|
|
@ -60,15 +64,20 @@ def _base64_data_uri_replacer(match: re.Match[str]) -> str:
|
|||
return f"data:{mime_type};base64,[base64_data truncated: {size_str}]"
|
||||
|
||||
|
||||
def _may_contain_base64_data_uri(value: str) -> bool:
|
||||
return ";" in value and _DATA_URI_PREFIX in value and _DATA_URI_BASE64_MARKER in value
|
||||
|
||||
|
||||
def _truncate_base64_in_string(value: str) -> str:
|
||||
"""Replace long base64 data-URI payloads in a string with a size placeholder."""
|
||||
if MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
|
||||
if MAX_BASE64_LENGTH_FOR_LOGGING <= 0 or not _may_contain_base64_data_uri(value):
|
||||
return value
|
||||
return _DATA_URI_RE.sub(_base64_data_uri_replacer, value)
|
||||
|
||||
|
||||
def _truncate_base64_in_value(
|
||||
value: str | dict[str, object] | list[object] | None,
|
||||
truncate_string: Callable[[str], str],
|
||||
) -> str | dict[str, object] | list[object] | None:
|
||||
"""Iteratively truncate base64 data URIs in a JSON-like value (str/list/dict).
|
||||
|
||||
|
|
@ -78,13 +87,13 @@ def _truncate_base64_in_value(
|
|||
# Stack entries: (source_value, depth, parent_container, key_or_index)
|
||||
# We mutate *copies* of dicts/lists in-place via parent references.
|
||||
if isinstance(value, str):
|
||||
return _truncate_base64_in_string(value)
|
||||
return truncate_string(value)
|
||||
if not isinstance(value, (dict, list)):
|
||||
return value
|
||||
|
||||
# Shallow-copy the root so we don't mutate the caller's data.
|
||||
root: Final = {k: v for k, v in value.items()} if isinstance(value, dict) else list(value)
|
||||
stack: Final[list] = [(root, 0)]
|
||||
stack: Final[list[tuple[dict[str, object] | list[object], int]]] = [(root, 0)] # mutable-ok: explicit stack
|
||||
|
||||
while stack:
|
||||
container, depth = stack.pop()
|
||||
|
|
@ -93,19 +102,19 @@ def _truncate_base64_in_value(
|
|||
if isinstance(container, dict):
|
||||
for k, v in container.items():
|
||||
if isinstance(v, str):
|
||||
container[k] = _truncate_base64_in_string(v)
|
||||
container[k] = truncate_string(v)
|
||||
elif isinstance(v, dict):
|
||||
copy: dict | list = {ck: cv for ck, cv in v.items()}
|
||||
copy: dict[str, object] | list[object] = {ck: cv for ck, cv in v.items()}
|
||||
container[k] = copy
|
||||
stack.append((copy, depth + 1))
|
||||
elif isinstance(v, list):
|
||||
copy = list(v)
|
||||
container[k] = copy
|
||||
stack.append((copy, depth + 1))
|
||||
elif isinstance(container, list):
|
||||
else:
|
||||
for i, v in enumerate(container):
|
||||
if isinstance(v, str):
|
||||
container[i] = _truncate_base64_in_string(v)
|
||||
container[i] = truncate_string(v)
|
||||
elif isinstance(v, dict):
|
||||
copy = {ck: cv for ck, cv in v.items()}
|
||||
container[i] = copy
|
||||
|
|
@ -127,8 +136,15 @@ def truncate_base64_in_messages(
|
|||
"""
|
||||
if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
|
||||
return messages
|
||||
return _copy_messages(messages, _truncate_base64_in_string)
|
||||
|
||||
|
||||
def _copy_messages(
|
||||
messages: str | dict[str, object] | list[object], # mutable-ok: same contract as truncate_base64_in_messages
|
||||
truncate_string: Callable[[str], str],
|
||||
) -> str | dict[str, object] | list[object] | None: # mutable-ok: same contract as truncate_base64_in_messages
|
||||
try:
|
||||
return _truncate_base64_in_value(messages)
|
||||
return _truncate_base64_in_value(messages, truncate_string)
|
||||
except Exception as e:
|
||||
verbose_logger.debug("Failed to truncate base64 in messages: %s", e)
|
||||
return messages
|
||||
|
|
@ -151,19 +167,31 @@ def _iter_string_leaves(value: _StringTree) -> Iterator[str]:
|
|||
pass
|
||||
|
||||
|
||||
def _is_few_plain_strings(leaves: tuple[str, ...]) -> bool:
|
||||
return len(leaves) < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS and not any(
|
||||
_may_contain_base64_data_uri(leaf) for leaf in leaves
|
||||
)
|
||||
|
||||
|
||||
async def truncate_base64_in_messages_async(
|
||||
messages: str | list | dict | None, # mutable-ok: same contract as truncate_base64_in_messages
|
||||
) -> str | list | dict | None: # mutable-ok: same contract as truncate_base64_in_messages
|
||||
"""
|
||||
Same result as truncate_base64_in_messages, but payloads whose string content
|
||||
reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are scanned in a worker
|
||||
thread so the regex pass over multi-MB base64 images does not block the event loop.
|
||||
reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are copied in a worker thread,
|
||||
so neither the regex pass over multi-MB base64 images nor the copy of a long
|
||||
history blocks the event loop. A few strings with no data URI, under
|
||||
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS in total, are copied inline, since
|
||||
checking them for data URIs and copying them costs less than the thread hop.
|
||||
"""
|
||||
if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
|
||||
return messages
|
||||
total_chars: Final = sum(len(leaf) for leaf in _iter_string_leaves(messages))
|
||||
leaves: Final = tuple(_iter_string_leaves(messages))
|
||||
total_chars: Final = sum(len(leaf) for leaf in leaves)
|
||||
if total_chars < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS:
|
||||
return truncate_base64_in_messages(messages)
|
||||
if total_chars < BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS and _is_few_plain_strings(leaves):
|
||||
return _copy_messages(messages, lambda text: text)
|
||||
return await asyncio.to_thread(truncate_base64_in_messages, messages)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from openai.types.responses.response_create_params import (
|
|||
ResponseCreateParamsNonStreaming,
|
||||
ResponseCreateParamsStreaming,
|
||||
)
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.types.llms.anthropic import AnthropicMessagesRequest
|
||||
|
|
@ -41,6 +42,16 @@ def peek_reasoning_summary_aliases(optional_params: Mapping[str, object]) -> obj
|
|||
return None
|
||||
|
||||
|
||||
_OPTIONAL_BOOL: Final = TypeAdapter[bool | None](bool | None)
|
||||
|
||||
|
||||
def embeds_thought_signature_in_tool_call_id(litellm_params: Mapping[str, object]) -> bool:
|
||||
try:
|
||||
return _OPTIONAL_BOOL.validate_python(litellm_params.get("thought_signature_in_tool_call_id")) is not False
|
||||
except ValidationError:
|
||||
return True
|
||||
|
||||
|
||||
class ModelParamHelper:
|
||||
# Cached at class level — deterministic set built from static OpenAI type annotations
|
||||
relevant_logging_args: ClassVar[frozenset[str]] = frozenset()
|
||||
|
|
|
|||
|
|
@ -88,17 +88,22 @@ def handle_any_messages_to_chat_completion_str_messages_conversion(
|
|||
return [{"input": str(messages)}]
|
||||
|
||||
|
||||
def _message_with_str_content(message: AllMessageValues) -> AllMessageValues:
|
||||
texts: Final = convert_content_list_to_str(message=message)
|
||||
if not texts:
|
||||
return message
|
||||
return cast( # cast-ok: the same message with its content flattened to one string
|
||||
AllMessageValues, {**message, "content": texts}
|
||||
)
|
||||
|
||||
|
||||
def handle_messages_with_content_list_to_str_conversion(
|
||||
messages: list[AllMessageValues],
|
||||
) -> list[AllMessageValues]:
|
||||
"""
|
||||
Handles messages with content list conversion
|
||||
Returns new messages with each content list flattened to a string, leaving the caller's messages untouched
|
||||
"""
|
||||
for message in messages:
|
||||
texts = convert_content_list_to_str(message=message)
|
||||
if texts:
|
||||
message["content"] = texts
|
||||
return messages
|
||||
return [_message_with_str_content(message) for message in messages]
|
||||
|
||||
|
||||
def strip_name_from_message(message: AllMessageValues, allowed_name_roles: list[str] = ["user"]) -> AllMessageValues:
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or
|
|||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
|
||||
anthropic_system_messages,
|
||||
find_anthropic_server_tool_result,
|
||||
is_anthropic_server_tool_use,
|
||||
is_anthropic_tool_search_use,
|
||||
message_field,
|
||||
parts_of,
|
||||
|
|
@ -1252,6 +1253,22 @@ def encode_tool_call_id_with_signature(tool_call_id: str, thought_signature: str
|
|||
_encode_tool_call_id_with_signature = encode_tool_call_id_with_signature
|
||||
|
||||
|
||||
def _tool_call_id_prefix(tool_call_id: object) -> str | None:
|
||||
return tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0] if isinstance(tool_call_id, str) else None
|
||||
|
||||
|
||||
def _called_tool_name(tool_call_id: str, tool_calls: Sequence[Mapping[str, object]]) -> str | None:
|
||||
exact_matches: Final = tuple(call for call in tool_calls if call.get("id") == tool_call_id)
|
||||
prefix: Final = _tool_call_id_prefix(tool_call_id)
|
||||
prefix_matches: Final = tuple(
|
||||
call for call in tool_calls if prefix and _tool_call_id_prefix(call.get("id")) == prefix
|
||||
)
|
||||
matches: Final = exact_matches or (prefix_matches if len(prefix_matches) == 1 else ())
|
||||
function: Final = matches[0].get("function") if matches else None
|
||||
name: Final = function.get("name") if isinstance(function, Mapping) else None
|
||||
return name if isinstance(name, str) else None
|
||||
|
||||
|
||||
def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None:
|
||||
"""Extract thought signature from tool call's provider_specific_fields.
|
||||
|
||||
|
|
@ -1691,16 +1708,14 @@ def convert_to_gemini_tool_call_result(
|
|||
)
|
||||
) is not None:
|
||||
media_parts.append(media_part)
|
||||
name: str | None = message.get("name", "")
|
||||
|
||||
# Recover name from last message with tool calls
|
||||
if last_message_with_tool_calls:
|
||||
tools: Final = last_message_with_tool_calls.get("tool_calls", [])
|
||||
msg_tool_call_id: Final = message.get("tool_call_id", None)
|
||||
for tool in tools:
|
||||
prev_tool_call_id = tool.get("id", None)
|
||||
if msg_tool_call_id and prev_tool_call_id and msg_tool_call_id == prev_tool_call_id:
|
||||
name = tool.get("function", {}).get("name", "")
|
||||
msg_tool_call_id: Final = message.get("tool_call_id", None)
|
||||
called_tool_name: Final = (
|
||||
_called_tool_name(msg_tool_call_id, last_message_with_tool_calls.get("tool_calls") or ())
|
||||
if last_message_with_tool_calls and isinstance(msg_tool_call_id, str) and msg_tool_call_id
|
||||
else None
|
||||
)
|
||||
name: Final[str | None] = called_tool_name if called_tool_name is not None else message.get("name", "")
|
||||
|
||||
# Echo the OpenAI tool_call_id on functionResponse (strip thought-signature suffix).
|
||||
gemini_call_id: str | None = None
|
||||
|
|
@ -2059,6 +2074,54 @@ def _placed_and_paired(
|
|||
_AnthropicToolInvokeItem: TypeAlias = AnthropicMessagesToolUseParam | dict[str, Any]
|
||||
|
||||
|
||||
def _is_server_tool_result(block: Mapping[str, object]) -> bool:
|
||||
block_type: Final = block.get("type")
|
||||
return isinstance(block_type, str) and block_type.endswith("_tool_result")
|
||||
|
||||
|
||||
def _split_unrun_server_tool_uses(
|
||||
groups: Sequence[Sequence[_AnthropicToolInvokeItem]],
|
||||
) -> tuple[tuple[Sequence[_AnthropicToolInvokeItem], ...], tuple[_AnthropicToolInvokeItem, ...]]:
|
||||
completed: Final = tuple(group for group in groups if any(_is_server_tool_result(block) for block in group))
|
||||
unrun: Final = tuple(group[0] for group in groups if not any(_is_server_tool_result(block) for block in group))
|
||||
return completed, unrun
|
||||
|
||||
|
||||
def _block_id(block: Mapping[str, object]) -> object:
|
||||
return block.get("id")
|
||||
|
||||
|
||||
def _unrun_server_tool_use_ids(blocks: Sequence[Mapping[str, object]]) -> frozenset[object]:
|
||||
result_ids: Final = frozenset(block.get("tool_use_id") for block in blocks if _is_server_tool_result(block))
|
||||
return frozenset(
|
||||
_block_id(block)
|
||||
for block in blocks
|
||||
if block.get("type") == "server_tool_use" and _block_id(block) not in result_ids
|
||||
)
|
||||
|
||||
|
||||
def _last_block_id_among(blocks: Sequence[Mapping[str, object]], ids: frozenset[object]) -> object:
|
||||
return next(block_id for block in reversed(blocks) if (block_id := _block_id(block)) in ids)
|
||||
|
||||
|
||||
def _paused_turn_replay(
|
||||
tool_invokes: Sequence[_AnthropicToolInvokeItem],
|
||||
thinking_blocks: Sequence[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
|
||||
*,
|
||||
is_final_message: bool,
|
||||
) -> tuple[
|
||||
tuple[_AnthropicToolInvokeItem, ...],
|
||||
tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...],
|
||||
]:
|
||||
unrun_ids: Final = _unrun_server_tool_use_ids(tool_invokes)
|
||||
if not unrun_ids:
|
||||
return tuple(tool_invokes), tuple(thinking_blocks)
|
||||
dropped_ids: Final = (
|
||||
unrun_ids - frozenset((_last_block_id_among(tool_invokes, unrun_ids),)) if is_final_message else unrun_ids
|
||||
)
|
||||
return tuple(block for block in tool_invokes if _block_id(block) not in dropped_ids), ()
|
||||
|
||||
|
||||
def convert_to_anthropic_tool_invoke(
|
||||
tool_calls: list[ChatCompletionAssistantToolCall],
|
||||
web_search_results: Sequence[object] | None = None,
|
||||
|
|
@ -2141,6 +2204,15 @@ def convert_to_anthropic_tool_invoke(
|
|||
anthropic_tool_invoke.append(server_tool_result)
|
||||
elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids:
|
||||
continue
|
||||
elif is_anthropic_server_tool_use(tool_id) and tool_id not in answered_tool_ids:
|
||||
anthropic_tool_invoke.append(
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": tool_id,
|
||||
"name": tool_name,
|
||||
"input": tool_input,
|
||||
}
|
||||
)
|
||||
else:
|
||||
sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id)
|
||||
_anthropic_tool_use_param = AnthropicMessagesToolUseParam(
|
||||
|
|
@ -3109,6 +3181,30 @@ def anthropic_messages_pt(
|
|||
_has_server_tool_calls = True
|
||||
break
|
||||
|
||||
# Get web_search_results and tool_results from provider_specific_fields
|
||||
# for server_tool_use reconstruction.
|
||||
# Fixes: https://github.com/BerriAI/litellm/issues/17737
|
||||
_provider_specific_fields_raw = assistant_content_block.get("provider_specific_fields")
|
||||
_provider_specific_fields: dict[str, Any] = (
|
||||
cast(dict[str, Any], _provider_specific_fields_raw)
|
||||
if isinstance(_provider_specific_fields_raw, dict)
|
||||
else {}
|
||||
)
|
||||
tool_invoke_results = (
|
||||
convert_to_anthropic_tool_invoke(
|
||||
assistant_tool_calls,
|
||||
web_search_results=_provider_specific_fields.get("web_search_results"),
|
||||
tool_results=_provider_specific_fields.get("tool_results"),
|
||||
answered_tool_ids=client_answered_tool_ids,
|
||||
)
|
||||
if assistant_tool_calls is not None
|
||||
else ()
|
||||
)
|
||||
turn_paused = bool(_unrun_server_tool_use_ids(tool_invoke_results))
|
||||
replayed_tool_invokes, replayed_thinking_blocks = _paused_turn_replay(
|
||||
tool_invoke_results, thinking_blocks or (), is_final_message=msg_i == len(messages) - 1
|
||||
)
|
||||
|
||||
if (
|
||||
thinking_blocks is not None
|
||||
and _has_server_tool_calls
|
||||
|
|
@ -3121,26 +3217,12 @@ def anthropic_messages_pt(
|
|||
# We must preserve this interleaved order because Anthropic
|
||||
# verifies thinking block signatures based on position.
|
||||
|
||||
# Build the tool call groups (server_tool_use + its result)
|
||||
_provider_specific_fields_raw_tc = assistant_content_block.get("provider_specific_fields")
|
||||
_provider_specific_fields_tc: dict[str, Any] = {}
|
||||
if isinstance(_provider_specific_fields_raw_tc, dict):
|
||||
_provider_specific_fields_tc = cast(dict[str, Any], _provider_specific_fields_raw_tc)
|
||||
_web_search_results_tc = _provider_specific_fields_tc.get("web_search_results")
|
||||
_tool_results_tc = _provider_specific_fields_tc.get("tool_results")
|
||||
tool_invoke_results = convert_to_anthropic_tool_invoke(
|
||||
assistant_tool_calls,
|
||||
web_search_results=_web_search_results_tc,
|
||||
tool_results=_tool_results_tc,
|
||||
answered_tool_ids=client_answered_tool_ids,
|
||||
)
|
||||
|
||||
# Group tool invoke results into (server_tool_use, result) pairs
|
||||
# and separate regular tool_use blocks
|
||||
server_tool_groups: list[list[_AnthropicToolInvokeItem]] = []
|
||||
regular_tool_uses: list[_AnthropicToolInvokeItem] = []
|
||||
_current_group: list[_AnthropicToolInvokeItem] = []
|
||||
for item in tool_invoke_results:
|
||||
for item in replayed_tool_invokes:
|
||||
item_type = item.get("type", "") if isinstance(item, dict) else getattr(item, "type", "")
|
||||
if item_type == "server_tool_use":
|
||||
if _current_group:
|
||||
|
|
@ -3154,6 +3236,9 @@ def anthropic_messages_pt(
|
|||
_current_group.append(item)
|
||||
if _current_group:
|
||||
server_tool_groups.append(_current_group)
|
||||
completed_server_tool_groups, pending_server_tool_uses = _split_unrun_server_tool_uses(
|
||||
server_tool_groups
|
||||
)
|
||||
|
||||
# Build the text block if content is a non-empty string
|
||||
text_element = None
|
||||
|
|
@ -3177,15 +3262,15 @@ def anthropic_messages_pt(
|
|||
# Any remaining groups (after all thinking blocks) go after.
|
||||
tb_idx = 0
|
||||
grp_idx = 0
|
||||
num_tb = len(thinking_blocks) if thinking_blocks else 0
|
||||
num_grp = len(server_tool_groups)
|
||||
num_tb = len(replayed_thinking_blocks)
|
||||
num_grp = len(completed_server_tool_groups)
|
||||
|
||||
while tb_idx < num_tb or grp_idx < num_grp:
|
||||
if tb_idx < num_tb and grp_idx < num_grp:
|
||||
# Emit thinking block then its tool group
|
||||
assistant_content.append(thinking_blocks[tb_idx])
|
||||
assistant_content.append(replayed_thinking_blocks[tb_idx])
|
||||
tb_idx += 1
|
||||
for block in server_tool_groups[grp_idx]:
|
||||
for block in completed_server_tool_groups[grp_idx]:
|
||||
item_id = block.get("id") if isinstance(block, dict) else getattr(block, "id", None)
|
||||
if item_id and item_id in unique_tool_ids:
|
||||
continue
|
||||
|
|
@ -3195,11 +3280,11 @@ def anthropic_messages_pt(
|
|||
grp_idx += 1
|
||||
elif tb_idx < num_tb:
|
||||
# More thinking blocks than tool groups - emit before text
|
||||
assistant_content.append(thinking_blocks[tb_idx])
|
||||
assistant_content.append(replayed_thinking_blocks[tb_idx])
|
||||
tb_idx += 1
|
||||
else:
|
||||
# More tool groups than thinking blocks - emit remaining
|
||||
for block in server_tool_groups[grp_idx]:
|
||||
for block in completed_server_tool_groups[grp_idx]:
|
||||
item_id = block.get("id") if isinstance(block, dict) else getattr(block, "id", None)
|
||||
if item_id and item_id in unique_tool_ids:
|
||||
continue
|
||||
|
|
@ -3212,8 +3297,8 @@ def anthropic_messages_pt(
|
|||
if text_element is not None:
|
||||
assistant_content.append(text_element)
|
||||
|
||||
# Add regular (non-server) tool calls at the end
|
||||
for item in regular_tool_uses:
|
||||
# Add regular tool calls, then a paused turn's unrun server tool call, at the end
|
||||
for item in (*regular_tool_uses, *pending_server_tool_uses):
|
||||
item_id = item.get("id") if isinstance(item, dict) else getattr(item, "id", None)
|
||||
if item_id and item_id in unique_tool_ids:
|
||||
continue
|
||||
|
|
@ -3233,18 +3318,18 @@ def anthropic_messages_pt(
|
|||
)
|
||||
_content_list = assistant_content_block.get("content") if _content_is_list else None
|
||||
|
||||
if (
|
||||
thinking_blocks is not None and not _content_is_list
|
||||
): # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR
|
||||
assistant_content.extend(thinking_blocks)
|
||||
if not _content_is_list: # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR
|
||||
assistant_content.extend(replayed_thinking_blocks)
|
||||
if _content_is_list and _content_list is not None:
|
||||
for m in _with_missing_thinking_blocks(_content_list, thinking_blocks or ()):
|
||||
for m in _with_missing_thinking_blocks(_content_list, replayed_thinking_blocks):
|
||||
if not isinstance(m, dict):
|
||||
continue
|
||||
# handle thinking blocks
|
||||
if (thinking_block := _as_thinking_block(m)) is not None:
|
||||
assistant_content.extend(
|
||||
_replayable_thinking_blocks([thinking_block], drop_params=drop_params)
|
||||
()
|
||||
if turn_paused
|
||||
else _replayable_thinking_blocks([thinking_block], drop_params=drop_params)
|
||||
)
|
||||
continue
|
||||
text_block = cast(str, m.get("text", ""))
|
||||
|
|
@ -3284,25 +3369,9 @@ def anthropic_messages_pt(
|
|||
assistant_content.append(_anthropic_text_content_element)
|
||||
|
||||
if assistant_tool_calls is not None: # support assistant tool invoke conversion
|
||||
# Get web_search_results and tool_results from provider_specific_fields
|
||||
# for server_tool_use reconstruction.
|
||||
# Fixes: https://github.com/BerriAI/litellm/issues/17737
|
||||
_provider_specific_fields_raw = assistant_content_block.get("provider_specific_fields")
|
||||
_provider_specific_fields: dict[str, Any] = {}
|
||||
if isinstance(_provider_specific_fields_raw, dict):
|
||||
_provider_specific_fields = cast(dict[str, Any], _provider_specific_fields_raw)
|
||||
_web_search_results = _provider_specific_fields.get("web_search_results")
|
||||
_tool_results = _provider_specific_fields.get("tool_results")
|
||||
tool_invoke_results = convert_to_anthropic_tool_invoke(
|
||||
assistant_tool_calls,
|
||||
web_search_results=_web_search_results,
|
||||
tool_results=_tool_results,
|
||||
answered_tool_ids=client_answered_tool_ids,
|
||||
)
|
||||
|
||||
# Prevent "tool_use ids must be unique" errors by filtering duplicates
|
||||
# This can happen when merging history that already contains the tool calls
|
||||
for item in tool_invoke_results:
|
||||
for item in replayed_tool_invokes:
|
||||
# tool_use items are typically dicts, but handle objects just in case
|
||||
item_id = item.get("id") if isinstance(item, dict) else getattr(item, "id", None)
|
||||
|
||||
|
|
|
|||
|
|
@ -286,6 +286,10 @@ def _separate_thinking_blocks_render(message: object) -> bool:
|
|||
return any(_thinking_block_renders(block) for block in parts_of(message_field(message, "thinking_blocks")))
|
||||
|
||||
|
||||
def is_anthropic_server_tool_use(tool_id: str) -> bool:
|
||||
return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
|
||||
|
||||
|
||||
def is_anthropic_tool_search_use(tool_id: str, tool_name: str) -> bool:
|
||||
return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) and tool_name in ANTHROPIC_TOOL_SEARCH_TOOL_NAMES
|
||||
|
||||
|
|
|
|||
|
|
@ -1509,11 +1509,20 @@ def _delta_reasoning_items(delta: object) -> Sequence[object]:
|
|||
return cast(Sequence[object], items) if isinstance(items, list) else () # cast-ok: items forwarded verbatim
|
||||
|
||||
|
||||
def _chunk_reasoning_items(chunk: object) -> Sequence[object]:
|
||||
def _chunk_first_delta(chunk: object) -> object:
|
||||
choices: Final = _field_of(chunk, "choices")
|
||||
if not isinstance(choices, list) or not choices:
|
||||
return ()
|
||||
return _delta_reasoning_items(_field_of(cast(Sequence[object], choices)[0], "delta")) # cast-ok: untyped chunk
|
||||
return None
|
||||
return _field_of(cast(Sequence[object], choices)[0], "delta") # cast-ok: untyped chunk
|
||||
|
||||
|
||||
def _chunk_reasoning_items(chunk: object) -> Sequence[object]:
|
||||
return _delta_reasoning_items(_chunk_first_delta(chunk))
|
||||
|
||||
|
||||
def _chunk_refusal(chunk: object) -> str:
|
||||
refusal: Final = _field_of(_chunk_first_delta(chunk), "refusal")
|
||||
return refusal if isinstance(refusal, str) else ""
|
||||
|
||||
|
||||
_NON_TEXT_DELTA_FIELDS: Final = (
|
||||
|
|
@ -1526,6 +1535,7 @@ _NON_TEXT_DELTA_FIELDS: Final = (
|
|||
"audio",
|
||||
"images",
|
||||
"provider_specific_fields",
|
||||
"refusal",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1674,6 +1684,9 @@ def _assembled_message_fields(processor: ChunkProcessor, chunks: Sequence) -> It
|
|||
yield "reasoning_content", processor.get_combined_reasoning_content(reasoning_chunks)
|
||||
|
||||
typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream
|
||||
combined_refusal: Final = "".join(_chunk_refusal(chunk) for chunk in typed_chunks)
|
||||
if combined_refusal:
|
||||
yield "refusal", combined_refusal
|
||||
reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
|
||||
if reasoning_items:
|
||||
yield "reasoning_items", list(reasoning_items)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ from litellm.types.utils import CallTypes
|
|||
from . import *
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import (
|
||||
BaseTranslation,
|
||||
)
|
||||
|
|
@ -30,7 +32,29 @@ def get_cost_for_google_maps_grounding_request(
|
|||
return cost_per_google_maps_grounding_request(usage=usage, model_info=model_info)
|
||||
|
||||
|
||||
def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo") -> float | None:
|
||||
def _with_served_model_search_pricing(
|
||||
model_info: "ModelInfo",
|
||||
response_model: str | None,
|
||||
served_model_provider: str,
|
||||
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None",
|
||||
) -> "ModelInfo":
|
||||
if model_info.get("search_context_cost_per_query") is not None or response_model is None:
|
||||
return model_info
|
||||
served_model_info: Final = (
|
||||
lookup_model_info(response_model, served_model_provider) if lookup_model_info is not None else None
|
||||
)
|
||||
if served_model_info is None:
|
||||
return model_info
|
||||
return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")}
|
||||
|
||||
|
||||
def get_cost_for_web_search_request(
|
||||
custom_llm_provider: str,
|
||||
usage: "Usage",
|
||||
model_info: "ModelInfo",
|
||||
response_model: str | None = None,
|
||||
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None" = None,
|
||||
) -> float | None:
|
||||
"""
|
||||
Get the cost for a web search request for a given model.
|
||||
|
||||
|
|
@ -38,6 +62,8 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
|
|||
custom_llm_provider: The custom LLM provider.
|
||||
usage: The usage object.
|
||||
model_info: The model info.
|
||||
response_model: The model the provider response names.
|
||||
lookup_model_info: Looks up a model's cost map entry, returning None when it is not mapped.
|
||||
"""
|
||||
if custom_llm_provider == "gemini":
|
||||
from .gemini.cost_calculator import cost_per_web_search_request
|
||||
|
|
@ -46,17 +72,25 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
|
|||
elif custom_llm_provider == "anthropic":
|
||||
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
|
||||
|
||||
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
|
||||
return get_cost_for_anthropic_web_search(
|
||||
model_info=_with_served_model_search_pricing(model_info, response_model, "anthropic", lookup_model_info),
|
||||
usage=usage,
|
||||
)
|
||||
elif custom_llm_provider.startswith("vertex_ai"):
|
||||
# Anthropic Claude models on Vertex AI populate server_tool_use.web_search_requests
|
||||
# (same as the direct Anthropic API), not prompt_tokens_details.web_search_requests
|
||||
# (which is the Gemini field). Route claude-* models to the Anthropic calculator.
|
||||
model_key: Final[str] = model_info.get("key", "") if model_info else ""
|
||||
if "claude" in model_key.lower():
|
||||
if "claude" in (response_model or model_key).lower():
|
||||
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
|
||||
|
||||
verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator")
|
||||
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
|
||||
return get_cost_for_anthropic_web_search(
|
||||
model_info=_with_served_model_search_pricing(
|
||||
model_info, response_model, "vertex_ai", lookup_model_info
|
||||
),
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
from .vertex_ai.gemini.cost_calculator import (
|
||||
cost_per_web_search_request as cost_per_web_search_request_vertex_ai,
|
||||
|
|
@ -71,7 +105,10 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
|
|||
return None
|
||||
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
|
||||
|
||||
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
|
||||
return get_cost_for_anthropic_web_search(
|
||||
model_info=_with_served_model_search_pricing(model_info, response_model, "azure_ai", lookup_model_info),
|
||||
usage=usage,
|
||||
)
|
||||
elif custom_llm_provider == "perplexity":
|
||||
# Perplexity handles search costs internally in its own cost calculator
|
||||
# Return 0.0 to indicate costs are already accounted for
|
||||
|
|
|
|||
|
|
@ -70,7 +70,12 @@ from litellm.types.utils import (
|
|||
)
|
||||
|
||||
from ...base import BaseLLM
|
||||
from ..common_utils import AnthropicError, process_anthropic_headers, refusal_provider_specific_fields
|
||||
from ..common_utils import (
|
||||
AnthropicError,
|
||||
process_anthropic_headers,
|
||||
refusal_params,
|
||||
stop_reason_provider_specific_fields,
|
||||
)
|
||||
from .transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY, AnthropicConfig, JsonModeRequest
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -926,6 +931,17 @@ class ModelResponseIterator:
|
|||
def _web_search_call_snapshot(self) -> dict[str, object]:
|
||||
return dict(self._web_search_calls)
|
||||
|
||||
def _refresh_pending_web_search_call(self, tool_use_id: str) -> bool:
|
||||
if tool_use_id not in self._web_search_calls:
|
||||
return False
|
||||
self._web_search_calls[tool_use_id] = build_web_search_call(
|
||||
tool_use_id,
|
||||
self._server_tool_inputs.get(tool_use_id, {}),
|
||||
{"content": []},
|
||||
status="in_progress",
|
||||
)
|
||||
return True
|
||||
|
||||
def _complete_web_search_call(self, result: dict[str, object]) -> None:
|
||||
tool_use_id: Final = result.get("tool_use_id")
|
||||
if not isinstance(tool_use_id, str) or tool_use_id not in self._web_search_calls:
|
||||
|
|
@ -1128,6 +1144,8 @@ class ModelResponseIterator:
|
|||
self._server_tool_inputs[self._current_server_tool_id] = json.loads(args)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
if self._refresh_pending_web_search_call(self._current_server_tool_id):
|
||||
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
|
||||
self._current_server_tool_id = None
|
||||
if self.is_response_format_tool:
|
||||
text = self._flush_response_format_buffer()
|
||||
|
|
@ -1140,8 +1158,8 @@ class ModelResponseIterator:
|
|||
# These are automatically handled by Anthropic API, we just pass them through
|
||||
pass
|
||||
elif type_chunk == "message_delta":
|
||||
finish_reason, usage, container, refusal_fields = self._handle_message_delta(chunk)
|
||||
provider_specific_fields.update(refusal_fields)
|
||||
finish_reason, usage, container, stop_reason_fields = self._handle_message_delta(chunk)
|
||||
provider_specific_fields.update(stop_reason_fields)
|
||||
text = self._release_answer_close() + self._drain_split_answer_blocks()
|
||||
if container:
|
||||
provider_specific_fields["container"] = container
|
||||
|
|
@ -1190,6 +1208,7 @@ class ModelResponseIterator:
|
|||
)
|
||||
|
||||
text, tool_use = self._handle_json_mode_chunk(text=text, tool_use=tool_use)
|
||||
refusal_fields: Final = refusal_params(provider_specific_fields)
|
||||
|
||||
returned_chunk: Final = ModelResponseStream(
|
||||
choices=[
|
||||
|
|
@ -1201,6 +1220,7 @@ class ModelResponseIterator:
|
|||
provider_specific_fields=(provider_specific_fields if provider_specific_fields else None),
|
||||
thinking_blocks=(thinking_blocks if thinking_blocks else None),
|
||||
reasoning_content=reasoning_content,
|
||||
**refusal_fields,
|
||||
),
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
|
|
@ -1371,7 +1391,7 @@ class ModelResponseIterator:
|
|||
self._handle_usage(anthropic_usage_chunk=message_delta["usage"]) if "usage" in message_delta else None
|
||||
)
|
||||
container: Final = message_delta["delta"].get("container")
|
||||
return finish_reason, usage, container, refusal_provider_specific_fields(message_delta["delta"])
|
||||
return finish_reason, usage, container, stop_reason_provider_specific_fields(message_delta["delta"])
|
||||
|
||||
def _handle_accumulated_json_chunk(self, data_str: str, is_final: bool = False) -> ModelResponseStream | None:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -120,13 +120,14 @@ from litellm.utils import (
|
|||
)
|
||||
|
||||
from ..common_utils import (
|
||||
REFUSAL_STOP_FIELDS,
|
||||
STOP_REASON_FIELDS,
|
||||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
eager_input_streaming_flag,
|
||||
process_anthropic_headers,
|
||||
refusal_provider_specific_fields,
|
||||
refusal_params,
|
||||
requires_native_compaction_beta,
|
||||
stop_reason_provider_specific_fields,
|
||||
strip_advisor_blocks_from_messages,
|
||||
tools_marked_for_a_tool_config_point,
|
||||
)
|
||||
|
|
@ -167,6 +168,9 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
|
|||
_TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_TOOLS_ADAPTER: Final = TypeAdapter(list[object])
|
||||
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_RESPONSE_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
_PENDING_WEB_SEARCH_RESULT: Final = MappingProxyType({"content": ()})
|
||||
_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
|
||||
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
|
||||
)
|
||||
|
|
@ -2254,6 +2258,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
headers=headers,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _messages_within_cache_control_cap(
|
||||
messages: Sequence[AllMessageValues], optional_params: Mapping[str, object]
|
||||
) -> Sequence[AllMessageValues]:
|
||||
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
|
||||
|
||||
return messages_within_cache_control_cap(
|
||||
messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params
|
||||
)
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -2271,9 +2285,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
anthropic_messages_pt,
|
||||
)
|
||||
|
||||
drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True
|
||||
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
capped_messages: Final = (
|
||||
list(
|
||||
self._messages_within_cache_control_cap(
|
||||
messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params)
|
||||
)
|
||||
)
|
||||
if drop_params
|
||||
else messages
|
||||
)
|
||||
|
||||
if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages):
|
||||
if "tools" not in optional_params and has_tool_call_blocks(capped_messages):
|
||||
optional_params["tools"], _ = self._map_tools(add_dummy_tool(custom_llm_provider="anthropic"))
|
||||
|
||||
# Drop thinking param if thinking is enabled but thinking_blocks are missing
|
||||
|
|
@ -2285,9 +2308,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
# Related issue: https://github.com/BerriAI/litellm/issues/18926
|
||||
if (
|
||||
optional_params.get("thinking") is not None
|
||||
and messages is not None
|
||||
and last_assistant_with_tool_calls_has_no_thinking_blocks(messages)
|
||||
and not any_assistant_message_has_thinking_blocks(messages)
|
||||
and last_assistant_with_tool_calls_has_no_thinking_blocks(capped_messages)
|
||||
and not any_assistant_message_has_thinking_blocks(capped_messages)
|
||||
):
|
||||
if litellm.modify_params:
|
||||
optional_params.pop("thinking", None)
|
||||
|
|
@ -2332,15 +2354,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
_name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request(
|
||||
optional_params=optional_params,
|
||||
)
|
||||
if _name_forward_map:
|
||||
messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map)
|
||||
request_messages: Final = (
|
||||
self._rewrite_tool_names_in_messages(capped_messages, _name_forward_map)
|
||||
if _name_forward_map
|
||||
else capped_messages
|
||||
)
|
||||
if _name_reverse_map and isinstance(litellm_params, dict):
|
||||
litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map
|
||||
|
||||
# Only the leading system run becomes the top-level system prompt. A later
|
||||
# system message stays in the conversation: hoisting it rewrites the cached
|
||||
# prefix and re-bills the whole history at cache-write pricing (#36559).
|
||||
leading_system_run, later_messages = split_leading_system_run(messages)
|
||||
leading_system_run, later_messages = split_leading_system_run(request_messages)
|
||||
anthropic_system_message_list: Final = self.translate_system_message(messages=list(leading_system_run))
|
||||
# Handling anthropic API Prompt Caching
|
||||
if len(anthropic_system_message_list) > 0:
|
||||
|
|
@ -2362,7 +2387,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
except Exception as e:
|
||||
raise AnthropicError(
|
||||
status_code=400,
|
||||
message=f"{e}\nReceived Messages={messages}",
|
||||
message=f"{e}\nReceived Messages={request_messages}",
|
||||
) # don't use verbose_logger.exception, if exception is raised
|
||||
|
||||
self.update_headers_with_optional_anthropic_beta(
|
||||
|
|
@ -2897,28 +2922,24 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
web_search_results: Sequence[object],
|
||||
completion_response: Mapping[str, object],
|
||||
) -> list[object]:
|
||||
content: Final = completion_response.get("content")
|
||||
blocks: Final = content if isinstance(content, Sequence) else ()
|
||||
inputs: Final = {
|
||||
call_id: tool_input
|
||||
for block in blocks
|
||||
if isinstance(block, Mapping)
|
||||
and block.get("type") == "server_tool_use"
|
||||
and block.get("name") == "web_search"
|
||||
and isinstance((call_id := block.get("id")), str)
|
||||
and isinstance((tool_input := block.get("input")), Mapping)
|
||||
blocks: Final = _RESPONSE_BLOCKS.validate_python(completion_response.get("content") or ())
|
||||
results_by_id: Final = {
|
||||
tool_use_id: result
|
||||
for result in _RESPONSE_BLOCKS.validate_python(web_search_results)
|
||||
if result.get("type") == "web_search_tool_result"
|
||||
and isinstance((tool_use_id := result.get("tool_use_id")), str)
|
||||
}
|
||||
return [
|
||||
build_web_search_call(
|
||||
tool_id=tool_use_id,
|
||||
tool_input=inputs.get(tool_use_id, {}),
|
||||
result=result,
|
||||
tool_id=call_id,
|
||||
tool_input=block.get("input"),
|
||||
result=results_by_id.get(call_id, _PENDING_WEB_SEARCH_RESULT),
|
||||
status=None if call_id in results_by_id else "in_progress",
|
||||
)
|
||||
for result in web_search_results
|
||||
if isinstance(result, dict)
|
||||
and result.get("type") == "web_search_tool_result"
|
||||
and isinstance((tool_use_id := result.get("tool_use_id")), str)
|
||||
and tool_use_id in inputs
|
||||
for block in blocks
|
||||
if block.get("type") == "server_tool_use"
|
||||
and block.get("name") == "web_search"
|
||||
and isinstance((call_id := block.get("id")), str)
|
||||
]
|
||||
|
||||
def _build_provider_specific_fields(
|
||||
|
|
@ -2942,10 +2963,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
|
||||
if web_search_results is not None:
|
||||
provider_specific_fields["web_search_results"] = web_search_results
|
||||
provider_specific_fields["web_search_calls"] = self._build_web_search_calls(
|
||||
web_search_results,
|
||||
completion_response,
|
||||
)
|
||||
|
||||
web_search_calls: Final = self._build_web_search_calls(web_search_results or (), completion_response)
|
||||
if web_search_results is not None or web_search_calls:
|
||||
provider_specific_fields["web_search_calls"] = web_search_calls
|
||||
|
||||
if tool_results is not None:
|
||||
provider_specific_fields["tool_results"] = tool_results
|
||||
|
|
@ -2967,10 +2988,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
if compaction_blocks is not None:
|
||||
provider_specific_fields["compaction_blocks"] = compaction_blocks
|
||||
|
||||
refusal_fields: Final = refusal_provider_specific_fields(
|
||||
REFUSAL_STOP_FIELDS.validate_python(completion_response)
|
||||
stop_reason_fields: Final = stop_reason_provider_specific_fields(
|
||||
STOP_REASON_FIELDS.validate_python(completion_response)
|
||||
)
|
||||
return {**provider_specific_fields, **refusal_fields}
|
||||
return {**provider_specific_fields, **stop_reason_fields}
|
||||
|
||||
def transform_parsed_response(
|
||||
self,
|
||||
|
|
@ -3045,22 +3066,26 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
if json_extra_content:
|
||||
merged_text = merged_text + json_extra_content if merged_text else json_extra_content
|
||||
|
||||
refusal_fields: Final = refusal_params(STOP_REASON_FIELDS.validate_python(completion_response))
|
||||
_message = litellm.Message(
|
||||
tool_calls=tool_calls_for_message,
|
||||
content=merged_text or None,
|
||||
provider_specific_fields=provider_specific_fields,
|
||||
thinking_blocks=thinking_blocks,
|
||||
reasoning_content=reasoning_content,
|
||||
**refusal_fields,
|
||||
)
|
||||
_message.provider_specific_fields = provider_specific_fields
|
||||
|
||||
if json_mode_message is not None:
|
||||
refusal_fields: Final = refusal_provider_specific_fields(
|
||||
REFUSAL_STOP_FIELDS.validate_python(completion_response)
|
||||
stop_reason_fields: Final = stop_reason_provider_specific_fields(
|
||||
STOP_REASON_FIELDS.validate_python(completion_response)
|
||||
)
|
||||
completion_response["stop_reason"] = "stop"
|
||||
if refusal_fields:
|
||||
json_mode_message.provider_specific_fields = dict(refusal_fields)
|
||||
if stop_reason_fields:
|
||||
json_mode_message.provider_specific_fields = dict(stop_reason_fields)
|
||||
if "refusal" in refusal_fields:
|
||||
json_mode_message.refusal = refusal_fields["refusal"]
|
||||
if thinking_blocks:
|
||||
json_mode_message.thinking_blocks = thinking_blocks
|
||||
json_mode_message.reasoning_content = reasoning_content
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ This file contains common utils for anthropic calls.
|
|||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Container, Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -19,6 +20,8 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
|
||||
import litellm
|
||||
from litellm.constants import (
|
||||
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH,
|
||||
ANTHROPIC_MAX_USER_ID_LENGTH,
|
||||
DEFAULT_MODEL_CREATED_AT_TIME,
|
||||
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
|
||||
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
|
||||
|
|
@ -178,6 +181,40 @@ def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: boo
|
|||
return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools]
|
||||
|
||||
|
||||
def anthropic_pass_through_api_base() -> str:
|
||||
return os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
|
||||
|
||||
|
||||
def _same_host_and_port(url: httpx.URL, other: str) -> bool:
|
||||
try:
|
||||
other_url: Final = httpx.URL(other)
|
||||
except httpx.InvalidURL:
|
||||
return False
|
||||
return (url.host, url.port) == (other_url.host, other_url.port)
|
||||
|
||||
|
||||
def anthropic_upstream_user_id_limit(url: str) -> int | None:
|
||||
upstream: Final = httpx.URL(url)
|
||||
if not upstream.path.removesuffix("/").endswith("/v1/messages"):
|
||||
return None
|
||||
if upstream.host == "api.anthropic.com":
|
||||
return ANTHROPIC_MAX_USER_ID_LENGTH
|
||||
return (
|
||||
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH
|
||||
if _same_host_and_port(upstream, anthropic_pass_through_api_base())
|
||||
else None
|
||||
)
|
||||
|
||||
|
||||
def _has_email_shape(token: str) -> bool:
|
||||
at_sign: Final = token.find("@", 1)
|
||||
return at_sign != -1 and token.find(".", at_sign + 2, -1) != -1
|
||||
|
||||
|
||||
def anthropic_accepts_user_id(user_id: str, max_length: int) -> bool:
|
||||
return len(user_id) <= max_length and user_id.isprintable() and not any(map(_has_email_shape, user_id.split()))
|
||||
|
||||
|
||||
def is_anthropic_messages_url(url: str) -> bool:
|
||||
"""Check whether a URL addresses Anthropic's Messages API."""
|
||||
parsed_url: Final = urlparse(url)
|
||||
|
|
@ -2481,7 +2518,7 @@ def create_anthropic_model_list_response(
|
|||
}
|
||||
|
||||
|
||||
class RefusalStopFields(TypedDict, total=False):
|
||||
class StopReasonFields(TypedDict, total=False):
|
||||
stop_reason: ReadOnly[object]
|
||||
stop_details: ReadOnly[object]
|
||||
|
||||
|
|
@ -2498,16 +2535,38 @@ def _dict_or_empty(value: object) -> object:
|
|||
return value if _is_dict(value) else _no_stop_fields()
|
||||
|
||||
|
||||
REFUSAL_STOP_FIELDS: Final[TypeAdapter[RefusalStopFields]] = TypeAdapter(
|
||||
Annotated[RefusalStopFields, BeforeValidator(_dict_or_empty)]
|
||||
STOP_REASON_FIELDS: Final[TypeAdapter[StopReasonFields]] = TypeAdapter(
|
||||
Annotated[StopReasonFields, BeforeValidator(_dict_or_empty)]
|
||||
)
|
||||
|
||||
_STOP_REASONS_KEPT_RAW: Final = frozenset({"refusal", "pause_turn"})
|
||||
|
||||
def refusal_provider_specific_fields(message: Mapping[str, object]) -> Mapping[str, object]:
|
||||
|
||||
def stop_reason_provider_specific_fields(message: Mapping[str, object]) -> Mapping[str, object]:
|
||||
stop_reason: Final = message.get("stop_reason")
|
||||
stop_details: Final = message.get("stop_details")
|
||||
if stop_details is not None:
|
||||
return {"stop_reason": stop_reason, "stop_details": stop_details}
|
||||
if stop_reason == "refusal":
|
||||
if stop_reason in _STOP_REASONS_KEPT_RAW:
|
||||
return {"stop_reason": stop_reason}
|
||||
return {}
|
||||
|
||||
|
||||
class _RefusalStopDetails(TypedDict, total=False):
|
||||
explanation: ReadOnly[object]
|
||||
|
||||
|
||||
_REFUSAL_STOP_DETAILS: Final[TypeAdapter[_RefusalStopDetails]] = TypeAdapter(
|
||||
Annotated[_RefusalStopDetails, BeforeValidator(_dict_or_empty)]
|
||||
)
|
||||
|
||||
|
||||
class RefusalParams(TypedDict, total=False):
|
||||
refusal: ReadOnly[str]
|
||||
|
||||
|
||||
def refusal_params(message: Mapping[str, object]) -> RefusalParams:
|
||||
if message.get("stop_reason") != "refusal":
|
||||
return {}
|
||||
explanation: Final = _REFUSAL_STOP_DETAILS.validate_python(message.get("stop_details")).get("explanation")
|
||||
return {"refusal": explanation} if isinstance(explanation, str) and explanation else {}
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
|
|||
from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.llms.anthropic import (
|
||||
AnthropicStopDetails,
|
||||
AppliedEdit,
|
||||
CompactionBlock,
|
||||
ContentBlockDelta,
|
||||
|
|
@ -284,9 +285,7 @@ class _CombinedChunkSplitter:
|
|||
@staticmethod
|
||||
def _is_combined(chunk: "ModelResponseStream") -> bool:
|
||||
"""True if ``chunk`` carries response content AND a finish_reason."""
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
openai_chat_refusal_text,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
|
||||
|
||||
choices: Final = _optional_attr_sequence(chunk, "choices")
|
||||
if not choices:
|
||||
|
|
@ -302,7 +301,7 @@ class _CombinedChunkSplitter:
|
|||
or _optional_attr(delta, "tool_calls")
|
||||
or _optional_attr(delta, "reasoning_content")
|
||||
or _optional_attr(delta, "thinking_blocks")
|
||||
or openai_chat_refusal_text(delta)
|
||||
or refusal_reply_text(delta)
|
||||
)
|
||||
|
||||
_PAYLOAD_FIELD_GROUPS: "tuple[tuple[str, ...], ...]" = (
|
||||
|
|
@ -659,6 +658,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
self.compaction_block = compaction_block
|
||||
self.iterations_usage = iterations_usage
|
||||
self._refusal_text: str = ""
|
||||
self._paused_turn: bool = False
|
||||
self._claude_refusal_stop_details: AnthropicStopDetails | None = None
|
||||
self.sent_compaction_block: bool = False
|
||||
# Per-phase flags so the compaction block's start/delta/stop events
|
||||
# are emitted (and the public state machine is advanced) in
|
||||
|
|
@ -953,7 +954,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
current_content_block_index=self.current_content_block_index,
|
||||
applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
|
||||
)
|
||||
processed_chunk = self._with_refusal_stop_details(processed_chunk)
|
||||
processed_chunk = self._with_stop_reason_fields(processed_chunk)
|
||||
|
||||
# Check if this is a usage chunk and we have a held stop_reason chunk
|
||||
if will_merge_into_held:
|
||||
|
|
@ -1188,7 +1189,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
current_content_block_index=self.current_content_block_index,
|
||||
applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
|
||||
)
|
||||
processed_chunk = self._with_refusal_stop_details(processed_chunk)
|
||||
processed_chunk = self._with_stop_reason_fields(processed_chunk)
|
||||
|
||||
# Check if this is a usage chunk and we have a held stop_reason chunk
|
||||
if will_merge_into_held:
|
||||
|
|
@ -1406,14 +1407,22 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
def _increment_content_block_index(self):
|
||||
self.current_content_block_index += 1
|
||||
|
||||
def _with_refusal_stop_details(
|
||||
def _with_stop_reason_fields(
|
||||
self,
|
||||
processed_chunk: ContentBlockDelta | MessageBlockDelta,
|
||||
) -> ContentBlockDelta | MessageBlockDelta:
|
||||
if processed_chunk["type"] != "message_delta" or not self._refusal_text:
|
||||
if processed_chunk["type"] != "message_delta":
|
||||
return processed_chunk
|
||||
delta: Final = processed_chunk["delta"]
|
||||
if delta.get("stop_reason") == "max_tokens":
|
||||
if self._paused_turn:
|
||||
paused_delta: Final[MessageDelta] = {**delta, "stop_reason": "pause_turn"}
|
||||
paused_chunk: Final[MessageBlockDelta] = {**processed_chunk, "delta": paused_delta}
|
||||
return paused_chunk
|
||||
if delta.get("stop_reason") == "refusal" and self._claude_refusal_stop_details is not None:
|
||||
claude_refusal_delta: Final[MessageDelta] = {**delta, "stop_details": self._claude_refusal_stop_details}
|
||||
claude_refusal_chunk: Final[MessageBlockDelta] = {**processed_chunk, "delta": claude_refusal_delta}
|
||||
return claude_refusal_chunk
|
||||
if not self._refusal_text or delta.get("stop_reason") == "max_tokens":
|
||||
return processed_chunk
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
refusal_stop_details,
|
||||
|
|
@ -1470,9 +1479,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
@staticmethod
|
||||
def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
|
||||
from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
openai_chat_refusal_text,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
|
||||
|
||||
choice: Final = chunk.choices[0]
|
||||
if choice.finish_reason is not None:
|
||||
|
|
@ -1482,7 +1489,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
return False
|
||||
if getattr(delta, "content", None):
|
||||
return False
|
||||
if openai_chat_refusal_text(delta):
|
||||
if refusal_reply_text(delta):
|
||||
return False
|
||||
if getattr(delta, "reasoning_content", None):
|
||||
return False
|
||||
|
|
@ -1497,6 +1504,18 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
return False
|
||||
return True
|
||||
|
||||
def _record_anthropic_stop_fields(self, delta: object) -> None:
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
anthropic_refusal_stop_details,
|
||||
anthropic_stop_reason,
|
||||
)
|
||||
|
||||
if anthropic_stop_reason(delta) == "pause_turn":
|
||||
self._paused_turn = True
|
||||
claude_refusal_stop_details: Final = anthropic_refusal_stop_details(delta)
|
||||
if claude_refusal_stop_details is not None:
|
||||
self._claude_refusal_stop_details = claude_refusal_stop_details
|
||||
|
||||
def _should_start_new_content_block(self, chunk: "ModelResponseStream") -> bool:
|
||||
"""
|
||||
Determine if we should start a new content block based on the processed chunk.
|
||||
|
|
@ -1507,16 +1526,15 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
- Different content types in the response
|
||||
- Specific markers in the content
|
||||
"""
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
openai_chat_refusal_text,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
|
||||
|
||||
from .transformation import LiteLLMAnthropicMessagesAdapter
|
||||
|
||||
self._record_anthropic_stop_fields(chunk.choices[0].delta)
|
||||
if chunk.choices[0].finish_reason is not None:
|
||||
return False
|
||||
|
||||
refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta)
|
||||
refusal_text: Final = refusal_reply_text(chunk.choices[0].delta)
|
||||
if refusal_text is not None:
|
||||
self._refusal_text = self._refusal_text + refusal_text
|
||||
|
||||
|
|
|
|||
|
|
@ -150,7 +150,9 @@ from litellm.llms.anthropic.messages.native.mid_conversation_system import (
|
|||
is_system_role_message,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
openai_chat_refusal_text,
|
||||
anthropic_refusal_stop_details,
|
||||
anthropic_stop_reason,
|
||||
refusal_reply_text,
|
||||
refusal_stop_details,
|
||||
)
|
||||
from litellm.types.llms.anthropic import (
|
||||
|
|
@ -398,6 +400,10 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
|
||||
return None
|
||||
|
||||
def _streamed_tool_call_signature(self, tool_call_id: str, tool_call: object) -> str | None:
|
||||
id_parts: Final = tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)
|
||||
return id_parts[1] if len(id_parts) > 1 else self._extract_signature_from_tool_call(tool_call)
|
||||
|
||||
def _extract_signature_from_tool_use_content(self, content: Mapping[str, object]) -> str | None:
|
||||
"""
|
||||
Extract signature from a tool_use content block's provider_specific_fields.
|
||||
|
|
@ -1467,7 +1473,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
new_content.append(
|
||||
AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump()
|
||||
)
|
||||
if (refusal_text := openai_chat_refusal_text(choice.message)) is not None:
|
||||
if (refusal_text := refusal_reply_text(choice.message)) is not None:
|
||||
new_content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump())
|
||||
# Handle tool calls (in parallel to text content)
|
||||
if choice.message.tool_calls is not None and len(choice.message.tool_calls) > 0:
|
||||
|
|
@ -1634,7 +1640,15 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
tool_name_mapping=tool_name_mapping,
|
||||
)
|
||||
refusal_text: Final = next(
|
||||
(text for choice in response.choices if (text := openai_chat_refusal_text(choice.message)) is not None),
|
||||
(text for choice in response.choices if (text := refusal_reply_text(choice.message)) is not None),
|
||||
None,
|
||||
)
|
||||
claude_refusal_stop_details: Final = next(
|
||||
(
|
||||
details
|
||||
for choice in response.choices
|
||||
if (details := anthropic_refusal_stop_details(choice.message)) is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
|
@ -1643,8 +1657,10 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
|
||||
## extract finish reason
|
||||
openai_finish_reason: Final = response.choices[0].finish_reason if response.choices else "stop"
|
||||
translated_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic(
|
||||
openai_finish_reason=openai_finish_reason
|
||||
translated_finish_reason: Final = (
|
||||
"pause_turn"
|
||||
if response.choices and anthropic_stop_reason(response.choices[0].message) == "pause_turn"
|
||||
else self._translate_openai_finish_reason_to_anthropic(openai_finish_reason=openai_finish_reason)
|
||||
)
|
||||
anthropic_finish_reason: Final = (
|
||||
"compaction"
|
||||
|
|
@ -1686,7 +1702,11 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
usage=anthropic_usage,
|
||||
content=anthropic_content,
|
||||
stop_reason=anthropic_finish_reason,
|
||||
stop_details=(refusal_stop_details(refusal_text) if anthropic_finish_reason == "refusal" else None),
|
||||
stop_details=(
|
||||
(claude_refusal_stop_details or refusal_stop_details(refusal_text))
|
||||
if anthropic_finish_reason == "refusal"
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None
|
||||
|
|
@ -1712,10 +1732,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
):
|
||||
raw_id = choice.delta.tool_calls[0].id or str(uuid.uuid4())
|
||||
tool_name = choice.delta.tool_calls[0].function.name or ""
|
||||
thought_sig: str | None = None
|
||||
if THOUGHT_SIGNATURE_SEPARATOR in raw_id:
|
||||
parts = raw_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)
|
||||
thought_sig = parts[1] if len(parts) > 1 else None
|
||||
thought_sig = self._streamed_tool_call_signature(raw_id, choice.delta.tool_calls[0])
|
||||
tool_block: dict[str, object] = {
|
||||
"type": "tool_use",
|
||||
"id": normalize_anthropic_tool_use_id(raw_id),
|
||||
|
|
@ -1727,7 +1744,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
"signature": thought_sig,
|
||||
}
|
||||
return "tool_use", cast("ContentBlockContentBlockDict", tool_block)
|
||||
elif (choice.delta.content is not None and len(choice.delta.content) > 0) or openai_chat_refusal_text(
|
||||
elif (choice.delta.content is not None and len(choice.delta.content) > 0) or refusal_reply_text(
|
||||
choice.delta
|
||||
) is not None:
|
||||
return "text", TextBlock(type="text", text="")
|
||||
|
|
@ -1810,7 +1827,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
return "thinking_delta", ContentThinkingBlockDelta(type="thinking_delta", thinking=reasoning_content)
|
||||
else:
|
||||
refusal_text: Final = "".join(
|
||||
refusal for choice in choices if (refusal := openai_chat_refusal_text(choice.delta)) is not None
|
||||
refusal for choice in choices if (refusal := refusal_reply_text(choice.delta)) is not None
|
||||
)
|
||||
return "text_delta", ContentTextBlockDelta(type="text_delta", text=text + refusal_text)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from collections.abc import Iterable, Mapping, Sequence
|
|||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING, Final, cast, get_type_hints
|
||||
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError, with_config
|
||||
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
|
|
@ -162,6 +162,34 @@ def openai_chat_refusal_text(message_or_delta: object) -> str | None:
|
|||
return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "refusal")
|
||||
|
||||
|
||||
def anthropic_stop_reason(message_or_delta: object) -> str | None:
|
||||
return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "stop_reason")
|
||||
|
||||
|
||||
def refusal_reply_text(message_or_delta: object) -> str | None:
|
||||
if anthropic_stop_reason(message_or_delta) == "refusal":
|
||||
return None
|
||||
return openai_chat_refusal_text(message_or_delta)
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow"))
|
||||
class _AnthropicRefusalStopDetails(AnthropicStopDetails, total=False):
|
||||
pass
|
||||
|
||||
|
||||
_ANTHROPIC_REFUSAL_STOP_DETAILS: Final = TypeAdapter(_AnthropicRefusalStopDetails)
|
||||
|
||||
|
||||
def anthropic_refusal_stop_details(message_or_delta: object) -> AnthropicStopDetails | None:
|
||||
provider_specific_fields: Final = getattr(message_or_delta, "provider_specific_fields", None)
|
||||
if _mapping_str_field(provider_specific_fields, "stop_reason") != "refusal":
|
||||
return None
|
||||
try:
|
||||
return _ANTHROPIC_REFUSAL_STOP_DETAILS.validate_python(_mapping_field(provider_specific_fields, "stop_details"))
|
||||
except ValidationError:
|
||||
return refusal_stop_details(openai_chat_refusal_text(message_or_delta))
|
||||
|
||||
|
||||
def _responses_message_refusal_text(item: object) -> str | None:
|
||||
from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal
|
||||
|
||||
|
|
|
|||
|
|
@ -155,6 +155,8 @@ def collapse_pure_text_chunks(
|
|||
continue
|
||||
|
||||
json_str = ev[idx + len("data:") :].strip()
|
||||
if not json_str.startswith("{"):
|
||||
continue
|
||||
try:
|
||||
data = json.loads(json_str)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
|
|
@ -206,6 +208,11 @@ def collapse_pure_text_chunks(
|
|||
return out
|
||||
|
||||
|
||||
def _has_non_object_data(event_str: str) -> bool:
|
||||
data_start: Final = event_str.find("data:")
|
||||
return data_start != -1 and not event_str[data_start + len("data:") :].lstrip().startswith("{")
|
||||
|
||||
|
||||
def build_complete_streaming_response_legacy(
|
||||
all_chunks: Sequence[str | bytes],
|
||||
litellm_logging_obj: LiteLLMLoggingObj,
|
||||
|
|
@ -243,6 +250,8 @@ def build_complete_streaming_response_legacy(
|
|||
# text payload happens to contain "[DONE]" is not dropped.
|
||||
if any(line.strip() == "data: [DONE]" for line in event_str.split("\n")):
|
||||
continue
|
||||
if _has_non_object_data(event_str):
|
||||
continue
|
||||
transformed_openai_chunk = anthropic_model_response_iterator.convert_str_chunk_to_generic_chunk(
|
||||
chunk=event_str
|
||||
)
|
||||
|
|
@ -268,21 +277,25 @@ def build_complete_streaming_response_legacy(
|
|||
return complete_streaming_response
|
||||
|
||||
|
||||
def extract_sse_data(event_str: str) -> dict | None:
|
||||
"""Parse the JSON object from the ``data:`` line of an Anthropic SSE event."""
|
||||
def _sse_data_payload(event_str: str) -> str | None:
|
||||
for line in split_sse_lines(event_str):
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("data:"):
|
||||
payload = stripped[len("data:") :].strip()
|
||||
if not payload or payload == "[DONE]":
|
||||
return None
|
||||
try:
|
||||
return json.loads(payload)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
return stripped[len("data:") :].strip()
|
||||
return None
|
||||
|
||||
|
||||
def extract_sse_data(event_str: str) -> dict | None:
|
||||
"""Parse the JSON object from the ``data:`` line of an Anthropic SSE event."""
|
||||
payload: Final = _sse_data_payload(event_str)
|
||||
if payload is None or not payload.startswith("{"):
|
||||
return None
|
||||
try:
|
||||
return json.loads(payload)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def build_usage_only_response_from_chunks(
|
||||
all_chunks: Sequence[str | bytes],
|
||||
model: str,
|
||||
|
|
@ -415,27 +428,81 @@ _ANTHROPIC_EVENT_TYPES: Final = frozenset(
|
|||
)
|
||||
|
||||
|
||||
_ERROR_PAYLOAD_KEYS: Final = frozenset({"type", "error", "request_id"})
|
||||
|
||||
|
||||
def _is_anthropic_event_type(event_type: object) -> bool:
|
||||
return isinstance(event_type, str) and event_type in _ANTHROPIC_EVENT_TYPES
|
||||
|
||||
|
||||
def is_raw_sse_stream(all_chunks: Sequence[object]) -> bool:
|
||||
return any(isinstance(chunk, (str, bytes)) for chunk in all_chunks)
|
||||
|
||||
|
||||
def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None:
|
||||
raw: Final = b"".join(
|
||||
def _joined_sse_bytes(all_chunks: Sequence[object]) -> bytes:
|
||||
return b"".join(
|
||||
chunk if isinstance(chunk, bytes) else chunk.encode("utf-8")
|
||||
for chunk in all_chunks
|
||||
if isinstance(chunk, (str, bytes))
|
||||
)
|
||||
|
||||
|
||||
def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None:
|
||||
try:
|
||||
return codecs.getincrementaldecoder("utf-8")().decode(raw, final=False)
|
||||
return codecs.getincrementaldecoder("utf-8")().decode(_joined_sse_bytes(all_chunks), final=False)
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def _sse_event_data(event: str) -> Mapping[str, object] | None:
|
||||
return as_str_mapping(extract_sse_data(event))
|
||||
|
||||
|
||||
_JSON_ESCAPED_LINE_BREAKS: Final = MappingProxyType(
|
||||
str.maketrans({"\x85": "\\u0085", "\u2028": "\\u2028", "\u2029": "\\u2029"})
|
||||
)
|
||||
|
||||
|
||||
def _sse_frames(sse_stream: str) -> Sequence[str]:
|
||||
return "\n".join(split_sse_lines(sse_stream)).split("\n\n")
|
||||
|
||||
|
||||
def _frames_of_the_widest_reader(sse_stream: str) -> Sequence[str]:
|
||||
"""The stream's frames as httpx's ``str.splitlines`` reads them, past a form feed or U+2028 in any line."""
|
||||
return "\n".join(sse_stream.splitlines()).split("\n\n")
|
||||
|
||||
|
||||
def _sse_field(line: str) -> tuple[str, str]:
|
||||
name, _, value = line.partition(":")
|
||||
return name, value.removeprefix(" ")
|
||||
|
||||
|
||||
def _sse_fields(frame: str) -> tuple[tuple[str, str], ...]:
|
||||
"""The frame's fields as a client reads them: a name matches exactly and a comment is no field."""
|
||||
return tuple(_sse_field(line) for line in frame.split("\n") if line and not line.startswith(":"))
|
||||
|
||||
|
||||
def _carries_no_data(fields: Sequence[tuple[str, str]]) -> bool:
|
||||
match fields:
|
||||
case () | (("event", _),) | (("data", "[DONE]"),):
|
||||
return True
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def _json_mapping(payload: str) -> Mapping[str, object] | None:
|
||||
try:
|
||||
value: Final[object] = json.loads(payload)
|
||||
except ValueError:
|
||||
return None
|
||||
return as_str_mapping(value)
|
||||
|
||||
|
||||
def _parsed_sse_events(sse_stream: str) -> tuple[Mapping[str, object], ...]:
|
||||
return tuple(
|
||||
event_data
|
||||
for event in split_sse_chunk_into_events(sse_stream)
|
||||
if (event_data := extract_sse_data(event)) is not None
|
||||
if (event_data := _sse_event_data(event)) is not None
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -444,7 +511,8 @@ def _anthropic_message_start(sse_stream: str) -> Mapping[str, object] | None:
|
|||
(
|
||||
message
|
||||
for event_data in _parsed_sse_events(sse_stream)
|
||||
if event_data.get("type") == "message_start" and isinstance(message := event_data.get("message"), dict)
|
||||
if event_data.get("type") == "message_start"
|
||||
and (message := as_str_mapping(event_data.get("message"))) is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
|
@ -482,11 +550,151 @@ def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool:
|
|||
sse_stream: Final = _joined_sse_stream(all_chunks)
|
||||
if sse_stream is None:
|
||||
return False
|
||||
return any(event.get("type") in _ANTHROPIC_EVENT_TYPES for event in _parsed_sse_events(sse_stream))
|
||||
return any(_is_anthropic_event_type(event.get("type")) for event in _parsed_sse_events(sse_stream))
|
||||
|
||||
|
||||
def _is_json_no_reader_can_take(payload: str) -> bool:
|
||||
try:
|
||||
value: Final[object] = json.loads(payload)
|
||||
except ValueError:
|
||||
return False
|
||||
if value is None:
|
||||
return False
|
||||
event: Final = as_str_mapping(value)
|
||||
return event is None or isinstance(event.get("type"), (list, dict))
|
||||
|
||||
|
||||
def has_unreadable_json_event(all_chunks: Sequence[object]) -> bool:
|
||||
"""Whether a frame's data is JSON that no reader of these streams can take as an event."""
|
||||
sse_stream: Final = codecs.getincrementaldecoder("utf-8")(errors="replace").decode(
|
||||
_joined_sse_bytes(all_chunks), final=False
|
||||
)
|
||||
return any(_is_json_no_reader_can_take(_sse_data_payload(event) or "") for event in _sse_frames(sse_stream))
|
||||
|
||||
|
||||
def anthropic_sse_event_type(event: str) -> object:
|
||||
return next((event_data.get("type") for event_data in _parsed_sse_events(event)), None)
|
||||
|
||||
|
||||
def anthropic_sse_event_types(all_chunks: Sequence[object]) -> frozenset[str]:
|
||||
"""The string ``type`` of every event the stream sent."""
|
||||
event_types: Final = (event.get("type") for event in _parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
|
||||
return frozenset(event_type for event_type in event_types if isinstance(event_type, str))
|
||||
|
||||
|
||||
def _is_named_error_event(event: str) -> bool:
|
||||
names: Final = tuple(value for name, value in _sse_fields(event) if name == "event")
|
||||
return names[-1:] == ("error",)
|
||||
|
||||
|
||||
def _is_error_event(event: str) -> bool:
|
||||
event_type: Final = anthropic_sse_event_type(event)
|
||||
return event_type == "error" or (_is_named_error_event(event) and not _is_anthropic_event_type(event_type))
|
||||
|
||||
|
||||
def _frames_a_client_dispatches(sse_stream: str) -> Sequence[str]:
|
||||
return "\n".join((*split_sse_lines(sse_stream), "")).split("\n\n")[:-1]
|
||||
|
||||
|
||||
def _error_events(sse_stream: str) -> tuple[str, ...]:
|
||||
return tuple(event for event in _frames_a_client_dispatches(sse_stream) if _is_error_event(event))
|
||||
|
||||
|
||||
def _plain_error_event(event: str) -> str:
|
||||
payload: Final = _sse_data_payload(event) or ""
|
||||
one_line: Final = payload.translate(_JSON_ESCAPED_LINE_BREAKS) if _json_mapping(payload) is not None else payload
|
||||
return "\n".join(("event: error", *(f"data: {line}" for line in one_line.splitlines()[:1])))
|
||||
|
||||
|
||||
def anthropic_sse_error_events(all_chunks: Sequence[object]) -> tuple[str, ...]:
|
||||
"""The ``error`` events of the stream, each as an ``event:`` line and the ``data:`` line assembly read.
|
||||
|
||||
A stream whose upstream fails after it started ends with an ``error`` event in place of
|
||||
``message_stop``. That frame is the outcome the client is owed, not content, so assembly through
|
||||
the upstream error leaves it out and a caller re-emitting a rewritten response puts it back,
|
||||
without any other line of the frame, since that is content no guardrail read.
|
||||
A frame counts by its data ``type``, or by its SSE ``event:`` name unless its data is another
|
||||
Anthropic event, since a client SDK raises on the name alone and an upstream that is not
|
||||
Anthropic may send anything as the data.
|
||||
"""
|
||||
return tuple(_plain_error_event(event) for event in _error_events(_joined_sse_stream(all_chunks) or ""))
|
||||
|
||||
|
||||
def anthropic_sse_stopped_block_index(event: str) -> object:
|
||||
return next(
|
||||
(
|
||||
event_data.get("index")
|
||||
for event_data in _parsed_sse_events(event)
|
||||
if event_data.get("type") == "content_block_stop"
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _block_index(event_data: Mapping[str, object]) -> int | None:
|
||||
index: Final = event_data.get("index")
|
||||
return index if isinstance(index, int) else None
|
||||
|
||||
|
||||
def _started_blocks(events: Sequence[Mapping[str, object]]) -> Mapping[int, Mapping[str, object]]:
|
||||
starts: Final = (event for event in events if event.get("type") == "content_block_start")
|
||||
return MappingProxyType(
|
||||
{
|
||||
index: block
|
||||
for event in starts
|
||||
if (index := _block_index(event)) is not None
|
||||
and (block := as_str_mapping(event.get("content_block"))) is not None
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def anthropic_sse_started_blocks(all_chunks: Sequence[object]) -> Mapping[int, Mapping[str, object]]:
|
||||
"""The ``content_block`` each ``content_block_start`` of the stream opened, by block index."""
|
||||
return _started_blocks(_parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
|
||||
|
||||
|
||||
def _unfinished_blocks(events: Sequence[Mapping[str, object]]) -> Mapping[int, Mapping[str, object]]:
|
||||
stopped: Final = frozenset(_block_index(event) for event in events if event.get("type") == "content_block_stop")
|
||||
return MappingProxyType({index: block for index, block in _started_blocks(events).items() if index not in stopped})
|
||||
|
||||
|
||||
def anthropic_sse_unfinished_blocks(all_chunks: Sequence[object]) -> Mapping[int, Mapping[str, object]]:
|
||||
"""The blocks the stream opened and never stopped, which only a failed upstream leaves."""
|
||||
return _unfinished_blocks(_parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
|
||||
|
||||
|
||||
def _sse_events_with_data(sse_stream: str) -> tuple[tuple[str, Mapping[str, object]], ...]:
|
||||
return tuple(
|
||||
(event, event_data)
|
||||
for event in split_sse_chunk_into_events(sse_stream)
|
||||
if (event_data := _sse_event_data(event)) is not None
|
||||
)
|
||||
|
||||
|
||||
def _content_events_before_upstream_error(sse_stream: str) -> tuple[str, ...]:
|
||||
"""The stream's events minus its upstream ``error`` and every cut-off block assembly cannot read.
|
||||
|
||||
A cut-off text block is still text the client was sent, so it stays. A cut-off tool call's
|
||||
input is a JSON prefix, and a cut-off thinking block has no signature, so neither is a block the
|
||||
client can act on, and both are left out.
|
||||
"""
|
||||
if not _error_events(sse_stream):
|
||||
return (sse_stream,)
|
||||
events: Final = _sse_events_with_data(sse_stream)
|
||||
cut_off: Final = frozenset(
|
||||
index
|
||||
for index, block in _unfinished_blocks(tuple(event_data for _, event_data in events)).items()
|
||||
if block.get("type") != "text"
|
||||
)
|
||||
return tuple(
|
||||
event
|
||||
for event, event_data in events
|
||||
if event_data.get("type") != "error" and _block_index(event_data) not in cut_off
|
||||
)
|
||||
|
||||
|
||||
def assemble_anthropic_sse_stream(
|
||||
all_chunks: Sequence[object], *, restore_identity: bool = False
|
||||
all_chunks: Sequence[object], *, restore_identity: bool = False, through_upstream_error: bool = False
|
||||
) -> ModelResponse | None:
|
||||
"""Assemble raw Anthropic SSE frames into a ModelResponse.
|
||||
|
||||
|
|
@ -494,6 +702,11 @@ def assemble_anthropic_sse_stream(
|
|||
assembler does not carry through. It is off by default so callers that re-emit the assembled
|
||||
response keep the wire shape they had before this helper was shared. The writes land on a
|
||||
freshly built object that is unreachable from caller state until returned.
|
||||
|
||||
A stream whose upstream failed after it started ends with an ``error`` event in place of
|
||||
``message_stop`` and assembles to None, unless ``through_upstream_error`` asks for the content
|
||||
the client was sent before the failure. That is for a guardrail that scans the stream and then
|
||||
passes the upstream error on; a caller recording what was served keeps None.
|
||||
"""
|
||||
sse_stream: Final = _joined_sse_stream(all_chunks)
|
||||
if sse_stream is None:
|
||||
|
|
@ -502,9 +715,12 @@ def assemble_anthropic_sse_stream(
|
|||
if message_start is None:
|
||||
return None
|
||||
model: Final = message_start.get("model") if restore_identity else None
|
||||
content_events: Final = (
|
||||
_content_events_before_upstream_error(sse_stream) if through_upstream_error else (sse_stream,)
|
||||
)
|
||||
try:
|
||||
assembled: Final = build_complete_streaming_response(
|
||||
all_chunks=(sse_stream,),
|
||||
all_chunks=content_events,
|
||||
litellm_logging_obj=None, # pyright: ignore[reportArgumentType] # only forwarded to stream_chunk_builder, which accepts None
|
||||
model=model if isinstance(model, str) else "",
|
||||
)
|
||||
|
|
@ -522,6 +738,21 @@ def assemble_anthropic_sse_stream(
|
|||
return assembled
|
||||
|
||||
|
||||
def _is_error_payload(payload: str) -> bool:
|
||||
event: Final = _json_mapping(payload)
|
||||
if event is None or not _ERROR_PAYLOAD_KEYS.issuperset(event):
|
||||
return False
|
||||
return event.get("type") == "error" or ("type" not in event and isinstance(event.get("error"), Mapping))
|
||||
|
||||
|
||||
def _is_error_frame(fields: Sequence[tuple[str, str]]) -> bool:
|
||||
match fields:
|
||||
case (("data", payload),) | (("event", "error"), ("data", payload)):
|
||||
return _is_error_payload(payload)
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def is_sse_error_stream(all_chunks: Sequence[object]) -> bool:
|
||||
"""Whether the buffered stream carries nothing but error frames.
|
||||
|
||||
|
|
@ -537,7 +768,6 @@ def is_sse_error_stream(all_chunks: Sequence[object]) -> bool:
|
|||
sse_stream: Final = _joined_sse_stream(all_chunks)
|
||||
if sse_stream is None:
|
||||
return False
|
||||
events: Final = _parsed_sse_events(sse_stream)
|
||||
return len(events) > 0 and all(
|
||||
event.get("type") == "error" or isinstance(event.get("error"), Mapping) for event in events
|
||||
)
|
||||
frames: Final = tuple(_sse_fields(frame) for frame in _frames_of_the_widest_reader(sse_stream))
|
||||
with_data: Final = tuple(fields for fields in frames if not _carries_no_data(fields))
|
||||
return len(with_data) > 0 and all(_is_error_frame(fields) for fields in with_data)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from pydantic_core import ErrorDetails
|
|||
from typing_extensions import assert_never
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
|
@ -297,13 +298,7 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]:
|
|||
assert_never(question)
|
||||
|
||||
|
||||
def ir_to_systemone_request(
|
||||
model: str, request: DecisionsIRRequest, *, supports_audio: bool = False, supports_video: bool = False
|
||||
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
|
||||
if request.audio and not supports_audio:
|
||||
return UnsupportedDecisionsRequest(reason="audio input is not supported by this Decisions model")
|
||||
if request.videos and not supports_video:
|
||||
return UnsupportedDecisionsRequest(reason="video input is not supported by this Decisions model")
|
||||
def ir_to_systemone_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object]:
|
||||
state: Final = _systemone_state(request.input)
|
||||
keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True)
|
||||
images: Final = input_images(request.input)
|
||||
|
|
@ -503,6 +498,16 @@ def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIR
|
|||
)
|
||||
|
||||
|
||||
def catalog_mode(model: str, custom_llm_provider: str) -> str | None:
|
||||
try:
|
||||
key: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)["key"]
|
||||
except Exception: # noqa: BLE001 # get_model_info raises bare Exception for an unmapped model, which the catalog does not list
|
||||
return None
|
||||
entry: Final = GetModelCostMap.loaded_model_cost_map().get(key)
|
||||
mode: Final = entry.get("mode") if entry is not None else None
|
||||
return mode if isinstance(mode, str) else None
|
||||
|
||||
|
||||
def _is_decisions_model(model: str, custom_llm_provider: str) -> bool:
|
||||
try:
|
||||
model_mode: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider).get("mode")
|
||||
|
|
@ -640,12 +645,16 @@ class BaseDecisionsConfig(ABC):
|
|||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
|
||||
if deployment_flag is not None:
|
||||
return deployment_flag
|
||||
return _is_decisions_model(model, custom_llm_provider) and litellm.supports_audio_input(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
|
||||
if deployment_flag is not None:
|
||||
return deployment_flag
|
||||
return _is_decisions_model(model, custom_llm_provider) and litellm.supports_video_input(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
|
@ -656,12 +665,7 @@ class BaseDecisionsConfig(ABC):
|
|||
request: DecisionsIRRequest,
|
||||
custom_llm_provider: str,
|
||||
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
|
||||
return ir_to_systemone_request(
|
||||
self.request_model(model),
|
||||
request,
|
||||
supports_audio=self.supports_audio_input(model, custom_llm_provider),
|
||||
supports_video=self.supports_video_input(model, custom_llm_provider),
|
||||
)
|
||||
return ir_to_systemone_request(self.request_model(model), request)
|
||||
|
||||
def unwrap_response(self, payload: object) -> object:
|
||||
return payload
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
as_str_mapping,
|
||||
drops_unsupported_params,
|
||||
filter_exceptions_from_params,
|
||||
filter_internal_params,
|
||||
map_finish_reason,
|
||||
|
|
@ -2129,6 +2130,24 @@ class AmazonConverseConfig(BaseConfig):
|
|||
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _claude_messages_within_cache_control_cap(
|
||||
messages: Sequence[AllMessageValues],
|
||||
capability_model: str,
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> Sequence[AllMessageValues]:
|
||||
if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower():
|
||||
return messages
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
messages_within_cache_control_cap,
|
||||
tool_message_with_single_cache_mark,
|
||||
)
|
||||
|
||||
return messages_within_cache_control_cap(
|
||||
tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools")
|
||||
)
|
||||
|
||||
async def _async_transform_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -2140,7 +2159,13 @@ class AmazonConverseConfig(BaseConfig):
|
|||
base_model: Final = litellm_params.get("base_model")
|
||||
capability_model: Final = bedrock_capability_model(model, base_model)
|
||||
messages, system_content_blocks = self._transform_system_message(
|
||||
messages, model=capability_model, deployment_model=model
|
||||
list(
|
||||
self._claude_messages_within_cache_control_cap(
|
||||
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
|
||||
)
|
||||
),
|
||||
model=capability_model,
|
||||
deployment_model=model,
|
||||
)
|
||||
|
||||
# Convert last user message to guarded_text if guardrailConfig is present
|
||||
|
|
@ -2221,7 +2246,13 @@ class AmazonConverseConfig(BaseConfig):
|
|||
base_model: Final = litellm_params.get("base_model")
|
||||
capability_model: Final = bedrock_capability_model(model, base_model)
|
||||
messages, system_content_blocks = self._transform_system_message(
|
||||
messages, model=capability_model, deployment_model=model
|
||||
list(
|
||||
self._claude_messages_within_cache_control_cap(
|
||||
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
|
||||
)
|
||||
),
|
||||
model=capability_model,
|
||||
deployment_model=model,
|
||||
)
|
||||
|
||||
# Convert last user message to guarded_text if guardrailConfig is present
|
||||
|
|
|
|||
|
|
@ -131,6 +131,8 @@ from litellm.responses.streaming_iterator import (
|
|||
ResponsesAPIStreamingIterator,
|
||||
ResponsesWebSocketStreaming,
|
||||
SyncResponsesAPIStreamingIterator,
|
||||
TokenLimitCallback,
|
||||
with_token_limit_checks,
|
||||
)
|
||||
from litellm.types.containers.main import (
|
||||
ContainerFileListResponse,
|
||||
|
|
@ -598,10 +600,10 @@ def _decoded_body_headers(response: httpx.Response) -> httpx.Headers:
|
|||
)
|
||||
|
||||
|
||||
def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
|
||||
"""Duck-type discover proxy hooks exposing per-frame project ITPM/OTPM
|
||||
enforcement, so the Responses WebSocket loop can charge every
|
||||
``response.create`` frame, not just the connection's first one.
|
||||
def _ws_callbacks_exposing(method_name: str) -> tuple[object, ...]:
|
||||
"""Duck-type discover proxy hooks exposing a per-frame check, so the
|
||||
Responses WebSocket loop can run it on every ``response.create`` frame,
|
||||
not just the connection's first one.
|
||||
|
||||
Uses duck-typing on ``litellm.callbacks`` (rather than importing the
|
||||
proxy hook directly) to avoid a layering violation (SDK importing from
|
||||
|
|
@ -612,13 +614,28 @@ def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
|
|||
callbacks: Final = cast( # cast-ok: callback registry is inspected before protocol use
|
||||
Sequence[object], _litellm.callbacks
|
||||
)
|
||||
return tuple(callback for callback in callbacks if callable(getattr(callback, method_name, None)))
|
||||
|
||||
|
||||
def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
|
||||
return tuple(
|
||||
cast(ProjectQuotaCallback, callback) # cast-ok: required callback method is callable
|
||||
for callback in callbacks
|
||||
if callable(getattr(callback, "enforce_project_io_token_quota_for_frame", None))
|
||||
for callback in _ws_callbacks_exposing("enforce_project_io_token_quota_for_frame")
|
||||
)
|
||||
|
||||
|
||||
def _collect_ws_token_limit_callbacks() -> tuple[TokenLimitCallback, ...]:
|
||||
return tuple(
|
||||
cast(TokenLimitCallback, callback) # cast-ok: required callback method is callable
|
||||
for callback in _ws_callbacks_exposing("enforce_token_limits_for_frame")
|
||||
)
|
||||
|
||||
|
||||
def _ws_model_group(litellm_metadata: Mapping[str, object] | None, model: str) -> str:
|
||||
model_group: Final = (litellm_metadata or {}).get("model_group")
|
||||
return model_group if isinstance(model_group, str) and model_group else model
|
||||
|
||||
|
||||
def _unwrap_provider_error(error: Exception) -> Exception:
|
||||
if not isinstance(error, httpx.TransportError):
|
||||
return error
|
||||
|
|
@ -7285,6 +7302,12 @@ class BaseLLMHTTPHandler:
|
|||
- Forwards events over the websocket connection
|
||||
"""
|
||||
_ws_quota_callbacks: Final = _collect_ws_project_quota_callbacks()
|
||||
_ws_turn_admission: Final = with_token_limit_checks(
|
||||
turn_admission,
|
||||
_collect_ws_token_limit_callbacks(),
|
||||
user_api_key_dict,
|
||||
_ws_model_group(litellm_metadata, model),
|
||||
)
|
||||
|
||||
_ws_guardrail_callbacks: list = []
|
||||
_ws_output_guardrail_callbacks: list = []
|
||||
|
|
@ -7332,7 +7355,7 @@ class BaseLLMHTTPHandler:
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
first_message=first_message,
|
||||
quota_callbacks=_ws_quota_callbacks,
|
||||
turn_admission=turn_admission,
|
||||
turn_admission=_ws_turn_admission,
|
||||
guardrail_callbacks=_ws_guardrail_callbacks, # pyright: ignore[reportUnknownArgumentType] # duck-typed callbacks from litellm.callbacks
|
||||
frame_guard=frame_guard,
|
||||
**kwargs,
|
||||
|
|
@ -7429,7 +7452,7 @@ class BaseLLMHTTPHandler:
|
|||
authorized_model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
request_defaults=request_defaults,
|
||||
turn_admission=turn_admission,
|
||||
turn_admission=_ws_turn_admission,
|
||||
frame_guard=frame_guard,
|
||||
)
|
||||
return await streaming.bidirectional_forward()
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
|
|||
aembedding=None,
|
||||
custom_endpoint: bool | None = None,
|
||||
headers: dict | None = None,
|
||||
drop_params: bool = False,
|
||||
) -> EmbeddingResponse:
|
||||
# Check for custom user agent in optional_params or environment
|
||||
# This allows partners building on LiteLLM to set their own telemetry
|
||||
|
|
@ -58,4 +59,5 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
|
|||
aembedding=aembedding,
|
||||
custom_endpoint=True,
|
||||
headers=headers,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -130,7 +130,8 @@ class DeepInfraConfig(OpenAIGPTConfig):
|
|||
- Input: {"role": "tool", "content": [{"type": "text", "text": "result"}]}
|
||||
- Output: {"role": "tool", "content": "[{\"type\": \"text\", \"text\": \"result\"}]"}
|
||||
"""
|
||||
for message in messages:
|
||||
copied_messages: Final = [message.copy() for message in messages]
|
||||
for message in copied_messages:
|
||||
if message.get("role") == "tool":
|
||||
content = message.get("content")
|
||||
|
||||
|
|
@ -149,7 +150,7 @@ class DeepInfraConfig(OpenAIGPTConfig):
|
|||
# For complex content, serialize the entire array as JSON string
|
||||
message["content"] = json.dumps(content)
|
||||
|
||||
return messages
|
||||
return copied_messages
|
||||
|
||||
@overload
|
||||
def _transform_messages(
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm.secret_managers.main import get_secret_str
|
|||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionToolParam,
|
||||
ChatCompletionToolParamFunctionChunk,
|
||||
OpenAIChatCompletionToolParam,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
|
|
@ -45,6 +46,7 @@ from ...openai.chat.gpt_transformation import (
|
|||
OpenAIChatCompletionStreamingHandler,
|
||||
OpenAIGPTConfig,
|
||||
)
|
||||
from ...openai.common_utils import without_cache_control
|
||||
from ..common_utils import (
|
||||
FIREROUTER,
|
||||
FireworksAIException,
|
||||
|
|
@ -480,15 +482,23 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
return (("response_format", _json_schema_response_format(choice_schema, "choice")),)
|
||||
|
||||
def _transform_tools(self, tools: list[OpenAIChatCompletionToolParam]) -> list[OpenAIChatCompletionToolParam]:
|
||||
for tool in tools:
|
||||
if tool.get("type") != "function":
|
||||
continue
|
||||
function = tool["function"]
|
||||
function.pop("strict", None)
|
||||
params = function.get("parameters")
|
||||
if isinstance(params, dict):
|
||||
unpack_legacy_defs(params)
|
||||
return tools
|
||||
return [self._fireworks_tool(tool) for tool in tools]
|
||||
|
||||
@staticmethod
|
||||
def _fireworks_tool(tool: OpenAIChatCompletionToolParam) -> OpenAIChatCompletionToolParam:
|
||||
if tool.get("type") != "function":
|
||||
return tool
|
||||
function: Final = tool["function"]
|
||||
params: Final = function.get("parameters")
|
||||
unstrict_function: Final = cast( # cast-ok: the same function definition minus `strict`
|
||||
ChatCompletionToolParamFunctionChunk, {key: value for key, value in function.items() if key != "strict"}
|
||||
)
|
||||
new_function: Final[ChatCompletionToolParamFunctionChunk] = (
|
||||
{**unstrict_function, "parameters": unpack_legacy_defs(params, copy=True)}
|
||||
if isinstance(params, dict)
|
||||
else unstrict_function
|
||||
)
|
||||
return {**tool, "function": new_function}
|
||||
|
||||
def _transform_messages_helper(
|
||||
self, messages: list[AllMessageValues], model: str, litellm_params: dict
|
||||
|
|
@ -496,10 +506,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
"""
|
||||
Strip fields not permitted by FireworksAI from messages.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
filter_value_from_dict,
|
||||
)
|
||||
|
||||
supports_vision_value: Final = self._get_model_cost_capability_exact(model=model, capability="supports_vision")
|
||||
for message in messages:
|
||||
if message["role"] == "user":
|
||||
|
|
@ -530,9 +536,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
model=model,
|
||||
llm_provider="fireworks_ai",
|
||||
)
|
||||
filter_value_from_dict(cast(dict, message), "cache_control")
|
||||
|
||||
return [strip_litellm_response_only_fields(message) for message in messages]
|
||||
return [
|
||||
strip_litellm_response_only_fields(cast(AllMessageValues, without_cache_control(message)))
|
||||
for message in messages
|
||||
]
|
||||
|
||||
# Cached index of fireworks_ai/* entries from litellm.model_cost. Building
|
||||
# this index requires a full scan of model_cost (tens of thousands of
|
||||
|
|
|
|||
|
|
@ -140,7 +140,8 @@ class GroqChatConfig(OpenAILikeChatConfig):
|
|||
def _transform_messages(
|
||||
self, messages: list[AllMessageValues], model: str, is_async: bool = False
|
||||
) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]:
|
||||
for idx, message in enumerate(messages):
|
||||
transformed_messages: Final = list(messages)
|
||||
for idx, message in enumerate(transformed_messages):
|
||||
"""
|
||||
1. Don't pass 'null' function_call assistant message to groq - https://github.com/BerriAI/litellm/issues/5839
|
||||
"""
|
||||
|
|
@ -154,12 +155,12 @@ class GroqChatConfig(OpenAILikeChatConfig):
|
|||
for k, v in _message.items():
|
||||
if v is not None:
|
||||
new_message[k] = v
|
||||
messages[idx] = strip_litellm_internal_message_fields(new_message)
|
||||
transformed_messages[idx] = strip_litellm_internal_message_fields(new_message)
|
||||
|
||||
if is_async:
|
||||
return super()._transform_messages(messages=messages, model=model, is_async=True)
|
||||
return super()._transform_messages(messages=transformed_messages, model=model, is_async=True)
|
||||
else:
|
||||
return super()._transform_messages(messages=messages, model=model, is_async=False)
|
||||
return super()._transform_messages(messages=transformed_messages, model=model, is_async=False)
|
||||
|
||||
def transform_messages(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Coroutine
|
||||
from typing import Final, Literal, cast, overload
|
||||
|
|
@ -88,7 +89,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
|
|||
) -> dict:
|
||||
_tools = non_default_params.pop("tools", None)
|
||||
if _tools is not None:
|
||||
_tools = remove_additional_properties(_tools)
|
||||
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
|
||||
_tools = remove_strict_from_schema(_tools)
|
||||
if isinstance(_tools, list):
|
||||
_tools = self._convert_custom_tools_to_function_tools(_tools)
|
||||
|
|
|
|||
|
|
@ -755,7 +755,8 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
if _name is not None:
|
||||
# Remove name if not a tool message
|
||||
if message["role"] != "tool" or isinstance(_name, str) and len(_name.strip()) == 0:
|
||||
message.pop("name", None)
|
||||
message_without_name: Final = {k: v for k, v in message.items() if k != "name"}
|
||||
return cast(AllMessageValues, message_without_name) # cast-ok: the same message minus name
|
||||
|
||||
return message
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
|
||||
import litellm
|
||||
from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS
|
||||
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
extract_reasoning_content,
|
||||
handle_invalid_parallel_tool_calls,
|
||||
|
|
@ -64,7 +64,7 @@ from litellm.types.utils import (
|
|||
)
|
||||
from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support
|
||||
|
||||
from ..common_utils import OpenAIError
|
||||
from ..common_utils import OpenAIError, without_cache_control
|
||||
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -128,6 +128,29 @@ def _litellm_params_str(litellm_params: Mapping[str, object] | None, key: str) -
|
|||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def _drops_params(litellm_params: Mapping[str, object]) -> bool:
|
||||
return drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
|
||||
|
||||
def _names_claude(model: str, litellm_params: Mapping[str, object]) -> bool:
|
||||
base_model: Final = litellm_params.get("base_model")
|
||||
return "claude" in model.lower() or (isinstance(base_model, str) and "claude" in base_model.lower())
|
||||
|
||||
|
||||
def _claude_messages_within_cache_control_cap(
|
||||
model: str,
|
||||
messages: Sequence[AllMessageValues],
|
||||
tools: Sequence["ChatCompletionToolParam"] | None,
|
||||
litellm_params: Mapping[str, object],
|
||||
request_params: Mapping[str, object],
|
||||
) -> Sequence[AllMessageValues]:
|
||||
if not _names_claude(model, litellm_params) or not _drops_params(litellm_params):
|
||||
return messages
|
||||
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
|
||||
|
||||
return list(messages_within_cache_control_cap(messages, tools, request_params.get("cache_control"), request_params))
|
||||
|
||||
|
||||
def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | None:
|
||||
try:
|
||||
extra_body: Final = _OBJECT_MAPPING.validate_python(optional_params.get("extra_body"))
|
||||
|
|
@ -311,52 +334,58 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
return content_copy
|
||||
|
||||
async def _async_handle_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
|
||||
file_id: Final = content_item.get("file_id")
|
||||
content_copy: Final = content_item.copy()
|
||||
file_id: Final = content_copy.get("file_id")
|
||||
if file_id is not None: # check for file id being url done in _handle_pdf_url
|
||||
base64_data: Final = await async_convert_url_to_base64(file_id)
|
||||
content_item["file_data"] = base64_data
|
||||
content_item["filename"] = "my_file.pdf"
|
||||
content_item.pop("file_id")
|
||||
return content_item
|
||||
content_copy["file_data"] = base64_data
|
||||
content_copy["filename"] = "my_file.pdf"
|
||||
content_copy.pop("file_id")
|
||||
return content_copy
|
||||
|
||||
def _common_file_data_check(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
|
||||
file_data: Final = content_item.get("file_data")
|
||||
filename: Final = content_item.get("filename")
|
||||
if file_data is not None and filename is None:
|
||||
content_item["filename"] = "my_file.pdf"
|
||||
return {**content_item, "filename": "my_file.pdf"}
|
||||
return content_item
|
||||
|
||||
@staticmethod
|
||||
def _image_item_for_openai(content_item: ChatCompletionImageObject) -> ChatCompletionImageObject:
|
||||
image_url: Final = content_item["image_url"]
|
||||
if isinstance(image_url, str):
|
||||
return {**content_item, "image_url": {"url": image_url}}
|
||||
if not isinstance(image_url, dict):
|
||||
return content_item
|
||||
new_image_url_obj: Final = cast(
|
||||
ChatCompletionImageUrlObject,
|
||||
{k: v for k, v in image_url.items() if k not in {"format", "video_metadata"}},
|
||||
)
|
||||
return {**content_item, "image_url": new_image_url_obj}
|
||||
|
||||
@staticmethod
|
||||
def _file_item_for_openai(content_item: ChatCompletionFileObject) -> ChatCompletionFileObject:
|
||||
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
|
||||
file_obj: Final = content_item.get("file")
|
||||
if file_obj is None:
|
||||
raise litellm.BadRequestError(
|
||||
message="Content block has type='file' but is missing the required 'file' field",
|
||||
model=None,
|
||||
llm_provider="openai",
|
||||
)
|
||||
new_file_obj: Final = ChatCompletionFileObjectFile(
|
||||
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
|
||||
)
|
||||
return {**content_item, "file": new_file_obj}
|
||||
|
||||
def _apply_common_transform_content_item(
|
||||
self,
|
||||
content_item: OpenAIMessageContentListBlock,
|
||||
) -> OpenAIMessageContentListBlock:
|
||||
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
|
||||
if content_item.get("type") == "image_url":
|
||||
content_item = cast(ChatCompletionImageObject, content_item)
|
||||
if isinstance(content_item["image_url"], str):
|
||||
content_item["image_url"] = {
|
||||
"url": content_item["image_url"],
|
||||
}
|
||||
elif isinstance(content_item["image_url"], dict):
|
||||
new_image_url_obj: Final = cast(
|
||||
ChatCompletionImageUrlObject,
|
||||
{k: v for k, v in content_item["image_url"].items() if k not in {"format", "video_metadata"}},
|
||||
)
|
||||
content_item["image_url"] = new_image_url_obj
|
||||
elif content_item.get("type") == "file":
|
||||
content_item = cast(ChatCompletionFileObject, content_item)
|
||||
file_obj: Final = content_item.get("file")
|
||||
if file_obj is None:
|
||||
raise litellm.BadRequestError(
|
||||
message="Content block has type='file' but is missing the required 'file' field",
|
||||
model=None,
|
||||
llm_provider="openai",
|
||||
)
|
||||
new_file_obj: Final = ChatCompletionFileObjectFile(
|
||||
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
|
||||
)
|
||||
content_item["file"] = new_file_obj
|
||||
|
||||
return self._image_item_for_openai(cast(ChatCompletionImageObject, content_item))
|
||||
if content_item.get("type") == "file":
|
||||
return self._file_item_for_openai(cast(ChatCompletionFileObject, content_item))
|
||||
return content_item
|
||||
|
||||
def _transform_content_item(
|
||||
|
|
@ -419,31 +448,34 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
hoisted_messages: Final = hoist_images_from_tool_messages(stripped_messages)
|
||||
|
||||
async def _async_transform():
|
||||
for message in hoisted_messages:
|
||||
message_content = message.get("content")
|
||||
message_role = message.get("role")
|
||||
|
||||
if message_role == "user" and message_content and isinstance(message_content, list):
|
||||
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
|
||||
for i, content_item in enumerate(message_content_types):
|
||||
message_content_types[i] = await self._async_transform_content_item(
|
||||
cast(OpenAIMessageContentListBlock, content_item),
|
||||
)
|
||||
return hoisted_messages
|
||||
return [await self._async_user_message_for_openai(message) for message in hoisted_messages]
|
||||
|
||||
if is_async:
|
||||
return _async_transform()
|
||||
else:
|
||||
for message in hoisted_messages:
|
||||
message_content = message.get("content")
|
||||
message_role = message.get("role")
|
||||
if message_role == "user" and message_content and isinstance(message_content, list):
|
||||
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
|
||||
for i, content_item in enumerate(message_content):
|
||||
message_content_types[i] = self._transform_content_item(
|
||||
cast(OpenAIMessageContentListBlock, content_item)
|
||||
)
|
||||
return hoisted_messages
|
||||
return [self._user_message_for_openai(message) for message in hoisted_messages]
|
||||
|
||||
@staticmethod
|
||||
def _user_content_parts(message: AllMessageValues) -> Sequence[OpenAIMessageContentListBlock] | None:
|
||||
message_content: Final = message.get("content")
|
||||
if message.get("role") != "user" or not message_content or not isinstance(message_content, list):
|
||||
return None
|
||||
return cast(Sequence[OpenAIMessageContentListBlock], message_content)
|
||||
|
||||
def _user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
|
||||
parts: Final = self._user_content_parts(message)
|
||||
if parts is None:
|
||||
return message
|
||||
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
|
||||
AllMessageValues, {**message, "content": [self._transform_content_item(part) for part in parts]}
|
||||
)
|
||||
|
||||
async def _async_user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
|
||||
parts: Final = self._user_content_parts(message)
|
||||
if parts is None:
|
||||
return message
|
||||
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
|
||||
AllMessageValues, {**message, "content": [await self._async_transform_content_item(part) for part in parts]}
|
||||
)
|
||||
|
||||
def transform_messages(
|
||||
self,
|
||||
|
|
@ -462,23 +494,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
tools: list["ChatCompletionToolParam"] | None = None,
|
||||
) -> tuple[list[AllMessageValues], list["ChatCompletionToolParam"] | None]:
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
filter_value_from_dict,
|
||||
)
|
||||
from litellm.types.llms.openai import ChatCompletionToolParam
|
||||
|
||||
for i, message in enumerate(messages):
|
||||
messages[i] = cast(
|
||||
AllMessageValues,
|
||||
filter_value_from_dict(message, "cache_control"),
|
||||
)
|
||||
if tools is not None:
|
||||
for i, tool in enumerate(tools):
|
||||
tools[i] = cast(
|
||||
ChatCompletionToolParam,
|
||||
filter_value_from_dict(tool, "cache_control"),
|
||||
)
|
||||
return messages, tools
|
||||
new_messages: Final = [cast(AllMessageValues, without_cache_control(message)) for message in messages]
|
||||
new_tools: Final = (
|
||||
[cast(ChatCompletionToolParam, without_cache_control(tool)) for tool in tools]
|
||||
if tools is not None
|
||||
else None
|
||||
)
|
||||
return new_messages, new_tools
|
||||
|
||||
def translate_developer_role_to_system_role(
|
||||
self,
|
||||
|
|
@ -527,6 +551,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
tools: Sequence["ChatCompletionToolParam"] | None,
|
||||
prompt_cache_options: object,
|
||||
litellm_params: Mapping[str, object],
|
||||
request_params: Mapping[str, object],
|
||||
) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]:
|
||||
"""
|
||||
A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on
|
||||
|
|
@ -543,10 +568,11 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
model, custom_llm_provider, api_base, prompt_cache_options
|
||||
)
|
||||
if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base):
|
||||
return messages, tools
|
||||
drops_breakpoints: Final = not reads_breakpoints and (
|
||||
litellm.drop_params is True or litellm_params.get("drop_params") is True
|
||||
)
|
||||
return (
|
||||
_claude_messages_within_cache_control_cap(model, messages, tools, litellm_params, request_params),
|
||||
tools,
|
||||
)
|
||||
drops_breakpoints: Final = not reads_breakpoints and _drops_params(litellm_params)
|
||||
renamed: Final = (
|
||||
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
|
||||
if reads_breakpoints
|
||||
|
|
@ -561,7 +587,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
if drops_breakpoints
|
||||
else list(tools)
|
||||
)
|
||||
return self.remove_cache_control_flag_from_messages_and_tools(model=model, messages=renamed, tools=kept_tools)
|
||||
sent_messages, sent_tools = self.remove_cache_control_flag_from_messages_and_tools(
|
||||
model=model, messages=renamed, tools=kept_tools
|
||||
)
|
||||
return (
|
||||
_claude_messages_within_cache_control_cap(model, sent_messages, sent_tools, litellm_params, request_params),
|
||||
sent_tools,
|
||||
)
|
||||
|
||||
def _sanitized_tools_update_for_openai(
|
||||
self,
|
||||
|
|
@ -623,6 +655,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
tools=optional_params.get("tools", []),
|
||||
prompt_cache_options=_prompt_cache_options(optional_params),
|
||||
litellm_params=litellm_params,
|
||||
request_params=optional_params,
|
||||
)
|
||||
if tools is not None and len(tools) > 0:
|
||||
optional_params["tools"] = tools
|
||||
|
|
@ -654,6 +687,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
tools=optional_params.get("tools", []),
|
||||
prompt_cache_options=_prompt_cache_options(optional_params),
|
||||
litellm_params=litellm_params,
|
||||
request_params=optional_params,
|
||||
)
|
||||
if tools is not None and len(tools) > 0:
|
||||
optional_params["tools"] = tools
|
||||
|
|
|
|||
|
|
@ -153,12 +153,13 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
|
|||
- role: system ==> translate to role 'user'
|
||||
"""
|
||||
_supports_system_messages: Final = supports_system_messages(model, "openai")
|
||||
for i, message in enumerate(messages):
|
||||
transformed_messages: Final = list(messages)
|
||||
for i, message in enumerate(transformed_messages):
|
||||
if message["role"] == "system" and not _supports_system_messages:
|
||||
new_message = ChatCompletionUserMessage(content=message["content"], role="user")
|
||||
messages[i] = new_message # Replace the old message with the new one
|
||||
transformed_messages[i] = new_message # Replace the old message with the new one
|
||||
|
||||
if is_async:
|
||||
return super()._transform_messages(messages, model, is_async=cast(Literal[True], True))
|
||||
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True))
|
||||
else:
|
||||
return super()._transform_messages(messages, model, is_async=cast(Literal[False], False))
|
||||
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[False], False))
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Common helpers / utils across al OpenAI endpoints
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
|
|
@ -31,7 +32,7 @@ if TYPE_CHECKING:
|
|||
import litellm
|
||||
from litellm.exceptions import error_response_without_credentials
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, iter_items
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
|
||||
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
|
@ -43,6 +44,47 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
)
|
||||
|
||||
|
||||
def _is_plain_dict(value: object) -> bool:
|
||||
return isinstance(value, dict)
|
||||
|
||||
|
||||
def _is_list(value: object) -> bool:
|
||||
return isinstance(value, list)
|
||||
|
||||
|
||||
def _contains_cache_control(value: object) -> bool:
|
||||
pending: Final[list[object]] = [value] # mutable-ok: a local worklist, never returned or stored
|
||||
while pending:
|
||||
node = pending.pop()
|
||||
mapping = as_str_mapping(node)
|
||||
if mapping is not None:
|
||||
if "cache_control" in mapping:
|
||||
return True
|
||||
pending.extend(mapping.values())
|
||||
elif _is_list(node):
|
||||
pending.extend(iter_items(node))
|
||||
return False
|
||||
|
||||
|
||||
def without_cache_control(item: object) -> object:
|
||||
"""Return `item` with every `cache_control` key removed, without touching the original.
|
||||
|
||||
`filter_value_from_dict` deletes the key in place and recurses into nested dicts and
|
||||
lists, so the caller's own message, tool or input object has to be copied first: it
|
||||
may reuse the same list on a provider that does support prompt caching. The copy is
|
||||
skipped when there is nothing to strip, which keeps the object identity the
|
||||
pass-through paths rely on and avoids a deep copy on every request.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import filter_value_from_dict
|
||||
|
||||
if not _is_plain_dict(item) or not _contains_cache_control(item):
|
||||
return item
|
||||
copied: Final = copy.deepcopy(item)
|
||||
if isinstance(copied, dict):
|
||||
filter_value_from_dict(copied, "cache_control")
|
||||
return copied
|
||||
|
||||
|
||||
def _get_client_init_params(cls: type) -> tuple[str, ...]:
|
||||
"""Extract __init__ parameter names (excluding 'self') from a class."""
|
||||
return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self")
|
||||
|
|
|
|||
|
|
@ -354,10 +354,10 @@ class OpenAIDecisionsConfig(BaseDecisionsConfig):
|
|||
) -> str | None:
|
||||
return resolve_openai_bearer_token(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
|
||||
|
||||
def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
|
||||
return False
|
||||
|
||||
def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
|
||||
return False
|
||||
|
||||
def transform_decisions_request(
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from openai._base_client import make_request_options
|
|||
from openai._constants import RAW_RESPONSE_HEADER
|
||||
from openai._legacy_response import LegacyAPIResponse
|
||||
from openai._types import RequestOptions
|
||||
from openai.pagination import SyncCursorPage
|
||||
from openai.types import CreateEmbeddingResponse
|
||||
from openai.types.beta.assistant_deleted import AssistantDeleted
|
||||
from openai.types.chat import ChatCompletion, ChatCompletionChunk
|
||||
|
|
@ -40,6 +41,7 @@ from litellm.types.utils import (
|
|||
LiteLLMBatch,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
openai_batch_status,
|
||||
)
|
||||
from litellm.utils import (
|
||||
CustomStreamWrapper,
|
||||
|
|
@ -2182,6 +2184,18 @@ class OpenAIFilesAPI(BaseLLM):
|
|||
return response
|
||||
|
||||
|
||||
def _with_openai_batch_fields(batches: Sequence[Batch]) -> Sequence[Batch]:
|
||||
return [
|
||||
batch.model_copy(
|
||||
update={
|
||||
"object": batch.object or "batch",
|
||||
"status": openai_batch_status(batch.status),
|
||||
}
|
||||
)
|
||||
for batch in batches
|
||||
]
|
||||
|
||||
|
||||
class OpenAIBatchesAPI(BaseLLM):
|
||||
"""
|
||||
OpenAI methods to support for batches
|
||||
|
|
@ -2372,7 +2386,7 @@ class OpenAIBatchesAPI(BaseLLM):
|
|||
):
|
||||
verbose_logger.debug("listing batches, after= %s, limit= %s", after, limit)
|
||||
response: Final = await openai_client.batches.list(after=after, limit=limit)
|
||||
return response
|
||||
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
|
||||
|
||||
def list_batches(
|
||||
self,
|
||||
|
|
@ -2384,7 +2398,7 @@ class OpenAIBatchesAPI(BaseLLM):
|
|||
organization: str | None,
|
||||
after: str | None = None,
|
||||
limit: int | None = None,
|
||||
client: OpenAI | None = None,
|
||||
client: OpenAI | AsyncOpenAI | None = None,
|
||||
litellm_params: Mapping[str, object] | None = None,
|
||||
):
|
||||
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
|
||||
|
|
@ -2409,7 +2423,9 @@ class OpenAIBatchesAPI(BaseLLM):
|
|||
)
|
||||
return self.alist_batches(openai_client=openai_client, after=after, limit=limit)
|
||||
response: Final = openai_client.batches.list(after=after, limit=limit)
|
||||
return response
|
||||
if not isinstance(response, SyncCursorPage):
|
||||
return response
|
||||
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
|
||||
|
||||
|
||||
class OpenAIAssistantsAPI(BaseLLM):
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ from itertools import accumulate, chain, groupby, repeat
|
|||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast
|
||||
|
||||
from openai.types.responses.response_output_refusal import ResponseOutputRefusal
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
|
|
@ -277,6 +278,7 @@ _TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | f
|
|||
)
|
||||
_OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"})
|
||||
_OUTPUT_TEXT_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.output_text.done"})
|
||||
_TEXT_DELTA_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.refusal.delta"})
|
||||
_TOOL_OUTPUT_TEXT_PART_TYPES: Final = frozenset({"input_text", "output_text", "text"})
|
||||
_PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"}
|
||||
|
|
@ -537,6 +539,14 @@ def _scanned_text_field(item: Mapping[str, object]) -> str:
|
|||
return "output" if item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES else "content"
|
||||
|
||||
|
||||
def _content_part_text_field(part: object) -> str:
|
||||
return "refusal" if stream_item_field(part, "type") == "refusal" else "text"
|
||||
|
||||
|
||||
def _has_writable_field(part: object, field: str) -> bool:
|
||||
return isinstance(part, dict) or hasattr(part, field)
|
||||
|
||||
|
||||
def _tool_output_part_text(part: object) -> str | None:
|
||||
if not isinstance(part, dict):
|
||||
return None
|
||||
|
|
@ -993,7 +1003,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
for content_idx, content_item in enumerate(content):
|
||||
if isinstance(content_item, dict):
|
||||
# Extract text
|
||||
text_str = content_item.get("text", None)
|
||||
text_str = content_item.get(_content_part_text_field(content_item), None)
|
||||
if text_str is not None:
|
||||
texts_to_check.append(text_str)
|
||||
task_mappings.append((msg_idx, int(content_idx)))
|
||||
|
|
@ -1042,8 +1052,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
|
||||
elif isinstance(content, list):
|
||||
# Replace specific text item in list content
|
||||
if isinstance(messages[msg_idx][field][content_idx_optional], dict):
|
||||
messages[msg_idx][field][content_idx_optional]["text"] = guardrail_response
|
||||
self._write_content_part_text(content[content_idx_optional], guardrail_response)
|
||||
|
||||
async def process_output_response(
|
||||
self,
|
||||
|
|
@ -1380,7 +1389,8 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
if isinstance(output_index, int) and isinstance(content_index, int)
|
||||
)
|
||||
if len(placeable_positions) != len(scanned_positions) or any(
|
||||
stream_item_field(event, "type") not in _OUTPUT_TEXT_EVENT_TYPES for event in scanned_events
|
||||
stream_item_field(event, "type") not in _OUTPUT_TEXT_EVENT_TYPES | _TEXT_DELTA_EVENT_TYPES
|
||||
for event in scanned_events
|
||||
):
|
||||
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
|
||||
|
||||
|
|
@ -1401,6 +1411,12 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
else:
|
||||
setattr(event, field, value)
|
||||
|
||||
@staticmethod
|
||||
def _write_content_part_text(part: object, text: str) -> None:
|
||||
field: Final = _content_part_text_field(part)
|
||||
if _has_writable_field(part, field):
|
||||
OpenAIResponsesHandler._write_event_field(part, field, text)
|
||||
|
||||
@staticmethod
|
||||
def _write_compaction_summary(item: object, summary: str) -> None:
|
||||
encrypted_content: Final = stream_item_field(item, "encrypted_content")
|
||||
|
|
@ -1416,11 +1432,12 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
) -> None:
|
||||
"""Sync pre-completion stream events with the rewritten completed
|
||||
response, keyed by ``(output_index, content_index)``: the first
|
||||
``output_text.delta`` for a rewritten item carries the full rewritten
|
||||
text and the rest are blanked, while ``output_text.done``,
|
||||
``content_part.done``, and ``output_item.done`` events carry the full
|
||||
rewritten text, so every event a client may read agrees with the
|
||||
rewritten ``response.completed`` payload."""
|
||||
``output_text.delta`` or ``refusal.delta`` for a rewritten item carries
|
||||
the full rewritten text and the rest are blanked, while
|
||||
``output_text.done``, ``refusal.done``, ``content_part.done``, and
|
||||
``output_item.done`` events carry the full rewritten text, so every
|
||||
event a client may read agrees with the rewritten ``response.completed``
|
||||
payload."""
|
||||
delta_replacements: Final = MappingProxyType(
|
||||
{position: chain((rewritten,), repeat("")) for position, rewritten in rewrites_by_position.items()}
|
||||
)
|
||||
|
|
@ -1436,14 +1453,14 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
if not isinstance(output_index, int) or not isinstance(content_index, int):
|
||||
continue
|
||||
position = (output_index, content_index)
|
||||
if event_type == "response.output_text.delta" and position in delta_replacements:
|
||||
if event_type in _TEXT_DELTA_EVENT_TYPES and position in delta_replacements:
|
||||
self._write_event_field(event, "delta", next(delta_replacements[position]))
|
||||
elif event_type == "response.output_text.done" and position in rewrites_by_position:
|
||||
self._write_event_field(event, "text", rewrites_by_position[position])
|
||||
elif event_type == "response.refusal.done" and position in rewrites_by_position:
|
||||
self._write_event_field(event, "refusal", rewrites_by_position[position])
|
||||
elif event_type == "response.content_part.done" and position in rewrites_by_position:
|
||||
part = event.get("part")
|
||||
if isinstance(part, dict) or hasattr(part, "text"):
|
||||
self._write_event_field(part, "text", rewrites_by_position[position])
|
||||
self._write_content_part_text(event.get("part"), rewrites_by_position[position])
|
||||
|
||||
@staticmethod
|
||||
def _sync_output_item_done_event(
|
||||
|
|
@ -1459,7 +1476,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
for (item_idx, content_idx), rewritten in rewrites_by_position.items():
|
||||
if item_idx != output_index or not 0 <= content_idx < len(content):
|
||||
continue
|
||||
OpenAIResponsesHandler._write_event_field(content[content_idx], "text", rewritten)
|
||||
OpenAIResponsesHandler._write_content_part_text(content[content_idx], rewritten)
|
||||
|
||||
def _deliver_ended_stream_tool_call_rewrites(
|
||||
self,
|
||||
|
|
@ -1814,7 +1831,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
return
|
||||
|
||||
# Handle both GenericResponseOutputItem and dict
|
||||
content: list[OutputText] | list[dict] | None = None
|
||||
content: list[OutputText | ResponseOutputRefusal] | list[dict[str, object]] | None = None
|
||||
if isinstance(output_item, BaseModel):
|
||||
try:
|
||||
output_item_dump: Final = output_item.model_dump()
|
||||
|
|
@ -1837,17 +1854,9 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
|
||||
verbose_proxy_logger.debug("OpenAI Responses API: Processing output item: %s", output_item)
|
||||
|
||||
# Iterate through content items (list of OutputText objects)
|
||||
for content_idx, content_item in enumerate(content):
|
||||
# Handle both OutputText objects and dicts
|
||||
if isinstance(content_item, OutputText):
|
||||
text_content = content_item.text
|
||||
elif isinstance(content_item, dict):
|
||||
text_content = content_item.get("text")
|
||||
else:
|
||||
continue
|
||||
|
||||
if text_content:
|
||||
text_content = stream_item_field(content_item, _content_part_text_field(content_item))
|
||||
if isinstance(text_content, str) and text_content:
|
||||
texts_to_check.append(text_content)
|
||||
task_mappings.append((output_idx, int(content_idx)))
|
||||
|
||||
|
|
@ -1887,33 +1896,24 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
# Handle both GenericResponseOutputItem, BaseModel, and dict
|
||||
if isinstance(output_item, GenericResponseOutputItem):
|
||||
if output_item.content and content_idx < len(output_item.content):
|
||||
content_item = output_item.content[content_idx]
|
||||
if isinstance(content_item, OutputText):
|
||||
content_item.text = guardrail_response
|
||||
elif isinstance(content_item, dict):
|
||||
content_item["text"] = guardrail_response
|
||||
self._write_content_part_text(output_item.content[content_idx], guardrail_response)
|
||||
elif isinstance(output_item, BaseModel):
|
||||
# Handle other Pydantic models by converting to GenericResponseOutputItem
|
||||
try:
|
||||
generic_item = GenericResponseOutputItem.model_validate(output_item.model_dump())
|
||||
if generic_item.content and content_idx < len(generic_item.content):
|
||||
content_item = generic_item.content[content_idx]
|
||||
if isinstance(content_item, OutputText):
|
||||
content_item.text = guardrail_response
|
||||
# Update the original response output
|
||||
if hasattr(output_item, "content") and output_item.content:
|
||||
original_content = output_item.content[content_idx]
|
||||
if hasattr(original_content, "text"):
|
||||
original_content.text = guardrail_response
|
||||
if (
|
||||
generic_item.content
|
||||
and content_idx < len(generic_item.content)
|
||||
and hasattr(output_item, "content")
|
||||
and output_item.content
|
||||
):
|
||||
self._write_content_part_text(output_item.content[content_idx], guardrail_response)
|
||||
except Exception:
|
||||
pass
|
||||
elif isinstance(output_item, dict):
|
||||
content = output_item.get("content", [])
|
||||
if content and content_idx < len(content):
|
||||
if isinstance(content[content_idx], dict):
|
||||
content[content_idx]["text"] = guardrail_response
|
||||
elif hasattr(content[content_idx], "text"):
|
||||
content[content_idx].text = guardrail_response
|
||||
self._write_content_part_text(content[content_idx], guardrail_response)
|
||||
|
||||
def build_block_sse_chunks(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ from litellm.types.router import GenericLiteLLMParams
|
|||
from litellm.types.utils import LlmProviders
|
||||
from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS
|
||||
|
||||
from ..common_utils import OpenAIError
|
||||
from ..common_utils import OpenAIError, without_cache_control
|
||||
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
|
||||
|
||||
OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16
|
||||
|
|
@ -658,13 +658,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search),
|
||||
tools=mapped_tools,
|
||||
)
|
||||
if marker_safe_input is not validated_input:
|
||||
self._strip_caller_input_as_if_unmarked(
|
||||
model=model,
|
||||
caller_input=validated_input,
|
||||
reads_breakpoints=reads_breakpoints,
|
||||
maps_anthropic_tool_search=maps_anthropic_tool_search,
|
||||
)
|
||||
object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools)
|
||||
sanitized_tools: Final = self._sanitized_tool_schemas_for_openai(
|
||||
model=model, tools=object_schema_tools, litellm_params=litellm_params
|
||||
|
|
@ -679,17 +672,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
return marked_input
|
||||
return input_without_unanswered_anthropic_tool_search_calls(marked_input)
|
||||
|
||||
def _strip_caller_input_as_if_unmarked(
|
||||
self,
|
||||
model: str,
|
||||
caller_input: str | ResponseInputParam,
|
||||
reads_breakpoints: bool,
|
||||
maps_anthropic_tool_search: bool,
|
||||
) -> None:
|
||||
self.remove_cache_control_flag_from_input_and_tools(
|
||||
model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search)
|
||||
)
|
||||
|
||||
def _input_with_wire_safe_prompt_cache_breakpoints(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -800,24 +782,29 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
the chat path. Strips Anthropic-only `cache_control` markers from
|
||||
Responses API input content blocks and tools.
|
||||
|
||||
`filter_value_from_dict` mutates each dict in place, so the same
|
||||
objects are returned.
|
||||
`filter_value_from_dict` deletes the key in place and recurses, so each
|
||||
item is copied first: the caller keeps its own input list and may reuse
|
||||
it on a provider that does support prompt caching.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
filter_value_from_dict,
|
||||
new_input: Final = (
|
||||
cast( # cast-ok: the comprehension rebuilds the input item for item
|
||||
ResponseInputParam,
|
||||
[without_cache_control(item) for item in input],
|
||||
)
|
||||
if isinstance(input, list)
|
||||
else input
|
||||
)
|
||||
|
||||
if isinstance(input, list):
|
||||
for item in input:
|
||||
if isinstance(item, dict):
|
||||
filter_value_from_dict(cast(dict, item), "cache_control")
|
||||
new_tools: Final = (
|
||||
cast( # cast-ok: the comprehension rebuilds the tool list item for item
|
||||
Sequence[ALL_RESPONSES_API_TOOL_PARAMS],
|
||||
[without_cache_control(tool) for tool in tools],
|
||||
)
|
||||
if tools is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if tools is not None:
|
||||
for tool in tools:
|
||||
if isinstance(tool, dict):
|
||||
filter_value_from_dict(cast(dict, tool), "cache_control")
|
||||
|
||||
return input, tools
|
||||
return new_input, new_tools
|
||||
|
||||
def _drop_bridge_minted_reasoning_items(self, input: str | ResponseInputParam) -> str | ResponseInputParam:
|
||||
if self.custom_llm_provider not in _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING or not isinstance(input, list):
|
||||
|
|
|
|||
|
|
@ -125,6 +125,20 @@ The provider inherits all request/response handling from OpenAI's Responses API
|
|||
|
||||
If `supported_endpoints` is omitted, it defaults to `[]` (only chat completions, which is always enabled for JSON providers).
|
||||
|
||||
When only some models serve an endpoint natively, list them under `model_supported_endpoints`. Every other model reaches `/v1/responses` and `/v1/messages` through chat completions:
|
||||
|
||||
```json
|
||||
{
|
||||
"your_provider": {
|
||||
"supported_endpoints": ["/v1/chat/completions"],
|
||||
"model_supported_endpoints": {
|
||||
"model-a": ["/v1/messages"],
|
||||
"model-b": ["/v1/responses"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### How It Works
|
||||
|
||||
1. `json_loader.py` checks `supported_endpoints` for `/v1/responses`
|
||||
|
|
|
|||
|
|
@ -3,11 +3,16 @@
|
|||
## Allows jina ai embedding calls - which don't allow 'encoding_format' in payload.
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
|
|
@ -17,6 +22,57 @@ from litellm.types.utils import EmbeddingResponse
|
|||
|
||||
from ..common_utils import OpenAILikeBase, OpenAILikeError
|
||||
|
||||
_KEY_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str])
|
||||
_EXTRA_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_TRANSPORT_HEADER_NAMES: Final = frozenset(
|
||||
{
|
||||
"host",
|
||||
"content-length",
|
||||
"transfer-encoding",
|
||||
"connection",
|
||||
"keep-alive",
|
||||
"proxy-connection",
|
||||
"te",
|
||||
"trailer",
|
||||
"upgrade",
|
||||
"expect",
|
||||
}
|
||||
)
|
||||
_HEADER_NAME: Final = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
|
||||
_HEADER_VALUE: Final = re.compile(r"[\t\x20-\x7e]*")
|
||||
|
||||
|
||||
def _header_text(name: str, value: object) -> str | None:
|
||||
if isinstance(value, bool) or not isinstance(value, str | int | float):
|
||||
return None
|
||||
if _HEADER_NAME.fullmatch(name) is None:
|
||||
return None
|
||||
return str(value).strip(" \t\r\n")
|
||||
|
||||
|
||||
def _extra_headers_as_text(extra_headers: Mapping[str, object], model: str, drop_params: bool) -> Mapping[str, str]:
|
||||
caller_set: Final = {
|
||||
name: value for name, value in extra_headers.items() if name.lower() not in _TRANSPORT_HEADER_NAMES
|
||||
}
|
||||
texts: Final = MappingProxyType({name: _header_text(name, value) for name, value in caller_set.items()})
|
||||
refused: Final = sorted(name for name, text in texts.items() if text is None)
|
||||
if refused and not drop_params:
|
||||
raise litellm.UnsupportedParamsError(
|
||||
message=f"extra_headers {', '.join(map(repr, refused))} cannot be sent. A header needs a name made of "
|
||||
"letters, digits or !#$%&'*+-.^_`|~ and a text or number value. "
|
||||
"To drop them from the call, set `litellm.drop_params = True`.",
|
||||
model=model,
|
||||
)
|
||||
named: Final = MappingProxyType({name: text for name, text in texts.items() if text is not None})
|
||||
unsendable: Final = sorted(name for name, text in named.items() if _HEADER_VALUE.fullmatch(text) is None)
|
||||
if unsendable:
|
||||
verbose_logger.warning(
|
||||
"extra_headers %s dropped for model %r, since a header value must be printable ASCII on one line",
|
||||
unsendable,
|
||||
model,
|
||||
)
|
||||
return MappingProxyType({name: text for name, text in named.items() if name not in unsendable})
|
||||
|
||||
|
||||
class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
||||
def __init__(self, **kwargs):
|
||||
|
|
@ -94,6 +150,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
|||
aembedding=None,
|
||||
custom_endpoint: bool | None = None,
|
||||
headers: dict | None = None,
|
||||
drop_params: bool = False,
|
||||
) -> EmbeddingResponse:
|
||||
api_base, headers = self._validate_environment(
|
||||
api_base=api_base,
|
||||
|
|
@ -103,8 +160,20 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
|||
custom_endpoint=custom_endpoint,
|
||||
)
|
||||
model = model
|
||||
filtered_optional_params: Final = {k: v for k, v in optional_params.items() if v not in (None, "")}
|
||||
data: Final = {"model": model, "input": input, **filtered_optional_params}
|
||||
data: Final = {
|
||||
"model": model,
|
||||
"input": input,
|
||||
**{k: v for k, v in optional_params.items() if k != "extra_headers" and v not in (None, "")},
|
||||
}
|
||||
extra_headers: Final = _extra_headers_as_text(
|
||||
_EXTRA_HEADERS_ADAPTER.validate_python(optional_params.get("extra_headers") or {}), model, drop_params
|
||||
)
|
||||
key_headers: Final = _KEY_HEADERS_ADAPTER.validate_python(headers)
|
||||
extra_header_names: Final = frozenset(name.lower() for name in extra_headers)
|
||||
request_headers: Final = {
|
||||
**{name: value for name, value in key_headers.items() if name.lower() not in extra_header_names},
|
||||
**extra_headers,
|
||||
}
|
||||
|
||||
## LOGGING
|
||||
logging_obj.pre_call(
|
||||
|
|
@ -123,7 +192,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
|||
api_key=api_key,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
headers=headers,
|
||||
headers=request_headers,
|
||||
)
|
||||
if client is None or isinstance(client, AsyncHTTPHandler):
|
||||
self.client = HTTPHandler(timeout=timeout)
|
||||
|
|
@ -134,7 +203,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
|||
try:
|
||||
response: Final = self.client.post(
|
||||
api_base,
|
||||
headers=headers,
|
||||
headers=request_headers,
|
||||
data=json.dumps(data),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers.
|
|||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -14,6 +14,7 @@ from litellm.types.utils import ProviderSpecificModelInfo
|
|||
|
||||
_PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo)
|
||||
_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool])
|
||||
_MODEL_ENDPOINTS_ADAPTER: Final = TypeAdapter(Mapping[str, tuple[str, ...]])
|
||||
|
||||
|
||||
class SimpleProviderConfig:
|
||||
|
|
@ -30,7 +31,10 @@ class SimpleProviderConfig:
|
|||
self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python(
|
||||
data.get("special_handling", {})
|
||||
)
|
||||
self.supported_endpoints = data.get("supported_endpoints", [])
|
||||
self.supported_endpoints: Sequence[str] = data.get("supported_endpoints", [])
|
||||
self.model_supported_endpoints: Mapping[str, tuple[str, ...]] = _MODEL_ENDPOINTS_ADAPTER.validate_python(
|
||||
data.get("model_supported_endpoints", {})
|
||||
)
|
||||
self.provider_info: ProviderSpecificModelInfo = (
|
||||
_PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {}
|
||||
)
|
||||
|
|
@ -90,6 +94,23 @@ class JSONProviderRegistry:
|
|||
return False
|
||||
return "/v1/responses" in provider.supported_endpoints
|
||||
|
||||
@classmethod
|
||||
def supports_embeddings(cls, slug: str) -> bool:
|
||||
provider: Final = cls._providers.get(slug)
|
||||
if provider is None:
|
||||
return False
|
||||
return "/v1/embeddings" in provider.supported_endpoints
|
||||
|
||||
@classmethod
|
||||
def serves_endpoint(cls, slug: str, model: str | None, endpoint: str) -> bool:
|
||||
provider: Final = cls._providers.get(slug)
|
||||
if provider is None:
|
||||
return False
|
||||
model_endpoints: Final = (
|
||||
provider.model_supported_endpoints.get(model.removeprefix(f"{slug}/"), ()) if model else ()
|
||||
)
|
||||
return endpoint in provider.supported_endpoints or endpoint in model_endpoints
|
||||
|
||||
@classmethod
|
||||
def list_providers(cls) -> list:
|
||||
"""List all registered provider slugs"""
|
||||
|
|
|
|||
|
|
@ -211,7 +211,7 @@
|
|||
"param_mappings": {
|
||||
"max_completion_tokens": "max_tokens"
|
||||
},
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"]
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
|
||||
},
|
||||
"scx-ai": {
|
||||
"base_url": "https://api.scx.ai/v1",
|
||||
|
|
@ -231,6 +231,16 @@
|
|||
"api_base_env": "PRISM_API_BASE",
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
|
||||
},
|
||||
"hostinger_router": {
|
||||
"base_url": "https://router.hostinger.com/v1",
|
||||
"api_key_env": "HOSTINGER_ROUTER_API_KEY",
|
||||
"api_base_env": "HOSTINGER_ROUTER_API_BASE",
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/embeddings"],
|
||||
"model_supported_endpoints": {
|
||||
"claude-sonnet-5": ["/v1/messages"],
|
||||
"gpt-5.2": ["/v1/responses"]
|
||||
}
|
||||
},
|
||||
"sail": {
|
||||
"base_url": "https://api.sailresearch.com/v1",
|
||||
"api_key_env": "SAIL_API_KEY",
|
||||
|
|
|
|||
|
|
@ -110,38 +110,23 @@ def separate_cached_messages(
|
|||
messages: list[AllMessageValues],
|
||||
) -> tuple[list[AllMessageValues], list[AllMessageValues]]:
|
||||
"""
|
||||
Returns separated cached and non-cached messages.
|
||||
|
||||
Args:
|
||||
messages: List of messages to be separated.
|
||||
|
||||
Returns:
|
||||
Tuple containing:
|
||||
- cached_messages: List of cached messages.
|
||||
- non_cached_messages: List of non-cached messages.
|
||||
Returns the first continuous block of cached messages, plus every system message once
|
||||
any message is cached, since generateContent rejects a system_instruction beside a
|
||||
cachedContent, and the remaining messages.
|
||||
"""
|
||||
cached_messages: list[AllMessageValues] = []
|
||||
non_cached_messages: list[AllMessageValues] = []
|
||||
filtered_messages: Final = [(idx, message) for idx, message in enumerate(messages) if is_cached_message(message)]
|
||||
if not filtered_messages:
|
||||
return [], messages
|
||||
|
||||
# Extract cached messages and their indices
|
||||
filtered_messages: Final[list[tuple[int, AllMessageValues]]] = []
|
||||
for idx, message in enumerate(messages):
|
||||
if is_cached_message(message=message):
|
||||
filtered_messages.append((idx, message))
|
||||
|
||||
# Validate only one block of continuous cached messages
|
||||
last_continuous_block_idx: Final = get_first_continuous_block_idx(filtered_messages)
|
||||
# Separate messages based on the block of cached messages
|
||||
if filtered_messages and last_continuous_block_idx is not None:
|
||||
first_cached_idx: Final = filtered_messages[0][0]
|
||||
last_cached_idx: Final = filtered_messages[last_continuous_block_idx][0]
|
||||
|
||||
cached_messages = messages[first_cached_idx : last_cached_idx + 1]
|
||||
non_cached_messages = messages[:first_cached_idx] + messages[last_cached_idx + 1 :]
|
||||
else:
|
||||
non_cached_messages = messages
|
||||
|
||||
return cached_messages, non_cached_messages
|
||||
cached_block: Final = range(filtered_messages[0][0], filtered_messages[last_continuous_block_idx][0] + 1)
|
||||
goes_in_cache: Final = tuple(
|
||||
idx in cached_block or message.get("role") == "system" for idx, message in enumerate(messages)
|
||||
)
|
||||
return (
|
||||
[message for message, cached in zip(messages, goes_in_cache) if cached],
|
||||
[message for message, cached in zip(messages, goes_in_cache) if not cached],
|
||||
)
|
||||
|
||||
|
||||
def cached_messages_end_on_supported_turn(cached_messages: Sequence[AllMessageValues]) -> bool:
|
||||
|
|
|
|||
|
|
@ -31,9 +31,8 @@ from litellm.exceptions import UnsupportedParamsError
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
encode_tool_call_id_with_signature,
|
||||
)
|
||||
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import encode_tool_call_id_with_signature
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
|
|
@ -2091,6 +2090,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
parts: list[HttpxPartType],
|
||||
cumulative_tool_call_idx: int,
|
||||
is_function_call: bool | None,
|
||||
thought_signature_in_tool_call_id: bool = True,
|
||||
) -> tuple[
|
||||
ChatCompletionToolCallFunctionChunk | None,
|
||||
list[ChatCompletionToolCallChunk] | None,
|
||||
|
|
@ -2129,9 +2129,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
# back on the matching `functionResponse`.
|
||||
if gemini_call_id:
|
||||
_tool_response_chunk["id"] = gemini_call_id
|
||||
# Embed thought signature in ID for OpenAI client compatibility
|
||||
if thought_signature:
|
||||
_tool_response_chunk["provider_specific_fields"] = {"thought_signature": thought_signature}
|
||||
if thought_signature and thought_signature_in_tool_call_id:
|
||||
_tool_response_chunk["id"] = encode_tool_call_id_with_signature(
|
||||
_tool_response_chunk["id"] or "", thought_signature
|
||||
)
|
||||
|
|
@ -2793,6 +2793,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
model_response: Union[ModelResponse, "ModelResponseStream"],
|
||||
standard_optional_params: dict,
|
||||
cumulative_tool_call_index: int = 0,
|
||||
thought_signature_in_tool_call_id: bool = True,
|
||||
) -> tuple[list[dict], list[dict], list, list, int]:
|
||||
"""
|
||||
Helper method to process candidates and extract metadata
|
||||
|
|
@ -2895,6 +2896,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
parts=candidate["content"]["parts"],
|
||||
cumulative_tool_call_idx=cumulative_tool_call_index,
|
||||
is_function_call=is_function_call(standard_optional_params),
|
||||
thought_signature_in_tool_call_id=thought_signature_in_tool_call_id,
|
||||
)
|
||||
|
||||
if "logprobsResult" in candidate:
|
||||
|
|
@ -2978,6 +2980,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
model_response: Union[ModelResponse, "ModelResponseStream"],
|
||||
standard_optional_params: dict[str, object], # mutable-ok: mirrors override contract
|
||||
cumulative_tool_call_index: int = 0,
|
||||
thought_signature_in_tool_call_id: bool = True,
|
||||
) -> tuple[ # mutable-ok: mirrors override contract
|
||||
list[dict[str, object]],
|
||||
list[dict[str, object]],
|
||||
|
|
@ -2986,7 +2989,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
int,
|
||||
]:
|
||||
return cls._process_candidates(
|
||||
_candidates, model_response, standard_optional_params, cumulative_tool_call_index
|
||||
_candidates,
|
||||
model_response,
|
||||
standard_optional_params,
|
||||
cumulative_tool_call_index,
|
||||
thought_signature_in_tool_call_id,
|
||||
)
|
||||
|
||||
def transform_response(
|
||||
|
|
@ -3069,7 +3076,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
safety_ratings,
|
||||
citation_metadata,
|
||||
_, # cumulative_tool_call_index not needed in non-streaming
|
||||
) = VertexGeminiConfig._process_candidates(_candidates, model_response, logging_obj.optional_params)
|
||||
) = VertexGeminiConfig._process_candidates(
|
||||
_candidates,
|
||||
model_response,
|
||||
logging_obj.optional_params,
|
||||
thought_signature_in_tool_call_id=embeds_thought_signature_in_tool_call_id(
|
||||
logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped
|
||||
),
|
||||
)
|
||||
|
||||
usage: Final = VertexGeminiConfig._calculate_usage(
|
||||
completion_response=completion_response,
|
||||
|
|
@ -3905,6 +3919,9 @@ class ModelResponseIterator:
|
|||
self.logging_obj = logging_obj
|
||||
self.response_headers = response_headers or {}
|
||||
self.is_function_call = check_is_function_call(logging_obj)
|
||||
self.thought_signature_in_tool_call_id: Final = embeds_thought_signature_in_tool_call_id(
|
||||
logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped
|
||||
)
|
||||
self.cumulative_tool_call_index: int = 0
|
||||
self.has_seen_tool_calls: bool = False
|
||||
self._resume = resume
|
||||
|
|
@ -3969,6 +3986,7 @@ class ModelResponseIterator:
|
|||
model_response,
|
||||
self.logging_obj.optional_params,
|
||||
cumulative_tool_call_index=self.cumulative_tool_call_index,
|
||||
thought_signature_in_tool_call_id=self.thought_signature_in_tool_call_id,
|
||||
)
|
||||
|
||||
# Track whether tool_calls have been seen across streaming chunks.
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ Translation from OpenAI's `/chat/completions` endpoint to IBM WatsonX's `/text/c
|
|||
Docs: https://cloud.ibm.com/apidocs/watsonx-ai#text-chat
|
||||
"""
|
||||
|
||||
import copy
|
||||
from typing import Final
|
||||
|
||||
from litellm import verbose_logger
|
||||
|
|
@ -57,7 +58,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
|
|||
_tools = non_default_params.pop("tools", None)
|
||||
if _tools is not None:
|
||||
# remove 'additionalProperties' from tools
|
||||
_tools = remove_additional_properties(_tools)
|
||||
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
|
||||
# remove 'strict' from tools
|
||||
_tools = remove_strict_from_schema(_tools)
|
||||
if _tools is not None:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
|
@ -208,7 +209,8 @@ class XAIChatConfig(OpenAIGPTConfig):
|
|||
tools = [
|
||||
filtered_tool
|
||||
for filtered_tool in (
|
||||
_FILTERED_TOOL.validate_python(filter_value_from_dict(tool, "strict")) for tool in value
|
||||
_FILTERED_TOOL.validate_python(filter_value_from_dict(copy.deepcopy(tool), "strict")) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
|
||||
for tool in value
|
||||
)
|
||||
if filtered_tool is not None
|
||||
]
|
||||
|
|
|
|||
124
litellm/main.py
124
litellm/main.py
|
|
@ -23,6 +23,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Mappin
|
|||
from concurrent import futures
|
||||
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
|
||||
|
|
@ -82,8 +83,8 @@ from litellm.litellm_core_utils.dd_tracing import tracer
|
|||
from litellm.litellm_core_utils.get_litellm_params import (
|
||||
AWS_CREDENTIAL_KWARGS_KEYS,
|
||||
OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS,
|
||||
OPTIONAL_KWARGS_KEYS,
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
|
||||
SUPPLEMENTAL_KWARGS_KEYS,
|
||||
InvalidControlOption,
|
||||
parse_control_options,
|
||||
with_control_options,
|
||||
|
|
@ -157,7 +158,7 @@ from litellm.types.completion import (
|
|||
)
|
||||
from litellm.types.litellm_params import ControlOptions, RetryStrategy
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.router import GenericLiteLLMParams, validate_litellm_params_strictly
|
||||
from litellm.types.utils import (
|
||||
AgenticLoopParams,
|
||||
BaseUrlRoutingHint,
|
||||
|
|
@ -399,6 +400,49 @@ def _resolve_control_options(kwargs: Mapping[str, object], model: str) -> Contro
|
|||
return assert_never(control)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SupplementalProviderParams:
|
||||
params: GenericLiteLLMParams
|
||||
dropped_keys: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _InvalidSupplementalParams:
|
||||
message: str
|
||||
param: str
|
||||
|
||||
|
||||
def _parse_supplemental_provider_params(
|
||||
kwargs: Mapping[str, object],
|
||||
) -> _SupplementalProviderParams | _InvalidSupplementalParams:
|
||||
supplemental: Final = {key: kwargs[key] for key in SUPPLEMENTAL_KWARGS_KEYS if key in kwargs}
|
||||
try:
|
||||
return _SupplementalProviderParams(params=validate_litellm_params_strictly(GenericLiteLLMParams, supplemental))
|
||||
except ValidationError as error:
|
||||
invalid: Final = tuple(detail for detail in error.errors() if detail["loc"])
|
||||
invalid_keys: Final = frozenset(str(detail["loc"][0]) for detail in invalid)
|
||||
if litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True:
|
||||
return _SupplementalProviderParams(
|
||||
params=GenericLiteLLMParams.model_validate(
|
||||
{key: value for key, value in supplemental.items() if key not in invalid_keys}
|
||||
),
|
||||
dropped_keys=invalid_keys,
|
||||
)
|
||||
return _InvalidSupplementalParams(
|
||||
message="; ".join(f"{detail['loc'][0]}: {detail['msg']}" for detail in invalid),
|
||||
param=", ".join(sorted(invalid_keys)),
|
||||
)
|
||||
|
||||
|
||||
def _supplemental_provider_params(kwargs: Mapping[str, object], model: str) -> _SupplementalProviderParams:
|
||||
parsed: Final = _parse_supplemental_provider_params(kwargs)
|
||||
if isinstance(parsed, _InvalidSupplementalParams):
|
||||
raise litellm.BadRequestError(
|
||||
message=parsed.message, model=model, llm_provider=None, body={"param": parsed.param}
|
||||
)
|
||||
return parsed
|
||||
|
||||
|
||||
PROVIDERS_WITHOUT_DOCUMENT_INPUT: Final = frozenset({"ollama", "ollama_chat"})
|
||||
PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT: Final = frozenset({"azure_ai"})
|
||||
|
||||
|
|
@ -830,21 +874,17 @@ async def acompletion(
|
|||
"enable_json_schema_validation": enable_json_schema_validation,
|
||||
}
|
||||
if custom_llm_provider is None:
|
||||
_supplemental: Final = cast( # cast-ok: kwargs values are request-scoped objects
|
||||
"dict[str, object]",
|
||||
{
|
||||
**{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs},
|
||||
"extra_headers": extra_headers,
|
||||
},
|
||||
)
|
||||
request_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: untyped request kwargs
|
||||
_, custom_llm_provider, _, _ = get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=cast( # cast-ok: api_base arrives as a str in the request kwargs
|
||||
"str | None", kwargs.get("api_base")
|
||||
"str | None", request_kwargs.get("api_base")
|
||||
)
|
||||
or base_url,
|
||||
litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None),
|
||||
litellm_params=_supplemental_provider_params(
|
||||
_REQUEST_KWARGS_ADAPTER.validate_python({**kwargs, "extra_headers": extra_headers}), model
|
||||
).params,
|
||||
)
|
||||
|
||||
if fallbacks is not None:
|
||||
|
|
@ -4382,6 +4422,7 @@ _ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str])
|
|||
_OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
_OLLAMA_EMBEDDING_HEADERS_ADAPTER: Final = TypeAdapter(dict[str, str])
|
||||
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
|
||||
_BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint)
|
||||
_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object)
|
||||
|
|
@ -5625,6 +5666,11 @@ def completion(
|
|||
|
||||
######### unpacking kwargs #####################
|
||||
args: Final = _locals_snapshot(locals())
|
||||
supplemental_params: Final = _supplemental_provider_params(
|
||||
_REQUEST_KWARGS_ADAPTER.validate_python({**kwargs, "extra_headers": extra_headers}), model
|
||||
)
|
||||
for dropped_key in supplemental_params.dropped_keys:
|
||||
kwargs.pop(dropped_key)
|
||||
|
||||
# Set by the responses->completion fallback so completion() does not bridge
|
||||
# back to the Responses API: that round-trip mutually recurses forever for a
|
||||
|
|
@ -5850,21 +5896,14 @@ def completion(
|
|||
if deployment_id is not None: # azure llms
|
||||
model = deployment_id
|
||||
custom_llm_provider = "azure"
|
||||
_supplemental_provider_params: Final = {
|
||||
**{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs},
|
||||
"extra_headers": extra_headers,
|
||||
}
|
||||
routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict
|
||||
requested_model: Final = model
|
||||
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
litellm_params=(
|
||||
GenericLiteLLMParams.model_validate(_supplemental_provider_params)
|
||||
if _supplemental_provider_params
|
||||
else None
|
||||
),
|
||||
litellm_params=supplemental_params.params,
|
||||
)
|
||||
_record_agentic_loop_params(
|
||||
logging_obj=logging,
|
||||
|
|
@ -5884,7 +5923,6 @@ def completion(
|
|||
api_base=api_base,
|
||||
deployment_model_info=deployment_model_info,
|
||||
)
|
||||
routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict
|
||||
drop_unsupported_reasoning: Final = (
|
||||
litellm.drop_params is True or normalize_drop_params(routing_kwargs.get("drop_params")) is True
|
||||
)
|
||||
|
|
@ -6220,6 +6258,7 @@ def completion(
|
|||
"github_copilot_user_session",
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
|
||||
"fireworks_forward_user_id",
|
||||
"thought_signature_in_tool_call_id",
|
||||
"responses_api_version",
|
||||
"allowed_openai_params",
|
||||
)
|
||||
|
|
@ -6809,6 +6848,23 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse:
|
|||
)
|
||||
|
||||
|
||||
def _embedding_headers(param: str, configured: object, model: str, drop_params: bool) -> Mapping[str, object]:
|
||||
if not configured:
|
||||
return {}
|
||||
if isinstance(configured, Mapping):
|
||||
return dict(cast(Mapping[str, object], configured)) # cast-ok: header names from untyped request JSON
|
||||
try:
|
||||
return dict(cast(Iterable[tuple[str, object]], configured)) # cast-ok: name and value pairs from request JSON
|
||||
except (TypeError, ValueError):
|
||||
if drop_params:
|
||||
return {}
|
||||
raise litellm.UnsupportedParamsError(
|
||||
message=f"{param} must be a JSON object of header names to values. "
|
||||
"To drop it from the call, set `litellm.drop_params = True`.",
|
||||
model=model,
|
||||
)
|
||||
|
||||
|
||||
# fmt: off
|
||||
|
||||
# Overload for when aembedding=True (returns coroutine)
|
||||
|
|
@ -6923,11 +6979,10 @@ def embedding(
|
|||
azure_ad_token_provider: Final = kwargs.get("azure_ad_token_provider", None)
|
||||
aembedding: Final[bool | None] = kwargs.get("aembedding", None)
|
||||
extra_headers: Final = kwargs.get("extra_headers", None)
|
||||
headers = kwargs.get("headers", None) or extra_headers
|
||||
if headers is None:
|
||||
headers = {}
|
||||
if extra_headers is not None:
|
||||
headers.update(extra_headers)
|
||||
headers = {
|
||||
**_embedding_headers("headers", kwargs.get("headers"), model, drop_unsupported_params),
|
||||
**_embedding_headers("extra_headers", extra_headers, model, drop_unsupported_params),
|
||||
}
|
||||
# Inject proxy auth headers if configured
|
||||
if litellm.proxy_auth is not None:
|
||||
try:
|
||||
|
|
@ -7091,7 +7146,7 @@ def embedding(
|
|||
# set API KEY
|
||||
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
|
||||
|
||||
if headers is not None and headers != {}:
|
||||
if headers:
|
||||
optional_params["extra_headers"] = headers
|
||||
|
||||
requested_encoding_format: Final = (
|
||||
|
|
@ -7183,14 +7238,15 @@ def embedding(
|
|||
custom_llm_provider == "openai_like"
|
||||
or custom_llm_provider == "llamafile"
|
||||
or custom_llm_provider == "lm_studio"
|
||||
or JSONProviderRegistry.supports_embeddings(custom_llm_provider)
|
||||
):
|
||||
api_base = api_base or litellm.api_base or get_secret_str("OPENAI_LIKE_API_BASE")
|
||||
|
||||
# set API KEY
|
||||
if api_key is None:
|
||||
api_key = api_key or litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY")
|
||||
if api_key is None and not JSONProviderRegistry.supports_embeddings(custom_llm_provider):
|
||||
api_key = litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY")
|
||||
|
||||
if headers is not None and headers != {}:
|
||||
if headers:
|
||||
optional_params["extra_headers"] = headers
|
||||
|
||||
## EMBEDDING CALL
|
||||
|
|
@ -7205,10 +7261,9 @@ def embedding(
|
|||
optional_params=optional_params,
|
||||
client=client,
|
||||
aembedding=aembedding,
|
||||
drop_params=drop_unsupported_params,
|
||||
)
|
||||
elif custom_llm_provider == "oci":
|
||||
if headers is None:
|
||||
headers = {}
|
||||
response = base_llm_http_handler.embedding(
|
||||
model=model,
|
||||
input=input,
|
||||
|
|
@ -7233,11 +7288,6 @@ def embedding(
|
|||
or litellm.api_key
|
||||
)
|
||||
|
||||
# Use the merged headers variable (already merged at the top of the function)
|
||||
# Don't overwrite it with just extra_headers
|
||||
if headers is None:
|
||||
headers = {}
|
||||
|
||||
response = base_llm_http_handler.embedding(
|
||||
model=model,
|
||||
input=input,
|
||||
|
|
|
|||
|
|
@ -39687,6 +39687,57 @@
|
|||
"/v1/images/generations"
|
||||
]
|
||||
},
|
||||
"hostinger_router/gpt-5.2": {
|
||||
"cache_read_input_token_cost": 0.00000021,
|
||||
"input_cost_per_token": 0.0000021,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 272000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.0000168,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/claude-sonnet-5": {
|
||||
"cache_creation_input_token_cost": 3.3e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 5.28e-06,
|
||||
"cache_read_input_token_cost": 2.64e-07,
|
||||
"input_cost_per_token": 2.64e-06,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.32e-05,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_response_schema": true,
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/messages"],
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/text-embedding-3-large": {
|
||||
"input_cost_per_token": 1.716e-07,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 8191,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/text-embedding-3-small": {
|
||||
"input_cost_per_token": 3e-08,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 8191,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
|
|
@ -51387,6 +51438,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 5e-06,
|
||||
"output_cost_per_token_batches": 2.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -51415,6 +51471,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 5e-06,
|
||||
"output_cost_per_token_batches": 2.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -52161,6 +52222,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -52266,6 +52332,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
|
|||
|
|
@ -1087,6 +1087,32 @@
|
|||
"generateContent": false
|
||||
}
|
||||
},
|
||||
"hostinger_router": {
|
||||
"display_name": "Hostinger Router (`hostinger_router`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/hostinger_router",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": true,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"crusoe": {
|
||||
"display_name": "Crusoe (`crusoe`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/crusoe",
|
||||
|
|
|
|||
|
|
@ -24268,7 +24268,14 @@
|
|||
"properties": {
|
||||
"content": {
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/OutputText"
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/OutputText"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/ResponseOutputRefusal"
|
||||
}
|
||||
]
|
||||
},
|
||||
"title": "Content",
|
||||
"type": "array"
|
||||
|
|
@ -58015,7 +58022,7 @@
|
|||
"paths": {
|
||||
"/toolset/{toolset_name}/mcp": {
|
||||
"delete": {
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
|
||||
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_delete",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
@ -58054,7 +58061,7 @@
|
|||
]
|
||||
},
|
||||
"get": {
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
|
||||
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_get",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
@ -58093,7 +58100,7 @@
|
|||
]
|
||||
},
|
||||
"head": {
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
|
||||
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_head",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
@ -58132,7 +58139,7 @@
|
|||
]
|
||||
},
|
||||
"options": {
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
|
||||
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_options",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
@ -58171,7 +58178,7 @@
|
|||
]
|
||||
},
|
||||
"patch": {
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
|
||||
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_patch",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
@ -58210,7 +58217,7 @@
|
|||
]
|
||||
},
|
||||
"post": {
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
|
||||
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_post",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
@ -58249,7 +58256,7 @@
|
|||
]
|
||||
},
|
||||
"put": {
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
|
||||
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -5031,8 +5031,9 @@ class ProxyBaseLLMRequestProcessing:
|
|||
if stream_usage is None:
|
||||
return None
|
||||
service_tier: Final = obj.get("service_tier")
|
||||
served_model: Final = litellm_logging_obj.model if litellm_logging_obj is not None else None
|
||||
cost_val: Final = ProxyBaseLLMRequestProcessing._streamed_usage_cost(
|
||||
ModelResponse(usage=stream_usage),
|
||||
ModelResponse(model=served_model, usage=stream_usage),
|
||||
model_name,
|
||||
service_tier if isinstance(service_tier, str) else None,
|
||||
litellm_logging_obj,
|
||||
|
|
|
|||
|
|
@ -12,18 +12,31 @@ from collections.abc import Mapping, Sequence
|
|||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details
|
||||
from litellm.llms.anthropic.messages.stream_assembly import anthropic_message_delta_fields
|
||||
from litellm.llms.anthropic.messages.stream_assembly import (
|
||||
anthropic_message_delta_fields,
|
||||
anthropic_sse_error_events,
|
||||
anthropic_sse_event_type,
|
||||
anthropic_sse_event_types,
|
||||
anthropic_sse_started_blocks,
|
||||
anthropic_sse_stopped_block_index,
|
||||
anthropic_sse_unfinished_blocks,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.stream_assembly import (
|
||||
assemble_anthropic_sse_stream as assemble_anthropic_sse_stream, # noqa: PLC0414 # public re-export
|
||||
)
|
||||
from litellm.llms.anthropic.messages.stream_assembly import (
|
||||
has_unreadable_json_event as has_unreadable_json_event, # noqa: PLC0414 # public re-export
|
||||
)
|
||||
from litellm.llms.anthropic.messages.stream_assembly import (
|
||||
is_anthropic_sse_stream as is_anthropic_sse_stream, # noqa: PLC0414 # public re-export
|
||||
)
|
||||
from litellm.llms.anthropic.messages.stream_assembly import (
|
||||
is_raw_sse_stream as is_raw_sse_stream, # noqa: PLC0414 # public re-export
|
||||
)
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
|
||||
from litellm.types.utils import Choices, ModelResponse
|
||||
|
||||
|
||||
|
|
@ -101,6 +114,48 @@ def _rewritten_message_delta_fields(
|
|||
)
|
||||
|
||||
|
||||
_MESSAGE_END_EVENT_TYPES: Final = frozenset({"message_delta", "message_stop"})
|
||||
|
||||
|
||||
def _text_block_left_open(rewritten_chunks: Sequence[bytes], source_chunks: Sequence[object]) -> int | None:
|
||||
"""Index of the rewritten text block to leave open when the failed upstream never stopped its text.
|
||||
|
||||
Assembly keeps a cut-off text block and drops every other cut-off block, and the round trip
|
||||
merges all text into one block, so that block carries text the upstream left open. It stays
|
||||
open only when it is the last block, since a client must see each block stop before the next
|
||||
one starts.
|
||||
"""
|
||||
upstream_left_text_open: Final = any(
|
||||
block.get("type") == "text" for block in anthropic_sse_unfinished_blocks(source_chunks).values()
|
||||
)
|
||||
rewritten_blocks: Final = anthropic_sse_started_blocks(rewritten_chunks)
|
||||
if not upstream_left_text_open or not rewritten_blocks:
|
||||
return None
|
||||
last_index: Final = max(rewritten_blocks)
|
||||
return last_index if rewritten_blocks[last_index].get("type") == "text" else None
|
||||
|
||||
|
||||
def _is_empty_text(block: object) -> bool:
|
||||
fields: Final = as_str_mapping(block)
|
||||
return fields is not None and fields.get("type") == "text" and not fields.get("text")
|
||||
|
||||
|
||||
def _without_empty_text(response: AnthropicMessagesResponse) -> AnthropicMessagesResponse:
|
||||
return {**response, "content": [block for block in response.get("content") or () if not _is_empty_text(block)]}
|
||||
|
||||
|
||||
def _sent_by_the_failed_upstream(chunk: bytes, block_left_open: int | None, end_events_sent: frozenset[str]) -> bool:
|
||||
event: Final = chunk.decode()
|
||||
event_type: Final = anthropic_sse_event_type(event)
|
||||
if event_type in _MESSAGE_END_EVENT_TYPES:
|
||||
return event_type in end_events_sent
|
||||
return block_left_open is None or anthropic_sse_stopped_block_index(event) != block_left_open
|
||||
|
||||
|
||||
def _sse_chunks_of_events(events: Sequence[str]) -> tuple[bytes, ...]:
|
||||
return tuple(f"{event}\n\n".encode() for event in events)
|
||||
|
||||
|
||||
def anthropic_sse_chunks_from_response(
|
||||
assembled: ModelResponse, source_chunks: Sequence[object] = (), *, keep_explanations: bool = False
|
||||
) -> tuple[bytes, ...]:
|
||||
|
|
@ -110,7 +165,26 @@ def anthropic_sse_chunks_from_response(
|
|||
nothing the OpenAI round trip made up, with a verdict kept only for a tool call delivered with
|
||||
the input the classifier judged. Their free-text ``explanation`` was written about the text the
|
||||
guardrail rewrote, so it is dropped unless the caller rewrites no text (``keep_explanations``).
|
||||
A source stream that ended in an upstream ``error`` event ends with that event again, never
|
||||
with a ``message_delta`` or ``message_stop`` the upstream did not send, with its cut-off text
|
||||
block left open unless another block follows it, and with no finished text block when it read
|
||||
no text, since Claude Code does not retry a stream that shows a finished block. A response that
|
||||
cannot be re-emitted, such as one with a tool input that cannot be parsed, leaves a failed stream
|
||||
its error alone.
|
||||
"""
|
||||
error_events: Final = anthropic_sse_error_events(source_chunks)
|
||||
try:
|
||||
return _rebuilt_sse_chunks(assembled, source_chunks, error_events, keep_explanations)
|
||||
except Exception:
|
||||
if not error_events:
|
||||
raise
|
||||
verbose_proxy_logger.warning("Failed Anthropic stream could not be rebuilt, sending its upstream error alone")
|
||||
return _sse_chunks_of_events(error_events)
|
||||
|
||||
|
||||
def _rebuilt_sse_chunks(
|
||||
assembled: ModelResponse, source_chunks: Sequence[object], error_events: Sequence[str], keep_explanations: bool
|
||||
) -> tuple[bytes, ...]:
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import (
|
||||
LiteLLMAnthropicMessagesAdapter,
|
||||
)
|
||||
|
|
@ -120,7 +194,7 @@ def anthropic_sse_chunks_from_response(
|
|||
|
||||
adapter: Final = LiteLLMAnthropicMessagesAdapter()
|
||||
anthropic_response: Final = adapter.translate_openai_response_to_anthropic(response=assembled)
|
||||
source_response: Final = assemble_anthropic_sse_stream(source_chunks)
|
||||
source_response: Final = assemble_anthropic_sse_stream(source_chunks, through_upstream_error=True)
|
||||
source_content: Final = (
|
||||
adapter.translate_openai_response_to_anthropic(response=source_response).get("content")
|
||||
if source_response is not None
|
||||
|
|
@ -129,11 +203,15 @@ def anthropic_sse_chunks_from_response(
|
|||
unchanged_tool_use_ids: Final = _unchanged_tool_use_ids(
|
||||
source_content or (), anthropic_response.get("content") or ()
|
||||
)
|
||||
return tuple(
|
||||
FakeAnthropicMessagesStreamIterator(
|
||||
response=anthropic_response,
|
||||
message_delta_fields=_rewritten_message_delta_fields(
|
||||
source_chunks, unchanged_tool_use_ids, keep_explanations
|
||||
),
|
||||
).chunks
|
||||
rewritten_chunks: Final = FakeAnthropicMessagesStreamIterator(
|
||||
response=_without_empty_text(anthropic_response) if error_events else anthropic_response,
|
||||
message_delta_fields=_rewritten_message_delta_fields(source_chunks, unchanged_tool_use_ids, keep_explanations),
|
||||
).chunks
|
||||
if not error_events:
|
||||
return tuple(rewritten_chunks)
|
||||
block_left_open: Final = _text_block_left_open(rewritten_chunks, source_chunks)
|
||||
end_events_sent: Final = _MESSAGE_END_EVENT_TYPES & anthropic_sse_event_types(source_chunks)
|
||||
return (
|
||||
*(chunk for chunk in rewritten_chunks if _sent_by_the_failed_upstream(chunk, block_left_open, end_events_sent)),
|
||||
*_sse_chunks_of_events(error_events),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames
|
|||
from litellm.proxy.guardrails.anthropic_sse import (
|
||||
anthropic_sse_chunks_from_response,
|
||||
assemble_anthropic_sse_stream,
|
||||
has_unreadable_json_event,
|
||||
is_anthropic_sse_stream,
|
||||
model_response_text,
|
||||
)
|
||||
|
|
@ -117,7 +118,7 @@ _MAX_FIRST_SSE_FRAME_BYTES: Final = 64 * 1024
|
|||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SsePreface:
|
||||
"""Complete leading SSE frames with no ``data:`` line, relayed verbatim before the stream shape is decided."""
|
||||
"""Complete leading SSE frames that carry no data, relayed verbatim before the stream shape is decided."""
|
||||
|
||||
raw: bytes
|
||||
|
||||
|
|
@ -125,17 +126,27 @@ class _SsePreface:
|
|||
_SSE_FRAME_END: Final = re.compile(rb"\r\n\r\n|\n\n|\r\r")
|
||||
|
||||
|
||||
def _carries_data(frame: bytes) -> bool:
|
||||
return any(line.startswith(b"data:") and line[len(b"data:") :].strip() for line in frame.splitlines())
|
||||
|
||||
|
||||
def _split_sse_preface(complete_frames: bytes) -> tuple[bytes, bytes]:
|
||||
"""Split complete frames into ``(frames before the first data-bearing frame, that frame and everything after)``."""
|
||||
start = 0
|
||||
for end in _SSE_FRAME_END.finditer(complete_frames):
|
||||
frame = complete_frames[start : end.end()]
|
||||
if any(line.startswith(b"data:") for line in frame.splitlines()):
|
||||
if _carries_data(frame):
|
||||
return complete_frames[:start], complete_frames[start:]
|
||||
start = end.end()
|
||||
return complete_frames, b""
|
||||
|
||||
|
||||
def _holds_json_object_frame(frames: bytes) -> bool:
|
||||
return any(
|
||||
line.startswith(b"data:") and line[len(b"data:") :].lstrip().startswith(b"{") for line in frames.splitlines()
|
||||
)
|
||||
|
||||
|
||||
def _flush_unmaskable_buffer(all_chunks: list[ModelResponseStream]) -> Iterator[ModelResponseStream]:
|
||||
"""Buffered chunks flushed unmasked when a mixed stream shape makes reconstruction impossible."""
|
||||
if not all_chunks:
|
||||
|
|
@ -150,12 +161,13 @@ def _flush_unmaskable_buffer(all_chunks: list[ModelResponseStream]) -> Iterator[
|
|||
|
||||
async def _coalesce_first_sse_frame(stream: AsyncIterator[object]) -> AsyncGenerator[object, None]:
|
||||
"""
|
||||
Relay leading data-less SSE frames (comment keepalives, events without a
|
||||
``data:`` line) as they complete, and join raw ``bytes`` chunks until they
|
||||
hold one complete SSE event with a data line, so the stream shape is
|
||||
decided on a whole frame rather than a transport fragment. Everything
|
||||
after that first frame is forwarded untouched. The byte cap can only be
|
||||
reached by a single unterminated frame.
|
||||
Relay leading data-less SSE frames (comment keepalives, events with no
|
||||
data) as they complete, and join raw ``bytes`` chunks until they
|
||||
hold one complete SSE event with a JSON object on its data line, so the
|
||||
stream shape is decided on a whole frame rather than a transport fragment
|
||||
or a line that says nothing about the shape. Everything after that frame
|
||||
is forwarded untouched. The byte cap bounds a single unterminated frame
|
||||
and a run of frames with no JSON object.
|
||||
"""
|
||||
pending = b""
|
||||
try:
|
||||
|
|
@ -169,7 +181,7 @@ async def _coalesce_first_sse_frame(stream: AsyncIterator[object]) -> AsyncGener
|
|||
if preface:
|
||||
yield _SsePreface(preface)
|
||||
pending = classifiable + tail
|
||||
if classifiable or len(pending) >= _MAX_FIRST_SSE_FRAME_BYTES:
|
||||
if _holds_json_object_frame(classifiable) or len(pending) >= _MAX_FIRST_SSE_FRAME_BYTES:
|
||||
break
|
||||
else:
|
||||
if pending:
|
||||
|
|
@ -1523,7 +1535,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
|
|||
first_frame_is_anthropic = (
|
||||
not passthrough_due_to_unknown_stream_shape
|
||||
and not all_chunks
|
||||
and is_anthropic_sse_stream((chunk,))
|
||||
and self._is_anthropic_first_frame(chunk)
|
||||
)
|
||||
if not first_frame_is_anthropic:
|
||||
passthrough_due_to_unknown_stream_shape = (
|
||||
|
|
@ -1565,13 +1577,29 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
|
|||
for chunk in all_chunks:
|
||||
yield chunk
|
||||
|
||||
def _is_anthropic_first_frame(self, chunk: bytes) -> bool:
|
||||
if is_anthropic_sse_stream((chunk,)):
|
||||
return True
|
||||
if has_unreadable_json_event((chunk,)):
|
||||
raise self._unreadable_stream_refusal()
|
||||
return False
|
||||
|
||||
def _unreadable_stream_refusal(self) -> GuardrailRaisedException:
|
||||
return GuardrailRaisedException(
|
||||
guardrail_name=self.guardrail_name,
|
||||
message="Presidio could not read the upstream stream; cannot verify PII when PII protection is configured",
|
||||
should_wrap_with_default_message=False,
|
||||
)
|
||||
|
||||
async def _mask_anthropic_sse_stream(
|
||||
self, first_chunk: bytes, rest: AsyncIterator[object], request_data: dict
|
||||
) -> tuple[object, ...]:
|
||||
rest_chunks: Final = [chunk async for chunk in rest]
|
||||
chunks: Final = (first_chunk, *rest_chunks)
|
||||
assembled: Final = assemble_anthropic_sse_stream(chunks, restore_identity=True)
|
||||
assembled: Final = assemble_anthropic_sse_stream(chunks, restore_identity=True, through_upstream_error=True)
|
||||
if assembled is None:
|
||||
if has_unreadable_json_event(chunks):
|
||||
raise self._unreadable_stream_refusal()
|
||||
verbose_proxy_logger.warning(
|
||||
"Presidio apply_to_output: raw SSE stream could not be assembled into a response. "
|
||||
"Output PII masking was skipped for this response."
|
||||
|
|
|
|||
|
|
@ -25,8 +25,10 @@ from litellm.proxy.common_utils.callback_utils import (
|
|||
)
|
||||
from litellm.proxy.guardrails.anthropic_sse import (
|
||||
anthropic_sse_chunks_from_response,
|
||||
anthropic_sse_error_events,
|
||||
assemble_anthropic_sse_stream,
|
||||
is_raw_sse_stream,
|
||||
is_sse_error_stream,
|
||||
)
|
||||
from litellm.types.guardrails import GuardrailEventHooks, LitellmParams
|
||||
from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import (
|
||||
|
|
@ -333,7 +335,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
|
|||
parsed_arguments = arguments
|
||||
else:
|
||||
return None, "arguments must be a JSON object"
|
||||
except (json.JSONDecodeError, TypeError) as exc:
|
||||
except (json.JSONDecodeError, TypeError, RecursionError) as exc:
|
||||
verbose_proxy_logger.warning(
|
||||
"Tool Permission Guardrail: Failed to decode arguments for tool %s: %s",
|
||||
tool_call.function.name,
|
||||
|
|
@ -957,9 +959,11 @@ class ToolPermissionGuardrail(CustomGuardrail):
|
|||
yield chunk
|
||||
return
|
||||
|
||||
anthropic_response: Final = assemble_anthropic_sse_stream(all_chunks)
|
||||
anthropic_response: Final = assemble_anthropic_sse_stream(
|
||||
all_chunks, restore_identity=bool(anthropic_sse_error_events(all_chunks)), through_upstream_error=True
|
||||
)
|
||||
if anthropic_response is None:
|
||||
if is_raw_sse_stream(all_chunks):
|
||||
if is_raw_sse_stream(all_chunks) and not is_sse_error_stream(all_chunks):
|
||||
raise GuardrailRaisedException(
|
||||
guardrail_name=self.guardrail_name,
|
||||
message=(
|
||||
|
|
@ -971,16 +975,24 @@ class ToolPermissionGuardrail(CustomGuardrail):
|
|||
yield chunk
|
||||
return
|
||||
|
||||
anthropic_denials: Final = self._check_assembled_stream(anthropic_response)
|
||||
if not anthropic_denials:
|
||||
for chunk in all_chunks:
|
||||
yield chunk
|
||||
return
|
||||
|
||||
self._modify_response_with_permission_errors(anthropic_response, anthropic_denials)
|
||||
for sse_chunk in anthropic_sse_chunks_from_response(anthropic_response, all_chunks, keep_explanations=True):
|
||||
for sse_chunk in self._anthropic_stream_to_deliver(anthropic_response, all_chunks):
|
||||
yield sse_chunk
|
||||
|
||||
def _anthropic_stream_to_deliver(
|
||||
self, anthropic_response: ModelResponse, source_chunks: Sequence[object]
|
||||
) -> Sequence[object]:
|
||||
"""The frames the client gets: on a failed upstream only what the check read, never the upstream's own.
|
||||
|
||||
Assembly reads past an upstream error, and a client can read a frame of a broken stream that assembly
|
||||
does not, so forwarding those frames would pass a tool call nobody checked.
|
||||
"""
|
||||
denials: Final = self._check_assembled_stream(anthropic_response)
|
||||
if denials:
|
||||
self._modify_response_with_permission_errors(anthropic_response, denials)
|
||||
elif not anthropic_sse_error_events(source_chunks):
|
||||
return source_chunks
|
||||
return anthropic_sse_chunks_from_response(anthropic_response, source_chunks, keep_explanations=True)
|
||||
|
||||
def _check_assembled_stream(
|
||||
self, assembled: ModelResponse
|
||||
) -> tuple[tuple[ChatCompletionMessageToolCall, PermissionError], ...]:
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from litellm.proxy.auth.auth_utils import (
|
|||
)
|
||||
from litellm.proxy.auth.budget_throttle import throttled_limit
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
|
||||
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit, socket_session_usage
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
||||
from litellm.types.utils import CallTypes, Usage
|
||||
|
||||
|
|
@ -41,10 +41,12 @@ def _precise_minute(now: datetime) -> str:
|
|||
return now.strftime("%Y-%m-%d-%H-%M")
|
||||
|
||||
|
||||
def _response_total_tokens(response_obj: object) -> int:
|
||||
if not isinstance(response_obj, (ModelResponse, EmbeddingResponse, TextCompletionResponse)):
|
||||
return 0
|
||||
response_usage: Final = getattr(response_obj, "usage", None)
|
||||
def _response_total_tokens(kwargs: Mapping[str, object], response_obj: object) -> int:
|
||||
response_usage: Final = (
|
||||
getattr(response_obj, "usage", None)
|
||||
if isinstance(response_obj, (ModelResponse, EmbeddingResponse, TextCompletionResponse))
|
||||
else socket_session_usage(kwargs, response_obj)
|
||||
)
|
||||
return response_usage.total_tokens if isinstance(response_usage, Usage) else 0
|
||||
|
||||
|
||||
|
|
@ -556,7 +558,10 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
|
|||
|
||||
precise_minute: Final = _precise_minute(self._clock())
|
||||
|
||||
total_tokens: int = _response_total_tokens(response_obj)
|
||||
total_tokens: Final = _response_total_tokens(
|
||||
kwargs, # pyright: ignore[reportUnknownArgumentType] # legacy callback kwargs are untyped
|
||||
response_obj,
|
||||
)
|
||||
|
||||
# ------------
|
||||
# Update usage - API Key
|
||||
|
|
@ -630,8 +635,6 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
|
|||
# Update usage - User
|
||||
# ------------
|
||||
if user_api_key_user_id is not None:
|
||||
total_tokens = _response_total_tokens(response_obj)
|
||||
|
||||
request_count_api_key = f"{user_api_key_user_id}::{precise_minute}::request_count"
|
||||
|
||||
current = await self.internal_usage_cache.async_get_cache(
|
||||
|
|
@ -639,7 +642,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
|
|||
litellm_parent_otel_span=litellm_parent_otel_span,
|
||||
) or {
|
||||
"current_requests": 1,
|
||||
"current_tpm": total_tokens,
|
||||
"current_tpm": 0,
|
||||
"current_rpm": 1,
|
||||
}
|
||||
|
||||
|
|
@ -656,8 +659,6 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
|
|||
# Update usage - Team
|
||||
# ------------
|
||||
if user_api_key_team_id is not None:
|
||||
total_tokens = _response_total_tokens(response_obj)
|
||||
|
||||
request_count_api_key = f"{user_api_key_team_id}::{precise_minute}::request_count"
|
||||
|
||||
current = await self.internal_usage_cache.async_get_cache(
|
||||
|
|
@ -665,7 +666,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
|
|||
litellm_parent_otel_span=litellm_parent_otel_span,
|
||||
) or {
|
||||
"current_requests": 1,
|
||||
"current_tpm": total_tokens,
|
||||
"current_tpm": 0,
|
||||
"current_rpm": 1,
|
||||
}
|
||||
|
||||
|
|
@ -682,8 +683,6 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
|
|||
# Update usage - End User
|
||||
# ------------
|
||||
if user_api_key_end_user_id is not None:
|
||||
total_tokens = _response_total_tokens(response_obj)
|
||||
|
||||
request_count_api_key = f"{user_api_key_end_user_id}::{precise_minute}::request_count"
|
||||
|
||||
current = await self.internal_usage_cache.async_get_cache(
|
||||
|
|
@ -691,7 +690,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
|
|||
litellm_parent_otel_span=litellm_parent_otel_span,
|
||||
) or {
|
||||
"current_requests": 1,
|
||||
"current_tpm": total_tokens,
|
||||
"current_tpm": 0,
|
||||
"current_rpm": 1,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -46,8 +46,10 @@ from litellm.constants import (
|
|||
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
NO_MAX_TOKENS_TPM_RESERVATION_FRACTION,
|
||||
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY,
|
||||
)
|
||||
from litellm.cost_calculator import get_usage_object
|
||||
from litellm.exceptions import ServiceUnavailableError
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
||||
ensure_response_additional_headers,
|
||||
|
|
@ -85,7 +87,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import (
|
|||
batch_response_view,
|
||||
canonical_provider_batch_id,
|
||||
)
|
||||
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
|
||||
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit, socket_session_usage
|
||||
from litellm.proxy.spend_tracking.input_tokens import (
|
||||
inline_pdf_file_count,
|
||||
models_counted_for,
|
||||
|
|
@ -745,6 +747,7 @@ class RequestRateLimiterStash:
|
|||
batch_tpd_refund_ops: tuple[ReservationAwareIncrementOperation, ...] = ()
|
||||
reservation_released: bool = False
|
||||
tpm_limited_tags: frozenset[str] = field(default_factory=frozenset)
|
||||
socket_agent_windows: tuple[RateLimitDescriptor, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -864,6 +867,62 @@ def _call_id_from_callback_kwargs(kwargs: object) -> str | None:
|
|||
return call_id if isinstance(call_id, str) else None
|
||||
|
||||
|
||||
def _socket_connection_call_id(kwargs: Mapping[str, object]) -> object:
|
||||
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params"))
|
||||
litellm_metadata: Final = None if litellm_params is None else as_str_mapping(litellm_params.get("litellm_metadata"))
|
||||
return None if litellm_metadata is None else litellm_metadata.get(RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY)
|
||||
|
||||
|
||||
def _is_socket_turn(kwargs: Mapping[str, object]) -> bool:
|
||||
if kwargs.get("call_type") == CallTypes.aresponses_websocket.value:
|
||||
return True
|
||||
connection_stash: Final = get_request_stash()
|
||||
connection_call_id: Final = _socket_connection_call_id(kwargs)
|
||||
return (
|
||||
connection_stash is not None
|
||||
and connection_call_id is not None
|
||||
and connection_call_id == connection_stash.owner_litellm_call_id
|
||||
)
|
||||
|
||||
|
||||
_AGENT_DESCRIPTOR_KEYS: Final = frozenset({"agent", "agent_session"})
|
||||
|
||||
_SOCKET_TURN_UNCOUNTED_DESCRIPTOR_KEYS: Final = frozenset(
|
||||
{PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY, PTU_TEAM_DESCRIPTOR_KEY}
|
||||
)
|
||||
|
||||
|
||||
def _socket_turn_tpm_windows(descriptors: Sequence[RateLimitDescriptor]) -> tuple[RateLimitDescriptor, ...]:
|
||||
return tuple(
|
||||
RateLimitDescriptor(
|
||||
key=descriptor["key"],
|
||||
value=descriptor["value"],
|
||||
rate_limit=RateLimitDescriptorRateLimitObject(
|
||||
tokens_per_unit=tokens_per_unit, window_size=(descriptor.get("rate_limit") or {}).get("window_size")
|
||||
),
|
||||
)
|
||||
for descriptor in descriptors
|
||||
if descriptor["key"] not in _SOCKET_TURN_UNCOUNTED_DESCRIPTOR_KEYS
|
||||
and (tokens_per_unit := (descriptor.get("rate_limit") or {}).get("tokens_per_unit")) is not None
|
||||
)
|
||||
|
||||
|
||||
def _unmodeled_response_usage(kwargs: Mapping[str, object], response_obj: object) -> Usage | None:
|
||||
combined_usage: Final = kwargs.get("combined_usage_object")
|
||||
if isinstance(combined_usage, Usage):
|
||||
return combined_usage
|
||||
return socket_session_usage(kwargs, response_obj)
|
||||
|
||||
|
||||
def _tpm_limited_tags(call_stash: RequestRateLimiterStash | None, kwargs: Mapping[str, object]) -> frozenset[str]:
|
||||
if call_stash is not None:
|
||||
return call_stash.tpm_limited_tags
|
||||
connection_stash: Final = get_request_stash()
|
||||
if connection_stash is None or not _is_socket_turn(kwargs):
|
||||
return frozenset()
|
||||
return connection_stash.tpm_limited_tags
|
||||
|
||||
|
||||
def _parse_output_cap_value(raw_value: object) -> int | None:
|
||||
if isinstance(raw_value, bool) or not isinstance(raw_value, (int, float, str)):
|
||||
return None
|
||||
|
|
@ -2967,6 +3026,40 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
if response["overall_code"] == "OVER_LIMIT":
|
||||
self._handle_rate_limit_error(response, descriptors, requested_model)
|
||||
|
||||
@with_service_target("rate_limits")
|
||||
async def enforce_token_limits_for_frame(
|
||||
self, user_api_key_dict: UserAPIKeyAuth, requested_model: str | None
|
||||
) -> None:
|
||||
connection_stash: Final = get_request_stash()
|
||||
tags: Final = sorted(connection_stash.tpm_limited_tags) if connection_stash is not None else []
|
||||
agent_windows: Final = connection_stash.socket_agent_windows if connection_stash is not None else ()
|
||||
hidden_stash: Final = _request_stash.set(None)
|
||||
try:
|
||||
descriptors: Final = await self._build_request_rate_limit_descriptors(
|
||||
user_api_key_dict,
|
||||
MappingProxyType({"model": requested_model, "metadata": {"tags": tags}}),
|
||||
CallTypes.aresponses_websocket.value,
|
||||
)
|
||||
finally:
|
||||
_request_stash.reset(hidden_stash)
|
||||
built_scopes: Final = frozenset((d["key"], d["value"]) for d in descriptors)
|
||||
tpm_windows: Final = _socket_turn_tpm_windows(
|
||||
(*descriptors, *(d for d in agent_windows if (d["key"], d["value"]) not in built_scopes))
|
||||
)
|
||||
try:
|
||||
response: Final = await self.should_rate_limit(
|
||||
tpm_windows,
|
||||
parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
read_only=True,
|
||||
skip_parallel_check=True,
|
||||
)
|
||||
except RateLimitUnverifiableError as unverifiable:
|
||||
raise ServiceUnavailableError(
|
||||
message=RATE_LIMIT_UNVERIFIABLE_MESSAGE, llm_provider="litellm_proxy", model=requested_model or ""
|
||||
) from unverifiable
|
||||
if response["overall_code"] == "OVER_LIMIT":
|
||||
self._handle_rate_limit_error(response, tpm_windows, requested_model)
|
||||
|
||||
def _rate_limited_model(self, requested_model: str | None) -> RateLimitedModel | None:
|
||||
if not requested_model:
|
||||
return None
|
||||
|
|
@ -3777,7 +3870,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
def _handle_rate_limit_error(
|
||||
self,
|
||||
response: RateLimitResponse,
|
||||
descriptors: list[RateLimitDescriptor],
|
||||
descriptors: Sequence[RateLimitDescriptor],
|
||||
requested_model: str | None = None,
|
||||
) -> None:
|
||||
"""Handle rate limit exceeded by raising :class:`ProxyRateLimitError` (a 429)."""
|
||||
|
|
@ -4329,6 +4422,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
for d in descriptors
|
||||
if d["key"] == "tag" and d["rate_limit"] is not None and d["rate_limit"].get("tokens_per_unit") is not None
|
||||
)
|
||||
stash.socket_agent_windows = tuple(d for d in descriptors if d["key"] in _AGENT_DESCRIPTOR_KEYS)
|
||||
|
||||
# Only check rate limits if we have descriptors with actual limits
|
||||
if descriptors:
|
||||
|
|
@ -5286,9 +5380,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
# Get total tokens from response. Responses LiteLLM does not model
|
||||
# (e.g. pass-through, whose usage is reported by the upstream rather
|
||||
# than parsed out of the body) carry their usage in
|
||||
# ``combined_usage_object`` instead, and would otherwise never charge
|
||||
# the TPM window.
|
||||
_combined_usage: Final = kwargs.get("combined_usage_object")
|
||||
# ``combined_usage_object`` instead (a Responses WebSocket turn in its
|
||||
# raw events) and would otherwise never charge the TPM window.
|
||||
_usage: Final[Usage | dict[str, object] | None] = (
|
||||
getattr(response_obj, "usage", None)
|
||||
if isinstance(
|
||||
|
|
@ -5302,9 +5395,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
else get_usage_object(response_obj)
|
||||
if isinstance(response_obj, ExtractionResponse)
|
||||
else _combined_usage
|
||||
if isinstance(_combined_usage, Usage)
|
||||
else None
|
||||
else _unmodeled_response_usage(kwargs, response_obj)
|
||||
)
|
||||
total_tokens = self._get_total_tokens_from_usage(usage=_usage, rate_limit_type=rate_limit_type)
|
||||
if total_tokens == 0:
|
||||
|
|
@ -5337,7 +5428,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
targets: Final = self._collect_tpm_scope_targets(
|
||||
standard_logging_metadata=standard_logging_metadata,
|
||||
kwargs=kwargs,
|
||||
tpm_limited_tags=stash.tpm_limited_tags if stash is not None else frozenset(),
|
||||
tpm_limited_tags=_tpm_limited_tags(stash, kwargs),
|
||||
model_group=reconcile_model.group if reconcile_model is not None else None,
|
||||
)
|
||||
targets.extend(
|
||||
|
|
@ -5365,16 +5456,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
reserved_tokens=reserved_tokens,
|
||||
)
|
||||
)
|
||||
ptu_skips_socket_usage: Final = kwargs.get("call_type") == CallTypes.aresponses_websocket.value
|
||||
pipeline_operations.extend(
|
||||
self._build_team_ptu_tpm_ops(
|
||||
standard_logging_metadata, # pyright: ignore[reportUnknownArgumentType] # untyped logging metadata
|
||||
usage_source=response_obj if _usage is None else _usage,
|
||||
usage_source=response_obj if _usage is None or ptu_skips_socket_usage else _usage,
|
||||
reconcile_model=reconcile_model,
|
||||
reserved_scopes=reserved_scopes,
|
||||
reserved_ceiling=stash.ptu_ceiling if stash is not None else None,
|
||||
reserved_tokens=stash.ptu_reserved_tokens if stash is not None else 0,
|
||||
reserved_windows=stash.ptu_reserved_window_identities if stash is not None else frozenset(),
|
||||
total_tokens=total_tokens,
|
||||
total_tokens=0 if ptu_skips_socket_usage else total_tokens,
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -5469,27 +5561,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
else None,
|
||||
)
|
||||
|
||||
pipeline_operations: Final = self._build_success_event_pipeline_operations(
|
||||
kwargs=kwargs,
|
||||
response_obj=response_obj,
|
||||
rate_limit_type=rate_limit_type,
|
||||
await self._apply_usage_operations(
|
||||
self._build_success_event_pipeline_operations(
|
||||
kwargs=kwargs,
|
||||
response_obj=response_obj,
|
||||
rate_limit_type=rate_limit_type,
|
||||
),
|
||||
litellm_parent_otel_span,
|
||||
)
|
||||
unguarded_operations: Final = [
|
||||
operation for operation in pipeline_operations if _as_window_guarded(operation) is None
|
||||
]
|
||||
window_guarded_operations: Final = tuple(
|
||||
guarded for operation in pipeline_operations if (guarded := _as_window_guarded(operation)) is not None
|
||||
)
|
||||
if unguarded_operations:
|
||||
await self.async_increment_tokens_with_ttl_preservation(
|
||||
pipeline_operations=unguarded_operations,
|
||||
parent_otel_span=litellm_parent_otel_span,
|
||||
)
|
||||
if window_guarded_operations:
|
||||
await self.async_increment_reservation_aware_tokens(
|
||||
pipeline_operations=window_guarded_operations,
|
||||
parent_otel_span=litellm_parent_otel_span,
|
||||
)
|
||||
io_token_operations: Final = self._build_io_token_reservation_ops(
|
||||
kwargs=kwargs,
|
||||
response_obj=response_obj,
|
||||
|
|
@ -5509,6 +5588,26 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error in rate limit success event: %s", e)
|
||||
|
||||
async def _apply_usage_operations(
|
||||
self, pipeline_operations: Sequence[RedisPipelineIncrementOperation], parent_otel_span: Span | None
|
||||
) -> None:
|
||||
unguarded_operations: Final = [
|
||||
operation for operation in pipeline_operations if _as_window_guarded(operation) is None
|
||||
]
|
||||
window_guarded_operations: Final = tuple(
|
||||
guarded for operation in pipeline_operations if (guarded := _as_window_guarded(operation)) is not None
|
||||
)
|
||||
if unguarded_operations:
|
||||
await self.async_increment_tokens_with_ttl_preservation(
|
||||
pipeline_operations=unguarded_operations,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
if window_guarded_operations:
|
||||
await self.async_increment_reservation_aware_tokens(
|
||||
pipeline_operations=window_guarded_operations,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
|
||||
@with_service_target("rate_limits")
|
||||
async def async_logging_hook(
|
||||
self,
|
||||
|
|
@ -5622,11 +5721,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
|
||||
try:
|
||||
litellm_parent_otel_span: Final[Span | None] = get_parent_otel_span_from_kwargs(kwargs)
|
||||
failed_call: Final = _callback_kwargs_mapping(kwargs)
|
||||
litellm_parent_otel_span: Final[Span | None] = get_parent_otel_span_from_kwargs(failed_call)
|
||||
|
||||
pipeline_operations: Final[list[RedisPipelineIncrementOperation]] = []
|
||||
|
||||
stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs))
|
||||
stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(failed_call))
|
||||
await self._release_stashed_parallel_slot(stash, litellm_parent_otel_span, in_logging_callback=True)
|
||||
|
||||
# Skip the reservation refund if async_post_call_failure_hook
|
||||
|
|
@ -5638,7 +5738,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
if stash is None or stash.reservation_released
|
||||
else (stash.reserved_tokens, stash.itpm_reserved_tokens, stash.otpm_reserved_tokens)
|
||||
)
|
||||
tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(kwargs)
|
||||
tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(failed_call)
|
||||
|
||||
if stash is not None and reserved_tokens > 0:
|
||||
verbose_proxy_logger.debug(
|
||||
|
|
@ -5722,6 +5822,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
if stash is not None and (reserved_tokens > 0 or itpm_reserved > 0 or otpm_reserved > 0):
|
||||
stash.reservation_released = True
|
||||
if stash is None and tpm_actual > 0 and _is_socket_turn(failed_call):
|
||||
await self._apply_usage_operations(
|
||||
self._build_success_event_pipeline_operations(
|
||||
kwargs=dict(failed_call), response_obj=None, rate_limit_type=self.get_rate_limit_type()
|
||||
),
|
||||
litellm_parent_otel_span,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error in rate limit failure event: %s", e)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,14 +2,32 @@
|
|||
Shared utility functions for rate limiter hooks.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import PROXY_LLM_PROVIDER_FALLBACK
|
||||
from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor
|
||||
from litellm.litellm_core_utils.core_helpers import iter_items
|
||||
from litellm.types.router import ModelGroupInfo
|
||||
from litellm.types.utils import PriorityReservationDict
|
||||
from litellm.types.utils import CallTypes, PriorityReservationDict, Usage
|
||||
|
||||
_SOCKET_SESSION_EVENTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
|
||||
|
||||
def socket_session_usage(kwargs: Mapping[str, object], response_obj: object) -> Usage | None:
|
||||
if not isinstance(response_obj, list):
|
||||
return None
|
||||
match kwargs.get("call_type"):
|
||||
case CallTypes.aresponses_websocket.value:
|
||||
return ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(
|
||||
_SOCKET_SESSION_EVENTS.validate_python(response_obj)
|
||||
)
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def resolve_llm_provider_for_rate_limit(
|
||||
|
|
|
|||
|
|
@ -1,14 +1,15 @@
|
|||
# What is this?
|
||||
## Helper utils for the management endpoints (keys/users/teams)
|
||||
from collections.abc import Callable, Mapping, MutableMapping, Sequence
|
||||
from collections.abc import Awaitable, Callable, Mapping, MutableMapping, Sequence
|
||||
from collections.abc import Set as AbstractSet
|
||||
from datetime import datetime
|
||||
from functools import wraps
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, Protocol
|
||||
from typing import Any, Final, ParamSpec, Protocol, TypeVar
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
import litellm
|
||||
from litellm._internal_context import with_service_target
|
||||
|
|
@ -687,6 +688,7 @@ async def _emit_management_endpoint_otel_span(
|
|||
end_time: datetime,
|
||||
result: Any = None,
|
||||
exception: Exception | None = None,
|
||||
log_payload: bool = True,
|
||||
) -> None:
|
||||
"""Stamp + end the parent OTEL SERVER span for a management endpoint.
|
||||
|
||||
|
|
@ -716,8 +718,8 @@ async def _emit_management_endpoint_otel_span(
|
|||
)
|
||||
|
||||
route = get_request_route(http_request)
|
||||
request_body: dict = await read_request_body( # rebind-ok: pre-existing rebinding on a rename-only line
|
||||
request=http_request
|
||||
request_body: dict = ( # rebind-ok: pre-existing rebinding on a rename-only line
|
||||
await read_request_body(request=http_request) if log_payload else {}
|
||||
)
|
||||
else:
|
||||
route = func.__name__
|
||||
|
|
@ -738,7 +740,7 @@ async def _emit_management_endpoint_otel_span(
|
|||
)
|
||||
|
||||
_response: dict[str, object] | None = None
|
||||
if exception is None and result is not None:
|
||||
if log_payload and exception is None and result is not None:
|
||||
try:
|
||||
raw: Final[Mapping[str, object]] = dict(result)
|
||||
_response = {k: v for k, v in raw.items() if k not in _CREDENTIAL_FIELDS}
|
||||
|
|
@ -767,37 +769,60 @@ async def _emit_management_endpoint_otel_span(
|
|||
)
|
||||
|
||||
|
||||
def management_endpoint_wrapper(func):
|
||||
_P: Final = ParamSpec("_P")
|
||||
_R: Final = TypeVar("_R")
|
||||
|
||||
|
||||
def management_endpoint_wrapper(func: Callable[_P, Awaitable[_R]]) -> Callable[_P, Awaitable[_R]]:
|
||||
"""
|
||||
This wrapper does the following:
|
||||
|
||||
1. Log I/O, Exceptions to OTEL
|
||||
2. Create an Audit log for success calls
|
||||
"""
|
||||
return _wrap_management_endpoint(func, log_payload=True)
|
||||
|
||||
@wraps(func)
|
||||
async def wrapper(*args, **kwargs):
|
||||
|
||||
def management_endpoint_span_wrapper(func: Callable[_P, Awaitable[_R]]) -> Callable[_P, Awaitable[_R]]:
|
||||
return _wrap_management_endpoint(func, log_payload=False)
|
||||
|
||||
|
||||
def _caller_auth(endpoint_kwargs: Mapping[str, object]) -> UserAPIKeyAuth:
|
||||
caller: Final = endpoint_kwargs.get("user_api_key_dict")
|
||||
return caller if isinstance(caller, UserAPIKeyAuth) else UserAPIKeyAuth()
|
||||
|
||||
|
||||
def _proxy_exception_handler_ends_server_span(exception: Exception) -> bool:
|
||||
return not isinstance(exception, StarletteHTTPException)
|
||||
|
||||
|
||||
def _wrap_management_endpoint(
|
||||
endpoint: Callable[_P, Awaitable[_R]], *, log_payload: bool
|
||||
) -> Callable[_P, Awaitable[_R]]:
|
||||
@wraps(endpoint)
|
||||
async def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R:
|
||||
start_time: Final = datetime.now()
|
||||
try:
|
||||
result: Final = await func(*args, **kwargs)
|
||||
result: Final = await endpoint(*args, **kwargs)
|
||||
end_time = datetime.now()
|
||||
try:
|
||||
user_api_key_dict: UserAPIKeyAuth = kwargs.get("user_api_key_dict") or UserAPIKeyAuth()
|
||||
user_api_key_dict: UserAPIKeyAuth = _caller_auth(kwargs)
|
||||
|
||||
await send_management_endpoint_alert(
|
||||
request_kwargs=kwargs,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
function_name=func.__name__,
|
||||
function_name=endpoint.__name__,
|
||||
)
|
||||
parent_otel_span: Span | None = getattr(user_api_key_dict, "parent_otel_span", None)
|
||||
if parent_otel_span is not None:
|
||||
await _emit_management_endpoint_otel_span(
|
||||
func=func,
|
||||
func=endpoint,
|
||||
kwargs=kwargs,
|
||||
parent_otel_span=parent_otel_span,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
result=result,
|
||||
log_payload=log_payload,
|
||||
)
|
||||
|
||||
# Delete updated/deleted info from cache
|
||||
|
|
@ -813,17 +838,18 @@ def management_endpoint_wrapper(func):
|
|||
except Exception as e:
|
||||
end_time = datetime.now()
|
||||
|
||||
user_api_key_dict: UserAPIKeyAuth = kwargs.get("user_api_key_dict") or UserAPIKeyAuth()
|
||||
user_api_key_dict: UserAPIKeyAuth = _caller_auth(kwargs)
|
||||
parent_otel_span = getattr(user_api_key_dict, "parent_otel_span", None)
|
||||
if parent_otel_span is not None:
|
||||
if parent_otel_span is not None and (log_payload or not _proxy_exception_handler_ends_server_span(e)):
|
||||
try:
|
||||
await _emit_management_endpoint_otel_span(
|
||||
func=func,
|
||||
func=endpoint,
|
||||
kwargs=kwargs,
|
||||
parent_otel_span=parent_otel_span,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
exception=e,
|
||||
log_payload=log_payload,
|
||||
)
|
||||
except Exception as otel_exc:
|
||||
# Non-Blocking Exception - never let OTEL failures swallow
|
||||
|
|
|
|||
|
|
@ -438,6 +438,14 @@ def is_mcp_admitted_user_subject(user_api_key_auth: UserAPIKeyAuth | None) -> bo
|
|||
_is_mcp_admitted_user_subject: Final = is_mcp_admitted_user_subject
|
||||
|
||||
|
||||
def has_team_unbound_mcp_admin_view(user_api_key_auth: UserAPIKeyAuth | None) -> bool:
|
||||
return (
|
||||
user_api_key_auth is not None
|
||||
and user_api_key_has_admin_view(user_api_key_auth)
|
||||
and (not user_api_key_auth.team_id or user_api_key_auth.team_id == UI_TEAM_ID)
|
||||
)
|
||||
|
||||
|
||||
def _gateway_dcr_challenge_target(
|
||||
route: str,
|
||||
mcp_servers: list[str] | None,
|
||||
|
|
@ -4290,19 +4298,19 @@ class MCPRequestHandler:
|
|||
"""Whether this principal's admin-view role grants the unscoped MCP resolution, whatever
|
||||
credential carries it (admin key, dashboard session, or OAuth-admitted session subject).
|
||||
|
||||
Two bounds disqualify, one per ownership of the row. A CREDENTIAL's explicit
|
||||
``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An
|
||||
admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is
|
||||
[] by DB default, so for that shape the row binds through the entitlement ceiling instead
|
||||
(any non-empty entitlement, or an unresolved one, disqualifies), exactly as
|
||||
``operator_open_server_ids`` reads the same row. The one owner of this predicate: the
|
||||
server-axis registry resolution in ``get_allowed_mcp_servers`` and the tools-axis open
|
||||
channel in ``resolve_admitted_subject_tools`` both consult it, so the two axes cannot
|
||||
disagree."""
|
||||
A credential on a real team never qualifies, a virtual key or a ``lite login`` session alike: its team
|
||||
bounds its MCP access whoever owns it, as it bounds the credential's model access. Two more bounds disqualify, one per ownership of the row. A CREDENTIAL's
|
||||
explicit ``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An
|
||||
admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is [] by DB
|
||||
default, so for that shape the row binds through the entitlement ceiling instead (any non-empty
|
||||
entitlement, or an unresolved one, disqualifies), exactly as ``operator_open_server_ids`` reads the same
|
||||
row. The one owner of this predicate: the server-axis registry resolution in ``get_allowed_mcp_servers``
|
||||
and the tools-axis open channel in ``resolve_admitted_subject_tools`` both consult it, so the two axes
|
||||
cannot disagree."""
|
||||
if (
|
||||
user_api_key_auth is None
|
||||
or user_api_key_auth.mcp_explicit_grants_only
|
||||
or not user_api_key_has_admin_view(user_api_key_auth)
|
||||
or not has_team_unbound_mcp_admin_view(user_api_key_auth)
|
||||
):
|
||||
return False
|
||||
object_permission: Final = user_api_key_auth.object_permission
|
||||
|
|
|
|||
|
|
@ -3705,7 +3705,7 @@ class MCPServerManager:
|
|||
|
||||
Priority:
|
||||
1. If object_permission.mcp_servers is explicitly set, use it (even for admins)
|
||||
2. If admin and no object_permission, return all servers
|
||||
2. If admin, not bound to a real team, and no object_permission, return all servers
|
||||
3. Otherwise, use standard permission checks
|
||||
"""
|
||||
if managed_agent_policy(user_api_key_auth) is not None:
|
||||
|
|
|
|||
|
|
@ -223,6 +223,7 @@ __all__ = (
|
|||
"mcp_get_prompt",
|
||||
"mcp_read_resource",
|
||||
"raise_denied_scoped_mcp_access",
|
||||
"raise_generic_mcp_access_denial",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -763,8 +764,14 @@ async def raise_denied_scoped_mcp_access(
|
|||
)
|
||||
}
|
||||
raise HTTPException(status_code=403, detail=group_denial)
|
||||
raise_generic_mcp_access_denial(requested_names)
|
||||
|
||||
|
||||
def raise_generic_mcp_access_denial(requested_names: Sequence[str]) -> NoReturn:
|
||||
generic_denial: Final[_McpDeniedDetail] = {
|
||||
"error": f"The key is not allowed to access the requested MCP servers: {', '.join(requested_names)}"
|
||||
"error": (
|
||||
f"The key is not allowed to access the requested MCP servers: {', '.join(dict.fromkeys(requested_names))}"
|
||||
)
|
||||
}
|
||||
raise HTTPException(status_code=403, detail=generic_denial)
|
||||
|
||||
|
|
|
|||
|
|
@ -520,6 +520,7 @@ if MCP_AVAILABLE:
|
|||
"mcp_get_prompt",
|
||||
"mcp_read_resource",
|
||||
"raise_denied_scoped_mcp_access",
|
||||
"raise_generic_mcp_access_denial",
|
||||
"redact_mcp_resource_url",
|
||||
)
|
||||
from mcp.server import Server
|
||||
|
|
@ -1197,6 +1198,7 @@ if MCP_AVAILABLE:
|
|||
apply_display_name_overrides,
|
||||
filter_tools_by_allowed_tools,
|
||||
raise_denied_scoped_mcp_access,
|
||||
raise_generic_mcp_access_denial,
|
||||
)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
|
|
@ -1824,10 +1826,10 @@ if MCP_AVAILABLE:
|
|||
row capping it.
|
||||
|
||||
Raises HTTPException(403) unless the principal holds toolset_id through one
|
||||
of its grant sources. Admins always pass.
|
||||
of its grant sources. Admins pass unless the key is bound to a real team.
|
||||
"""
|
||||
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
|
||||
from litellm.proxy.management_endpoints.common_utils import user_api_key_has_admin_view
|
||||
from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import has_team_unbound_mcp_admin_view
|
||||
|
||||
# A key scoped to no MCP servers opts out of every MCP path. Enforce it
|
||||
# here too, since toolset scoping replaces mcp_servers and would otherwise
|
||||
|
|
@ -1841,7 +1843,7 @@ if MCP_AVAILABLE:
|
|||
)
|
||||
|
||||
acting: Final = await acting_user(user_api_key_auth)
|
||||
is_admin: Final = user_api_key_has_admin_view(acting)
|
||||
is_admin: Final = has_team_unbound_mcp_admin_view(acting)
|
||||
if not is_admin and toolset_id not in await granted(acting):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
|
|
|
|||
|
|
@ -52,7 +52,11 @@ from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
|
|||
from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention
|
||||
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
|
||||
from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
AnthropicModelInfo,
|
||||
anthropic_pass_through_api_base,
|
||||
merge_anthropic_beta_headers,
|
||||
)
|
||||
from litellm.llms.azure.passthrough.transformation import (
|
||||
foreign_azure_deployment,
|
||||
is_azure_body_model_inference_endpoint,
|
||||
|
|
@ -1047,7 +1051,7 @@ async def anthropic_proxy_route(
|
|||
"""
|
||||
[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)
|
||||
"""
|
||||
base_target_url = os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
|
||||
base_target_url = anthropic_pass_through_api_base()
|
||||
encoded_endpoint = httpx.URL(endpoint).path
|
||||
|
||||
# Ensure endpoint starts with '/' for proper URL construction
|
||||
|
|
|
|||
|
|
@ -71,7 +71,11 @@ from litellm.litellm_core_utils.litellm_logging import get_masked_values
|
|||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.llms.anthropic.common_utils import is_anthropic_messages_url
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
anthropic_accepts_user_id,
|
||||
anthropic_upstream_user_id_limit,
|
||||
is_anthropic_messages_url,
|
||||
)
|
||||
from litellm.llms.base_llm.managed_resources.utils import (
|
||||
resolve_passthrough_managed_id_provider,
|
||||
)
|
||||
|
|
@ -148,6 +152,8 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
|||
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
|
||||
EndpointType,
|
||||
PassthroughStandardLoggingPayload,
|
||||
WebSocketSessionMeter,
|
||||
WebSocketSessionUsage,
|
||||
)
|
||||
from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, CallTypes, Usage
|
||||
|
||||
|
|
@ -501,7 +507,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
|
|||
url: httpx.URL,
|
||||
headers: dict,
|
||||
requested_query_params: dict | None = None,
|
||||
_parsed_body: dict | None = None,
|
||||
_parsed_body: Mapping[str, object] | None = None,
|
||||
forward_multipart: bool = False,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
|
|
@ -715,8 +721,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
|
|||
# unenforced. Set after the client merge so a request body cannot supply its own budget.
|
||||
#
|
||||
# Only for the built-in provider routes. `get_model_from_request` returns
|
||||
# None for a user-defined pass-through, deliberately: its body is forwarded
|
||||
# verbatim, so `model` there names an UPSTREAM model rather than a
|
||||
# None for a user-defined pass-through, deliberately: the client's `model`
|
||||
# goes upstream unchanged, so it names an UPSTREAM model rather than a
|
||||
# LiteLLM-managed one. Enforcement is therefore skipped on those routes, and
|
||||
# charging a counter anyway would track spend that nothing can refuse, and
|
||||
# would attribute it to a budget the operator scoped to a LiteLLM model that
|
||||
|
|
@ -1185,6 +1191,36 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
|
|||
return headers
|
||||
|
||||
|
||||
def _json_object(value: object) -> Mapping[str, object]:
|
||||
return cast("Mapping[str, object]", value) if isinstance(value, Mapping) else MappingProxyType({}) # cast-ok: json
|
||||
|
||||
|
||||
def _anthropic_user_id_to_forward(
|
||||
body: Mapping[str, object],
|
||||
url: str,
|
||||
has_pre_call_guardrails: Callable[[Mapping[str, object]], bool],
|
||||
) -> str | None:
|
||||
user_id: Final = _json_object(body.get("metadata")).get("user_id")
|
||||
if not isinstance(user_id, str):
|
||||
return None
|
||||
max_length: Final = anthropic_upstream_user_id_limit(url)
|
||||
if max_length is None:
|
||||
return None
|
||||
if not anthropic_accepts_user_id(user_id, max_length):
|
||||
verbose_proxy_logger.warning("Not forwarding metadata.user_id because the upstream would refuse it")
|
||||
return None
|
||||
if has_pre_call_guardrails(body):
|
||||
verbose_proxy_logger.debug("Not forwarding metadata.user_id because a pre-call guardrail cannot scan it")
|
||||
return None
|
||||
return user_id
|
||||
|
||||
|
||||
def _with_anthropic_user_id(body: Mapping[str, object] | None, user_id: str | None) -> Mapping[str, object] | None:
|
||||
if body is None or user_id is None:
|
||||
return body
|
||||
return {**body, "metadata": {"user_id": user_id}}
|
||||
|
||||
|
||||
def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None
|
||||
|
||||
|
|
@ -1421,6 +1457,10 @@ async def pass_through_request(
|
|||
request_method=getattr(request, "method", None),
|
||||
cost_per_request=cost_per_request,
|
||||
)
|
||||
hooked_body: Final = _json_object(_parsed_body)
|
||||
client_user_id: Final = _anthropic_user_id_to_forward(
|
||||
hooked_body, str(url), proxy_logging_obj.request_has_pre_call_guardrails
|
||||
)
|
||||
kwargs = HttpPassThroughEndpointHelpers.init_kwargs_for_pass_through_endpoint( # rebind-ok: pre-existing rebinding on a rename-only line
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
_parsed_body=_parsed_body,
|
||||
|
|
@ -1588,11 +1628,13 @@ async def pass_through_request(
|
|||
else:
|
||||
logging_url = str(url) + "?" + requested_query_params_str
|
||||
|
||||
outgoing_body: Final = cast("Mapping[str, object] | None", _parsed_body) # cast-ok: json
|
||||
upstream_body: Final = _with_anthropic_user_id(outgoing_body, client_user_id)
|
||||
logging_obj.pre_call(
|
||||
input=[{"role": "user", "content": safe_dumps(_parsed_body)}],
|
||||
api_key="",
|
||||
additional_args={
|
||||
"complete_input_dict": _parsed_body,
|
||||
"complete_input_dict": upstream_body,
|
||||
"api_base": str(logging_url),
|
||||
"headers": redact_credential_headers(upstream_headers),
|
||||
},
|
||||
|
|
@ -1632,7 +1674,7 @@ async def pass_through_request(
|
|||
url,
|
||||
params=requested_query_params,
|
||||
headers=upstream_headers,
|
||||
json=_parsed_body,
|
||||
json=upstream_body,
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -1731,7 +1773,7 @@ async def pass_through_request(
|
|||
url=url,
|
||||
headers=upstream_headers,
|
||||
requested_query_params=requested_query_params,
|
||||
_parsed_body=_parsed_body,
|
||||
_parsed_body=upstream_body,
|
||||
forward_multipart=is_multipart,
|
||||
)
|
||||
verbose_proxy_logger.debug("response.headers= %s", response.headers)
|
||||
|
|
@ -2595,6 +2637,13 @@ def _with_trace_context(headers: Mapping[str, str], parent_span: object) -> dict
|
|||
return inject_trace_context(headers, parent_span=parent_span)
|
||||
|
||||
|
||||
def _log_session_usage(logging_obj: LiteLLMLoggingObj, usage: WebSocketSessionUsage) -> None:
|
||||
logging_obj.model = usage.model # rebind-ok: the spend logger reads model and cost off the shared logging object
|
||||
logging_obj.model_call_details["model"] = usage.model # rebind-ok: same shared logging object
|
||||
logging_obj.model_call_details["custom_llm_provider"] = usage.custom_llm_provider # rebind-ok: same
|
||||
logging_obj.model_call_details["response_cost"] = usage.response_cost # rebind-ok: same
|
||||
|
||||
|
||||
async def websocket_passthrough_request(
|
||||
websocket: WebSocket,
|
||||
target: str,
|
||||
|
|
@ -2607,6 +2656,7 @@ async def websocket_passthrough_request(
|
|||
setup_model_rewriter: Callable[[str], str] | None = None,
|
||||
builtin_provider_route: bool = False,
|
||||
model: str | None = None,
|
||||
meter: WebSocketSessionMeter | None = None,
|
||||
):
|
||||
"""
|
||||
WebSocket passthrough request handler.
|
||||
|
|
@ -2622,6 +2672,7 @@ async def websocket_passthrough_request(
|
|||
setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream
|
||||
builtin_provider_route: Whether a built-in provider route serves the connection, so its URL names the provider
|
||||
model: The model the caller named before connecting, logged on a failure before the setup frame arrives
|
||||
meter: Prices the session from the client text frames delivered upstream, for an upstream that reports no usage
|
||||
"""
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj
|
||||
|
|
@ -2632,6 +2683,7 @@ async def websocket_passthrough_request(
|
|||
# Initialize tracking variables
|
||||
start_time: Final = datetime.now()
|
||||
websocket_messages: Final[list[dict[str, object]]] = []
|
||||
metered_total = 0 # rebind-ok: forward_client_to_upstream adds the measure of each client text frame it delivers
|
||||
litellm_call_id: Final = str(uuid.uuid4())
|
||||
|
||||
verbose_proxy_logger.info("WebSocket passthrough (%s): Starting WebSocket connection to %s", endpoint, target)
|
||||
|
|
@ -2750,6 +2802,7 @@ async def websocket_passthrough_request(
|
|||
|
||||
async def forward_client_to_upstream() -> None:
|
||||
"""Forward messages from client to upstream WebSocket"""
|
||||
nonlocal metered_total
|
||||
try:
|
||||
while True:
|
||||
message = await websocket.receive()
|
||||
|
|
@ -2819,6 +2872,8 @@ async def websocket_passthrough_request(
|
|||
# Not a JSON message or doesn't contain setup data
|
||||
|
||||
await upstream_ws.send(_rewrite_vertex_live_setup_model(text_data, setup_model_rewriter))
|
||||
if meter is not None:
|
||||
metered_total += meter.measure(text_data)
|
||||
elif bytes_data is not None:
|
||||
await upstream_ws.send(bytes_data)
|
||||
except asyncio.CancelledError:
|
||||
|
|
@ -2945,6 +3000,8 @@ async def websocket_passthrough_request(
|
|||
self.url = target_url
|
||||
|
||||
mock_response: Final = MockWebSocketResponse(target)
|
||||
if meter is not None:
|
||||
_log_session_usage(logging_obj, meter.price(metered_total))
|
||||
|
||||
# Use the same success handler as HTTP passthrough endpoints
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
|
||||
|
|
|
|||
|
|
@ -181,6 +181,7 @@ from litellm.proxy.common_utils.token_count_settings import (
|
|||
)
|
||||
from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog
|
||||
from litellm.proxy.management_helpers.model_group_custom_fields import model_group_custom_fields
|
||||
from litellm.proxy.management_helpers.utils import management_endpoint_span_wrapper
|
||||
from litellm.repositories import database as database_runtime
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
|
|
@ -12339,6 +12340,7 @@ async def _deployment_hidden_by_listing_callbacks(deployment: Deployment, user_a
|
|||
@router.get(
|
||||
"/models", dependencies=[Depends(user_api_key_auth)], tags=["model management"]
|
||||
) # if project requires model list
|
||||
@management_endpoint_span_wrapper
|
||||
async def model_list(
|
||||
request: Request = None, # pyright: ignore[reportArgumentType] # FastAPI always injects the Request; the None default only serves direct in-process callers
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
|
|
@ -12649,6 +12651,7 @@ async def model_list(
|
|||
dependencies=[Depends(user_api_key_auth)],
|
||||
tags=["model management"],
|
||||
)
|
||||
@management_endpoint_span_wrapper
|
||||
async def model_info(
|
||||
model_id: str,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
|
|
@ -21985,9 +21988,9 @@ async def toolset_mcp_route(toolset_name: str, request: Request):
|
|||
Namespace a toolset as its own MCP endpoint.
|
||||
|
||||
Connecting to /toolset/<name>/mcp exposes exactly the tools defined in
|
||||
the toolset. Access is enforced: non-admin API keys must have the toolset
|
||||
listed in their object_permission.mcp_toolsets grant list, or the request
|
||||
will be rejected with a 403.
|
||||
the toolset. Access is enforced: API keys other than an admin's key with no
|
||||
team must hold the toolset through one of their grant sources, or the
|
||||
request will be rejected with a 403.
|
||||
"""
|
||||
try:
|
||||
from litellm.proxy.mcp_server.mcp_server_manager import (
|
||||
|
|
|
|||
|
|
@ -3343,6 +3343,34 @@
|
|||
],
|
||||
"default_model_placeholder": "prism/deepseek-v4.1-flash"
|
||||
},
|
||||
{
|
||||
"provider": "HOSTINGER_ROUTER",
|
||||
"provider_display_name": "Hostinger Router",
|
||||
"litellm_provider": "hostinger_router",
|
||||
"credential_fields": [
|
||||
{
|
||||
"key": "api_base",
|
||||
"label": "API Base",
|
||||
"placeholder": "https://router.hostinger.com/v1",
|
||||
"tooltip": null,
|
||||
"required": false,
|
||||
"field_type": "text",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
},
|
||||
{
|
||||
"key": "api_key",
|
||||
"label": "API Key",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": true,
|
||||
"field_type": "password",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
}
|
||||
],
|
||||
"default_model_placeholder": "hostinger_router/claude-sonnet-5"
|
||||
},
|
||||
{
|
||||
"provider": "RECRAFT",
|
||||
"provider_display_name": "Recraft",
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ from litellm._logging import verbose_proxy_logger
|
|||
from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.constants import EMPTY_MAPPING, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, TERMINAL_RESPONSE_STATUSES
|
||||
from litellm.exceptions import EndpointNotSupportedByModelError
|
||||
from litellm.exceptions import EndpointNotSupportedByModelError, RateLimitError
|
||||
from litellm.integrations.custom_guardrail import ModifyResponseException, guardrail_request_data_with_streaming
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata_variable_name_from_kwargs
|
||||
from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background
|
||||
|
|
@ -103,6 +103,7 @@ from litellm.responses.streaming_iterator import (
|
|||
ResponsesWsFrameGuard,
|
||||
ResponsesWsFrameRejection,
|
||||
spawn_logging_task,
|
||||
turn_refusal_frame,
|
||||
)
|
||||
from litellm.router import Router
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
|
@ -2627,6 +2628,12 @@ async def _responses_websocket_session(
|
|||
route_type="_aresponses_websocket",
|
||||
skip_guardrails=True,
|
||||
)
|
||||
except RateLimitError as e:
|
||||
verbose_proxy_logger.info("Responses WebSocket refused at open by a rate limit: %s", e)
|
||||
with contextlib.suppress(WebSocketDisconnect, RuntimeError, OSError):
|
||||
await websocket.send_text(turn_refusal_frame(e))
|
||||
await websocket.close(code=1008, reason="Rate limit exceeded")
|
||||
return
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Responses WebSocket pre-call error")
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -3031,14 +3031,21 @@ class ProxyLogging:
|
|||
hook to consult, but it judges the payload the same way, so a proxy configured only with
|
||||
one of those still has something to say about every record.
|
||||
"""
|
||||
if request_metadata.get("_guardrail_pipelines"):
|
||||
return self.request_has_pre_call_guardrails({"metadata": dict(request_metadata)}, event_type)
|
||||
|
||||
def request_has_pre_call_guardrails(
|
||||
self,
|
||||
request_data: Mapping[str, object],
|
||||
event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call,
|
||||
) -> bool:
|
||||
if _policy_state_metadata(request_data).get("_guardrail_pipelines"):
|
||||
return True
|
||||
caps: Final = ProxyLogging.callback_capabilities()
|
||||
if caps.has_content_enforcer:
|
||||
return True
|
||||
probe: Final = {"metadata": dict(request_metadata)}
|
||||
return any(
|
||||
isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=event_type)
|
||||
isinstance(callback, CustomGuardrail)
|
||||
and callback.should_run_guardrail(data=request_data, event_type=event_type)
|
||||
for callback in caps.resolved_callbacks
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from pydantic import TypeAdapter
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
|
||||
from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools
|
||||
from litellm.responses.litellm_completion_transformation.client_tool_search import bridge_client_tool_search
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary
|
||||
|
|
@ -148,6 +149,9 @@ class LiteLLMCompletionTransformationHandler:
|
|||
responses_api_request=bridged_request,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_metadata=kwargs.get("litellm_metadata", {}),
|
||||
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(
|
||||
_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)
|
||||
),
|
||||
)
|
||||
raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")
|
||||
|
||||
|
|
@ -240,5 +244,8 @@ class LiteLLMCompletionTransformationHandler:
|
|||
responses_api_request=responses_api_request,
|
||||
custom_llm_provider=litellm_completion_request.get("custom_llm_provider"),
|
||||
litellm_metadata=kwargs.get("litellm_metadata", {}),
|
||||
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(
|
||||
_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)
|
||||
),
|
||||
)
|
||||
raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")
|
||||
|
|
|
|||
|
|
@ -200,6 +200,7 @@ class ResponsesSessionHandler:
|
|||
previous_response_id: str,
|
||||
item_cache_control: bool = False,
|
||||
client_response_ids: tuple[str, ...] = (),
|
||||
tool_call_provider_fields: bool = False,
|
||||
stored_turns: StoredTurns | None = None,
|
||||
) -> ChatCompletionSession:
|
||||
"""
|
||||
|
|
@ -241,9 +242,13 @@ class ResponsesSessionHandler:
|
|||
for index in range(start.index, replayed_turns)
|
||||
],
|
||||
item_cache_control,
|
||||
tool_call_provider_fields,
|
||||
)
|
||||
history_before_replay: Final = ResponsesSessionHandler._history_of_turns(
|
||||
all_spend_logs[: start.index], proxy_server_requests[: start.index], item_cache_control
|
||||
all_spend_logs[: start.index],
|
||||
proxy_server_requests[: start.index],
|
||||
item_cache_control,
|
||||
tool_call_provider_fields,
|
||||
)
|
||||
chat_completion_message_history: Final = (
|
||||
list(with_stored_calls_the_input_answers(replayed_history, history_before_replay))
|
||||
|
|
@ -305,6 +310,7 @@ class ResponsesSessionHandler:
|
|||
spend_logs: Sequence["SpendLogsPayload"],
|
||||
proxy_server_requests: Sequence[Mapping[str, object] | None],
|
||||
item_cache_control: bool,
|
||||
tool_call_provider_fields: bool,
|
||||
) -> list["SessionMessage"]:
|
||||
return list(
|
||||
chain.from_iterable(
|
||||
|
|
@ -313,6 +319,7 @@ class ResponsesSessionHandler:
|
|||
proxy_server_request_dict=proxy_server_request,
|
||||
chat_completion_message_history=[],
|
||||
item_cache_control=item_cache_control,
|
||||
tool_call_provider_fields=tool_call_provider_fields,
|
||||
)
|
||||
for spend_log, proxy_server_request in zip(spend_logs, proxy_server_requests, strict=True)
|
||||
)
|
||||
|
|
@ -330,6 +337,7 @@ class ResponsesSessionHandler:
|
|||
| Message
|
||||
],
|
||||
item_cache_control: bool = False,
|
||||
tool_call_provider_fields: bool = False,
|
||||
):
|
||||
"""
|
||||
Extend the chat completion message history with the spend log payload
|
||||
|
|
@ -362,6 +370,7 @@ class ResponsesSessionHandler:
|
|||
responses_api_request={},
|
||||
replay_reasoning=True,
|
||||
item_cache_control=item_cache_control,
|
||||
tool_call_provider_fields=tool_call_provider_fields,
|
||||
only_since_compaction=True,
|
||||
)
|
||||
)
|
||||
|
|
@ -377,6 +386,7 @@ class ResponsesSessionHandler:
|
|||
responses_api_request={},
|
||||
replay_reasoning=True,
|
||||
item_cache_control=item_cache_control,
|
||||
tool_call_provider_fields=tool_call_provider_fields,
|
||||
only_since_compaction=True,
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import litellm
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items
|
||||
from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
|
||||
from litellm.llms.anthropic.messages.native.utils import openai_chat_refusal_text
|
||||
from litellm.main import stream_chunk_builder
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import compaction_output_item
|
||||
from litellm.responses.litellm_completion_transformation.custom_tools import (
|
||||
|
|
@ -35,6 +36,7 @@ from litellm.types.llms.openai import (
|
|||
ContentPartAddedEvent,
|
||||
ContentPartDoneEvent,
|
||||
ContentPartDonePartOutputText,
|
||||
ContentPartDonePartRefusal,
|
||||
FunctionCallArgumentsDeltaEvent,
|
||||
FunctionCallArgumentsDoneEvent,
|
||||
OutputItemAddedEvent,
|
||||
|
|
@ -45,6 +47,8 @@ from litellm.types.llms.openai import (
|
|||
ReasoningSummaryPartDoneEvent,
|
||||
ReasoningSummaryTextDeltaEvent,
|
||||
ReasoningSummaryTextDoneEvent,
|
||||
RefusalDeltaEvent,
|
||||
RefusalDoneEvent,
|
||||
ResponseCompletedEvent,
|
||||
ResponseCreatedEvent,
|
||||
ResponseIncompleteEvent,
|
||||
|
|
@ -88,6 +92,20 @@ def _output_items_with_id(items: tuple[Any, ...], item_type: str, item_id: str |
|
|||
)
|
||||
|
||||
|
||||
def _tool_call_objects(tool_calls: object) -> tuple[object, ...]:
|
||||
return tuple(tool_calls) if isinstance(tool_calls, list) else ()
|
||||
|
||||
|
||||
def _tool_call_provider_specific_fields(tool_call: object) -> Mapping[str, object]:
|
||||
mapping: Final = as_str_mapping(tool_call)
|
||||
fields: Final[object] = (
|
||||
mapping.get("provider_specific_fields")
|
||||
if mapping is not None
|
||||
else getattr(tool_call, "provider_specific_fields", None)
|
||||
)
|
||||
return {"provider_specific_fields": fields} if fields else {}
|
||||
|
||||
|
||||
def _is_signed_thinking_block(block: object) -> bool:
|
||||
fields: Final = as_str_mapping(block)
|
||||
return fields is not None and isinstance(block, dict) and bool(fields.get("signature") or fields.get("data"))
|
||||
|
|
@ -110,6 +128,7 @@ def _delta_carries_output(delta: ChatCompletionDelta) -> bool:
|
|||
delta.function_call,
|
||||
getattr(delta, "images", None),
|
||||
getattr(delta, "audio", None),
|
||||
openai_chat_refusal_text(delta),
|
||||
)
|
||||
return any(fields) or _delta_has_signed_thinking_block(delta)
|
||||
|
||||
|
|
@ -119,6 +138,13 @@ def _delta_carries_reasoning_and_answer(delta: ChatCompletionDelta) -> bool:
|
|||
return has_reasoning and bool(delta.content or delta.tool_calls)
|
||||
|
||||
|
||||
_WEB_SEARCH_PROGRESS_EVENTS: Final = (
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, WebSearchCallInProgressEvent),
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent),
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),
|
||||
)
|
||||
|
||||
|
||||
class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
||||
"""
|
||||
Async iterator for processing streaming responses from the Responses API.
|
||||
|
|
@ -132,6 +158,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
responses_api_request: ResponsesAPIOptionalRequestParams,
|
||||
custom_llm_provider: str | None = None,
|
||||
litellm_metadata: dict | None = None,
|
||||
tool_call_provider_fields: bool = False,
|
||||
):
|
||||
self.model: str = model
|
||||
self.litellm_custom_stream_wrapper: litellm.CustomStreamWrapper = litellm_custom_stream_wrapper
|
||||
|
|
@ -139,6 +166,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
self.responses_api_request: ResponsesAPIOptionalRequestParams = responses_api_request
|
||||
self.custom_llm_provider: str | None = custom_llm_provider
|
||||
self.litellm_metadata: dict | None = litellm_metadata or {}
|
||||
self.tool_call_provider_fields: Final = tool_call_provider_fields
|
||||
# Store lightweight dict snapshots for stream_chunk_builder to reduce
|
||||
# repeated Pydantic attribute access in end-of-stream assembly.
|
||||
self.collected_chat_completion_chunks: list[dict[str, object]] = []
|
||||
|
|
@ -152,6 +180,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
self.sent_output_text_done_event: bool = False
|
||||
self.sent_output_content_part_done_event: bool = False
|
||||
self.sent_output_item_done_event: bool = False
|
||||
self._sent_refusal_events: bool = False
|
||||
self._annotation_index: int = 0
|
||||
self._annotations_awaiting_message: tuple[ChatCompletionAnnotation, ...] = ()
|
||||
self.litellm_model_response: ModelResponse | TextCompletionResponse | None = None
|
||||
|
|
@ -490,10 +519,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
except Exception:
|
||||
tool_calls = None
|
||||
|
||||
if not tool_calls or not isinstance(tool_calls, list):
|
||||
return
|
||||
|
||||
for tc in tool_calls:
|
||||
for tc in _tool_call_objects(tool_calls):
|
||||
call_id_raw = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)
|
||||
if not call_id_raw:
|
||||
continue
|
||||
|
|
@ -568,7 +594,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
output_index=output_index,
|
||||
sequence_number=self._sequence_number,
|
||||
item=BaseLiteLLMOpenAIResponseObject(**item_kwargs),
|
||||
item=BaseLiteLLMOpenAIResponseObject(
|
||||
**item_kwargs,
|
||||
**(_tool_call_provider_specific_fields(tc) if self.tool_call_provider_fields else {}),
|
||||
),
|
||||
)
|
||||
self._pending_tool_events.append(item_done_event)
|
||||
|
||||
|
|
@ -592,11 +621,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
}
|
||||
),
|
||||
)
|
||||
for event_type, event_class in (
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, WebSearchCallInProgressEvent),
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent),
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),
|
||||
):
|
||||
progress_events: Final = (
|
||||
_WEB_SEARCH_PROGRESS_EVENTS[:1] if item.status == "in_progress" else _WEB_SEARCH_PROGRESS_EVENTS
|
||||
)
|
||||
for event_type, event_class in progress_events:
|
||||
self._sequence_number += 1
|
||||
event = event_class(type=event_type, output_index=output_index, item_id=item.id)
|
||||
event.__dict__["sequence_number"] = self._sequence_number
|
||||
|
|
@ -798,9 +826,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
return event
|
||||
|
||||
def _queue_message_item_added_events(self) -> None:
|
||||
if not self.sent_message_item_added_event:
|
||||
self._queue_message_item_added_event()
|
||||
self.sent_content_part_added_event = True
|
||||
self._pending_response_events.append(self.create_content_part_added_event())
|
||||
|
||||
def _queue_message_item_added_event(self) -> None:
|
||||
item_id: Final = self._message_item_id()
|
||||
self.sent_message_item_added_event = True
|
||||
self.sent_content_part_added_event = True
|
||||
self._sequence_number += 1
|
||||
event: Final = OutputItemAddedEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
|
||||
|
|
@ -817,7 +850,47 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
)
|
||||
event.__dict__["sequence_number"] = self._sequence_number
|
||||
self._pending_response_events.append(event)
|
||||
self._pending_response_events.append(self.create_content_part_added_event())
|
||||
|
||||
def _queue_refusal_events(self, refusal: str) -> None:
|
||||
item_id: Final = self._message_item_id()
|
||||
output_index: Final = self._message_index()
|
||||
content_index: Final = 1 if self.sent_content_part_added_event else 0
|
||||
events: Final = (
|
||||
ContentPartAddedEvent(
|
||||
type=ResponsesAPIStreamEvents.CONTENT_PART_ADDED,
|
||||
item_id=item_id,
|
||||
output_index=output_index,
|
||||
content_index=content_index,
|
||||
part=ContentPartDonePartRefusal(type="refusal", refusal=""),
|
||||
),
|
||||
RefusalDeltaEvent(
|
||||
type=ResponsesAPIStreamEvents.REFUSAL_DELTA,
|
||||
item_id=item_id,
|
||||
output_index=output_index,
|
||||
content_index=content_index,
|
||||
delta=refusal,
|
||||
),
|
||||
RefusalDoneEvent(
|
||||
type=ResponsesAPIStreamEvents.REFUSAL_DONE,
|
||||
item_id=item_id,
|
||||
output_index=output_index,
|
||||
content_index=content_index,
|
||||
refusal=refusal,
|
||||
),
|
||||
ContentPartDoneEvent(
|
||||
type=ResponsesAPIStreamEvents.CONTENT_PART_DONE,
|
||||
item_id=item_id,
|
||||
output_index=output_index,
|
||||
content_index=content_index,
|
||||
part=ContentPartDonePartRefusal(type="refusal", refusal=refusal),
|
||||
),
|
||||
)
|
||||
first_sequence_number: Final = self._sequence_number + 1
|
||||
self._sequence_number += len(events)
|
||||
self._pending_response_events.extend(
|
||||
event.model_copy(update={"sequence_number": sequence_number})
|
||||
for sequence_number, event in enumerate(events, start=first_sequence_number)
|
||||
)
|
||||
|
||||
def _merge_provider_specific_fields(self, src: dict) -> None:
|
||||
"""Merge provider_specific_fields using last-value-wins for lists.
|
||||
|
|
@ -974,14 +1047,22 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
if self._cached_item_id is None:
|
||||
self._cached_item_id = f"msg_{uuid.uuid4()}"
|
||||
|
||||
text: Final = litellm_complete_object.choices[0].message.content or ""
|
||||
annotations = getattr(litellm_complete_object.choices[0].message, "annotations", None)
|
||||
message: Final = litellm_complete_object.choices[0].message
|
||||
text: Final = message.content or ""
|
||||
annotations = getattr(message, "annotations", None)
|
||||
|
||||
response_annotations: Final = (
|
||||
LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations(
|
||||
annotations=annotations
|
||||
)
|
||||
)
|
||||
text_parts: Final = (
|
||||
({"type": "output_text", "text": text, "annotations": response_annotations},)
|
||||
if self.sent_content_part_added_event
|
||||
else ()
|
||||
)
|
||||
refusal: Final = openai_chat_refusal_text(message)
|
||||
refusal_parts: Final = ({"type": "refusal", "refusal": refusal},) if refusal is not None else ()
|
||||
return OutputItemDoneEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
output_index=self._message_index(),
|
||||
|
|
@ -992,13 +1073,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
"status": "completed",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": text,
|
||||
"annotations": response_annotations,
|
||||
}
|
||||
],
|
||||
"content": [*text_parts, *refusal_parts],
|
||||
}
|
||||
),
|
||||
)
|
||||
|
|
@ -1050,22 +1125,33 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
def return_default_done_events(
|
||||
self, litellm_complete_object: ModelResponse
|
||||
) -> BaseLiteLLMOpenAIResponseObject | None:
|
||||
message: Final = litellm_complete_object.choices[0].message
|
||||
refusal: Final = openai_chat_refusal_text(message)
|
||||
if self.sent_message_item_added_event is False:
|
||||
final_content: Final = litellm_complete_object.choices[0].message.content or ""
|
||||
if not final_content:
|
||||
final_content: Final = message.content or ""
|
||||
if not final_content and refusal is None:
|
||||
self.sent_output_text_done_event = True
|
||||
self.sent_output_content_part_done_event = True
|
||||
self.sent_output_item_done_event = True
|
||||
return None
|
||||
self._queue_message_item_added_events()
|
||||
if final_content:
|
||||
self._queue_message_item_added_events()
|
||||
else:
|
||||
self._queue_message_item_added_event()
|
||||
return self._pending_response_events.pop(0)
|
||||
if self.sent_output_text_done_event is False:
|
||||
if self.sent_output_text_done_event is False and self.sent_content_part_added_event:
|
||||
self.sent_output_text_done_event = True
|
||||
return self.create_output_text_done_event(litellm_complete_object)
|
||||
if self.sent_output_content_part_done_event is False:
|
||||
if self.sent_output_content_part_done_event is False and self.sent_content_part_added_event:
|
||||
self.sent_output_content_part_done_event = True
|
||||
return self.create_output_content_part_done_event(litellm_complete_object)
|
||||
if refusal is not None and not self._sent_refusal_events:
|
||||
self._sent_refusal_events = True
|
||||
self._queue_refusal_events(refusal)
|
||||
return self._pending_response_events.pop(0)
|
||||
if self.sent_output_item_done_event is False:
|
||||
self.sent_output_text_done_event = True
|
||||
self.sent_output_content_part_done_event = True
|
||||
self.sent_output_item_done_event = True
|
||||
return self.create_output_item_done_event(litellm_complete_object)
|
||||
return None
|
||||
|
|
@ -1169,7 +1255,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
# DO NOT create message item
|
||||
return
|
||||
|
||||
# Default: message
|
||||
if openai_chat_refusal_text(delta) is not None and not delta.content:
|
||||
self._queue_message_item_added_event()
|
||||
return
|
||||
self._queue_message_item_added_events()
|
||||
return
|
||||
|
||||
|
|
@ -1413,7 +1501,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
# Priority 2: Handle text deltas
|
||||
delta_content: Final = self._get_delta_string_from_streaming_choices(chunk.choices)
|
||||
if delta_content:
|
||||
if not self.sent_message_item_added_event:
|
||||
if not self.sent_content_part_added_event:
|
||||
self._queue_message_item_added_events()
|
||||
self._hold_tool_calls_behind_text(getattr(chunk.choices[0].delta, "tool_calls", None))
|
||||
self._sequence_number += 1
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import uuid
|
|||
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from itertools import accumulate, chain, groupby, islice
|
||||
from itertools import accumulate, chain, groupby
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
|
|
@ -28,7 +28,7 @@ from openai.types.chat.chat_completion_named_tool_choice_param import (
|
|||
from openai.types.chat.chat_completion_named_tool_choice_param import (
|
||||
Function as NamedToolChoiceFunction,
|
||||
)
|
||||
from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch
|
||||
from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch, ResponseOutputRefusal
|
||||
from openai.types.responses.response_compaction_item import ResponseCompactionItem
|
||||
from openai.types.responses.response_create_params import ResponseInputParam
|
||||
from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam
|
||||
|
|
@ -45,8 +45,16 @@ from litellm.constants import (
|
|||
REDACTED_BY_LITELLM,
|
||||
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER,
|
||||
)
|
||||
from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
AnthropicCacheControlHook,
|
||||
block_with_cache_control,
|
||||
cache_control_count,
|
||||
dropped_cache_control,
|
||||
newest_marks_within_cache_control_limit,
|
||||
whole_messages_within_cache_control_limit,
|
||||
with_cache_control,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params
|
||||
from litellm.litellm_core_utils.get_supported_openai_params import (
|
||||
get_supported_openai_params,
|
||||
)
|
||||
|
|
@ -56,7 +64,9 @@ from litellm.litellm_core_utils.hidden_params import (
|
|||
set_hidden_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
|
||||
from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control
|
||||
from litellm.llms.anthropic.messages.native.utils import anthropic_stop_reason, openai_chat_refusal_text
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support
|
||||
from litellm.llms.openrouter.chat.transformation import openrouter_model_supports_cache_control
|
||||
|
|
@ -365,8 +375,9 @@ class _ResponsesToolCallItem(Protocol):
|
|||
def get(self, key: str, /) -> object: ...
|
||||
|
||||
|
||||
class _MarkedToolCallChunk(ChatCompletionToolCallChunk):
|
||||
class _ReplayedToolCallChunk(ChatCompletionToolCallChunk, total=False):
|
||||
cache_control: ReadOnly[ChatCompletionCachedContent]
|
||||
provider_specific_fields: ReadOnly[Mapping[str, object]]
|
||||
|
||||
|
||||
class _MarkedToolMessage(ChatCompletionToolMessage):
|
||||
|
|
@ -438,6 +449,32 @@ def _input_item_role(input_item: Mapping[str, object]) -> str:
|
|||
return cast(str, input_item.get("role") or "user") # cast-ok: client-supplied role forwarded verbatim, unvalidated
|
||||
|
||||
|
||||
def _starts_a_turn(input_item: Mapping[str, object]) -> bool:
|
||||
item_type: Final = input_item.get("type")
|
||||
if item_type in TOOL_CALL_OUTPUT_ITEM_TYPES:
|
||||
return True
|
||||
return item_type in (None, "message") and input_item.get("role") != "assistant"
|
||||
|
||||
|
||||
def _is_unrun_hosted_search(input_item: Mapping[str, object]) -> bool:
|
||||
return input_item.get("type") == "web_search_call" and input_item.get("status") == "in_progress"
|
||||
|
||||
|
||||
def _without_encrypted_content(input_item: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {key: value for key, value in input_item.items() if key != "encrypted_content"}
|
||||
|
||||
|
||||
def _without_paused_turn_thinking(input_items: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]:
|
||||
turns: Final = tuple(accumulate(int(_starts_a_turn(item)) for item in input_items))
|
||||
paused_turns: Final = frozenset(
|
||||
turn for turn, item in zip(turns, input_items, strict=True) if _is_unrun_hosted_search(item)
|
||||
)
|
||||
return tuple(
|
||||
_without_encrypted_content(item) if turn in paused_turns and item.get("type") == "reasoning" else item
|
||||
for turn, item in zip(turns, input_items, strict=True)
|
||||
)
|
||||
|
||||
|
||||
SessionMessage: TypeAlias = (
|
||||
AllMessageValues
|
||||
| GenericChatCompletionMessage
|
||||
|
|
@ -603,21 +640,28 @@ def _validated_cache_control(value: object) -> ChatCompletionCachedContent | Non
|
|||
return None
|
||||
|
||||
|
||||
def _dropped_cache_control(_: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _without_cache_control_key(item: object) -> object:
|
||||
def _without_keys(item: object, keys: frozenset[str]) -> object:
|
||||
if not isinstance(item, dict):
|
||||
return item
|
||||
return {key: value for key, value in _STR_KEY_DICT_ADAPTER.validate_python(item).items() if key != "cache_control"}
|
||||
return {key: value for key, value in _STR_KEY_DICT_ADAPTER.validate_python(item).items() if key not in keys}
|
||||
|
||||
|
||||
def without_item_cache_control(input: str | ResponseInputParam) -> str | ResponseInputParam:
|
||||
if not isinstance(input, list):
|
||||
def _without_item_keys(input: str | ResponseInputParam, keys: frozenset[str]) -> str | ResponseInputParam:
|
||||
if not keys or not isinstance(input, list):
|
||||
return input
|
||||
return cast( # cast-ok: the same input items, each minus its own cache_control key
|
||||
ResponseInputParam, [_without_cache_control_key(item) for item in input]
|
||||
return cast( # cast-ok: the same input items, each minus the given keys
|
||||
ResponseInputParam, [_without_keys(item, keys) for item in input]
|
||||
)
|
||||
|
||||
|
||||
def _dropped_item_keys(item_cache_control: bool, tool_call_provider_fields: bool) -> frozenset[str]:
|
||||
return frozenset(
|
||||
key
|
||||
for key, kept in (
|
||||
("cache_control", item_cache_control),
|
||||
("provider_specific_fields", tool_call_provider_fields),
|
||||
)
|
||||
if not kept
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -694,83 +738,6 @@ def _content_blocks_carrying(content: object, cache_control: ChatCompletionCache
|
|||
return _with_cache_control_on_last_block(_OBJECT_LIST_ADAPTER.validate_python(content), cache_control)
|
||||
|
||||
|
||||
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
|
||||
|
||||
|
||||
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
|
||||
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
|
||||
return tuple(
|
||||
chain.from_iterable(_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list))
|
||||
)
|
||||
|
||||
|
||||
def _block_carries_cache_control(block: object) -> bool:
|
||||
return isinstance(block, dict) and _STR_KEY_DICT_ADAPTER.validate_python(block).get("cache_control") is not None
|
||||
|
||||
|
||||
def _cache_control_count(message: object) -> int:
|
||||
if not isinstance(message, dict):
|
||||
return 0
|
||||
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
|
||||
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
|
||||
return int(fields.get("cache_control") is not None) + block_marks
|
||||
|
||||
|
||||
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
|
||||
|
||||
|
||||
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
|
||||
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
|
||||
if "cache_control" not in fields:
|
||||
return others
|
||||
cache_control: Final = rewrite(fields["cache_control"])
|
||||
return others if cache_control is None else {**others, "cache_control": cache_control}
|
||||
|
||||
|
||||
def _block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
|
||||
if not isinstance(block, dict):
|
||||
return block
|
||||
return _fields_with_cache_control(_STR_KEY_DICT_ADAPTER.validate_python(block), rewrite)
|
||||
|
||||
|
||||
def _with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
|
||||
if not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
|
||||
blocks_rewritten: Final = {
|
||||
key: (
|
||||
[_block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
|
||||
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
|
||||
else value
|
||||
)
|
||||
for key, value in fields.items()
|
||||
}
|
||||
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
|
||||
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
|
||||
)
|
||||
|
||||
|
||||
def _kept_within_budget(state: tuple[int, bool], marks: int) -> tuple[int, bool]:
|
||||
remaining, _ = state
|
||||
if marks > remaining:
|
||||
return remaining, False
|
||||
return remaining - marks, True
|
||||
|
||||
|
||||
def _messages_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
|
||||
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
|
||||
newest_first: Final = tuple(reversed(messages))
|
||||
budget_after_each: Final = accumulate(
|
||||
(_cache_control_count(message) for message in newest_first), _kept_within_budget, initial=(budget, True)
|
||||
)
|
||||
kept_flags: Final = tuple(kept for _, kept in islice(budget_after_each, 1, None))
|
||||
kept_newest_first: Final = tuple(
|
||||
message if kept else _with_cache_control(message, _dropped_cache_control)
|
||||
for message, kept in zip(newest_first, kept_flags)
|
||||
)
|
||||
return tuple(reversed(kept_newest_first))
|
||||
|
||||
|
||||
_SYSTEM_MESSAGE_FIELDS: Final = frozenset(("content", "cache_control"))
|
||||
|
||||
|
||||
|
|
@ -799,7 +766,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
|
|||
if fields is None:
|
||||
return message
|
||||
if fields.get("role") == "tool":
|
||||
return _with_cache_control(message, _dropped_cache_control)
|
||||
return with_cache_control(message, dropped_cache_control)
|
||||
tool_calls: Final = fields.get("tool_calls")
|
||||
if not isinstance(tool_calls, list):
|
||||
return message
|
||||
|
|
@ -808,7 +775,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
|
|||
{
|
||||
**fields,
|
||||
"tool_calls": [
|
||||
_block_with_cache_control(tool_call, _dropped_cache_control)
|
||||
block_with_cache_control(tool_call, dropped_cache_control)
|
||||
for tool_call in _OBJECT_LIST_ADAPTER.validate_python(tool_calls)
|
||||
],
|
||||
},
|
||||
|
|
@ -822,7 +789,7 @@ class _ItemMarkRules:
|
|||
external_marks: int
|
||||
|
||||
def sanitized(self, message: _MessageT) -> _MessageT:
|
||||
valid: Final = _with_cache_control(message, _validated_cache_control)
|
||||
valid: Final = with_cache_control(message, _validated_cache_control)
|
||||
return _without_tool_block_marks(valid) if self.refuses_tool_block_marks else valid
|
||||
|
||||
def checked(self, message: _MessageT) -> _MessageT:
|
||||
|
|
@ -845,15 +812,15 @@ def _request_messages_for_cache_points(messages: Sequence[_MessageT], rules: _It
|
|||
as_system: Final = tuple(_developer_message_as_system_message(message) for message in messages)
|
||||
if not rules.drop_params:
|
||||
return as_system
|
||||
return _messages_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
|
||||
return newest_marks_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
|
||||
|
||||
|
||||
def _replayed_messages_for_cache_points(
|
||||
replayed_messages: Sequence[_MessageT], current_messages: Sequence[object], rules: _ItemMarkRules
|
||||
) -> Sequence[_MessageT]:
|
||||
as_system: Final = tuple(_developer_message_as_system_message(message) for message in replayed_messages)
|
||||
current_marks: Final = sum(_cache_control_count(message) for message in current_messages)
|
||||
return _messages_within_cache_control_limit(
|
||||
current_marks: Final = sum(cache_control_count(message) for message in current_messages)
|
||||
return whole_messages_within_cache_control_limit(
|
||||
tuple(map(rules.sanitized, as_system)), current_marks + rules.external_marks
|
||||
)
|
||||
|
||||
|
|
@ -1187,6 +1154,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
responses_api_request=responses_api_request,
|
||||
replay_reasoning=True,
|
||||
item_cache_control=keeps_item_marks,
|
||||
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs),
|
||||
native_compaction=native_compaction,
|
||||
only_since_compaction=not native_compaction,
|
||||
)
|
||||
|
|
@ -1250,6 +1218,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
responses_api_request: ResponsesAPIOptionalRequestParams | dict,
|
||||
replay_reasoning: bool = False,
|
||||
item_cache_control: bool = False,
|
||||
tool_call_provider_fields: bool = False,
|
||||
native_compaction: bool = False,
|
||||
only_since_compaction: bool = False,
|
||||
) -> list[
|
||||
|
|
@ -1267,7 +1236,9 @@ class LiteLLMCompletionResponsesConfig:
|
|||
assistant ``reasoning_content`` and signed ``thinking_blocks`` so the
|
||||
provider gets its own chain-of-thought back instead of reading it as
|
||||
visible text. ``item_cache_control`` belongs to the same callers, when
|
||||
the target keeps an input item's ``cache_control`` on the message.
|
||||
the target keeps an input item's ``cache_control`` on the message, and
|
||||
``tool_call_provider_fields`` when the target reads a function call's
|
||||
thought signature from ``provider_specific_fields`` instead of its id.
|
||||
|
||||
Callers that only inspect the messages (token counting, rate limiting,
|
||||
guardrail scanning) leave both off, because they need every piece of text
|
||||
|
|
@ -1305,7 +1276,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
read_input: Final = _input_to_read(dumped_input, only_since_compaction, native_compaction)
|
||||
messages.extend(
|
||||
LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message(
|
||||
input=read_input if item_cache_control else without_item_cache_control(read_input),
|
||||
input=_without_item_keys(read_input, _dropped_item_keys(item_cache_control, tool_call_provider_fields)),
|
||||
replay_reasoning=replay_reasoning,
|
||||
native_compaction=native_compaction,
|
||||
)
|
||||
|
|
@ -1363,6 +1334,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id,
|
||||
item_cache_control=keeps_item_marks,
|
||||
client_response_ids=previous_response_client_ids,
|
||||
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs),
|
||||
stored_turns=stored_turns,
|
||||
)
|
||||
chat_completion_session: Final = (
|
||||
|
|
@ -1489,7 +1461,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
messages.append(ChatCompletionUserMessage(role="user", content=input))
|
||||
elif isinstance(input, list):
|
||||
existing_tool_call_ids: Final[set[str]] = set()
|
||||
for _input in input:
|
||||
for _input in _without_paused_turn_thinking(input):
|
||||
chat_completion_messages = (
|
||||
LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message(
|
||||
input_item=_input,
|
||||
|
|
@ -2546,7 +2518,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
tool_call: Final = (
|
||||
ChatCompletionToolCallChunk(id=tool_call_id, type="function", function=tool_call_function, index=0)
|
||||
if function_call_cache_control is None
|
||||
else _MarkedToolCallChunk(
|
||||
else _ReplayedToolCallChunk(
|
||||
id=tool_call_id,
|
||||
type="function",
|
||||
function=tool_call_function,
|
||||
|
|
@ -2554,9 +2526,17 @@ class LiteLLMCompletionResponsesConfig:
|
|||
cache_control=function_call_cache_control,
|
||||
)
|
||||
)
|
||||
provider_specific_fields: Final = as_str_mapping(function_call.get("provider_specific_fields"))
|
||||
replayed_tool_call: Final = (
|
||||
tool_call
|
||||
if provider_specific_fields is None
|
||||
else cast( # cast-ok: the same tool call plus the provider_specific_fields the client sent back
|
||||
_ReplayedToolCallChunk, {**tool_call, "provider_specific_fields": provider_specific_fields}
|
||||
)
|
||||
)
|
||||
|
||||
# Create an assistant message with the tool call
|
||||
return [ChatCompletionResponseMessage(tool_calls=[tool_call], role="assistant", content=None)]
|
||||
return [ChatCompletionResponseMessage(tool_calls=[replayed_tool_call], role="assistant", content=None)]
|
||||
|
||||
@staticmethod
|
||||
def _transform_input_file_item_to_file_item(item: Mapping[str, object]) -> _ResponsesFilePart:
|
||||
|
|
@ -2601,45 +2581,55 @@ class LiteLLMCompletionResponsesConfig:
|
|||
for item in content:
|
||||
if isinstance(item, str):
|
||||
content_list.append(item)
|
||||
elif isinstance(item, dict):
|
||||
if (
|
||||
media := decode_responses_media(
|
||||
fields := _validate_responses_media(_STR_KEY_DICT_ADAPTER, item)
|
||||
elif (
|
||||
isinstance(item, dict)
|
||||
and (
|
||||
part
|
||||
:= LiteLLMCompletionResponsesConfig._transform_responses_api_content_part_to_chat_completion_part(
|
||||
_validate_responses_media(_STR_KEY_DICT_ADAPTER, item)
|
||||
)
|
||||
) is not None:
|
||||
content_list.append(dict(media))
|
||||
elif fields.get("type") == "input_image":
|
||||
content_list.append(dict(_responses_image_part(fields)))
|
||||
elif fields.get("type") == "input_file":
|
||||
content_list.append(dict(_responses_file_part(fields)))
|
||||
elif fields.get("type") == "input_audio":
|
||||
content_list.append(dict(_validate_responses_media(_AUDIO_PART_ADAPTER, fields)))
|
||||
elif fields.get("type") == "encrypted_content":
|
||||
encrypted_content = fields.get("encrypted_content")
|
||||
if encrypted_content is not None:
|
||||
content_list.append(
|
||||
OpenAIChatCompletionTextObject(type="text", text=str(encrypted_content))
|
||||
)
|
||||
else:
|
||||
# Skip text blocks with None text to avoid downstream errors
|
||||
text_value = fields.get("text")
|
||||
if text_value is None:
|
||||
continue
|
||||
content_block: dict[str, object] = {
|
||||
"type": LiteLLMCompletionResponsesConfig._get_chat_completion_request_content_type(
|
||||
content_type
|
||||
if isinstance(content_type := fields.get("type"), str) and content_type
|
||||
else "text"
|
||||
),
|
||||
"text": text_value,
|
||||
}
|
||||
if "cache_control" in fields:
|
||||
content_block["cache_control"] = fields["cache_control"]
|
||||
content_list.append(content_block)
|
||||
)
|
||||
is not None
|
||||
):
|
||||
content_list.append(part)
|
||||
return content_list
|
||||
else:
|
||||
raise ValueError(f"Invalid content type: {type(content)}")
|
||||
|
||||
@staticmethod
|
||||
def _transform_responses_api_content_part_to_chat_completion_part(
|
||||
fields: dict[str, object],
|
||||
) -> dict[str, object] | None:
|
||||
if (media := decode_responses_media(fields)) is not None:
|
||||
return dict(media)
|
||||
part_type: Final = fields.get("type")
|
||||
if part_type == "input_image":
|
||||
return dict(_responses_image_part(fields))
|
||||
if part_type == "input_file":
|
||||
return dict(_responses_file_part(fields))
|
||||
if part_type == "input_audio":
|
||||
return dict(_validate_responses_media(_AUDIO_PART_ADAPTER, fields))
|
||||
if part_type == "refusal":
|
||||
refusal: Final = fields.get("refusal")
|
||||
return dict(OpenAIChatCompletionTextObject(type="text", text=refusal)) if isinstance(refusal, str) else None
|
||||
if part_type == "encrypted_content":
|
||||
encrypted_content: Final = fields.get("encrypted_content")
|
||||
return (
|
||||
None
|
||||
if encrypted_content is None
|
||||
else dict(OpenAIChatCompletionTextObject(type="text", text=str(encrypted_content)))
|
||||
)
|
||||
text_value: Final = fields.get("text")
|
||||
if text_value is None:
|
||||
return None
|
||||
return {
|
||||
"type": LiteLLMCompletionResponsesConfig._get_chat_completion_request_content_type(
|
||||
part_type if isinstance(part_type, str) and part_type else "text"
|
||||
),
|
||||
"text": text_value,
|
||||
**({"cache_control": fields["cache_control"]} if "cache_control" in fields else {}),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _get_chat_completion_request_content_type(
|
||||
content_type: str,
|
||||
|
|
@ -3118,6 +3108,18 @@ class LiteLLMCompletionResponsesConfig:
|
|||
# Default to completed for unknown finish reasons
|
||||
return "completed"
|
||||
|
||||
@staticmethod
|
||||
def _is_paused_turn(choice: Choices) -> bool:
|
||||
return anthropic_stop_reason(choice.message) == "pause_turn"
|
||||
|
||||
@staticmethod
|
||||
def _responses_status_for_choice(choice: Choices) -> ResponsesAPIStatus:
|
||||
if LiteLLMCompletionResponsesConfig._is_paused_turn(choice):
|
||||
return "completed"
|
||||
return LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
|
||||
choice.finish_reason
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _incomplete_details_for_finish_reason(
|
||||
finish_reason: str | None,
|
||||
|
|
@ -3254,19 +3256,21 @@ class LiteLLMCompletionResponsesConfig:
|
|||
"""
|
||||
if isinstance(chat_completion_response, dict):
|
||||
chat_completion_response = ModelResponse(**chat_completion_response)
|
||||
# Get finish_reason from the first choice to determine overall status
|
||||
finish_reason: str | None = None
|
||||
choices: Final[list[Choices]] = getattr(chat_completion_response, "choices", [])
|
||||
if choices and len(choices) > 0:
|
||||
finish_reason = choices[0].finish_reason
|
||||
finish_reason: Final = choices[0].finish_reason if choices else None
|
||||
paused_turn: Final = bool(choices) and LiteLLMCompletionResponsesConfig._is_paused_turn(choices[0])
|
||||
|
||||
echoed: Final = _echoable_request_params(responses_api_request)
|
||||
chat_usage: Final = getattr(chat_completion_response, "usage", None)
|
||||
incomplete_details: Final = LiteLLMCompletionResponsesConfig._incomplete_details_for_finish_reason(
|
||||
finish_reason=finish_reason,
|
||||
existing=getattr(chat_completion_response, "incomplete_details", None),
|
||||
completion_tokens=getattr(chat_usage, "completion_tokens", None),
|
||||
max_output_tokens=echoed.get("max_output_tokens"),
|
||||
incomplete_details: Final = (
|
||||
None
|
||||
if paused_turn
|
||||
else LiteLLMCompletionResponsesConfig._incomplete_details_for_finish_reason(
|
||||
finish_reason=finish_reason,
|
||||
existing=getattr(chat_completion_response, "incomplete_details", None),
|
||||
completion_tokens=getattr(chat_usage, "completion_tokens", None),
|
||||
max_output_tokens=echoed.get("max_output_tokens"),
|
||||
)
|
||||
)
|
||||
|
||||
responses_api_response: Final[ResponsesAPIResponse] = ResponsesAPIResponse(
|
||||
|
|
@ -3293,8 +3297,8 @@ class LiteLLMCompletionResponsesConfig:
|
|||
max_output_tokens=echoed.get("max_output_tokens"),
|
||||
previous_response_id=echoed.get("previous_response_id"),
|
||||
reasoning=echoed.get("reasoning"),
|
||||
status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
|
||||
finish_reason
|
||||
status=(
|
||||
LiteLLMCompletionResponsesConfig._responses_status_for_choice(choices[0]) if choices else "incomplete"
|
||||
),
|
||||
text=echoed.get("text") or {},
|
||||
truncation=echoed.get("truncation"),
|
||||
|
|
@ -3440,9 +3444,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
GenericResponseOutputItem(
|
||||
type="reasoning",
|
||||
id=mint_reasoning_item_id(),
|
||||
status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
|
||||
choice.finish_reason
|
||||
),
|
||||
status=LiteLLMCompletionResponsesConfig._responses_status_for_choice(choice),
|
||||
role="assistant",
|
||||
content=[
|
||||
OutputText(
|
||||
|
|
@ -3565,24 +3567,34 @@ class LiteLLMCompletionResponsesConfig:
|
|||
choice=choice,
|
||||
)
|
||||
message_output_items.extend(image_generation_items)
|
||||
elif choice.message.content is not None or getattr(choice.message, "annotations", None):
|
||||
elif content := LiteLLMCompletionResponsesConfig._message_output_content(choice.message):
|
||||
message_output_items.append(
|
||||
GenericResponseOutputItem(
|
||||
type="message",
|
||||
id=f"msg_{uuid.uuid4()}",
|
||||
status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
|
||||
choice.finish_reason
|
||||
),
|
||||
status=LiteLLMCompletionResponsesConfig._responses_status_for_choice(choice),
|
||||
role=choice.message.role,
|
||||
content=[
|
||||
LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text(
|
||||
choice.message
|
||||
)
|
||||
],
|
||||
content=list(content),
|
||||
)
|
||||
)
|
||||
return message_output_items
|
||||
|
||||
@staticmethod
|
||||
def _message_output_content(message: Message) -> tuple[OutputText | ResponseOutputRefusal, ...]:
|
||||
refusal: Final = openai_chat_refusal_text(message)
|
||||
refusal_parts: Final = (ResponseOutputRefusal(type="refusal", refusal=refusal),) if refusal is not None else ()
|
||||
has_text_part: Final = (
|
||||
bool(message.content)
|
||||
or bool(getattr(message, "annotations", None))
|
||||
or (message.content is not None and not refusal_parts)
|
||||
)
|
||||
text_parts: Final = (
|
||||
(LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text(message),)
|
||||
if has_text_part
|
||||
else ()
|
||||
)
|
||||
return (*text_parts, *refusal_parts)
|
||||
|
||||
@staticmethod
|
||||
def _transform_responses_api_outputs_to_chat_completion_messages(
|
||||
responses_api_output: ResponsesAPIResponse,
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm.responses.mcp.tool_precheck import (
|
|||
LITELLM_PROXY_MCP_SERVER_URL_PREFIX,
|
||||
may_name_litellm_mcp_gateway,
|
||||
names_gateway_explicitly,
|
||||
names_whole_gateway,
|
||||
proxy_path_mcp_name,
|
||||
)
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
|
||||
|
|
@ -182,8 +183,8 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
|
||||
@staticmethod
|
||||
async def _apply_toolset_permissions(
|
||||
resolved_toolset_ids: list[str],
|
||||
resolved_mcp_servers: list[str],
|
||||
resolved_toolset_ids: Sequence[str],
|
||||
resolved_mcp_servers: Sequence[str],
|
||||
user_api_key_auth: "UserAPIKeyAuth",
|
||||
) -> "UserAPIKeyAuth":
|
||||
"""Apply resolved toolset permissions to user_api_key_auth and return updated auth."""
|
||||
|
|
@ -195,7 +196,7 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
)
|
||||
|
||||
tool_permissions: Final = await global_mcp_server_manager.resolve_toolset_tool_permissions(
|
||||
toolset_ids=resolved_toolset_ids
|
||||
toolset_ids=list(resolved_toolset_ids)
|
||||
)
|
||||
all_server_ids: Final = list(set(tool_permissions.keys()) | set(resolved_mcp_servers))
|
||||
existing_op: Final = user_api_key_auth.object_permission
|
||||
|
|
@ -224,6 +225,62 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
verbose_logger.debug("Could not apply toolset permissions: %s", _e)
|
||||
return user_api_key_auth
|
||||
|
||||
@staticmethod
|
||||
async def _resolve_proxy_mcp_name(
|
||||
name: str,
|
||||
user_api_key_auth: "UserAPIKeyAuth | None",
|
||||
granted_toolsets: "GrantedToolsetIds | None",
|
||||
) -> tuple[Literal["server", "toolset", "denied_toolset"], str]:
|
||||
from litellm.proxy.mcp_server.mcp_server_manager import (
|
||||
global_mcp_server_manager,
|
||||
)
|
||||
|
||||
if global_mcp_server_manager.get_mcp_server_by_name(name):
|
||||
return ("server", name)
|
||||
try:
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
if prisma_client is None:
|
||||
return ("server", name)
|
||||
toolset: Final = await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name)
|
||||
if toolset is None:
|
||||
return ("server", name)
|
||||
from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import (
|
||||
has_team_unbound_mcp_admin_view,
|
||||
)
|
||||
from litellm.proxy.mcp_server.ui_session_utils import (
|
||||
granted_toolset_ids,
|
||||
)
|
||||
|
||||
denied: Final = (
|
||||
user_api_key_auth is not None
|
||||
and not has_team_unbound_mcp_admin_view(user_api_key_auth)
|
||||
and toolset.toolset_id not in await (granted_toolsets or granted_toolset_ids)(user_api_key_auth)
|
||||
)
|
||||
except Exception as _e:
|
||||
verbose_logger.debug("Could not resolve '%s' as toolset: %s", name, _e)
|
||||
return ("server", name)
|
||||
if not denied:
|
||||
return ("toolset", toolset.toolset_id)
|
||||
if await LiteLLM_Proxy_MCP_Handler._holds_access_group_named(name, user_api_key_auth):
|
||||
return ("server", name)
|
||||
verbose_logger.debug("Key does not have access to toolset '%s', denying it.", name)
|
||||
return ("denied_toolset", name)
|
||||
|
||||
@staticmethod
|
||||
async def _holds_access_group_named(name: str, user_api_key_auth: "UserAPIKeyAuth | None") -> bool:
|
||||
from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import (
|
||||
MCPRequestHandler,
|
||||
)
|
||||
from litellm.proxy.mcp_server.mcp_server_manager import (
|
||||
global_mcp_server_manager,
|
||||
)
|
||||
|
||||
group_server_ids: Final = await MCPRequestHandler.get_mcp_servers_from_access_groups([name])
|
||||
return bool(group_server_ids) and not frozenset(group_server_ids).isdisjoint(
|
||||
await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@catalog_operation(global_manager)
|
||||
async def _get_mcp_tools_from_manager(
|
||||
|
|
@ -256,6 +313,7 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
from litellm.proxy.mcp_server.server import (
|
||||
_get_allowed_mcp_servers_from_mcp_server_names,
|
||||
_get_tools_from_mcp_servers,
|
||||
raise_generic_mcp_access_denial,
|
||||
)
|
||||
|
||||
mcp_servers: Final = [
|
||||
|
|
@ -268,36 +326,15 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
# Resolve toolset names: collect all toolset IDs first, then apply their
|
||||
# combined permissions in a single pass so multiple toolsets are unioned
|
||||
# rather than the last one overwriting the others.
|
||||
resolved_mcp_servers: Final[list[str]] = []
|
||||
resolved_toolset_ids: Final[list[str]] = []
|
||||
for name in mcp_servers:
|
||||
if not global_mcp_server_manager.get_mcp_server_by_name(name):
|
||||
try:
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
if prisma_client is not None:
|
||||
toolset = await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name)
|
||||
if toolset is not None:
|
||||
if user_api_key_auth is not None:
|
||||
from litellm.proxy.management_endpoints.common_utils import (
|
||||
user_api_key_has_admin_view,
|
||||
)
|
||||
from litellm.proxy.mcp_server.ui_session_utils import (
|
||||
granted_toolset_ids,
|
||||
)
|
||||
|
||||
if not user_api_key_has_admin_view(user_api_key_auth) and toolset.toolset_id not in (
|
||||
await (granted_toolsets or granted_toolset_ids)(user_api_key_auth)
|
||||
):
|
||||
verbose_logger.debug("Key does not have access to toolset '%s', skipping.", name)
|
||||
continue
|
||||
resolved_toolset_ids.append(toolset.toolset_id)
|
||||
# Don't add to resolved_mcp_servers — toolset scope
|
||||
# restricts via object_permission, not server name filter.
|
||||
continue
|
||||
except Exception as _e:
|
||||
verbose_logger.debug("Could not resolve '%s' as toolset: %s", name, _e)
|
||||
resolved_mcp_servers.append(name)
|
||||
resolutions: Final = tuple(
|
||||
[
|
||||
await LiteLLM_Proxy_MCP_Handler._resolve_proxy_mcp_name(name, user_api_key_auth, granted_toolsets)
|
||||
for name in mcp_servers
|
||||
]
|
||||
)
|
||||
resolved_mcp_servers: Final = tuple(value for kind, value in resolutions if kind == "server")
|
||||
resolved_toolset_ids: Final = tuple(value for kind, value in resolutions if kind == "toolset")
|
||||
denied_toolset_names: Final = tuple(value for kind, value in resolutions if kind == "denied_toolset")
|
||||
|
||||
# Apply all resolved toolsets at once (union), avoiding permission overwrite.
|
||||
if resolved_toolset_ids and user_api_key_auth is not None:
|
||||
|
|
@ -311,7 +348,10 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
# full union (toolset server IDs + direct server names). Passing a name-based
|
||||
# filter here would exclude those toolset server IDs (which are UUIDs, not
|
||||
# names), so use None and let the auth object's mcp_servers do the filtering.
|
||||
effective_server_filter: Final = None if resolved_toolset_ids else (resolved_mcp_servers or None)
|
||||
asks_whole_gateway: Final = any(map(names_whole_gateway, mcp_tools_with_litellm_proxy or ()))
|
||||
if denied_toolset_names and not (asks_whole_gateway or resolved_mcp_servers or resolved_toolset_ids):
|
||||
raise_generic_mcp_access_denial(denied_toolset_names)
|
||||
effective_server_filter: Final = None if resolved_toolset_ids else (list(resolved_mcp_servers) or None)
|
||||
|
||||
listing: Final = await _get_tools_from_mcp_servers(
|
||||
user_api_key_auth=user_api_key_auth,
|
||||
|
|
|
|||
|
|
@ -22,6 +22,13 @@ def names_gateway_explicitly(tool: object) -> bool:
|
|||
return (_mcp_server_url(tool) or "").startswith(LITELLM_PROXY_MCP_SERVER_URL)
|
||||
|
||||
|
||||
def names_whole_gateway(tool: object) -> bool:
|
||||
server_url: Final = _mcp_server_url(tool) or ""
|
||||
return server_url.startswith(LITELLM_PROXY_MCP_SERVER_URL) and not server_url.startswith(
|
||||
LITELLM_PROXY_MCP_SERVER_URL_PREFIX
|
||||
)
|
||||
|
||||
|
||||
def proxy_path_mcp_name(tool: object) -> str | None:
|
||||
server_url: Final = _mcp_server_url(tool)
|
||||
match: Final = None if server_url is None else _PROXY_MCP_PATH_RE.match(server_url)
|
||||
|
|
|
|||
|
|
@ -35,9 +35,10 @@ import litellm
|
|||
from litellm.constants import (
|
||||
EMPTY_MAPPING,
|
||||
LITELLM_MAX_STREAMING_DURATION_SECONDS,
|
||||
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY,
|
||||
STREAM_SSE_DONE_STRING,
|
||||
)
|
||||
from litellm.exceptions import MidStreamFallbackError, RateLimitError
|
||||
from litellm.exceptions import MidStreamFallbackError, RateLimitError, ServiceUnavailableError
|
||||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
process_response_headers,
|
||||
|
|
@ -100,6 +101,12 @@ class ProjectQuotaCallback(Protocol):
|
|||
) -> None: ...
|
||||
|
||||
|
||||
class TokenLimitCallback(Protocol):
|
||||
async def enforce_token_limits_for_frame(
|
||||
self, user_api_key_dict: UserAPIKeyAuth, requested_model: str | None
|
||||
) -> None: ...
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_openai_response_types():
|
||||
from litellm.types.llms import openai as openai_types
|
||||
|
|
@ -2158,6 +2165,25 @@ class _RefusedTurn:
|
|||
ResponsesWsTurnAdmission: TypeAlias = Callable[[], Awaitable[Mapping[str, object]]]
|
||||
|
||||
|
||||
def with_token_limit_checks(
|
||||
turn_admission: ResponsesWsTurnAdmission | None,
|
||||
token_limit_callbacks: Sequence[TokenLimitCallback],
|
||||
user_api_key_dict: UserAPIKeyAuth | None,
|
||||
requested_model: str | None,
|
||||
) -> ResponsesWsTurnAdmission | None:
|
||||
if not token_limit_callbacks or user_api_key_dict is None:
|
||||
return turn_admission
|
||||
|
||||
async def admit() -> Mapping[str, object]:
|
||||
for callback in token_limit_callbacks:
|
||||
await callback.enforce_token_limits_for_frame(
|
||||
user_api_key_dict=user_api_key_dict, requested_model=requested_model
|
||||
)
|
||||
return MappingProxyType({}) if turn_admission is None else await turn_admission()
|
||||
|
||||
return admit
|
||||
|
||||
|
||||
async def _admit_frame_turn(
|
||||
turn_admission: ResponsesWsTurnAdmission | None, raw_message: str
|
||||
) -> _AdmittedTurn | _RefusedTurn:
|
||||
|
|
@ -2173,22 +2199,24 @@ async def _admit_frame_turn(
|
|||
return _RefusedTurn(e)
|
||||
|
||||
|
||||
def _turn_refusal_frame(refusal: RateLimitError) -> str:
|
||||
def turn_refusal_frame(refusal: RateLimitError) -> str:
|
||||
return json.dumps(
|
||||
{"type": "error", "status": 429, "error": {"type": "rate_limit_exceeded", "message": str(refusal)}}
|
||||
)
|
||||
|
||||
|
||||
def _turn_admission_error_frame(failure: Exception) -> str:
|
||||
return json.dumps({"type": "error", "status": 500, "error": {"type": "server_error", "message": str(failure)}})
|
||||
def _turn_admission_error_frame(failure: Exception, status: int) -> str:
|
||||
return json.dumps({"type": "error", "status": status, "error": {"type": "server_error", "message": str(failure)}})
|
||||
|
||||
|
||||
def _turn_rejection_frame(rejection: Exception) -> str:
|
||||
match rejection:
|
||||
case RateLimitError():
|
||||
return _turn_refusal_frame(rejection)
|
||||
return turn_refusal_frame(rejection)
|
||||
case ServiceUnavailableError():
|
||||
return _turn_admission_error_frame(rejection, rejection.status_code)
|
||||
case _:
|
||||
return _turn_admission_error_frame(rejection)
|
||||
return _turn_admission_error_frame(rejection, 500)
|
||||
|
||||
|
||||
RESPONSES_WS_LOGGED_EVENT_TYPES: Final = [
|
||||
|
|
@ -3015,7 +3043,7 @@ class ResponsesWebSocketStreaming:
|
|||
|
||||
async def _send_turn_refusal(self, refusal: RateLimitError) -> None:
|
||||
try:
|
||||
await self.websocket.send_text(_turn_refusal_frame(refusal))
|
||||
await self.websocket.send_text(turn_refusal_frame(refusal))
|
||||
except Exception: # noqa: BLE001, S110 # client may already be gone
|
||||
pass
|
||||
|
||||
|
|
@ -3569,7 +3597,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
self.quota_callbacks, self.user_api_key_dict, self.model_group or self.model, raw_message
|
||||
)
|
||||
except RateLimitError as e:
|
||||
await self._send_frame(_turn_refusal_frame(e))
|
||||
await self._send_frame(turn_refusal_frame(e))
|
||||
return
|
||||
guarded: Final = await self._guard_frame(raw_message)
|
||||
if guarded is None:
|
||||
|
|
@ -3613,6 +3641,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
call_kwargs["litellm_metadata"] = {
|
||||
**(turn_litellm_metadata if _is_json_object(turn_litellm_metadata) else {}),
|
||||
**verdict.reservation,
|
||||
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY: self.logging_obj.litellm_call_id,
|
||||
}
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -147,6 +147,7 @@ from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
|
|||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference
|
||||
from litellm.litellm_core_utils.dd_tracing import tracer
|
||||
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models
|
||||
from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
declared_authenticating_provider,
|
||||
inferred_provider,
|
||||
|
|
@ -483,6 +484,7 @@ from litellm.types.router import (
|
|||
SearchToolTypedDict,
|
||||
TaggedPreRoutingStrategy,
|
||||
holds_secret_pointer,
|
||||
validate_litellm_params_strictly,
|
||||
)
|
||||
from litellm.types.services import ServiceTypes
|
||||
from litellm.types.utils import (
|
||||
|
|
@ -640,6 +642,7 @@ _PRE_CALL_CHECK_FILTERED_PARAMS: Final = frozenset({"response_format"})
|
|||
_STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"})
|
||||
_video_call_deployment_id: Final[ContextVar[str | None]] = ContextVar("video_call_deployment_id", default=None)
|
||||
_routed_call_type: Final[ContextVar[str | None]] = ContextVar("routed_call_type", default=None)
|
||||
_DECISIONS_MEDIA_CALL_TYPES: Final = frozenset({"adecisions", "decisions", "asystemone", "systemone"})
|
||||
_ROUTE_CAPABILITY_RESOLVERS: Final[Mapping[str, Callable[[str, LlmProviders], object | None]]] = MappingProxyType(
|
||||
{
|
||||
"adecisions": ProviderConfigManager.get_provider_decisions_config,
|
||||
|
|
@ -727,6 +730,35 @@ def _drops_malformed_request_tools(
|
|||
return drop_policy({**deployment_params, **set_defaults, **request}).drops("tools")
|
||||
|
||||
|
||||
def _malformed_request_params_the_deployment_sets(
|
||||
deployment_params: Mapping[str, object], defaults: Mapping[str, object], request: Mapping[str, object]
|
||||
) -> frozenset[str]:
|
||||
overrides: Final = {
|
||||
key: value
|
||||
for key, value in request.items()
|
||||
if key in SUPPLEMENTAL_KWARGS_KEYS and deployment_params.get(key) is not None
|
||||
}
|
||||
if not overrides:
|
||||
return frozenset()
|
||||
try:
|
||||
validate_litellm_params_strictly(GenericLiteLLMParams, overrides)
|
||||
except ValidationError as error:
|
||||
set_defaults: Final = {name: value for name, value in defaults.items() if value is not None}
|
||||
policy: Final = drop_policy({**deployment_params, **set_defaults, **request})
|
||||
malformed: Final = (str(detail["loc"][0]) for detail in error.errors() if detail["loc"])
|
||||
return frozenset(key for key in malformed if policy.drops(key))
|
||||
return frozenset()
|
||||
|
||||
|
||||
def _drop_malformed_request_params_the_deployment_sets(
|
||||
kwargs: dict[str, object], # mutable-ok: drops the keys from the router's request kwargs in place
|
||||
deployment_params: Mapping[str, object],
|
||||
defaults: Mapping[str, object],
|
||||
) -> None:
|
||||
for key in _malformed_request_params_the_deployment_sets(deployment_params, defaults, kwargs):
|
||||
kwargs.pop(key)
|
||||
|
||||
|
||||
def _merged_tools(deployment_tools: Sequence[object], request_tools: object) -> list[object] | None:
|
||||
if not isinstance(request_tools, (list, tuple)):
|
||||
return None
|
||||
|
|
@ -4162,13 +4194,16 @@ class Router(DeprecatedRouterMembers):
|
|||
raise e.original_exception from e
|
||||
raise e
|
||||
|
||||
def _active_default_litellm_params(self) -> Mapping[str, object]:
|
||||
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params)
|
||||
return {key: value for key, value in default_params.items() if value is not None}
|
||||
|
||||
def _params_sent_to_deployment(
|
||||
self, request_kwargs: Mapping[str, object], deployment: Deployment | None
|
||||
) -> Mapping[str, object]:
|
||||
"""The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies."""
|
||||
deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {}
|
||||
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params)
|
||||
defaults: Final = {key: value for key, value in default_params.items() if value is not None}
|
||||
defaults: Final = self._active_default_litellm_params()
|
||||
deployment_tool_choice: Final = (
|
||||
{"tool_choice": deployment_params["tool_choice"]}
|
||||
if deployment_params.get("tool_choice") is not None
|
||||
|
|
@ -5223,6 +5258,11 @@ class Router(DeprecatedRouterMembers):
|
|||
if isinstance(raw_deployment_params, LiteLLM_Params)
|
||||
else raw_deployment_params
|
||||
)
|
||||
_drop_malformed_request_params_the_deployment_sets(
|
||||
kwargs, # pyright: ignore[reportUnknownArgumentType] # kwargs is the router's untyped request dict
|
||||
deployment_context,
|
||||
defaults,
|
||||
)
|
||||
declared_provider: Final = deployment_context.get("custom_llm_provider")
|
||||
declared_model: Final = deployment_context.get("model")
|
||||
deployment_provider: Final = (
|
||||
|
|
@ -10980,7 +11020,9 @@ class Router(DeprecatedRouterMembers):
|
|||
merged_params: Final[Mapping[str, object]] = (
|
||||
_litellm_params if zeroed_pricing is None else MappingProxyType({**_litellm_params, **zeroed_pricing})
|
||||
)
|
||||
litellm_params: Final[LiteLLM_Params] = LiteLLM_Params.model_validate(dict(**merged_params))
|
||||
litellm_params: Final[LiteLLM_Params] = validate_litellm_params_strictly(
|
||||
LiteLLM_Params, dict(**merged_params)
|
||||
)
|
||||
declared_params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(_litellm_params)
|
||||
warn_on_provider_credential_mismatch(model_name=_model_name, litellm_params=declared_params)
|
||||
warn_on_ignored_model_capability_flags(model_name=_model_name, litellm_params=declared_params)
|
||||
|
|
@ -14952,6 +14994,62 @@ class Router(DeprecatedRouterMembers):
|
|||
return healthy_deployments
|
||||
return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates
|
||||
|
||||
@staticmethod
|
||||
def _decisions_deployment_refuses_media(
|
||||
deployment: DeploymentTypedDict, group: str, request_kwargs: Mapping[str, object], *, audio: bool, videos: bool
|
||||
) -> bool:
|
||||
from litellm.decisions.call import deployment_refuses_media
|
||||
|
||||
plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts
|
||||
model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict
|
||||
deployment_params: Final = cast(Mapping[str, object], deployment["litellm_params"]) # cast-ok: plain dict
|
||||
try:
|
||||
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
|
||||
return deployment_refuses_media(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
model_info,
|
||||
{**deployment_params, **request_kwargs},
|
||||
audio=audio,
|
||||
videos=videos,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # best-effort filter: a deployment it cannot resolve answers for itself
|
||||
verbose_router_logger.debug(
|
||||
"litellm.router.py::_decisions_deployment_refuses_media: keeping a deployment of model=%s. Got - %s",
|
||||
group,
|
||||
e,
|
||||
)
|
||||
return False
|
||||
|
||||
def _without_decisions_deployments_refusing_media(
|
||||
self, healthy_deployments: _RouteDeploymentsT, group: str, request_kwargs: Mapping[str, object] | None
|
||||
) -> _RouteDeploymentsT:
|
||||
call_kwargs: Final = request_kwargs or {}
|
||||
audio: Final = bool(call_kwargs.get("audio"))
|
||||
videos: Final = bool(call_kwargs.get("videos"))
|
||||
if (
|
||||
_routed_call_type.get() not in _DECISIONS_MEDIA_CALL_TYPES
|
||||
or not (audio or videos)
|
||||
or isinstance(healthy_deployments, Mapping)
|
||||
):
|
||||
return healthy_deployments
|
||||
|
||||
sent_kwargs: Final = {**self._active_default_litellm_params(), **call_kwargs}
|
||||
deployments_taking_media: Final = [
|
||||
deployment
|
||||
for deployment in healthy_deployments
|
||||
if not Router._decisions_deployment_refuses_media(
|
||||
cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape
|
||||
group,
|
||||
sent_kwargs,
|
||||
audio=audio,
|
||||
videos=videos,
|
||||
)
|
||||
]
|
||||
if not deployments_taking_media:
|
||||
return healthy_deployments
|
||||
return cast(_RouteDeploymentsT, deployments_taking_media) # cast-ok: same list shape as the input candidates
|
||||
|
||||
@staticmethod
|
||||
def _without_declared_text_only_deployments(
|
||||
healthy_deployments: _RouteDeploymentsT,
|
||||
|
|
@ -15676,10 +15774,14 @@ class Router(DeprecatedRouterMembers):
|
|||
|
||||
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
|
||||
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
|
||||
batch_candidates: Final = self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
batch_candidates: Final = self._without_decisions_deployments_refusing_media(
|
||||
self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
),
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
)
|
||||
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(
|
||||
|
|
@ -16932,10 +17034,14 @@ class Router(DeprecatedRouterMembers):
|
|||
|
||||
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
|
||||
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
|
||||
batch_candidates: Final = self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
batch_candidates: Final = self._without_decisions_deployments_refusing_media(
|
||||
self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
),
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
)
|
||||
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(
|
||||
|
|
|
|||
|
|
@ -288,6 +288,7 @@ class PromptOptions:
|
|||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ResponseOptions:
|
||||
merge_reasoning_content_in_choices: bool | None = None
|
||||
thought_signature_in_tool_call_id: bool | None = None
|
||||
enable_json_schema_validation: bool | None = None
|
||||
complete_response: bool | None = None
|
||||
keepalive_seconds: float | None = None
|
||||
|
|
|
|||
|
|
@ -751,7 +751,9 @@ class AnthropicOutputTokensDetails(LiteLLMBaseModel):
|
|||
thinking_tokens: int | None = None
|
||||
|
||||
|
||||
AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"]
|
||||
AnthropicFinishReason = Literal[
|
||||
"end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "pause_turn", "compaction"
|
||||
]
|
||||
|
||||
|
||||
class AnthropicResponse(LiteLLMBaseModel):
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ class AnthropicMessagesResponse(TypedDict, total=False):
|
|||
model: str | None # This represents the Model type from Anthropic
|
||||
role: Literal["assistant"] | None
|
||||
stop_reason: ReadOnly[
|
||||
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"] | None
|
||||
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction", "pause_turn"] | None
|
||||
]
|
||||
stop_details: NotRequired[ReadOnly[AnthropicStopDetails | None]]
|
||||
stop_sequence: str | None
|
||||
|
|
|
|||
|
|
@ -70,6 +70,7 @@ from pydantic import (
|
|||
FieldSerializationInfo,
|
||||
NonNegativeInt,
|
||||
PrivateAttr,
|
||||
SerializeAsAny,
|
||||
SerializerFunctionWrapHandler,
|
||||
field_serializer,
|
||||
field_validator,
|
||||
|
|
@ -402,7 +403,7 @@ class OpenAIFileObject(LiteLLMBaseModel):
|
|||
filename: str
|
||||
"""The name of the file."""
|
||||
|
||||
object: Literal["file"]
|
||||
object: Literal["file"] = "file"
|
||||
"""The object type, which is always `file`."""
|
||||
|
||||
purpose: OpenAIFilesPurpose
|
||||
|
|
@ -440,6 +441,11 @@ class OpenAIFileObject(LiteLLMBaseModel):
|
|||
|
||||
_hidden_params: dict = PrivateAttr(default={"response_cost": 0.0}) # no cost for writing a file
|
||||
|
||||
@field_validator("object", mode="before")
|
||||
@classmethod
|
||||
def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue:
|
||||
return "file" if value is None else value
|
||||
|
||||
@property
|
||||
def hidden_params(self) -> dict[str, builtins.object]: # mutable-ok: API requires mutation
|
||||
return self._hidden_params
|
||||
|
|
@ -1850,7 +1856,7 @@ class ContentPartAddedEvent(BaseLiteLLMOpenAIResponseObject):
|
|||
item_id: str
|
||||
output_index: int
|
||||
content_index: int
|
||||
part: BaseLiteLLMOpenAIResponseObject
|
||||
part: SerializeAsAny[BaseLiteLLMOpenAIResponseObject]
|
||||
|
||||
|
||||
class ContentPartDonePartOutputText(BaseLiteLLMOpenAIResponseObject):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Final
|
||||
from typing import Final, Protocol
|
||||
|
||||
from pydantic import Field
|
||||
from typing_extensions import TypedDict
|
||||
|
|
@ -75,6 +76,19 @@ class PassthroughStandardLoggingPayload(TypedDict, total=False):
|
|||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WebSocketSessionUsage:
|
||||
model: str
|
||||
custom_llm_provider: str
|
||||
response_cost: float | None
|
||||
|
||||
|
||||
class WebSocketSessionMeter(Protocol):
|
||||
def measure(self, client_text: str) -> int: ...
|
||||
|
||||
def price(self, total: int) -> WebSocketSessionUsage: ...
|
||||
|
||||
|
||||
class PassThroughGuardrailSettings(LiteLLMPydanticObjectBase):
|
||||
"""
|
||||
Settings for a specific guardrail on a passthrough endpoint.
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from typing import Final, Literal, Optional, Union
|
|||
|
||||
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
|
||||
from openai.types.responses.response_function_web_search import ActionSearchSource, ResponseFunctionWebSearch
|
||||
from openai.types.responses.response_output_refusal import ResponseOutputRefusal
|
||||
from pydantic import ConfigDict, PrivateAttr
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
|
|
@ -154,7 +155,7 @@ class GenericResponseOutputItem(BaseLiteLLMOpenAIResponseObject):
|
|||
id: str
|
||||
status: str # "completed", "in_progress", etc.
|
||||
role: str # "assistant", "user", etc.
|
||||
content: list[OutputText]
|
||||
content: list[OutputText | ResponseOutputRefusal]
|
||||
phase: Phase = None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,20 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
|||
|
||||
import httpx
|
||||
from openai import Timeout as SDKTimeout
|
||||
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictInt, field_validator, model_validator
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
BeforeValidator,
|
||||
ConfigDict,
|
||||
Field,
|
||||
JsonValue,
|
||||
StrictInt,
|
||||
ValidationError,
|
||||
ValidationInfo,
|
||||
ValidatorFunctionWrapHandler,
|
||||
WrapValidator,
|
||||
field_validator,
|
||||
model_validator,
|
||||
)
|
||||
from typing_extensions import NotRequired, Protocol, ReadOnly, Required, TypedDict, runtime_checkable
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -531,6 +544,27 @@ def holds_secret_pointer(param_name: str) -> bool:
|
|||
|
||||
_RESERVED_INIT_KEYS: Final = frozenset({"self", "params", "__class__"})
|
||||
|
||||
_STRICT_LITELLM_PARAMS: Final = MappingProxyType({"strict_litellm_params": True})
|
||||
|
||||
_StrictlyValidatedParams: Final = TypeVar("_StrictlyValidatedParams", bound=BaseModel)
|
||||
|
||||
|
||||
def validate_litellm_params_strictly(
|
||||
params_model: type[_StrictlyValidatedParams], params: Mapping[str, object]
|
||||
) -> _StrictlyValidatedParams:
|
||||
return params_model.model_validate(params, context=_STRICT_LITELLM_PARAMS)
|
||||
|
||||
|
||||
def _none_when_malformed_outside_strict_validation(
|
||||
value: object, handler: ValidatorFunctionWrapHandler, info: ValidationInfo
|
||||
) -> object:
|
||||
try:
|
||||
return handler(value)
|
||||
except ValidationError:
|
||||
if info.context is _STRICT_LITELLM_PARAMS:
|
||||
raise
|
||||
return None
|
||||
|
||||
|
||||
_DAILY_RESET_TIME_PATTERN: Final = re.compile(r"\d{2}:\d{2}")
|
||||
|
||||
|
|
@ -612,6 +646,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
|
|||
)
|
||||
model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True)
|
||||
merge_reasoning_content_in_choices: bool | None = False
|
||||
thought_signature_in_tool_call_id: Annotated[
|
||||
bool | None, WrapValidator(_none_when_malformed_outside_strict_validation)
|
||||
] = None
|
||||
model_info: dict | None = None
|
||||
mock_response: str | ModelResponse | Exception | object | None = None
|
||||
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@ from .llms.openai import (
|
|||
OpenAIRealtimeStreamList,
|
||||
ResponsesAPIResponse,
|
||||
WebSearchOptions,
|
||||
_JsonValue,
|
||||
)
|
||||
from .rerank import RerankResponse as RerankResponse
|
||||
|
||||
|
|
@ -4682,6 +4683,7 @@ class LlmProviders(str, Enum):
|
|||
PRISM = "prism"
|
||||
OPENINFER = "openinfer"
|
||||
INFERENCEAPIS = "inferenceapis"
|
||||
HOSTINGER_ROUTER = "hostinger_router"
|
||||
ZEROGPU = "zerogpu"
|
||||
DARKBLOOM = "darkbloom"
|
||||
META = "meta"
|
||||
|
|
@ -4963,29 +4965,50 @@ def _batch_usage_from_chat_usage(usage: Usage) -> BatchUsage:
|
|||
)
|
||||
|
||||
|
||||
OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType(
|
||||
{"pending": "validating", "running": "in_progress"}
|
||||
)
|
||||
|
||||
|
||||
def openai_batch_status(status: _JsonValue) -> _JsonValue:
|
||||
return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(status, status) if isinstance(status, str) else status
|
||||
|
||||
|
||||
class LiteLLMBatch(Batch):
|
||||
_hidden_params: dict = {}
|
||||
|
||||
object: Literal["batch"] = "batch"
|
||||
|
||||
@field_validator("object", mode="before")
|
||||
@classmethod
|
||||
def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue:
|
||||
return "batch" if value is None else value
|
||||
|
||||
@field_validator("status", mode="before")
|
||||
@classmethod
|
||||
def _map_vllm_router_status(cls, value: _JsonValue) -> _JsonValue:
|
||||
return openai_batch_status(value)
|
||||
|
||||
@property
|
||||
def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation
|
||||
def hidden_params(self) -> dict[str, _JsonValue]: # mutable-ok: API requires mutation
|
||||
return self._hidden_params
|
||||
|
||||
@hidden_params.setter
|
||||
def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation
|
||||
def hidden_params(self, hidden_params: dict[str, _JsonValue]) -> None: # mutable-ok: API requires mutation
|
||||
self._hidden_params = hidden_params
|
||||
|
||||
usage: Usage | None = None
|
||||
|
||||
@field_validator("usage", mode="before")
|
||||
@classmethod
|
||||
def _usage_from_batch_token_counts(cls, value: object) -> object:
|
||||
def _usage_from_batch_token_counts(cls, value: _JsonValue) -> _JsonValue:
|
||||
counts: Final = _parse_batch_token_counts(value)
|
||||
return value if counts is None else _usage_with_batch_token_counts(Usage.model_validate(value), counts)
|
||||
|
||||
@field_serializer("usage")
|
||||
def _serialize_usage_with_batch_fields(
|
||||
self, usage: Usage | None, info: FieldSerializationInfo
|
||||
) -> Mapping[str, object] | None:
|
||||
) -> Mapping[str, _JsonValue] | None:
|
||||
if not isinstance(usage, Usage):
|
||||
return usage
|
||||
chat_fields: Final = usage.model_dump(
|
||||
|
|
|
|||
|
|
@ -10159,7 +10159,7 @@ class ProviderConfigManager:
|
|||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
|
||||
json_provider: Final = JSONProviderRegistry.get(provider.value)
|
||||
if json_provider is not None and "/v1/messages" in json_provider.supported_endpoints:
|
||||
if json_provider is not None and JSONProviderRegistry.serves_endpoint(provider.value, model, "/v1/messages"):
|
||||
from litellm.llms.openai_like.messages.transformation import (
|
||||
JSONProviderAnthropicMessagesConfig,
|
||||
)
|
||||
|
|
@ -10298,7 +10298,7 @@ class ProviderConfigManager:
|
|||
return result
|
||||
|
||||
# Fall back to JSON providers (generic OpenAI-compatible)
|
||||
if JSONProviderRegistry.exists(provider_str) and JSONProviderRegistry.supports_responses_api(provider_str):
|
||||
if JSONProviderRegistry.serves_endpoint(provider_str, model, "/v1/responses"):
|
||||
provider_config: Final = JSONProviderRegistry.get(provider_str)
|
||||
if provider_config is not None:
|
||||
return create_responses_config_class(provider_config)()
|
||||
|
|
|
|||
|
|
@ -39687,6 +39687,57 @@
|
|||
"/v1/images/generations"
|
||||
]
|
||||
},
|
||||
"hostinger_router/gpt-5.2": {
|
||||
"cache_read_input_token_cost": 0.00000021,
|
||||
"input_cost_per_token": 0.0000021,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 272000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.0000168,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/claude-sonnet-5": {
|
||||
"cache_creation_input_token_cost": 3.3e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 5.28e-06,
|
||||
"cache_read_input_token_cost": 2.64e-07,
|
||||
"input_cost_per_token": 2.64e-06,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.32e-05,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_response_schema": true,
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/messages"],
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/text-embedding-3-large": {
|
||||
"input_cost_per_token": 1.716e-07,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 8191,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/text-embedding-3-small": {
|
||||
"input_cost_per_token": 3e-08,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 8191,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
|
|
@ -51387,6 +51438,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 5e-06,
|
||||
"output_cost_per_token_batches": 2.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -51415,6 +51471,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 5e-06,
|
||||
"output_cost_per_token_batches": 2.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -52161,6 +52222,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -52266,6 +52332,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
|
|||
|
|
@ -1087,6 +1087,32 @@
|
|||
"generateContent": false
|
||||
}
|
||||
},
|
||||
"hostinger_router": {
|
||||
"display_name": "Hostinger Router (`hostinger_router`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/hostinger_router",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": true,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"crusoe": {
|
||||
"display_name": "Crusoe (`crusoe`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/crusoe",
|
||||
|
|
|
|||
|
|
@ -383,6 +383,8 @@ exclude-newer = "3 days"
|
|||
[tool.uv.exclude-newer-package]
|
||||
# TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window
|
||||
brotli = "2026-10-22T14:01:00Z"
|
||||
# TODO: remove after 2026-10-31, once pypdf 6.21.4 clears the 3-day window
|
||||
pypdf = "2026-10-28T17:01:00Z"
|
||||
|
||||
[tool.uv.sources]
|
||||
litellm-proxy-extras = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -124,6 +124,8 @@ ignored_function_names = [
|
|||
"_is_fallback_hop", # Tested through the order fallback hop tests in test_router_order_fallback.py
|
||||
"_deployment_that_just_failed", # Tested through the same-boundary hop test in test_router_order_fallback.py
|
||||
"_merged_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py
|
||||
"_malformed_request_params_the_deployment_sets", # Run by _update_kwargs_with_deployment, tested through the drop_params deployment test in test_thought_signature_in_tool_call_id.py
|
||||
"_drop_malformed_request_params_the_deployment_sets", # Run by _update_kwargs_with_deployment, tested through the drop_params deployment test in test_thought_signature_in_tool_call_id.py
|
||||
"_drops_malformed_request_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py
|
||||
"_collect_ptu_share_teams", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py
|
||||
"_resolve_team_ptu_ceiling", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py
|
||||
|
|
@ -185,6 +187,9 @@ ignored_function_names = [
|
|||
"_aresponses_background_stream_can_release", # Tested through the background Responses stream tests in test_router.py
|
||||
"_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py
|
||||
"_mid_stream_error_before_output", # Tested through the Responses mid-stream fallback and background stream tests
|
||||
"_without_decisions_deployments_refusing_media", # Tested through the mixed media-flag group tests in tests/unit/decisions/test_main.py
|
||||
"_decisions_deployment_refuses_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests
|
||||
"_active_default_litellm_params", # Tested through the router default drop_params test in tests/unit/decisions/test_main.py
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,9 @@
|
|||
- {id: llm.chat_completions.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Vertex AI"}
|
||||
- {id: llm.chat_completions.gemini.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini OpenAI-compatible chat translation"}
|
||||
- {id: llm.chat_completions.gemini.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini chat cost lands in SpendLogs"}
|
||||
- {id: llm.chat_completions.gemini.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Gemini tool-call ids fit the AgentCore tool-use id shape"}
|
||||
- {id: llm.chat_completions.gemini.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: stream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Streamed Gemini tool-call ids fit the AgentCore tool-use id shape"}
|
||||
- {id: llm.chat_completions.gemini.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "A short Gemini tool-call id still round-trips when the client drops provider fields"}
|
||||
- {id: llm.chat_completions.hosted_vllm.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "OpenAI-compatible hosted_vllm chat is a confirmed self-hosted backend path"}
|
||||
- {id: llm.chat_completions.cohere.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: cohere, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Cohere chat via OpenAI-compatible /chat/completions"}
|
||||
|
||||
|
|
|
|||
187
tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py
Normal file
187
tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from e2e_config import unique_marker
|
||||
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from llm_translation.sdk_clients import NO_PROXY_CACHE, SdkClients
|
||||
from models import LiteLLMParamsBody
|
||||
from openai.types.chat import (
|
||||
ChatCompletion,
|
||||
ChatCompletionAssistantMessageParam,
|
||||
ChatCompletionChunk,
|
||||
ChatCompletionMessageFunctionToolCallParam,
|
||||
ChatCompletionToolMessageParam,
|
||||
ChatCompletionToolParam,
|
||||
ChatCompletionUserMessageParam,
|
||||
)
|
||||
from openai.types.chat.chat_completion_chunk import ChoiceDeltaToolCall
|
||||
from openai.types.chat.chat_completion_message_function_tool_call import ChatCompletionMessageFunctionToolCall
|
||||
from passthrough_client import PassthroughClient
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
BACKEND_MODEL: Final = "gemini/gemini-3.8-flash"
|
||||
GEMINI_API_KEY: Final = "os.environ/GEMINI_API_KEY"
|
||||
# bedrock-agentcore HarnessToolUseId shape in botocore 1.43.6, read 2026-10-20
|
||||
AGENTCORE_TOOL_USE_ID: Final = re.compile(r"[a-zA-Z0-9_-]{1,64}")
|
||||
WEATHER_TOOL: Final[ChatCompletionToolParam] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather for a city",
|
||||
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _subject(mode: Mode) -> Subject:
|
||||
return Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(BACKEND_MODEL,),
|
||||
mode=mode,
|
||||
capabilities=(Capability.FUNCTION_CALLING, Capability.TOOL_CHOICE),
|
||||
)
|
||||
|
||||
|
||||
def _register_model(client: PassthroughClient, resources: ResourceManager) -> str:
|
||||
model_name: Final = f"e2e-gemini-tool-call-id-{unique_marker()}"
|
||||
model_id: Final = client.proxy.create_model(
|
||||
model_name,
|
||||
LiteLLMParamsBody(
|
||||
model=BACKEND_MODEL,
|
||||
api_key=GEMINI_API_KEY,
|
||||
thought_signature_in_tool_call_id=False,
|
||||
),
|
||||
)
|
||||
resources.defer(lambda: client.proxy.delete_model(model_id))
|
||||
return model_name
|
||||
|
||||
|
||||
def _user_message() -> ChatCompletionUserMessageParam:
|
||||
return {"role": "user", "content": f"Use get_weather for San Francisco and return its result. {unique_marker()}"}
|
||||
|
||||
|
||||
def _tool_calls(completion: ChatCompletion) -> tuple[ChatCompletionMessageFunctionToolCall, ...]:
|
||||
assert completion.choices, f"Gemini returned no choices: {completion.model_dump_json()[:400]}"
|
||||
calls: Final = tuple(
|
||||
call
|
||||
for call in completion.choices[0].message.tool_calls or ()
|
||||
if isinstance(call, ChatCompletionMessageFunctionToolCall)
|
||||
)
|
||||
assert calls, f"Gemini returned no tool calls for tool_choice='required': {completion.model_dump_json()[:400]}"
|
||||
return calls
|
||||
|
||||
|
||||
def _assert_agentcore_accepts_ids(ids: tuple[str, ...]) -> None:
|
||||
rejected: Final = tuple(call_id for call_id in ids if not AGENTCORE_TOOL_USE_ID.fullmatch(call_id))
|
||||
assert not rejected, f"tool call id(s) AgentCore would reject: {tuple(call_id[:80] for call_id in rejected)}"
|
||||
|
||||
|
||||
def _streamed_tool_calls(chunks: tuple[ChatCompletionChunk, ...]) -> Iterator[ChoiceDeltaToolCall]:
|
||||
for chunk in chunks:
|
||||
for choice in chunk.choices:
|
||||
yield from choice.delta.tool_calls or ()
|
||||
|
||||
|
||||
def _short_id_tool_call(call: ChatCompletionMessageFunctionToolCall) -> ChatCompletionMessageFunctionToolCallParam:
|
||||
return {
|
||||
"id": call.id,
|
||||
"type": "function",
|
||||
"function": {"name": call.function.name, "arguments": call.function.arguments},
|
||||
}
|
||||
|
||||
|
||||
def _tool_result(call: ChatCompletionMessageFunctionToolCall) -> ChatCompletionToolMessageParam:
|
||||
return {"role": "tool", "tool_call_id": call.id, "content": '{"temperature": 21, "condition": "sunny"}'}
|
||||
|
||||
|
||||
class TestGeminiToolCallIdLength:
|
||||
@pytest.mark.covers(
|
||||
"llm.chat_completions.gemini.tool_use.nonstream.works",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
@meta(_subject(Mode.NONSTREAM))
|
||||
def test_non_streaming_tool_call_id_fits_agentcore(
|
||||
self, client: PassthroughClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model: Final = _register_model(client, resources)
|
||||
completion: Final = sdk.openai(resources.key()).chat.completions.create(
|
||||
model=model,
|
||||
messages=[_user_message()],
|
||||
tools=[WEATHER_TOOL],
|
||||
tool_choice="required",
|
||||
max_tokens=2048,
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
|
||||
_assert_agentcore_accepts_ids(tuple(call.id for call in _tool_calls(completion)))
|
||||
|
||||
@pytest.mark.covers(
|
||||
"llm.chat_completions.gemini.tool_use.stream.works",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
@meta(_subject(Mode.STREAM))
|
||||
def test_streaming_tool_call_id_fits_agentcore(
|
||||
self, client: PassthroughClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model: Final = _register_model(client, resources)
|
||||
chunks: Final[tuple[ChatCompletionChunk, ...]] = tuple(
|
||||
sdk.openai(resources.key()).chat.completions.create(
|
||||
model=model,
|
||||
messages=[_user_message()],
|
||||
tools=[WEATHER_TOOL],
|
||||
tool_choice="required",
|
||||
max_tokens=2048,
|
||||
stream=True,
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
)
|
||||
ids: Final = tuple(call.id for call in _streamed_tool_calls(chunks) if call.id)
|
||||
assert ids, f"Gemini stream returned no tool call ids in {len(chunks)} chunks"
|
||||
|
||||
_assert_agentcore_accepts_ids(ids)
|
||||
|
||||
@pytest.mark.covers(
|
||||
"llm.chat_completions.gemini.multi_turn.nonstream.works",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
@meta(_subject(Mode.NONSTREAM))
|
||||
def test_tool_call_round_trip_succeeds(
|
||||
self, client: PassthroughClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model: Final = _register_model(client, resources)
|
||||
openai_client: Final = sdk.openai(resources.key())
|
||||
question: Final = _user_message()
|
||||
first: Final = openai_client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[question],
|
||||
tools=[WEATHER_TOOL],
|
||||
tool_choice="required",
|
||||
max_tokens=2048,
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
calls: Final = _tool_calls(first)
|
||||
assistant: Final[ChatCompletionAssistantMessageParam] = {
|
||||
"role": "assistant",
|
||||
"tool_calls": [_short_id_tool_call(call) for call in calls],
|
||||
}
|
||||
|
||||
second: Final = openai_client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[question, assistant, *(_tool_result(call) for call in calls)],
|
||||
tools=[WEATHER_TOOL],
|
||||
tool_choice="none",
|
||||
max_tokens=2048,
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
|
||||
assert second.choices, f"Gemini turn 2 returned no choices: {second.model_dump_json()[:400]}"
|
||||
answer: Final = second.choices[0].message.content
|
||||
assert answer and answer.strip(), f"Gemini turn 2 returned an empty answer: {second.model_dump_json()[:400]}"
|
||||
|
|
@ -1363,6 +1363,7 @@ class LiteLLMParamsBody(BaseModel):
|
|||
realtime_protocol: str | None = None
|
||||
allowed_openai_params: list[str] | None = None
|
||||
drop_params: bool | None = None
|
||||
thought_signature_in_tool_call_id: bool | None = None
|
||||
aws_access_key_id: str | None = Field(default=None, repr=False)
|
||||
aws_secret_access_key: str | None = Field(default=None, repr=False)
|
||||
aws_region_name: str | None = None
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
PARITY pairs the streamed case with its plain twin where both bill the same row."""
|
||||
|
||||
import json
|
||||
from typing import Final, cast
|
||||
from typing import Final, Literal, cast
|
||||
|
||||
from pydantic import JsonValue
|
||||
|
||||
|
|
@ -41,6 +41,7 @@ from integration.cost_calculation.cases.anthropic import (
|
|||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
CostTrackingTestCase,
|
||||
Deployment,
|
||||
ExactExpected,
|
||||
JsonResponse,
|
||||
RecountExpected,
|
||||
|
|
@ -3387,6 +3388,232 @@ VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS: Final = bridge_pairs(
|
|||
)
|
||||
|
||||
|
||||
VERTEX_AI_CLAUDE_DEPLOYMENT_RATES: Final = Deployment(
|
||||
input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.01
|
||||
)
|
||||
|
||||
|
||||
def _vertex_claude_web_search_message(web_search_requests: int) -> JsonResponse:
|
||||
return JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
"id": "msg_$REQUEST_ID",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5-5",
|
||||
"content": [{"type": "text", "text": "scripted answer for cost fixtures"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {
|
||||
"input_tokens": 1840,
|
||||
"output_tokens": 412,
|
||||
"server_tool_use": {"web_search_requests": web_search_requests},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _vertex_claude_web_search_stream(web_search_requests: int) -> SseResponse:
|
||||
return SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=sse_frames(
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_$REQUEST_ID",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5-5",
|
||||
"content": [],
|
||||
"stop_reason": None,
|
||||
"usage": {"input_tokens": 1840},
|
||||
},
|
||||
},
|
||||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 0,
|
||||
"delta": {"type": "text_delta", "text": "scripted answer for cost fixtures"},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "end_turn"},
|
||||
"usage": {"output_tokens": 412, "server_tool_use": {"web_search_requests": web_search_requests}},
|
||||
},
|
||||
{"type": "message_stop"},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _vertex_claude_deployment_web_search_case(
|
||||
name: str,
|
||||
endpoint: Literal["/v1/chat/completions", "/v1/responses", "/v1/messages"],
|
||||
request: dict[str, JsonValue],
|
||||
response: JsonResponse | SseResponse,
|
||||
tool_usage_cost: float,
|
||||
deployment: Deployment = VERTEX_AI_CLAUDE_DEPLOYMENT_RATES,
|
||||
) -> CostTrackingTestCase:
|
||||
return CostTrackingTestCase(
|
||||
name=name,
|
||||
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
model="vertex_ai/claude-sonnet-5-5",
|
||||
endpoint=endpoint,
|
||||
deployment=deployment,
|
||||
request={"model": "$MODEL", **request},
|
||||
response=response,
|
||||
expected=ExactExpected(
|
||||
spend=round(0.0039 + tool_usage_cost, 10),
|
||||
input_cost=0.00184,
|
||||
output_cost=0.00206,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
tool_usage_cost=tool_usage_cost,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL: Final[dict[str, JsonValue]] = {
|
||||
"type": "web_search_20250305",
|
||||
"name": "web_search",
|
||||
"max_uses": 3,
|
||||
}
|
||||
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests",
|
||||
"/v1/messages",
|
||||
{
|
||||
"max_tokens": 412,
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment rates messages web search"}],
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
},
|
||||
_vertex_claude_web_search_message(2),
|
||||
0.02,
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests_stream",
|
||||
"/v1/messages",
|
||||
{
|
||||
"max_tokens": 412,
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment rates messages web search stream"}],
|
||||
"stream": True,
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
},
|
||||
_vertex_claude_web_search_stream(2),
|
||||
0.02,
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE: Final = (
|
||||
_vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_zero_search_rate_messages_two_requests",
|
||||
"/v1/messages",
|
||||
{
|
||||
"max_tokens": 412,
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment zero search rate messages"}],
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
},
|
||||
_vertex_claude_web_search_message(2),
|
||||
0.0,
|
||||
Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.0),
|
||||
)
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY: Final = (
|
||||
_vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_token_rates_only_messages_web_search_two_requests",
|
||||
"/v1/messages",
|
||||
{
|
||||
"max_tokens": 412,
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment token rates only messages web search"}],
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
},
|
||||
_vertex_claude_web_search_message(2),
|
||||
0.034,
|
||||
Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06),
|
||||
)
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_three_requests",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment rates chat web search"}],
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
"allowed_openai_params": ["web_search_options"],
|
||||
},
|
||||
_vertex_claude_web_search_message(3),
|
||||
0.03,
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_two_requests_stream",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment rates chat web search stream"}],
|
||||
"stream": True,
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
"allowed_openai_params": ["web_search_options"],
|
||||
},
|
||||
_vertex_claude_web_search_stream(2),
|
||||
0.02,
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_rates_responses_web_search_two_requests",
|
||||
"/v1/responses",
|
||||
{"input": "vertex claude deployment rates responses web search", "tools": [{"type": "web_search"}]},
|
||||
_vertex_claude_web_search_message(2),
|
||||
0.02,
|
||||
)
|
||||
|
||||
GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = CostTrackingTestCase(
|
||||
name="vertex_ai/gemini-3.1-pro-preview-deployment_rates_messages_web_search_two_queries_stream",
|
||||
covers="quota_management.spend_tracking.scripted_wire.logs_cost",
|
||||
model="vertex_ai/gemini-3.1-pro-preview",
|
||||
endpoint="/v1/messages",
|
||||
deployment=Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.03),
|
||||
request={
|
||||
"model": "$MODEL",
|
||||
"max_tokens": 412,
|
||||
"messages": [{"role": "user", "content": "vertex gemini deployment rates messages web search stream"}],
|
||||
"stream": True,
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
},
|
||||
response=SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=sse_frames(
|
||||
{
|
||||
"candidates": [
|
||||
{
|
||||
"content": {"parts": [{"text": "scripted answer for cost fixtures"}], "role": "model"},
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
"modelVersion": "gemini-3.1-pro-preview",
|
||||
},
|
||||
{
|
||||
"candidates": [
|
||||
{
|
||||
"content": {"parts": [{"text": ""}], "role": "model"},
|
||||
"finishReason": "STOP",
|
||||
"index": 0,
|
||||
"groundingMetadata": {"webSearchQueries": ["query 0", "query 1"]},
|
||||
}
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 1840,
|
||||
"candidatesTokenCount": 412,
|
||||
"totalTokenCount": 2252,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}],
|
||||
},
|
||||
"modelVersion": "gemini-3.1-pro-preview",
|
||||
},
|
||||
),
|
||||
),
|
||||
expected=ExactExpected(
|
||||
spend=0.0339,
|
||||
input_cost=0.00184,
|
||||
output_cost=0.00206,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
tool_usage_cost=0.03,
|
||||
),
|
||||
)
|
||||
|
||||
GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT: Final = _gemini_case(
|
||||
GEMINI_3_8_FLASH_INPUT_TEXT, "vertex_ai/gemini-3.1-pro-preview-input_text", "vertex_ai/gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412),
|
||||
|
|
@ -3725,6 +3952,14 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_STREAM,
|
||||
*(pair.bridged for pair in VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS),
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH,
|
||||
GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM,
|
||||
GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT,
|
||||
GEMINI_3_1_PRO_PREVIEW_STREAM,
|
||||
GEMINI_3_1_PRO_PREVIEW_CACHE_READ,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,15 @@ from integration._support.wire import Reply, Request, wire_server
|
|||
from pydantic import JsonValue
|
||||
|
||||
MODEL: Final = "claude-haiku-4-5"
|
||||
TEXT: Final = {"type": "text", "text": "alpha bravo"}
|
||||
EXPLANATION: Final = "This request was declined under the usage policy."
|
||||
PENDING_SEARCH: Final = {
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_pending",
|
||||
"name": "web_search",
|
||||
"input": {"query": "next city"},
|
||||
}
|
||||
TERMINAL_EVENTS: Final = ("response.completed", "response.incomplete", "response.failed")
|
||||
|
||||
|
||||
def _sse(event: Mapping[str, JsonValue]) -> bytes:
|
||||
|
|
@ -21,36 +30,64 @@ def _message(identity: str, stop_reason: str | None) -> dict[str, JsonValue]:
|
|||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": MODEL,
|
||||
"content": [{"type": "text", "text": "alpha bravo"}],
|
||||
"content": [TEXT],
|
||||
"stop_reason": stop_reason,
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 199878, "output_tokens": 12799},
|
||||
}
|
||||
|
||||
|
||||
def _stream(identity: str, stop_reason: str) -> tuple[bytes, ...]:
|
||||
def _block_events(index: int, block: Mapping[str, JsonValue]) -> tuple[bytes, ...]:
|
||||
if block["type"] == "text":
|
||||
return (
|
||||
_sse({"type": "content_block_start", "index": index, "content_block": {"type": "text", "text": ""}}),
|
||||
_sse(
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": index,
|
||||
"delta": {"type": "text_delta", "text": block["text"]},
|
||||
}
|
||||
),
|
||||
_sse({"type": "content_block_stop", "index": index}),
|
||||
)
|
||||
return (
|
||||
_sse({"type": "message_start", "message": {**_message(identity, None), "content": []}}),
|
||||
_sse({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}),
|
||||
_sse({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "alpha bravo"}}),
|
||||
_sse({"type": "content_block_stop", "index": 0}),
|
||||
_sse({"type": "content_block_start", "index": index, "content_block": {**block, "input": {}}}),
|
||||
_sse(
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": stop_reason, "stop_sequence": None},
|
||||
"usage": {"output_tokens": 12799},
|
||||
"type": "content_block_delta",
|
||||
"index": index,
|
||||
"delta": {"type": "input_json_delta", "partial_json": json.dumps(block["input"])},
|
||||
}
|
||||
),
|
||||
_sse({"type": "message_stop"}),
|
||||
_sse({"type": "content_block_stop", "index": index}),
|
||||
)
|
||||
|
||||
|
||||
def _anthropic(identity: str, stop_reason: str):
|
||||
def _turn(
|
||||
identity: str,
|
||||
stop_reason: str,
|
||||
content: tuple[Mapping[str, JsonValue], ...] = (TEXT,),
|
||||
stop_details: Mapping[str, JsonValue] | None = None,
|
||||
):
|
||||
stop: Final = {"stop_reason": stop_reason, "stop_sequence": None}
|
||||
details: Final = {"stop_details": stop_details} if stop_details is not None else {}
|
||||
message: Final = {**_message(identity, stop_reason), "content": list(content), **details}
|
||||
stream: Final = (
|
||||
_sse({"type": "message_start", "message": {**_message(identity, None), "content": []}}),
|
||||
*(
|
||||
event
|
||||
for index, block in enumerate(content)
|
||||
for event in _block_events(index, block) # comprehension-ok: block -> events flatten
|
||||
),
|
||||
_sse({"type": "message_delta", "delta": {**stop, **details}, "usage": {"output_tokens": 12799}}),
|
||||
_sse({"type": "message_stop"}),
|
||||
)
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/v1/messages", request.target
|
||||
if json.loads(request.body).get("stream"):
|
||||
return Reply(content_type="text/event-stream", chunks=_stream(identity, stop_reason))
|
||||
return Reply(body=json.dumps(_message(identity, stop_reason)).encode())
|
||||
return Reply(content_type="text/event-stream", chunks=stream)
|
||||
return Reply(body=json.dumps(message).encode())
|
||||
|
||||
return respond
|
||||
|
||||
|
|
@ -67,45 +104,70 @@ def _events(gateway: Gateway, path: str, body: Mapping[str, JsonValue]) -> tuple
|
|||
)
|
||||
|
||||
|
||||
def _chat_finish_reason(gateway: Gateway, model: str, *, stream: bool) -> JsonValue:
|
||||
body: Final = {"model": model, "max_tokens": 64000, "messages": [{"role": "user", "content": "copy the list"}]}
|
||||
def _chat_choices(gateway: Gateway, body: Mapping[str, JsonValue], *, stream: bool) -> tuple[dict[str, JsonValue], ...]:
|
||||
if stream:
|
||||
reasons: Final = tuple(
|
||||
choice["finish_reason"]
|
||||
return tuple(
|
||||
choice
|
||||
for chunk in _events(gateway, "/v1/chat/completions", body)
|
||||
for choice in chunk["choices"] # comprehension-ok: chunk -> choices flatten
|
||||
if choice.get("finish_reason")
|
||||
)
|
||||
assert len(reasons) == 1, reasons
|
||||
return reasons[0]
|
||||
response: Final = gateway.request("POST", "/v1/chat/completions", body)
|
||||
assert response.status_code == 200, response.text
|
||||
return response.json()["choices"][0]["finish_reason"]
|
||||
return tuple(response.json()["choices"])
|
||||
|
||||
|
||||
def _responses_status(gateway: Gateway, model: str, *, stream: bool) -> JsonValue:
|
||||
body: Final = {"model": model, "max_output_tokens": 64000, "input": "copy the list"}
|
||||
def _chat_body(model: str) -> dict[str, JsonValue]:
|
||||
return {"model": model, "max_tokens": 64000, "messages": [{"role": "user", "content": "copy the list"}]}
|
||||
|
||||
|
||||
def _chat_finish_reason(gateway: Gateway, model: str, *, stream: bool) -> JsonValue:
|
||||
reasons: Final = tuple(
|
||||
choice["finish_reason"]
|
||||
for choice in _chat_choices(gateway, _chat_body(model), stream=stream)
|
||||
if choice.get("finish_reason")
|
||||
)
|
||||
assert len(reasons) == 1, reasons
|
||||
return reasons[0]
|
||||
|
||||
|
||||
def _chat_messages(gateway: Gateway, model: str, *, stream: bool) -> tuple[dict[str, JsonValue], ...]:
|
||||
return tuple(
|
||||
choice.get("delta") or choice.get("message") or {}
|
||||
for choice in _chat_choices(gateway, _chat_body(model), stream=stream)
|
||||
)
|
||||
|
||||
|
||||
def _responses_body(model: str) -> dict[str, JsonValue]:
|
||||
return {"model": model, "max_output_tokens": 64000, "input": "copy the list", "tools": [{"type": "web_search"}]}
|
||||
|
||||
|
||||
def _responses_result(gateway: Gateway, model: str, *, stream: bool) -> dict[str, JsonValue]:
|
||||
if stream:
|
||||
terminal: Final = tuple(
|
||||
event["response"]
|
||||
for event in _events(gateway, "/v1/responses", body)
|
||||
if event["type"] in ("response.completed", "response.incomplete", "response.failed")
|
||||
for event in _events(gateway, "/v1/responses", _responses_body(model))
|
||||
if event["type"] in TERMINAL_EVENTS
|
||||
)
|
||||
assert len(terminal) == 1, terminal
|
||||
return terminal[0]["status"]
|
||||
response: Final = gateway.request("POST", "/v1/responses", body)
|
||||
return terminal[0]
|
||||
response: Final = gateway.request("POST", "/v1/responses", _responses_body(model))
|
||||
assert response.status_code == 200, response.text
|
||||
return response.json()["status"]
|
||||
return response.json()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
|
||||
@pytest.mark.parametrize(
|
||||
("stop_reason", "finish_reason"),
|
||||
[("model_context_window_exceeded", "length"), ("max_tokens", "length"), ("end_turn", "stop")],
|
||||
[
|
||||
("model_context_window_exceeded", "length"),
|
||||
("max_tokens", "length"),
|
||||
("pause_turn", "length"),
|
||||
("end_turn", "stop"),
|
||||
],
|
||||
)
|
||||
def test_chat_maps_anthropic_stop_reason(gateway: Gateway, stop_reason: str, finish_reason: str, stream: bool) -> None:
|
||||
identity: Final = uuid.uuid4().hex
|
||||
with wire_server(_anthropic(identity, stop_reason)) as wire, gateway.scenario() as scenario:
|
||||
with wire_server(_turn(identity, stop_reason)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{MODEL}", api_base=wire.url, api_key="scripted-anthropic-key")
|
||||
assert _chat_finish_reason(gateway, model, stream=stream) == finish_reason
|
||||
|
||||
|
|
@ -113,10 +175,81 @@ def test_chat_maps_anthropic_stop_reason(gateway: Gateway, stop_reason: str, fin
|
|||
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
|
||||
@pytest.mark.parametrize(
|
||||
("stop_reason", "status"),
|
||||
[("model_context_window_exceeded", "incomplete"), ("max_tokens", "incomplete"), ("end_turn", "completed")],
|
||||
[
|
||||
("model_context_window_exceeded", "incomplete"),
|
||||
("max_tokens", "incomplete"),
|
||||
("pause_turn", "completed"),
|
||||
("end_turn", "completed"),
|
||||
],
|
||||
)
|
||||
def test_responses_maps_anthropic_stop_reason(gateway: Gateway, stop_reason: str, status: str, stream: bool) -> None:
|
||||
identity: Final = uuid.uuid4().hex
|
||||
with wire_server(_anthropic(identity, stop_reason)) as wire, gateway.scenario() as scenario:
|
||||
with wire_server(_turn(identity, stop_reason)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{MODEL}", api_base=wire.url, api_key="scripted-anthropic-key")
|
||||
assert _responses_status(gateway, model, stream=stream) == status
|
||||
assert _responses_result(gateway, model, stream=stream)["status"] == status
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
|
||||
def test_chat_keeps_pause_turn_so_the_client_knows_to_resend_the_turn(gateway: Gateway, stream: bool) -> None:
|
||||
identity: Final = uuid.uuid4().hex
|
||||
paused: Final = _turn(identity, "pause_turn", (PENDING_SEARCH,))
|
||||
with wire_server(paused) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{MODEL}", api_base=wire.url, api_key="scripted-anthropic-key")
|
||||
stop_reasons: Final = tuple(
|
||||
(message.get("provider_specific_fields") or {}).get("stop_reason")
|
||||
for message in _chat_messages(gateway, model, stream=stream)
|
||||
)
|
||||
assert "pause_turn" in stop_reasons, stop_reasons
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
|
||||
def test_responses_list_a_paused_turn_unrun_search_as_an_in_progress_web_search(gateway: Gateway, stream: bool) -> None:
|
||||
identity: Final = uuid.uuid4().hex
|
||||
paused: Final = _turn(identity, "pause_turn", (PENDING_SEARCH,))
|
||||
with wire_server(paused) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{MODEL}", api_base=wire.url, api_key="scripted-anthropic-key")
|
||||
result: Final = _responses_result(gateway, model, stream=stream)
|
||||
assert (result["status"], result.get("incomplete_details")) == ("completed", None)
|
||||
assert [(item["type"], item["status"], item["action"]["query"]) for item in result["output"]] == [
|
||||
("web_search_call", "in_progress", PENDING_SEARCH["input"]["query"])
|
||||
]
|
||||
|
||||
|
||||
_REFUSAL: Final = {"type": "refusal", "category": "cyber", "explanation": EXPLANATION}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
|
||||
def test_chat_returns_the_refusal_explanation_as_the_message_refusal(gateway: Gateway, stream: bool) -> None:
|
||||
identity: Final = uuid.uuid4().hex
|
||||
with wire_server(_turn(identity, "refusal", (), _REFUSAL)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{MODEL}", api_base=wire.url, api_key="scripted-anthropic-key")
|
||||
refusal: Final = "".join(
|
||||
message["refusal"] for message in _chat_messages(gateway, model, stream=stream) if message.get("refusal")
|
||||
)
|
||||
assert refusal == EXPLANATION
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
|
||||
@pytest.mark.parametrize(
|
||||
("content", "expected_parts"),
|
||||
[
|
||||
((), [("refusal", EXPLANATION)]),
|
||||
((TEXT,), [("output_text", TEXT["text"]), ("refusal", EXPLANATION)]),
|
||||
],
|
||||
ids=["refusal_only", "answer_then_refusal"],
|
||||
)
|
||||
def test_responses_return_the_refusal_explanation_as_a_refusal_part(
|
||||
gateway: Gateway,
|
||||
content: tuple[Mapping[str, JsonValue], ...],
|
||||
expected_parts: list[tuple[str, str]],
|
||||
stream: bool,
|
||||
) -> None:
|
||||
identity: Final = uuid.uuid4().hex
|
||||
with wire_server(_turn(identity, "refusal", content, _REFUSAL)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{MODEL}", api_base=wire.url, api_key="scripted-anthropic-key")
|
||||
result: Final = _responses_result(gateway, model, stream=stream)
|
||||
assert (result["status"], result["incomplete_details"]) == ("incomplete", {"reason": "content_filter"})
|
||||
assert [
|
||||
[(part["type"], part.get("text") or part.get("refusal")) for part in item["content"]]
|
||||
for item in result["output"]
|
||||
] == [expected_parts]
|
||||
|
|
|
|||
|
|
@ -317,7 +317,6 @@ def test_credentialless_proxy_forwards_the_caller_anthropic_credential_and_never
|
|||
|
||||
|
||||
def test_anthropic_sdk_metadata_user_id_reaches_upstream(api_key_proxy: Gateway, wire: Wire) -> None:
|
||||
pytest.skip("BUG: /anthropic/v1/messages drops the native metadata.user_id field before forwarding upstream")
|
||||
with api_key_proxy.scenario() as scenario:
|
||||
key: Final = scenario.key()
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
|
|
|||
|
|
@ -0,0 +1,123 @@
|
|||
import json
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.client import Gateway, eventually, object_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_API_KEY: Final = "synthetic-decisions-key"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_INPUT: Final = "Listen to the attached clip."
|
||||
_AUDIO: Final = ("data:audio/wav;base64,UklGRiQAAABXQVZF",)
|
||||
_VIDEOS: Final = ("data:video/mp4;base64,AAAAIGZ0eXBpc29t",)
|
||||
_QUESTION: Final[dict[str, JsonValue]] = {
|
||||
"type": "predicate",
|
||||
"name": "banana",
|
||||
"instructions": "Does the clip name a banana?",
|
||||
}
|
||||
_SPEND_QUERY: Final = 'SELECT status, model_group FROM "LiteLLM_SpendLogs" WHERE request_id = %s'
|
||||
|
||||
Route = Literal["/v1/decisions", "/v1/systemone"]
|
||||
Media = Literal["audio", "videos"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Upstream:
|
||||
model: str
|
||||
|
||||
def reply(self, request: Request) -> Reply:
|
||||
questions: Final = object_value(_JSON_OBJECT.validate_json(request.body)["questions"])
|
||||
plain: Final[dict[str, JsonValue]] = {
|
||||
"model": self.model,
|
||||
"answers": {name: {"type": "noul", "noul": 0.9} for name in questions},
|
||||
"usage": {"input_tokens": 41, "output_tokens": 0},
|
||||
}
|
||||
return Reply(body=json.dumps(plain).encode())
|
||||
|
||||
|
||||
def _unlisted_openrouter_model() -> _Upstream:
|
||||
return _Upstream(f"openrouter/example/omni-{uuid.uuid4().hex[:12]}")
|
||||
|
||||
|
||||
def _decide(gateway: Gateway, model: str, route: Route, media: Mapping[str, JsonValue]) -> httpx.Response:
|
||||
body: Final[dict[str, JsonValue]] = (
|
||||
{"model": model, "input": _INPUT, "questions": [_QUESTION], "num_retries": 0, **media}
|
||||
if route == "/v1/decisions"
|
||||
else {
|
||||
"model": model,
|
||||
"state": _INPUT,
|
||||
"questions": {"banana": {"type": "noul", "instructions": _QUESTION["instructions"]}},
|
||||
"num_retries": 0,
|
||||
**media,
|
||||
}
|
||||
)
|
||||
return gateway.request("POST", route, body)
|
||||
|
||||
|
||||
def _sent_media(request: Request) -> dict[str, JsonValue]:
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
return {kind: body[kind] for kind in ("audio", "videos") if kind in body}
|
||||
|
||||
|
||||
def _assert_refused(response: httpx.Response, kinds: tuple[Media, ...]) -> None:
|
||||
assert response.status_code == 400, response.text
|
||||
assert f"does not support parameters: {list(kinds)}" in response.text, response.text
|
||||
|
||||
|
||||
_BOTH: Final[dict[str, JsonValue]] = {"audio": list(_AUDIO), "videos": list(_VIDEOS)}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ("/v1/decisions", "/v1/systemone"))
|
||||
def test_a_deployment_flagged_for_audio_and_video_sends_both_clips_for_a_model_the_cost_map_does_not_list(
|
||||
gateway: Gateway, route: Route
|
||||
) -> None:
|
||||
upstream: Final = _unlisted_openrouter_model()
|
||||
with wire_server(upstream.reply) as wire, gateway.scenario() as scenario:
|
||||
flags: Final[dict[str, JsonValue]] = {"supports_audio_input": True, "supports_video_input": True}
|
||||
flagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags)
|
||||
flagged_dropping: Final = scenario.model(
|
||||
model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags, drop_params=True
|
||||
)
|
||||
audio_only: Final = scenario.model(
|
||||
model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info={"supports_audio_input": True}
|
||||
)
|
||||
unflagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY)
|
||||
|
||||
_assert_refused(_decide(gateway, unflagged, route, _BOTH), ("audio", "videos"))
|
||||
_assert_refused(_decide(gateway, audio_only, route, _BOTH), ("videos",))
|
||||
assert wire.drain() == ()
|
||||
|
||||
answered: Final = _decide(gateway, flagged, route, _BOTH)
|
||||
assert answered.status_code == 200, answered.text
|
||||
(call,) = wire.drain()
|
||||
assert _sent_media(call) == _BOTH
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(_SPEND_QUERY, (answered.headers["x-litellm-call-id"],)),
|
||||
lambda found: len(found) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
assert (rows[0]["status"], rows[0]["model_group"]) == ("success", flagged)
|
||||
|
||||
kept: Final = _decide(gateway, flagged_dropping, route, _BOTH)
|
||||
assert kept.status_code == 200, kept.text
|
||||
(kept_call,) = wire.drain()
|
||||
assert _sent_media(kept_call) == _BOTH
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value",
|
||||
({"supports_audio_input": True, "supports_video_input": True}, 5, [{"supports_audio_input": True}], "x" * 5000),
|
||||
ids=("flags", "int", "list", "5kb"),
|
||||
)
|
||||
def test_a_caller_sent_model_info_never_turns_media_on(gateway: Gateway, value: JsonValue) -> None:
|
||||
upstream: Final = _unlisted_openrouter_model()
|
||||
with wire_server(upstream.reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY)
|
||||
_assert_refused(_decide(gateway, model, "/v1/decisions", {**_BOTH, "model_info": value}), ("audio", "videos"))
|
||||
assert wire.drain() == ()
|
||||
|
|
@ -71,6 +71,13 @@ class FaultingTurnCheck(CustomLogger):
|
|||
|
||||
instance = FaultingTurnCheck()
|
||||
"""
|
||||
HEAVY_TURN_TOKENS: Final = 600
|
||||
HEAVY_TURN_USAGE: Final[dict[str, JsonValue]] = {
|
||||
"input_tokens": 400,
|
||||
"output_tokens": 200,
|
||||
"total_tokens": HEAVY_TURN_TOKENS,
|
||||
}
|
||||
SOCKET_TPM_LIMIT: Final = 1000
|
||||
FrameReader = Callable[[], int]
|
||||
|
||||
|
||||
|
|
@ -124,7 +131,16 @@ def _admitting_proxy(
|
|||
(directory / f"{module}.py").write_text(source)
|
||||
config: Final = _config(directory, router_settings, tuple(f"{module}.instance" for module in callback_sources))
|
||||
with gateway_from_environment() as base:
|
||||
with owned_proxy(base, directory, {"SSL_CERT_FILE": str(cert[0])}, config=config) as proxy:
|
||||
with owned_proxy(
|
||||
base,
|
||||
directory,
|
||||
{
|
||||
"SSL_CERT_FILE": str(cert[0]),
|
||||
"REDIS_HOST": os.environ["REDIS_HOST"],
|
||||
"REDIS_PORT": os.environ["REDIS_PORT"],
|
||||
},
|
||||
config=config,
|
||||
) as proxy:
|
||||
yield proxy
|
||||
|
||||
|
||||
|
|
@ -659,6 +675,159 @@ def test_usage_based_routing_refuses_a_turn_only_once_earlier_turns_used_up_the_
|
|||
assert deployment.forwarded_turns() == 2
|
||||
|
||||
|
||||
def _heavy_turn(response_id: str) -> tuple[dict[str, JsonValue], ...]:
|
||||
*started, done = events(response_id, "heavy")
|
||||
return (*started, {**done, "response": {**object_value(done["response"]), "usage": HEAVY_TURN_USAGE}})
|
||||
|
||||
|
||||
def _counted_tokens(counter: str) -> int:
|
||||
with _redis() as redis:
|
||||
return int(redis.get(counter) or 0)
|
||||
|
||||
|
||||
def _chat(gateway: Gateway, key: str, model: str, headers: Mapping[str, str] | None = None) -> httpx.Response:
|
||||
return gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": f"after the socket {uuid.uuid4().hex}"}]},
|
||||
key=key,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
def _two_heavy_turns_then_chat(
|
||||
gateway: Gateway,
|
||||
cert: tuple[Path, Path],
|
||||
counter: Callable[[str], str],
|
||||
headers: Mapping[str, str],
|
||||
**limits: JsonValue,
|
||||
) -> httpx.Response:
|
||||
with responses_peer(cert, _heavy_turn) as peer, gateway.scenario() as scenario:
|
||||
deployment: Final = _native(scenario, peer)
|
||||
chat: Final = scenario.model()
|
||||
key: Final = scenario.key(models=[deployment.name, chat], **limits)
|
||||
_fresh_minute()
|
||||
|
||||
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
|
||||
async with _socket(gateway, key, deployment.name, headers) as connection:
|
||||
return (await _say(connection, deployment.name), await _say(connection, deployment.name))
|
||||
|
||||
turns: Final = asyncio.run(session())
|
||||
|
||||
assert [completed(received)["status"] for received in turns] == ["completed"] * 2
|
||||
eventually(
|
||||
partial(_counted_tokens, counter(key)),
|
||||
lambda tokens: tokens >= 2 * HEAVY_TURN_TOKENS,
|
||||
seconds=15,
|
||||
)
|
||||
return _chat(gateway, key, chat, headers)
|
||||
|
||||
|
||||
def test_websocket_turn_tokens_count_toward_the_key_tpm_so_its_next_http_call_is_refused(
|
||||
limited: Gateway, cert: tuple[Path, Path]
|
||||
) -> None:
|
||||
refused: Final = _two_heavy_turns_then_chat(
|
||||
limited, cert, lambda key: f"{{api_key:{_key_hash(key)}}}:tokens", {}, tpm_limit=SOCKET_TPM_LIMIT
|
||||
)
|
||||
|
||||
assert refused.status_code == 429, refused.text
|
||||
assert "token" in refused.text.lower(), refused.text
|
||||
|
||||
|
||||
def test_websocket_turn_tokens_count_toward_the_tag_tpm_so_its_next_http_call_is_refused(
|
||||
limited: Gateway, cert: tuple[Path, Path]
|
||||
) -> None:
|
||||
tag: Final = f"ws-tag-tpm-{uuid.uuid4().hex}"
|
||||
limited.post("/tag/new", {"name": tag, "tpm_limit": SOCKET_TPM_LIMIT})
|
||||
try:
|
||||
refused: Final = _two_heavy_turns_then_chat(
|
||||
limited, cert, lambda _: f"{{tag:{tag}}}:tokens", {"x-litellm-tags": tag}
|
||||
)
|
||||
finally:
|
||||
limited.post("/tag/delete", {"name": tag})
|
||||
|
||||
assert refused.status_code == 429, refused.text
|
||||
assert "tag" in refused.text.lower(), refused.text
|
||||
|
||||
|
||||
def _key_tokens(key: str) -> int:
|
||||
return _counted_tokens(f"{{api_key:{_key_hash(key)}}}:tokens")
|
||||
|
||||
|
||||
def test_a_turn_is_refused_on_the_open_socket_once_earlier_turns_used_up_the_key_tpm(
|
||||
limited: Gateway, cert: tuple[Path, Path]
|
||||
) -> None:
|
||||
with responses_peer(cert, _heavy_turn) as peer, limited.scenario() as scenario:
|
||||
deployment: Final = _native(scenario, peer)
|
||||
key: Final = scenario.key(models=[deployment.name], tpm_limit=SOCKET_TPM_LIMIT)
|
||||
|
||||
async def session() -> tuple[tuple[tuple[dict[str, JsonValue], ...], ...], int | None]:
|
||||
async with _socket(limited, key, deployment.name) as connection:
|
||||
answered: Final = (await _say(connection, deployment.name), await _say(connection, deployment.name))
|
||||
await _until(partial(_key_tokens, key), lambda tokens: tokens >= 2 * HEAVY_TURN_TOKENS)
|
||||
refused: Final = (await _say(connection, deployment.name), await _say(connection, deployment.name))
|
||||
return (*answered, *refused), connection.close_code
|
||||
|
||||
(first, second, third, fourth), close_code = asyncio.run(session())
|
||||
|
||||
assert [completed(received)["status"] for received in (first, second)] == ["completed"] * 2
|
||||
_assert_refused(third)
|
||||
_assert_refused(fourth)
|
||||
assert close_code is None
|
||||
assert deployment.forwarded_turns() == 2
|
||||
|
||||
|
||||
def test_managed_socket_turns_count_toward_the_sockets_tag_tpm_like_the_key_tpm(limited: Gateway) -> None:
|
||||
tag: Final = f"ws-managed-tag-tpm-{uuid.uuid4().hex}"
|
||||
limited.post("/tag/new", {"name": tag, "tpm_limit": 1_000_000})
|
||||
try:
|
||||
with limited.scenario() as scenario:
|
||||
deployment: Final = _managed(scenario)
|
||||
key: Final = scenario.key(models=[deployment.name], tpm_limit=1_000_000)
|
||||
|
||||
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
|
||||
async with _socket(limited, key, deployment.name, {"x-litellm-tags": tag}) as connection:
|
||||
return (await _say(connection, deployment.name), await _say(connection, deployment.name))
|
||||
|
||||
turns: Final = asyncio.run(session())
|
||||
turn_tokens: Final = sum(
|
||||
int(str(object_value(completed(received)["usage"])["total_tokens"])) for received in turns
|
||||
)
|
||||
assert turn_tokens > 0, turns
|
||||
|
||||
eventually(
|
||||
lambda: (_key_tokens(key), _counted_tokens(f"{{tag:{tag}}}:tokens")),
|
||||
lambda counters: counters[0] >= turn_tokens and counters[0] == counters[1],
|
||||
seconds=20,
|
||||
)
|
||||
finally:
|
||||
limited.post("/tag/delete", {"name": tag})
|
||||
|
||||
|
||||
def test_a_socket_opened_after_turns_used_up_the_key_tpm_gets_the_turn_refusal_frame_and_closes_1008(
|
||||
limited: Gateway, cert: tuple[Path, Path]
|
||||
) -> None:
|
||||
with responses_peer(cert, _heavy_turn) as peer, limited.scenario() as scenario:
|
||||
deployment: Final = _native(scenario, peer)
|
||||
key: Final = scenario.key(models=[deployment.name], tpm_limit=SOCKET_TPM_LIMIT)
|
||||
|
||||
async def spend_then_reopen() -> tuple[tuple[dict[str, JsonValue], ...], int | None]:
|
||||
async with _socket(limited, key, deployment.name) as spending:
|
||||
completed(await _say(spending, deployment.name))
|
||||
completed(await _say(spending, deployment.name))
|
||||
await _until(partial(_key_tokens, key), lambda tokens: tokens >= 2 * HEAVY_TURN_TOKENS)
|
||||
async with _socket(limited, key, deployment.name) as refused:
|
||||
received: Final = await _say(refused, deployment.name)
|
||||
await asyncio.wait_for(refused.wait_closed(), timeout=20)
|
||||
return received, refused.close_code
|
||||
|
||||
received, close_code = asyncio.run(spend_then_reopen())
|
||||
|
||||
_assert_refused(received)
|
||||
assert close_code == 1008, close_code
|
||||
assert deployment.forwarded_turns() == 2
|
||||
|
||||
|
||||
def _assert_http_chat_counted_once(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
identity: Final = f"ws-admission-{uuid.uuid4().hex}"
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue