diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index 0c5b70a67fe..73159bd64d2 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -51,7 +51,7 @@ jobs: suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }} env: UV_PYTHON: "3.12" - INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"32","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }} + INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"48","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }} services: postgres: image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 diff --git a/README.md b/README.md index 3cfb497c9f2..1061ab793e5 100644 --- a/README.md +++ b/README.md @@ -416,6 +416,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th | [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | | | [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | | +| [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) | | | | | | | | | | | | [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | | | [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | | | [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index 4653f919e17..7db2de1f874 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.80" +version = "0.1.81" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.80" +version = "0.1.81" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-enterprise==", diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index cc87a2ed16b..c2740ac1d41 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -142,6 +142,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset( "/test", "/debug/memory/summary", "/api/event_logging/batch", + "/eu_jev/v1/systemone", } ) diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 6de44be0c12..a12e1e27cfc 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.111" +version = "0.4.112" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -30,7 +30,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.111" +version = "0.4.112" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm/__init__.py b/litellm/__init__.py index 6576dc3df93..576dcc891be 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1784,6 +1784,9 @@ if TYPE_CHECKING: from .llms.bespoke.decisions.transformation import ( BespokeDecisionsConfig as BespokeDecisionsConfig, ) + from .llms.eu_jev.decisions.transformation import ( + EuJevDecisionsConfig as EuJevDecisionsConfig, + ) from .llms.openai.decisions.transformation import ( OpenAIDecisionsConfig as OpenAIDecisionsConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 599234bab1a..7e81602fe77 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -174,6 +174,7 @@ LLM_CONFIG_NAMES: Final = ( "OllamaDecisionsConfig", "LayaDecisionsConfig", "BespokeDecisionsConfig", + "EuJevDecisionsConfig", "OpenAIDecisionsConfig", "AzureDecisionsConfig", "AzureAIDecisionsConfig", @@ -753,6 +754,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"), "LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"), "BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"), + "EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"), "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), "AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"), "AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"), diff --git a/litellm/_logging.py b/litellm/_logging.py index 9750d4a3b68..a1d2b1e2b08 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -30,9 +30,11 @@ from litellm.litellm_core_utils.env_utils import get_env_int from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.secret_redaction import ( + REDACTED, python_redact_string, python_redact_structured_value, redact_internal_details, + redact_structured_value, ) from litellm.litellm_core_utils.secret_redaction import ( redact_string as redact_secret_string, @@ -224,6 +226,32 @@ def redact_internal_details_from_client_message(value: str) -> str: return redact_internal_details(value) +_CLIENT_FIELDS: Final = TypeAdapter(dict[str, object]) +_CLIENT_ITEMS: Final = TypeAdapter(list[object]) + + +def _redacted_client_field(key: str | None, value: object) -> object: + if isinstance(value, str): + return redact_internal_details(redact_structured_value(key, value)) + if isinstance(value, dict): + return { + child_key: _redacted_client_field(child_key, child) + for child_key, child in _CLIENT_FIELDS.validate_python(value).items() + } + if isinstance(value, (list, tuple)): + return [_redacted_client_field(key, child) for child in _CLIENT_ITEMS.validate_python(value)] + if not isinstance(value, (int, float)): + return value + text: Final = str(value) + return value if redact_structured_value(key, text) == text else REDACTED + + +def redact_internal_details_from_client_fields(fields: Mapping[str, object]) -> object: + if not _ENABLE_SECRET_REDACTION: + return fields + return _redacted_client_field(None, dict(fields)) + + def _substituted_color_message(record: logging.LogRecord) -> str | None: """Render a record's ``color_message`` against its args, or None if absent. diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a4a9a5bee39..defe1e63b86 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1887,7 +1887,10 @@ def completion_cost( video_resolution=video_resolution, ) elif call_type in _SPEECH_CALL_TYPES: - prompt_characters = litellm.utils.count_characters(text=prompt) + prompt_characters = _speech_billed_characters( + completion_response if isinstance(completion_response, HttpxBinaryResponseContent) else None, + prompt, + ) elif call_type in _TRANSCRIPTION_CALL_TYPES: # Check _hidden_params first (duration stored there to # avoid polluting the response body), then fall back to @@ -2366,6 +2369,11 @@ def response_cost_calculator( raise e +def _speech_billed_characters(speech_response: HttpxBinaryResponseContent | None, prompt: str) -> int: + reported: Final = speech_response.hidden_params.get("billed_characters") if speech_response is not None else None + return reported if isinstance(reported, int) else litellm.utils.count_characters(text=prompt) + + def _video_input_image_cost( model: str, custom_llm_provider: str | None, diff --git a/litellm/images/main.py b/litellm/images/main.py index 4aad5210028..165543aadba 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -53,7 +53,7 @@ from litellm.main import ( from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ImageEditOptionalRequestParams from litellm.types.llms.openai import ImageGenerationRequestQuality -from litellm.types.router import GenericLiteLLMParams +from litellm.types.router import CustomPricingLiteLLMParams, GenericLiteLLMParams from litellm.types.utils import ( LITELLM_IMAGE_VARIATION_PROVIDERS, LlmProviders, @@ -1010,6 +1010,7 @@ def image_edit( optional_params=dict(image_edit_request_params), litellm_params={ **image_edit_request_params, + **litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True), "litellm_call_id": litellm_call_id, "model_info": model_info, "vertex_location": litellm_params.vertex_location, diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 4b0b875af6d..404dfab0a6d 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -75,6 +75,14 @@ _IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = ( "output_cost_per_image_token", ) + +def _deployment_prices_images_without_an_output_price(pricing: ModelInfo | None) -> bool: + return pricing is not None and ( + "input_cost_per_pixel" in pricing + or ("input_cost_per_image" in pricing and "output_cost_per_image" not in pricing) + ) + + # Pre-resolved service-tier cost-key suffixes (e.g. "_priority"). Used per # request in the cost-calc path, so the f-strings are built once here instead # of being rebuilt for every model_info key on every call. Longest-first so a @@ -127,7 +135,7 @@ def _get_token_detail_value(details: object, key: str) -> int | None: return value if isinstance(value, int) else None -_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-)\d+") +_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-|\*)\d+") def _requested_image_param(optional_params: Mapping[str, object] | None, key: str) -> str | None: @@ -137,7 +145,7 @@ def _requested_image_param(optional_params: Mapping[str, object] | None, key: st def _requested_image_size(optional_params: Mapping[str, object] | None) -> str | None: value: Final = _requested_image_param(optional_params, "size") - return value if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None + return value.replace("*", "x") if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None def get_web_search_requests(server_tool_use: object) -> int | None: @@ -2090,7 +2098,7 @@ class _ModelPricedImageCostCalculator(Protocol): def _model_priced_image_cost_calculator( - custom_llm_provider: str | None, model: str + custom_llm_provider: str | None, model: str, pricing: ModelInfo | None ) -> _ModelPricedImageCostCalculator | None: match custom_llm_provider: case litellm.LlmProviders.RECRAFT.value: @@ -2116,6 +2124,14 @@ def _model_priced_image_cost_calculator( from litellm.llms.tencent.image_generation.transformation import is_tokenhub_image_model return tencent_image_cost_calculator if is_tokenhub_image_model(model) else None + case litellm.LlmProviders.DASHSCOPE.value: + if _deployment_prices_images_without_an_output_price(pricing): + return None + from litellm.llms.dashscope.image_generation.cost_calculator import ( + cost_calculator as dashscope_image_cost_calculator, + ) + + return dashscope_image_cost_calculator case _: return None @@ -2178,7 +2194,7 @@ class CostCalculatorUtils: ) resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0) pricing: Final = deployment_pricing(model_info) - model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model) + model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model, pricing) if model_priced_calculator is not None: return model_priced_calculator(model=model, image_response=completion_response, model_info=pricing) diff --git a/litellm/litellm_core_utils/stream_usage_chunks.py b/litellm/litellm_core_utils/stream_usage_chunks.py new file mode 100644 index 00000000000..eda57c7ea12 --- /dev/null +++ b/litellm/litellm_core_utils/stream_usage_chunks.py @@ -0,0 +1,27 @@ +from collections.abc import Iterable +from typing import Final + +from litellm.types.utils import ModelResponseStream, StreamingChoices + + +def is_empty_streaming_choice(choice: StreamingChoices) -> bool: + if choice.finish_reason is not None: + return False + if getattr(choice, "logprobs", None) is not None: + return False + delta: Final = choice.delta if hasattr(choice, "delta") else None + if delta is None: + return True + return _all_values_none(delta.model_dump().values()) + + +def _all_values_none(values: Iterable[object]) -> bool: + return all(value is None for value in values) + + +def is_injected_stream_usage_artifact(chunk: object) -> bool: + if not isinstance(chunk, ModelResponseStream): + return False + if chunk.provider_specific_fields is not None: + return False + return all(is_empty_streaming_choice(choice) for choice in chunk.choices or []) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 591d77bdb16..5ad4e130e62 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -1224,10 +1224,6 @@ def _validate_anthropic_content(content: Mapping[str, object]) -> type: if expected_cls is None: raise ValueError(f"Unknown Anthropic content type: '{content_type}'") - missing: Final = [k for k in getattr(expected_cls, "__required_keys__", set()) if k not in content] - if missing: - raise ValueError(f"Missing required fields in {content_type} block: {', '.join(missing)}") - return expected_cls @@ -1552,12 +1548,13 @@ def _format_single_function_definition(function: Mapping[str, object]) -> tuple[ def _format_object_parameters(parameters, indent): properties: Final = parameters.get("properties") - if not properties: + if not isinstance(properties, Mapping) or not properties: return "" - required_params: Final = parameters.get("required", []) + required: Final = parameters.get("required") + required_params: Final = tuple(required) if isinstance(required, list) else () lines: Final = [] for key, props in properties.items(): - description = props.get("description") + description = props.get("description") if isinstance(props, Mapping) else None if description: lines.append(f"// {description}") question = "?" @@ -1568,19 +1565,21 @@ def _format_object_parameters(parameters, indent): def _format_type(props, indent): + if not isinstance(props, Mapping): + return "any" type: Final = props.get("type") + enum: Final = props.get("enum") if type == "string": - if "enum" in props: - return " | ".join([f'"{item}"' for item in props["enum"]]) + if isinstance(enum, list): + return " | ".join([f'"{item}"' for item in enum]) return "string" elif type == "array": - # items is required, OpenAI throws an error if it's missing - return f"{_format_type(props['items'], indent)}[]" + return f"{_format_type(props.get('items'), indent)}[]" elif type == "object": return f"{{\n{_format_object_parameters(props, indent + 2)}\n}}" elif type in ["integer", "number"]: - if "enum" in props: - return " | ".join([f'"{item}"' for item in props["enum"]]) + if isinstance(enum, list): + return " | ".join([f'"{item}"' for item in enum]) return "number" elif type == "boolean": return "boolean" diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 3b7fcdea0e4..0bc519187f4 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -6,6 +6,7 @@ import copy import json import re from collections.abc import Container, Mapping, MutableMapping, Sequence +from dataclasses import dataclass from datetime import datetime, timezone from types import MappingProxyType from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar @@ -1774,6 +1775,48 @@ def anthropic_request_errors_on_thinking_binding_mismatch(request_body: object) return binding is not None and binding.get("prefix_mismatch_behavior") == "error" +_CLAUDE_STRICT_TOOLS_REJECTION: Final = re.compile( + r"Too many strict tools \(\d+\)" + r"|Schemas contains too many (?:optional parameters|parameters with union types)" + r"|[Cc]ompiled grammar (?:is too large|size)" + r"|tools\.\d+\.custom(?:\.strict)?: " + r"|Grammar compilation timed out" +) +_CLAUDE_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + + +def is_claude_strict_tools_rejection(status_code: int, error_text: str) -> bool: + return status_code == 400 and _CLAUDE_STRICT_TOOLS_REJECTION.search(error_text) is not None + + +@dataclass(frozen=True, slots=True) +class ToolsWithoutStrict: + tools: tuple[Mapping[str, object], ...] + dropped_count: int + + +def tools_without_strict(tools: object) -> ToolsWithoutStrict | None: + try: + validated: Final = _CLAUDE_TOOLS_ADAPTER.validate_python(tools) + except ValidationError: + return None + dropped_count: Final = sum(tool.get("strict") is True for tool in validated) + if dropped_count == 0: + return None + return ToolsWithoutStrict( + tools=tuple({key: value for key, value in tool.items() if key != "strict"} for tool in validated), + dropped_count=dropped_count, + ) + + +def warn_claude_strict_tools_dropped(dropped_count: int) -> None: + litellm.verbose_logger.warning( + "Claude rejected the request's strict schemas, so litellm removed `strict` from every tool (%d had " + "strict true) and sent the request again because drop_params is set.", + dropped_count, + ) + + def strip_thinking_blocks_from_anthropic_messages(messages: Sequence[object]) -> list[object]: """ Return a new message list with thinking / redacted_thinking content blocks removed diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 71ccfbaf915..704bec9056f 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -160,6 +160,7 @@ async def prepare_context_managed_request( """Apply client compaction history, then optional context_management polyfill.""" from litellm.llms.anthropic.messages.context_management.editors.compact import ( apply_client_compaction_block_history, + slice_around_compaction_block, ) # Skip the client-history pre-processing when a ``compact_20260112`` @@ -175,7 +176,7 @@ async def prepare_context_managed_request( if polyfill_will_run: history_result: PolyfillResult | None = None - working_messages: _AnthropicMessages = messages + working_messages: _AnthropicMessages = list(slice_around_compaction_block(messages)[0]) working_system: _AnthropicSystem = system else: history_result = apply_client_compaction_block_history( diff --git a/litellm/llms/anthropic/messages/context_management/dispatcher.py b/litellm/llms/anthropic/messages/context_management/dispatcher.py index 1557cf206ac..2b713646d50 100644 --- a/litellm/llms/anthropic/messages/context_management/dispatcher.py +++ b/litellm/llms/anthropic/messages/context_management/dispatcher.py @@ -8,6 +8,7 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify from litellm.types.llms.anthropic import AppliedEdit +from ..convertible_blocks import drop_non_string_tool_result_texts from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE from .editors import ( CheckedEdit, @@ -103,7 +104,7 @@ async def apply_context_management( Every ``clear_tool_uses_20250919`` edit is validated before any editor runs, so a malformed one raises its 400 before a ``compact_20260112`` summary call is paid for, or runs without its malformed fields under - ``drop_invalid_edits``. + ``drop_invalid_edits``, which also drops non-string ``tool_result`` text first, as the provider never receives it. The dispatcher is async so async editors (``compact_20260112``) can ``await`` the configured summarization model. Sync editors run in a @@ -119,7 +120,7 @@ async def apply_context_management( raise first_error well_formed_edits: Final = tuple(check.edit for check in checked) - current_messages = messages + current_messages = list(drop_non_string_tool_result_texts(messages, drop_invalid_edits)) current_system = system aggregated_applied: Final[list[AppliedEdit]] = [] aggregated_compaction_block = None diff --git a/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py index c4f7966f429..fabc112d443 100644 --- a/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py +++ b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py @@ -39,6 +39,21 @@ if TYPE_CHECKING: T = TypeVar("T") # api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep _TINY_RESULT_BYTES: Final = 46 +_TOKEN_COUNTER_BLOCK_TYPES: Final = frozenset( + { + "text", + "image_url", + "image", + "document", + "file", + "tool_use", + "tool_result", + "thinking", + "redacted_thinking", + "tool_reference", + "search_result", + } +) ToolNames = Literal["all"] | frozenset[str] @@ -52,16 +67,19 @@ class _ClearAtLeast(BaseModel): class _ClearToolUsesFields(BaseModel): model_config = ConfigDict(frozen=True) + trigger: Mapping[str, object] | None = None + keep: Mapping[str, object] | None = None clear_at_least: _ClearAtLeast | None = None exclude_tools: tuple[str, ...] | None = None clear_tool_inputs: bool | tuple[str, ...] | None = None _BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) -# api.anthropic.com, measured 2026-10-19: its wording for the two shapes pydantic words differently +# api.anthropic.com, measured 2026-10-19 and 2026-10-28: its wording for the shapes pydantic words differently _API_MESSAGES: Final = MappingProxyType( { "tuple_type": "Input should be a valid array", + "dict_type": "Input should be an object", "model_type": "Input does not match the expected shape.", } ) @@ -178,10 +196,12 @@ def _cleared_result(block: Mapping[str, object]) -> dict[str, object]: def _freed_tokens(model: str, result: Mapping[str, object], use: Mapping[str, object], clears_input: bool) -> int: """Tokens that clearing this use frees: its result, and its input when ``clear_tool_inputs`` covers it.""" - before: Final = [result, use] if clears_input else [result] - after: Final = [_cleared_result(result), {**use, "input": {}}] if clears_input else [_cleared_result(result)] + before: Final[Sequence[Mapping[str, object]]] = (result, use) if clears_input else (result,) + after: Final[Sequence[Mapping[str, object]]] = ( + (_cleared_result(result), {**use, "input": {}}) if clears_input else (_cleared_result(result),) + ) counts: Final = tuple( - litellm.token_counter(model=model, messages=[{"role": "user", "content": blocks}]) for blocks in (before, after) + _count_tokens(model, [{"role": "user", "content": blocks}], None) for blocks in (before, after) ) return max(counts[0] - counts[1], 0) @@ -199,6 +219,35 @@ def _select_in_batches(candidates: Sequence[str], tokens: Mapping[str, int], cle return tuple(candidates[:batch_end]) +def _without_uncountable_blocks(block: Mapping[str, object]) -> Mapping[str, object]: + nested: Final = _parse(_BLOCKS, block.get("content")) + if nested is None: + return block + return {**block, "content": [inner for inner in nested if inner.get("type") in _TOKEN_COUNTER_BLOCK_TYPES]} + + +def _countable_message(message: Mapping[str, object]) -> Mapping[str, object]: + blocks: Final = _parse(_BLOCKS, message.get("content")) + if blocks is None: + return message + return { + **message, + "content": [ + _without_uncountable_blocks(block) for block in blocks if block.get("type") in _TOKEN_COUNTER_BLOCK_TYPES + ], + } + + +def _count_tokens( + model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None +) -> int: + return litellm.token_counter( + model=model, + messages=[_countable_message(message) for message in messages], + tools=cast("list[ChatCompletionToolParam] | None", tools), + ) + + def _trigger_met( trigger: dict[str, object], model: str, @@ -218,11 +267,7 @@ def _trigger_met( if not isinstance(threshold, int): threshold = DEFAULT_INPUT_TOKENS_TRIGGER - message_tokens: Final = litellm.token_counter( - model=model, - messages=messages, - tools=cast("list[ChatCompletionToolParam] | None", tools), - ) + message_tokens: Final = _count_tokens(model, messages, tools) if message_tokens > threshold: return True, message_tokens current_tokens: Final = message_tokens + count_system_tokens(model, system) @@ -375,17 +420,9 @@ def apply_clear_tool_uses_20250919( [_message_without_inputs(message, ids_to_clear, input_tools) for message in edited] if input_tools else edited ) - tokens_before: Final = ( - counted_tokens - if counted_tokens is not None - else litellm.token_counter( - model=model, messages=kept, tools=cast("list[ChatCompletionToolParam] | None", tools) - ) - ) + tokens_before: Final = counted_tokens if counted_tokens is not None else _count_tokens(model, kept, tools) kept_after, _ = history_after_client_compaction(final_messages, system) - tokens_after: Final = litellm.token_counter( - model=model, messages=kept_after, tools=cast("list[ChatCompletionToolParam] | None", tools) - ) + tokens_after: Final = _count_tokens(model, kept_after, tools) cleared_input_tokens: Final = max(tokens_before - tokens_after, 0) return final_messages, { diff --git a/litellm/llms/anthropic/messages/context_management/editors/compact.py b/litellm/llms/anthropic/messages/context_management/editors/compact.py index 0bf8cac7344..d1cfcc2f2bc 100644 --- a/litellm/llms/anthropic/messages/context_management/editors/compact.py +++ b/litellm/llms/anthropic/messages/context_management/editors/compact.py @@ -137,7 +137,7 @@ def _find_latest_compaction_index( return None, None -def _slice_around_compaction_block( +def slice_around_compaction_block( messages: Sequence[_MsgT], ) -> tuple[Sequence[_MsgT | dict[str, object]], dict[str, object] | None]: """Apply Anthropic's "drop everything before the compaction block" rule. @@ -194,7 +194,7 @@ def history_after_client_compaction( system: str | Sequence[Mapping[str, object]] | None, ) -> tuple[list[dict[str, object]], str | Sequence[Mapping[str, object]] | None]: """Return the history the API keeps after the latest compaction block, without it, and the system with its summary.""" - sliced, block = _slice_around_compaction_block(messages) + sliced, block = slice_around_compaction_block(messages) if block is None: return list(messages), system tail: Final = _strip_compaction_blocks(sliced) @@ -615,7 +615,7 @@ def apply_client_compaction_block_history( tail is forwarded unchanged (with compaction blocks stripped) so recent turns the summary does not cover are preserved. """ - effective_messages, prior_compaction_block = _slice_around_compaction_block(messages) + effective_messages, prior_compaction_block = slice_around_compaction_block(messages) if prior_compaction_block is None: return None @@ -686,7 +686,7 @@ async def apply_compact_20260112( # opt-in gate below so that even when summarization is disabled we still # strip Anthropic-only ``compaction`` blocks from messages going to # non-Anthropic backends (which would reject them). - effective_messages, prior_compaction_block = _slice_around_compaction_block(messages) + effective_messages, prior_compaction_block = slice_around_compaction_block(messages) prior_summary_text: Final = prior_compaction_block.get("content") if prior_compaction_block else None augmented_system: str | list[dict[str, object]] | None = system if isinstance(prior_summary_text, str) and prior_summary_text: diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 9b22170c9db..0eef3fd87c0 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -10,11 +10,15 @@ from typing import Final, TypeAlias from pydantic import TypeAdapter, ValidationError import litellm +from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, supports_openai_prompt_cache_breakpoint, ) -from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.asyncify import ( + run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper +) +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( cache_control_as_prompt_cache_breakpoint, prompt_cache_breakpoint_for_wire, @@ -28,14 +32,19 @@ from litellm.types.llms.anthropic import ( AnthropicMessagesRequest, AnthropicOutputConfig, AnthropicOutputSchema, + ContextManagementResponse, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.utils import CountContextManagement from litellm.utils import ProviderConfigManager -from ..adapters.handler import request_safeguards_evaluator +from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator +from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management +from ..context_management.constants import COMPACT_EDIT_TYPE +from ..context_management.dispatcher import normalize_spec from ..convertible_blocks import reject_malformed_tool_result_text from ..safeguards import with_safeguard_results from ..utils import litellm_logging_obj_from_kwargs, local_model_name @@ -45,10 +54,111 @@ from .transformation import LiteLLMAnthropicToResponsesAPIAdapter AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]] BlockRewrite: TypeAlias = Callable[[Mapping[str, object]], Mapping[str, object]] _BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]]) +_SYSTEM_ITEMS: Final = TypeAdapter(list[object]) +_SPEC: Final = TypeAdapter(dict[str, object]) +_SplitEdits: TypeAlias = tuple[dict[str, object] | None, Sequence[Mapping[str, object]]] +AnthropicContextManagementSpec: TypeAlias = dict[str, object] | list[dict[str, object]] | None _ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter() +def _split_context_management( + spec: AnthropicContextManagementSpec, + kwargs: Mapping[str, object], +) -> _SplitEdits: + """Return (the edits the Responses API runs natively, the edits the gateway polyfill runs). + + The Responses API compacts natively (``compact_20260112`` becomes its ``compaction``); any other + edit, such as ``clear_tool_uses_20250919``, has no Responses API form and runs in the polyfill, as + it does on the chat completions path. An OpenAI-style list spec is not Anthropic's shape and stays + unsent, as it was before the polyfill ran here. + """ + if not isinstance(spec, dict): + return None, () + if not spec or context_management_explicitly_dropped(kwargs.get("additional_drop_params")): + return spec, () + edits: Final = normalize_spec(spec) or [] + native: Final = [edit for edit in edits if edit.get("type") == COMPACT_EDIT_TYPE] + return ({"edits": native} if native else None), tuple( + edit for edit in edits if edit.get("type") != COMPACT_EDIT_TYPE + ) + + +def _system_blocks(system: object) -> str | Sequence[Mapping[str, object]] | None: + if system is None or isinstance(system, str): + return system + try: + items: Final = _SYSTEM_ITEMS.validate_python(system) + except ValidationError: + return None + return tuple(block for block in map(as_str_mapping, items) if block is not None) + + +async def _polyfilled( + *, + model: str, + messages: AnthropicRequestMessages, + tools: Sequence[AllAnthropicToolsValues | Mapping[str, object]] | None, + system: object, + edits: Sequence[Mapping[str, object]], + kwargs: Mapping[str, object], +) -> PolyfillResult | None: + """Run the edits the Responses API can't run natively; raise like the chat completions path does.""" + if not edits: + return None + reject_malformed_tool_result_text(messages, model, kwargs) + drop_invalid_edits: Final = drops_unsupported_params(kwargs.get("drop_params")) + blocks: Final = _system_blocks(system) + try: + return await apply_context_management( + model=model, + messages=_BLOCK_LIST.validate_python(messages), + tools=_BLOCK_LIST.validate_python(tools) if tools else None, + system=blocks if blocks is None or isinstance(blocks, str) else _BLOCK_LIST.validate_python(blocks), + context_management_spec={"edits": [dict(edit) for edit in edits]}, + drop_invalid_edits=drop_invalid_edits, + ) + except AnthropicContextManagementError: + raise + except Exception as e: + verbose_logger.exception("context_management polyfill failed on the Responses API path: %s", e) + raise AnthropicContextManagementError( + status_code=500, message=f"context_management polyfill failed: {e}" + ) from e + + +async def count_view_messages( + *, + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + system: object, + context_management: CountContextManagement | None, +) -> Sequence[Mapping[str, object]]: + """The messages /v1/messages sends the Responses API once the polyfill has run the edits it can't run natively.""" + if context_management is None or not isinstance(context_management.spec, dict): + return messages + _, edits = _split_context_management(_SPEC.validate_python(context_management.spec), {}) + result: Final = await _polyfilled( + model=model, + messages=_BLOCK_LIST.validate_python(messages), + tools=tools, + system=system, + edits=edits, + kwargs={"drop_params": context_management.drop_invalid_edits}, + ) + return messages if result is None else result.messages + + +def _with_applied_edits( + response: AnthropicMessagesResponse, polyfill_result: PolyfillResult | None +) -> AnthropicMessagesResponse: + applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None + if not applied_edits: + return response + return {**response, "context_management": ContextManagementResponse(applied_edits=list(applied_edits))} + + def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None: logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs) if logging_obj is not None: @@ -281,7 +391,7 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens: int, messages: AnthropicRequestMessages, model: str, - context_management: dict[str, object] | None = None, + context_management: AnthropicContextManagementSpec = None, metadata: dict[str, object] | None = None, output_config: AnthropicOutputConfig | None = None, stop_sequences: list[str] | None = None, @@ -296,11 +406,15 @@ class LiteLLMMessagesToResponsesAPIHandler: output_format: AnthropicOutputSchema | None = None, **kwargs: object, ) -> AnthropicMessagesResponse | AsyncIterator[bytes]: + native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs) + polyfill_result: Final = await _polyfilled( + model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs + ) responses_kwargs: Final = _build_responses_kwargs( max_tokens=max_tokens, - messages=messages, + messages=polyfill_result.messages if polyfill_result is not None else messages, model=model, - context_management=context_management, + context_management=native_context_management, metadata=metadata, output_config=output_config, stop_sequences=stop_sequences, @@ -325,13 +439,16 @@ class LiteLLMMessagesToResponsesAPIHandler: model=local_model_name(model, kwargs.get("custom_llm_provider")), litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs), safeguards_evaluator=safeguards_evaluator, + applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None, ) return wrapper.async_anthropic_sse_wrapper() if not isinstance(result, ResponsesAPIResponse): raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") - served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator) + served: Final = await with_safeguard_results( + _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result), safeguards_evaluator + ) _record_bridged_response(responses_kwargs, result) return served @@ -340,7 +457,7 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens: int, messages: AnthropicRequestMessages, model: str, - context_management: dict[str, object] | None = None, + context_management: AnthropicContextManagementSpec = None, metadata: dict[str, object] | None = None, output_config: AnthropicOutputConfig | None = None, stop_sequences: list[str] | None = None, @@ -382,11 +499,26 @@ class LiteLLMMessagesToResponsesAPIHandler: ) # Sync path + native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs) + ran: Final[object] = ( # pyright: ignore[reportUnknownVariableType] # run_async_function is untyped + run_async_function( + _polyfilled, + model=model, + messages=messages, + tools=tools, + system=system, + edits=polyfilled_edits, + kwargs=kwargs, + ) + if polyfilled_edits + else None + ) + polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None responses_kwargs: Final = _build_responses_kwargs( max_tokens=max_tokens, - messages=messages, + messages=polyfill_result.messages if polyfill_result is not None else messages, model=model, - context_management=context_management, + context_management=native_context_management, metadata=metadata, output_config=output_config, stop_sequences=stop_sequences, @@ -409,10 +541,11 @@ class LiteLLMMessagesToResponsesAPIHandler: responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider")), litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs), + applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None, ) return wrapper.async_anthropic_sse_wrapper() if not isinstance(result, ResponsesAPIResponse): raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") - return _ADAPTER.translate_response(result) + return _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result) diff --git a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py index 44f233f2254..dae96ba4ce1 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py @@ -2,7 +2,7 @@ ## Translates OpenAI call to Anthropic `/v1/messages` format import asyncio from collections import deque -from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping +from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import anyio @@ -27,6 +27,7 @@ from litellm.responses.streaming_iterator import ( partial_responses_incomplete_event_on_disconnect, stream_error_status_and_message, ) +from litellm.types.llms.anthropic import AppliedEdit, ContextManagementResponse from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from litellm.types.llms.base import LiteLLMBaseModel @@ -129,8 +130,10 @@ class AnthropicResponsesStreamWrapper: model: str, litellm_logging_obj: "LiteLLMLoggingObject | None" = None, safeguards_evaluator: "SafeguardsEvaluator | None" = None, + applied_edits: Sequence[AppliedEdit] | None = None, ) -> None: self.responses_stream = responses_stream + self._applied_edits: Final = tuple(applied_edits or ()) self.model = model self.litellm_logging_obj = litellm_logging_obj self.safeguards_evaluator: Final = safeguards_evaluator @@ -445,6 +448,11 @@ class AnthropicResponsesStreamWrapper: "type": "message_delta", "delta": message_delta_payload, "usage": dict(anthropic_usage), + **( + {"context_management": ContextManagementResponse(applied_edits=list(self._applied_edits))} + if self._applied_edits + else {} + ), } ) self._chunk_queue.append({"type": "message_stop"}) diff --git a/litellm/llms/base_llm/anthropic_messages/transformation.py b/litellm/llms/base_llm/anthropic_messages/transformation.py index bda7015ab2b..d234ac40c3d 100644 --- a/litellm/llms/base_llm/anthropic_messages/transformation.py +++ b/litellm/llms/base_llm/anthropic_messages/transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) @@ -201,11 +202,23 @@ class BaseAnthropicMessagesConfig(ABC): anthropic_request_errors_on_thinking_binding_mismatch, is_anthropic_invalid_thinking_block_error, is_anthropic_thinking_binding_mismatch_error, + is_claude_strict_tools_rejection, + tools_without_strict, + validated_object_mapping, ) if e.response.status_code != 400: return False error_text: Final[str] = e.response.text + body: Final = validated_object_mapping(request_body) + drops_rejected_strict_tools: Final = ( + drops_unsupported_params(litellm_params.get("drop_params")) + and is_claude_strict_tools_rejection(e.response.status_code, error_text) + and body is not None + and tools_without_strict(body.get("tools")) is not None + ) + if drops_rejected_strict_tools: + return True caller_asked_for_this_error: Final = anthropic_request_errors_on_thinking_binding_mismatch( request_body ) and is_anthropic_thinking_binding_mismatch_error(error_text) @@ -219,11 +232,22 @@ class BaseAnthropicMessagesConfig(ABC): """ from litellm.llms.anthropic.common_utils import ( is_anthropic_invalid_thinking_block_error, + is_claude_strict_tools_rejection, strip_thinking_blocks_from_anthropic_messages_request_dict, + tools_without_strict, + warn_claude_strict_tools_dropped, ) if e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text): strip_thinking_blocks_from_anthropic_messages_request_dict(request_data) + without_strict: Final = ( + tools_without_strict(request_data.get("tools")) + if is_claude_strict_tools_rejection(e.response.status_code, e.response.text) + else None + ) + if without_strict is not None: + warn_claude_strict_tools_dropped(without_strict.dropped_count) + request_data["tools"] = list(without_strict.tools) # rebind-ok: the retry loop re-signs this same dict return request_data def native_output_fallback_litellm_params( diff --git a/litellm/llms/base_llm/text_to_speech/transformation.py b/litellm/llms/base_llm/text_to_speech/transformation.py index be49098b62b..6df83966bf8 100644 --- a/litellm/llms/base_llm/text_to_speech/transformation.py +++ b/litellm/llms/base_llm/text_to_speech/transformation.py @@ -135,6 +135,18 @@ class BaseTextToSpeechConfig(ABC): Transform provider response to standard format """ + async def async_transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> "HttpxBinaryResponseContent": + return self.transform_text_to_speech_response( + model=model, + raw_response=raw_response, + logging_obj=logging_obj, + ) + def get_error_class(self, error_message: str, status_code: int, headers: dict) -> BaseLLMException: from ..chat.transformation import BaseLLMException diff --git a/litellm/llms/bedrock/chat/native_output_fallback.py b/litellm/llms/bedrock/chat/native_output_fallback.py index 6892cd37beb..8069c3cf7e7 100644 --- a/litellm/llms/bedrock/chat/native_output_fallback.py +++ b/litellm/llms/bedrock/chat/native_output_fallback.py @@ -2,11 +2,11 @@ from collections.abc import Callable, Coroutine from functools import partial from typing import Final, TypeAlias +from litellm.llms.anthropic.common_utils import is_claude_strict_tools_rejection from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.common_utils import ( NativeOutputFallback, is_bedrock_native_output_rejection, - is_bedrock_strict_tools_rejection, warn_bedrock_native_output_rejected, warn_bedrock_strict_tools_dropped, ) @@ -34,7 +34,7 @@ def _native_output_retry( def _strict_tools_retry( error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str ) -> NativeOutputFallback | None: - if fallback is None or not is_bedrock_strict_tools_rejection(error.status_code, error.message): + if fallback is None or not is_claude_strict_tools_rejection(error.status_code, error.message): return None retry: Final = fallback() if retry is not None: diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 4175479f068..d88d7e6280c 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -366,21 +366,10 @@ class NativeOutputFallback: litellm_params: Mapping[str, object] -_BEDROCK_STRICT_TOOLS_REJECTION: Final = re.compile( - r"Too many strict tools \(\d+\)" - r"|Schemas contains too many (?:optional parameters|parameters with union types)" - r"|[Cc]ompiled grammar (?:is too large|size)" - r"|tools\.\d+\.custom(?:\.strict)?: " - r"|Grammar compilation timed out" -) _TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _TOOL_SPEC_ADAPTER: Final = TypeAdapter(Mapping[str, object]) -def is_bedrock_strict_tools_rejection(status_code: int, error_text: str) -> bool: - return status_code == 400 and _BEDROCK_STRICT_TOOLS_REJECTION.search(error_text) is not None - - def warn_bedrock_strict_tools_dropped(model: str) -> None: verbose_logger.warning( "Bedrock rejected the strict tool schemas for model=%s, so litellm sent the request again without `strict` " diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 4178095c9ef..c92315e53b3 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2467,8 +2467,7 @@ class BaseLLMHTTPHandler: ) else: verbose_logger.debug( - "Anthropic /v1/messages: invalid thinking signature; " - "stripping thinking blocks and retrying (attempt %s/%s).", + "Anthropic /v1/messages: retrying after a recoverable HTTP error (attempt %s/%s).", len(recovered_errors) + 2, max_attempts, ) @@ -12850,7 +12849,7 @@ class BaseLLMHTTPHandler: provider_config=text_to_speech_provider_config, ) - speech_response: Final = text_to_speech_provider_config.transform_text_to_speech_response( + speech_response: Final = await text_to_speech_provider_config.async_transform_text_to_speech_response( model=model, raw_response=response, logging_obj=logging_obj, diff --git a/litellm/llms/dashscope/common_utils.py b/litellm/llms/dashscope/common_utils.py index 5ad2aa17a3a..d3e1fb8eada 100644 --- a/litellm/llms/dashscope/common_utils.py +++ b/litellm/llms/dashscope/common_utils.py @@ -24,7 +24,8 @@ DASHSCOPE_ANTHROPIC_PATH: Final = "/apps/anthropic" DASHSCOPE_NATIVE_V1_PATH: Final = "/api/v1" DASHSCOPE_COMPATIBLE_API_PATH: Final = "/compatible-api/v1" DASHSCOPE_RERANK_PATH: Final = f"{DASHSCOPE_COMPATIBLE_API_PATH}/reranks" -DASHSCOPE_IMAGE_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation" +DASHSCOPE_MULTIMODAL_GENERATION_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation" +DASHSCOPE_IMAGE_PATH: Final = DASHSCOPE_MULTIMODAL_GENERATION_PATH DASHSCOPE_DEFAULT_ORIGIN: Final = "https://dashscope.aliyuncs.com" _STANDARD_DASHSCOPE_HOSTS: Final = frozenset( @@ -100,6 +101,21 @@ def is_dashscope_native_endpoint_url(api_base: str) -> bool: return urlparse(api_base).path.rstrip("/").startswith(f"{DASHSCOPE_NATIVE_V1_PATH}/") +def dashscope_api_root(api_base: str) -> str: + endpoints: Final = resolve_dashscope_endpoints(api_base) + if endpoints is not None: + return endpoints.origin + trimmed: Final = api_base.rstrip("/") + if trimmed.endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH): + return trimmed.removesuffix(DASHSCOPE_CHAT_COMPATIBLE_PATH) + root, separator, _ = trimmed.partition(DASHSCOPE_NATIVE_V1_PATH) + return root if separator else trimmed + + +def is_aliyun_host(host: str | None) -> bool: + return host is not None and (host == "aliyuncs.com" or host.endswith(".aliyuncs.com")) + + def resolve_dashscope_openai_api_base( api_base: str | None, env_var: str = "DASHSCOPE_API_BASE", @@ -140,9 +156,7 @@ def _chat_shaped_aliyun_origin(api_base: str | None) -> str | None: if api_base is None: return None parsed: Final = urlparse(api_base) - host: Final = parsed.hostname or "" - on_aliyun_host: Final = host == "aliyuncs.com" or host.endswith(".aliyuncs.com") - if not on_aliyun_host or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH: + if not is_aliyun_host(parsed.hostname) or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH: return None return f"{parsed.scheme}://{parsed.netloc}" diff --git a/litellm/llms/dashscope/image_edit/__init__.py b/litellm/llms/dashscope/image_edit/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/dashscope/image_edit/transformation.py b/litellm/llms/dashscope/image_edit/transformation.py new file mode 100644 index 00000000000..0d34017a321 --- /dev/null +++ b/litellm/llms/dashscope/image_edit/transformation.py @@ -0,0 +1,160 @@ +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final + +import httpx +from httpx._types import RequestFiles +from pydantic import TypeAdapter +from typing_extensions import override + +import litellm +from litellm.images.utils import image_to_data_url +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.dashscope.common_utils import ( + DASHSCOPE_CHAT_COMPATIBLE_PATH, + DASHSCOPE_MULTIMODAL_GENERATION_PATH, + dashscope_api_root, + is_dashscope_native_endpoint_url, + resolve_dashscope_endpoints, +) +from litellm.llms.dashscope.image_generation.transformation import ( + DEFAULT_API_BASE, + OPENAI_TO_DASHSCOPE_SIZE, + DashScopeImageGenerationConfig, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import ImageResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +MAX_INPUT_IMAGES: Final = 3 +DASHSCOPE_IMAGE_EDIT_PARAMS: Final = ("n", "size", "negative_prompt", "prompt_extend", "watermark", "seed") + +_IMAGES: Final = TypeAdapter(tuple[object, ...]) + + +def _dashscope_size(size: object) -> object: + if not isinstance(size, str): + return size + return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*")) + + +def _deployment_edit_url(api_base: str) -> str: + derives_from_root: Final = not is_dashscope_native_endpoint_url(api_base) and ( + resolve_dashscope_endpoints(api_base) is not None + or api_base.rstrip("/").endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH) + ) + return f"{dashscope_api_root(api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if derives_from_root else api_base + + +class DashScopeImageEditConfig(BaseImageEditConfig): + def __init__(self, generation_config: DashScopeImageGenerationConfig | None = None) -> None: + super().__init__() + self._generation_config: Final = generation_config or DashScopeImageGenerationConfig() + + @override + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract + return list(DASHSCOPE_IMAGE_EDIT_PARAMS) + + @override + def map_openai_params( + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: mirrors override contract + return { + key: _dashscope_size(value) if key == "size" else value + for key, value in image_edit_optional_params.items() + if key in DASHSCOPE_IMAGE_EDIT_PARAMS + } + + @override + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: mirrors override contract + model: str, + api_key: str | None = None, + litellm_params: dict[str, object] | None = None, # mutable-ok: mirrors override contract + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: mirrors override contract + return self._generation_config.validate_environment( + headers=dict(headers), + model=model, + messages=[], + optional_params={}, + litellm_params=dict(litellm_params or {}), + api_key=api_key, + api_base=api_base, + ) + + @override + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict[str, object], # mutable-ok: mirrors override contract + ) -> str: + if api_base is not None: + return _deployment_edit_url(api_base) + env_api_base: Final = get_secret_str("DASHSCOPE_API_BASE") + derived: Final = ( + f"{dashscope_api_root(env_api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if env_api_base else None + ) + return get_secret_str("DASHSCOPE_API_BASE_IMAGE") or derived or DEFAULT_API_BASE + + @override + def use_multipart_form_data(self) -> bool: + return False + + @override + def transform_image_edit_request( + self, + model: str, + prompt: str | None, + image: object, + image_edit_optional_request_params: Mapping[str, object], + litellm_params: GenericLiteLLMParams, + headers: Mapping[str, str], + ) -> tuple[dict[str, object], RequestFiles]: # mutable-ok: mirrors override contract + candidates: Final = _IMAGES.validate_python(image) if isinstance(image, list) else (image,) + images: Final = tuple(candidate for candidate in candidates if candidate is not None) + if len(images) > MAX_INPUT_IMAGES: + raise litellm.BadRequestError( + message=f"DashScope image edit accepts at most {MAX_INPUT_IMAGES} input images, got {len(images)}", + model=model, + llm_provider="dashscope", + ) + prompt_content: Final = ({"text": prompt},) if prompt is not None else () + return { + "model": model, + "input": { + "messages": [ + { + "role": "user", + "content": [*({"image": image_to_data_url(item)} for item in images), *prompt_content], + } + ] + }, + "parameters": dict(image_edit_optional_request_params), + }, () + + @override + def transform_image_edit_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> ImageResponse: + return self._generation_config.transform_image_generation_response( + model=model, + raw_response=raw_response, + model_response=ImageResponse(), + logging_obj=logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) diff --git a/litellm/llms/dashscope/image_generation/cost_calculator.py b/litellm/llms/dashscope/image_generation/cost_calculator.py new file mode 100644 index 00000000000..f699ef7e1c5 --- /dev/null +++ b/litellm/llms/dashscope/image_generation/cost_calculator.py @@ -0,0 +1,19 @@ +from typing import Final + +import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.types.utils import ImageResponse, ModelInfo + + +def cost_calculator(model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float: + prices: Final = resolve_image_model_info( + model=model, custom_llm_provider=litellm.LlmProviders.DASHSCOPE.value, model_info=model_info + ) + output_cost_per_image: Final = prices.get("output_cost_per_image") + if output_cost_per_image is None: + raise ValueError(f"No output_cost_per_image found for dashscope/{model}") + input_images: Final = image_response.hidden_params.get("billable_input_image_count") + input_image_cost: Final = (prices.get("input_cost_per_image") or 0.0) * ( + input_images if isinstance(input_images, int) else 0 + ) + return output_cost_per_image * len(image_response.data or ()) + input_image_cost diff --git a/litellm/llms/dashscope/image_generation/transformation.py b/litellm/llms/dashscope/image_generation/transformation.py index 2f528b5a2c3..d1b5a7b40e7 100644 --- a/litellm/llms/dashscope/image_generation/transformation.py +++ b/litellm/llms/dashscope/image_generation/transformation.py @@ -23,7 +23,7 @@ Response format: } """ -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final import httpx @@ -61,7 +61,7 @@ _DECODED_JSON: Final = TypeAdapter(object) DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" # Maps OpenAI size strings (WxH) to DashScope size strings (W*H) -OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = { +OPENAI_TO_DASHSCOPE_SIZE: Final[Mapping[str, str]] = { "256x256": "256*256", "512x512": "512*512", "1024x1024": "1024*1024", @@ -71,6 +71,24 @@ OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = { } +def _message_image_urls(message: Mapping[str, object]) -> Iterator[str]: + contents: Final = _JSON_OBJECTS.validate_python(message.get("content", [])) + return (url for url in (item.get("image") for item in contents) if isinstance(url, str) and url) + + +def _image_urls(response_object: Mapping[str, object]) -> Iterator[str]: + output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {})) + choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", [])) + for choice in choices: + yield from _message_image_urls(_JSON_OBJECT.validate_python(choice.get("message", {}))) + + +def _dashscope_size(size: object) -> object: + if not isinstance(size, str): + return size + return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*")) + + class DashScopeImageGenerationConfig(BaseImageGenerationConfig): """ Configuration for DashScope image generation (qwen-image-2.0, qwen-image-2.0-pro, @@ -82,24 +100,16 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): def map_openai_params( self, - non_default_params: dict, - optional_params: dict, + non_default_params: dict[str, object], + optional_params: dict[str, object], model: str, drop_params: bool, - ) -> dict: + ) -> dict[str, object]: supported_params: Final = self.get_supported_openai_params(model) - mapped: Final[dict] = {} - for k, v in non_default_params.items(): - if k in optional_params: - continue - if k not in supported_params: - continue - if k == "size": - # Convert "WxH" → "W*H" - mapped["size"] = OPENAI_TO_DASHSCOPE_SIZE.get(v, v.replace("x", "*")) - else: - mapped[k] = v - return mapped + requested: Final = { + k: v for k, v in non_default_params.items() if k in supported_params and k not in optional_params + } + return {k: _dashscope_size(v) if k == "size" else v for k, v in requested.items()} def _resolve_api_key(self, api_key: str | None) -> str: resolved_api_key: Final = api_key or get_secret_str("DASHSCOPE_API_KEY") @@ -121,8 +131,8 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): api_base: str | None, api_key: str | None, model: str, - optional_params: dict, - litellm_params: dict, + optional_params: dict[str, object], + litellm_params: dict[str, object], stream: bool | None = None, ) -> str: image_env_api_base: Final = self._image_api_base_env() @@ -140,14 +150,14 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): def validate_environment( self, - headers: dict, + headers: dict[str, str], model: str, messages: list[AllMessageValues], - optional_params: dict, - litellm_params: dict, + optional_params: dict[str, object], + litellm_params: dict[str, object], api_key: str | None = None, api_base: str | None = None, - ) -> dict: + ) -> dict[str, str]: headers["Authorization"] = f"Bearer {self._resolve_api_key(api_key)}" headers["Content-Type"] = "application/json" return headers @@ -186,9 +196,9 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): raw_response: httpx.Response, model_response: ImageResponse, logging_obj: LiteLLMLoggingObj, - request_data: dict, - optional_params: dict, - litellm_params: dict, + request_data: dict[str, object], + optional_params: dict[str, object], + litellm_params: dict[str, object], encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, @@ -225,17 +235,21 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): headers=raw_response.headers, ) - if not model_response.data: - model_response.data = [] - - output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {})) - choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", [])) - for choice in choices: - message = _JSON_OBJECT.validate_python(choice.get("message", {})) - content_list = _JSON_OBJECTS.validate_python(message.get("content", [])) - for content_item in content_list: - image_url = content_item.get("image") - if image_url: - model_response.data.append(ImageObject(url=image_url)) - - return model_response + images: Final = [*(model_response.data or ()), *(ImageObject(url=url) for url in _image_urls(response_object))] + usage: Final = response_object.get("usage") + usage_fields: Final[Mapping[str, object]] = usage if isinstance(usage, Mapping) else {} + width: Final = usage_fields.get("output_width", usage_fields.get("width")) + height: Final = usage_fields.get("output_height", usage_fields.get("height")) + image_response: Final = model_response.model_copy( + update={ + "data": images, + **({"size": f"{width}x{height}"} if isinstance(width, int) and isinstance(height, int) else {}), + } + ) + input_image_count: Final = usage_fields.get("input_image_count") + if isinstance(input_image_count, int) and input_image_count > 0: + image_response.hidden_params = { + **model_response.hidden_params, + "billable_input_image_count": input_image_count, + } + return image_response diff --git a/litellm/llms/dashscope/text_to_speech/__init__.py b/litellm/llms/dashscope/text_to_speech/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/dashscope/text_to_speech/transformation.py b/litellm/llms/dashscope/text_to_speech/transformation.py new file mode 100644 index 00000000000..053682040bd --- /dev/null +++ b/litellm/llms/dashscope/text_to_speech/transformation.py @@ -0,0 +1,297 @@ +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from itertools import chain +from typing import TYPE_CHECKING, Final +from urllib.parse import urlparse + +import httpx +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from typing_extensions import override + +import litellm +from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.text_to_speech.transformation import ( + BaseTextToSpeechConfig, + TextToSpeechRequestData, +) +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped + get_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped +) +from litellm.llms.dashscope.common_utils import ( + DASHSCOPE_MULTIMODAL_GENERATION_PATH, + DashScopeError, + dashscope_api_root, + is_aliyun_host, + missing_dashscope_family_key_message, + resolve_dashscope_family_api_key, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.utils import Usage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +DASHSCOPE_TTS_DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com" +_SPEECH_SYNTHESIZER_MODEL_PREFIX: Final = "qwen-audio-" +_TIMEOUT_PHASES: Final = TypeAdapter(dict[str, float | None]) + + +@dataclass(frozen=True, slots=True) +class _TextToSpeechFamily: + path: str + response_formats: frozenset[str] + sends_format: bool + takes_instructions: bool + + +MULTIMODAL_GENERATION_FAMILY: Final = _TextToSpeechFamily( + path=DASHSCOPE_MULTIMODAL_GENERATION_PATH, + response_formats=frozenset({"wav"}), + sends_format=False, + takes_instructions=False, +) +SPEECH_SYNTHESIZER_FAMILY: Final = _TextToSpeechFamily( + path="/api/v1/services/audio/tts/SpeechSynthesizer", + response_formats=frozenset({"mp3", "wav", "opus", "pcm"}), + sends_format=True, + takes_instructions=True, +) + + +class _Audio(BaseModel): + model_config = ConfigDict(frozen=True) + url: str + + +class _Output(BaseModel): + model_config = ConfigDict(frozen=True) + audio: _Audio + + +class _Usage(BaseModel): + model_config = ConfigDict(frozen=True) + input_tokens: int | None = None + output_tokens: int | None = None + characters: int | None = None + + +class _SpeechResponse(BaseModel): + model_config = ConfigDict(frozen=True) + output: _Output | None = None + usage: _Usage | None = None + code: str | None = None + message: str | None = None + + +def text_to_speech_family(model: str) -> _TextToSpeechFamily: + if model.startswith(_SPEECH_SYNTHESIZER_MODEL_PREFIX): + return SPEECH_SYNTHESIZER_FAMILY + return MULTIMODAL_GENERATION_FAMILY + + +def _mapped_field(family: _TextToSpeechFamily, key: str, value: object) -> Mapping[str, object] | None: + match key: + case "response_format" if value in family.response_formats: + return {"format": value} if family.sends_format else {} + case "instructions" if family.takes_instructions: + return {"instruction": value} + case _: + return None + + +def _unsupported(model: str, unsupported: tuple[str, ...]) -> UnsupportedParamsError: + return UnsupportedParamsError( + message=f"DashScope {model} does not support {', '.join(unsupported)}, set drop_params=True to ignore it", + model=model, + llm_provider="dashscope", + ) + + +def _speech_response(raw_response: httpx.Response) -> _SpeechResponse: + try: + return _SpeechResponse.model_validate_json(raw_response.content) + except ValidationError as e: + raise DashScopeError( + status_code=502, + message=f"Unexpected DashScope text-to-speech response: {e}", + headers=raw_response.headers, + ) from e + + +def _https_on_aliyun(url: str) -> str: + parsed: Final = urlparse(url) + return ( + parsed._replace(scheme="https").geturl() if parsed.scheme == "http" and is_aliyun_host(parsed.hostname) else url + ) + + +def _audio_url(raw_response: httpx.Response) -> tuple[str, _Usage | None]: + parsed: Final = _speech_response(raw_response) + if parsed.output is None: + raise DashScopeError( + status_code=max(raw_response.status_code, 400), + message=parsed.message or parsed.code or raw_response.text, + headers=raw_response.headers, + ) + return _https_on_aliyun(parsed.output.audio.url), parsed.usage + + +def _request_timeout(raw_response: httpx.Response) -> httpx.Timeout | None: + extension: Final = raw_response.request.extensions.get("timeout") + if extension is None: + return None + phases: Final = _TIMEOUT_PHASES.validate_python(extension) + return httpx.Timeout( + connect=phases.get("connect"), read=phases.get("read"), write=phases.get("write"), pool=phases.get("pool") + ) + + +def _token_usage(usage: _Usage | None) -> Usage | None: + if usage is None or usage.input_tokens is None or usage.output_tokens is None: + return None + return Usage( + prompt_tokens=usage.input_tokens, + completion_tokens=usage.output_tokens, + total_tokens=usage.input_tokens + usage.output_tokens, + ) + + +def _refused_audio_url(error: SSRFError) -> DashScopeError: + return DashScopeError(status_code=502, message=f"DashScope returned an audio URL that was refused: {error}") + + +def _binary_audio(audio_response: httpx.Response, usage: _Usage | None) -> HttpxBinaryResponseContent: + if audio_response.status_code != 200: + raise DashScopeError( + status_code=502, + message=f"DashScope audio download failed with HTTP {audio_response.status_code}", + headers=audio_response.headers, + ) + binary: Final = HttpxBinaryResponseContent(audio_response) + binary.usage = _token_usage(usage) + if usage is not None and usage.characters is not None: + binary.hidden_params = {**binary.hidden_params, "billed_characters": usage.characters} + return binary + + +def _shared_async_client() -> AsyncHTTPHandler: + return get_async_httpx_client(llm_provider=litellm.LlmProviders.DASHSCOPE) + + +class DashScopeTextToSpeechConfig(BaseTextToSpeechConfig): + def __init__( + self, + sync_client: Callable[[], HTTPHandler] = get_httpx_client, + async_client: Callable[[], AsyncHTTPHandler] = _shared_async_client, + ) -> None: + super().__init__() + self._sync_client: Final = sync_client + self._async_client: Final = async_client + + @override + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract + family: Final = text_to_speech_family(model) + return ["voice", "response_format", *(("instructions",) if family.takes_instructions else ())] + + @override + def map_openai_params( + self, + model: str, + optional_params: dict[str, object], # mutable-ok: mirrors override contract + voice: str | dict[str, object] | None = None, # mutable-ok: mirrors override contract + drop_params: bool = False, + kwargs: Mapping[str, object] | None = None, + ) -> tuple[str | None, dict[str, object]]: # mutable-ok: mirrors override contract + family: Final = text_to_speech_family(model) + fields: Final = tuple((key, value, _mapped_field(family, key, value)) for key, value in optional_params.items()) + unsupported: Final = ( + *(f"{key}={value!r}" for key, value, field in fields if field is None), + *(("voice as an object",) if isinstance(voice, dict) else ()), + ) + if unsupported and not (drop_params or drops_unsupported_params((kwargs or {}).get("drop_params"))): + raise _unsupported(model, unsupported) + supported: Final = tuple(field for _, _, field in fields if field is not None) + return voice if isinstance(voice, str) else None, dict(chain.from_iterable(f.items() for f in supported)) + + @override + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: mirrors override contract + model: str, + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: mirrors override contract + resolved_api_key: Final = resolve_dashscope_family_api_key("dashscope", api_key) + if resolved_api_key is None: + raise ValueError(missing_dashscope_family_key_message("dashscope")) + return {**headers, "Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"} + + @override + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict[str, object], # mutable-ok: mirrors override contract + ) -> str: + configured: Final = api_base or get_secret_str("DASHSCOPE_API_BASE") + root: Final = dashscope_api_root(configured) if configured else DASHSCOPE_TTS_DEFAULT_API_BASE + return f"{root}{text_to_speech_family(model).path}" + + @override + def transform_text_to_speech_request( + self, + model: str, + input: str, + voice: str | None, + optional_params: dict[str, object], # mutable-ok: mirrors override contract + litellm_params: dict[str, object], # mutable-ok: mirrors override contract + headers: dict[str, str], # mutable-ok: mirrors override contract + ) -> TextToSpeechRequestData: + voice_field: Final = {"voice": voice} if voice is not None else {} + return {"dict_body": {"model": model, "input": {"text": input, **voice_field, **optional_params}}} + + @override + def transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> HttpxBinaryResponseContent: + audio_url, usage = _audio_url(raw_response) + try: + audio_response: Final = safe_get(self._sync_client(), audio_url, timeout=_request_timeout(raw_response)) + except SSRFError as e: + raise _refused_audio_url(e) from e + return _binary_audio(audio_response, usage) + + @override + async def async_transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> HttpxBinaryResponseContent: + audio_url, usage = _audio_url(raw_response) + try: + audio_response: Final = await async_safe_get( + self._async_client(), audio_url, timeout=_request_timeout(raw_response) + ) + except SSRFError as e: + raise _refused_audio_url(e) from e + return _binary_audio(audio_response, usage) + + @override + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, str] | httpx.Headers, # mutable-ok: mirrors override contract + ) -> BaseLLMException: + return DashScopeError(status_code=status_code, message=error_message, headers=httpx.Headers(headers)) diff --git a/litellm/llms/eu_jev/__init__.py b/litellm/llms/eu_jev/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/eu_jev/common_utils.py b/litellm/llms/eu_jev/common_utils.py new file mode 100644 index 00000000000..18385ef8e5d --- /dev/null +++ b/litellm/llms/eu_jev/common_utils.py @@ -0,0 +1,34 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +EU_JEV_LATEST_MODEL: Final = "eu-jeff-latest" +_EU_JEV_CURRENT_RELEASE_NAMES: Final = frozenset((EU_JEV_LATEST_MODEL, "jeff-latest", "jeff-1.0.0")) +_EU_JEV_PREFIX: Final = "eu_jev/" + + +def eu_jev_registry_model(requested_model: str | None) -> str: + release: Final = requested_model.removeprefix(_EU_JEV_PREFIX) if requested_model is not None else None + if release is None or release in _EU_JEV_CURRENT_RELEASE_NAMES: + return EU_JEV_LATEST_MODEL + return release + + +def eu_jev_scoped_model(decision_body: Mapping[str, object]) -> str: + requested_model: Final = decision_body.get("model") + return f"{_EU_JEV_PREFIX}{eu_jev_registry_model(requested_model if isinstance(requested_model, str) else None)}" + + +def eu_jev_upstream_body(hook_body: Mapping[str, object], client_body: Mapping[str, object]) -> Mapping[str, object]: + hook_model: Final = hook_body.get("model") + without_model: Final = {key: value for key, value in hook_body.items() if key != "model"} + if hook_model == eu_jev_scoped_model(client_body): + client_model: Final = client_body.get("model") + if client_model is None: + return MappingProxyType(without_model) + if isinstance(client_model, str): + return MappingProxyType({**without_model, "model": client_model.removeprefix(_EU_JEV_PREFIX)}) + return MappingProxyType({**without_model, "model": client_model}) + if isinstance(hook_model, str): + return MappingProxyType({**without_model, "model": hook_model.removeprefix(_EU_JEV_PREFIX)}) + return hook_body diff --git a/litellm/llms/eu_jev/decisions/transformation.py b/litellm/llms/eu_jev/decisions/transformation.py new file mode 100644 index 00000000000..18ef391b040 --- /dev/null +++ b/litellm/llms/eu_jev/decisions/transformation.py @@ -0,0 +1,9 @@ +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig + + +class EuJevDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("EU_JEV_API_KEY",) + api_base_env = ("EU_JEV_API_BASE",) + + def get_default_api_base(self) -> str | None: + return "https://eu-jev.com" diff --git a/litellm/llms/litellm_proxy/responses/transformation.py b/litellm/llms/litellm_proxy/responses/transformation.py index 3092b49b743..fc94eb30504 100644 --- a/litellm/llms/litellm_proxy/responses/transformation.py +++ b/litellm/llms/litellm_proxy/responses/transformation.py @@ -5,11 +5,30 @@ LiteLLM Proxy supports the OpenAI Responses API natively when the underlying mod This config enables pass-through behavior to the proxy's /v1/responses endpoint. """ +from typing import Final + +import httpx +from pydantic import TypeAdapter, ValidationError + from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.utils import LlmProviders +_ERROR_BODY: Final = TypeAdapter(dict[str, object]) + + +def _openai_sdk_error_body( + error_message: str, +) -> dict[str, object] | None: # mutable-ok: BaseLLMException.body is a dict + try: + decoded: Final = _ERROR_BODY.validate_json(error_message) + except ValidationError: + return None + nested: Final = decoded.get("error") + return _ERROR_BODY.validate_python(nested) if isinstance(nested, dict) else decoded + class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig): """ @@ -43,6 +62,19 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig): return join_api_base(api_base, "responses") + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, object] | httpx.Headers, # mutable-ok: inherited contract + ) -> BaseLLMException: + return BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + body=_openai_sdk_error_body(error_message), + ) + def supports_native_websocket(self) -> bool: """LiteLLM Proxy does not support native WebSocket for Responses API""" return False diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 6ae61e15953..87a75dae95e 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -15,7 +15,9 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.tool_fields import DropPolicy +from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error +from litellm.llms.anthropic.messages.responses_adapters.handler import count_view_messages from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) @@ -95,6 +97,32 @@ def _count_error(model_to_use: str, request_model: str, message: str, status_cod ) +async def _sent_messages( + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + system: object, + messages_format: TokenCountMessagesFormat, + context_management: CountContextManagement | None, +) -> Sequence[Mapping[str, object]] | AnthropicContextManagementError: + match messages_format: + case "openai": + return messages + case "anthropic": + try: + return await count_view_messages( + model=model, + messages=messages, + tools=tools, + system=system, + context_management=context_management, + ) + except AnthropicContextManagementError as error: + return error + case _: + assert_never(messages_format) + + class OpenAITokenCounter(BaseTokenCounter): """Token counter implementation for OpenAI provider using the Responses API.""" @@ -175,13 +203,28 @@ class OpenAITokenCounter(BaseTokenCounter): rejected_before_provider_call=True, ) + sent_messages: Final = await _sent_messages( + model_to_use, messages, tools, system, messages_format, context_management + ) + if isinstance(sent_messages, AnthropicContextManagementError): + return TokenCountResponse( + total_tokens=0, + request_model=request_model, + model_used=model_to_use, + tokenizer_type="openai_api", + error=True, + error_message=sent_messages.message, + status_code=sent_messages.status_code, + rejected_before_provider_call=True, + ) + resolved: Final = _count_credential(deployment, request_model or model_to_use) if isinstance(resolved, OpenAIError): return _count_error(model_to_use, request_model, resolved.message, resolved.status_code) input_items, instructions, counted_tools, text = self._responses_request_parts( model_to_use=model_to_use, - messages=messages, + messages=sent_messages, tools=tools, system=system, messages_format=messages_format, diff --git a/litellm/llms/pass_through/typesafe_logging_handler.py b/litellm/llms/pass_through/typesafe_logging_handler.py index 397ec408ae9..9eae57774c2 100644 --- a/litellm/llms/pass_through/typesafe_logging_handler.py +++ b/litellm/llms/pass_through/typesafe_logging_handler.py @@ -11,6 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, # pyright: ignore[reportUnknownVariableType] # legacy helper has an untyped signature ) +from litellm.llms.eu_jev.common_utils import EU_JEV_LATEST_MODEL, eu_jev_registry_model from litellm.llms.laya.common_utils import laya_response_model from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject, Usage @@ -82,13 +83,21 @@ class TypeSafePassthroughLoggingHandler: response_model: Final = ( laya_response_model(response_body, request_model) if custom_llm_provider == "laya" else response.model ) - logged_model: Final = response_model or request_model or "unknown" + registry_request_model: Final = ( + eu_jev_registry_model(request_model) if custom_llm_provider == "eu_jev" else request_model + ) + logged_model: Final = response_model or registry_request_model or "unknown" model_name: Final = f"{custom_llm_provider}/{logged_model}" usage: Final = response.usage or _TypeSafeUsage() input_tokens: Final = usage.input_tokens output_tokens: Final = usage.output_tokens + candidate_models: Final = ( + (response_model, registry_request_model, EU_JEV_LATEST_MODEL) + if custom_llm_provider == "eu_jev" + else (response_model, registry_request_model) + ) candidate_model_keys: Final = tuple( - f"{custom_llm_provider}/{model}" for model in (response_model, request_model) if model is not None + f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None ) pricing: Final = _pricing_for(candidate_model_keys) response_cost: Final = ( diff --git a/litellm/main.py b/litellm/main.py index 703af1339e3..f5a7d63ada1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7980,6 +7980,8 @@ async def atext_completion(*args, **kwargs) -> TextCompletionResponse | TextComp model=model, custom_llm_provider=custom_llm_provider, stream_options=kwargs.get("stream_options"), + usage_injected_by_proxy="_litellm_strip_stream_usage" in kwargs + and kwargs["_litellm_strip_stream_usage"] is True, ) else: ## OpenAI / Azure Text Completion Returns here @@ -8829,8 +8831,9 @@ def speech( extra_headers: Final = kwargs.get("extra_headers", None) model_info: Final = kwargs.get("model_info", None) shared_session: Final = kwargs.get("shared_session", None) + caller_api_base: Final = api_base or base_url model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider, api_base=api_base or base_url + model=model, custom_llm_provider=custom_llm_provider, api_base=caller_api_base ) kwargs.pop("tags", []) @@ -8885,13 +8888,13 @@ def speech( custom_llm_provider=custom_llm_provider, ) response: HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent] | None = None - if custom_llm_provider == "edenai": - litellm_params_dict["api_base"] = api_base + if custom_llm_provider in ("edenai", "dashscope") and text_to_speech_provider_config is not None: + litellm_params_dict["api_base"] = caller_api_base if custom_llm_provider == "dashscope" else api_base response = base_llm_http_handler.text_to_speech_handler( model=model, input=input, voice=voice if isinstance(voice, str) else None, - text_to_speech_provider_config=text_to_speech_provider_config or litellm.EdenAITextToSpeechConfig(), + text_to_speech_provider_config=text_to_speech_provider_config, text_to_speech_optional_params=optional_params, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params_dict, diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index 8e6becc60b4..35136c41dc2 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -121,6 +121,7 @@ _ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType( Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook, Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception, Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted, + Feature.STRICT_TOOLS_RETRY: route_detectors.strict_tools_retry, } ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c6bbb63b6fc..b5d09d6ff4f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18346,25 +18346,31 @@ "dashscope/qwen-image-2.0": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.035, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-2.0-pro": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.075, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-3.0": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.03, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-3.0-pro": { @@ -18372,7 +18378,54 @@ "mode": "image_generation", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "dashscope/qwen3-tts-flash": { + "input_cost_per_character": 1e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.1-tts-flash": { + "input_cost_per_token": 2.3e-07, + "output_cost_per_token": 1.87e-06, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.1-tts-plus": { + "input_cost_per_character": 3e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.0-tts-plus": { + "input_cost_per_character": 2e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.0-tts-flash": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" ] }, "dashscope/wan3.0-video": { @@ -72330,6 +72383,28 @@ "source": "https://docs.typesafe.ai/models", "supports_box_questions": true }, + "eu_jev/eu-jeff-1.0.0": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://eu-jev.com/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, + "eu_jev/eu-jeff-latest": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://eu-jev.com/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 2e-07, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 1d487f33835..18edb2cd7dc 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -1148,9 +1148,9 @@ "text_completion": true, "embeddings": false, "image_generations": true, - "image_edits": false, + "image_edits": true, "audio_transcriptions": false, - "audio_speech": false, + "audio_speech": true, "rerank": false, "ocr": false, "search": false, @@ -2241,6 +2241,32 @@ "generateContent": false } }, + "eu_jev": { + "display_name": "eu/jev (`eu_jev`)", + "url": "https://docs.litellm.ai/docs/pass_through/eu_jev", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": true, + "a2a": false, + "generateContent": false + } + }, "laya": { "display_name": "Laya (`laya`)", "url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers", diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index db01475154b..dc6575f1e1b 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -238,6 +238,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( "/tinyfish/", "/transcribe", "/typesafe/", + "/eu_jev/", "/laya/", "/bespoke/", "/openrouter/", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index a9694634de7..bc64a4c7ef0 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -23170,6 +23170,39 @@ "title": "CachedTokensDetails", "type": "object" }, + "ChoiceAnswer": { + "additionalProperties": true, + "properties": { + "choice": { + "title": "Choice", + "type": "string" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "choice", + "confidence", + "probabilities" + ], + "title": "ChoiceAnswer", + "type": "object" + }, "Click": { "additionalProperties": true, "description": "A click action.", @@ -23934,6 +23967,112 @@ "title": "ErrorResponse", "type": "object" }, + "EuJevDecisionResponse": { + "properties": { + "answers": { + "additionalProperties": { + "discriminator": { + "mapping": { + "choice": "#/components/schemas/ChoiceAnswer", + "noul": "#/components/schemas/NoulAnswer", + "score": "#/components/schemas/ScoreAnswer" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/components/schemas/ChoiceAnswer" + }, + { + "$ref": "#/components/schemas/NoulAnswer" + }, + { + "$ref": "#/components/schemas/ScoreAnswer" + } + ] + }, + "description": "One answer per question, under the same names", + "title": "Answers", + "type": "object" + }, + "meta": { + "$ref": "#/components/schemas/EuJevMeta" + }, + "model": { + "description": "The release that answered, such as eu-jeff-1.0.0", + "title": "Model", + "type": "string" + }, + "usage": { + "$ref": "#/components/schemas/EuJevUsage" + } + }, + "required": [ + "model", + "answers", + "usage", + "meta" + ], + "title": "EuJevDecisionResponse", + "type": "object" + }, + "EuJevMeta": { + "properties": { + "cost_eur": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The exact charge in euros, on authenticated requests", + "title": "Cost Eur" + }, + "latency_ms": { + "description": "Server-side time for token quoting, credit reservation and inference", + "title": "Latency Ms", + "type": "integer" + }, + "mode": { + "const": "live", + "title": "Mode", + "type": "string" + }, + "request_id": { + "title": "Request Id", + "type": "string" + } + }, + "required": [ + "request_id", + "mode", + "latency_ms" + ], + "title": "EuJevMeta", + "type": "object" + }, + "EuJevUsage": { + "properties": { + "input_tokens": { + "minimum": 0.0, + "title": "Input Tokens", + "type": "integer" + }, + "output_tokens": { + "minimum": 0.0, + "title": "Output Tokens", + "type": "integer" + } + }, + "required": [ + "input_tokens", + "output_tokens" + ], + "title": "EuJevUsage", + "type": "object" + }, "FileSearchTool": { "additionalProperties": true, "description": "A tool that searches for relevant content from uploaded files.\n\nLearn more about the [file search tool](https://platform.openai.com/docs/guides/tools-file-search).", @@ -25594,6 +25733,26 @@ "title": "NamespaceTool", "type": "object" }, + "NoulAnswer": { + "additionalProperties": true, + "properties": { + "noul": { + "title": "Noul", + "type": "number" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "noul" + ], + "title": "NoulAnswer", + "type": "object" + }, "OperationCreateFile": { "additionalProperties": true, "description": "Instruction describing how to create a file via the apply_patch tool.", @@ -28866,6 +29025,59 @@ "title": "Result", "type": "object" }, + "ScoreAnswer": { + "additionalProperties": true, + "properties": { + "confidence": { + "title": "Confidence", + "type": "number" + }, + "legend": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "title": "Legend", + "type": "object" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "score": { + "title": "Score", + "type": "number" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "score", + "confidence", + "legend", + "probabilities" + ], + "title": "ScoreAnswer", + "type": "object" + }, "Screenshot": { "additionalProperties": true, "description": "A screenshot action.", @@ -31758,6 +31970,315 @@ ] } }, + "/eu_jev/v1/systemone": { + "post": { + "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and\n`stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "operationId": "eu_jev_proxy_route_eu_jev_v1_systemone_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "eu-jeff-latest", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "team": { + "criteria": { + "billing": "Payments, invoices, and refunds", + "support": "Technical issues and bugs" + }, + "instructions": "Which team should handle this?", + "type": "choice" + } + }, + "state": "I was charged twice. Can I get a refund?" + }, + "schema": { + "properties": { + "model": { + "default": "eu-jeff-latest", + "description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "description": "Two to 64 options, option name to description. A null description uses the option name", + "maxProperties": 64, + "minProperties": 2, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevChoiceQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what true and false mean", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "EuJevNoulQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "description": "Two to 10 level descriptions, ordered from low to high", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "maxItems": 10, + "minItems": 2, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevScoreQuestion", + "type": "object" + } + ] + }, + "description": "One to eight questions, keyed by names of up to 80 characters", + "maxProperties": 8, + "minProperties": 1, + "propertyNames": { + "maxLength": 80, + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "maxLength": 100000, + "minLength": 1, + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array", + "title": "State" + } + }, + "required": [ + "state", + "questions" + ], + "title": "EuJevDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EuJevDecisionResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Eu Jev Proxy Route", + "tags": [ + "llm_passthrough" + ] + } + }, "/fal_ai/{endpoint}": { "delete": { "operationId": "fal_ai_proxy_route_fal_ai__endpoint__delete", diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 33bec02e6bc..9350c9c935c 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -33,6 +33,7 @@ from litellm.litellm_core_utils.url_utils import ( from litellm.llms.azure.passthrough.transformation import azure_router_model_in_endpoint from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path from litellm.proxy._types import * +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_metadata from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_ENDPOINT_MARKER, @@ -884,7 +885,6 @@ def route_in_additonal_public_routes(current_route: str): public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate", "/api/*"] ``` """ - from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.proxy_server import general_settings, premium_user try: @@ -2175,6 +2175,13 @@ def _extract_model_candidates_from_request( except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc return _dedupe_model_candidates((f"{provider}/{decision_model}",)) + if RouteChecks.is_builtin_eu_jev_decision_route(route=route): + from litellm.llms.eu_jev.common_utils import eu_jev_registry_model + + requested_model: Final = TypeAdapter(Mapping[str, object]).validate_python(request_data).get("model") + if requested_model is not None and not isinstance(requested_model, str): + raise HTTPException(status_code=400, detail="eu_jev model must be a string") + return [f"eu_jev/{eu_jev_registry_model(requested_model)}"] if route == "/cost/predict-cache": prediction_models: Final = _cache_prediction_model_candidates(request_data, llm_router, team_id) # pyright: ignore[reportUnknownArgumentType] # the typed reader validates each deployment ID from this legacy payload return _dedupe_model_candidates(prediction_models) diff --git a/litellm/proxy/auth/oauth2_check.py b/litellm/proxy/auth/oauth2_check.py index 3633f7b6127..970f046c3d6 100644 --- a/litellm/proxy/auth/oauth2_check.py +++ b/litellm/proxy/auth/oauth2_check.py @@ -12,6 +12,11 @@ from litellm.llms.custom_httpx.http_handler import ( from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth +class OAuth2TokenValidationError(ValueError): + """The token-info or introspection endpoint did not vouch for the token: it rejected it, reported it + inactive, or could not be reached.""" + + class Oauth2Handler: """ Handles OAuth2 token validation. @@ -207,7 +212,7 @@ class Oauth2Handler: ) except httpx.HTTPStatusError as e: # This will catch any 4xx or 5xx errors - raise ValueError(f"Oauth 2.0 Token validation failed: {e}") + raise OAuth2TokenValidationError(f"Oauth 2.0 Token validation failed: {e}") except Exception as e: # This will catch any other errors (like network issues) - raise ValueError(f"An error occurred during token validation: {e}") + raise OAuth2TokenValidationError(f"An error occurred during token validation: {e}") diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 74e4c4c6d11..56be44600fd 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -73,6 +73,7 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES: Final = frozenset( _PROXY_ADMIN_VIEW_ONLY_BLOCKED_KEY_SUFFIXES: Final = ("/regenerate", "/reset_spend") _LLM_API_ROUTE_GROUPS: Final = frozenset(("llm_api_routes", "llm_api_routes_without_mcp")) +_EU_JEV_DECISION_ROUTE: Final = "/eu_jev/v1/systemone" _AUTH_ENFORCED_PASS_THROUGH_ROUTE_GROUPS: Final = frozenset(("openai_routes", *_LLM_API_ROUTE_GROUPS)) # Writes that only touch the caller's own credential row (BYOK connections and @@ -503,7 +504,19 @@ class RouteChecks: for _llm_passthrough_route in LiteLLMRoutes.mapped_pass_through_routes.value: if route == _llm_passthrough_route or route.startswith(_llm_passthrough_route + "/"): return True - return False + return RouteChecks.is_builtin_eu_jev_decision_route(route=route) + + @staticmethod + def is_builtin_eu_jev_decision_route(route: str) -> bool: + if route.rstrip("/") != _EU_JEV_DECISION_ROUTE: + return False + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( # noqa: PLC0415 # pass_through_endpoints imports the auth stack + InitPassThroughEndpointHelpers, + ) + + return not InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash( + route=_EU_JEV_DECISION_ROUTE + ) @staticmethod def _is_get_mcp_server_discovery_route(route: str, request: Request | None) -> bool: diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index e5b0beadebf..2b6471952fd 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -1160,6 +1160,24 @@ def _matches_routing_override(token_claims: dict, override: "JWTRoutingOverride" ) +def oauth2_introspection_applies( + api_key: str, route: str, *, enable_oauth2_auth: bool, enable_jwt_auth: bool, jwt_handler: JWTHandler +) -> bool: + """Whether LiteLLM validates ``api_key`` on ``route`` by OAuth2 token introspection, on LLM API, info and + lens routes only: a non-JWT token when OAuth2 auth is on globally, or a JWT matched by a routing override.""" + is_jwt: Final = enable_jwt_auth and jwt_handler.is_jwt(token=api_key) + # Routing uses unverified JWT claims only to choose auth path. + # Final authentication is enforced by the selected validator. + route_jwt_to_oauth2: Final = is_jwt and _should_route_jwt_to_oauth2_override(token=api_key, jwt_handler=jwt_handler) + covered_route: Final = ( + RouteChecks.is_llm_api_route(route=route) + or RouteChecks.is_info_route(route=route) + or route == "/lens" + or route.startswith("/lens/") + ) + return covered_route and ((enable_oauth2_auth and not is_jwt) or route_jwt_to_oauth2) + + def _should_route_jwt_to_oauth2_override(token: str, jwt_handler: JWTHandler) -> bool: routing_overrides: Final = jwt_handler.litellm_jwtauth.routing_overrides if not routing_overrides: @@ -2278,23 +2296,13 @@ async def user_api_key_auth_builder( enable_oauth2_auth: Final = general_settings.get("enable_oauth2_auth", False) is True enable_jwt_auth: Final = general_settings.get("enable_jwt_auth", False) is True - is_jwt = jwt_handler.is_jwt(token=api_key) if enable_jwt_auth else False - - # Routing uses unverified JWT claims only to choose auth path. - # Final authentication is enforced by the selected validator. - route_jwt_to_oauth2 = is_jwt and _should_route_jwt_to_oauth2_override(token=api_key, jwt_handler=jwt_handler) - - # OAuth2 applies for: - # 1) when global OAuth2 auth is enabled on LLM + info routes - # 2) JWT tokens that explicitly match routing_overrides on LLM + info routes - route_is_lens: Final = route == "/lens" or route.startswith("/lens/") - should_apply_override_oauth2: Final = route_jwt_to_oauth2 and ( - RouteChecks.is_llm_api_route(route=route) or RouteChecks.is_info_route(route=route) or route_is_lens - ) - should_apply_global_oauth2: Final = enable_oauth2_auth and ( - RouteChecks.is_llm_api_route(route=route) or RouteChecks.is_info_route(route=route) or route_is_lens - ) - if (should_apply_global_oauth2 and not is_jwt) or should_apply_override_oauth2: + if oauth2_introspection_applies( + api_key, + route, + enable_oauth2_auth=enable_oauth2_auth, + enable_jwt_auth=enable_jwt_auth, + jwt_handler=jwt_handler, + ): from litellm.proxy.proxy_server import premium_user if premium_user is not True: diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index a46ea4d9bd7..519cfb52240 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -115,6 +115,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( error_status_code, openai_error_param, openai_error_type, + relayed_provider_specific_fields, ) from litellm.proxy.common_utils.sse_keepalive import ( SSE_COMMENT_PING_BYTES, @@ -589,6 +590,7 @@ def model_deployments_support_stream_options( _STREAM_USAGE_FLAGS: Final = ("_litellm_strip_stream_usage", STREAM_USAGE_INJECTED_FIELD) +_STREAM_USAGE_TRACKED_ROUTES: Final = frozenset({"acompletion", "atext_completion"}) def stream_usage_tracking_updates( @@ -614,7 +616,7 @@ def stream_usage_tracking_updates( return scrub if always_include is False and route_type == "agenerate_content_stream" and "stream_options" not in data: return {**scrub, "stream_options": {"include_usage": False}} - if always_include is False or route_type != "acompletion": + if always_include is False or route_type not in _STREAM_USAGE_TRACKED_ROUTES: return scrub if isinstance(stream_options, dict) and stream_options.get("include_usage") is True: return scrub @@ -4486,7 +4488,8 @@ class ProxyBaseLLMRequestProcessing: param=openai_error_param(e), openai_code=getattr(e, "code", None), code=_code, - provider_specific_fields=getattr(e, "provider_specific_fields", None), + provider_specific_fields=getattr(e, "provider_specific_fields", None) + or relayed_provider_specific_fields(e), headers=safe_headers, ) diff --git a/litellm/proxy/common_utils/openai_error_payload.py b/litellm/proxy/common_utils/openai_error_payload.py index 3c91e87ee51..f92fbeef9a1 100644 --- a/litellm/proxy/common_utils/openai_error_payload.py +++ b/litellm/proxy/common_utils/openai_error_payload.py @@ -8,7 +8,9 @@ from types import MappingProxyType from typing import Final from fastapi import HTTPException, status +from pydantic import TypeAdapter +from litellm._logging import redact_internal_details_from_client_fields from litellm.constants import STRINGIFIED_NONE from litellm.proxy._types import ProxyErrorTypes, ProxyException, proxy_exception_status_code from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded @@ -26,6 +28,8 @@ _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType( _AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN}) +_ERROR_BODY: Final = TypeAdapter(dict[str, object]) + _FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"}) _FAILOVER_REFUSALS: Final = ( @@ -77,6 +81,18 @@ def openai_error_param(exc: object) -> str | None: return carried if isinstance(carried, str) and carried != STRINGIFIED_NONE else None +def relayed_provider_specific_fields( + exc: object, +) -> dict[str, object] | None: # mutable-ok: ProxyException.provider_specific_fields is a dict + body: Final = attribute_of(exc, "body") + if not isinstance(body, dict): + return None + fields: Final = _ERROR_BODY.validate_python(body).get("provider_specific_fields") + if not isinstance(fields, dict): + return None + return _ERROR_BODY.validate_python(redact_internal_details_from_client_fields(_ERROR_BODY.validate_python(fields))) + + def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyException: match exc: case ProxyException(): diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py index ecd23d86540..7e052796545 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py @@ -664,4 +664,7 @@ class GenericGuardrailAPI(CustomGuardrail): GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call, GuardrailEventHooks.during_call, + GuardrailEventHooks.pre_mcp_call, + GuardrailEventHooks.during_mcp_call, + GuardrailEventHooks.post_mcp_call, ] diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 8f60311e491..a87fc815402 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2505,6 +2505,7 @@ def refresh_proxy_server_request_body_snapshot( "litellm_logging_obj", VECTOR_STORE_CALLER_METADATA_KWARG, STREAM_USAGE_INJECTED_FIELD, + "_litellm_strip_stream_usage", } ) | _TRANSPORT_ONLY_CREDENTIAL_KEYS diff --git a/litellm/proxy/mcp_server/AGENTS.md b/litellm/proxy/mcp_server/AGENTS.md index 25150c11f1d..b66128454b8 100644 --- a/litellm/proxy/mcp_server/AGENTS.md +++ b/litellm/proxy/mcp_server/AGENTS.md @@ -68,8 +68,11 @@ module materially harder to understand. behind a single generic branch unless tests prove every mode still behaves correctly. - Be especially careful with legacy `delegate_auth_to_upstream: true`. `auth_type: oauth2` - with `delegate_auth_to_upstream: true` is deprecated: LiteLLM admission is required - for matching MCP routes. Use `auth_type: oauth_delegate` for client-forwarded OAuth. + with `delegate_auth_to_upstream: true` is deprecated but still supported. A bare + `Authorization` that is a LiteLLM credential is validated and never forwarded. Any other + bearer is the upstream token: it is forwarded, and the caller is admitted as the JWT or + custom-auth user it also proves when that user is granted the server, else anonymously. + New servers should use `auth_type: true_passthrough` or `auth_type: oauth_delegate`. OAuth discovery endpoints stay public so clients can start the RFC 9728 flow. - Keep database-backed fields in sync across migrations, typed models under `litellm/types/mcp.py` or `litellm/types/mcp_server/`, config loading, this diff --git a/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py index cc7080738d0..db640b6d689 100644 --- a/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py @@ -2,14 +2,15 @@ import asyncio import functools import re import secrets -from collections.abc import Awaitable, Callable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from collections.abc import Set as AbstractSet from dataclasses import dataclass from datetime import datetime, timezone from itertools import chain from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal +from typing import TYPE_CHECKING, Final, Literal, TypeAlias +import jwt from fastapi import HTTPException from pydantic import SecretStr, TypeAdapter from starlette.datastructures import Headers @@ -45,9 +46,11 @@ from litellm.proxy.agent_endpoints.auth.agent_access_groups import ( from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_auth from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_agent_policy from litellm.proxy.auth.ip_address_utils import IPAddressUtils +from litellm.proxy.auth.oauth2_check import OAuth2TokenValidationError from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401 # legacy module exports _get_bearer_token_or_received_api_key, # pyright: ignore[reportPrivateUsage] # shared x-litellm-api-key parser lives with user_api_key_auth _run_centralized_common_checks, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + oauth2_introspection_applies, run_centralized_common_checks, user_api_key_auth, ) @@ -100,6 +103,9 @@ if TYPE_CHECKING: _EMPTY_TOOLSET_GRANTS: Final[Mapping[str, Sequence[str]]] = MappingProxyType({}) OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) +_PROXY_SETTINGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_CAUSE_CHAIN_MAX_DEPTH: Final = 16 +_IdentityCheck: TypeAlias = Literal["oauth2", "oauth2_proxy", "jwt"] _CACHED_VALUE: Final = TypeAdapter(object) @@ -233,6 +239,136 @@ def _is_legacy_delegate_cold_start(mcp_servers: list[str] | None, client_ip: str return True +def _cause_chain(exc: BaseException) -> Iterator[BaseException]: + link: BaseException | None = exc # rebind-ok: advances one link of the cause chain per iteration + for _ in range(_CAUSE_CHAIN_MAX_DEPTH): + if link is None: + return + yield link + link = link.__cause__ or link.__context__ + + +def _caused_by(exc: BaseException, error_type: type[BaseException]) -> bool: + return any(isinstance(link, error_type) for link in _cause_chain(exc)) + + +def _signed_with(token: str, secret: str) -> bool: + """True when ``token`` is a JWT whose HS256 signature verifies with ``secret``, whatever its claims say.""" + try: + jwt.decode( + token, + secret, + algorithms=["HS256"], + options={"verify_exp": False, "verify_nbf": False, "verify_iat": False, "verify_aud": False}, + ) + except jwt.PyJWTError: + return False + return True + + +async def _bearer_is_litellm_credential(token: str) -> bool: + """True when ``token`` is, or may be, a credential LiteLLM issued: the master key or a JWT it signed + (dashboard sessions, BYOK sessions), a sealed gateway bearer, or a virtual key the key table holds (a + lookup fault counts as held). Only ``sk-`` tokens are looked up, because no other token can be a + virtual key.""" + from litellm.proxy.mcp_server.bridge_token_flow import ( # noqa: PLC0415 # bridge flow imports this module + is_sealed_gateway_bearer, + virtual_key_may_exist, + ) + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # manager imports this module + LITELLM_VIRTUAL_KEY_PREFIX, + ) + from litellm.proxy.proxy_server import master_key # noqa: PLC0415 # startup owns the master key + + if master_key and (secrets.compare_digest(token.encode(), master_key.encode()) or _signed_with(token, master_key)): + return True + if is_sealed_gateway_bearer(token): + return True + return token.startswith(LITELLM_VIRTUAL_KEY_PREFIX) and await virtual_key_may_exist(token) + + +def _identity_check(token: str, route: str, settings: Mapping[str, object]) -> _IdentityCheck | None: + """The identity check LiteLLM's own auth would run on ``token`` for ``route``, in its order: OAuth2 + introspection, then trusted reverse-proxy identity headers, then a JWT whose issuer may be LiteLLM's.""" + from litellm.proxy.auth.handle_jwt import JWTHandler # noqa: PLC0415 # proxy import cycle + from litellm.proxy.mcp_server.bridge_token_flow import ( # noqa: PLC0415 # bridge flow imports this module + jwt_issuer_may_be_litellm, + ) + from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the active auth configuration + jwt_handler, + premium_user, + ) + + enable_jwt_auth: Final = settings.get("enable_jwt_auth") is True + if oauth2_introspection_applies( + token, + route, + enable_oauth2_auth=settings.get("enable_oauth2_auth") is True, + enable_jwt_auth=enable_jwt_auth, + jwt_handler=jwt_handler, + ): + return "oauth2" if premium_user is True else None + if settings.get("enable_oauth2_proxy_auth") is True: + return "oauth2_proxy" + if enable_jwt_auth and premium_user is True and JWTHandler.is_jwt(token=token) and jwt_issuer_may_be_litellm(token): + return "jwt" + return None + + +async def _verify_identity(check: Literal["oauth2_proxy", "jwt"], token: str, request: Request) -> None: + from litellm.proxy.auth.oauth2_proxy_hook import ( # noqa: PLC0415 # proxy import cycle + handle_oauth2_proxy_request, + ) + from litellm.proxy.proxy_server import jwt_handler # noqa: PLC0415 # startup owns the active auth configuration + + match check: + case "oauth2_proxy": + await handle_oauth2_proxy_request(request=request) + case "jwt": + await jwt_handler.auth_jwt(token=token) # pyright: ignore[reportUnknownMemberType] # returns a bare dict; only whether it raises matters + case _: # pragma: no cover - exhaustiveness guard, unreachable while the Literal holds + assert_never(check) # pragma: no cover + + +async def _upstream_bearer_identity(token: str, litellm_api_key: str, request: Request) -> UserAPIKeyAuth | None: + """The LiteLLM user an upstream bearer also proves, or ``None`` when it proves none. + + Custom auth decides alone, so any rejection is ``None``. Otherwise the identity check LiteLLM would run + must vouch for the caller. Reverse-proxy headers and a JWT are verified first, and OAuth2 introspection + runs once inside the full check, where a provider that does not vouch is ``None``. Past that, only a + user LiteLLM does not know is ``None``; every other refusal (budget, blocked team, disallowed route) + propagates.""" + from litellm.proxy.auth.auth_utils import get_request_route # noqa: PLC0415 # proxy import cycle + from litellm.proxy.mcp_server.bridge_token_flow import ( # noqa: PLC0415 # bridge flow imports this module + custom_auth_configured, + ) + from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the active auth configuration + general_settings, # pyright: ignore[reportUnknownVariableType] # legacy global is declared as a bare dict + ) + + if custom_auth_configured(): + try: + return await user_api_key_auth(api_key=litellm_api_key, request=request) + except (HTTPException, ProxyException): + return None + check: Final = _identity_check( + token, get_request_route(request), _PROXY_SETTINGS_ADAPTER.validate_python(general_settings) + ) + if check is None: + return None + if check != "oauth2": + try: + await _verify_identity(check, token, request) + except Exception: # noqa: BLE001 # any verification failure means LiteLLM does not vouch for this bearer + return None + try: + return await user_api_key_auth(api_key=litellm_api_key, request=request) + except (HTTPException, ProxyException, OAuth2TokenValidationError) as exc: + if _caused_by(exc, UserNotFoundError) or _caused_by(exc, OAuth2TokenValidationError): + return None + raise + + def _is_litellm_auth_admission_error(exc: Exception) -> bool: if isinstance(exc, HTTPException): return exc.status_code == 401 @@ -704,6 +840,19 @@ class MCPRequestHandler: route=request_route, mcp_servers=mcp_servers, ) + elif oauth2_headers and ( + legacy_delegate_targets := MCPRequestHandler._legacy_delegate_targets( + path=request_route, + mcp_servers=mcp_servers, + client_ip=IPAddressUtils.get_mcp_client_ip(request), + ) + ): + validated_user_api_key_auth = await MCPRequestHandler._admit_legacy_delegate_bearer( # rebind-ok: one assignment per admission arm, like its siblings + authorization_value=oauth2_headers["Authorization"], + litellm_api_key=litellm_api_key, + targets=legacy_delegate_targets, + request=request, + ) elif oauth2_headers: # Authorization on a non-delegated server: the bearer must be a real # LiteLLM credential, so a failed validation is a genuine 401/403 and @@ -741,10 +890,14 @@ class MCPRequestHandler: custom_key_header_name: Final = OPTIONAL_STRING_ADAPTER.validate_python( general_settings.get("litellm_key_header_name") ) - admitted_credential: Final = MCPRequestHandler.caller_admission_credential( - headers, - validated_user_api_key_auth, - custom_key_header_name=custom_key_header_name, + admitted_credential: Final = ( + None + if validated_user_api_key_auth.mcp_forwards_caller_authorization + else MCPRequestHandler.caller_admission_credential( + headers, + validated_user_api_key_auth, + custom_key_header_name=custom_key_header_name, + ) ) ( oauth2_headers, @@ -934,6 +1087,53 @@ class MCPRequestHandler: return False return True + @staticmethod + def _legacy_delegate_targets( + path: str, mcp_servers: Sequence[str] | None, client_ip: str | None + ) -> tuple[MCPServer, ...]: + """Every server the request targets when ALL of them are legacy ``delegate_auth_to_upstream`` + servers, else ``()``. Names resolve the way downstream routing resolves them, so a permissive + ``x-mcp-servers`` header cannot open the gate for a stricter path target.""" + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # manager imports this module + global_mcp_server_manager, + is_legacy_upstream_delegate, + ) + + servers: Final = tuple( + global_mcp_server_manager.get_mcp_server_by_name(name, client_ip=client_ip) + for name in MCPRequestHandler._resolve_target_server_names(path=path, mcp_servers_header=mcp_servers) + ) + delegates: Final = tuple( + server for server in servers if server is not None and is_legacy_upstream_delegate(server) + ) + return delegates if servers and len(delegates) == len(servers) else () + + @staticmethod + async def _admit_legacy_delegate_bearer( + authorization_value: str, + litellm_api_key: str, + targets: tuple[MCPServer, ...], + request: Request, + ) -> UserAPIKeyAuth: + """Admit a bare ``Authorization`` bearer sent to legacy ``delegate_auth_to_upstream`` servers. + + A LiteLLM credential runs normal admission, and its errors propagate. Any other bearer is the + upstream token and is marked for forwarding: a user it also proves who is granted every target is + admitted as that user, and every other caller is admitted anonymously.""" + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # manager imports this module + global_mcp_server_manager, + ) + + token: Final = _normalize_caller_admission_credential(authorization_value) + if await _bearer_is_litellm_credential(token): + return await user_api_key_auth(api_key=litellm_api_key, request=request) + identity: Final = await _upstream_bearer_identity(token, litellm_api_key, request) + if identity is not None: + allowed: Final = frozenset(await global_mcp_server_manager.get_allowed_mcp_servers(identity)) + if all(server.server_id in allowed for server in targets): + return identity.model_copy(update={"mcp_forwards_caller_authorization": True}) + return UserAPIKeyAuth().model_copy(update={"mcp_forwards_caller_authorization": True}) + @staticmethod def _single_dcr_bridge_delegate_target( path: str, mcp_servers: list[str] | None, client_ip: str | None @@ -1553,7 +1753,7 @@ class MCPRequestHandler: return expiry >= datetime.now(timezone.utc) @staticmethod - def _resolve_target_server_names(path: str, mcp_servers_header: list[str] | None) -> list[str]: + def _resolve_target_server_names(path: str, mcp_servers_header: Sequence[str] | None) -> Sequence[str]: """ Resolve the target MCP server names exactly as downstream routing does (``server.py::extract_mcp_auth_context``). diff --git a/litellm/proxy/mcp_server/bridge_token_flow.py b/litellm/proxy/mcp_server/bridge_token_flow.py index 63428dfb28a..dd9be4cb3e1 100644 --- a/litellm/proxy/mcp_server/bridge_token_flow.py +++ b/litellm/proxy/mcp_server/bridge_token_flow.py @@ -61,7 +61,6 @@ async def oauth_authorization_uses_gateway_credential(request: Request) -> bool: from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the active auth configuration jwt_handler, master_key, - user_custom_auth, ) if "x-litellm-api-key" in request.headers: @@ -71,10 +70,26 @@ async def oauth_authorization_uses_gateway_credential(request: Request) -> bool: return "authorization" in request.headers if token.startswith("sk-") or (master_key and secrets.compare_digest(token.encode(), master_key.encode())): return True - if user_custom_auth is not None or jwt_handler.litellm_jwtauth.oidc_userinfo_enabled: + if custom_auth_configured() or jwt_handler.litellm_jwtauth.oidc_userinfo_enabled: return True if not JWTHandler.is_jwt(token): - return await _opaque_bearer_is_gateway_credential(token) + return is_sealed_gateway_bearer(token) or await virtual_key_may_exist(token) + return jwt_issuer_may_be_litellm(token) + + +def custom_auth_configured() -> bool: + from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the active auth configuration + user_custom_auth, + ) + + return user_custom_auth is not None + + +def jwt_issuer_may_be_litellm(token: str) -> bool: + """False only when the JWT names an issuer LiteLLM's JWT auth is not configured to accept.""" + from litellm.proxy.auth.handle_jwt import JWTHandler # noqa: PLC0415 # proxy import cycle + from litellm.proxy.proxy_server import jwt_handler # noqa: PLC0415 # startup owns the active auth configuration + claims: Final = JWTHandler.get_unverified_claims(token) issuer: Final = claims.get("iss") if claims is not None else None global_issuer: Final = os.getenv("JWT_ISSUER") @@ -86,25 +101,35 @@ async def oauth_authorization_uses_gateway_credential(request: Request) -> bool: ) -async def _opaque_bearer_is_gateway_credential(token: str) -> bool: - from litellm.proxy._types import hash_token # noqa: PLC0415 # proxy import cycle +def is_sealed_gateway_bearer(token: str) -> bool: + """True for a bearer only the gateway issues: a bridge or refresh envelope, a v2 GCM blob, or a + dashboard UI token. A UI token that cannot be read still counts, so a lookup fault never reads as foreign.""" from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken # noqa: PLC0415 # proxy import cycle - from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError # noqa: PLC0415 # proxy import cycle - from litellm.proxy.auth.resolvers.store import IdentityStore # noqa: PLC0415 # proxy import cycle from litellm.proxy.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # envelope imports bridge types is_envelope, is_refresh_envelope, ) - from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the identity store dependencies - prisma_client, - user_api_key_cache, - ) if is_envelope(token) or is_refresh_envelope(token) or token.startswith(V2_GCM_PREFIX): return True try: - if ExperimentalUIJWTToken.get_key_object_from_ui_hash_key(token) is not None: - return True + return ExperimentalUIJWTToken.get_key_object_from_ui_hash_key(token) is not None + except Exception as exc: # noqa: BLE001 # an identity lookup fault must not permit cookie fallback + verbose_logger.debug("OAuth bearer ownership could not be checked (%s)", type(exc).__name__) + return True + + +async def virtual_key_may_exist(token: str) -> bool: + """False only when the key table answers that ``token`` is not a virtual key; a lookup fault counts as held.""" + from litellm.proxy._types import hash_token # noqa: PLC0415 # proxy import cycle + from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError # noqa: PLC0415 # proxy import cycle + from litellm.proxy.auth.resolvers.store import IdentityStore # noqa: PLC0415 # proxy import cycle + from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the identity store dependencies + prisma_client, + user_api_key_cache, + ) + + try: await IdentityStore(prisma_client, user_api_key_cache).resolve(hashed_token=hash_token(token)) except KeyNotFoundError: return False diff --git a/litellm/proxy/mcp_server/mcp_server_manager.py b/litellm/proxy/mcp_server/mcp_server_manager.py index ceac2b2a8e1..c772e2e9d09 100644 --- a/litellm/proxy/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/mcp_server/mcp_server_manager.py @@ -1196,8 +1196,11 @@ def should_strip_caller_authorization( pass-through cold-start case (RFC 9728) the bearer in ``Authorization`` is the upstream OAuth token and must be forwarded, so we keep it. - - **Delegated OAuth servers**: admission always runs and there is no - anonymous path, so the caller's separate ``Authorization`` is + - **Legacy delegate servers**: forward when MCP admission marked the + caller's ``Authorization`` as the upstream token + (``mcp_forwards_caller_authorization``); otherwise follow the + delegated OAuth rule below. + - **Delegated OAuth servers**: the caller's separate ``Authorization`` is forwarded only when a distinct ``x-litellm-api-key`` carried admission. Without that header the ``Authorization`` *was* the admission credential — a virtual key, an IdP JWT, or an SSO / OIDC / @@ -1206,6 +1209,12 @@ def should_strip_caller_authorization( - **Every other auth-enabled mode**: strip when ``Authorization`` is the credential admission validated. """ + if ( + user_api_key_auth is not None + and user_api_key_auth.mcp_forwards_caller_authorization + and is_legacy_upstream_delegate(mcp_server) + ): + return False if _authorization_is_admission_jwt(raw_headers, user_api_key_auth): return True if mcp_server.auth_type == MCPAuth.oauth2_token_exchange: @@ -1557,6 +1566,18 @@ async def _materialize_auth_headers(auth: httpx2.Auth | None) -> dict[str, str] return {header_name: header_value} if header_value else None +def is_legacy_upstream_delegate(server: MCPServer) -> bool: + """True for an interactive ``auth_type=oauth2`` server with ``delegate_auth_to_upstream=true``, whose + caller ``Authorization`` is the upstream token. The flow is resolved, not read from the column, so an + unstamped M2M-shaped row stays excluded: its stored client credentials drive egress, and admitting it + anonymously would let any caller act as LiteLLM's service account.""" + return ( + server.auth_type == MCPAuth.oauth2 + and server.delegate_auth_to_upstream is True + and MCPServerManager.effective_oauth2_flow(server) != "client_credentials" + ) + + def _consumes_caller_authorization(server: MCPServer) -> bool: """True when this server's egress forwards the caller's request-wide ``Authorization`` upstream: the client-forwarded token modes, legacy OAuth pass-through, and legacy upstream-delegated @@ -1726,18 +1747,15 @@ def _warn_overlapping_tool_policies(server: MCPServer) -> None: def _warn_legacy_delegate_auth_if_applicable(server: MCPServer, *, source: str) -> None: - """Direct legacy delegated OAuth configurations to the admitted replacement.""" - if server.auth_type != MCPAuth.oauth2: - return - if getattr(server, "delegate_auth_to_upstream", False) is not True: - return - if server.has_client_credentials: + """Point legacy delegated OAuth configurations at the client-forwarded replacements.""" + if not is_legacy_upstream_delegate(server): return label: Final = get_server_prefix(server) verbose_logger.warning( "MCP server %r (id=%s, source=%s) uses deprecated auth_type=oauth2 with " - "delegate_auth_to_upstream=true. LiteLLM admission is now required; migrate to " - "auth_type=oauth_delegate for client-forwarded OAuth.", + "delegate_auth_to_upstream=true. It still works: a caller without a LiteLLM credential is " + "admitted anonymously and its upstream token is forwarded. Migrate to auth_type=true_passthrough " + "or auth_type=oauth_delegate.", label, server.server_id, source, @@ -3766,12 +3784,12 @@ class MCPServerManager: ) ) if is_anonymous: - passthrough_server_ids: Final = [ + anonymous_server_ids: Final = [ server.server_id for server in self.get_registry().values() - if server.auth_type == MCPAuth.true_passthrough + if server.auth_type == MCPAuth.true_passthrough or is_legacy_upstream_delegate(server) ] - combined_servers.update(passthrough_server_ids) + combined_servers.update(anonymous_server_ids) restrict_allow_all: Final = ( resolved_general_settings.get("mcp_allow_all_keys_respects_mcp_scope", False) diff --git a/litellm/proxy/mcp_server/server.py b/litellm/proxy/mcp_server/server.py index 32f790a8e81..2105d875b58 100644 --- a/litellm/proxy/mcp_server/server.py +++ b/litellm/proxy/mcp_server/server.py @@ -588,6 +588,7 @@ if MCP_AVAILABLE: from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, global_mcp_server_manager, + is_legacy_upstream_delegate, ) ###################################################### @@ -2162,8 +2163,9 @@ if MCP_AVAILABLE: oauth2_headers: Mapping[str, str] | None, mcp_server_auth_headers: dict[str, dict[str, str]] | None, raw_headers: Mapping[str, str] | None, + requested_servers: Sequence[str] | None = None, ) -> None: - """Probe pass-through upstream servers in parallel before the MCP session starts. + """Probe pass-through and legacy delegate upstream servers in parallel before the MCP session starts. Only servers the caller's key is already authorized to reach are probed — the list is derived from _get_allowed_mcp_servers so that a user cannot @@ -2186,6 +2188,18 @@ if MCP_AVAILABLE: mcp_servers=mcp_servers, client_ip=client_ip, ) + # A legacy delegate is probed only when the single requested name resolves to it the way admission + # resolves it, so neither a multi-server connect nor an access group can be failed by one rejected + # token; its challenge echoes the requested name, matching the tokenless challenge. + requested_scope: Final = requested_servers if requested_servers is not None else mcp_servers + single_requested_name: Final = ( + requested_scope[0] if requested_scope is not None and len(requested_scope) == 1 else None + ) + requested_server: Final = ( + operations.global_mcp_server_manager.get_mcp_server_by_name(single_requested_name, client_ip=client_ip) + if single_requested_name is not None + else None + ) prepared_headers: Final = tuple( ( srv, @@ -2201,6 +2215,11 @@ if MCP_AVAILABLE: ) for srv in allowed_servers if srv.is_oauth_passthrough + or ( + requested_server is not None + and srv.server_id == requested_server.server_id + and is_legacy_upstream_delegate(srv) + ) ) probe_headers: Final = tuple( ( @@ -2213,7 +2232,7 @@ if MCP_AVAILABLE: for srv, (auth, extra) in prepared_headers ) probe_targets: Final[tuple[tuple[MCPServer, str, str], ...]] = tuple( - (srv, auth_header, srv.name) + (srv, auth_header, srv.name if srv.is_oauth_passthrough else single_requested_name or srv.name) for srv, headers in probe_headers if ( auth_header := next( @@ -2311,6 +2330,7 @@ if MCP_AVAILABLE: if context.mcp_server_auth_headers is not None else None, raw_headers=context.raw_headers, + requested_servers=context.mcp_servers, ) return None diff --git a/litellm/proxy/middleware/billable_request_metrics_middleware.py b/litellm/proxy/middleware/billable_request_metrics_middleware.py index 96c3276efac..fd78b6b58da 100644 --- a/litellm/proxy/middleware/billable_request_metrics_middleware.py +++ b/litellm/proxy/middleware/billable_request_metrics_middleware.py @@ -25,6 +25,7 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send from litellm._logging import verbose_proxy_logger from litellm.proxy._types import LiteLLMRoutes +from litellm.proxy.auth.route_checks import RouteChecks class BillableCategory(str, Enum): @@ -116,6 +117,8 @@ def _classify_llm_route(path: str) -> str | None: suffix_match = next((suffix for suffix in _LLM_ROUTE_SUFFIXES if path == suffix or path.endswith(suffix)), None) if suffix_match is not None: return suffix_match + if RouteChecks.is_builtin_eu_jev_decision_route(route=path): + return path # Deep passthrough paths only: the bare prefix itself is not an inference call. return next((prefix for prefix in _PASSTHROUGH_PREFIXES if path.startswith(f"{prefix}/")), None) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 0a09743217c..21078dfc848 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -91,6 +91,7 @@ from litellm.proxy.common_request_processing import ( open_sse_before_first_byte, resolve_litellm_call_id, ) +from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports _read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -119,6 +120,7 @@ from litellm.proxy.pass_through_endpoints.injected_stream_usage import ( ) from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( HttpPassThroughEndpointHelpers, + YieldsToConfiguredPassThroughRoute, create_pass_through_route, create_websocket_passthrough_route, websocket_passthrough_request, @@ -132,6 +134,7 @@ from litellm.proxy.vector_store_endpoints.utils import ( is_allowed_to_call_vector_store_endpoint, ) from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str, str_to_bool +from litellm.types.passthrough_endpoints.eu_jev import EuJevDecisionRequest, EuJevDecisionResponse from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, @@ -714,6 +717,85 @@ def typesafe_allowlist_model_names() -> tuple[str, ...]: return ("typesafe/*", *jev_models_typesafe_serves) +_EU_JEV_DECISION_REQUEST_BODY: Final = inline_request_body( + EuJevDecisionRequest, + { + "model": "eu-jeff-latest", + "state": "I was charged twice. Can I get a refund?", + "questions": { + "team": { + "type": "choice", + "instructions": "Which team should handle this?", + "criteria": {"billing": "Payments, invoices, and refunds", "support": "Technical issues and bugs"}, + }, + "is_refund_request": {"type": "noul", "instructions": "Is the customer asking for a refund?"}, + "frustration": { + "type": "score", + "instructions": "How frustrated is the customer?", + "criteria": ["calm", "mildly annoyed", "angry"], + }, + }, + }, +) + + +eu_jev_router: Final = APIRouter(route_class=YieldsToConfiguredPassThroughRoute) + + +@eu_jev_router.post( + "/eu_jev/v1/systemone", + tags=["eu/jev Pass-through", "pass-through"], + responses={200: {"model": EuJevDecisionResponse}}, + openapi_extra={"requestBody": _EU_JEV_DECISION_REQUEST_BODY}, +) +async def eu_jev_proxy_route( + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> Response: + """ + Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + + `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and + `stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it. + + [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + """ + body: Final = TypeAdapter(dict[str, object]).validate_python(await read_request_body(request)) + if "custom_body" in body: + raise HTTPException(status_code=400, detail="custom_body is not supported for eu_jev requests") + if body.get("stream"): + raise HTTPException(status_code=400, detail="Streaming is not supported for eu_jev requests") + eu_jev_credentials: Final = passthrough_endpoint_router.get_proxy_wide_credentials(custom_llm_provider="eu_jev") + if eu_jev_credentials is None: + raise HTTPException(status_code=503, detail="eu_jev is not configured, set EU_JEV_API_KEY on the proxy") + base_url: Final = httpx.URL( + (eu_jev_credentials.api_base or get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com") + .rstrip("/") + .removesuffix("/v1") + ) + updated_url: Final = base_url.copy_with( + path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, "/v1/systemone"), + ) + endpoint_func: Final = create_pass_through_route( + endpoint="v1/systemone", + target=str(updated_url), + custom_headers={ + "Authorization": f"Bearer {eu_jev_credentials.api_key}", + "Content-Type": "application/json", + }, + custom_llm_provider="eu_jev", + is_streaming_request=False, + ) + return TypeAdapter(Response, config=ConfigDict(arbitrary_types_allowed=True)).validate_python( + await endpoint_func(request, fastapi_response, user_api_key_dict) + ) + + +router.include_router(eu_jev_router) + + @router.post( "/laya/v1/systemone", tags=["Laya Pass-through", "pass-through"], diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 23dffb4a2bb..bda2bfb9e04 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -26,9 +26,11 @@ from fastapi import ( status, ) from fastapi.responses import StreamingResponse +from fastapi.routing import APIRoute from pydantic import TypeAdapter from starlette.datastructures import UploadFile as StarletteUploadFile -from starlette.routing import BaseRoute, Route +from starlette.routing import BaseRoute, Match, Route +from starlette.types import Scope from starlette.websockets import WebSocketState from typing_extensions import ReadOnly from websockets.asyncio.client import connect @@ -73,6 +75,7 @@ from litellm.llms.base_llm.managed_resources.utils import ( resolve_passthrough_managed_id_provider, ) from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.llms.eu_jev.common_utils import eu_jev_scoped_model, eu_jev_upstream_body from litellm.llms.oss_decision import validate_oss_request from litellm.passthrough import BasePassthroughUtils from litellm.proxy._lazy_features import lazy_owned_routes @@ -90,6 +93,7 @@ from litellm.proxy.auth.auth_utils import ( get_request_route, request_dispatched_to_pass_through_endpoint, ) +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, @@ -433,9 +437,9 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): parsed_url: Final = urlparse(url) if custom_llm_provider == "chatgpt": return EndpointType.OPENAI - if custom_llm_provider in ("typesafe", "laya", "bespoke") and parsed_url.path.removesuffix("/").endswith( - "/v1/systemone" - ): + if custom_llm_provider in ("typesafe", "eu_jev", "laya", "bespoke") and parsed_url.path.removesuffix( + "/" + ).endswith("/v1/systemone"): return EndpointType.DECISIONS if ( ("generateContent") in url @@ -1137,6 +1141,16 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]: return headers +def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None: + return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None + + +def _eu_jev_scoped_model_override(eu_jev_client_body: Mapping[str, object] | None) -> Mapping[str, str]: + if eu_jev_client_body is None: + return MappingProxyType({}) + return MappingProxyType({"model": eu_jev_scoped_model(eu_jev_client_body)}) + + async def pass_through_request( request: Request, target: str, @@ -1255,7 +1269,12 @@ async def pass_through_request( strip_client_guardrail_information(client_body) if not _key_or_team_allows_client_pricing_override(user_api_key_dict): _strip_client_pricing_overrides(client_body) - _parsed_body = client_body + eu_jev_client_body: Final = _eu_jev_client_body(custom_llm_provider, client_body) + scoped_client_body: Final = { + **client_body, + **_eu_jev_scoped_model_override(eu_jev_client_body), + } + _parsed_body = scoped_client_body if custom_llm_provider in ("laya", "bespoke"): decision_request: Final = TypeAdapter(Mapping[str, object]).validate_python(_parsed_body) checkpoint: Final = validate_oss_request(custom_llm_provider, decision_request) @@ -1339,6 +1358,12 @@ async def pass_through_request( ) _ = validate_oss_request(custom_llm_provider, decision_body) _parsed_body = TypeAdapter(dict[str, object]).validate_python(decision_body) + if eu_jev_client_body is not None: + _parsed_body = TypeAdapter(dict[str, object]).validate_python( # rebind-ok: body is rebound per stage + eu_jev_upstream_body( + TypeAdapter(Mapping[str, object]).validate_python(_parsed_body), eu_jev_client_body + ) + ) resolved_timeout: Final = resolve_pass_through_request_timeout(timeout) async_client_obj: Final = get_async_httpx_client( llm_provider=httpxSpecialProvider.PassThroughEndpoint, @@ -2240,17 +2265,23 @@ def create_pass_through_route( stream, ) = await _parse_request_data_by_content_type(request) - if not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=path): + if not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=path) or ( + custom_llm_provider is None and RouteChecks.is_builtin_eu_jev_decision_route(route=path) + ): raise HTTPException( status_code=404, detail=f"Pass-through endpoint {endpoint} not found. This could have been deleted or not yet added to the proxy.", ) - passthrough_params: Final = InitPassThroughEndpointHelpers.get_registered_pass_through_route( - route=path, method=request.method + serves_configured_route: Final = custom_llm_provider != "eu_jev" + passthrough_params: Final = ( + InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=path, method=request.method) + if serves_configured_route + else None ) if ( - passthrough_params is None + serves_configured_route + and passthrough_params is None and InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=path) is not None ): raise HTTPException( @@ -3392,6 +3423,37 @@ class InitPassThroughEndpointHelpers: del _registered_pass_through_routes[key] verbose_proxy_logger.debug("Removed pass-through route from registry: %s", key) + @staticmethod + def _llm_api_routes_of_route(route_info: Mapping[str, object]) -> tuple[str, ...]: + path: Final = route_info.get("path") + if not isinstance(path, str): + return () + if route_info.get("type") == "subpath": + return (path, path.rstrip("/") + "/*") + return (path,) + + @staticmethod + def llm_api_routes_of_endpoint(endpoint_id: str) -> frozenset[str]: + route_infos: Final = ( + route_info + for route_info in _registered_pass_through_routes.values() + if route_info["endpoint_id"] == endpoint_id + ) + return frozenset(chain.from_iterable(map(InitPassThroughEndpointHelpers._llm_api_routes_of_route, route_infos))) + + @staticmethod + def auth_enforced_llm_api_routes() -> frozenset[str]: + route_infos: Final = ( + route_info for route_info in _registered_pass_through_routes.values() if route_info.get("auth") is True + ) + return frozenset(chain.from_iterable(map(InitPassThroughEndpointHelpers._llm_api_routes_of_route, route_infos))) + + @staticmethod + def drop_llm_api_routes(routes: frozenset[str]) -> None: + openai_routes: Final = LiteLLMRoutes.openai_routes.value + for route in routes.intersection(openai_routes): + openai_routes.remove(route) + @staticmethod def clear_all_pass_through_routes(): """Clear all pass-through routes from the registry""" @@ -3436,6 +3498,8 @@ class InitPassThroughEndpointHelpers: return True comparison_route: Final = InitPassThroughEndpointHelpers._route_for_registry_lookup(route) + if RouteChecks.is_builtin_eu_jev_decision_route(route=comparison_route): + return True # Fast path: check if any registered route key contains this path # Keys are in format: "{endpoint_id}:exact:{path}:{methods}" or "{endpoint_id}:subpath:{path}:{methods}" @@ -3512,6 +3576,25 @@ class InitPassThroughEndpointHelpers: return None + @staticmethod + def has_configured_route_ignoring_trailing_slash(route: str) -> bool: + bare_route: Final = route.rstrip("/") + return any( + InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=form) is not None + for form in (bare_route, f"{bare_route}/") + ) + + +class YieldsToConfiguredPassThroughRoute(APIRoute): + def matches(self, scope: Scope) -> tuple[Match, Scope]: + match, child_scope = super().matches(scope) + if match is Match.NONE: + return match, child_scope + configured: Final = InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash( + route=get_request_route(Request(scope)) + ) + return (Match.NONE, {}) if configured else (match, child_scope) + def _get_combined_pass_through_endpoints( pass_through_endpoints: list[dict] | list[PassThroughGenericEndpoint], @@ -3971,9 +4054,7 @@ async def update_pass_through_endpoints( # Update the list pass_through_endpoint_data[endpoint_index] = endpoint_dict - - # Remove old routes from registry before they get re-registered - InitPassThroughEndpointHelpers.remove_endpoint_routes(endpoint_id) + replaced_llm_api_routes: Final = InitPassThroughEndpointHelpers.llm_api_routes_of_endpoint(endpoint_id) ## Update db updated_data: Final = ConfigFieldUpdate( @@ -4023,6 +4104,9 @@ async def update_pass_through_endpoints( auth=updated_endpoint.auth, timeout=updated_endpoint.timeout, ) + InitPassThroughEndpointHelpers.drop_llm_api_routes( + replaced_llm_api_routes - InitPassThroughEndpointHelpers.auth_enforced_llm_api_routes() + ) return PassThroughEndpointResponse(endpoints=[updated_endpoint] if updated_endpoint else []) diff --git a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py index 7092055bf6f..3b49ca1450a 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py @@ -1,6 +1,7 @@ import json import os -from collections.abc import Callable +from collections.abc import Callable, Mapping +from dataclasses import dataclass from typing import TYPE_CHECKING, Final from fastapi import HTTPException @@ -11,6 +12,7 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i LiteLLM_ManagedVectorStore, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.secret_managers.main import ( GOOGLE_OIDC_SECRET_PREFIX, UnresolvableSecretReference, @@ -25,12 +27,23 @@ if TYPE_CHECKING: from litellm.router import Router +@dataclass(frozen=True, slots=True) +class _PassThroughCredentials: + api_key: str + api_base: str | None + + def _get_proxy_llm_router() -> "Router | None": from litellm.proxy.proxy_server import llm_router return llm_router +def _is_team_deployment(deployment: Mapping[str, object]) -> bool: + model_info: Final = deployment.get("model_info") + return is_str_keyed_mapping(model_info) and bool(model_info.get("team_id")) + + def _get_str_value(values: dict[str, object] | None, key: str) -> str | None: value: Final = values.get(key) if values is not None else None return value if isinstance(value, str) else None @@ -84,12 +97,13 @@ class PassthroughEndpointRouter: custom_llm_provider: str, region_name: str | None, ) -> str | None: - deployment_api_key: Final = self._get_deployment_api_key( + deployment_credentials: Final = self._get_deployment_credentials( custom_llm_provider=custom_llm_provider, region_name=region_name, + include_team_deployments=True, ) - if deployment_api_key is not None: - return _resolve_oidc_reference(deployment_api_key) + if deployment_credentials is not None: + return _resolve_oidc_reference(deployment_credentials.api_key) verbose_router_logger.debug( "No pass-through deployment credentials found for %s, looking for env variable", custom_llm_provider ) @@ -98,29 +112,56 @@ class PassthroughEndpointRouter: ) return _resolve_oidc_reference(get_secret_str(_env_variable_name)) + def get_proxy_wide_credentials(self, custom_llm_provider: str) -> _PassThroughCredentials | None: + deployment_credentials: Final = self._get_deployment_credentials( + custom_llm_provider=custom_llm_provider, + region_name=None, + include_team_deployments=False, + ) + api_key: Final = _resolve_oidc_reference( + deployment_credentials.api_key + if deployment_credentials is not None + else get_secret_str( + self._get_default_env_variable_name_passthrough_endpoint(custom_llm_provider=custom_llm_provider) + ) + ) + if not api_key: + return None + return _PassThroughCredentials( + api_key=api_key, + api_base=deployment_credentials.api_base if deployment_credentials is not None else None, + ) + def has_credentials(self, custom_llm_provider: str) -> bool: - if self._get_deployment_api_key(custom_llm_provider=custom_llm_provider, region_name=None) is not None: + if ( + self._get_deployment_credentials( + custom_llm_provider=custom_llm_provider, region_name=None, include_team_deployments=True + ) + is not None + ): return True env_variable_name: Final = self._get_default_env_variable_name_passthrough_endpoint( custom_llm_provider=custom_llm_provider, ) return bool(os.environ.get(env_variable_name)) - def _get_deployment_api_key( + def _get_deployment_credentials( self, custom_llm_provider: str, region_name: str | None, - ) -> str | None: + include_team_deployments: bool, + ) -> _PassThroughCredentials | None: llm_router: Final = self.llm_router_getter() if llm_router is None: return None deployments: Final = llm_router.registry().deployments() or () return next( ( - api_key + credentials for deployment in deployments - if ( - api_key := self._resolve_matching_deployment_api_key( + if (include_team_deployments or not _is_team_deployment(deployment)) + and ( + credentials := self._resolve_matching_deployment_credentials( litellm_params=deployment["litellm_params"], custom_llm_provider=custom_llm_provider, region_name=region_name, @@ -131,12 +172,12 @@ class PassthroughEndpointRouter: None, ) - def _resolve_matching_deployment_api_key( + def _resolve_matching_deployment_credentials( self, litellm_params: LiteLLMParamsTypedDict, custom_llm_provider: str, region_name: str | None, - ) -> str | None: + ) -> _PassThroughCredentials | None: if litellm_params.get("use_in_pass_through") is not True: return None if self._get_deployment_provider(litellm_params) != custom_llm_provider: @@ -152,7 +193,8 @@ class PassthroughEndpointRouter: ) if deployment_region != region_name: return None - return _get_str_value(credential_values, "api_key") or litellm_params.get("api_key") + api_key: Final = _get_str_value(credential_values, "api_key") or litellm_params.get("api_key") + return _PassThroughCredentials(api_key=api_key, api_base=api_base) if api_key is not None else None def _get_deployment_provider(self, litellm_params: LiteLLMParamsTypedDict) -> str | None: model: Final = litellm_params.get("model") diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index bafdd9e197d..1f4e7b6df95 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -375,7 +375,7 @@ class PassThroughEndpointLogging: kwargs = transcribe_handler_result["kwargs"] # rebind-ok: elif-chain contract elif ( self.is_typesafe_route(custom_llm_provider) - or custom_llm_provider in ("laya", "bespoke") + or custom_llm_provider in ("eu_jev", "laya", "bespoke") or self.is_openrouter_decisions_route(url_route, custom_llm_provider) ): from .llm_provider_handlers.typesafe_passthrough_logging_handler import ( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d91af4fa144..a5e929c1c2b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -201,7 +201,6 @@ from litellm.types.utils import ( CountContextManagement, ModelResponse, ModelResponseStream, - StreamingChoices, TextCompletionResponse, TokenCountMessagesFormat, TokenCountResponse, @@ -384,6 +383,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( SensitiveDataMasker, mask_sensitive_keys, ) +from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact from litellm.llms.anthropic.messages.convertible_blocks import ( deployment_drops_params, drop_non_string_tool_result_texts, @@ -6464,6 +6464,8 @@ class ProxyConfig: _log_db_config_param(section, param_value) for message, args in pending_db_warnings: verbose_proxy_logger.warning(message, *args) + if prisma_client is not None and store_model_in_db is True: + self._warn_about_shadowed_db_settings() self.update_config_state(config=config) @@ -8544,7 +8546,8 @@ class ProxyConfig: self._prepared_db_settings_values(section, param_value, warning_sink=warning_sink), ) - self._warn_about_shadowed_db_settings(warning_sink=warning_sink) + if warning_sink is None: + self._warn_about_shadowed_db_settings() return self._config_with_resolved_settings(config) async def _apply_db_environment_variables_row( @@ -8593,15 +8596,14 @@ class ProxyConfig: revision: Final = _config_row_revision(row) return revision is None or revision < applied_revision - def _warn_about_shadowed_db_settings(self, warning_sink: Callable[[_DbConfigWarning], None] | None = None) -> None: + def _warn_about_shadowed_db_settings(self) -> None: shadowed: Final[frozenset[tuple[Section, str]]] = frozenset( (section, key) for section, store in self._settings_stores.items() for key in store.shadowed_db_keys() ) for section, key in sorted(shadowed - self._warned_shadowed_keys): - _warn_db_config( + verbose_proxy_logger.warning( "%s", config_ownership_message(section=section, key=key, shadows_db_value=True), - warning_sink=warning_sink, ) self._warned_shadowed_keys = shadowed @@ -10271,29 +10273,6 @@ def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes: return chunk.model_dump_json(exclude_none=True, exclude_unset=True) -def _is_injected_stream_usage_artifact(chunk: object) -> bool: - if not isinstance(chunk, ModelResponseStream): - return False - if chunk.provider_specific_fields is not None: - return False - return all(_is_empty_streaming_choice(choice) for choice in chunk.choices or []) - - -def _is_empty_streaming_choice(choice: StreamingChoices) -> bool: - if choice.finish_reason is not None: - return False - if getattr(choice, "logprobs", None) is not None: - return False - delta: Final = choice.delta if hasattr(choice, "delta") else None - if delta is None: - return True - return _all_values_none(delta.model_dump().values()) - - -def _all_values_none(values: Iterable[object]) -> bool: - return all(value is None for value in values) - - async def _apply_streaming_chunk_hooks( *, chunk: Any, @@ -10673,7 +10652,7 @@ async def async_data_generator( fallback_model_from_metadata=fallback_model_from_metadata, ) - if strip_stream_usage and _is_injected_stream_usage_artifact(chunk): + if strip_stream_usage and is_injected_stream_usage_artifact(chunk): if pending_fallback_event: yield _format_fallback_metadata_sse_event( fallback_model=fallback_model_from_metadata, diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 04e6c5d8ba9..d8ff32c3819 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -1553,6 +1553,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "EU_JEV", + "provider_display_name": "eu/jev", + "litellm_provider": "eu_jev", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://eu-jev.com", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "eu_jev/eu-jeff-latest" + }, { "provider": "FalAI", "provider_display_name": "Fal AI", diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index fe52f8416d0..629d9098970 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -98,6 +98,8 @@ class Feature(str, Enum): """Replayed thinking blocks, which Python strips and retries once if Anthropic rejects their signature.""" TOOL_CUSTOM_FIELD = "tool_custom_field" """Tools carrying a non-spec custom object, which Python drops or hoists for the provider.""" + STRICT_TOOLS_RETRY = "strict_tools_retry" + """Strict tools under drop_params, which Python sends again without strict if Claude rejects their limits.""" CLIENT_HEADERS = "client_headers" """Proxy-forwarded headers that Python merges into extra_headers.""" DEPLOYMENT_HOOK = "deployment_hook" @@ -179,6 +181,7 @@ MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { Feature.STREAM_COST_INJECTION, Feature.THINKING_SIGNATURE_RETRY, Feature.TOOL_CUSTOM_FIELD, + Feature.STRICT_TOOLS_RETRY, } RESPONSES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { diff --git a/litellm/rust_bridge/messages/detectors.py b/litellm/rust_bridge/messages/detectors.py index 52917126b3b..8edd2707405 100644 --- a/litellm/rust_bridge/messages/detectors.py +++ b/litellm/rust_bridge/messages/detectors.py @@ -5,6 +5,7 @@ from __future__ import annotations from typing import Final import litellm +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.llms.opencode.common_utils import is_opencode_api_base from litellm.rust_bridge.detectors import api_base, overridden, present, streaming from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str @@ -48,6 +49,13 @@ def tool_custom_field(request: NativeCall) -> bool: return any("custom" in (optional_mapping(tool) or {}) for tool in tools) +def strict_tools_retry(request: NativeCall) -> bool: + tools: Final = optional_sequence(request.resolved.get("tools")) or () + return drops_unsupported_params(request.resolved.get("drop_params")) and any( + (optional_mapping(tool) or {}).get("strict") is True for tool in tools + ) + + def stream_cost_injection(request: NativeCall) -> bool: """Python adds response cost to streaming usage when the global setting is truthy.""" return streaming(request) and bool(litellm.include_cost_in_streaming_usage) diff --git a/litellm/text_completions/streaming_iterator.py b/litellm/text_completions/streaming_iterator.py index ec1b96fd432..61e47e9d7af 100644 --- a/litellm/text_completions/streaming_iterator.py +++ b/litellm/text_completions/streaming_iterator.py @@ -1,6 +1,7 @@ from typing import Final -from litellm.types.utils import Choices, ModelResponse, TextChoices, TextCompletionResponse +from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact +from litellm.types.utils import Choices, ModelResponse, ModelResponseStream, TextChoices, TextCompletionResponse class TextCompletionStreamWrapper: @@ -10,11 +11,18 @@ class TextCompletionStreamWrapper: model, stream_options: dict | None = None, custom_llm_provider: str | None = None, + usage_injected_by_proxy: bool = False, ): self.completion_stream = completion_stream self.model = model self.stream_options = stream_options self.custom_llm_provider = custom_llm_provider + self.usage_injected_by_proxy: Final = usage_injected_by_proxy + self.include_usage: Final = ( + not usage_injected_by_proxy + and stream_options is not None + and stream_options.get("include_usage", False) is True + ) def __iter__(self): return self @@ -22,7 +30,7 @@ class TextCompletionStreamWrapper: def __aiter__(self): return self - def convert_to_text_completion_object(self, chunk: ModelResponse): + def convert_to_text_completion_object(self, chunk: ModelResponse | ModelResponseStream): try: response: Final = TextCompletionResponse() response["id"] = chunk.get("id", None) @@ -43,8 +51,7 @@ class TextCompletionStreamWrapper: else: response["choices"] = [] - # only pass usage when stream_options["include_usage"] is True - if self.stream_options and self.stream_options.get("include_usage", False) is True: + if self.include_usage: response.usage = chunk.get("usage", None) else: response.usage = None @@ -53,6 +60,9 @@ class TextCompletionStreamWrapper: except Exception as e: raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}") + def _is_injected_usage_artifact(self, chunk: ModelResponseStream) -> bool: + return self.usage_injected_by_proxy and is_injected_stream_usage_artifact(chunk) + def __next__(self): # model_response = ModelResponse(stream=True, model=self.model) TextCompletionResponse() @@ -60,8 +70,9 @@ class TextCompletionStreamWrapper: for chunk in self.completion_stream: if chunk == "None" or chunk is None: raise Exception - processed_chunk = self.convert_to_text_completion_object(chunk=chunk) - return processed_chunk + if isinstance(chunk, ModelResponseStream) and self._is_injected_usage_artifact(chunk): + continue + return self.convert_to_text_completion_object(chunk=chunk) raise StopIteration except StopIteration: raise StopIteration @@ -81,8 +92,9 @@ class TextCompletionStreamWrapper: async for chunk in self.completion_stream: if chunk == "None" or chunk is None: raise Exception - processed_chunk = self.convert_to_text_completion_object(chunk=chunk) - return processed_chunk + if isinstance(chunk, ModelResponseStream) and self._is_injected_usage_artifact(chunk): + continue + return self.convert_to_text_completion_object(chunk=chunk) raise StopIteration except StopIteration: raise StopAsyncIteration diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index d29e4db55a4..0d39b0a0616 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -204,9 +204,11 @@ class MCPServer(LiteLLMBaseModel): access_groups: list[str] | None = None allow_all_keys: bool = False available_on_public_internet: bool = True - # Legacy opt-in to upstream-delegated authentication for ``oauth2`` - # servers. LiteLLM admission still applies; use ``oauth_delegate`` for the - # supported client-forwarded OAuth flow. + # Deprecated opt-in to upstream-delegated authentication for ``oauth2`` + # servers. A bare ``Authorization`` that is not a LiteLLM credential is the + # upstream token and is forwarded; see + # ``MCPRequestHandler._admit_legacy_delegate_bearer``. New servers should use + # ``true_passthrough`` or ``oauth_delegate``. # # Honored only for ``auth_type == oauth2``; ignored for any other # ``auth_type``. OAuth pass-through for non-oauth2 servers diff --git a/litellm/types/passthrough_endpoints/eu_jev.py b/litellm/types/passthrough_endpoints/eu_jev.py new file mode 100644 index 00000000000..3b81d68deea --- /dev/null +++ b/litellm/types/passthrough_endpoints/eu_jev.py @@ -0,0 +1,100 @@ +from collections.abc import Mapping, Sequence +from typing import Annotated, Literal, TypeAlias + +from pydantic import ConfigDict, Field + +from litellm.types.decisions import ChoiceAnswer, DecisionsJSON, NoulAnswer, NoulCriteria, ScoreAnswer +from litellm.types.llms.base import LiteLLMPydanticObjectBase + +EuJevEntry: TypeAlias = Annotated[ + DecisionsJSON | None, + Field(description="Free text, or a JSON object, array or null that eu/jev serializes to text before inference"), +] + + +class EuJevChoiceQuestion(LiteLLMPydanticObjectBase): + type: Literal["choice"] + instructions: EuJevEntry + criteria: Annotated[ + Mapping[str, EuJevEntry], + Field( + min_length=2, + max_length=64, + description="Two to 64 options, option name to description. A null description uses the option name", + ), + ] + + model_config = ConfigDict(frozen=True) + + +class EuJevNoulQuestion(LiteLLMPydanticObjectBase): + type: Literal["noul"] + instructions: EuJevEntry + criteria: Annotated[NoulCriteria | None, Field(description="Optional descriptions of what true and false mean")] = ( + None + ) + + model_config = ConfigDict(frozen=True) + + +class EuJevScoreQuestion(LiteLLMPydanticObjectBase): + type: Literal["score"] + instructions: EuJevEntry + criteria: Annotated[ + Sequence[EuJevEntry], + Field(min_length=2, max_length=10, description="Two to 10 level descriptions, ordered from low to high"), + ] + + model_config = ConfigDict(frozen=True) + + +EuJevQuestion: TypeAlias = Annotated[ + EuJevChoiceQuestion | EuJevNoulQuestion | EuJevScoreQuestion, + Field(discriminator="type"), +] + + +class EuJevDecisionRequest(LiteLLMPydanticObjectBase): + model: Annotated[str, Field(description="eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it")] = ( + "eu-jeff-latest" + ) + state: Annotated[ + Annotated[str, Field(min_length=1, max_length=100_000)] | Mapping[str, object] | Sequence[object], + Field(description="The content to evaluate, a string of up to 100,000 characters or a JSON object or array"), + ] + questions: Annotated[ + Mapping[Annotated[str, Field(min_length=1, max_length=80)], EuJevQuestion], + Field(min_length=1, max_length=8, description="One to eight questions, keyed by names of up to 80 characters"), + ] + + model_config = ConfigDict(frozen=True) + + +EuJevAnswer: TypeAlias = Annotated[ChoiceAnswer | NoulAnswer | ScoreAnswer, Field(discriminator="type")] + + +class EuJevUsage(LiteLLMPydanticObjectBase): + input_tokens: Annotated[int, Field(ge=0)] + output_tokens: Annotated[int, Field(ge=0)] + + model_config = ConfigDict(frozen=True) + + +class EuJevMeta(LiteLLMPydanticObjectBase): + request_id: str + mode: Literal["live"] + latency_ms: Annotated[ + int, Field(description="Server-side time for token quoting, credit reservation and inference") + ] + cost_eur: Annotated[str | None, Field(description="The exact charge in euros, on authenticated requests")] = None + + model_config = ConfigDict(frozen=True) + + +class EuJevDecisionResponse(LiteLLMPydanticObjectBase): + model: Annotated[str, Field(description="The release that answered, such as eu-jeff-1.0.0")] + answers: Annotated[Mapping[str, EuJevAnswer], Field(description="One answer per question, under the same names")] + usage: EuJevUsage + meta: EuJevMeta + + model_config = ConfigDict(frozen=True) diff --git a/litellm/types/proxy/auth/user_api_key_auth.py b/litellm/types/proxy/auth/user_api_key_auth.py index 2c1c75fbe04..1183c36eda7 100644 --- a/litellm/types/proxy/auth/user_api_key_auth.py +++ b/litellm/types/proxy/auth/user_api_key_auth.py @@ -169,6 +169,10 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob # above; a forged value could at most narrow, but the stripping keeps the field's provenance # single-owner so its meaning stays trustworthy. mcp_session_resource_server_ids: tuple[str, ...] | None = Field(default=None, exclude=True) + # Set only by MCP admission when a legacy delegate_auth_to_upstream caller's Authorization is the + # upstream token rather than a LiteLLM credential, so egress forwards it. Stripped from validated + # input like the markers above: a forged value would forward a LiteLLM credential upstream. + mcp_forwards_caller_authorization: bool = Field(default=False, exclude=True) mcp_toolset_id: str | None = Field(default=None, exclude=True) authenticated_by_custom_auth: bool = Field(default=False, exclude=True) via_virtual_key: bool = Field( @@ -230,6 +234,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob values.pop("mcp_explicit_grants_only", None) values.pop("mcp_source_team_rpm_limits", None) values.pop("mcp_session_resource_server_ids", None) + values.pop("mcp_forwards_caller_authorization", None) values.pop("mcp_toolset_id", None) values.pop("via_virtual_key", None) values.pop("authenticated_by_custom_auth", None) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 1e59ec0c2cb..1055ffe7b2f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4489,6 +4489,7 @@ class LlmProviders(str, Enum): STRANDS_DECIDER = "strands_decider" LAYA = "laya" BESPOKE = "bespoke" + EU_JEV = "eu_jev" MISTRAL = "mistral" MILVUS = "milvus" GROQ = "groq" diff --git a/litellm/utils.py b/litellm/utils.py index 5228562857e..4207b5c8c1c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9977,11 +9977,12 @@ class ProviderConfigManager: return litellm.LayaDecisionsConfig() if provider == LlmProviders.BESPOKE: return litellm.BespokeDecisionsConfig() - if provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI): + if provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI, LlmProviders.EU_JEV): configs: Final = { LlmProviders.OPENAI: litellm.OpenAIDecisionsConfig, LlmProviders.AZURE: litellm.AzureDecisionsConfig, LlmProviders.AZURE_AI: litellm.AzureAIDecisionsConfig, + LlmProviders.EU_JEV: litellm.EuJevDecisionsConfig, } return configs[provider]() return None @@ -10932,19 +10933,8 @@ class ProviderConfigManager: return None @staticmethod - def get_provider_image_edit_config( - model: str, - provider: LlmProviders, - ) -> BaseImageEditConfig | None: - if LlmProviders.OPENAI == provider: - from litellm.llms.openai.image_edit import OpenAIImageEditConfig - - return OpenAIImageEditConfig() - elif LlmProviders.HOSTED_VLLM == provider: - from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config - - return get_hosted_vllm_image_edit_config(model=model) - elif LlmProviders.AZURE == provider: + def _get_model_independent_image_edit_config(provider: LlmProviders) -> BaseImageEditConfig | None: + if LlmProviders.AZURE == provider: from litellm.llms.azure.image_edit.transformation import ( AzureImageEditConfig, ) @@ -10962,6 +10952,36 @@ class ProviderConfigManager: ) return BlackForestLabsImageEditConfig() + elif LlmProviders.LITELLM_PROXY == provider: + from litellm.llms.litellm_proxy.image_edit.transformation import ( + LiteLLMProxyImageEditConfig, + ) + + return LiteLLMProxyImageEditConfig() + elif LlmProviders.DASHSCOPE == provider: + from litellm.llms.dashscope.image_edit.transformation import ( + DashScopeImageEditConfig, + ) + + return DashScopeImageEditConfig() + return None + + @staticmethod + def get_provider_image_edit_config( + model: str, + provider: LlmProviders, + ) -> BaseImageEditConfig | None: + model_independent_config: Final = ProviderConfigManager._get_model_independent_image_edit_config(provider) + if model_independent_config is not None: + return model_independent_config + if LlmProviders.OPENAI == provider: + from litellm.llms.openai.image_edit import OpenAIImageEditConfig + + return OpenAIImageEditConfig() + elif LlmProviders.HOSTED_VLLM == provider: + from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config + + return get_hosted_vllm_image_edit_config(model=model) elif LlmProviders.FAL_AI == provider: from litellm.llms.fal_ai.image_edit import get_fal_ai_image_edit_config @@ -10974,12 +10994,6 @@ class ProviderConfigManager: from litellm.llms.gemini.image_edit import get_gemini_image_edit_config return get_gemini_image_edit_config(model) - elif LlmProviders.LITELLM_PROXY == provider: - from litellm.llms.litellm_proxy.image_edit.transformation import ( - LiteLLMProxyImageEditConfig, - ) - - return LiteLLMProxyImageEditConfig() elif LlmProviders.VERTEX_AI == provider: from litellm.llms.vertex_ai.image_edit import ( get_vertex_ai_image_edit_config, @@ -11201,6 +11215,12 @@ class ProviderConfigManager: return AWSPollyTextToSpeechConfig() elif litellm.LlmProviders.EDENAI == provider: return litellm.EdenAITextToSpeechConfig() + elif litellm.LlmProviders.DASHSCOPE == provider: + from litellm.llms.dashscope.text_to_speech.transformation import ( + DashScopeTextToSpeechConfig, + ) + + return DashScopeTextToSpeechConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c6bbb63b6fc..b5d09d6ff4f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18346,25 +18346,31 @@ "dashscope/qwen-image-2.0": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.035, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-2.0-pro": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.075, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-3.0": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.03, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-3.0-pro": { @@ -18372,7 +18378,54 @@ "mode": "image_generation", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "dashscope/qwen3-tts-flash": { + "input_cost_per_character": 1e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.1-tts-flash": { + "input_cost_per_token": 2.3e-07, + "output_cost_per_token": 1.87e-06, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.1-tts-plus": { + "input_cost_per_character": 3e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.0-tts-plus": { + "input_cost_per_character": 2e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.0-tts-flash": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" ] }, "dashscope/wan3.0-video": { @@ -72330,6 +72383,28 @@ "source": "https://docs.typesafe.ai/models", "supports_box_questions": true }, + "eu_jev/eu-jeff-1.0.0": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://eu-jev.com/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, + "eu_jev/eu-jeff-latest": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://eu-jev.com/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 2e-07, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 1d487f33835..18edb2cd7dc 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1148,9 +1148,9 @@ "text_completion": true, "embeddings": false, "image_generations": true, - "image_edits": false, + "image_edits": true, "audio_transcriptions": false, - "audio_speech": false, + "audio_speech": true, "rerank": false, "ocr": false, "search": false, @@ -2241,6 +2241,32 @@ "generateContent": false } }, + "eu_jev": { + "display_name": "eu/jev (`eu_jev`)", + "url": "https://docs.litellm.ai/docs/pass_through/eu_jev", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": true, + "a2a": false, + "generateContent": false + } + }, "laya": { "display_name": "Laya (`laya`)", "url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers", diff --git a/pyproject.toml b/pyproject.toml index 44dfa498904..6e4732b4f7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,8 +84,8 @@ proxy = [ "mcp>=2.3.0,<3", "httpx2>=2.10.0,<3", "pydantic>=2.12.0,<3", - "litellm-proxy-extras==0.4.111", - "litellm-enterprise==0.1.80", + "litellm-proxy-extras==0.4.112", + "litellm-enterprise==0.1.81", "RestrictedPython>=8.5,<9.0", "rich>=13.9.4,<14.0", "InquirerPy>=0.3.4,<1.0", diff --git a/scripts/openapi_docs_gate.py b/scripts/openapi_docs_gate.py index 5f5d1021c22..de340bf84e7 100755 --- a/scripts/openapi_docs_gate.py +++ b/scripts/openapi_docs_gate.py @@ -101,6 +101,23 @@ def _install_termination_handlers() -> None: signal.signal(termination, _exit_on_termination) +def _install_base_native(worktree: Path, repo_root: Path) -> None: + env: Final = { + **os.environ, + "CARGO_TARGET_DIR": str( + Path(os.environ.get("CARGO_TARGET_DIR", repo_root / "litellm-rust" / "target")).resolve() + ), + } + _run( + [ + "uv", "pip", "install", "--python", sys.executable, "--no-deps", + "--target", str(worktree.parent / "native-install"), "--editable", str(worktree), + ], + cwd=worktree, + env=env, + ) + + def base_counts(ref: str, repo_root: Path = REPO_ROOT, checker: Path = CHECKER) -> Mapping[str, int]: """Rule counts at `ref`, measured with the *current* checker over the base tree.""" _install_termination_handlers() @@ -111,6 +128,7 @@ def base_counts(ref: str, repo_root: Path = REPO_ROOT, checker: Path = CHECKER) (worktree / "scripts").mkdir(parents=True, exist_ok=True) base_checker: Final = worktree / "scripts" / "check_openapi_docs.py" shutil.copy(checker, base_checker) + _install_base_native(worktree, repo_root) return count_by_rule(_check(worktree, base_checker)) finally: # Teardown must never raise, or it masks the real error when the body failed. diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index a5bb5e0b340..c943ff2cbd0 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -81,6 +81,7 @@ IGNORE_FUNCTIONS = [ "completion_cost", # max depth 1: recursion only fires for mixed-tier Responses WS logging objects, and each split part carries a single service_tier so _split_responses_ws_logging_object_by_service_tier returns None. "_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible). "_replace_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible). + "_redacted_client_field", # bounded by the nesting depth of a JSON-decoded error body (a finite JSON tree, no cycles possible). "_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible). "scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap. "_without_requester_ip_address", # walks only the fresh tree _sanitize_request_body_for_spend_logs_payload returns (mapping cycles already cut), so it never goes deeper than that sanitizer did. diff --git a/tests/integration/configuration/test_openapi_docs_gate.py b/tests/integration/configuration/test_openapi_docs_gate.py new file mode 100644 index 00000000000..eefa663cb10 --- /dev/null +++ b/tests/integration/configuration/test_openapi_docs_gate.py @@ -0,0 +1,9 @@ +from typing import Final + +import openapi_docs_gate as gate + + +def test_base_counts_imports_native_extension_from_detached_worktree() -> None: + expected_counts: Final = dict(gate.count_by_rule(gate.head_violations())) + assert expected_counts + assert dict(gate.base_counts("HEAD")) == expected_counts diff --git a/tests/integration/providers/test_dashscope_image_edit_and_speech_wire.py b/tests/integration/providers/test_dashscope_image_edit_and_speech_wire.py new file mode 100644 index 00000000000..ea16633d81d --- /dev/null +++ b/tests/integration/providers/test_dashscope_image_edit_and_speech_wire.py @@ -0,0 +1,259 @@ +import base64 +import json +import os +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Final +from urllib.parse import urlsplit, urlunsplit + +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, gateway_from_environment, list_value, object_value +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from integration.providers._dashscope_video_support import deployed, json_reply, rate, settled_spend +from pydantic import JsonValue + +_API_KEY: Final = "synthetic-dashscope-key" +_MULTIMODAL_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation" +_SYNTHESIZER_PATH: Final = "/api/v1/services/audio/tts/SpeechSynthesizer" +_OSS_PATH: Final = "/oss/" +_PNG: Final = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) +_PNG_DATA_URI: Final = f"data:image/png;base64,{base64.b64encode(_PNG).decode()}" +_EDITED_URL: Final = "https://dashscope-result.example.com/edited.png" +_PROMPT: Final = "Turn the red circle into a blue square" +_TEXT: Final = "Hello from LiteLLM." +_BILLED_CHARACTERS: Final = len(_TEXT) +_WAV: Final = b"RIFF\x24\x00\x00\x00WAVEfmt " + uuid.uuid4().bytes +_MP3: Final = b"ID3\x04\x00\x00\x00\x00\x00\x00" + uuid.uuid4().bytes +_SIGNED_QUERY: Final = "Expires=1793316671&OSSAccessKeyId=LTAI5t&Signature=ab%2Bc%3D" +_QWEN_TTS: Final = "dashscope-qwen-tts" +_QWEN_AUDIO_TTS: Final = "dashscope-qwen-audio-tts" +_INPUT_TOKENS: Final = 8 +_OUTPUT_TOKENS: Final = 30 +_OWNED_TIMEOUT: Final = int(2 * graceful_stop_seconds() + 120) + + +def _edit_api(request: Request) -> Reply: + assert (request.method, request.target) == ("POST", _MULTIMODAL_PATH), (request.method, request.target) + return json_reply( + { + "output": {"choices": [{"message": {"role": "assistant", "content": [{"image": _EDITED_URL}]}}]}, + "usage": {"width": 1024, "height": 1024, "image_count": 1}, + } + ) + + +def _synthesized(request: Request, file_name: str, usage: dict[str, JsonValue]) -> Reply: + audio_url: Final = f"http://{request.headers['host']}{_OSS_PATH}{file_name}?{_SIGNED_QUERY}" + return json_reply({"output": {"audio": {"url": audio_url, "data": "", "id": "audio-1"}}, "usage": usage}) + + +def _speech_api(request: Request) -> Reply: + if request.method == "GET" and request.target.startswith(f"{_OSS_PATH}speech.wav"): + return Reply(body=_WAV, content_type="audio/wav") + if request.method == "GET" and request.target.startswith(f"{_OSS_PATH}speech.mp3"): + return Reply(body=_MP3, content_type="audio/mpeg") + if (request.method, request.target) == ("POST", _MULTIMODAL_PATH): + return _synthesized(request, "speech.wav", {"characters": _BILLED_CHARACTERS}) + assert (request.method, request.target) == ("POST", _SYNTHESIZER_PATH), (request.method, request.target) + return _synthesized( + request, + "speech.mp3", + {"characters": _BILLED_CHARACTERS, "input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS}, + ) + + +def _pooled_database_url(url: str) -> str: + parts: Final = urlsplit(url) + query: Final = "&".join(part for part in (parts.query, "connection_limit=5") if part) + return urlunsplit(parts._replace(query=query)) + + +def _tts_deployment(name: str, model: str, api_base: str) -> dict[str, JsonValue]: + return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, "api_key": _API_KEY}} + + +def _error_message(body: bytes) -> str: + return str(object_value(JSON_OBJECT.validate_json(body)["error"])["message"]) + + +@pytest.fixture(scope="module") +def rig() -> Iterator[Gateway]: + with gateway_from_environment() as gateway: + yield gateway + + +@pytest.fixture(scope="module") +def speech_wire() -> Iterator[Wire]: + with wire_server(_speech_api) as served: + yield served + + +@pytest.fixture(scope="module") +def speech_proxy(rig: Gateway, speech_wire: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("dashscope-speech") + config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + path: Final = directory / "dashscope-speech.yaml" + path.write_text( + yaml.safe_dump( + { + **config, + "model_list": [ + _tts_deployment(_QWEN_TTS, "dashscope/qwen3-tts-flash", speech_wire.url), + _tts_deployment(_QWEN_AUDIO_TTS, "dashscope/qwen-audio-3.1-tts-flash", speech_wire.url), + ], + "general_settings": { + **object_value(config["general_settings"]), + "user_url_allowed_hosts": ["127.0.0.1"], + }, + } + ) + ) + with owned_proxy( + rig, directory, {"DATABASE_URL": _pooled_database_url(os.environ["DATABASE_URL"])}, config=path + ) as owned: + yield owned + + +def test_a_multipart_edit_reaches_dashscope_as_inline_images_and_bills_the_edited_image(gateway: Gateway) -> None: + with wire_server(_edit_api) as wire, gateway.scenario() as scenario: + model: Final = deployed( + gateway, + scenario, + model="dashscope/qwen-image-2.0", + api_base=f"{wire.url}{_MULTIMODAL_PATH}", + api_key=_API_KEY, + ) + key: Final = scenario.key(models=[model]) + response: Final = gateway.client.post( + "/v1/images/edits", + data={"model": model, "prompt": _PROMPT, "size": "1024x1024", "negative_prompt": "blurry"}, + files=[("image[]", ("a.png", _PNG, "image/png")), ("image[]", ("b.png", _PNG, "image/png"))], + headers={"Authorization": f"Bearer {key}"}, + ) + + assert response.status_code == 200, response.text + data: Final = list_value(JSON_OBJECT.validate_json(response.content)["data"]) + assert [object_value(item)["url"] for item in data] == [_EDITED_URL] + (request,) = wire.drain() + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + assert json.loads(request.body) == { + "model": "qwen-image-2.0", + "input": { + "messages": [ + { + "role": "user", + "content": [{"image": _PNG_DATA_URI}, {"image": _PNG_DATA_URI}, {"text": _PROMPT}], + } + ] + }, + "parameters": {"size": "1024*1024", "negative_prompt": "blurry"}, + } + settled_spend(key, rate("dashscope", "output_cost_per_image", model="qwen-image-2.0")) + + +def test_a_fourth_image_is_refused_with_a_400_before_dashscope_is_called(gateway: Gateway) -> None: + with wire_server(_edit_api) as wire, gateway.scenario() as scenario: + model: Final = deployed( + gateway, + scenario, + model="dashscope/qwen-image-2.0", + api_base=f"{wire.url}{_MULTIMODAL_PATH}", + api_key=_API_KEY, + ) + response: Final = gateway.client.post( + "/v1/images/edits", + data={"model": model, "prompt": _PROMPT}, + files=[("image[]", (f"{index}.png", _PNG, "image/png")) for index in range(4)], + headers={"Authorization": f"Bearer {gateway.key}"}, + ) + + assert response.status_code == 400, response.text + assert "at most 3 input images, got 4" in _error_message(response.content) + assert wire.drain() == () + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_qwen3_tts_returns_the_downloaded_wav_and_bills_each_input_character( + speech_proxy: Gateway, speech_wire: Wire +) -> None: + speech_wire.drain() + with speech_proxy.scenario() as scenario: + key: Final = scenario.key(models=[_QWEN_TTS]) + response: Final = speech_proxy.request( + "POST", "/v1/audio/speech", {"model": _QWEN_TTS, "input": _TEXT, "voice": "Cherry"}, key=key + ) + + assert response.status_code == 200, response.text + assert response.content == _WAV + assert response.headers["content-type"] == "audio/wav" + synthesis, download = speech_wire.drain() + assert (synthesis.method, synthesis.target) == ("POST", _MULTIMODAL_PATH) + assert synthesis.headers["authorization"] == f"Bearer {_API_KEY}" + assert json.loads(synthesis.body) == {"model": "qwen3-tts-flash", "input": {"text": _TEXT, "voice": "Cherry"}} + assert (download.method, download.target) == ("GET", f"{_OSS_PATH}speech.wav?{_SIGNED_QUERY}") + assert "authorization" not in download.headers + settled_spend(key, _BILLED_CHARACTERS * rate("dashscope", "input_cost_per_character", "qwen3-tts-flash")) + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_qwen_audio_tts_sends_format_and_instruction_returns_the_mp3_and_bills_reported_tokens( + speech_proxy: Gateway, speech_wire: Wire +) -> None: + speech_wire.drain() + model: Final = "qwen-audio-3.1-tts-flash" + with speech_proxy.scenario() as scenario: + key: Final = scenario.key(models=[_QWEN_AUDIO_TTS]) + response: Final = speech_proxy.request( + "POST", + "/v1/audio/speech", + { + "model": _QWEN_AUDIO_TTS, + "input": _TEXT, + "voice": "longanhuan_v3.1", + "response_format": "mp3", + "instructions": "Speak slowly.", + }, + key=key, + ) + + assert response.status_code == 200, response.text + assert response.content == _MP3 + assert response.headers["content-type"] == "audio/mpeg" + synthesis, download = speech_wire.drain() + assert (synthesis.method, synthesis.target) == ("POST", _SYNTHESIZER_PATH) + assert json.loads(synthesis.body) == { + "model": model, + "input": {"text": _TEXT, "voice": "longanhuan_v3.1", "format": "mp3", "instruction": "Speak slowly."}, + } + assert (download.method, download.target) == ("GET", f"{_OSS_PATH}speech.mp3?{_SIGNED_QUERY}") + settled_spend( + key, + _INPUT_TOKENS * rate("dashscope", "input_cost_per_token", model) + + _OUTPUT_TOKENS * rate("dashscope", "output_cost_per_token", model), + ) + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_a_format_qwen3_tts_cannot_produce_answers_400_without_reaching_dashscope( + speech_proxy: Gateway, speech_wire: Wire +) -> None: + speech_wire.drain() + refused: Final = speech_proxy.request( + "POST", "/v1/audio/speech", {"model": _QWEN_TTS, "input": _TEXT, "voice": "Cherry", "response_format": "mp3"} + ) + served: Final = speech_proxy.request( + "POST", "/v1/audio/speech", {"model": _QWEN_TTS, "input": _TEXT, "voice": "Cherry", "response_format": "wav"} + ) + + assert refused.status_code == 400, refused.text + assert "response_format='mp3'" in _error_message(refused.content) + assert served.status_code == 200, served.text + assert [(request.method, request.target.split("?")[0]) for request in speech_wire.drain()] == [ + ("POST", _MULTIMODAL_PATH), + ("GET", f"{_OSS_PATH}speech.wav"), + ] diff --git a/tests/integration/spend/test_text_completion_stream_usage_cost.py b/tests/integration/spend/test_text_completion_stream_usage_cost.py new file mode 100644 index 00000000000..4d66e60f3e7 --- /dev/null +++ b/tests/integration/spend/test_text_completion_stream_usage_cost.py @@ -0,0 +1,185 @@ +import json +from pathlib import Path +from typing import Final +from uuid import uuid4 + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +PROMPT_TOKENS: Final = 19 +COMPLETION_TOKENS: Final = 5 +INPUT_RATE: Final = 0.001 +OUTPUT_RATE: Final = 0.002 +EXPECTED_SPEND: Final = PROMPT_TOKENS * INPUT_RATE + COMPLETION_TOKENS * OUTPUT_RATE +USAGE: Final[dict[str, JsonValue]] = { + "prompt_tokens": PROMPT_TOKENS, + "completion_tokens": COMPLETION_TOKENS, + "total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS, +} +PROMPT: Final = "Say hi" + + +def _sse_frame(payload: dict[str, JsonValue]) -> bytes: + return f"data: {json.dumps(payload, separators=(',', ':'))}\n\n".encode() + + +def _chunk(request_id: str, model: str, delta: dict[str, JsonValue], finish_reason: str | None) -> dict[str, JsonValue]: + return { + "id": request_id, + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], + } + + +def _respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return Reply(body=json.dumps({"object": "list", "data": [{"id": "gpt-5.4-mini", "object": "model"}]}).encode()) + assert request.method == "POST", request.method + assert request.target == "/v1/chat/completions", request.target + body: Final = json.loads(request.body) + assert body["messages"] == [{"role": "user", "content": PROMPT}], body + request_id: Final = f"chatcmpl-{uuid4().hex}" + model: Final = str(body["model"]) + if body.get("stream") is not True: + return Reply( + body=json.dumps( + { + "id": request_id, + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Hi there"}, "finish_reason": "length"} + ], + "usage": USAGE, + } + ).encode() + ) + stream_options: Final = body.get("stream_options") or {} + usage_frames: Final = ( + ( + _sse_frame( + { + "id": request_id, + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [], + "usage": USAGE, + } + ), + ) + if stream_options.get("include_usage") is True + else () + ) + return Reply( + content_type="text/event-stream", + chunks=( + _sse_frame(_chunk(request_id, model, {"role": "assistant", "content": "Hi"}, None)), + _sse_frame(_chunk(request_id, model, {"content": " there"}, None)), + _sse_frame(_chunk(request_id, model, {}, "length")), + *usage_frames, + b"data: [DONE]\n\n", + ), + ) + + +def _proxy_config(directory: Path, model: str, upstream_url: str) -> Path: + config: Final = directory / "text_completion_stream_usage_cost_config.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": model, + "litellm_params": { + "model": "openai/gpt-5.4-mini", + "api_base": f"{upstream_url}/v1", + "api_key": "scripted-upstream", + "input_cost_per_token": INPUT_RATE, + "output_cost_per_token": OUTPUT_RATE, + }, + } + ], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "store_model_in_db": True, + "disable_spend_logs": False, + "proxy_batch_write_at": 1, + "proxy_batch_polling_interval": 1, + }, + "router_settings": {"disable_cooldowns": True}, + } + ) + ) + return config + + +def _data_events(body: str) -> tuple[dict[str, JsonValue], ...]: + return tuple( + json.loads(line.removeprefix("data:")) + for line in body.splitlines() + if line.startswith("data:") and not line.endswith("[DONE]") + ) + + +def _carries_text_or_finish(frame: dict[str, JsonValue]) -> bool: + choices: Final = frame["choices"] + assert isinstance(choices, list) and len(choices) == 1, frame + choice: Final = choices[0] + assert isinstance(choice, dict), frame + return choice.get("text") is not None or choice.get("finish_reason") is not None + + +def _billed(row: dict[str, object]) -> tuple[object, object, float]: + return (row["prompt_tokens"], row["completion_tokens"], float(str(row["spend"]))) + + +def test_text_completion_stream_without_stream_options_bills_like_its_twins(gateway: Gateway, tmp_path: Path) -> None: + model: Final = f"gpt-5.4-mini-{uuid4().hex}" + body: Final[dict[str, JsonValue]] = {"model": model, "prompt": PROMPT, "max_tokens": COMPLETION_TOKENS} + + with wire_server(_respond) as wire: + config: Final = _proxy_config(tmp_path, model, wire.url) + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate: + silent: Final = candidate.request("POST", "/v1/completions", {**body, "stream": True}) + asked: Final = candidate.request( + "POST", "/v1/completions", {**body, "stream": True, "stream_options": {"include_usage": True}} + ) + whole: Final = candidate.request("POST", "/v1/completions", body) + assert (silent.status_code, asked.status_code, whole.status_code) == (200, 200, 200), ( + silent.text, + asked.text, + whole.text, + ) + silent_frames: Final = _data_events(silent.text) + asked_frames: Final = _data_events(asked.text) + request_ids: Final = (silent_frames[0]["id"], asked_frames[0]["id"], whole.json()["id"]) + rows: Final = eventually( + lambda: read_rows( + 'SELECT request_id, prompt_tokens, completion_tokens, spend FROM "LiteLLM_SpendLogs" ' + "WHERE request_id = ANY(%s)", + (list(request_ids),), + database_url=candidate.database_url, + ), + lambda values: len(values) == 3, + seconds=70, + ) + upstream_stream_options: Final = [ + json.loads(sent.body).get("stream_options") for sent in wire.drain() if sent.method == "POST" + ] + + assert upstream_stream_options == [{"include_usage": True}, {"include_usage": True}, None] + assert all(_carries_text_or_finish(frame) for frame in silent_frames), silent.text + assert all("usage" not in frame for frame in silent_frames), silent.text + assert asked_frames[-1]["usage"]["total_tokens"] == PROMPT_TOKENS + COMPLETION_TOKENS, asked.text + billed: Final = {str(row["request_id"]): _billed(row) for row in rows} + assert billed[str(request_ids[0])] == billed[str(request_ids[1])] == billed[str(request_ids[2])], billed + assert billed[str(request_ids[0])] == (PROMPT_TOKENS, COMPLETION_TOKENS, pytest.approx(EXPECTED_SPEND)), billed diff --git a/tests/test_litellm_rust/messages/test_dispatch.py b/tests/test_litellm_rust/messages/test_dispatch.py index 15f58afb4e5..5b11e628b8d 100644 --- a/tests/test_litellm_rust/messages/test_dispatch.py +++ b/tests/test_litellm_rust/messages/test_dispatch.py @@ -442,6 +442,7 @@ class _Agentic(CustomLogger): ADVISOR_TOOL: Final = {"type": ANTHROPIC_ADVISOR_TOOL_TYPE, "name": "advisor", "model": "claude-opus-5-5"} +STRICT_TOOL: Final = {"name": "a", "input_schema": {"type": "object"}, "strict": True} @pytest.mark.parametrize( @@ -527,6 +528,45 @@ def test_each_detected_feature_reaches_its_rule( assert served(kwargs) is python_result +@pytest.mark.asyncio +@pytest.mark.parametrize( + "kwargs", + ( + pytest.param({"tools": [STRICT_TOOL]}, id="drop-params-unset"), + pytest.param({"drop_params": False, "tools": [STRICT_TOOL]}, id="drop-params-off"), + pytest.param( + {"drop_params": True, "tools": [{"name": "a", "input_schema": {"type": "object"}}]}, id="no-strict" + ), + ), +) +async def test_strict_tools_stay_native_unless_python_could_drop_strict( + monkeypatch: pytest.MonkeyPatch, kwargs: Mapping[str, object] +) -> None: + rules: Final[Rules] = ( + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({Feature.STRICT_TOOLS_RETRY})), + RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), + ) + native_result: Final = response() + monkeypatch.setattr(litellm, "drop_params", False) + + async def python(*call_args: object, **python_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub + pytest.fail("Python cannot drop strict here") + + async def native(request: NativeCall) -> AnthropicMessagesResponse: + return native_result + + served: Final = await _ADISPATCH.arun( + (16, MESSAGES, "anthropic/claude-sonnet-4-5"), + kwargs, + python=python, + binding=amessages_binding(native), + native=lambda hook, request, call_args, native_kwargs: hook(request), + rules=rules, + ) + + assert served is native_result + + ASYNC_ONLY_FEATURES: Final = ( pytest.param(Feature.PRE_REQUEST_HOOK, {}, {"callbacks": [_Rewrite()]}, id="pre-request-hook"), pytest.param( @@ -538,6 +578,12 @@ ASYNC_ONLY_FEATURES: Final = ( {}, id="messages-interceptor", ), + pytest.param( + Feature.STRICT_TOOLS_RETRY, {"drop_params": True, "tools": [STRICT_TOOL]}, {}, id="strict-tools-retry" + ), + pytest.param( + Feature.STRICT_TOOLS_RETRY, {"tools": [STRICT_TOOL]}, {"drop_params": True}, id="strict-tools-retry-global" + ), ) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index e5501310556..0c357837d31 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -123,6 +123,7 @@ _PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = ( ), ("typesafe", "typesafe/jev-1.13", "https://api.typesafe.ai/v1/systemone", "jev-1.13"), ("nace", "nace/drex-v2.0", "https://console.nace.ai/v1/systemone", "drex-v2.0"), + ("eu_jev", "eu_jev/eu-jeff-latest", "https://eu-jev.com/v1/systemone", "eu-jeff-latest"), ( "openrouter", "openrouter/typesafe/jev-1.13", diff --git a/tests/unit/images/test_image_edit.py b/tests/unit/images/test_image_edit.py index 4f858050e3e..06372a29d92 100644 --- a/tests/unit/images/test_image_edit.py +++ b/tests/unit/images/test_image_edit.py @@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, TypedDict, override import litellm from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.types.utils import FileTypes, ImageResponse _PNG_SIGNATURE: Final = b"\x89PNG\r\n\x1a\n" @@ -34,6 +35,26 @@ _EDIT_RESPONSE: Final = { }, } +_DASHSCOPE_IMAGE_URL: Final = ( + "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" +) +_DASHSCOPE_EDIT_RESPONSE: Final = { + "output": { + "choices": [ + { + "finish_reason": "stop", + "message": {"role": "assistant", "content": [{"image": "https://example.com/edited.png"}]}, + } + ] + }, + "usage": {"image_count": 1}, +} + +_FAL_EDIT_URL: Final = "https://fal.run/fal-ai/flux-2/edit" +_FAL_EDIT_RESPONSE: Final = { + "images": [{"url": "https://example.com/edited.png", "width": 1024, "height": 768, "content_type": "image/png"}] +} + class _LoggedImageEdit(TypedDict): model: ReadOnly[str] @@ -229,3 +250,101 @@ async def test_router_image_edit_returns_the_provider_image( assert isinstance(result, ImageResponse) assert result.data is not None and result.data[0].b64_json == _EDITED_IMAGE_B64 assert route.call_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_price", "size", "expected_cost"), + ( + ({"input_cost_per_image": 0.0123}, None, 0.0123), + ({"input_cost_per_pixel": 1e-08}, "2048x2048", 2048 * 2048 * 1e-08), + ), +) +async def test_an_edit_bills_the_deployment_price_at_its_size_on_a_logging_object_made_before_routing( + respx_mock: respx.MockRouter, + httpx_transport: None, + monkeypatch: pytest.MonkeyPatch, + deployment_price: dict[str, float], + size: str | None, + expected_cost: float, +) -> None: + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + monkeypatch.delenv("DASHSCOPE_API_BASE_IMAGE", raising=False) + respx_mock.post(_DASHSCOPE_IMAGE_URL).mock(return_value=httpx.Response(200, json=_DASHSCOPE_EDIT_RESPONSE)) + logging_made_before_routing: Final = Logging( + model="qwen-image-edit", + messages=[], + stream=False, + call_type="aimage_edit", + start_time=datetime.now(), + litellm_call_id="edit-call", + function_id="edit-call", + ) + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "qwen-image-edit", + "litellm_params": { + "model": "dashscope/qwen-image-2.0", + "api_key": "fake-key", + **deployment_price, + }, + } + ] + ) + + result: Final = await router.aimage_edit( + model="qwen-image-edit", + image=_FIRST_IMAGE, + prompt="Turn the circle into a square", + size=size, + litellm_logging_obj=logging_made_before_routing, + litellm_call_id="edit-call", + ) + + assert isinstance(result, ImageResponse) + assert result._hidden_params["response_cost"] == pytest.approx(expected_cost) + + +@pytest.mark.asyncio +async def test_a_fal_edit_bills_the_deployment_price_per_image_on_a_logging_object_made_before_routing( + respx_mock: respx.MockRouter, + httpx_transport: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("FAL_AI_API_BASE", raising=False) + respx_mock.post(_FAL_EDIT_URL).mock(return_value=httpx.Response(200, json=_FAL_EDIT_RESPONSE)) + logging_made_before_routing: Final = Logging( + model="flux-edit", + messages=[], + stream=False, + call_type="aimage_edit", + start_time=datetime.now(), + litellm_call_id="fal-edit-call", + function_id="fal-edit-call", + ) + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "flux-edit", + "litellm_params": { + "model": "fal_ai/fal-ai/flux-2/edit", + "api_key": "fake-key", + "output_cost_per_image": 0.0456, + }, + } + ] + ) + + result: Final = await router.aimage_edit( + model="flux-edit", + image=_FIRST_IMAGE, + prompt="Turn the circle into a square", + litellm_logging_obj=logging_made_before_routing, + litellm_call_id="fal-edit-call", + ) + + assert isinstance(result, ImageResponse) + assert result._hidden_params["response_cost"] == pytest.approx(0.0456) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index d97970753f8..f03c7421899 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -523,6 +523,43 @@ def test_token_counter_skips_non_mapping_tools(): assert mixed_tokens == valid_tokens +@pytest.mark.parametrize( + ("shorthand", "explicit"), + [({"type": "array"}, {"type": "array", "items": {}}), (True, {})], + ids=["array_without_items", "boolean_subschema"], +) +def test_token_counter_counts_json_schema_shorthands_like_their_explicit_form(shorthand: object, explicit: object) -> None: + assert _count_with_property_schema(shorthand) == _count_with_property_schema(explicit) + + +@pytest.mark.parametrize( + ("malformed", "without_keyword"), + [ + ( + {"type": "object", "properties": {"id": {"type": "string"}}, "required": True}, + {"type": "object", "properties": {"id": {"type": "string"}}}, + ), + ({"type": "object", "properties": [{"id": {"type": "string"}}]}, {"type": "object"}), + ({"type": "string", "enum": None}, {"type": "string"}), + ({"type": "integer", "enum": 3}, {"type": "integer"}), + ], + ids=["required_true", "properties_list", "string_enum_null", "integer_enum_number"], +) +def test_token_counter_counts_a_malformed_schema_keyword_as_absent(malformed: object, without_keyword: object) -> None: + assert _count_with_property_schema(malformed) == _count_with_property_schema(without_keyword) + + +def _count_with_property_schema(property_schema: object) -> int: + tool: Final = { + "type": "function", + "function": { + "name": "read_files", + "parameters": {"type": "object", "properties": {"paths": property_schema}}, + }, + } + return token_counter_new(model="gpt-4o", messages=[{"role": "user", "content": "Read the config files"}], tools=[tool]) + + _WEATHER_FUNCTION: Final = { "name": "get_weather", "description": "Get the current weather for a city", @@ -2132,6 +2169,20 @@ def test_token_counter_counts_anthropic_search_result_text() -> None: ) +def test_token_counter_counts_a_search_result_without_a_title() -> None: + from litellm.litellm_core_utils.token_counter import _count_content_list + + search_result: Final = { + "type": "search_result", + "source": "https://example.invalid/notes", + "content": [{"type": "text", "text": "The code word is 7f3a."}], + } + + assert _count_content_list(len, [search_result], False, None) == len( + "https://example.invalid/notesThe code word is 7f3a." + ) + + @pytest.mark.parametrize("nested", (False, True)) def test_unknown_content_type_remains_a_value_error_with_an_optional_fallback(nested: bool) -> None: from litellm.litellm_core_utils.token_counter import _count_content_list diff --git a/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py index f03ffb25b2b..4cbf4cfa5f6 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py @@ -217,6 +217,106 @@ def test_cleared_input_tokens_is_nonnegative(): assert applied["cleared_input_tokens"] >= 0 +SERVER_TOOL_TURN: Final = { + "role": "assistant", + "content": [ + {"type": "server_tool_use", "id": "srvtoolu_01", "name": "web_search", "input": {"query": "Mumbai rain"}}, + { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_01", + "content": [{"type": "web_search_result", "url": "https://example.com", "title": "rain " * 300}], + }, + ], +} + + +def _clear_above( + messages: list[dict[str, object]], threshold: int +) -> tuple[list[dict[str, object]], AppliedEdit | None]: + return apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "input_tokens", "value": threshold}, + "keep": {"type": "tool_uses", "value": 1}, + }, + ) + + +def test_an_input_tokens_trigger_skips_blocks_no_adapter_sends() -> None: + history: Final = _make_history(n_pairs=3) + sent_tokens: Final = litellm.token_counter(model=MODEL, messages=history) + turn: Final = SERVER_TOOL_TURN + messages: Final = [history[0], turn, *history[1:]] + + untouched, not_applied = _clear_above(messages, sent_tokens + 100) + cleared, applied = _clear_above(messages, sent_tokens - 100) + + assert not_applied is None + assert untouched == messages + assert applied is not None + assert applied["cleared_tool_uses"] == 2 + assert _cleared_ids(cleared) == {"toolu_00", "toolu_01"} + assert cleared[1] == turn + + +def _first_result_with(blocks: list[dict[str, object]]) -> dict[str, object]: + return {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_00", "content": blocks}]} + + +def test_an_input_tokens_trigger_skips_blocks_no_adapter_sends_inside_a_tool_result() -> None: + history: Final = _make_history(n_pairs=3) + result_text: Final = {"type": "text", "text": "Result 0: " + "x" * 200} + search_result: Final = {"type": "web_search_result", "url": "https://example.com", "title": "rain " * 300} + sent_tokens: Final = litellm.token_counter( + model=MODEL, messages=[*history[:2], _first_result_with([result_text]), *history[3:]] + ) + messages: Final = [*history[:2], _first_result_with([result_text, search_result]), *history[3:]] + + untouched, not_applied = _clear_above(messages, sent_tokens + 100) + cleared, applied = _clear_above(messages, sent_tokens - 100) + + assert not_applied is None + assert untouched == messages + assert applied is not None + assert _cleared_ids(cleared) == {"toolu_00", "toolu_01"} + + +def _clear_at_least_one_token( + messages: list[dict[str, object]], +) -> tuple[list[dict[str, object]], AppliedEdit | None]: + return apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_at_least": {"type": "input_tokens", "value": 1}, + }, + ) + + +def test_clear_at_least_frees_only_blocks_an_adapter_sends_inside_a_tool_result() -> None: + history: Final = _make_history(n_pairs=3) + result_text: Final = {"type": "text", "text": "Result 0: " + "x" * 200} + search_result: Final = {"type": "web_search_result", "url": "https://example.com", "title": "rain " * 300} + + _, sent_only = _clear_at_least_one_token([*history[:2], _first_result_with([result_text]), *history[3:]]) + cleared, applied = _clear_at_least_one_token( + [*history[:2], _first_result_with([result_text, search_result]), *history[3:]] + ) + + assert applied is not None + assert applied == sent_only + assert _cleared_ids(cleared) == {"toolu_00", "toolu_01"} + + def _blocks_of(messages: list[dict[str, object]], block_type: str) -> tuple[dict[str, object], ...]: contents: Final = (message["content"] for message in messages) blocks: Final = chain.from_iterable(content for content in contents if isinstance(content, list)) @@ -559,6 +659,9 @@ async def _malformed_spec_error(edits: list[dict[str, object]]) -> AnthropicCont ("exclude_tools", "get_weather"), ("exclude_tools", [1]), ("clear_tool_inputs", "get_weather"), + ("trigger", "soon"), + ("trigger", [1]), + ("keep", 5), ], ) async def test_malformed_fields_are_a_400_even_below_the_trigger(field: str, value: object) -> None: @@ -576,12 +679,15 @@ async def test_malformed_fields_are_a_400_even_below_the_trigger(field: str, val ("clear_at_least", 1200, "clear_at_least: Input does not match the expected shape."), ("clear_at_least", {"type": "input_tokens", "value": "1200"}, "value: Input should be a valid integer"), ("clear_tool_inputs", 5, "clear_tool_inputs.bool: Input should be a valid boolean, unable to interpret input"), + ("trigger", "soon", "trigger: Input should be an object"), + ("keep", 5, "keep: Input should be an object"), ], ) async def test_malformed_field_400_names_the_edit_and_field_as_the_api_does( field: str, value: object, message: str ) -> None: - # api.anthropic.com, measured 2026-10-19, answers these four with these messages for the edit at index 1. + # api.anthropic.com answers these with these messages for the edit at index 1 (trigger and keep measured + # 2026-10-28, the rest 2026-10-19). error = await _malformed_spec_error( [{"type": "clear_thinking_20251015"}, {"type": "clear_tool_uses_20250919", field: value}] ) @@ -625,6 +731,28 @@ async def test_drop_params_drops_only_the_malformed_field(field: str, value: obj assert _tool_inputs(result.messages) == {"toolu_00": {}, "toolu_01": {}, "toolu_02": {"location": "City2"}} +@pytest.mark.parametrize(("field", "value"), [("trigger", "soon"), ("keep", 5)]) +async def test_drop_params_treats_a_malformed_trigger_or_keep_as_never_sent(field: str, value: object) -> None: + async def run(edit: dict[str, object], drop_invalid_edits: bool) -> PolyfillResult | None: + return await _run_polyfill_if_enabled( + model=MODEL, + messages=_make_history(n_pairs=5), + tools=None, + system=None, + context_management_spec={"edits": [edit]}, + litellm_metadata={}, + additional_drop_params=None, + llm_router=None, + drop_invalid_edits=drop_invalid_edits, + ) + + malformed: Final = {**_VALID_EDIT, field: value} + with pytest.raises(AnthropicContextManagementError, match=rf"\.{field}: Input should be an object$"): + await run(malformed, drop_invalid_edits=False) + never_sent: Final = {key: spec for key, spec in _VALID_EDIT.items() if key != field} + assert await run(malformed, drop_invalid_edits=True) == await run(never_sent, drop_invalid_edits=True) + + async def test_malformed_edit_is_rejected_before_a_compact_summary_is_paid_for( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/llms/anthropic/messages/context_management/test_compact.py b/tests/unit/llms/anthropic/messages/context_management/test_compact.py index dff931a6074..32899a8e3c8 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_compact.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_compact.py @@ -29,7 +29,7 @@ from litellm.llms.anthropic.messages.context_management.editors.compact import ( _augment_system_with_summary, _extract_summary_text, _select_last_user_question, - _slice_around_compaction_block, + slice_around_compaction_block, _strip_compaction_blocks, apply_client_compaction_block_history, apply_compact_20260112, @@ -153,7 +153,7 @@ def test_applied_edits_for_response_includes_compact_when_block_present(): def test_slice_around_compaction_block_found(): messages = _messages_with_compaction("my summary") - sliced, block = _slice_around_compaction_block(messages) + sliced, block = slice_around_compaction_block(messages) assert block is not None assert block["type"] == "compaction" assert block["content"] == "my summary" @@ -164,7 +164,7 @@ def test_slice_around_compaction_block_found(): def test_slice_around_compaction_block_not_found(): messages = _simple_messages() - sliced, block = _slice_around_compaction_block(messages) + sliced, block = slice_around_compaction_block(messages) assert block is None assert sliced is messages # same object, no copy diff --git a/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py b/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py index fabb05c2e61..e55af602013 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py @@ -5,6 +5,7 @@ Unit tests for the context_management polyfill dispatcher. from litellm.llms.anthropic.messages.context_management import ( apply_context_management, ) +from litellm.llms.anthropic.messages.convertible_blocks import drop_non_string_tool_result_texts MODEL = "xai/grok-4" @@ -160,3 +161,57 @@ async def test_sync_editor_counts_tokens_off_the_event_loop(): assert result.messages == messages assert_loop_stayed_free(took, lags) + + +def _history_with_null_text_beside_a_short_first_result(): + return [ + {"role": "user", "content": "Hi"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "t0", "name": "f", "input": {}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "t0", + "content": [{"type": "text", "text": "ok"}, {"type": "text", "text": None}], + } + ], + }, + *_history_with_two_tool_pairs()[1:], + ] + + +async def test_drop_invalid_edits_clears_as_if_malformed_tool_result_text_was_never_sent(): + spec = { + "edits": [ + { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 2}, + "keep": {"type": "tool_uses", "value": 1}, + } + ] + } + history = _history_with_null_text_beside_a_short_first_result() + + from_raw = await apply_context_management( + model=MODEL, + messages=history, + tools=None, + system=None, + context_management_spec=spec, + drop_invalid_edits=True, + ) + from_sent = await apply_context_management( + model=MODEL, + messages=list(drop_non_string_tool_result_texts(history, True)), + tools=None, + system=None, + context_management_spec=spec, + drop_invalid_edits=True, + ) + + assert from_raw.applied_edits == from_sent.applied_edits + assert from_raw.messages == from_sent.messages diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index c11f58faec4..5ce6d03f477 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -3,6 +3,7 @@ import copy import json import os import uuid +from dataclasses import dataclass from typing import Any, Dict, Final, List import httpx @@ -10,7 +11,7 @@ import responses import respx import pytest from fastapi.testclient import TestClient -from pydantic import ValidationError +from pydantic import TypeAdapter, ValidationError from unittest.mock import AsyncMock, MagicMock, patch @@ -2396,3 +2397,61 @@ def test_native_messages_tool_config_point_marks_last_tool_and_drops_the_key(mon assert "cache_control" not in captured["params"]["tools"][0] assert captured["messages"][0]["content"][-1]["cache_control"] == {"type": "ephemeral"} assert tools == tools_before + + +# Anthropic's 400 for a request past Claude's strict tool limits, seen live on claude-sonnet-4-6 on 2026-10-29 +_TOO_MANY_STRICT_TOOLS: Final = { + "type": "error", + "error": { + "type": "invalid_request_error", + "message": "Too many strict tools (21). The maximum number of strict tools supported is 20. Try reducing the " + "number of tools marked as strict.", + }, + "request_id": "req_011CgWXzv8cmAr9vFs786WV4", +} +_JSON_BODY: Final = TypeAdapter(dict[str, object]) + + +@dataclass(slots=True) +class _RejectsStrictToolsOnce: + sent: tuple[dict[str, object], ...] = () + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.sent = (*self.sent, _JSON_BODY.validate_json(request.content)) + if len(self.sent) == 1: + return httpx.Response(400, json=_TOO_MANY_STRICT_TOOLS, request=request) + return httpx.Response( + 200, + json={ + "id": "msg_strict_retry", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "done"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + request=request, + ) + + +@pytest.mark.asyncio +async def test_anthropic_messages_strict_tools_rejection_is_sent_again_without_strict_under_drop_params() -> None: + anthropic: Final = _RejectsStrictToolsOnce() + tool: Final = {"name": "lookup_record", "input_schema": {"type": "object"}, "strict": True} + + response: Final = await litellm.anthropic_messages( + max_tokens=16, + messages=[{"role": "user", "content": "Use lookup_record to find invoice 42"}], + model="anthropic/claude-sonnet-4-6", + api_key="sk-test", + tools=[tool], + drop_params=True, + client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)), + ) + + first, second = anthropic.sent + assert first["tools"] == [tool] + assert second == {**first, "tools": [{"name": "lookup_record", "input_schema": {"type": "object"}}]} + assert _JSON_BODY.validate_python(response)["id"] == "msg_strict_retry" diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index bba5413fbdf..382917fc756 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -2,7 +2,8 @@ import datetime import json import os import sys -from collections.abc import Mapping +from collections.abc import AsyncIterator, Mapping +from itertools import chain from typing import Final, cast from unittest.mock import AsyncMock, patch @@ -761,3 +762,325 @@ async def test_recomputed_cost_of_a_served_bridge_response_prices_the_reasoning_ assert logging_obj.cost_breakdown is not None assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(expected_output) assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning) + + +TEXT_RESPONSE: Final = { + "id": "resp_cleared", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.6-luna", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "done", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7}, +} +CLEAR_EDIT: Final = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 1}, + "keep": {"type": "tool_uses", "value": 1}, +} +COMPACT_EDIT: Final = {"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}} + + +def _read(i: int) -> tuple[dict[str, object], dict[str, object]]: + return ( + { + "role": "assistant", + "content": [{"type": "tool_use", "id": f"toolu_{i}", "name": "Bash", "input": {"command": f"cat {i}"}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": f"toolu_{i}", "content": f"log {i} " * 40}], + }, + ) + + +def _three_reads() -> list[dict[str, object]]: + return [{"role": "user", "content": "Read the logs."}, *chain.from_iterable(_read(i) for i in range(3))] + + +async def _cleared_bridge( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + *, + stream: bool, + edits: list[dict[str, object]], + drop_params: bool = False, + system: object = None, +) -> tuple[object, Mapping[str, object]]: + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses") + if stream: + upstream.respond(status_code=200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY) + else: + upstream.respond(status_code=200, json=TEXT_RESPONSE) + response: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + stream=stream, + custom_llm_provider="openai", + context_management={"edits": edits}, + drop_params=drop_params, + system=system, + ) + body: Final = json.loads(upstream.calls.last.request.content) + if not stream: + return response, body + return [json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in response], body + + +@pytest.mark.asyncio +async def test_clear_tool_uses_clears_old_results_before_the_responses_api_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER + + served, body = await _cleared_bridge(respx_mock, monkeypatch, stream=False, edits=[dict(CLEAR_EDIT)]) + + outputs: Final = { + item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output" + } + assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [ + True, + True, + False, + ] + assert "context_management" not in body + assert isinstance(served, dict) + assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +@pytest.mark.asyncio +async def test_streamed_clear_tool_uses_reports_its_edits_on_the_final_message_delta( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + events, _ = await _cleared_bridge(respx_mock, monkeypatch, stream=True, edits=[dict(CLEAR_EDIT)]) + + assert isinstance(events, list) + message_delta: Final = next(event for event in events if event["type"] == "message_delta") + assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +@pytest.mark.asyncio +async def test_compact_stays_native_while_clear_tool_uses_runs_in_the_gateway( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + served, body = await _cleared_bridge( + respx_mock, monkeypatch, stream=False, edits=[dict(CLEAR_EDIT), dict(COMPACT_EDIT)] + ) + + assert body["context_management"] == [{"type": "compaction", "compact_threshold": 150000}] + assert isinstance(served, dict) + assert [edit["type"] for edit in served["context_management"]["applied_edits"]] == ["clear_tool_uses_20250919"] + + +@pytest.mark.asyncio +async def test_a_malformed_clear_tool_uses_edit_is_a_400_before_the_responses_api_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError + + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, json=TEXT_RESPONSE + ) + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + with pytest.raises(AnthropicContextManagementError) as raised: + await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + custom_llm_provider="openai", + context_management={"edits": [{**CLEAR_EDIT, "exclude_tools": "Bash"}]}, + ) + assert raised.value.status_code == 400 + assert not upstream.called + + +@pytest.mark.asyncio +async def test_a_malformed_text_in_a_result_clearing_would_drop_is_still_a_400( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, json=TEXT_RESPONSE + ) + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + reads: Final = _three_reads() + malformed_oldest_result: Final = { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_0", "content": [{"type": "text", "text": None}]}], + } + with pytest.raises(litellm.BadRequestError) as rejected: + await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=[*reads[:2], malformed_oldest_result, *reads[3:]], + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + custom_llm_provider="openai", + context_management={"edits": [dict(CLEAR_EDIT)]}, + ) + assert rejected.value.status_code == 400 + assert "tool_result toolu_0 content.0.text: Input should be a valid string" in rejected.value.message + assert not upstream.called + + +def test_the_sync_path_clears_old_results_too(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER + + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, json=TEXT_RESPONSE + ) + + served: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + custom_llm_provider="openai", + context_management={"edits": [dict(CLEAR_EDIT)]}, + ) + + body: Final = json.loads(upstream.calls.last.request.content) + outputs: Final = { + item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output" + } + assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [ + True, + True, + False, + ] + assert isinstance(served, dict) + assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +@pytest.mark.asyncio +async def test_an_openai_style_context_management_list_is_still_not_sent_to_the_responses_api( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, json=TEXT_RESPONSE + ) + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + + served: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + custom_llm_provider="openai", + context_management=[{"type": "compaction", "compact_threshold": 500}], + ) + + assert "context_management" not in json.loads(upstream.calls.last.request.content) + assert isinstance(served, dict) + assert "context_management" not in served + + +@pytest.mark.asyncio +async def test_under_drop_params_a_malformed_edit_still_clears_without_its_bad_field( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER + + served, body = await _cleared_bridge( + respx_mock, monkeypatch, stream=False, edits=[{**CLEAR_EDIT, "exclude_tools": "Bash"}], drop_params=True + ) + + outputs: Final = { + item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output" + } + assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [ + True, + True, + False, + ] + assert isinstance(served, dict) + assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +def _cleared_outputs(body: Mapping[str, object]) -> list[bool]: + from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER + + outputs: Final = { + item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output" + } + return [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "system", + [7, {"type": "text", "text": "Be brief."}, ["Be brief."], ["Be brief.", {"type": "text", "text": "Be kind."}]], + ids=["number", "object", "strings", "mixed_list"], +) +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +async def test_clear_tool_uses_still_clears_when_the_system_prompt_is_not_text_or_a_block_list( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, system: object, stream: bool +) -> None: + _, body = await _cleared_bridge(respx_mock, monkeypatch, stream=stream, edits=[dict(CLEAR_EDIT)], system=system) + + assert _cleared_outputs(body) == [True, True, False] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("system", "expect_cleared"), + [ + (["Stray.", {"type": "text", "text": "Follow the policy. " * 1500}], [True, True, False]), + (["Follow the policy. " * 1500], [False, False, False]), + ], + ids=["text_block_counts", "stray_string_does_not"], +) +async def test_input_tokens_trigger_counts_the_text_blocks_of_a_system_list_with_stray_items( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, system: object, expect_cleared: list[bool] +) -> None: + trigger_between_messages_and_system: Final = litellm.token_counter(model="gpt-4o", messages=_three_reads()) + 1000 + edit: Final = {**CLEAR_EDIT, "trigger": {"type": "input_tokens", "value": trigger_between_messages_and_system}} + + _, body = await _cleared_bridge(respx_mock, monkeypatch, stream=False, edits=[edit], system=system) + + assert _cleared_outputs(body) == expect_cleared + + +@pytest.mark.asyncio +async def test_the_sync_path_reports_its_edits_on_the_streamed_message_delta( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY + ) + + sse: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + stream=True, + custom_llm_provider="openai", + context_management={"edits": [dict(CLEAR_EDIT)]}, + ) + events: Final = [ + json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in cast(AsyncIterator[bytes], sse) + ] + message_delta: Final = next(event for event in events if event["type"] == "message_delta") + assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index b37b2534b73..3ec72f29d34 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -25,7 +25,7 @@ import pytest import respx import litellm -from litellm.llms.anthropic.common_utils import is_claude_tool_target +from litellm.llms.anthropic.common_utils import ToolsWithoutStrict, is_claude_tool_target, tools_without_strict from litellm.llms.custom_httpx.http_handler import HTTPHandler sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../.."))) @@ -4559,3 +4559,27 @@ def test_shared_legacy_thinking_translation_drops_a_non_numeric_budget_under_dro ) def test_is_claude_tool_target(custom_llm_provider: str, model: str, expected: bool): assert is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model) is expected + + +@pytest.mark.parametrize( + "tools", + [ + None, + "lookup_record", + [], + [{"name": "a"}], + [{"name": "a", "strict": False}], + [{"name": "a", "strict": True}, "b"], + ], + ids=["absent", "not_a_list", "empty", "no_strict_key", "strict_false_only", "malformed_tool"], +) +def test_tools_without_strict_is_none_unless_every_tool_is_an_object_and_one_is_strict(tools: object) -> None: + assert tools_without_strict(tools) is None + + +def test_tools_without_strict_drops_only_the_strict_key_and_counts_the_strict_tools() -> None: + tools: Final = [{"name": "a", "strict": True}, {"name": "b"}, {"name": "c", "strict": False}] + + assert tools_without_strict(tools) == ToolsWithoutStrict( + tools=({"name": "a"}, {"name": "b"}, {"name": "c"}), dropped_count=1 + ) diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py index 74f6f3030cb..0bd480dcdbc 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py @@ -8,7 +8,7 @@ import zlib from collections.abc import AsyncIterator, Mapping, Sequence from dataclasses import dataclass from datetime import datetime -from types import SimpleNamespace +from types import MappingProxyType, SimpleNamespace from typing import Final from unittest.mock import Mock @@ -4051,12 +4051,14 @@ _THINKING_HISTORY: Final = ( @dataclass(slots=True) class ScriptedBedrockInvoke: - replies: tuple[tuple[int, Mapping[str, object]], ...] + replies: tuple[tuple[int, Mapping[str, object] | bytes], ...] bodies: tuple[dict[str, object], ...] = () def __call__(self, request: httpx.Request) -> httpx.Response: self.bodies = (*self.bodies, REASONING_BODY_ADAPTER.validate_json(request.content)) status_code, body = self.replies[len(self.bodies) - 1] + if isinstance(body, bytes): + return httpx.Response(status_code, content=body) return httpx.Response(status_code, json=body) @@ -4065,6 +4067,7 @@ async def _send_bedrock_messages( model_info: Mapping[str, object] | None, replies: Sequence[tuple[int, Mapping[str, object]]], messages: Sequence[Mapping[str, object]] = ({"role": "user", "content": "Summarise the weather as JSON."},), + extra: Mapping[str, object] = MappingProxyType({}), ) -> tuple[ScriptedBedrockInvoke, Exception | None]: bedrock: Final = ScriptedBedrockInvoke(replies=tuple(replies)) try: @@ -4080,6 +4083,7 @@ async def _send_bedrock_messages( aws_region_name="us-east-1", client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock)), **({} if model_info is None else {"model_info": model_info}), + **extra, ) except Exception as error: return bedrock, error @@ -4171,6 +4175,158 @@ async def test_bedrock_messages_400_is_not_retried_unless_it_rejects_opt_in_nati assert len(bedrock.bodies) == 1 +# Bedrock's 400 bodies for requests past Claude's strict tool limits, seen live on us.anthropic.claude-sonnet-4-6 on 2026-10-29 +_STRICT_TOOLS_REJECTIONS: Final = ( + { + "message": "Too many strict tools (21). The maximum number of strict tools supported is 20. Try reducing the " + "number of tools marked as strict." + }, + { + "message": "Schemas contains too many optional parameters (96), which would make grammar compilation " + "inefficient. Reduce the number of optional parameters in your tool schemas (limit: 24)." + }, + { + "message": "Schemas contains too many parameters with union types (24 parameters with type arrays or anyOf). " + "This causes exponential compilation cost. Reduce the number of nullable or union-typed parameters (limit: 16 " + "parameters with unions)." + }, + { + "message": "The compiled grammar is too large, which would cause performance issues. Simplify your tool schemas " + "or reduce the number of strict tools." + }, +) +_LOOKUP_TOOL: Final = { + "name": "lookup_record", + "description": "Look up records", + "input_schema": { + "type": "object", + "additionalProperties": False, + "required": ["query"], + "properties": {"query": {"type": "string"}}, + }, +} +_STRICT_TOOLS: Final = ({**_LOOKUP_TOOL, "strict": True}, {**_LOOKUP_TOOL, "name": "lookup_invoice", "strict": True}) +_STREAMED_MESSAGE: Final = _bedrock_event_frame( + {"type": "message_start", "message": {**_ANTHROPIC_MESSAGE, "content": [], "stop_reason": None}} +) + _bedrock_event_frame({"type": "message_stop"}) + + +async def _send_strict_tools_to_bedrock( + replies: Sequence[tuple[int, Mapping[str, object] | bytes]], + drop_params: bool | None, + tools: Sequence[Mapping[str, object]] = _STRICT_TOOLS, + stream: bool = False, +) -> tuple[ScriptedBedrockInvoke, object]: + bedrock: Final = ScriptedBedrockInvoke(replies=tuple(replies)) + try: + response: Final = await litellm.anthropic_messages( + model="bedrock/invoke/us.anthropic.claude-sonnet-4-6", + messages=[{"role": "user", "content": "Use lookup_record to find invoice 42"}], + max_tokens=200, + tools=[dict(tool) for tool in tools], + stream=stream, + drop_params=drop_params, + aws_access_key_id="test-access-key", + aws_secret_access_key="test-secret-key", + aws_region_name="us-east-1", + client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock)), + ) + except litellm.BadRequestError as error: + return bedrock, error + if isinstance(response, AsyncIterator): + return bedrock, b"".join([chunk async for chunk in response]) + return bedrock, response + + +def _assert_resent_without_strict(bedrock: ScriptedBedrockInvoke) -> None: + first, second = bedrock.bodies + first_tools: Final = TypeAdapter(list[dict[str, object]]).validate_python(first["tools"]) + assert [tool["strict"] for tool in first_tools] == [True, True] + assert second == { + **first, + "tools": [{key: value for key, value in tool.items() if key != "strict"} for tool in first_tools], + } + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("rejection", _STRICT_TOOLS_REJECTIONS, ids=["count", "optional", "union", "grammar"]) +async def test_bedrock_messages_strict_tools_rejection_is_sent_again_without_strict_under_drop_params( + rejection: Mapping[str, object], +) -> None: + bedrock, response = await _send_strict_tools_to_bedrock([(400, rejection), (200, _ANTHROPIC_MESSAGE)], True) + + _assert_resent_without_strict(bedrock) + assert REASONING_BODY_ADAPTER.validate_python(response)["id"] == _ANTHROPIC_MESSAGE["id"] + + +@pytest.mark.usefixtures("local_model_cost_map") +async def test_bedrock_messages_strict_tools_rejection_is_sent_again_under_global_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + + bedrock, response = await _send_strict_tools_to_bedrock( + [(400, _STRICT_TOOLS_REJECTIONS[0]), (200, _ANTHROPIC_MESSAGE)], None + ) + + _assert_resent_without_strict(bedrock) + assert REASONING_BODY_ADAPTER.validate_python(response)["id"] == _ANTHROPIC_MESSAGE["id"] + + +@pytest.mark.usefixtures("local_model_cost_map") +async def test_bedrock_messages_streamed_strict_tools_rejection_is_sent_again_without_strict_under_drop_params() -> ( + None +): + bedrock, streamed = await _send_strict_tools_to_bedrock( + [(400, _STRICT_TOOLS_REJECTIONS[0]), (200, _STREAMED_MESSAGE)], True, stream=True + ) + + _assert_resent_without_strict(bedrock) + assert isinstance(streamed, bytes) and b"event: message_stop" in streamed + + +@pytest.mark.usefixtures("local_model_cost_map") +async def test_bedrock_messages_body_rebuilt_without_native_output_still_drops_the_rejected_strict() -> None: + bedrock, error = await _send_bedrock_messages( + "us.anthropic.claude-opus-4-8", + None, + [(400, _STRICT_TOOLS_REJECTIONS[0]), (400, _NATIVE_OUTPUT_REJECTION), (200, _ANTHROPIC_MESSAGE)], + extra={"tools": [dict(tool) for tool in _STRICT_TOOLS], "drop_params": True}, + ) + + assert error is None + first, _, rebuilt = bedrock.bodies + assert "format" in REASONING_BODY_ADAPTER.validate_python(first["output_config"]) + assert "format" not in REASONING_BODY_ADAPTER.validate_python(rebuilt.get("output_config", {})) + assert rebuilt["tools"] == [ + {key: value for key, value in tool.items() if key != "strict"} for tool in _STRICT_TOOLS + ] + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + ("drop_params", "tools", "rejection"), + [ + (None, _STRICT_TOOLS, _STRICT_TOOLS_REJECTIONS[0]), + (False, _STRICT_TOOLS, _STRICT_TOOLS_REJECTIONS[0]), + (True, _STRICT_TOOLS, {"message": "messages.0.content: Field required"}), + (True, (_LOOKUP_TOOL,), _STRICT_TOOLS_REJECTIONS[0]), + (True, ({**_LOOKUP_TOOL, "strict": False},), _STRICT_TOOLS_REJECTIONS[1]), + ], + ids=["drop_params_unset", "drop_params_off", "other_400", "no_strict_tool", "strict_false_only"], +) +async def test_bedrock_messages_400_reaches_the_caller_unless_drop_params_can_drop_the_rejected_strict( + drop_params: bool | None, tools: Sequence[Mapping[str, object]], rejection: Mapping[str, object] +) -> None: + bedrock, raised = await _send_strict_tools_to_bedrock( + [(400, rejection), (200, _ANTHROPIC_MESSAGE)], drop_params, tools + ) + + assert isinstance(raised, litellm.BadRequestError) + assert str(rejection["message"]) in str(raised) + assert len(bedrock.bodies) == 1 + + @pytest.mark.parametrize("trace", ["enabled", "disabled"]) def test_bedrock_messages_applies_the_deployment_guardrail_config_as_invoke_headers(trace: str) -> None: from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( diff --git a/tests/unit/llms/dashscope/test_dashscope_image_edit_transformation.py b/tests/unit/llms/dashscope/test_dashscope_image_edit_transformation.py new file mode 100644 index 00000000000..c3d186fdd8a --- /dev/null +++ b/tests/unit/llms/dashscope/test_dashscope_image_edit_transformation.py @@ -0,0 +1,358 @@ +import base64 +import json +from collections.abc import Callable +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.dashscope.image_edit.transformation import DashScopeImageEditConfig +from litellm.llms.dashscope.text_to_speech.transformation import DashScopeTextToSpeechConfig +from litellm.types.utils import ModelInfo + +PNG: Final = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) +PNG_DATA_URI: Final = f"data:image/png;base64,{base64.b64encode(PNG).decode()}" +INTL_ENDPOINT: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" +EDITED_URL: Final = "https://dashscope-result.example.com/edited.png" +EDITED: Final = { + "output": {"choices": [{"message": {"role": "assistant", "content": [{"image": EDITED_URL}]}}]}, + "usage": {"width": 1024, "height": 1024, "image_count": 1}, +} + + +def _recording( + seen: list[httpx.Request], reply: Callable[[httpx.Request], httpx.Response] +) -> httpx.MockTransport: # mutable-ok: test records the requests the transport receives + def respond(request: httpx.Request) -> httpx.Response: + seen.append(request) + return reply(request) + + return httpx.MockTransport(respond) + + +def _edited(_: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=EDITED) + + +def _sync_client( + seen: list[httpx.Request], reply: Callable[[httpx.Request], httpx.Response] = _edited +) -> HTTPHandler: # mutable-ok: shared request log + return HTTPHandler(client=httpx.Client(transport=_recording(seen, reply))) + + +@pytest.fixture(autouse=True) +def _no_ambient_dashscope_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("DASHSCOPE_API_BASE_IMAGE", raising=False) + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + + +@pytest.mark.parametrize("asynchronous", (False, True)) +async def test_an_edit_posts_every_image_inline_with_provider_params_at_the_top_of_parameters( + asynchronous: bool, +) -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + arguments: Final = { + "model": "dashscope/qwen-image-2.0", + "image": [PNG, PNG], + "prompt": "Turn the circle into a square", + "size": "1024x1024", + "n": 1, + "negative_prompt": "blurry", + "prompt_extend": False, + "watermark": False, + "seed": 42, + "api_key": "sk-dashscope-test", + } + response: Final = ( + await litellm.aimage_edit(**arguments, client=AsyncHTTPHandler(transport=_recording(seen, _edited))) + if asynchronous + else litellm.image_edit(**arguments, client=_sync_client(seen)) + ) + + assert [image.url for image in response.data or []] == [EDITED_URL] + (request,) = seen + assert str(request.url) == INTL_ENDPOINT + assert request.headers["authorization"] == "Bearer sk-dashscope-test" + assert json.loads(request.content) == { + "model": "qwen-image-2.0", + "input": { + "messages": [ + { + "role": "user", + "content": [ + {"image": PNG_DATA_URI}, + {"image": PNG_DATA_URI}, + {"text": "Turn the circle into a square"}, + ], + } + ] + }, + "parameters": { + "n": 1, + "size": "1024*1024", + "negative_prompt": "blurry", + "prompt_extend": False, + "watermark": False, + "seed": 42, + }, + } + + +@pytest.mark.parametrize(("size", "dashscope_size"), (("1792x1024", "1792*1024"), ("1328x1328", "1328*1328"))) +def test_an_openai_size_reaches_dashscope_in_its_star_format(size: str, dashscope_size: str) -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + + litellm.image_edit( + model="dashscope/qwen-image-2.0", image=PNG, prompt="p", size=size, api_key="k", client=_sync_client(seen) + ) + + assert json.loads(seen[0].content)["parameters"] == {"size": dashscope_size} + + +CHINA_ENDPOINT: Final = "https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" + + +@pytest.mark.parametrize( + ("api_base", "env_base", "image_base", "endpoint"), + ( + ("https://dashscope.aliyuncs.com/compatible-mode/v1", None, None, CHINA_ENDPOINT), + ("https://dashscope-intl.aliyuncs.com/compatible-mode/v1/", None, None, INTL_ENDPOINT), + ("https://dashscope.aliyuncs.com", None, None, CHINA_ENDPOINT), + ("https://dashscope-intl.aliyuncs.com/apps/anthropic", None, None, INTL_ENDPOINT), + (None, "https://dashscope.aliyuncs.com/apps/anthropic", None, CHINA_ENDPOINT), + ( + "https://gw.example.com/compatible-mode/v1", + None, + None, + "https://gw.example.com/api/v1/services/aigc/multimodal-generation/generation", + ), + (None, "https://dashscope.aliyuncs.com/compatible-mode/v1", None, CHINA_ENDPOINT), + (None, "https://dashscope.aliyuncs.com/api/v1", None, CHINA_ENDPOINT), + (None, None, None, INTL_ENDPOINT), + ( + "https://images.example.com/edit", + "https://dashscope.aliyuncs.com/compatible-mode/v1", + None, + "https://images.example.com/edit", + ), + ("https://dashscope.aliyuncs.com/compatible-mode/v1", None, "https://images.example.com/edit", CHINA_ENDPOINT), + ( + None, + "https://dashscope.aliyuncs.com/compatible-mode/v1", + "https://images.example.com/edit", + "https://images.example.com/edit", + ), + ), +) +def test_an_edit_goes_to_the_image_endpoint_on_the_host_its_dashscope_base_names( + monkeypatch: pytest.MonkeyPatch, api_base: str | None, env_base: str | None, image_base: str | None, endpoint: str +) -> None: + if env_base is not None: + monkeypatch.setenv("DASHSCOPE_API_BASE", env_base) + if image_base is not None: + monkeypatch.setenv("DASHSCOPE_API_BASE_IMAGE", image_base) + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + + litellm.image_edit( + model="dashscope/qwen-image-2.0", + image=PNG, + prompt="p", + api_key="k", + api_base=api_base, + client=_sync_client(seen), + ) + + assert str(seen[0].url) == endpoint + + +@pytest.mark.parametrize("api_base", ("https://dashscope.aliyuncs.com/compatible-mode/v1", None)) +def test_an_edit_and_a_speech_call_with_the_same_dashscope_base_reach_the_same_host( + monkeypatch: pytest.MonkeyPatch, api_base: str | None +) -> None: + monkeypatch.setenv("DASHSCOPE_API_BASE", "https://gw.example.com/compatible-mode/v1") + edit_url: Final = DashScopeImageEditConfig().get_complete_url( + model="qwen-image-2.0", api_base=api_base, litellm_params={} + ) + speech_url: Final = DashScopeTextToSpeechConfig().get_complete_url( + model="qwen3-tts-flash", api_base=api_base, litellm_params={} + ) + + assert httpx.URL(edit_url).host == httpx.URL(speech_url).host + + +def test_a_fourth_image_is_refused_with_a_400_before_any_request_even_under_drop_params() -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + + with pytest.raises(litellm.BadRequestError) as refused: + litellm.image_edit( + model="dashscope/qwen-image-2.0", + image=[PNG, PNG, PNG, PNG], + prompt="p", + api_key="k", + drop_params=True, + client=_sync_client(seen), + ) + + assert refused.value.status_code == 400 + assert "at most 3 input images, got 4" in str(refused.value) + assert seen == [] + + +def test_three_images_are_accepted() -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + + litellm.image_edit( + model="dashscope/qwen-image-2.0", image=[PNG, PNG, PNG], prompt="p", api_key="k", client=_sync_client(seen) + ) + + content: Final = json.loads(seen[0].content)["input"]["messages"][0]["content"] + assert content == [{"image": PNG_DATA_URI}] * 3 + [{"text": "p"}] + + +def test_an_unsupported_openai_param_answers_400_and_is_dropped_under_drop_params() -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + client: Final = _sync_client(seen) + + with pytest.raises(litellm.UnsupportedParamsError) as refused: + litellm.image_edit( + model="dashscope/qwen-image-2.0", image=PNG, prompt="p", quality="high", api_key="k", client=client + ) + litellm.image_edit( + model="dashscope/qwen-image-2.0", + image=PNG, + prompt="p", + quality="high", + api_key="k", + drop_params=True, + client=client, + ) + + assert refused.value.status_code == 400 + assert "quality" in str(refused.value) + (request,) = seen + assert json.loads(request.content)["parameters"] == {} + + +@pytest.mark.parametrize( + ("reply", "error", "message"), + ( + ( + httpx.Response(200, json={"code": "InvalidParameter", "message": "Image is too small"}), + litellm.APIError, + "Image is too small", + ), + ( + httpx.Response(400, json={"code": "InvalidParameter", "message": "Model not exist."}), + litellm.BadRequestError, + "Model not exist.", + ), + ), +) +def test_a_dashscope_rejection_reaches_the_caller_with_its_message( + reply: httpx.Response, error: type[Exception], message: str +) -> None: + with pytest.raises(error, match=message): + litellm.image_edit( + model="dashscope/qwen-image-2.0", + image=PNG, + prompt="p", + api_key="k", + client=_sync_client([], lambda _: reply), + ) + + +@pytest.mark.parametrize("model", ("qwen-image-2.0", "qwen-image-2.0-pro", "qwen-image-3.0")) +@pytest.mark.usefixtures("local_model_cost_map") +def test_each_edited_image_is_billed_at_the_model_output_image_price(model: str) -> None: + response: Final = litellm.ImageResponse(data=[{"url": EDITED_URL}, {"url": EDITED_URL}]) + + cost: Final = litellm.completion_cost( + completion_response=response, + model=f"dashscope/{model}", + call_type="aimage_edit", + custom_llm_provider="dashscope", + ) + + assert cost == pytest.approx(2 * litellm.model_cost[f"dashscope/{model}"]["output_cost_per_image"]) + assert cost > 0 + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_the_input_images_dashscope_reports_are_billed_on_top_of_the_edited_images() -> None: + def reply(_: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={**EDITED, "usage": {"input_image_count": 2, "output_image_count": 1}}) + + response: Final = litellm.image_edit( + model="dashscope/qwen-image-3.0", image=[PNG, PNG], prompt="p", api_key="k", client=_sync_client([], reply) + ) + + assert isinstance(response, litellm.ImageResponse) + assert CostCalculatorUtils.route_image_generation_cost_calculator( + model="qwen-image-3.0", + completion_response=response, + custom_llm_provider="dashscope", + call_type="aimage_edit", + model_info=ModelInfo(input_cost_per_image=0.004, output_cost_per_image=0.04), + ) == pytest.approx(2 * 0.004 + 0.04) + + +def _edited_images(count: int, input_images: int) -> litellm.ImageResponse: + response: Final = litellm.ImageResponse(data=[{"url": EDITED_URL}] * count) + response.hidden_params = {"billable_input_image_count": input_images} + return response + + +@pytest.mark.parametrize( + ("model", "deployment_price", "size", "expected_cost"), + ( + ("qwen-image-2.0", ModelInfo(input_cost_per_image=0.02), None, 2 * 0.02), + ("qwen-image-3.0-pro", ModelInfo(input_cost_per_pixel=1e-08), "1024x1024", 2 * 1024 * 1024 * 1e-08), + ), +) +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_deployment_image_price_bills_the_edit_the_way_it_bills_any_image_call( + model: str, deployment_price: ModelInfo, size: str | None, expected_cost: float +) -> None: + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model=model, + completion_response=_edited_images(count=2, input_images=3), + custom_llm_provider="dashscope", + size=size, + call_type="aimage_edit", + model_info=deployment_price, + ) + + assert cost == pytest.approx(expected_cost) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_deployment_that_copies_both_image_prices_bills_edited_and_input_images_at_its_own_prices() -> None: + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model="qwen-image-3.0", + completion_response=_edited_images(count=2, input_images=3), + custom_llm_provider="dashscope", + call_type="aimage_edit", + model_info=ModelInfo(input_cost_per_image=0.004, output_cost_per_image=0.04), + ) + + assert cost == pytest.approx(2 * 0.04 + 3 * 0.004) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_deployment_token_price_keeps_the_shared_per_image_billing() -> None: + prices: Final = litellm.model_cost["dashscope/qwen-image-3.0"] + + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model="qwen-image-3.0", + completion_response=_edited_images(count=1, input_images=2), + custom_llm_provider="dashscope", + call_type="aimage_edit", + model_info=ModelInfo(input_cost_per_token=1e-06), + ) + + assert cost == pytest.approx(2 * prices.get("input_cost_per_image", 0.0) + prices["output_cost_per_image"]) diff --git a/tests/unit/llms/dashscope/test_dashscope_text_to_speech_transformation.py b/tests/unit/llms/dashscope/test_dashscope_text_to_speech_transformation.py new file mode 100644 index 00000000000..d47c66be3cd --- /dev/null +++ b/tests/unit/llms/dashscope/test_dashscope_text_to_speech_transformation.py @@ -0,0 +1,415 @@ +import datetime +import json +from collections.abc import Callable +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.dashscope.common_utils import DashScopeError +from litellm.llms.dashscope.text_to_speech.transformation import DashScopeTextToSpeechConfig +from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.utils import Usage + +INTL: Final = "https://dashscope-intl.aliyuncs.com" +MULTIMODAL_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation" +SYNTHESIZER_PATH: Final = "/api/v1/services/audio/tts/SpeechSynthesizer" +AUDIO_URL: Final = ( + "http://93.184.216.34/prod/qwen-tts/a1.mp3?Expires=1793316671&OSSAccessKeyId=LTAI&Signature=ab%2Bc%3D" +) +AUDIO: Final = b"ID3\x04\x00synthetic-mp3-frames" +SYNTHESIZED: Final = { + "output": {"audio": {"url": AUDIO_URL, "data": "", "id": "audio_1"}}, + "usage": {"characters": 6, "input_tokens": 8, "output_tokens": 30, "total_tokens": 38}, +} + +API_REQUEST: Final = httpx.Request("POST", f"{INTL}{MULTIMODAL_PATH}") + +Reply = Callable[[httpx.Request], httpx.Response] + + +def _api(reply: dict[str, object]) -> Reply: # mutable-ok: JSON body fixture + return lambda _: httpx.Response(200, json=reply) + + +def _audio(_: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=AUDIO, headers={"content-type": "audio/mpeg"}) + + +def _recording(seen: list[httpx.Request], reply: Reply) -> httpx.MockTransport: # mutable-ok: request log + def respond(request: httpx.Request) -> httpx.Response: + seen.append(request) + return reply(request) + + return httpx.MockTransport(respond) + + +def _refusing_blocking_download(request: httpx.Request) -> httpx.Response: + raise AssertionError(f"blocking client used for {request.url} on the event loop") + + +def _logging() -> Logging: + return Logging( + model="dashscope-tts", + messages=[], + stream=False, + call_type="aspeech", + start_time=datetime.datetime(2026, 10, 28, tzinfo=datetime.timezone.utc), + litellm_call_id="call-tts", + function_id="fn-tts", + ) + + +@pytest.fixture(autouse=True) +def _no_ambient_dashscope_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + + +@pytest.mark.parametrize( + ("model", "optional_params", "path", "provider_input"), + ( + ( + "qwen3-tts-flash", + {"response_format": "wav"}, + MULTIMODAL_PATH, + {"text": "Hello.", "voice": "Cherry"}, + ), + ( + "qwen-audio-3.1-tts-flash", + {"response_format": "mp3", "instructions": "Speak slowly."}, + SYNTHESIZER_PATH, + {"text": "Hello.", "voice": "Cherry", "format": "mp3", "instruction": "Speak slowly."}, + ), + ( + "qwen-audio-3.0-tts-plus", + {}, + SYNTHESIZER_PATH, + {"text": "Hello.", "voice": "Cherry"}, + ), + ), +) +async def test_each_family_posts_to_its_endpoint_and_the_async_handler_returns_the_downloaded_audio( + model: str, + optional_params: dict[str, object], # mutable-ok: parametrized fixture + path: str, + provider_input: dict[str, object], # mutable-ok: parametrized fixture +) -> None: + api_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + download_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + config: Final = DashScopeTextToSpeechConfig( + sync_client=lambda: HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_refusing_blocking_download)) + ), + async_client=lambda: AsyncHTTPHandler(transport=_recording(download_seen, _audio)), + ) + voice, mapped = config.map_openai_params(model=model, optional_params=optional_params, voice="Cherry") + + speech: Final = await BaseLLMHTTPHandler().async_text_to_speech_handler( + model=model, + input="Hello.", + voice=voice, + text_to_speech_provider_config=config, + text_to_speech_optional_params=mapped, + custom_llm_provider="dashscope", + litellm_params={"api_key": "sk-dashscope-test", "api_base": None}, + logging_obj=_logging(), + timeout=30, + client=AsyncHTTPHandler(transport=_recording(api_seen, _api(SYNTHESIZED))), + ) + + assert speech.content == AUDIO + (api_request,) = api_seen + assert str(api_request.url) == f"{INTL}{path}" + assert api_request.headers["authorization"] == "Bearer sk-dashscope-test" + assert json.loads(api_request.content) == {"model": model, "input": provider_input} + (download,) = download_seen + assert str(download.url) == AUDIO_URL + assert "authorization" not in download.headers + assert download.extensions["timeout"] == api_request.extensions["timeout"] + + +def test_the_sync_handler_downloads_through_the_injected_blocking_client_within_the_call_timeout() -> None: + api_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + download_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + config: Final = DashScopeTextToSpeechConfig( + sync_client=lambda: HTTPHandler(client=httpx.Client(transport=_recording(download_seen, _audio))) + ) + + speech: Final = BaseLLMHTTPHandler().text_to_speech_handler( + model="qwen3-tts-flash", + input="Hello.", + voice="Cherry", + text_to_speech_provider_config=config, + text_to_speech_optional_params={}, + custom_llm_provider="dashscope", + litellm_params={"api_key": "k", "api_base": None}, + logging_obj=_logging(), + timeout=30, + client=HTTPHandler(client=httpx.Client(transport=_recording(api_seen, _api(SYNTHESIZED)))), + ) + + assert isinstance(speech, HttpxBinaryResponseContent) + assert speech.content == AUDIO + (api_request,) = api_seen + (download,) = download_seen + assert str(download.url) == AUDIO_URL + assert download.extensions["timeout"] == api_request.extensions["timeout"] + + +@pytest.mark.parametrize( + ("audio_url", "download_url"), + ( + ( + "http://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/a1.wav?Expires=1&Signature=ab%2Bc%3D", + "https://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/a1.wav?Expires=1&Signature=ab%2Bc%3D", + ), + ( + "https://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/a1.wav", + "https://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/a1.wav", + ), + ("http://audio.notaliyuncs.com/a1.wav", "http://audio.notaliyuncs.com/a1.wav"), + (AUDIO_URL, AUDIO_URL), + ), +) +def test_an_alibaba_cloud_audio_url_is_downloaded_over_https( + monkeypatch: pytest.MonkeyPatch, audio_url: str, download_url: str +) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + download_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + config: Final = DashScopeTextToSpeechConfig( + sync_client=lambda: HTTPHandler(client=httpx.Client(transport=_recording(download_seen, _audio))) + ) + + speech: Final = BaseLLMHTTPHandler().text_to_speech_handler( + model="qwen3-tts-flash", + input="Hello.", + voice="Cherry", + text_to_speech_provider_config=config, + text_to_speech_optional_params={}, + custom_llm_provider="dashscope", + litellm_params={"api_key": "k", "api_base": None}, + logging_obj=_logging(), + timeout=30, + client=HTTPHandler( + client=httpx.Client( + transport=_recording([], _api({**SYNTHESIZED, "output": {"audio": {"url": audio_url}}})) + ) + ), + ) + + assert speech.content == AUDIO + (download,) = download_seen + assert str(download.url) == download_url + + +@pytest.mark.parametrize( + ("api_base", "environment", "model", "url"), + ( + (None, None, "qwen3-tts-flash", f"{INTL}{MULTIMODAL_PATH}"), + (None, None, "qwen-audio-3.1-tts-plus", f"{INTL}{SYNTHESIZER_PATH}"), + ( + "https://dashscope.aliyuncs.com/compatible-mode/v1", + None, + "qwen-audio-3.1-tts-plus", + f"https://dashscope.aliyuncs.com{SYNTHESIZER_PATH}", + ), + (f"{INTL}{SYNTHESIZER_PATH}", None, "qwen3-tts-flash", f"{INTL}{MULTIMODAL_PATH}"), + ( + "https://dashscope.aliyuncs.com/apps/anthropic", + None, + "qwen3-tts-flash", + f"https://dashscope.aliyuncs.com{MULTIMODAL_PATH}", + ), + ( + "https://relay.example.com/dashscope/", + None, + "qwen3-tts-flash", + f"https://relay.example.com/dashscope{MULTIMODAL_PATH}", + ), + ( + None, + "https://dashscope.aliyuncs.com/compatible-mode/v1", + "qwen3-tts-flash", + f"https://dashscope.aliyuncs.com{MULTIMODAL_PATH}", + ), + ), +) +def test_the_endpoint_follows_the_model_family_on_the_configured_host( + monkeypatch: pytest.MonkeyPatch, api_base: str | None, environment: str | None, model: str, url: str +) -> None: + if environment is not None: + monkeypatch.setenv("DASHSCOPE_API_BASE", environment) + + assert DashScopeTextToSpeechConfig().get_complete_url(model=model, api_base=api_base, litellm_params={}) == url + + +@pytest.mark.parametrize( + ("model", "optional_params", "voice", "refused"), + ( + ("qwen3-tts-flash", {"response_format": "mp3"}, "Cherry", "response_format='mp3'"), + ("qwen3-tts-flash", {"instructions": "Whisper."}, "Cherry", "instructions='Whisper.'"), + ("qwen-audio-3.1-tts-flash", {"response_format": "aac"}, "Cherry", "response_format='aac'"), + ("qwen-audio-3.1-tts-flash", {"speed": 1.5}, "Cherry", "speed=1.5"), + ("qwen-audio-3.1-tts-flash", {}, {"voice_id": "Cherry"}, "voice as an object"), + ), +) +def test_an_unsupported_param_answers_400_and_is_dropped_under_drop_params( + model: str, + optional_params: dict[str, object], # mutable-ok: parametrized fixture + voice: str | dict[str, object], # mutable-ok: parametrized fixture + refused: str, +) -> None: + config: Final = DashScopeTextToSpeechConfig() + + with pytest.raises(litellm.UnsupportedParamsError) as rejected: + config.map_openai_params(model=model, optional_params=optional_params, voice=voice) + dropped_voice, dropped = config.map_openai_params( + model=model, optional_params=optional_params, voice=voice, kwargs={"drop_params": True} + ) + + assert rejected.value.status_code == 400 + assert refused in str(rejected.value) + assert dropped == {} + assert dropped_voice == (voice if isinstance(voice, str) else None) + + +async def test_aspeech_uses_the_native_endpoint_on_the_caller_api_base_and_refuses_a_private_audio_url() -> None: + api_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + private_audio: Final = {"output": {"audio": {"url": "http://169.254.169.254/latest/meta-data"}}} + + with pytest.raises(litellm.BadGatewayError) as refused: + await litellm.aspeech( + model="dashscope/qwen-audio-3.1-tts-flash", + input="Hello.", + voice="longanhuan_v3.1", + response_format="opus", + api_key="sk-dashscope-test", + api_base="https://dashscope.aliyuncs.com/compatible-mode/v1", + client=AsyncHTTPHandler(transport=_recording(api_seen, _api(private_audio))), + ) + + assert refused.value.status_code == 502 + assert "audio URL that was refused" in str(refused.value) + (api_request,) = api_seen + assert str(api_request.url) == f"https://dashscope.aliyuncs.com{SYNTHESIZER_PATH}" + assert json.loads(api_request.content) == { + "model": "qwen-audio-3.1-tts-flash", + "input": {"text": "Hello.", "voice": "longanhuan_v3.1", "format": "opus"}, + } + + +@pytest.mark.parametrize( + ("api_reply", "audio_reply", "status", "message"), + ( + ( + httpx.Response(200, json={"code": "InvalidParameter", "message": "The voice property is required."}), + _audio, + 400, + "The voice property is required.", + ), + ( + httpx.Response(200, content=b"gateway"), + _audio, + 502, + "Unexpected DashScope text-to-speech response", + ), + ( + httpx.Response(200, json=SYNTHESIZED, request=API_REQUEST), + lambda _: httpx.Response(403), + 502, + "audio download failed with HTTP 403", + ), + ), +) +async def test_a_failed_synthesis_or_download_raises_a_dashscope_error( + api_reply: httpx.Response, audio_reply: Reply, status: int, message: str +) -> None: + config: Final = DashScopeTextToSpeechConfig( + async_client=lambda: AsyncHTTPHandler(transport=httpx.MockTransport(audio_reply)) + ) + + with pytest.raises(DashScopeError) as failed: + await config.async_transform_text_to_speech_response( + model="qwen3-tts-flash", raw_response=api_reply, logging_obj=_logging() + ) + + assert failed.value.status_code == status + assert message in failed.value.message + + +async def test_provider_token_usage_rides_on_the_audio_so_token_priced_speech_is_billed() -> None: + config: Final = DashScopeTextToSpeechConfig( + async_client=lambda: AsyncHTTPHandler(transport=httpx.MockTransport(_audio)) + ) + + speech: Final = await config.async_transform_text_to_speech_response( + model="qwen-audio-3.1-tts-flash", + raw_response=httpx.Response(200, json=SYNTHESIZED, request=API_REQUEST), + logging_obj=_logging(), + ) + + assert speech.usage == Usage(prompt_tokens=8, completion_tokens=30, total_tokens=38) + + +@pytest.mark.usefixtures("local_model_cost_map") +async def test_character_priced_speech_bills_the_characters_dashscope_reports() -> None: + text: Final = "你好 世界。" + reported: Final = {**SYNTHESIZED, "usage": {"characters": 10}} + config: Final = DashScopeTextToSpeechConfig( + async_client=lambda: AsyncHTTPHandler(transport=httpx.MockTransport(_audio)) + ) + speech: Final = await config.async_transform_text_to_speech_response( + model="qwen3-tts-flash", + raw_response=httpx.Response(200, json=reported, request=API_REQUEST), + logging_obj=_logging(), + ) + + cost: Final = litellm.completion_cost( + completion_response=speech, + model="dashscope/qwen3-tts-flash", + call_type="aspeech", + prompt=text, + custom_llm_provider="dashscope", + ) + + assert litellm.utils.count_characters(text) != 10 + assert cost == pytest.approx(10 * litellm.model_cost["dashscope/qwen3-tts-flash"]["input_cost_per_character"]) + + +@pytest.mark.parametrize( + "model", ("qwen3-tts-flash", "qwen-audio-3.1-tts-plus", "qwen-audio-3.0-tts-plus", "qwen-audio-3.0-tts-flash") +) +@pytest.mark.usefixtures("local_model_cost_map") +def test_character_priced_speech_bills_each_input_character_at_the_model_rate(model: str) -> None: + cost: Final = litellm.completion_cost( + completion_response=HttpxBinaryResponseContent(httpx.Response(200, content=AUDIO)), + model=f"dashscope/{model}", + call_type="aspeech", + prompt="Hello.", + custom_llm_provider="dashscope", + ) + + assert cost == pytest.approx(6 * litellm.model_cost[f"dashscope/{model}"]["input_cost_per_character"]) + assert cost > 0 + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_token_priced_speech_bills_the_provider_reported_input_and_output_tokens() -> None: + speech: Final = HttpxBinaryResponseContent(httpx.Response(200, content=AUDIO)) + speech.usage = Usage(prompt_tokens=8, completion_tokens=30, total_tokens=38) + row: Final = litellm.model_cost["dashscope/qwen-audio-3.1-tts-flash"] + + cost: Final = litellm.completion_cost( + completion_response=speech, + model="dashscope/qwen-audio-3.1-tts-flash", + call_type="aspeech", + prompt="Hello.", + custom_llm_provider="dashscope", + ) + + assert cost == pytest.approx(8 * row["input_cost_per_token"] + 30 * row["output_cost_per_token"]) + assert row["output_cost_per_token"] > row["input_cost_per_token"] > 0 diff --git a/tests/unit/llms/litellm_proxy/responses/test_transformation.py b/tests/unit/llms/litellm_proxy/responses/test_transformation.py index 3d774b79eed..5537d9d1916 100644 --- a/tests/unit/llms/litellm_proxy/responses/test_transformation.py +++ b/tests/unit/llms/litellm_proxy/responses/test_transformation.py @@ -1,4 +1,5 @@ import json +from collections.abc import Callable, Iterable from typing import Final import httpx @@ -94,3 +95,86 @@ def test_compact_request_forwards_metadata_and_user_to_the_downstream_proxy(drop "user": "end-user-1", "temperature": 0.2, } + + +_GUARDRAIL_BLOCK: Final = { + "error": { + "message": "Content blocked: secret_project_codename pattern detected", + "type": "invalid_request_error", + "param": None, + "code": "400", + "provider_specific_fields": {"guardrail_name": "block-secret-project", "guardrail_mode": "pre_call"}, + } +} + + +def _proxy_rejection(route: str, response: httpx.Response, call: Callable[[], object]) -> litellm.BadRequestError: + with respx.mock(assert_all_called=True) as mock: + _ = mock.post(f"https://my-proxy.example.com/{route}").mock(return_value=response) + with pytest.raises(litellm.BadRequestError) as caught: + _ = call() + return caught.value + + +def _consume(stream: Iterable[object]) -> None: + for _ in stream: + pass + + +_PROXY_PARAMS: Final = {"api_base": "https://my-proxy.example.com/", "api_key": "sk-proxy-key"} + + +@pytest.mark.usefixtures(httpx_transport.__name__) +@pytest.mark.parametrize("stream", [False, True]) +def test_proxy_400_reaches_responses_caller_with_the_same_body_as_chat_completions(stream: bool) -> None: + chat_error: Final = _proxy_rejection( + "chat/completions", + httpx.Response(400, json=_GUARDRAIL_BLOCK), + lambda: litellm.completion( + model="litellm_proxy/gpt-5.5", messages=[{"role": "user", "content": "project bluebird"}], **_PROXY_PARAMS + ), + ) + responses_error: Final = _proxy_rejection( + "responses", + httpx.Response(400, json=_GUARDRAIL_BLOCK), + lambda: ( + _consume(litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", stream=True, **_PROXY_PARAMS)) + if stream + else litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", **_PROXY_PARAMS) + ), + ) + assert responses_error.body == chat_error.body == _GUARDRAIL_BLOCK["error"] + assert (responses_error.code, responses_error.type) == (chat_error.code, chat_error.type) + assert str(responses_error) == str(chat_error) + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_proxy_error_without_a_json_object_body_still_raises_with_its_text() -> None: + error: Final = _proxy_rejection( + "responses", + httpx.Response(400, text="upstream proxy rejected the request"), + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="hi", **_PROXY_PARAMS), + ) + assert error.body is None + assert "upstream proxy rejected the request" in str(error) + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_proxy_error_object_without_an_error_member_is_the_body_as_sent() -> None: + sent: Final = {"message": "upstream proxy rejected the request", "type": "invalid_request_error"} + error: Final = _proxy_rejection( + "responses", + httpx.Response(400, json=sent), + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="hi", **_PROXY_PARAMS), + ) + assert error.body == sent + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_proxy_error_whose_error_member_is_text_still_raises_with_that_text() -> None: + error: Final = _proxy_rejection( + "responses", + httpx.Response(400, json={"error": "upstream proxy rejected the request"}), + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="hi", **_PROXY_PARAMS), + ) + assert "upstream proxy rejected the request" in str(error) diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index 17c1b35e211..29c25c5fdbf 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -2,6 +2,7 @@ import copy import json import operator from collections.abc import Mapping +from itertools import chain from typing import Final import httpx @@ -11,9 +12,10 @@ from pydantic import JsonValue, ValidationError import litellm from litellm.litellm_core_utils.tool_fields import DropPolicy +from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER from litellm.llms.openai.responses.count_tokens.token_counter import OpenAITokenCounter from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse +from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse COUNT_URL: Final = "https://api.openai.com/v1/responses/input_tokens" @@ -484,6 +486,122 @@ async def test_anthropic_format_tool_result_text_block_without_text_still_counts assert {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\n\ndone"} in sent["input"] +def _tool_pair(index: int) -> tuple[dict[str, object], dict[str, object]]: + return ( + {"role": "assistant", "content": [{"type": "tool_use", "id": f"toolu_{index}", "name": "log", "input": {}}]}, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": f"toolu_{index}", "content": f"log {index} " * 40}], + }, + ) + + +TOOL_PAIRS: Final[list[dict[str, object]]] = [ + {"role": "user", "content": "Compare the logs."}, + *chain.from_iterable(_tool_pair(index) for index in range(3)), +] + + +async def _count_with_edit( + monkeypatch: pytest.MonkeyPatch, edit: dict[str, JsonValue], drop_invalid_edits: bool, system: object = None +) -> tuple[TokenCountResponse | None, respx.Route]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 9})) + result: Final = await OpenAITokenCounter().count_tokens( + model_to_use="gpt-4o", + messages=copy.deepcopy(TOOL_PAIRS), + contents=None, + deployment={"litellm_params": {"api_key": "sk-test"}}, + request_model="openai/gpt-4o", + system=system, + messages_format="anthropic", + context_management=CountContextManagement(spec={"edits": [edit]}, drop_invalid_edits=drop_invalid_edits), + ) + return result, route + + +def _sent_outputs(route: respx.Route) -> dict[str, str]: + sent: Final = json.loads(route.calls.last.request.content) + return {item["call_id"]: item["output"] for item in sent["input"] if item["type"] == "function_call_output"} + + +CLEAR_EDIT: Final[dict[str, JsonValue]] = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 1}, + "keep": {"type": "tool_uses", "value": 1}, +} + + +@pytest.mark.asyncio +async def test_anthropic_format_counts_the_history_the_clear_edit_leaves_for_inference( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, route = await _count_with_edit(monkeypatch, CLEAR_EDIT, drop_invalid_edits=False) + + assert result is not None + assert (result.error, result.total_tokens) == (False, 9) + assert _sent_outputs(route) == { + "toolu_0": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_1": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_2": "log 2 " * 40, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "system", + [ + "Follow the policy. " * 1500, + [{"type": "text", "text": "Follow the policy. " * 1500}], + ["Stray.", {"type": "text", "text": "Follow the policy. " * 1500}], + ], + ids=["string", "blocks", "blocks_beside_a_stray_string"], +) +async def test_anthropic_format_input_tokens_trigger_counts_the_system_prompt_like_inference( + monkeypatch: pytest.MonkeyPatch, system: object +) -> None: + message_tokens: Final = litellm.token_counter(model="gpt-4o", messages=TOOL_PAIRS) + trigger_between_messages_and_system: Final = message_tokens + 1000 + edit: Final[dict[str, JsonValue]] = { + **CLEAR_EDIT, + "trigger": {"type": "input_tokens", "value": trigger_between_messages_and_system}, + } + + _, without_system = await _count_with_edit(monkeypatch, edit, drop_invalid_edits=False) + _, with_system = await _count_with_edit(monkeypatch, edit, drop_invalid_edits=False, system=system) + + assert _sent_outputs(without_system)["toolu_0"] == "log 0 " * 40 + assert _sent_outputs(with_system) == { + "toolu_0": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_1": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_2": "log 2 " * 40, + } + + +@pytest.mark.asyncio +async def test_anthropic_format_malformed_clear_edit_answers_400_without_calling_openai( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, route = await _count_with_edit(monkeypatch, {**CLEAR_EDIT, "exclude_tools": "log"}, drop_invalid_edits=False) + + assert route.called is False + assert result is not None + assert (result.error, result.status_code) == (True, 400) + assert "exclude_tools" in str(result.error_message) + + +@pytest.mark.asyncio +async def test_anthropic_format_drop_params_counts_the_clear_without_its_malformed_field( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, route = await _count_with_edit(monkeypatch, {**CLEAR_EDIT, "exclude_tools": "log"}, drop_invalid_edits=True) + + assert result is not None + assert result.error is False + assert _sent_outputs(route)["toolu_0"] == CLEARED_TOOL_RESULT_PLACEHOLDER + + @pytest.mark.asyncio async def test_count_uses_the_global_openai_key_chat_would_use(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/tests/unit/llms/runwayml/test_text_to_speech_transformation.py b/tests/unit/llms/runwayml/test_text_to_speech_transformation.py index 2e4d68a02da..981ac3e4855 100644 --- a/tests/unit/llms/runwayml/test_text_to_speech_transformation.py +++ b/tests/unit/llms/runwayml/test_text_to_speech_transformation.py @@ -2,13 +2,51 @@ Test RunwayML text-to-speech transformation """ +from collections.abc import Iterator +from typing import Final +import httpx +import pytest +import respx +import litellm from litellm.llms.runwayml.text_to_speech.transformation import ( RunwayMLTextToSpeechConfig, ) +@pytest.fixture +def httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + yield + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.asyncio +async def test_async_speech_polls_the_runwayml_task_with_the_caller_key_and_returns_its_audio( + respx_mock: respx.MockRouter, httpx_transport: None +) -> None: + respx_mock.post("https://api.dev.runwayml.com/v1/text_to_speech").mock( + return_value=httpx.Response(200, json={"id": "task_1", "status": "PENDING"}) + ) + poll: Final = respx_mock.get("https://api.dev.runwayml.com/v1/tasks/task_1").mock( + return_value=httpx.Response( + 200, json={"id": "task_1", "status": "SUCCEEDED", "output": ["https://storage.example.com/a1.mp3"]} + ) + ) + respx_mock.get("https://storage.example.com/a1.mp3").mock( + return_value=httpx.Response(200, content=b"ID3runway-audio") + ) + + speech: Final = await litellm.aspeech( + model="runwayml/eleven_multilingual_v2", input="Hello.", voice="alloy", api_key="rw-key" + ) + + assert speech.content == b"ID3runway-audio" + assert poll.calls.last.request.headers["authorization"] == "Bearer rw-key" + + def test_openai_voice_mapping_to_runwayml(): """ Test that OpenAI voice names are correctly mapped to RunwayML preset IDs diff --git a/tests/unit/llms/test_get_complete_url_trailing_slash.py b/tests/unit/llms/test_get_complete_url_trailing_slash.py index 43a74e56028..e35e21c67f9 100644 --- a/tests/unit/llms/test_get_complete_url_trailing_slash.py +++ b/tests/unit/llms/test_get_complete_url_trailing_slash.py @@ -137,6 +137,7 @@ API_BASE_IS_THE_FULL_URL: Final[Mapping[str, str]] = MappingProxyType( "brave.search.transformation.BraveSearchConfig", "cohere.chat.v2_transformation.CohereV2ChatConfig", "cohere.embed.transformation.CohereEmbeddingConfig", + "dashscope.image_edit.transformation.DashScopeImageEditConfig", "dashscope.image_generation.transformation.DashScopeImageGenerationConfig", "dataforseo.search.transformation.DataForSEOSearchConfig", "datarobot.chat.transformation.DataRobotConfig", diff --git a/tests/unit/llms/vertex_ai/count_tokens/test_context.py b/tests/unit/llms/vertex_ai/count_tokens/test_context.py index b2a0b735b0f..38eb87debfd 100644 --- a/tests/unit/llms/vertex_ai/count_tokens/test_context.py +++ b/tests/unit/llms/vertex_ai/count_tokens/test_context.py @@ -649,7 +649,7 @@ async def test_anthropic_view_counts_the_unedited_body_without_an_error_log_when assert not any(record.levelno >= logging.ERROR for record in caplog.records) -async def test_anthropic_view_counts_the_turns_after_a_compaction_block_when_an_edit_before_compact_fails() -> None: +async def test_anthropic_view_clears_only_the_turns_after_a_compaction_block_when_clear_runs_before_compact() -> None: clear_before_compact: Final[JsonValue] = { "edits": [ { diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index de37b710f36..8612351eda6 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -1028,6 +1028,40 @@ def test_oss_native_model_cannot_implicitly_select_an_unauthorized_checkpoint(pr assert denied.value.status_code == 400 +# eu-jev.com/docs (read 2026-10-21): an omitted model means eu-jeff-latest, and the legacy names +# jeff-latest and jeff-1.0.0 answer with the current release. +@pytest.mark.parametrize( + ("request_data", "permission_model"), + [ + ({}, "eu_jev/eu-jeff-latest"), + ({"model": None}, "eu_jev/eu-jeff-latest"), + ({"model": "eu-jeff-latest"}, "eu_jev/eu-jeff-latest"), + ({"model": "jeff-latest"}, "eu_jev/eu-jeff-latest"), + ({"model": "jeff-1.0.0"}, "eu_jev/eu-jeff-latest"), + ({"model": "eu-jeff-1.0.0"}, "eu_jev/eu-jeff-1.0.0"), + ({"model": "eu_jev/eu-jeff-1.0.0"}, "eu_jev/eu-jeff-1.0.0"), + ({"model": "eu_jev/jeff-latest"}, "eu_jev/eu-jeff-latest"), + ], +) +@pytest.mark.parametrize("suffix", ["", "/"]) +def test_eu_jev_decision_is_checked_against_the_model_eu_jev_runs( + request_data: dict[str, object], permission_model: str, suffix: str +) -> None: + assert get_model_from_request(request_data=request_data, route=f"/eu_jev/v1/systemone{suffix}") == permission_model + + +@pytest.mark.parametrize("model", [["eu-jeff-latest"], 7, {"name": "eu-jeff-latest"}]) +def test_eu_jev_decision_rejects_a_model_that_is_not_a_string(model: object) -> None: + with pytest.raises(HTTPException) as denied: + get_model_from_request(request_data={"model": model}, route="/eu_jev/v1/systemone") + assert denied.value.status_code == 400 + + +@pytest.mark.usefixtures("configured_eu_jev_route") +def test_a_pass_through_configured_at_the_eu_jev_decision_path_keeps_the_requested_model() -> None: + assert get_model_from_request(request_data={"model": "jeff-latest"}, route="/eu_jev/v1/systemone") == "jeff-latest" + + def test_laya_model_normalization_does_not_change_other_provider_routes() -> None: assert get_model_from_request(request_data={"model": "jev-latest"}, route="/typesafe/v1/systemone") == "jev-latest" assert get_model_from_request(request_data={}, route="/laya/health") is None diff --git a/tests/unit/proxy/auth/test_route_checks.py b/tests/unit/proxy/auth/test_route_checks.py index b2c67b4e009..529f639999d 100644 --- a/tests/unit/proxy/auth/test_route_checks.py +++ b/tests/unit/proxy/auth/test_route_checks.py @@ -662,6 +662,41 @@ def test_virtual_key_llm_api_routes_allows_model_info(route): assert result is True +@pytest.mark.parametrize( + ("route", "is_llm_route"), + [ + ("/eu_jev/v1/systemone", True), + ("/eu_jev/v1/systemone/extra", False), + ("/eu_jev/v1/systemone2", False), + ("/eu_jev/custom", False), + ], +) +def test_only_the_built_in_eu_jev_decision_path_is_an_llm_route(route: str, is_llm_route: bool) -> None: + assert RouteChecks.is_llm_api_route(route=route) is is_llm_route + + +@pytest.mark.usefixtures("configured_eu_jev_route") +def test_a_pass_through_configured_at_the_eu_jev_decision_path_is_not_an_llm_route() -> None: + assert RouteChecks.is_llm_api_route(route="/eu_jev/v1/systemone") is False + + +@pytest.mark.usefixtures("configured_eu_jev_route_with_trailing_slash") +@pytest.mark.parametrize("route", ["/eu_jev/v1/systemone", "/eu_jev/v1/systemone/"]) +def test_a_pass_through_configured_at_the_eu_jev_decision_path_with_a_trailing_slash_is_not_an_llm_route( + route: str, +) -> None: + assert RouteChecks.is_llm_api_route(route=route) is False + + +@pytest.mark.parametrize("allowed_route", ["llm_api_routes", "llm_api_routes_without_mcp"]) +def test_an_llm_api_key_can_call_the_built_in_eu_jev_decision_route(allowed_route: str) -> None: + valid_token: Final = UserAPIKeyAuth(user_id="test_user", allowed_routes=[allowed_route]) + + assert RouteChecks.is_virtual_key_allowed_to_call_route( + route="/eu_jev/v1/systemone", valid_token=valid_token, request=_mock_request("POST") + ) + + @pytest.mark.parametrize("route", ["/model/info", "/v1/model/info", "/model_group/info"]) def test_model_info_not_classified_as_llm_api(route): """Membership in `llm_api_routes` must not promote /model/info to an diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index e13eddef43b..298d32def7d 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -8,17 +8,23 @@ import os import tempfile import warnings from collections.abc import Awaitable, Callable, Iterator +from contextlib import contextmanager from typing import Dict, Final, Optional from unittest.mock import AsyncMock, MagicMock, patch import pytest import yaml +from fastapi import FastAPI from fastapi.testclient import TestClient from prisma.errors import ClientNotConnectedError import litellm import litellm.proxy.proxy_server from litellm._logging import verbose_proxy_logger +from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + _registered_pass_through_routes, +) from litellm._service_logger import ServiceTypes from litellm.integrations.otel.model.payloads import ServiceSpanData from litellm.integrations.otel.model.spans import service_span_name @@ -239,6 +245,40 @@ def secret_vault_factory() -> type[FakeSecretVault]: return FakeSecretVault +@contextmanager +def _eu_jev_route_configured_at(path: str) -> Iterator[None]: + saved_registry: Final = dict(_registered_pass_through_routes) + InitPassThroughEndpointHelpers.add_exact_path_route( + app=FastAPI(), + path=path, + target="https://operator.example/decide", + custom_headers=None, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=None, + endpoint_id="operator-eu-jev", + methods=["POST"], + ) + try: + yield + finally: + _registered_pass_through_routes.clear() + _registered_pass_through_routes.update(saved_registry) + + +@pytest.fixture +def configured_eu_jev_route() -> Iterator[None]: + with _eu_jev_route_configured_at("/eu_jev/v1/systemone"): + yield + + +@pytest.fixture +def configured_eu_jev_route_with_trailing_slash() -> Iterator[None]: + with _eu_jev_route_configured_at("/eu_jev/v1/systemone/"): + yield + + @pytest.fixture def httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py index 8308a7ba386..b5f794cb136 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py @@ -16,12 +16,16 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +from mcp.types import CallToolResult, TextContent import litellm from litellm import ModelResponse from litellm._version import version as litellm_version from litellm.exceptions import GuardrailRaisedException, Timeout from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, MaskedHTTPStatusError +from litellm.proxy._experimental.mcp_server.guardrail_translation.handler import ( + MCPGuardrailTranslationHandler, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( GenericGuardrailAPI, @@ -29,6 +33,8 @@ from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api.generic_guardrail_api import ( _HEADER_PRESENT_PLACEHOLDER, ) +from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import UnifiedLLMGuardrails +from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import Choices, GenericGuardrailAPIInputs, Message @@ -2621,3 +2627,68 @@ class TestNumRetries: assert sent.qsize() == num_retries + 1 assert result == {"texts": ["hi"]} + + +def _mcp_guardrail(mode: str) -> GenericGuardrailAPI: + return GenericGuardrailAPI(api_base="http://x", guardrail_name="g", event_hook=mode, default_on=True) + + +def _mock_api_response(payload: dict[str, object]) -> MagicMock: + response: Final = MagicMock() + response.json.return_value = payload + response.raise_for_status = MagicMock() + return response + + +@pytest.mark.parametrize("mode", ["pre_mcp_call", "during_mcp_call", "post_mcp_call"]) +def test_mcp_call_modes_pass_event_hook_validation(mode: str) -> None: + guardrail: Final = _mcp_guardrail(mode) + + assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks(mode)) is True + + +@pytest.mark.asyncio +async def test_pre_mcp_call_blocked_raises() -> None: + guardrail: Final = _mcp_guardrail("pre_mcp_call") + response: Final = _mock_api_response({"action": "BLOCKED", "blocked_reason": "credential detected in tool input"}) + + with patch.object(guardrail.async_handler, "post", return_value=response): + with pytest.raises(GuardrailRaisedException) as exc_info: + await MCPGuardrailTranslationHandler().process_input_messages( + data={"mcp_tool_name": "t", "mcp_arguments": {"a": "secret"}}, + guardrail_to_apply=guardrail, + ) + + assert exc_info.value.message == "credential detected in tool input" + + +@pytest.mark.asyncio +async def test_post_mcp_call_redacts_result() -> None: + guardrail: Final = _mcp_guardrail("post_mcp_call") + response: Final = _mock_api_response({"action": "GUARDRAIL_INTERVENED", "texts": ["x [R] y"]}) + result: Final = CallToolResult(content=[TextContent(type="text", text="x secret y")], isError=False) + + with patch.object(guardrail.async_handler, "post", return_value=response): + returned: Final = await MCPGuardrailTranslationHandler().process_output_response( + response=result, + guardrail_to_apply=guardrail, + request_data={"mcp_tool_name": "t"}, + ) + + assert returned.content[0].text == "x [R] y" + + +@pytest.mark.asyncio +async def test_during_mcp_call_blocked_raises() -> None: + guardrail: Final = _mcp_guardrail("during_mcp_call") + response: Final = _mock_api_response({"action": "BLOCKED", "blocked_reason": "credential detected in tool input"}) + + with patch.object(guardrail.async_handler, "post", return_value=response): + with pytest.raises(GuardrailRaisedException) as exc_info: + await UnifiedLLMGuardrails().async_moderation_hook( + data={"mcp_tool_name": "t", "mcp_arguments": {"a": "secret"}, "guardrail_to_apply": guardrail}, + user_api_key_dict=UserAPIKeyAuth(), + call_type="call_mcp_tool", + ) + + assert exc_info.value.message == "credential detected in tool input" diff --git a/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py index b7795b4d2f0..ee3ef3f5323 100644 --- a/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -1,7 +1,7 @@ import contextlib import json import os -from collections.abc import Sequence +from collections.abc import Callable, Iterator, Sequence from collections.abc import Set as AbstractSet from datetime import datetime, timedelta, timezone from types import SimpleNamespace @@ -1599,9 +1599,7 @@ class TestMCPRequestHandler: team_id="test-team-id", ) - with patch( - "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth" - ) as mock_user_api_key_auth: + with patch("litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth") as mock_user_api_key_auth: mock_user_api_key_auth.return_value = mock_auth_result # Call the method @@ -1663,10 +1661,10 @@ class TestMCPOAuth2AuthFlow: as LiteLLM API keys, causing auth failures and empty tool listings. """ - async def test_oauth2_token_in_authorization_header_requires_litellm_admission(self): + async def test_oauth2_upstream_token_on_legacy_delegate_is_admitted_anonymously(self) -> None: """ - A bare Authorization token on the legacy delegated mode must establish - a LiteLLM principal rather than entering anonymously. + A bare upstream OAuth token on the legacy delegated mode is admitted + anonymously without LiteLLM key validation and kept for egress. """ from litellm.types.mcp import MCPAuth @@ -1702,10 +1700,11 @@ class TestMCPOAuth2AuthFlow: raw_headers, ) = await MCPRequestHandler.process_mcp_request(scope) - assert isinstance(auth_result, UserAPIKeyAuth) - assert auth_result.user_id == "admitted-user" - mock_auth.assert_awaited_once() - assert oauth2_headers.get("Authorization") == "Bearer atlassian-oauth2-access-token-xyz" + assert auth_result.user_id is None + assert auth_result.api_key is None + assert auth_result.mcp_forwards_caller_authorization is True + mock_auth.assert_not_awaited() + assert oauth2_headers == {"Authorization": "Bearer atlassian-oauth2-access-token-xyz"} async def test_explicit_litellm_key_with_oauth2_authorization(self): """ @@ -2545,9 +2544,10 @@ class TestMCPDelegateAuthToUpstream: """ Tests for the ``delegate_auth_to_upstream`` per-server flag. - The legacy flag no longer bypasses LiteLLM admission. OAuth discovery may - still use the anonymous cold-start challenge, but a presented bearer must - authenticate to LiteLLM unless a separate admission credential is supplied. + A bare bearer that is a LiteLLM credential takes normal admission and is + never forwarded. Any other bearer is the upstream token: it is forwarded, + and the caller is admitted as the JWT or custom-auth user it also proves + when that user is granted the server, else anonymously. """ @staticmethod @@ -2652,37 +2652,497 @@ class TestMCPDelegateAuthToUpstream: assert isinstance(auth_result, UserAPIKeyAuth) mock_auth.assert_awaited_once() - async def test_delegate_with_only_upstream_token_requires_litellm_auth(self): - """ - An upstream token cannot establish a LiteLLM principal and must not - reopen anonymous admission. - """ - from litellm.types.mcp import MCPAuth - - scope = { + @staticmethod + def _bearer_scope(token: str) -> dict[str, object]: + return { "type": "http", "method": "POST", "path": "/mcp/delegated_oauth_server", - "headers": [(b"authorization", b"Bearer upstream-pkce-token")], + "headers": [(b"authorization", f"Bearer {token}".encode())], } + @staticmethod + def _jwt(issuer: str = "https://idp.example") -> str: + import jwt + + return jwt.encode({"iss": issuer, "sub": "jwt-user"}, "test-signing-secret", algorithm="HS256") + + @staticmethod + @contextlib.contextmanager + def _delegate_admission( + admission: AsyncMock, allowed_server_ids: Sequence[str] = ("test-server-id",) + ) -> Iterator[None]: + from litellm.types.mcp import MCPAuth + with ( - patch( - "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", - new_callable=AsyncMock, - side_effect=HTTPException(status_code=401, detail="Invalid API key"), - ) as mock_auth, + patch("litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", admission), patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( auth_type=MCPAuth.oauth2, delegate_auth_to_upstream=True, ) - with pytest.raises(HTTPException) as exc_info: - await MCPRequestHandler.process_mcp_request(scope) + mock_mgr.get_allowed_mcp_servers = AsyncMock(return_value=list(allowed_server_ids)) + yield - assert exc_info.value.status_code == 401 - mock_auth.assert_awaited_once() + @staticmethod + @contextlib.contextmanager + def _jwt_auth_enabled(auth_jwt: AsyncMock) -> Iterator[None]: + from litellm.proxy import proxy_server + from litellm.proxy._types import LiteLLM_JWTAuth + + with ( + patch.dict(proxy_server.general_settings, {"enable_jwt_auth": True}), + patch.object(proxy_server, "premium_user", True), + patch.object(proxy_server, "user_custom_auth", None), + patch.object(proxy_server.jwt_handler, "auth_jwt", auth_jwt), + patch.object(proxy_server.jwt_handler, "litellm_jwtauth", LiteLLM_JWTAuth(), create=True), + patch.dict(os.environ), + ): + os.environ.pop("JWT_ISSUER", None) + yield + + async def test_delegate_forwards_unknown_sk_token_after_key_table_miss(self) -> None: + """An sk- token the key table does not hold, such as a provider API key, belongs to the upstream: + it is forwarded anonymously rather than rejected for its prefix.""" + from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError + + admission: Final = AsyncMock() + with ( + TestMCPDelegateAuthToUpstream._delegate_admission(admission), + patch( + "litellm.proxy.auth.resolvers.store.IdentityStore.resolve", + new_callable=AsyncMock, + side_effect=KeyNotFoundError("hashed-token"), + ), + ): + auth_result, _, _, _, oauth2_headers, _ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope("sk-proj-upstream-provider-key") + ) + + assert auth_result.api_key is None + assert auth_result.mcp_forwards_caller_authorization is True + assert oauth2_headers == {"Authorization": "Bearer sk-proj-upstream-provider-key"} + admission.assert_not_awaited() + + @pytest.mark.parametrize( + "lookup_effect", + [None, RuntimeError("key table unreachable")], + ids=["key-held", "lookup-fault"], + ) + async def test_delegate_litellm_key_refusal_propagates(self, lookup_effect: Exception | None) -> None: + """A key the table holds, or one the lookup could not check, takes normal admission, so its + refusal reaches the caller instead of becoming an anonymous request that forwards the key.""" + refusal: Final = ProxyException( + message="Budget has been exceeded", type="budget_exceeded", param=None, code=400 + ) + with ( + TestMCPDelegateAuthToUpstream._delegate_admission(AsyncMock(side_effect=refusal)), + patch( + "litellm.proxy.auth.resolvers.store.IdentityStore.resolve", + new_callable=AsyncMock, + side_effect=lookup_effect, + ), + ): + with pytest.raises(ProxyException) as exc_info: + await MCPRequestHandler.process_mcp_request(TestMCPDelegateAuthToUpstream._bearer_scope("sk-held-key")) + + assert exc_info.value is refusal + + async def test_delegate_master_key_is_admitted_and_never_forwarded(self) -> None: + """The master key is not in the key table, so only recognizing it directly stops it from being + treated as the upstream's token and sent to the upstream server.""" + from litellm.proxy import proxy_server + from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError + + admission: Final = AsyncMock(return_value=UserAPIKeyAuth(api_key="sk-master-secret", user_role="proxy_admin")) + with ( + patch.object(proxy_server, "master_key", "sk-master-secret"), + TestMCPDelegateAuthToUpstream._delegate_admission(admission), + patch( + "litellm.proxy.auth.resolvers.store.IdentityStore.resolve", + new_callable=AsyncMock, + side_effect=KeyNotFoundError("hashed-master-key"), + ), + ): + auth_result, _, _, _, oauth2_headers, _ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope("sk-master-secret") + ) + + admission.assert_awaited_once() + assert auth_result.mcp_forwards_caller_authorization is False + assert oauth2_headers is None + + @pytest.mark.parametrize( + "signing_secret, expires_in, forwarded", + [ + ("sk-master-secret", timedelta(hours=1), False), + ("sk-master-secret", timedelta(hours=-1), False), + ("upstream-idp-signing-secret", timedelta(hours=1), True), + ], + ids=["master-key-signed", "master-key-signed-expired", "signed-elsewhere"], + ) + async def test_delegate_master_key_signed_jwt_is_never_forwarded( + self, signing_secret: str, expires_in: timedelta, forwarded: bool + ) -> None: + """LiteLLM signs dashboard and OAuth tokens with the master key, so such a JWT is LiteLLM's own even + once expired: it takes normal admission and never reaches the upstream. A JWT signed by anyone + else is the upstream's token.""" + import jwt + + from litellm.proxy import proxy_server + + token: Final = jwt.encode( + {"sub": "dashboard-user", "exp": datetime.now(timezone.utc) + expires_in}, signing_secret, algorithm="HS256" + ) + admission: Final = AsyncMock(return_value=UserAPIKeyAuth(user_id="dashboard-user")) + with ( + patch.object(proxy_server, "master_key", "sk-master-secret"), + TestMCPDelegateAuthToUpstream._delegate_admission(admission), + ): + auth_result, *_ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope(token) + ) + + assert auth_result.mcp_forwards_caller_authorization is forwarded + assert admission.await_count == (0 if forwarded else 1) + + @pytest.mark.parametrize( + "token, ui_token_lookup", + [ + ("llm_env_sealed-bridge-envelope", None), + ("llm_refresh_sealed-refresh-envelope", None), + ("v2:gcm:sealed-credential-blob", None), + ("dashboard-login-token", lambda: MagicMock(return_value=UserAPIKeyAuth(user_id="dashboard-user"))), + ("unreadable-dashboard-token", lambda: MagicMock(side_effect=ValueError("Invalid hash key"))), + ], + ids=["bridge-envelope", "refresh-envelope", "gcm-blob", "dashboard-token", "unreadable-dashboard-token"], + ) + async def test_delegate_sealed_litellm_token_takes_normal_admission( + self, token: str, ui_token_lookup: Callable[[], MagicMock] | None + ) -> None: + """A token only LiteLLM issues is LiteLLM's own, even one it fails to read, so it goes through + normal admission and is never marked for forwarding upstream.""" + from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken + + admission: Final = AsyncMock(return_value=UserAPIKeyAuth(user_id="litellm-user")) + ui_lookup: Final = ( + patch.object(ExperimentalUIJWTToken, "get_key_object_from_ui_hash_key", ui_token_lookup()) + if ui_token_lookup is not None + else contextlib.nullcontext() + ) + with TestMCPDelegateAuthToUpstream._delegate_admission(admission), ui_lookup: + auth_result, *_ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope(token) + ) + + admission.assert_awaited_once() + assert auth_result.mcp_forwards_caller_authorization is False + + @staticmethod + def _unknown_user_refusal() -> ProxyException: + from litellm.types.proxy.auth.auth_checks import UserNotFoundError + + refusal: Final = ProxyException(message="User doesn't exist in db", type="auth_error", param=None, code=401) + refusal.__cause__ = UserNotFoundError(user_id="jwt-user") + return refusal + + @pytest.mark.parametrize( + "auth_jwt_effect, admission, allowed_server_ids, expected_user_id", + [ + ( + None, + lambda: AsyncMock(return_value=UserAPIKeyAuth(user_id="jwt-user", jwt_claims={"sub": "jwt-user"})), + ["test-server-id"], + "jwt-user", + ), + ( + None, + lambda: AsyncMock(return_value=UserAPIKeyAuth(user_id="jwt-user", jwt_claims={"sub": "jwt-user"})), + [], + None, + ), + ( + None, + lambda: AsyncMock(side_effect=TestMCPDelegateAuthToUpstream._unknown_user_refusal()), + ["test-server-id"], + None, + ), + ( + ValueError("Signature verification failed"), + lambda: AsyncMock(return_value=UserAPIKeyAuth(user_id="jwt-user")), + ["test-server-id"], + None, + ), + ], + ids=["granted-user", "user-not-granted", "user-unknown-to-litellm", "signature-rejected"], + ) + async def test_delegate_jwt_is_forwarded_as_granted_user_or_anonymously( + self, + auth_jwt_effect: Exception | None, + admission: Callable[[], AsyncMock], + allowed_server_ids: Sequence[str], + expected_user_id: str | None, + ) -> None: + """A JWT LiteLLM also accepts admits that user when granted the server; a user who is not + granted, unknown to LiteLLM, or whose signature fails is admitted anonymously. The JWT is always + forwarded, because on this server it is the upstream's token.""" + token: Final = TestMCPDelegateAuthToUpstream._jwt() + with ( + TestMCPDelegateAuthToUpstream._jwt_auth_enabled(AsyncMock(side_effect=auth_jwt_effect)), + TestMCPDelegateAuthToUpstream._delegate_admission(admission(), allowed_server_ids), + ): + auth_result, _, _, _, oauth2_headers, _ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope(token) + ) + + assert auth_result.user_id == expected_user_id + assert auth_result.mcp_forwards_caller_authorization is True + assert oauth2_headers == {"Authorization": f"Bearer {token}"} + + @pytest.mark.parametrize( + "refusal", + [ + ProxyException(message="Budget has been exceeded", type="budget_exceeded", param=None, code=400), + ProxyException(message="Team=t1 is blocked", type="auth_error", param=None, code=401), + ], + ids=["over-budget", "blocked-team"], + ) + async def test_delegate_jwt_refusal_after_valid_signature_propagates(self, refusal: ProxyException) -> None: + """Once the signature verifies, LiteLLM knows who the caller is, so a budget or an explicit + disallowment is enforced and the request fails instead of continuing anonymously.""" + with ( + TestMCPDelegateAuthToUpstream._jwt_auth_enabled(AsyncMock(return_value={"sub": "jwt-user"})), + TestMCPDelegateAuthToUpstream._delegate_admission(AsyncMock(side_effect=refusal)), + ): + with pytest.raises(ProxyException) as exc_info: + await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope(TestMCPDelegateAuthToUpstream._jwt()) + ) + + assert exc_info.value is refusal + + @staticmethod + @contextlib.contextmanager + def _idp_auth_enabled( + mechanism: Literal["oauth2", "oauth2_proxy"], verify: AsyncMock, premium: bool = True + ) -> Iterator[None]: + from litellm.proxy import proxy_server + from litellm.proxy.auth.oauth2_check import Oauth2Handler + + settings: Final = {"enable_oauth2_auth": True} if mechanism == "oauth2" else {"enable_oauth2_proxy_auth": True} + verifier: Final = ( + patch.object(Oauth2Handler, "check_oauth2_token", verify) + if mechanism == "oauth2" + else patch("litellm.proxy.auth.oauth2_proxy_hook.handle_oauth2_proxy_request", verify) + ) + with ( + patch.dict(proxy_server.general_settings, settings), + patch.object(proxy_server, "premium_user", premium), + patch.object(proxy_server, "user_custom_auth", None), + verifier, + ): + yield + + @pytest.mark.parametrize( + "verify_effect, allowed_server_ids, expected_user_id", + [ + (None, ["test-server-id"], "idp-user"), + (None, [], None), + (ValueError("Missing trusted proxy header"), ["test-server-id"], None), + ], + ids=["granted-user", "user-not-granted", "identity-rejected"], + ) + async def test_delegate_proxy_header_identity_follows_the_jwt_rules( + self, + verify_effect: Exception | None, + allowed_server_ids: Sequence[str], + expected_user_id: str | None, + ) -> None: + """Trusted reverse-proxy headers admit a granted user by name, and a user who is not granted or not + vouched for anonymously. The bearer is forwarded either way.""" + admission: Final = AsyncMock(return_value=UserAPIKeyAuth(user_id="idp-user")) + with ( + TestMCPDelegateAuthToUpstream._idp_auth_enabled("oauth2_proxy", AsyncMock(side_effect=verify_effect)), + TestMCPDelegateAuthToUpstream._delegate_admission(admission, allowed_server_ids), + ): + auth_result, _, _, _, oauth2_headers, _ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope("opaque-idp-token") + ) + + assert auth_result.user_id == expected_user_id + assert auth_result.mcp_forwards_caller_authorization is True + assert oauth2_headers == {"Authorization": "Bearer opaque-idp-token"} + + @pytest.mark.parametrize( + "provider_status, active, allowed_server_ids, expected_user_id", + [ + (200, True, ["test-server-id"], "idp-user"), + (200, True, [], None), + (200, False, ["test-server-id"], None), + (401, True, ["test-server-id"], None), + ], + ids=["granted-user", "user-not-granted", "token-inactive", "provider-rejects-token"], + ) + async def test_delegate_oauth2_introspects_once_and_follows_the_jwt_rules( + self, provider_status: int, active: bool, allowed_server_ids: Sequence[str], expected_user_id: str | None + ) -> None: + """The provider is asked about the token once per request. A token it vouches for admits a granted + user by name, and one it reports inactive or rejects outright is forwarded anonymously instead of + failing the request.""" + import httpx + + from litellm.proxy import proxy_server + from litellm.types.mcp import MCPAuth + + async def introspect(outbound: httpx.Request, **kwargs: object) -> httpx.Response: + return httpx.Response(provider_status, request=outbound, json={"active": active, "sub": "idp-user"}) + + with ( + patch.dict(proxy_server.general_settings, {"enable_oauth2_auth": True}), + patch.object(proxy_server, "premium_user", True), + patch.object(proxy_server, "user_custom_auth", None), + patch.object(proxy_server, "master_key", "sk-fixture-master"), + patch.object(proxy_server, "prisma_client", None), + patch.object(proxy_server, "user_api_key_cache", DualCache()), + patch.dict( + os.environ, + { + "OAUTH_TOKEN_INFO_ENDPOINT": "https://idp.fixture.test/introspect", + "OAUTH_CLIENT_ID": "fixture-client", + "OAUTH_CLIENT_SECRET": "fixture-client-secret", + }, + ), + patch("httpx.AsyncClient.send", side_effect=introspect) as network, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + ): + mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( + auth_type=MCPAuth.oauth2, delegate_auth_to_upstream=True + ) + mock_mgr.get_allowed_mcp_servers = AsyncMock(return_value=list(allowed_server_ids)) + auth_result, _, _, _, oauth2_headers, _ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope("opaque-idp-token") + ) + + assert network.call_count == 1 + assert auth_result.user_id == expected_user_id + assert auth_result.mcp_forwards_caller_authorization is True + assert oauth2_headers == {"Authorization": "Bearer opaque-idp-token"} + + async def test_delegate_oauth2_rejection_admits_anonymously_whether_or_not_auth_wraps_it(self) -> None: + """A provider rejection is the provider declining to vouch, so the bearer is forwarded anonymously + even when it reaches admission as the bare validation error rather than a wrapped auth error.""" + from litellm.proxy.auth.oauth2_check import OAuth2TokenValidationError + + with ( + TestMCPDelegateAuthToUpstream._idp_auth_enabled("oauth2", AsyncMock()), + TestMCPDelegateAuthToUpstream._delegate_admission( + AsyncMock(side_effect=OAuth2TokenValidationError("Token is not active")) + ), + ): + auth_result, _, _, _, oauth2_headers, _ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope("opaque-idp-token") + ) + + assert auth_result.user_id is None + assert auth_result.mcp_forwards_caller_authorization is True + assert oauth2_headers == {"Authorization": "Bearer opaque-idp-token"} + + @pytest.mark.parametrize("mechanism", ["oauth2", "oauth2_proxy"]) + async def test_delegate_idp_user_refusal_propagates(self, mechanism: Literal["oauth2", "oauth2_proxy"]) -> None: + """Once the provider vouches for the caller, LiteLLM's budget and permission refusals apply.""" + refusal: Final = ProxyException( + message="Budget has been exceeded", type="budget_exceeded", param=None, code=400 + ) + with ( + TestMCPDelegateAuthToUpstream._idp_auth_enabled(mechanism, AsyncMock()), + TestMCPDelegateAuthToUpstream._delegate_admission(AsyncMock(side_effect=refusal)), + ): + with pytest.raises(ProxyException) as exc_info: + await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope("opaque-idp-token") + ) + + assert exc_info.value is refusal + + async def test_delegate_oauth2_without_premium_never_calls_the_provider(self) -> None: + """OAuth2 introspection is enterprise-only, so without a license the bearer is the upstream's alone.""" + introspect: Final = AsyncMock() + with ( + TestMCPDelegateAuthToUpstream._idp_auth_enabled("oauth2", introspect, premium=False), + TestMCPDelegateAuthToUpstream._delegate_admission( + AsyncMock(return_value=UserAPIKeyAuth(user_id="idp-user")) + ), + ): + auth_result, *_ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope("opaque-idp-token") + ) + + assert auth_result.user_id is None + assert auth_result.mcp_forwards_caller_authorization is True + introspect.assert_not_awaited() + + async def test_delegate_jwt_from_unconfigured_issuer_is_forwarded_without_jwt_auth(self) -> None: + """A JWT from an issuer LiteLLM's JWT auth is not configured for never reaches JWT auth.""" + auth_jwt: Final = AsyncMock(return_value={"sub": "jwt-user"}) + admission: Final = AsyncMock(return_value=UserAPIKeyAuth(user_id="jwt-user")) + with ( + TestMCPDelegateAuthToUpstream._jwt_auth_enabled(auth_jwt), + patch.dict(os.environ, {"JWT_ISSUER": "https://litellm-idp.example"}), + TestMCPDelegateAuthToUpstream._delegate_admission(admission), + ): + auth_result, *_ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope( + TestMCPDelegateAuthToUpstream._jwt(issuer="https://upstream-idp.example") + ) + ) + + assert auth_result.user_id is None + assert auth_result.mcp_forwards_caller_authorization is True + auth_jwt.assert_not_awaited() + + @pytest.mark.parametrize( + "admission, expected_user_id", + [ + ( + lambda: AsyncMock(return_value=UserAPIKeyAuth(user_id="custom-user", api_key="upstream-custom-token")), + "custom-user", + ), + ( + lambda: AsyncMock( + side_effect=ProxyException(message="custom auth rejected", type="auth_error", param=None, code=401) + ), + None, + ), + ], + ids=["accepted", "rejected"], + ) + async def test_delegate_custom_auth_decides_alone( + self, admission: Callable[[], AsyncMock], expected_user_id: str | None + ) -> None: + """With custom auth configured, its answer alone decides: accepted admits that user, any failure + admits anonymously. The bearer is forwarded either way.""" + from litellm.proxy import proxy_server + + with ( + patch.object(proxy_server, "user_custom_auth", AsyncMock()), + TestMCPDelegateAuthToUpstream._delegate_admission(admission()), + ): + auth_result, _, _, _, oauth2_headers, _ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope("upstream-custom-token") + ) + + assert auth_result.user_id == expected_user_id + assert auth_result.mcp_forwards_caller_authorization is True + assert oauth2_headers == {"Authorization": "Bearer upstream-custom-token"} + + def test_forwarding_marker_cannot_be_forged_from_input(self) -> None: + """Only admission may mark a caller's Authorization for forwarding; caller-shaped input never can.""" + assert UserAPIKeyAuth(mcp_forwards_caller_authorization=True).mcp_forwards_caller_authorization is False + assert ( + UserAPIKeyAuth.model_validate({"mcp_forwards_caller_authorization": True}).mcp_forwards_caller_authorization + is False + ) async def test_delegate_off_still_requires_litellm_auth(self): """ @@ -2753,7 +3213,12 @@ class TestMCPDelegateAuthToUpstream: await MCPRequestHandler.process_mcp_request(scope) assert exc_info.value.status_code == 401 - async def test_delegate_mixed_targets_fail_closed(self): + @pytest.mark.parametrize( + "bearer_headers", + [[], [(b"authorization", b"Bearer upstream-pkce-token")]], + ids=["no-bearer", "upstream-bearer"], + ) + async def test_delegate_mixed_targets_fail_closed(self, bearer_headers: list[tuple[bytes, bytes]]) -> None: """ x-mcp-servers can list multiple targets. If ANY of them does not opt in to delegate_auth_to_upstream, the bypass must NOT fire — otherwise an @@ -2769,6 +3234,7 @@ class TestMCPDelegateAuthToUpstream: "path": "/mcp", "headers": [ (b"x-mcp-servers", b"delegated_oauth,plain_oauth"), + *bearer_headers, ], } @@ -2863,9 +3329,8 @@ class TestMCPDelegateAuthToUpstream: async def test_authorization_bearer_on_delegate_server_establishes_litellm_principal(self): """ - A bare Authorization bearer now follows normal LiteLLM admission. A - separate x-litellm-api-key is required when Authorization is intended - for the upstream server. + A bare sk- bearer the key table may hold takes normal LiteLLM admission + and is never marked for forwarding upstream. """ from litellm.types.mcp import MCPAuth @@ -2898,6 +3363,7 @@ class TestMCPDelegateAuthToUpstream: ) = await MCPRequestHandler.process_mcp_request(scope) assert isinstance(auth_result, UserAPIKeyAuth) assert auth_result.user_id == "real-user" + assert auth_result.mcp_forwards_caller_authorization is False assert oauth2_headers.get("Authorization") == "Bearer sk-9876" mock_auth.assert_awaited_once() @@ -3079,8 +3545,9 @@ class TestMCPDelegateAuthToUpstream: mock_auth.assert_awaited_once() assert auth.api_key is None - async def test_get_allowed_servers_excludes_legacy_delegates(self): - """Legacy delegated servers are never added to anonymous access.""" + async def test_get_allowed_servers_includes_interactive_legacy_delegates_for_anonymous(self) -> None: + """Anonymous callers reach interactive legacy delegates, never an M2M server whose stored + client credentials would let them act as LiteLLM's service account.""" from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -3118,7 +3585,7 @@ class TestMCPDelegateAuthToUpstream: ): result = await manager.get_allowed_mcp_servers(None) - assert result == [] + assert result == ["pkce-server"] async def test_true_passthrough_skips_litellm_auth_anonymously(self): """auth_type=true_passthrough performs no admission auth: the caller's Authorization is an @@ -7644,9 +8111,7 @@ class TestMCPDcrBridgeDualCredential: new_callable=AsyncMock, return_value=UserAPIKeyAuth(user_id=presented_user, api_key="any-hash"), ), - patch( - "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), ): mock_mgr.get_mcp_server_by_name.return_value = self._DELEGATE._bridge_delegate_server() @@ -7697,7 +8162,9 @@ class TestMCPDcrBridgeDualCredential: @pytest.mark.parametrize("root_path", ["", "/litellm"]) @pytest.mark.parametrize("invalid_token", [False, True]) def test_gateway_challenge_metadata_contains_the_public_resource( - public_path: str, root_path: str, invalid_token: bool, + public_path: str, + root_path: str, + invalid_token: bool, ) -> None: from mcp.shared.auth_utils import check_resource_allowed from starlette.requests import Request @@ -7710,8 +8177,12 @@ def test_gateway_challenge_metadata_contains_the_public_resource( ) scope: Final[Scope] = { - "type": "http", "method": "POST", "scheme": "https", "path": "/mcp", - "root_path": root_path, "_original_path": f"{root_path}{public_path}", + "type": "http", + "method": "POST", + "scheme": "https", + "path": "/mcp", + "root_path": root_path, + "_original_path": f"{root_path}{public_path}", "headers": [(b"host", b"gateway.example.com")], } request: Final = Request(scope) @@ -7720,8 +8191,7 @@ def test_gateway_challenge_metadata_contains_the_public_resource( metadata_url: Final = header.split('resource_metadata="', 1)[1].split('"', 1)[0] toolset: Final = public_path in ("/toolset/analysis/mcp", "/analysis/mcp") assert metadata_url == ( - f"https://gateway.example.com{root_path}/.well-known/oauth-protected-resource" - + ("" if toolset else "/mcp") + f"https://gateway.example.com{root_path}/.well-known/oauth-protected-resource" + ("" if toolset else "/mcp") ) metadata: Final = ( oauth_protected_resource_root(request) if toolset else _build_aggregate_protected_resource_response(request) @@ -7924,9 +8394,7 @@ class TestAggregateGatewayDcrChallenge: ): with ( patch(self._AUTH_PATCH_TARGET, side_effect=self._auth_401()), - patch( - "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = server with pytest.raises(HTTPException) as exc_info: @@ -7972,9 +8440,7 @@ class TestAggregateGatewayDcrChallenge: with ( patch.dict(os.environ, {"SERVER_ROOT_PATH": "/litellm"}), patch(self._AUTH_PATCH_TARGET, side_effect=self._auth_401()), - patch( - "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = server with pytest.raises(HTTPException) as exc_info: @@ -8005,9 +8471,7 @@ class TestAggregateGatewayDcrChallenge: ): with ( patch(self._AUTH_PATCH_TARGET, side_effect=self._auth_401()), - patch( - "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = resolved mock_mgr.get_mcp_server_by_identifier.return_value = resolved @@ -8052,9 +8516,7 @@ class TestAggregateGatewayDcrChallenge: (None, None), ] for resolved, expected in cases: - with patch( - "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr: + with patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr: mock_mgr.get_mcp_server_by_identifier.return_value = resolved assert _gateway_dcr_challenge_target("/mcp/srv", None, None) == expected, resolved assert _gateway_dcr_challenge_target("/mcp/a,b", None, None) is None @@ -10950,14 +11412,15 @@ async def test_catalog_refresh_uses_current_virtual_key_policy_and_keeps_session from litellm.proxy import proxy_server from litellm.types.mcp_server.mcp_toolset import MCPToolset - permission = LiteLLM_ObjectPermissionTable( - object_permission_id="current-policy", mcp_toolsets=["session-toolset"] - ) + permission = LiteLLM_ObjectPermissionTable(object_permission_id="current-policy", mcp_toolsets=["session-toolset"]) toolset: Final = MCPToolset( - toolset_id="session-toolset", toolset_name="session-toolset", + toolset_id="session-toolset", + toolset_name="session-toolset", tools=[{"server_id": "current-server", "tool_name": "read"}], ) - tables: Final = SimpleNamespace(litellm_mcptoolsettable=SimpleNamespace(find_many=AsyncMock(return_value=[toolset]))) + tables: Final = SimpleNamespace( + litellm_mcptoolsettable=SimpleNamespace(find_many=AsyncMock(return_value=[toolset])) + ) monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=tables, writer_db=tables)) monkeypatch.setattr(proxy_server, "user_api_key_cache", DualCache()) current = UserAPIKeyAuth( @@ -10984,10 +11447,13 @@ async def test_catalog_refresh_uses_current_virtual_key_policy_and_keeps_session caller.mcp_toolset_id = "session-toolset" refreshed = await MCPRequestHandler.refresh_catalog_authority(caller) reload_key.assert_awaited_once_with("owned-key-hash", check_db_only=True) - assert refreshed.object_permission == permission.model_copy(update={ - "mcp_servers": ["current-server"], "mcp_toolsets": [], - "mcp_tool_permissions": {"current-server": ["read"]}, - }) + assert refreshed.object_permission == permission.model_copy( + update={ + "mcp_servers": ["current-server"], + "mcp_toolsets": [], + "mcp_tool_permissions": {"current-server": ["read"]}, + } + ) assert refreshed.object_permission_id == "current-policy" assert (refreshed.team_id, refreshed.org_id, refreshed.project_id, refreshed.user_id) == ( "new-team", @@ -12091,9 +12557,7 @@ async def test_personal_key_guardrail_auth_preserves_native_tool_name_matching_s from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: - return _personal_key_team_granting_tools( - team_id, ("math-multiply",) if team_id == "team-b" else ("other",) - ) + return _personal_key_team_granting_tools(team_id, ("math-multiply",) if team_id == "team-b" else ("other",)) monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) @@ -12132,9 +12596,7 @@ async def test_personal_key_mcp_billing_inherits_adopted_organization_limits( ) async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: - return _personal_key_team_granting_tools(team_id, ("divide",)).model_copy( - update={"organization_id": "org-1"} - ) + return _personal_key_team_granting_tools(team_id, ("divide",)).model_copy(update={"organization_id": "org-1"}) org_loader: Final = AsyncMock(return_value=organization) monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) @@ -12180,16 +12642,12 @@ async def test_personal_key_mcp_billing_keeps_key_organization_limits( ) async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: - return _personal_key_team_granting_tools(team_id, ("divide",)).model_copy( - update={"organization_id": "org-1"} - ) + return _personal_key_team_granting_tools(team_id, ("divide",)).model_copy(update={"organization_id": "org-1"}) org_loader: Final = AsyncMock(return_value=organization) monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) monkeypatch.setattr(auth_checks, "get_org_object_for_request", org_loader) - auth: Final = personal_key_owner_teams.model_copy( - update={"org_id": "org-k", "organization_rpm_limit": 9} - ) + auth: Final = personal_key_owner_teams.model_copy(update={"org_id": "org-k", "organization_rpm_limit": 9}) server: Final = global_mcp_server_manager.registry["srv-b"] with patch.object(global_mcp_server_manager, "get_mcp_server_from_tool_name", return_value=server): @@ -12321,9 +12779,7 @@ async def test_personal_key_mcp_billing_uses_fresh_team_and_membership_for_stric } ) - async def _get_team_membership( - user_id: str, team_id: str, **kwargs: object - ) -> LiteLLM_TeamMembership: + async def _get_team_membership(user_id: str, team_id: str, **kwargs: object) -> LiteLLM_TeamMembership: database_value: Final = bool(kwargs.get("check_db_only")) return LiteLLM_TeamMembership( user_id=user_id, diff --git a/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py b/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py index 78c37184d6c..621a7b65005 100644 --- a/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py +++ b/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py @@ -3,6 +3,7 @@ import contextlib import contextvars import json import os +from collections.abc import Coroutine, Iterator, Sequence from datetime import datetime, timedelta from types import SimpleNamespace from typing import Callable, Final, Literal @@ -669,6 +670,75 @@ def test_prepare_mcp_server_headers_legacy_delegate_strips_repeated_admission_ke assert extra_headers is None +@pytest.mark.parametrize( + "caller_fields", + [{}, {"user_id": "jwt-user", "api_key": "upstream-jwt", "jwt_claims": {"sub": "jwt-user"}}], + ids=["anonymous", "admitted-jwt-user"], +) +def test_prepare_mcp_server_headers_legacy_delegate_forwards_marked_upstream_authorization( + caller_fields: dict[str, object], +) -> None: + """When admission marked the caller's Authorization as the upstream token, a legacy delegate forwards + it, including a JWT that also admitted the caller and would otherwise be stripped as an admission JWT.""" + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( + _prepare_mcp_server_headers, + ) + + server = MCPServer( + server_id="legacy-delegate-marked", + name="legacy-delegate", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + delegate_auth_to_upstream=True, + ) + caller: Final = UserAPIKeyAuth.model_validate(caller_fields).model_copy( + update={"mcp_forwards_caller_authorization": True} + ) + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=None, + mcp_auth_header=None, + oauth2_headers={"Authorization": "Bearer upstream-jwt"}, + raw_headers={"authorization": "Bearer upstream-jwt"}, + user_api_key_auth=caller, + ) + + assert server_auth_header is None + assert extra_headers == {"Authorization": "Bearer upstream-jwt"} + + +def test_prepare_mcp_server_headers_forwarding_marker_is_ignored_off_legacy_delegates() -> None: + """The forwarding marker only applies to legacy delegates, so it can never carry a LiteLLM key to an + oauth_delegate upstream.""" + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( + _prepare_mcp_server_headers, + ) + + server = MCPServer( + server_id="oauth-delegate-marked", + name="oauth-delegate", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth_delegate, + ) + caller: Final = UserAPIKeyAuth(api_key="sk-litellm-key").model_copy( + update={"mcp_forwards_caller_authorization": True} + ) + + _, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=None, + mcp_auth_header=None, + oauth2_headers={"Authorization": "Bearer sk-litellm-key"}, + raw_headers={"authorization": "Bearer sk-litellm-key"}, + user_api_key_auth=caller, + ) + + assert extra_headers is None + + def test_prepare_mcp_server_headers_m2m_skips_authorization_from_raw_extra_headers(): """M2M must not merge caller Authorization from raw_headers when extra_headers lists it.""" try: @@ -7265,15 +7335,19 @@ async def test_legacy_delegate_bare_token_is_not_probed_upstream(): # test-qual @pytest.mark.asyncio -async def test_legacy_delegate_dual_credentials_are_not_probed_upstream(): # test-quality-ok: this removed security-sensitive egress has no return value; non-invocation is the contract - """A separate upstream bearer never triggers the removed legacy probe.""" +@pytest.mark.usefixtures("httpx_transport") +async def test_legacy_delegate_dual_credentials_probe_only_the_upstream_token() -> None: + """With a separate x-litellm-api-key, the preflight probes the upstream with its own token and never + with the LiteLLM key.""" + import respx + from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.mcp_server.server import ( _check_passthrough_upstream_auth, ) - server = _delegate_auth_mcp_server() - scope = _delegate_scope( + server: Final = _delegate_auth_mcp_server() + scope: Final = _delegate_scope( [ (b"x-litellm-api-key", b"sk-litellm-proxy-key"), (b"authorization", b"Bearer upstream-token"), @@ -7281,15 +7355,14 @@ async def test_legacy_delegate_dual_credentials_are_not_probed_upstream(): # te ) with ( + _patch_delegate_resolver(server, "delegate_test"), patch( # test-quality-ok: isolate authorized-server resolution so this test targets the preflight boundary "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[server]), ), - patch( # test-quality-ok: the removed probe call is the security regression under test - "litellm.proxy.mcp_server.server._probe_upstream_auth", - new=AsyncMock(), - ) as probe, + respx.mock as upstream, ): + probe: Final = upstream.post(server.url).respond(200) await _check_passthrough_upstream_auth( scope=scope, user_api_key_auth=UserAPIKeyAuth(user_id="admitted-user"), @@ -7300,7 +7373,160 @@ async def test_legacy_delegate_dual_credentials_are_not_probed_upstream(): # te raw_headers={"x-litellm-api-key": "sk-litellm-proxy-key", "Authorization": "Bearer upstream-token"}, ) - probe.assert_not_awaited() + assert probe.call_count == 1 + sent_headers: Final = probe.calls[0].request.headers + assert sent_headers["Authorization"] == "Bearer upstream-token" + assert all("sk-litellm-proxy-key" not in value for value in sent_headers.values()) + + +def _expired_delegate_token_preflight(mcp_servers: Sequence[str]) -> Coroutine[object, object, None]: + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( + _check_passthrough_upstream_auth, + ) + + return _check_passthrough_upstream_auth( + scope=_delegate_scope([(b"authorization", b"Bearer expired-upstream-token")]), + user_api_key_auth=UserAPIKeyAuth().model_copy(update={"mcp_forwards_caller_authorization": True}), + mcp_servers=list(mcp_servers), + client_ip=None, + oauth2_headers={"Authorization": "Bearer expired-upstream-token"}, + mcp_server_auth_headers=None, + raw_headers={"authorization": "Bearer expired-upstream-token"}, + ) + + +@contextlib.contextmanager +def _rejecting_delegate_upstream() -> Iterator[None]: + server: Final = _delegate_auth_mcp_server() + with ( + _patch_delegate_resolver(server, "delegate_test"), + patch( # test-quality-ok: isolate authorized-server resolution so this test targets the preflight boundary + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", + new=AsyncMock(return_value=[server]), + ), + patch( # test-quality-ok: stands in for the upstream rejecting an expired token + "litellm.proxy.mcp_server.server._probe_upstream_auth", + new=AsyncMock(return_value=(401, 'Bearer error="invalid_token"')), + ), + ): + yield + + +@pytest.mark.asyncio +async def test_legacy_delegate_expired_upstream_token_gets_reauthorization_challenge() -> None: + """An expired upstream token on a single-server route gets the RFC 9728 re-authorization challenge + for the requested name, so the client signs in again instead of seeing an empty tool list.""" + with _rejecting_delegate_upstream(), pytest.raises(HTTPException) as exc_info: + await _expired_delegate_token_preflight(["delegate_test"]) + + assert exc_info.value.status_code == 401 + challenge: Final = exc_info.value.headers["www-authenticate"] + assert 'error="invalid_token"' in challenge + assert ( + 'resource_metadata="http://localhost:4000/.well-known/oauth-protected-resource/mcp/delegate_test"' in challenge + ) + + +@pytest.mark.asyncio +async def test_legacy_delegate_multi_server_scope_is_not_failed_by_one_upstream_token() -> None: + """A multi-server scope is not probed, so one rejected upstream token cannot fail the whole connect.""" + with _rejecting_delegate_upstream(): + assert await _expired_delegate_token_preflight(["delegate_test", "other_server"]) is None + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport") +async def test_legacy_delegate_probe_checks_the_callers_token_not_a_static_authorization() -> None: + """The probe is what checks a delegate caller's own token, so a static Authorization the server sends on + tool calls cannot stand in for it: a junk token still gets the re-authorization challenge.""" + import respx + + server: Final = _delegate_auth_mcp_server().model_copy( + update={"static_headers": {"Authorization": "Bearer operator-token"}} + ) + with ( + _patch_delegate_resolver(server, "delegate_test"), + patch( # test-quality-ok: isolate authorized-server resolution so this test targets the preflight boundary + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", + new=AsyncMock(return_value=[server]), + ), + respx.mock as upstream, + ): + upstream.post(server.url).mock( + side_effect=lambda request: httpx.Response( + 200 if request.headers["Authorization"] == "Bearer operator-token" else 401 + ) + ) + outcome: Final = await asyncio.gather( + _expired_delegate_token_preflight(["delegate_test"]), return_exceptions=True + ) + + assert getattr(outcome[0], "status_code", None) == 401 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport") +@pytest.mark.parametrize( + "requested, expected_status", + [(("delegate_test",), 401), (("delegate_test", "other_server"), None)], + ids=["single-server", "two-requested-one-reachable"], +) +async def test_modern_interaction_probes_legacy_delegate_only_for_a_single_requested_server( + requested: tuple[str, ...], expected_status: int | None +) -> None: + """The single-server rule follows what the caller asked for, not what it can reach, so a request + naming two servers is not failed by one expired delegate token.""" + import respx + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server import server + from litellm.proxy.mcp_server.contracts import OperationContext + + target: Final = _delegate_auth_mcp_server() + scope: Final = _delegate_scope( + [ + (b"authorization", b"Bearer expired-upstream-token"), + (b"mcp-protocol-version", b"2026-07-28"), + (b"mcp-method", b"tools/call"), + (b"mcp-name", b"delegate-add"), + ] + ) + context: Final = OperationContext( + _caller=UserAPIKeyAuth().model_copy(update={"mcp_forwards_caller_authorization": True}), + mcp_servers=requested, + oauth2_headers={"Authorization": "Bearer expired-upstream-token"}, + raw_headers={"authorization": "Bearer expired-upstream-token"}, + ) + body: Final = json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "delegate-add", + "arguments": {"a": 2, "b": 3}, + "_meta": { + "io.modelcontextprotocol/protocolVersion": "2026-07-28", + "io.modelcontextprotocol/clientCapabilities": {}, + }, + }, + } + ).encode() + with ( + _patch_delegate_resolver(target, "delegate_test"), + patch( # test-quality-ok: isolate authorized-server resolution so this test targets the preflight boundary + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", + new=AsyncMock(return_value=[target]), + ), + respx.mock as upstream, + ): + upstream.post(target.url).respond(401, headers={"www-authenticate": 'Bearer error="invalid_token"'}) + outcome: Final = await asyncio.gather( + server._preflight_modern_interaction(scope, body, context), return_exceptions=True + ) + + assert getattr(outcome[0], "status_code", None) == expected_status @pytest.mark.parametrize( @@ -7537,39 +7763,47 @@ async def test_delegate_not_probed_when_named_only_via_server_id(): @pytest.mark.asyncio -async def test_delegate_probe_not_fanned_out_to_access_group_members(): - """A single access-group name passes the one-target route gate but must not fan - the delegate probe out to group-expanded member servers; the group name resolves - to no server under admission's resolver, so no probe fires.""" +@pytest.mark.usefixtures("httpx_transport") +async def test_delegate_probe_not_fanned_out_to_access_group_members() -> None: + """A single access-group name resolves to no server under admission's resolver, so the delegate probe is + not fanned out to the group's members and one member rejecting the token cannot fail the whole group.""" + import respx + from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.mcp_server.server import ( _check_passthrough_upstream_auth, ) - group_member = _delegate_auth_mcp_server() - + rejecting: Final = _delegate_auth_mcp_server() + accepting: Final = _delegate_auth_mcp_server("delegate-2").model_copy( + update={"name": "delegate_two", "url": "http://upstream-two:9402/mcp"} + ) with ( - _patch_delegate_resolver(group_member, "delegate_test"), - patch( + _patch_delegate_resolver(rejecting, "delegate_test"), + patch( # test-quality-ok: isolate authorized-server resolution so this test targets the preflight boundary "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", - new=AsyncMock(return_value=[group_member]), + new=AsyncMock(return_value=[rejecting, accepting]), ), - patch( - "litellm.proxy.mcp_server.server._probe_upstream_auth", - new=AsyncMock(return_value=(401, None)), - ) as probe, + respx.mock as upstream, ): - await _check_passthrough_upstream_auth( - scope=_delegate_scope([(b"authorization", b"Bearer bogus-token")]), - user_api_key_auth=UserAPIKeyAuth(), - mcp_servers=["prod_tools_group"], - client_ip=None, - oauth2_headers=None, - mcp_server_auth_headers=None, - raw_headers=None, + upstream.post(rejecting.url).respond(401, headers={"www-authenticate": 'Bearer error="invalid_token"'}) + upstream.post(accepting.url).respond(200) + assert ( + await _check_passthrough_upstream_auth( + scope=_delegate_scope( + [(b"x-litellm-api-key", b"sk-litellm-proxy-key"), (b"authorization", b"Bearer upstream-token")] + ), + user_api_key_auth=UserAPIKeyAuth(user_id="admitted-user"), + mcp_servers=["prod_tools_group"], + client_ip=None, + oauth2_headers={"Authorization": "Bearer upstream-token"}, + mcp_server_auth_headers=None, + raw_headers={"x-litellm-api-key": "sk-litellm-proxy-key", "Authorization": "Bearer upstream-token"}, + ) + is None ) - probe.assert_not_awaited() + assert upstream.calls.call_count == 0 @pytest.mark.asyncio @@ -8846,8 +9080,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_unknown_name_fails_ ] with patch( - "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp." - "MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ): @@ -8905,8 +9138,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_known_alias_returns ] with patch( - "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp." - "MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ): @@ -8938,8 +9170,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_mixed_known_and_unk ] with patch( - "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp." - "MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ): @@ -8971,8 +9202,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_access_group_resolv ] with patch( - "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp." - "MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=["id-b"], ): @@ -9240,15 +9470,11 @@ async def test_stateful_mcp_tracing_follows_each_message_exclusion( is_otel_v2_enabled.cache_clear() notified: Final = asyncio.Event() - async def call_tool( - ctx: ServerRequestContext[object, object], params: CallToolRequestParams - ) -> CallToolResult: + async def call_tool(ctx: ServerRequestContext[object, object], params: CallToolRequestParams) -> CallToolResult: with tracer.start_as_current_span("tool-handler"): return CallToolResult(content=[TextContent(type="text", text="done")]) - async def notification( - ctx: ServerRequestContext[object, object], params: NotificationParams - ) -> HandlerResult: + async def notification(ctx: ServerRequestContext[object, object], params: NotificationParams) -> HandlerResult: with tracer.start_as_current_span("notification-handler"): notified.set() return None @@ -9266,51 +9492,85 @@ async def test_stateful_mcp_tracing_follows_each_message_exclusion( try: async with manager.run(): async with httpx2.AsyncClient(transport=httpx2.ASGITransport(app=app)) as client: - initialized: Final = await client.post(initial_url, headers=headers, json={ - "jsonrpc": "2.0", "id": 0, "method": "initialize", "params": { - "protocolVersion": "2025-11-25", "capabilities": {}, - "clientInfo": {"name": "trace-exclusion", "version": "1"}, - }, - }) - assert initialized.status_code == 200, initialized.text - assert any( - span.name == "initialize" for span in exporter.get_finished_spans() - ) is not initialized_excluded - exporter.clear() - session_headers: Final = {**headers, "MCP-Session-Id": initialized.headers["mcp-session-id"]} - ready: Final = await client.post(initial_url, headers=session_headers, - json={"jsonrpc": "2.0", "method": "notifications/initialized"}) - assert ready.status_code == 202, ready.text - response: Final = await client.post(current_url, headers=session_headers, json={ - "jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": { - "name": "probe", "arguments": {}, "_meta": { - "traceparent": "00-11111111111111111111111111111111-2222222222222222-01", + initialized: Final = await client.post( + initial_url, + headers=headers, + json={ + "jsonrpc": "2.0", + "id": 0, + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": {"name": "trace-exclusion", "version": "1"}, }, }, - }) + ) + assert initialized.status_code == 200, initialized.text + assert ( + any(span.name == "initialize" for span in exporter.get_finished_spans()) is not initialized_excluded + ) + exporter.clear() + session_headers: Final = {**headers, "MCP-Session-Id": initialized.headers["mcp-session-id"]} + ready: Final = await client.post( + initial_url, headers=session_headers, json={"jsonrpc": "2.0", "method": "notifications/initialized"} + ) + assert ready.status_code == 202, ready.text + response: Final = await client.post( + current_url, + headers=session_headers, + json={ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "probe", + "arguments": {}, + "_meta": { + "traceparent": "00-11111111111111111111111111111111-2222222222222222-01", + }, + }, + }, + ) assert response.status_code == 200 and '"text":"done"' in response.text, response.text names: Final = {span.name for span in exporter.get_finished_spans()} assert ("tools/call probe" in names) is initialized_excluded assert ("tool-handler" in names) is initialized_excluded exporter.clear() - notification_response: Final = await client.post(current_url, headers=session_headers, - json={"jsonrpc": "2.0", "method": "notifications/exclusion-probe"}) + notification_response: Final = await client.post( + current_url, + headers=session_headers, + json={"jsonrpc": "2.0", "method": "notifications/exclusion-probe"}, + ) assert notification_response.status_code == 202, notification_response.text await notified.wait() - assert any( - span.name == "notification-handler" for span in exporter.get_finished_spans() - ) is initialized_excluded + assert ( + any(span.name == "notification-handler" for span in exporter.get_finished_spans()) + is initialized_excluded + ) exporter.clear() - modern_response: Final = await client.post(current_url, headers={ - "Accept": "application/json, text/event-stream", "MCP-Protocol-Version": "2026-07-28", - "MCP-Method": "tools/call", "MCP-Name": "probe", - }, json={ - "jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": { - "name": "probe", "arguments": {}, "_meta": { - PROTOCOL_VERSION_META_KEY: "2026-07-28", CLIENT_CAPABILITIES_META_KEY: {}, + modern_response: Final = await client.post( + current_url, + headers={ + "Accept": "application/json, text/event-stream", + "MCP-Protocol-Version": "2026-07-28", + "MCP-Method": "tools/call", + "MCP-Name": "probe", + }, + json={ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "probe", + "arguments": {}, + "_meta": { + PROTOCOL_VERSION_META_KEY: "2026-07-28", + CLIENT_CAPABILITIES_META_KEY: {}, + }, }, }, - }) + ) assert modern_response.status_code == 200, modern_response.text assert '"text":"done"' in modern_response.text, modern_response.text modern_names: Final = {span.name for span in exporter.get_finished_spans()} @@ -9413,8 +9673,7 @@ async def test_get_allowed_mcp_servers_includes_active_servers_submitted_by_user return_value={"submitted-1": submitted_server}, ), patch( - "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp." - "MCPRequestHandler.get_allowed_mcp_servers", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_allowed_mcp_servers", AsyncMock(return_value=[]), ), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), @@ -10374,9 +10633,7 @@ async def test_handle_list_tools_all_faults_on_continuation_stays_degraded( outcomes={"a": ServerListFault(tag="upstream_error", status_code=429)}, ) - result: Final = await _list_tools_with_listing( - _mcp_request_ctx, listing, PaginatedRequestParams(cursor="cursor-1") - ) + result: Final = await _list_tools_with_listing(_mcp_request_ctx, listing, PaginatedRequestParams(cursor="cursor-1")) assert isinstance(result, ListToolsResult) assert result.tools == [] diff --git a/tests/unit/proxy/middleware/test_billable_request_metrics_middleware.py b/tests/unit/proxy/middleware/test_billable_request_metrics_middleware.py index 8c81a2c9b36..d2e961ad158 100644 --- a/tests/unit/proxy/middleware/test_billable_request_metrics_middleware.py +++ b/tests/unit/proxy/middleware/test_billable_request_metrics_middleware.py @@ -123,6 +123,7 @@ def test_is_pure_asgi_not_base_http_middleware(): ("/azure_speech/speechtotext/v3.2/transcriptions", (BillableCategory.LLM, "/azure_speech")), ("/transcribe", (BillableCategory.LLM, "/transcribe")), ("/transcribe/StartTranscriptionJob", (BillableCategory.LLM, "/transcribe")), + ("/eu_jev/v1/systemone", (BillableCategory.LLM, "/eu_jev/v1/systemone")), ("/mcp", (BillableCategory.MCP, "/mcp")), ("/mcp/", (BillableCategory.MCP, "/mcp")), ("/mcp/tools/list", (BillableCategory.MCP, "/mcp")), @@ -165,6 +166,8 @@ def test_classify_billable(path: str, expected: Tuple[BillableCategory, str]): "/langfuse/api/public/ingestion", # a bare provider prefix is not an inference call "/bedrock", + "/eu_jev/v1/systemone/extra", + "/eu_jev/v1/systemone2", "/v1/mcp", "/v1/mcp/tools", "/v1/mcp/server", @@ -184,6 +187,11 @@ def test_classify_non_billable_returns_none(path: str): assert classify_billable_request(path) is None +@pytest.mark.usefixtures("configured_eu_jev_route") +def test_a_pass_through_configured_at_the_eu_jev_decision_path_is_not_billed() -> None: + assert classify_billable_request("/eu_jev/v1/systemone") is None + + @pytest.mark.parametrize( "path", [ diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index 79209a32f28..4011bc0294a 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -366,3 +366,94 @@ def test_success_handler_skips_typesafe_pricing_for_non_decisions_openrouter_rou assert normalized["standard_logging_response_object"] is None assert "combined_usage_object" not in normalized["kwargs"] assert normalized["kwargs"].get("model") != "openrouter/typesafe/jev-1.13-20260917" + + +def test_eu_jev_decision_is_priced_from_its_registry_row(): + model_cost: Final = litellm.model_cost["eu_jev/eu-jeff-1.0.0"] + response: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, + logging_obj=_logging_obj(), + url_route="https://eu-jev.com/v1/systemone", + result='{"answers": {}}', + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"model": "eu-jeff-latest"}, + custom_llm_provider="eu_jev", + ) + + assert response["kwargs"]["model"] == "eu_jev/eu-jeff-1.0.0" + assert response["kwargs"]["custom_llm_provider"] == "eu_jev" + assert response["kwargs"]["response_cost"] == pytest.approx(64 * model_cost["input_cost_per_token"]) + assert response["kwargs"]["response_cost"] > 0 + assert response["kwargs"]["combined_usage_object"].prompt_tokens == 64 + assert response["kwargs"]["combined_usage_object"].completion_tokens == 0 + + +# eu-jev.com/docs (read 2026-10-21): an omitted model and the legacy names answer with the current release, +# which a future eu/jev release reports under a version the registry does not list yet, and a caller may pin it. +@pytest.mark.parametrize( + "request_body", [{}, {"model": "jeff-latest"}, {"model": "jeff-1.0.0"}, {"model": "eu-jeff-99.0.0"}] +) +def test_eu_jev_release_without_its_own_row_is_priced_as_the_latest_alias(request_body: dict[str, object]) -> None: + unlisted_release: Final = "eu-jeff-99.0.0" + assert f"eu_jev/{unlisted_release}" not in litellm.model_cost + response: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={"model": unlisted_release, "usage": {"input_tokens": 64, "output_tokens": 0}}, + logging_obj=_logging_obj(), + url_route="https://eu-jev.com/v1/systemone", + result='{"answers": {}}', + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body=request_body, + custom_llm_provider="eu_jev", + ) + + latest_cost: Final = litellm.model_cost["eu_jev/eu-jeff-latest"]["input_cost_per_token"] + assert response["kwargs"]["model"] == f"eu_jev/{unlisted_release}" + assert latest_cost > 0 + assert response["kwargs"]["response_cost"] == pytest.approx(64 * latest_cost) + + +def test_eu_jev_pinned_release_is_not_priced_as_the_latest_alias(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem( + litellm.model_cost, + "eu_jev/eu-jeff-latest", + {**litellm.model_cost["eu_jev/eu-jeff-latest"], "input_cost_per_token": 1.0}, + ) + response: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, + logging_obj=_logging_obj(), + url_route="https://eu-jev.com/v1/systemone", + result='{"answers": {}}', + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"model": "eu-jeff-1.0.0"}, + custom_llm_provider="eu_jev", + ) + + pinned_cost: Final = litellm.model_cost["eu_jev/eu-jeff-1.0.0"]["input_cost_per_token"] + assert response["kwargs"]["response_cost"] == pytest.approx(64 * pinned_cost) + + +def test_success_handler_dispatches_eu_jev_to_the_shared_handler(): + normalized: Final = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + httpx_response=_response(), + response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, + request_body={"model": "eu-jeff-latest"}, + logging_obj=_logging_obj(), + url_route="https://eu-jev.com/v1/systemone", + result="{}", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + custom_llm_provider="eu_jev", + ) + + assert normalized["kwargs"]["custom_llm_provider"] == "eu_jev" + assert normalized["kwargs"]["model"] == "eu_jev/eu-jeff-1.0.0" diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 8d330cbd749..806d88b2467 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -15,7 +15,7 @@ from urllib.parse import parse_qs import httpx import pytest import respx -from fastapi import HTTPException, Request, Response +from fastapi import FastAPI, HTTPException, Request, Response from fastapi.routing import APIRoute from fastapi.responses import StreamingResponse from fastapi.testclient import TestClient @@ -52,6 +52,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( bedrock_proxy_route, create_pass_through_route, cursor_proxy_route, + eu_jev_proxy_route, fal_ai_proxy_route, get_azure_ai_search_index_from_endpoint, get_vertex_base_url, @@ -74,6 +75,7 @@ from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.types.guardrails import GuardrailEventHooks from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials +from litellm.llms.eu_jev.common_utils import eu_jev_scoped_model, eu_jev_upstream_body def _assert_bedrock_processing_data_classification(data: dict[str, object], is_streaming: bool) -> None: @@ -8601,6 +8603,387 @@ class TestTypeSafePassthroughRoute: ) +class _RecordsPreCallModel(CustomLogger): + def __init__(self, switch_to: str | None = None) -> None: + super().__init__() + self.switch_to: Final = switch_to + self.models: Final[list[object]] = [] + + async def async_pre_call_hook( + self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, data: dict, call_type: str + ) -> dict | None: + self.models.append(data.get("model")) + return None if self.switch_to is None else {**data, "model": self.switch_to} + + +class TestEuJevPassthroughRoute: + @pytest.fixture + def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + from litellm.proxy.proxy_server import app + + monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") + monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base") + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) + yield TestClient(app) + + def test_forwards_a_decision_upstream_with_the_eu_jev_key(self, client: TestClient) -> None: + body: Final = {"model": "eu-jeff-latest", "state": "x"} + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").mock( + return_value=httpx.Response(200, json={"id": "upstream_123"}) + ) + response: Final = client.post("/eu_jev/v1/systemone", json=body) + + assert (response.status_code, response.json()) == (200, {"id": "upstream_123"}) + sent: Final = route.calls.last.request + assert sent.headers["authorization"] == "Bearer eu-jev-test-key" + assert json.loads(sent.content) == body + + @pytest.mark.parametrize( + ("body", "hook_model"), + [ + ({"state": "x"}, "eu_jev/eu-jeff-latest"), + ({"model": "jeff-latest", "state": "x"}, "eu_jev/eu-jeff-latest"), + ({"model": "eu-jeff-1.0.0", "state": "x"}, "eu_jev/eu-jeff-1.0.0"), + ], + ) + def test_pre_call_hooks_see_the_model_key_limits_name_while_upstream_gets_the_client_body( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch, body: dict[str, str], hook_model: str + ) -> None: + recorder: Final = _RecordsPreCallModel() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json=body) + + assert response.status_code == 200 + assert recorder.models == [hook_model] + assert json.loads(route.calls.last.request.content) == body + + @pytest.mark.parametrize("model", [7, ["eu-jeff-latest"], False]) + def test_a_non_string_client_model_reaches_upstream_unchanged(self, model: int | list[str] | bool) -> None: + client_body: Final = {"model": model, "state": "x"} + hook_body: Final = {"model": eu_jev_scoped_model(client_body), "state": "x"} + + assert eu_jev_upstream_body(hook_body, client_body) == client_body + + def test_a_null_model_is_limited_as_the_latest_release_and_left_out_upstream( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + recorder: Final = _RecordsPreCallModel() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": None, "state": "x"}) + + assert response.status_code == 200 + assert recorder.models == ["eu_jev/eu-jeff-latest"] + assert json.loads(route.calls.last.request.content) == {"state": "x"} + + def test_forwards_the_release_a_pre_call_hook_switched_to( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "callbacks", [_RecordsPreCallModel(switch_to="eu_jev/eu-jeff-1.0.0")]) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": "jeff-latest", "state": "x"}) + + assert response.status_code == 200 + assert json.loads(route.calls.last.request.content) == {"model": "eu-jeff-1.0.0", "state": "x"} + + def test_a_provider_prefixed_model_is_limited_once_and_forwarded_without_the_prefix( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + recorder: Final = _RecordsPreCallModel() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": "eu_jev/eu-jeff-1.0.0", "state": "x"}) + + assert response.status_code == 200 + assert recorder.models == ["eu_jev/eu-jeff-1.0.0"] + assert json.loads(route.calls.last.request.content) == {"model": "eu-jeff-1.0.0", "state": "x"} + + def test_a_pass_through_deployment_key_goes_to_that_deployment_base( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy import proxy_server + + deployment_router: Final = litellm.Router( + model_list=[ + { + "model_name": "eu-jev-friendly", + "litellm_params": { + "model": "eu_jev/eu-jeff-latest", + "api_key": "eu-jev-deployment-key", + "api_base": "https://eu-jev-deployment.example/v2", + "use_in_pass_through": True, + }, + } + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", deployment_router) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev-deployment.example/v2/v1/systemone").respond( + 200, json={"answers": {}} + ) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 200 + assert route.calls.last.request.headers["authorization"] == "Bearer eu-jev-deployment-key" + + def test_a_team_deployment_never_redirects_another_team_s_calls( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy import proxy_server + + team_router: Final = litellm.Router( + model_list=[ + { + "model_name": "team-a-eu-jev", + "litellm_params": { + "model": "eu_jev/eu-jeff-latest", + "api_key": "team-a-own-key", + "api_base": "https://team-a-server.example", + "use_in_pass_through": True, + }, + "model_info": {"team_id": "team-a"}, + } + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", team_router) + with respx.mock(assert_all_called=False) as upstream: + team_server: Final = upstream.route(host="team-a-server.example").respond(200, json={"answers": {}}) + eu_jev: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond( + 200, json={"answers": {}} + ) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "team b data"}) + + assert response.status_code == 200 + assert not team_server.called + assert eu_jev.calls.last.request.headers["authorization"] == "Bearer eu-jev-test-key" + + def test_a_base_ending_in_v1_is_not_doubled(self, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base/v1/") + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 200 + assert route.called + + def test_forwards_query_params_to_the_decision_endpoint(self, client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone", params={"trace": "yes"}).respond( + 200, json={"answers": {}} + ) + response: Final = client.post("/eu_jev/v1/systemone?trace=yes", json={"state": "x"}) + + assert response.status_code == 200 + assert route.called + + def test_uses_the_hosted_api_when_base_is_unset(self, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("EU_JEV_API_BASE") + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.com/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 200 + assert route.called + + def test_answers_503_without_calling_upstream_when_the_key_is_unset( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("EU_JEV_API_KEY") + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.route(host="eu-jev.example").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 503 + assert "EU_JEV_API_KEY" in response.text + assert not route.called + + @pytest.mark.parametrize( + ("method", "path"), + [("GET", "/eu_jev/api/account/keys"), ("DELETE", "/eu_jev/v1/keys/abc"), ("GET", "/eu_jev/v1/systemone")], + ) + def test_forwards_nothing_but_a_decision(self, client: TestClient, method: str, path: str) -> None: + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.route(host="eu-jev.example").respond(200, json={}) + response: Final = client.request(method, path) + + assert response.status_code in (404, 405) + assert not route.called + + def test_rejects_custom_body_that_would_replace_the_authorized_request(self, client: TestClient) -> None: + smuggled: Final = {"model": "eu-jeff-latest", "state": "refund", "questions": {"q": {"type": "noul"}}} + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": "gpt-4o", "custom_body": smuggled}) + + assert response.status_code == 400 + assert "custom_body" in response.text + assert not route.called + + def test_rejects_streaming_that_would_skip_spend_tracking(self, client: TestClient) -> None: + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post( + "/eu_jev/v1/systemone", json={"model": "eu-jeff-latest", "state": "x", "stream": True} + ) + + assert response.status_code == 400 + assert "Streaming" in response.text + assert not route.called + + @pytest.fixture + def operator_app(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[Callable[[list[str], str], FastAPI]]: + from litellm.proxy._lazy_features import LAZY_FEATURES, attach_lazy_features, reserve_lazy_slot + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + _registered_pass_through_routes, + ) + + monkeypatch.delenv("EU_JEV_API_KEY", raising=False) + monkeypatch.delenv("LITELLM_DISABLE_LAZY_ROUTES", raising=False) + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + features: Final = tuple(feature for feature in LAZY_FEATURES if feature.name == "llm_passthrough") + + def build(methods: list[str], path: str) -> FastAPI: + app: Final = FastAPI() + reserve_lazy_slot(app, "llm_passthrough", features=features) + attach_lazy_features(app, features) + monkeypatch.setitem( + app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual") + ) + InitPassThroughEndpointHelpers.add_exact_path_route( + app=app, + path=path, + target="https://operator.example/decide", + custom_headers={"x-operator": "on"}, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=None, + endpoint_id="operator-eu-jev", + methods=methods, + ) + return app + + saved_registry: Final = dict(_registered_pass_through_routes) + try: + yield build + finally: + _registered_pass_through_routes.clear() + _registered_pass_through_routes.update(saved_registry) + + @pytest.mark.parametrize("root_path", ["", "/tenant-a"]) + def test_a_pass_through_endpoint_configured_at_the_same_path_keeps_serving_it( + self, operator_app: Callable[[list[str], str], FastAPI], root_path: str + ) -> None: + with ( + respx.mock(assert_all_called=True) as upstream, + TestClient(operator_app(["POST"], "/eu_jev/v1/systemone"), root_path=root_path) as client, + ): + route: Final = upstream.post("https://operator.example/decide").respond(200, json={"operator": True}) + response: Final = client.post(f"{root_path}/eu_jev/v1/systemone", json={"state": "x", "stream": True}) + + assert (response.status_code, response.json()) == (200, {"operator": True}) + assert route.calls.last.request.headers["x-operator"] == "on" + + def test_a_post_to_a_get_only_pass_through_at_the_same_path_is_still_refused( + self, operator_app: Callable[[list[str], str], FastAPI] + ) -> None: + with TestClient(operator_app(["GET"], "/eu_jev/v1/systemone")) as client: + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 405 + + def test_a_pass_through_configured_with_a_trailing_slash_still_gets_the_path_without_one( + self, operator_app: Callable[[list[str], str], FastAPI] + ) -> None: + with ( + respx.mock(assert_all_called=True) as upstream, + TestClient(operator_app(["POST"], "/eu_jev/v1/systemone/")) as client, + ): + route: Final = upstream.post("https://operator.example/decide").respond(200, json={"operator": True}) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert (response.status_code, response.json()) == (200, {"operator": True}) + assert route.calls.last.request.headers["x-operator"] == "on" + + @pytest.mark.parametrize("method", ["GET", "PUT"]) + def test_a_pass_through_configured_with_a_trailing_slash_still_gets_other_methods_without_one( + self, operator_app: Callable[[list[str], str], FastAPI], method: str + ) -> None: + with ( + respx.mock(assert_all_called=True) as upstream, + TestClient(operator_app(["GET", "POST", "PUT"], "/eu_jev/v1/systemone/")) as client, + ): + route: Final = upstream.route(method=method, url="https://operator.example/decide").respond( + 200, json={"operator": True} + ) + response: Final = client.request(method, "/eu_jev/v1/systemone", json={"state": "x"}) + + assert (response.status_code, response.json()) == (200, {"operator": True}) + assert route.calls.last.request.headers["x-operator"] == "on" + + def test_a_request_the_built_in_route_took_is_not_forwarded_with_a_pass_through_added_meanwhile( + self, operator_app: Callable[[list[str], str], FastAPI], monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers + + monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") + monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base") + app: Final = operator_app(["POST"], "/eu_jev/v1/systemone") + InitPassThroughEndpointHelpers.remove_endpoint_routes("operator-eu-jev") + + def add_the_pass_through_after_routing() -> UserAPIKeyAuth: + InitPassThroughEndpointHelpers.add_exact_path_route( + app=app, + path="/eu_jev/v1/systemone", + target="https://operator.example/decide", + custom_headers={"x-operator": "on"}, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=None, + endpoint_id="operator-eu-jev", + methods=["POST"], + ) + return UserAPIKeyAuth(api_key="sk-virtual") + + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, add_the_pass_through_after_routing) + with respx.mock(assert_all_called=False) as upstream, TestClient(app) as client: + built_in: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + operator: Final = upstream.route(host="operator.example").respond(200, json={"operator": True}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": "eu-jeff-latest", "state": "x"}) + + assert (response.status_code, response.json()) == (200, {"answers": {}}) + assert built_in.calls.last.request.headers["authorization"] == "Bearer eu-jev-test-key" + assert not operator.called + + def test_a_deleted_pass_through_at_the_same_path_stops_forwarding_when_built_in_routes_come_first( + self, operator_app: Callable[[list[str], str], FastAPI], monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers + + monkeypatch.setenv("LITELLM_BUILTIN_PASS_THROUGH_ROUTES_FIRST", "true") + app: Final = operator_app(["GET", "PUT"], "/eu_jev/v1/systemone") + InitPassThroughEndpointHelpers.remove_endpoint_routes("operator-eu-jev") + with respx.mock(assert_all_called=False) as upstream, TestClient(app) as client: + operator: Final = upstream.route(host="operator.example").respond(200, json={"operator": True}) + responses: Final = (client.get("/eu_jev/v1/systemone"), client.put("/eu_jev/v1/systemone", json={})) + + assert [response.status_code for response in responses] == [404, 404] + assert not operator.called + + @pytest.mark.parametrize("provider", ["laya", "bespoke"]) class TestOssDecisionPassthroughRoute: @pytest.fixture diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 72e5d13661f..7b62853b0e9 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -82,6 +82,7 @@ PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES: Final = MappingProxyType( "/tinyfish/{endpoint:path}": frozenset({"GET", "POST"}), "/laya/v1/systemone": frozenset({"POST"}), "/bespoke/v1/systemone": frozenset({"POST"}), + "/eu_jev/v1/systemone": frozenset({"POST"}), } ) @@ -2977,6 +2978,56 @@ async def test_update_pass_through_endpoint_preserves_auth_false(): assert RouteChecks.is_auth_enforced_pass_through_route(route="/public-passthrough", method="POST") is False +@pytest.mark.asyncio +async def test_update_pass_through_endpoint_keeps_serving_the_old_route_until_the_new_one_is_registered(): + from litellm.proxy._types import ConfigFieldInfo, PassThroughGenericEndpoint, UserAPIKeyAuth + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + update_pass_through_endpoints, + ) + + def registered_target() -> object: + registered: Final = InitPassThroughEndpointHelpers.get_registered_pass_through_route(route="/operator") or {} + return registered.get("passthrough_params", {}).get("target") + + targets_during_db_write: Final[list[object]] = [] + + async def record_registered_target(**_: object) -> None: + targets_during_db_write.append(registered_target()) + + with ( + patch("litellm.proxy.proxy_server.get_config_general_settings") as mock_get_config, + patch("litellm.proxy.proxy_server.update_config_general_settings", side_effect=record_registered_target), + patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes", {}), + ): + InitPassThroughEndpointHelpers.add_exact_path_route( + app=FastAPI(), + path="/operator", + target="http://old.example/api", + custom_headers=None, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=None, + endpoint_id="operator-route", + methods=["POST"], + ) + mock_get_config.return_value = ConfigFieldInfo( + field_name="pass_through_endpoints", + field_value=[ + {"id": "operator-route", "path": "/operator", "target": "http://old.example/api", "methods": ["POST"]} + ], + ) + await update_pass_through_endpoints( + endpoint_id="operator-route", + data=PassThroughGenericEndpoint(path="/operator", target="http://new.example/api", methods=["POST"]), + request=MagicMock(spec=Request), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + ) + + assert (targets_during_db_write, registered_target()) == (["http://old.example/api"], "http://new.example/api") + + @pytest.mark.asyncio async def test_update_pass_through_endpoint_not_found(): """ @@ -10548,6 +10599,138 @@ async def test_direct_passthrough_keeps_mapped_baggage_tags_outside_upstream_bod assert await read_request_body(request) == {"model": "qa-model"} +@pytest.mark.parametrize( + ("update", "expected"), + [ + ({"path": "/moved-to"}, (True, True, False, False)), + ({"path": "/moved-from"}, (True, True, True, True)), + ({"path": "/moved-from", "auth": False}, (False, False, False, False)), + ], +) +@pytest.mark.asyncio +async def test_update_pass_through_endpoint_keeps_llm_api_route_checks_on_the_updated_path( + monkeypatch: pytest.MonkeyPatch, + update: dict[str, str | bool], + expected: tuple[bool, bool, bool, bool], +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._types import ( + ConfigFieldInfo, + ConfigFieldUpdate, + LiteLLMRoutes, + PassThroughGenericEndpoint, + UserAPIKeyAuth, + ) + from litellm.proxy.auth.route_checks import RouteChecks + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import update_pass_through_endpoints + + async def serve_stored_endpoints(data: ConfigFieldUpdate, **_: object) -> None: + await initialize_pass_through_endpoints(data.field_value) + + stored_endpoint: Final = { + "id": "moved-route", + "path": "/moved-from", + "target": "http://example.com/api", + "auth": True, + "include_subpath": True, + } + saved_registry: Final = dict(_registered_pass_through_routes) + saved_openai_routes: Final = tuple(LiteLLMRoutes.openai_routes.value) + try: + _registered_pass_through_routes.clear() + monkeypatch.setattr(proxy_server, "app", FastAPI()) + monkeypatch.setattr(proxy_server, "premium_user", True) + monkeypatch.setattr(proxy_server, "config_passthrough_endpoints", None) + await initialize_pass_through_endpoints([stored_endpoint]) + with ( + patch( + "litellm.proxy.proxy_server.get_config_general_settings", + return_value=ConfigFieldInfo(field_name="pass_through_endpoints", field_value=[stored_endpoint]), + ), + patch("litellm.proxy.proxy_server.update_config_general_settings", side_effect=serve_stored_endpoints), + ): + await update_pass_through_endpoints( + endpoint_id="moved-route", + data=PassThroughGenericEndpoint.model_validate({"target": "http://example.com/api", **update}), + request=MagicMock(spec=Request), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + ) + + updated_path: Final = str(update["path"]) + assert ( + RouteChecks.is_llm_api_route(route=updated_path), + f"{updated_path}/*" in LiteLLMRoutes.openai_routes.value, + RouteChecks.is_llm_api_route(route="/moved-from"), + "/moved-from/*" in LiteLLMRoutes.openai_routes.value, + ) == expected + finally: + _registered_pass_through_routes.clear() + _registered_pass_through_routes.update(saved_registry) + LiteLLMRoutes.openai_routes.value[:] = saved_openai_routes + + +@pytest.mark.asyncio +async def test_update_pass_through_endpoint_keeps_llm_api_route_checks_on_a_path_another_endpoint_shares( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._types import ( + ConfigFieldInfo, + ConfigFieldUpdate, + LiteLLMRoutes, + PassThroughGenericEndpoint, + UserAPIKeyAuth, + ) + from litellm.proxy.auth.route_checks import RouteChecks + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import update_pass_through_endpoints + + async def serve_stored_endpoints(data: ConfigFieldUpdate, **_: object) -> None: + await initialize_pass_through_endpoints(data.field_value) + + stored_endpoints: Final = [ + { + "id": f"shared-{method.lower()}", + "path": "/shared", + "target": "http://example.com/api", + "auth": True, + "methods": [method], + } + for method in ("GET", "POST") + ] + saved_registry: Final = dict(_registered_pass_through_routes) + saved_openai_routes: Final = tuple(LiteLLMRoutes.openai_routes.value) + try: + _registered_pass_through_routes.clear() + monkeypatch.setattr(proxy_server, "app", FastAPI()) + monkeypatch.setattr(proxy_server, "premium_user", True) + monkeypatch.setattr(proxy_server, "config_passthrough_endpoints", None) + await initialize_pass_through_endpoints(stored_endpoints) + with ( + patch( + "litellm.proxy.proxy_server.get_config_general_settings", + return_value=ConfigFieldInfo(field_name="pass_through_endpoints", field_value=stored_endpoints), + ), + patch("litellm.proxy.proxy_server.update_config_general_settings", side_effect=serve_stored_endpoints), + ): + await update_pass_through_endpoints( + endpoint_id="shared-get", + data=PassThroughGenericEndpoint.model_validate( + {"target": "http://example.com/api", "path": "/moved", "auth": True, "methods": ["GET"]} + ), + request=MagicMock(spec=Request), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + ) + + assert (RouteChecks.is_llm_api_route(route="/shared"), RouteChecks.is_llm_api_route(route="/moved")) == ( + True, + True, + ) + finally: + _registered_pass_through_routes.clear() + _registered_pass_through_routes.update(saved_registry) + LiteLLMRoutes.openai_routes.value[:] = saved_openai_routes + + @pytest.mark.asyncio @pytest.mark.parametrize( ("flat_cost", "upstream_cost", "expected"), diff --git a/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py b/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py index 0e337195bb8..7771ff0274e 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py +++ b/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py @@ -26,6 +26,7 @@ import pytest from fastapi import FastAPI from fastapi.routing import APIRoute from fastapi.testclient import TestClient +from starlette.requests import Request from litellm.proxy._types import PassThroughGenericEndpoint, ProxyException @@ -137,3 +138,15 @@ async def test_runtime_check_explicit_auth_false_still_skips_validation(): ) assert isinstance(result, UserAPIKeyAuth) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ["/eu_jev/v1/systemone", "/eu_jev/v1/systemone/extra", "/eu_jev/v1/systemone2"]) +async def test_no_eu_jev_route_reads_the_litellm_user_api_key_header(route: str) -> None: + request: Final = Request({"type": "http", "headers": [(b"litellm_user_api_key", b"sk-from-header")]}) + + result: Final = await check_api_key_for_custom_headers_or_pass_through_endpoints( + request=request, route=route, pass_through_endpoints=None, api_key="sk-from-bearer" + ) + + assert result == "sk-from-bearer" diff --git a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py index ab4f3f7a675..dd48a3e55ef 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py +++ b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py @@ -1,6 +1,7 @@ import asyncio import os import unittest +from typing import Final from unittest.mock import patch import pytest @@ -60,6 +61,49 @@ def test_credential_loaded_after_deployment_registration_still_resolves(): ) +def test_proxy_wide_credentials_pair_a_deployment_key_with_that_deployment_base() -> None: + llm_router: Final = litellm.Router( + model_list=[ + _flagged_deployment("openai/gpt-4o", api_base="https://keyless.example/v1"), + _flagged_deployment("openai/gpt-4o-mini", api_key="sk-deployment", api_base="https://keyed.example/v1"), + ] + ) + credentials: Final = _passthrough_router(llm_router).get_proxy_wide_credentials(custom_llm_provider="openai") + + assert credentials is not None + assert (credentials.api_key, credentials.api_base) == ("sk-deployment", "https://keyed.example/v1") + + +def test_proxy_wide_credentials_never_come_from_a_team_deployment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-proxy-env") + team_deployment: Final = { + **_flagged_deployment("openai/gpt-4o", api_key="sk-team-a", api_base="https://team-a.example/v1"), + "model_info": {"team_id": "team-a"}, + } + passthrough_router: Final = _passthrough_router(litellm.Router(model_list=[team_deployment])) + + credentials: Final = passthrough_router.get_proxy_wide_credentials(custom_llm_provider="openai") + + assert credentials is not None + assert (credentials.api_key, credentials.api_base) == ("sk-proxy-env", None) + + +def test_credentials_for_a_provider_route_still_include_a_team_deployment_key() -> None: + team_deployment: Final = { + **_flagged_deployment("openai/gpt-4o", api_key="sk-team-a"), + "model_info": {"team_id": "team-a"}, + } + passthrough_router: Final = _passthrough_router(litellm.Router(model_list=[team_deployment])) + + assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-team-a" + + +def test_proxy_wide_credentials_are_missing_without_a_deployment_or_env_key() -> None: + passthrough_router: Final = _passthrough_router(litellm.Router(model_list=[])) + + assert passthrough_router.get_proxy_wide_credentials(custom_llm_provider="anthropic") is None + + def test_credential_rotation_is_reflected_without_deployment_update(): CredentialAccessor.upsert_credentials([_credential("cred_openai", "sk-before-rotation")]) llm_router = litellm.Router( diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index 93d0b778e26..20eb7b80016 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -6377,3 +6377,54 @@ async def test_proxy_config_privacy_precedes_database_settings_diagnostics( assert stored["general_settings"]["custom_auth"] == remote_module assert any(record.levelno == logging.WARNING for record in caplog.records) assert loaded["router_settings"] == stored["router_settings"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("privacy", [False, True]) +async def test_shadow_warning_survives_failed_config_load_and_emits_once( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, privacy: bool +) -> None: + from litellm.caching.dual_cache import DualCache + from litellm.proxy import proxy_server + from litellm.proxy import utils as proxy_utils + + async def get_generic_data(key: str, value: str, table_name: str) -> SimpleNamespace | None: + assert (key, table_name) == ("param_name", "config") + return ( + SimpleNamespace(param_name=value, param_value={"max_parallel_requests": 2}) + if value == "general_settings" + else None + ) + + monkeypatch.setattr(litellm, "redact_messages_in_debug_logs", False) + monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(get_generic_data=get_generic_data)) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_utils, "litellm_config_cache", DualCache()) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + monkeypatch.setenv("LITELLM_QA_SHADOW_PRIVACY", "invalid-boolean") + config_file: Final = tmp_path / "shadow-warning.yaml" + config_file.write_text( + json.dumps( + { + "litellm_settings": {"redact_messages_in_debug_logs": "os.environ/LITELLM_QA_SHADOW_PRIVACY"}, + "general_settings": {"max_parallel_requests": 5}, + } + ) + ) + config: Final = ProxyConfig() + caplog.set_level(logging.WARNING, logger="LiteLLM Proxy") + + with pytest.raises(ValidationError): + await config.get_config(str(config_file)) + assert caplog.records == [] + + monkeypatch.setenv("LITELLM_QA_SHADOW_PRIVACY", str(privacy).lower()) + loaded: Final = await config.get_config(str(config_file)) + assert loaded["general_settings"]["max_parallel_requests"] == 5 + assert len(caplog.records) == 1 + assert caplog.records[0].levelno == logging.WARNING + assert ("max_parallel_requests" in caplog.records[0].getMessage()) is not privacy + + caplog.clear() + await config.get_config(str(config_file)) + assert caplog.records == [] diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index da608dbca0e..48672c7a305 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -481,9 +481,16 @@ def _decisions_provider_entry(provider: str) -> ProviderCreateInfo: return entry -@pytest.mark.parametrize("display_name,provider", (("TypeSafe", LlmProviders.TYPESAFE), ("Nace", LlmProviders.NACE))) -def test_decisions_provider_fields(display_name: str, provider: LlmProviders) -> None: - entry: Final = _decisions_provider_entry(display_name) +@pytest.mark.parametrize( + ("provider_key", "display_name", "provider"), + ( + ("TypeSafe", "TypeSafe", LlmProviders.TYPESAFE), + ("Nace", "Nace", LlmProviders.NACE), + ("EU_JEV", "eu/jev", LlmProviders.EU_JEV), + ), +) +def test_decisions_provider_fields(provider_key: str, display_name: str, provider: LlmProviders) -> None: + entry: Final = _decisions_provider_entry(provider_key) assert entry.provider_display_name == display_name assert entry.litellm_provider == provider.value diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index a4d5d073352..2d8349367c6 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -14,6 +14,7 @@ from tests.unit.proxy.router_entry_point_stubs import route_entry_points_to_stub import anthropic import httpx import pytest +import respx from fastapi import HTTPException, Request, Response, status from fastapi.responses import JSONResponse, StreamingResponse @@ -9400,6 +9401,14 @@ class TestApplyStreamUsageTracking: assert data["_litellm_strip_stream_usage"] is True assert data["_litellm_stream_usage_injected"] is True + def test_default_injects_usage_and_marks_strip_for_text_completions(self): + data: Final = {"stream": True, "model": "gpt-5.4-nano"} + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="atext_completion") + + assert data["stream_options"] == {"include_usage": True} + assert data["_litellm_strip_stream_usage"] is True + def test_default_preserves_other_client_stream_options_keys(self): data = {"stream": True, "stream_options": {"include_obfuscation": True}} @@ -12720,3 +12729,103 @@ async def test_nonstream_failure_preserves_client_error_and_safe_status_without_ else: assert any("PRIVATE_NONSTREAM_FAILURE_42896" in message for message in messages) assert all("event=request_failed" not in message for message in messages) + + +_DOWNSTREAM_GUARDRAIL_FIELDS: Final = MappingProxyType( + {"guardrail_name": "block-secret-project", "guardrail_mode": "pre_call", "pattern": "secret_project_codename"} +) + + +def _downstream_proxy_rejection( + route: str, call: Callable[[], object], fields: Mapping[str, object] = _DOWNSTREAM_GUARDRAIL_FIELDS +) -> Exception: + body: Final = { + "error": { + "message": "Content blocked: secret_project_codename pattern detected", + "type": "invalid_request_error", + "param": None, + "code": "400", + "provider_specific_fields": dict(fields), + } + } + with respx.mock(assert_all_called=True) as mock: + _ = mock.post(f"https://downstream-proxy.example.com/{route}").mock(return_value=httpx.Response(400, json=body)) + with pytest.raises(litellm.BadRequestError) as caught: + _ = call() + return caught.value + + +_DOWNSTREAM: Final = MappingProxyType( + {"api_base": "https://downstream-proxy.example.com/", "api_key": "sk-downstream-key"} +) + + +@pytest.mark.usefixtures("httpx_transport") +@pytest.mark.parametrize( + ("route", "call"), + [ + ( + "responses", + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", **_DOWNSTREAM), + ), + ( + "chat/completions", + lambda: litellm.completion( + model="litellm_proxy/gpt-5.5", messages=[{"role": "user", "content": "project bluebird"}], **_DOWNSTREAM + ), + ), + ], +) +async def test_a_chained_proxy_relays_which_downstream_guardrail_blocked_the_request( + route: str, call: Callable[[], object] +) -> None: + rejection: Final = _downstream_proxy_rejection(route, call) + with pytest.raises(ProxyException) as relayed: + await ProxyBaseLLMRequestProcessing(data={"model": "chained"})._handle_llm_api_exception( + e=rejection, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-front"), + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + assert relayed.value.to_dict()["provider_specific_fields"] == dict(_DOWNSTREAM_GUARDRAIL_FIELDS) + + +_DOWNSTREAM_LEAKABLE_VALUES: Final = ( + "guard-scan.internal.corp", + "/srv/secret/app/guard.py", + "10.0.4.17", + "abcd1234efgh5678", + "cs-0123456789", + "tok-0123456789abcdef", + "4815162342108", +) + + +@pytest.mark.usefixtures("httpx_transport") +async def test_a_chained_proxy_redacts_secrets_and_internal_details_from_relayed_guardrail_fields() -> None: + fields: Final = { + **_DOWNSTREAM_GUARDRAIL_FIELDS, + "exception": "connect to http://guard-scan.internal.corp:9000/v1/check failed at /srv/secret/app/guard.py", + "api_key": "abcd1234efgh5678", + "access_token": ["tok-0123456789abcdef"], + "scanner": { + "hosts": ["10.0.4.17:8080"], + "client_secret": "cs-0123456789", + "api_secret": 4815162342108, + "attempts": 2, + }, + } + rejection: Final = _downstream_proxy_rejection( + "responses", + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", **_DOWNSTREAM), + fields, + ) + with pytest.raises(ProxyException) as relayed: + await ProxyBaseLLMRequestProcessing(data={"model": "chained"})._handle_llm_api_exception( + e=rejection, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-front"), + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + answered: Final = json.loads(json.dumps(relayed.value.to_dict()["provider_specific_fields"])) + assert {key: answered[key] for key in _DOWNSTREAM_GUARDRAIL_FIELDS} == dict(_DOWNSTREAM_GUARDRAIL_FIELDS) + assert answered["scanner"]["attempts"] == 2 + assert [value for value in _DOWNSTREAM_LEAKABLE_VALUES if value in json.dumps(answered)] == [] diff --git a/tests/unit/proxy/test_component_allowlists.py b/tests/unit/proxy/test_component_allowlists.py index eb058d01770..4c0b3ad913b 100644 --- a/tests/unit/proxy/test_component_allowlists.py +++ b/tests/unit/proxy/test_component_allowlists.py @@ -397,6 +397,22 @@ def test_gateway_keeps_memory_summary_and_trims_the_other_debug_routes(): assert not _is_gateway_route(debug_memory_routes[path]), f"{path} must not be served by the gateway" +@pytest.mark.parametrize( + ("path", "served"), + [ + ("/eu_jev/v1/systemone", True), + ("/eu_jev/v1/systemone2", False), + ("/eu_jev/v1/systemone/{subpath:path}", False), + ("/eu_jev/custom", False), + ], +) +def test_gateway_serves_the_eu_jev_decision_route_and_no_other_eu_jev_path(path: str, served: bool) -> None: + async def endpoint() -> None: + return None + + assert _is_gateway_route(Route(path, endpoint, methods=["POST"])) is served + + def test_every_app_mount_is_assigned_to_a_component(): """Every Mount on the proxy app must be consciously assigned to a component. diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 00051462b3c..1ce61a8e9b3 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -1203,24 +1203,35 @@ def test_body_snapshot_excludes_team_callback_credentials() -> None: }, proxy_request -def test_body_snapshot_keeps_the_stream_usage_injection_marker_out_of_the_stored_request() -> None: +@pytest.mark.parametrize( + ("route_type", "request_input"), + [ + ("acompletion", {"messages": [{"role": "user", "content": "hi"}]}), + ("atext_completion", {"prompt": "hi"}), + ], +) +def test_body_snapshot_keeps_the_stream_usage_injection_marker_out_of_the_stored_request( + route_type: str, request_input: dict[str, object] +) -> None: from litellm.proxy.common_request_processing import stream_usage_tracking_updates from litellm.proxy.litellm_pre_call_utils import refresh_proxy_server_request_body_snapshot from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD - request: Final = {"model": "gpt-5-mini", "messages": [{"role": "user", "content": "hi"}], "stream": True} + request: Final = {"model": "gpt-5-mini", **request_input, "stream": True} proxy_request: Final = {"body": dict(request)} data: Final = { **request, - **stream_usage_tracking_updates(request, {}, "acompletion", lambda: True), + **stream_usage_tracking_updates(request, {}, route_type, lambda: True), "proxy_server_request": proxy_request, } refresh_proxy_server_request_body_snapshot(data) assert data[STREAM_USAGE_INJECTED_FIELD] is True + assert data["_litellm_strip_stream_usage"] is True assert proxy_request["body"]["stream_options"] == {"include_usage": True} assert STREAM_USAGE_INJECTED_FIELD not in proxy_request["body"] + assert "_litellm_strip_stream_usage" not in proxy_request["body"] @pytest.mark.asyncio diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 02f88d444e4..023ef0b4606 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -13583,29 +13583,29 @@ def _stream_usage_generator_chunks(): def test_is_injected_stream_usage_artifact(): - from litellm.proxy.proxy_server import _is_injected_stream_usage_artifact + from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact from litellm.types.utils import ModelResponseStream, Usage content_chunk, finish_chunk, empty_choices_usage_chunk = _stream_usage_test_chunks() - assert _is_injected_stream_usage_artifact(empty_choices_usage_chunk) is True + assert is_injected_stream_usage_artifact(empty_choices_usage_chunk) is True synthetic_final_chunk = ModelResponseStream(model="gpt-5.4-nano") synthetic_final_chunk.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238) - assert _is_injected_stream_usage_artifact(synthetic_final_chunk) is True + assert is_injected_stream_usage_artifact(synthetic_final_chunk) is True azure_prompt_filter_chunk = ModelResponseStream(model="gpt-5.4-nano", choices=[]) - assert _is_injected_stream_usage_artifact(azure_prompt_filter_chunk) is True + assert is_injected_stream_usage_artifact(azure_prompt_filter_chunk) is True - assert _is_injected_stream_usage_artifact(content_chunk) is False - assert _is_injected_stream_usage_artifact(finish_chunk) is False + assert is_injected_stream_usage_artifact(content_chunk) is False + assert is_injected_stream_usage_artifact(finish_chunk) is False content_chunk_with_usage, finish_chunk_with_usage, _ = _stream_usage_test_chunks() content_chunk_with_usage.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238) finish_chunk_with_usage.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238) - assert _is_injected_stream_usage_artifact(content_chunk_with_usage) is False - assert _is_injected_stream_usage_artifact(finish_chunk_with_usage) is False + assert is_injected_stream_usage_artifact(content_chunk_with_usage) is False + assert is_injected_stream_usage_artifact(finish_chunk_with_usage) is False - assert _is_injected_stream_usage_artifact({"usage": {"prompt_tokens": 1}}) is False + assert is_injected_stream_usage_artifact({"usage": {"prompt_tokens": 1}}) is False async def _collect_async_data_generator_frames(request_data: dict) -> list: diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 43c017468ff..c182cd1ec75 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -2064,22 +2064,6 @@ async def test_framing_keeps_the_400_for_blocks_the_provider_counter_cannot_conv }, id="web_search_blocks", ), - pytest.param( - { - "messages": [{"role": "user", "content": "What is new?"}], - "tools": [{"name": "search", "input_schema": {"type": "object", "properties": {"query": True}}}], - }, - id="boolean_subschema_tool", - ), - pytest.param( - { - "messages": [{"role": "user", "content": "What is new?"}], - "tools": [ - {"name": "search", "input_schema": {"type": "object", "properties": {"tags": {"type": "array"}}}} - ], - }, - id="array_schema_without_items", - ), pytest.param( {"messages": [], "contents": [{"role": "user", "parts": [{"text": "What is new?"}]}]}, id="google_contents", @@ -2138,6 +2122,44 @@ async def test_framed_estimate_that_cannot_read_the_body_answers_the_provider_er assert readable.count_source == "local_estimate" +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("shorthand", "explicit"), + [ + pytest.param({"type": "array"}, {"type": "array", "items": {}}, id="array_schema_without_items"), + pytest.param(True, {}, id="boolean_subschema"), + ], +) +async def test_framed_estimate_counts_a_json_schema_shorthand_like_its_explicit_form( + monkeypatch: pytest.MonkeyPatch, shorthand: object, explicit: dict[str, object] +): + counter: Final = _ScriptedCounter({"us.moonshotai.kimi-k3": _provider_reply("kimi", status_code=400)}) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router(model_list=[{"model_name": "kimi", "litellm_params": {"model": "bedrock/us.moonshotai.kimi-k3"}}]), + ) + monkeypatch.setattr( + litellm.proxy.proxy_server, "_get_provider_token_counter", lambda deployment, model: (counter, model, "bedrock") + ) + + async def count(schema: object) -> TokenCountResponse: + return await token_counter( + request=TokenCountRequest( + model="kimi", + messages=[{"role": "user", "content": "What is new?"}], + tools=[{"name": "search", "input_schema": {"type": "object", "properties": {"tags": schema}}}], + ), + call_endpoint=True, + ) + + from_shorthand: Final = await count(shorthand) + from_explicit: Final = await count(explicit) + + assert from_shorthand.count_source == "local_estimate" + assert from_shorthand.total_tokens == from_explicit.total_tokens + + @pytest.mark.asyncio @pytest.mark.parametrize( "contents", diff --git a/tests/unit/rust_bridge/test_detectors.py b/tests/unit/rust_bridge/test_detectors.py index c02d2c12dbf..634e69910b9 100644 --- a/tests/unit/rust_bridge/test_detectors.py +++ b/tests/unit/rust_bridge/test_detectors.py @@ -33,6 +33,7 @@ MAPPED_FILE_INPUT: Final = [{"role": "user", "content": [{"type": "input_file", CACHED_TEXT: Final = {"type": "input_text", "text": "hi", "cache_control": {"type": "ephemeral"}} THINKING_TURN: Final = {"role": "assistant", "content": [{"type": "thinking", "thinking": "x", "signature": "s"}]} REDACTED: Final = {"type": "redacted_thinking", "data": "x"} +STRICT_TOOL: Final = {"name": "b", "input_schema": {"type": "object"}, "strict": True} TOOL_SEARCH_TYPE: Final = sorted(ANTHROPIC_TOOL_SEARCH_TOOL_TYPES)[0] @@ -192,6 +193,21 @@ def _wrap_nested_level(value: object, depth: int) -> object: (messages_detectors.tool_custom_field, {"tools": [{"name": "a"}, {"name": "b", "custom": {}}]}, True), (messages_detectors.tool_custom_field, {"tools": [{"name": "a", "defer_loading": True}]}, False), (messages_detectors.tool_custom_field, {}, False), + (messages_detectors.strict_tools_retry, {"drop_params": True, "tools": [{"name": "a"}, STRICT_TOOL]}, True), + ( + messages_detectors.strict_tools_retry, + {"drop_params": True, "tools": [{**STRICT_TOOL, "strict": False}]}, + False, + ), + ( + messages_detectors.strict_tools_retry, + {"additional_drop_params": ["metadata.user_id"], "tools": [STRICT_TOOL]}, + False, + ), + (messages_detectors.strict_tools_retry, {"tools": [STRICT_TOOL]}, False), + (messages_detectors.strict_tools_retry, {"drop_params": False, "tools": [STRICT_TOOL]}, False), + (messages_detectors.strict_tools_retry, {"drop_params": True, "tools": [{"name": "a"}]}, False), + (messages_detectors.strict_tools_retry, {"drop_params": True}, False), (detectors.drop_params, {"drop_params": True}, True), (detectors.drop_params, {"drop_params": False}, False), (detectors.drop_params, {"drop_params": " True "}, True), diff --git a/tests/unit/test_dashscope_image_generation.py b/tests/unit/test_dashscope_image_generation.py index 9c65399350c..0f35136bac3 100644 --- a/tests/unit/test_dashscope_image_generation.py +++ b/tests/unit/test_dashscope_image_generation.py @@ -14,6 +14,7 @@ import respx from pydantic import ValidationError import litellm +from litellm.cost_calculator import default_image_cost_calculator from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.dashscope.common_utils import ( DASHSCOPE_DEFAULT_ORIGIN, @@ -23,7 +24,8 @@ from litellm.llms.dashscope.image_generation.transformation import ( DEFAULT_API_BASE, DashScopeImageGenerationConfig, ) -from litellm.types.utils import ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils +from litellm.types.utils import ImageResponse, ModelInfo from litellm.utils import get_llm_provider # --------------------------------------------------------------------------- @@ -614,6 +616,16 @@ def test_transform_response_without_image_content_has_no_images( assert _transform_response(payload).data == [] +@pytest.mark.parametrize("usage", [None, "n/a", ["input_image_count", 2]]) +def test_transform_response_returns_the_images_whatever_shape_usage_has(usage: object): + response = _transform_response( + {"output": {"choices": [{"message": {"content": [{"image": "https://a.example/1.png"}]}}]}, "usage": usage} + ) + + assert [image.url for image in response.data] == ["https://a.example/1.png"] + assert "billable_input_image_count" not in response.hidden_params + + def test_transform_response_skips_content_items_without_an_image(): response = _transform_response( { @@ -672,3 +684,50 @@ def test_transform_response_rejects_malformed_payloads_without_echoing_them( _transform_response(payload) assert "input_value" not in str(exc_info.value) + + +@pytest.mark.parametrize( + ("model", "usage"), + ( + ("qwen-image-2.0", {"width": 2048, "height": 1536, "image_count": 1}), + ( + "qwen-image-3.0", + {"output_width": 2048, "output_height": 1536, "output_image_count": 1, "input_image_count": 0}, + ), + ), +) +def test_a_per_pixel_deployment_bills_the_size_dashscope_rendered_when_the_caller_sent_none( + model: str, usage: dict[str, int] +): + response = _transform_response( + { + "output": {"choices": [{"message": {"content": [{"image": "https://a.example/1.png"}]}}]}, + "usage": usage, + } + ) + + cost = CostCalculatorUtils.route_image_generation_cost_calculator( + model=model, + completion_response=response, + custom_llm_provider="dashscope", + call_type="image_generation", + model_info=ModelInfo(input_cost_per_pixel=1e-08), + ) + + assert cost == pytest.approx(2048 * 1536 * 1e-08) + + +@pytest.mark.parametrize("model", ("qwen-image-2.0", "qwen-image-2.0-pro", "qwen-image-3.0")) +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_shared_row_bills_a_generated_image_the_same_on_the_default_calculator_released_proxies_use(model: str): + dashscope_cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model=model, + completion_response=ImageResponse(data=[{"url": "https://a.example/1.png"}]), + custom_llm_provider="dashscope", + call_type="image_generation", + ) + + assert dashscope_cost > 0 + assert default_image_cost_calculator(model=f"dashscope/{model}", custom_llm_provider="dashscope") == pytest.approx( + dashscope_cost + ) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index f4e6acb93b4..a7c164639e0 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -48,7 +48,14 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.litellm_params import ControlOptions from litellm.types.llms.openai import AllMessageValues, HttpxBinaryResponseContent from litellm.types.prompts.init_prompts import PromptSpec -from litellm.types.utils import Delta, ModelResponseStream, StandardCallbackDynamicParams, StreamingChoices, Usage +from litellm.types.utils import ( + Delta, + ModelResponseStream, + StandardCallbackDynamicParams, + StreamingChoices, + TextCompletionResponse, + Usage, +) from tests.unit.llms.bedrock.slow_upstream import ( CONVERSE_ANSWER, STREAM_TIMEOUT_SECONDS, @@ -4136,6 +4143,95 @@ def test_mock_text_completion_stream_and_non_stream_report_the_same_zero_admissi assert non_stream.usage.prompt_tokens == 0 +def _carries_text_or_finish(frame: TextCompletionResponse) -> bool: + return frame.choices[0].text is not None or frame.choices[0].finish_reason is not None + + +@pytest.mark.asyncio +async def test_mock_atext_completion_stream_hides_the_usage_chunk_the_proxy_asked_for_on_the_clients_behalf(): + response: Final = await litellm.atext_completion( + model="openai/gpt-5.4-mini", + prompt="hi", + mock_response="ok then", + api_key="mock", + stream=True, + stream_options={"include_usage": True}, + _litellm_strip_stream_usage=True, + ) + + frames: Final[list[TextCompletionResponse]] = [frame async for frame in response] + + assert len(frames) == 4 + assert all(_carries_text_or_finish(frame) for frame in frames) + assert all(frame.usage is None for frame in frames) + + +def _native_completions_reply(outbound: list[dict[str, object]]) -> Callable[[httpx.Request], httpx.Response]: + def respond(request: httpx.Request) -> httpx.Response: + outbound.append(json.loads(request.content)) + base: Final = {"id": "cmpl-native", "object": "text_completion", "created": 1, "model": "code-model"} + frames: Final = ( + {**base, "choices": [{"index": 0, "text": "def ", "finish_reason": None, "logprobs": None}]}, + {**base, "choices": [{"index": 0, "text": "main", "finish_reason": "length", "logprobs": None}]}, + {**base, "choices": [], "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}}, + ) + content: Final = "".join(f"data: {json.dumps(frame)}\n\n" for frame in frames) + "data: [DONE]\n\n" + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=content.encode()) + + return respond + + +def _native_completions_upstream(outbound: list[dict[str, object]]) -> OpenAI: + transport: Final = httpx.MockTransport(_native_completions_reply(outbound)) + return OpenAI(api_key="transport-only", http_client=httpx.Client(transport=transport)) + + +def _async_native_completions_upstream(outbound: list[dict[str, object]]) -> AsyncOpenAI: + transport: Final = httpx.MockTransport(_native_completions_reply(outbound)) + return AsyncOpenAI(api_key="transport-only", http_client=httpx.AsyncClient(transport=transport)) + + +@pytest.mark.asyncio +async def test_native_atext_completion_stream_hides_the_usage_chunk_the_proxy_asked_for_on_the_clients_behalf(): + outbound: Final[list[dict[str, object]]] = [] + response: Final = await litellm.atext_completion( + model="text-completion-openai/code-model", + prompt="hi", + client=_async_native_completions_upstream(outbound), + api_key="transport-only", + stream=True, + stream_options={"include_usage": True}, + _litellm_strip_stream_usage=True, + ) + + frames: Final[list[TextCompletionResponse]] = [frame async for frame in response] + + assert [sent.get("stream_options") for sent in outbound] == [{"include_usage": True}] + assert [(frame.choices[0].text, frame.choices[0].finish_reason) for frame in frames] == [ + ("def ", None), + ("main", None), + (None, "length"), + ] + assert all(frame.usage is None for frame in frames) + + +def test_native_text_completion_stream_keeps_the_usage_chunk_the_client_asked_for(): + frames: Final = list( + litellm.text_completion( + model="text-completion-openai/code-model", + prompt="hi", + client=_native_completions_upstream([]), + api_key="transport-only", + stream=True, + stream_options={"include_usage": True}, + ) + ) + + assert len(frames) == 4 + assert all(_carries_text_or_finish(frame) for frame in frames[:3]) + assert (frames[-1].usage.prompt_tokens, frames[-1].usage.completion_tokens) == (10, 5) + + def test_mock_completion_stream_with_model_response(): """Test that mock_completion correctly handles stream=True with a ModelResponse as mock_response.""" from litellm import completion @@ -4293,7 +4389,6 @@ class TestCallTypesOCR: def test_stream_chunk_builder_text_completion_combines_text_and_usage(): from litellm.main import stream_chunk_builder_text_completion - from litellm.types.utils import TextCompletionResponse chunks = [ TextCompletionResponse( diff --git a/tests/unit/test_openapi_docs_gate.py b/tests/unit/test_openapi_docs_gate.py index fa8a5b1c7e3..b376178c8a9 100644 --- a/tests/unit/test_openapi_docs_gate.py +++ b/tests/unit/test_openapi_docs_gate.py @@ -5,10 +5,35 @@ here is the checker output parse and `introduced`, the diff scan that names the endpoint behind a breach. """ +import subprocess from pathlib import Path from typing import Final +from unittest.mock import patch import openapi_docs_gate as gate +import pytest + + +def test_base_counts_stops_when_the_native_build_fails(tmp_path: Path) -> None: + parent: Final = tmp_path / "oa_base" + parent.mkdir() + checker: Final = tmp_path / "check_openapi_docs.py" + checker.write_text("print('POST /v1/test OA001')\n") + with ( + patch("openapi_docs_gate.tempfile.mkdtemp", return_value=str(parent)), + patch( + "subprocess.run", + side_effect=( + subprocess.CompletedProcess([], 0, stdout="", stderr=""), + subprocess.CompletedProcess([], 1, stdout="", stderr="build failed"), + subprocess.CompletedProcess([], 0, stdout="", stderr=""), + ), + ) as run, + patch("signal.signal"), + ): + with pytest.raises(SystemExit, match="uv exited 1"): + gate.base_counts("HEAD", repo_root=tmp_path, checker=checker) + assert run.call_count == 3 def test_editing_the_checker_rekeys_the_base_counts(tmp_path: Path) -> None: diff --git a/tests/unit/text_completions/test_streaming_iterator.py b/tests/unit/text_completions/test_streaming_iterator.py index b7edffc0753..e5459a5a0cd 100644 --- a/tests/unit/text_completions/test_streaming_iterator.py +++ b/tests/unit/text_completions/test_streaming_iterator.py @@ -142,3 +142,90 @@ def test_sync_text_completion_stream_maps_upstream_failure_after_content() -> No next(wrapper) assert (error.value.model, error.value.llm_provider) == ("test-model", "openai") + + +def _stream_with_trailing_usage_chunk() -> tuple[ModelResponseStream, ...]: + return ( + ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="Hi"))]), + ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="length")]), + ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta())], + usage=Usage(prompt_tokens=19, completion_tokens=5, total_tokens=24), + ), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_stream", (False, True)) +@pytest.mark.parametrize( + ("usage_injected_by_proxy", "expected_frames", "expected_usage"), + ( + (True, [("Hi", None), (None, "length")], [None, None]), + (False, [("Hi", None), (None, "length"), (None, None)], [None, None, (19, 5)]), + ), +) +async def test_text_completion_stream_hides_only_the_usage_chunk_the_proxy_asked_for( + async_stream: bool, + usage_injected_by_proxy: bool, + expected_frames: list[tuple[str | None, str | None]], + expected_usage: list[tuple[int, int] | None], +) -> None: + async def chunks() -> AsyncIterator[ModelResponseStream]: + for chunk in _stream_with_trailing_usage_chunk(): + yield chunk + + wrapper: Final = TextCompletionStreamWrapper( + completion_stream=chunks() if async_stream else iter(_stream_with_trailing_usage_chunk()), + model="gpt-5.4-mini", + stream_options={"include_usage": True}, + usage_injected_by_proxy=usage_injected_by_proxy, + ) + frames: Final = [frame async for frame in wrapper] if async_stream else list(wrapper) + + assert [(frame.choices[0].text, frame.choices[0].finish_reason) for frame in frames] == expected_frames + assert [ + None if frame.usage is None else (frame.usage.prompt_tokens, frame.usage.completion_tokens) for frame in frames + ] == expected_usage + + +def test_text_completion_stream_keeps_a_finish_frame_but_hides_the_usage_the_proxy_asked_for() -> None: + finish_with_usage: Final = ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")], + usage=Usage(prompt_tokens=27, completion_tokens=5, total_tokens=32), + ) + wrapper: Final = TextCompletionStreamWrapper( + completion_stream=iter((finish_with_usage,)), + model="claude-haiku-4-5", + stream_options={"include_usage": True}, + usage_injected_by_proxy=True, + ) + + frames: Final = list(wrapper) + + assert [(frame.choices[0].finish_reason, frame.usage) for frame in frames] == [("stop", None)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_stream", (False, True)) +async def test_text_completion_stream_hides_the_empty_prompt_filter_frame_when_the_proxy_asked_for_usage( + async_stream: bool, +) -> None: + prompt_filter_frame: Final = ModelResponseStream(id="", choices=[]) + upstream: Final = (prompt_filter_frame, *_stream_with_trailing_usage_chunk()) + + async def chunks() -> AsyncIterator[ModelResponseStream]: + for chunk in upstream: + yield chunk + + wrapper: Final = TextCompletionStreamWrapper( + completion_stream=chunks() if async_stream else iter(upstream), + model="gpt-5.4-mini", + stream_options={"include_usage": True}, + usage_injected_by_proxy=True, + ) + frames: Final = [frame async for frame in wrapper] if async_stream else list(wrapper) + + assert [(frame.choices[0].text, frame.choices[0].finish_reason) for frame in frames] == [ + ("Hi", None), + (None, "length"), + ] diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 8704dc55c38..48def46e3fe 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -25,6 +25,7 @@ "clsx": "^2.1.1", "date-fns": "^4.4.0", "dayjs": "1.11.19", + "i18next": "26.5.11", "jwt-decode": "4.0.0", "lucide-react": "0.513.0", "moment": "2.31.0", @@ -43,6 +44,7 @@ "react-dom": "19.2.8", "react-hook-form": "7.82.0", "react-hotkeys-hook": "5.3.3", + "react-i18next": "17.1.9", "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", "react-reconciler": "0.33.0", @@ -412,9 +414,9 @@ } }, "node_modules/@babel/runtime": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", - "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", + "version": "7.29.10", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.10.tgz", + "integrity": "sha512-NpugTzbKgGrNe45Ui32cN8vCFFMxVtP/9rtfwv0nP5FRDAjvYBxJcLVpYA9mr1jxP6cnIT7x9LUzuDTbWYYNRA==", "license": "MIT", "engines": { "node": ">=6.9.0" @@ -7319,6 +7321,15 @@ "dev": true, "license": "MIT" }, + "node_modules/html-parse-stringify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/html-parse-stringify/-/html-parse-stringify-4.0.1.tgz", + "integrity": "sha512-0zHsZJrK7S3K2aucXWL6ycoYJ/iNtIcFHC/nYQgFklPtrv5LpJctIiSCroWZWeuoXvuyFdzp6KzjJQ+OT5MfFw==", + "license": "MIT", + "funding": { + "url": "https://locize.com" + } + }, "node_modules/html-url-attributes": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/html-url-attributes/-/html-url-attributes-3.0.1.tgz", @@ -7357,6 +7368,34 @@ "node": ">= 14" } }, + "node_modules/i18next": { + "version": "26.5.11", + "resolved": "https://registry.npmjs.org/i18next/-/i18next-26.5.11.tgz", + "integrity": "sha512-Wc0klBPKxqZ2tL2qdAnF8xvvnWgr1IMGSY2ikTiAvaFVbz7xiFARtkoU4RidBDw1SyziGROp0qxu6qnJ7l2Wug==", + "funding": [ + { + "type": "individual", + "url": "https://www.locize.com/i18next" + }, + { + "type": "individual", + "url": "https://www.i18next.com/how-to/faq#i18next-is-awesome.-how-can-i-support-the-project" + }, + { + "type": "individual", + "url": "https://www.locize.com" + } + ], + "license": "MIT", + "peerDependencies": { + "typescript": "^5 || ^6 || ^7" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -10597,6 +10636,33 @@ "react-dom": ">=16.8.0" } }, + "node_modules/react-i18next": { + "version": "17.1.9", + "resolved": "https://registry.npmjs.org/react-i18next/-/react-i18next-17.1.9.tgz", + "integrity": "sha512-OgYqIH13XHPlWTaaI795Dz+k2AdKnXcOxNQVnf750T8Ij0TYWp1nYvXRpoJBulPqlyk2OAbn1PM4tFKiIj/byA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7", + "html-parse-stringify": "^4.0.1", + "use-sync-external-store": "^1.6.0" + }, + "peerDependencies": { + "i18next": ">= 26.2.0", + "react": ">= 16.8.0", + "typescript": "^5 || ^6 || ^7" + }, + "peerDependenciesMeta": { + "react-dom": { + "optional": true + }, + "react-native": { + "optional": true + }, + "typescript": { + "optional": true + } + } + }, "node_modules/react-intersection-observer": { "version": "11.0.1", "resolved": "https://registry.npmjs.org/react-intersection-observer/-/react-intersection-observer-11.0.1.tgz", @@ -12105,7 +12171,7 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index b1b8fde44ee..f65316439bf 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -42,6 +42,7 @@ "clsx": "^2.1.1", "date-fns": "^4.4.0", "dayjs": "1.11.19", + "i18next": "26.5.11", "jwt-decode": "4.0.0", "lucide-react": "0.513.0", "moment": "2.31.0", @@ -60,6 +61,7 @@ "react-dom": "19.2.8", "react-hook-form": "7.82.0", "react-hotkeys-hook": "5.3.3", + "react-i18next": "17.1.9", "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", "react-reconciler": "0.33.0", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.test.tsx index 6707330a54c..f4a299a5dcc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.test.tsx @@ -14,7 +14,7 @@ interface FakeRun { status: string; workflow_type: string; created_at: string; - metadata: { title?: string; state?: string } | null; + metadata: Record | null; } const RUNS: FakeRun[] = [ @@ -34,16 +34,16 @@ const RUNS: FakeRun[] = [ }, ]; -function mockFetch(runs: FakeRun[]) { +function mockFetch(runs: FakeRun[], events: FakeEvent[] = [], messages: FakeMessage[] = []) { return vi.fn((url: string) => { if (url.includes("/runs?limit")) { return Promise.resolve({ ok: true, json: () => Promise.resolve({ runs }) }); } if (url.includes("/events")) { - return Promise.resolve({ ok: true, json: () => Promise.resolve({ events: [] }) }); + return Promise.resolve({ ok: true, json: () => Promise.resolve({ events }) }); } if (url.includes("/messages")) { - return Promise.resolve({ ok: true, json: () => Promise.resolve({ messages: [] }) }); + return Promise.resolve({ ok: true, json: () => Promise.resolve({ messages }) }); } return Promise.resolve({ ok: false, status: 404, json: () => Promise.resolve({}) }); }); @@ -114,7 +114,7 @@ interface FakeMessage { created_at: string; } -const DETAIL_RUN = { +const DETAIL_RUN: FakeRun = { run_id: "run-aaaaaaaa-1111", status: "completed", workflow_type: "grill", @@ -151,33 +151,21 @@ const DETAIL_MESSAGES: FakeMessage[] = [ }, ]; -function mockDetailFetch(events: FakeEvent[], messages: FakeMessage[]) { - return vi.fn((url: string) => { - if (url.includes("/runs?limit")) { - return Promise.resolve({ ok: true, json: () => Promise.resolve({ runs: [DETAIL_RUN] }) }); - } - if (url.includes("/events")) { - return Promise.resolve({ ok: true, json: () => Promise.resolve({ events }) }); - } - if (url.includes("/messages")) { - return Promise.resolve({ ok: true, json: () => Promise.resolve({ messages }) }); - } - return Promise.resolve({ ok: false, status: 404, json: () => Promise.resolve({}) }); - }); -} - -async function openDetailDrawer(events = DETAIL_EVENTS, messages = DETAIL_MESSAGES) { +async function openRunDrawer(run: FakeRun, events: FakeEvent[], messages: FakeMessage[]) { const user = userEvent.setup(); - const fetchSpy = mockDetailFetch(events, messages); + const fetchSpy = mockFetch([run], events, messages); vi.stubGlobal("fetch", fetchSpy); render(); - await user.click(await screen.findByText("First run")); + await user.click(await screen.findByText(run.metadata?.title ?? run.workflow_type)); const drawer = await screen.findByRole("dialog"); - await waitFor(() => expect(within(drawer).getByText("Timeline")).toBeInTheDocument()); + expect(await within(drawer).findByText("Timeline")).toBeInTheDocument(); return { user, fetchSpy, drawer }; } +const openDetailDrawer = (events = DETAIL_EVENTS, messages = DETAIL_MESSAGES) => + openRunDrawer(DETAIL_RUN, events, messages); + describe("WorkflowRuns detail drawer", () => { it("shows the run's identity and metadata fields", async () => { const { drawer } = await openDetailDrawer(); @@ -244,3 +232,219 @@ describe("WorkflowRuns detail drawer", () => { await waitFor(() => expect(screen.queryAllByRole("dialog")).toHaveLength(0)); }); }); + +const FULL_RUN: FakeRun = { + run_id: "run-cccccccc-3333", + status: "running", + workflow_type: "autoship", + created_at: "2026-01-01T00:00:00.000Z", + metadata: { + title: "Full run", + state: "reviewing", + pr_url: "https://example.com/pr/2", + worktree_path: "/tmp/wt/full", + grill_session_id: "grill-1", + session_id: "sess-1", + plan_text: "step ".repeat(26), + }, +}; + +const TIMED_EVENTS: FakeEvent[] = [ + { + event_id: "ev-a", + event_type: "step.started", + step_name: "plan", + sequence_number: 1, + created_at: "2026-01-01T00:00:00.100Z", + data: null, + }, + { + event_id: "ev-b", + event_type: "step.started", + step_name: "implement", + sequence_number: 2, + created_at: "2026-01-01T00:00:00.165Z", + data: null, + }, + { + event_id: "ev-c", + event_type: "hook.waiting", + step_name: "review", + sequence_number: 3, + created_at: "2026-01-01T00:00:01.365Z", + data: null, + }, +]; + +describe("WorkflowRuns page copy", () => { + it("announces the loading state while runs are being fetched", async () => { + vi.stubGlobal( + "fetch", + vi.fn(() => new Promise(() => {})), + ); + render(); + + expect(await screen.findByText("Loading workflow runs…")).toBeInTheDocument(); + }); + + it("labels the page header, the table columns and the search box", async () => { + vi.stubGlobal("fetch", mockFetch(RUNS)); + render(); + await screen.findByText("First run"); + + expect(screen.getByText("Workflow Runs")).toBeInTheDocument(); + expect(screen.getByText("Durable state tracking for agents and automated workflows")).toBeInTheDocument(); + expect(screen.getAllByRole("columnheader").map((header) => header.textContent)).toEqual([ + "Run", + "Type", + "Status", + "Created", + ]); + expect(screen.getByPlaceholderText("Search runs…")).toBeInTheDocument(); + }); + + it("names every column in the Columns menu", async () => { + const user = userEvent.setup(); + vi.stubGlobal("fetch", mockFetch(RUNS)); + render(); + await screen.findByText("First run"); + + await user.click(screen.getByRole("button", { name: "Columns" })); + + expect((await screen.findAllByRole("menuitemcheckbox")).map((item) => item.textContent)).toEqual([ + "Run", + "Type", + "Status", + "Created", + ]); + }); + + it("labels the filters drawer and offers every run status as a filter", async () => { + const user = userEvent.setup(); + vi.stubGlobal("fetch", mockFetch(RUNS)); + render(); + await screen.findByText("First run"); + + await user.click(screen.getByRole("button", { name: "Filters" })); + const filters = await screen.findByRole("dialog", { name: "Filters" }); + expect(filters).toHaveAccessibleDescription("Narrow down workflow runs"); + expect(within(filters).getByText("Status")).toBeInTheDocument(); + expect(within(filters).getByText("Type")).toBeInTheDocument(); + expect(within(filters).getByPlaceholderText("Filter by type…")).toBeInTheDocument(); + + const statusSelect = within(filters).getByRole("combobox"); + expect(statusSelect).toHaveTextContent("All statuses"); + await user.click(statusSelect); + expect((await screen.findAllByRole("option")).map((option) => option.textContent)).toEqual([ + "All statuses", + "Pending", + "Running", + "Paused", + "Completed", + "Failed", + ]); + + await user.click(screen.getByRole("option", { name: "Running" })); + expect(statusSelect).toHaveTextContent("Running"); + }); + + it("labels the detail drawer, its controls and every primary metadata field", async () => { + const { user, drawer } = await openRunDrawer(FULL_RUN, [TIMED_EVENTS[0]], []); + + expect(drawer).toHaveAccessibleName("Workflow run details"); + expect(drawer).toHaveAccessibleDescription("Metadata, timeline and messages for the selected workflow run"); + for (const label of ["status", "created", "pr", "state", "worktree", "grill session", "session"]) { + expect(within(drawer).getByText(label)).toBeInTheDocument(); + } + expect(within(drawer).getByRole("button", { name: "close" })).toBeInTheDocument(); + expect(within(drawer).getByRole("button", { name: "Refresh" })).toBeInTheDocument(); + expect(within(drawer).getByText("1 event")).toBeInTheDocument(); + + await user.click(within(drawer).getByRole("button", { name: /Messages/ })); + expect(await within(drawer).findByText("No messages")).toBeInTheDocument(); + }); + + it("counts zero events in the timeline header", async () => { + const { drawer } = await openRunDrawer(FULL_RUN, [], []); + + expect(within(drawer).getByText("0 events")).toBeInTheDocument(); + }); + + it("collapses a long metadata value behind a more toggle that flips to less", async () => { + const { user, drawer } = await openRunDrawer(FULL_RUN, [], []); + + await user.click(within(drawer).getByRole("button", { name: "more" })); + + expect(within(drawer).getByRole("button", { name: "less" })).toBeInTheDocument(); + }); + + it("formats timeline durations in milliseconds under a second and in seconds above it", async () => { + const { drawer } = await openRunDrawer(FULL_RUN, TIMED_EVENTS, []); + + expect(within(drawer).getByText("65ms")).toBeInTheDocument(); + expect(within(drawer).getByText("1.2s")).toBeInTheDocument(); + expect(within(drawer).getAllByText("1.4s")).toHaveLength(2); + }); +}); + +describe("WorkflowRuns relative times", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("shows created times in the largest whole unit and passes an unparseable date through", async () => { + const now = Date.parse("2026-03-10T12:00:00Z"); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(now); + const runCreated = (id: string, createdAt: string): FakeRun => ({ + run_id: `run-${id}`, + status: "completed", + workflow_type: `type-${id}`, + created_at: createdAt, + metadata: null, + }); + const secondsAgo = (seconds: number) => new Date(now - seconds * 1000).toISOString(); + vi.stubGlobal( + "fetch", + mockFetch([ + runCreated("a", secondsAgo(59)), + runCreated("b", secondsAgo(60)), + runCreated("c", secondsAgo(3600)), + runCreated("d", secondsAgo(86400)), + runCreated("e", "not-a-date"), + ]), + ); + render(); + + expect(await screen.findByText("59s ago")).toBeInTheDocument(); + expect(screen.getByText("1m ago")).toBeInTheDocument(); + expect(screen.getByText("1h ago")).toBeInTheDocument(); + expect(screen.getByText("1d ago")).toBeInTheDocument(); + expect(screen.getByText("not-a-date")).toBeInTheDocument(); + }); +}); + +describe("WorkflowRuns detail drawer relative times", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("labels the run's age, each message's age and every field of a hovered timeline event", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.parse("2026-01-01T00:00:10Z")); + const { user, drawer } = await openDetailDrawer(); + + expect(within(drawer).getByText("10s ago")).toBeInTheDocument(); + + await user.hover(within(drawer).getByText("step.started")); + const tooltip = await screen.findByText( + (_, element) => element?.getAttribute("data-slot") === "tooltip-content", + {}, + { timeout: 3000 }, + ); + expect(tooltip).toHaveTextContent('type: step.startedstep: planseq: 1time: 9s agodata: {"attempt":1}'); + + await user.click(within(drawer).getByRole("button", { name: /Messages/ })); + expect(await within(drawer).findByText("8s ago")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.tsx index ba4c23fcc9c..f02bbd0688c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.tsx @@ -1,6 +1,8 @@ import React, { useState, useEffect, useCallback, useMemo } from "react"; import { ArrowLeft, ChevronDown, RefreshCw } from "lucide-react"; import type { ColumnDef, ColumnFiltersState } from "@tanstack/react-table"; +import type { ParseKeys, TFunction } from "i18next"; +import { useTranslation } from "react-i18next"; import { getGlobalLitellmHeaderName, proxyBaseUrl } from "@/components/networking"; import { DataTable, @@ -70,12 +72,12 @@ const STATUS_DOT: Record = { }; const RUN_STATUS_OPTIONS: RunStatus[] = ["pending", "running", "paused", "completed", "failed"]; -const STATUS_LABELS: Record = { - pending: "Pending", - running: "Running", - paused: "Paused", - completed: "Completed", - failed: "Failed", +const STATUS_LABEL_KEYS: Record = { + pending: "workflows.status.pending", + running: "workflows.status.running", + paused: "workflows.status.paused", + completed: "workflows.status.completed", + failed: "workflows.status.failed", }; const EVENT_COLOR: Record = { @@ -91,22 +93,22 @@ function eventStyle(type: string) { // ── helpers ─────────────────────────────────────────────────────────────────── -function timeAgo(iso: string): string { +function timeAgo(iso: string, t: TFunction): string { const diff = Date.now() - new Date(iso).getTime(); if (isNaN(diff)) return iso; const s = Math.floor(diff / 1000); - if (s < 60) return `${s}s ago`; + if (s < 60) return t("workflows.timeAgo.seconds", { count: s }); const m = Math.floor(s / 60); - if (m < 60) return `${m}m ago`; + if (m < 60) return t("workflows.timeAgo.minutes", { count: m }); const h = Math.floor(m / 60); - if (h < 24) return `${h}h ago`; - return `${Math.floor(h / 24)}d ago`; + if (h < 24) return t("workflows.timeAgo.hours", { count: h }); + return t("workflows.timeAgo.days", { count: Math.floor(h / 24) }); } -function fmtDuration(ms: number): string { +function fmtDuration(ms: number, t: TFunction): string { if (ms < 0) return ""; - if (ms < 1000) return `${ms}ms`; - return `${(ms / 1000).toFixed(1)}s`; + if (ms < 1000) return t("workflows.duration.ms", { count: ms }); + return t("workflows.duration.seconds", { value: (ms / 1000).toFixed(1) }); } function runTitle(run: WorkflowRun): string { @@ -130,6 +132,7 @@ const StatusDot: React.FC<{ status: RunStatus; className?: string }> = ({ status const TRUNCATE_AT = 120; const TruncatedValue: React.FC<{ value: string }> = ({ value }) => { + const { t } = useTranslation(); const [expanded, setExpanded] = useState(false); if (value.length <= TRUNCATE_AT) { return {value}; @@ -138,7 +141,7 @@ const TruncatedValue: React.FC<{ value: string }> = ({ value }) => { {expanded ? value : value.slice(0, TRUNCATE_AT) + "…"} ); @@ -147,13 +150,14 @@ const TruncatedValue: React.FC<{ value: string }> = ({ value }) => { // ── metadata card ───────────────────────────────────────────────────────────── const MetadataCard: React.FC<{ run: WorkflowRun }> = ({ run }) => { + const { t } = useTranslation(); const meta = run.metadata ?? {}; const primaryFields: { key: string; label: string }[] = [ - { key: "state", label: "state" }, - { key: "worktree_path", label: "worktree" }, - { key: "grill_session_id", label: "grill session" }, - { key: "session_id", label: "session" }, + { key: "state", label: t("workflows.metadata.state") }, + { key: "worktree_path", label: t("workflows.metadata.worktree") }, + { key: "grill_session_id", label: t("workflows.metadata.grillSession") }, + { key: "session_id", label: t("workflows.metadata.session") }, ]; const primaryKeys = new Set(["title", ...primaryFields.map((f) => f.key)]); @@ -175,15 +179,15 @@ const MetadataCard: React.FC<{ run: WorkflowRun }> = ({ run }) => { {/* key fields grid */}
- + {run.status} - - {timeAgo(run.created_at)} + + {timeAgo(run.created_at, t)} {meta.pr_url && ( - + = ({ run, events }) => { + const { t } = useTranslation(); if (events.length === 0) { - return
No events recorded
; + return
{t("workflows.drawer.noEvents")}
; } const runStart = new Date(run.created_at).getTime(); const eventTimes = events.map((e) => new Date(e.created_at).getTime()); const lastTime = Math.max(...eventTimes); const totalSpan = Math.max(lastTime - runStart, 1); - const totalDur = fmtDuration(lastTime - runStart); + const totalDur = fmtDuration(lastTime - runStart, t); return ( @@ -275,7 +280,7 @@ const GanttTimeline: React.FC<{ : lastTime + Math.max(totalSpan * 0.12, 500); const widthPct = Math.max(8, ((nextTime - evTime) / totalSpan) * 100); const style = eventStyle(ev.event_type); - const dur = fmtDuration(nextTime - evTime); + const dur = fmtDuration(nextTime - evTime, t); return ( @@ -303,24 +308,24 @@ const GanttTimeline: React.FC<{
- type: + {t("workflows.tooltip.type")} {ev.event_type}
- step: + {t("workflows.tooltip.step")} {ev.step_name}
- seq: + {t("workflows.tooltip.seq")} {ev.sequence_number}
- time: - {timeAgo(ev.created_at)} + {t("workflows.tooltip.time")} + {timeAgo(ev.created_at, t)}
{ev.data && Object.keys(ev.data).length > 0 && (
- data: + {t("workflows.tooltip.data")} {JSON.stringify(ev.data)}
)} @@ -346,15 +351,18 @@ const ROLE_COLOR: Record = { tool_result: "text-warning", }; -const MessageRow: React.FC<{ msg: WorkflowRunMessage }> = ({ msg }) => ( -
- [{msg.role}] -
- {msg.content} - {timeAgo(msg.created_at)} +const MessageRow: React.FC<{ msg: WorkflowRunMessage }> = ({ msg }) => { + const { t } = useTranslation(); + return ( +
+ [{msg.role}] +
+ {msg.content} + {timeAgo(msg.created_at, t)} +
-
-); + ); +}; // ── collapsible section ─────────────────────────────────────────────────────── @@ -379,6 +387,7 @@ const DetailSection: React.FC<{ // ── main component ──────────────────────────────────────────────────────────── const WorkflowRuns: React.FC = ({ accessToken }) => { + const { t } = useTranslation(); const [runs, setRuns] = useState([]); const [loadingRuns, setLoadingRuns] = useState(false); const [selectedRun, setSelectedRun] = useState(null); @@ -455,8 +464,8 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { { id: "run", accessorFn: (row) => `${runTitle(row)} ${row.run_id}`, - header: "Run", - meta: { title: "Run", skeleton: "twoLine" }, + header: t("workflows.table.columns.run"), + meta: { title: t("workflows.table.columns.run"), skeleton: "twoLine" }, cell: ({ row }) => { const run = row.original; return ( @@ -472,8 +481,8 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { }, { accessorKey: "workflow_type", - header: "Type", - meta: { title: "Type" }, + header: t("workflows.table.columns.type"), + meta: { title: t("workflows.table.columns.type") }, filterFn: "includesString", cell: ({ row }) => ( {row.original.workflow_type} @@ -482,8 +491,8 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { { id: "status", accessorKey: "status", - header: "Status", - meta: { title: "Status" }, + header: t("workflows.table.columns.status"), + meta: { title: t("workflows.table.columns.status") }, filterFn: "equalsString", cell: ({ row }) => { const run = row.original; @@ -497,22 +506,22 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { }, { accessorKey: "created_at", - header: "Created", - meta: { title: "Created" }, - cell: ({ row }) => {timeAgo(row.original.created_at)}, + header: t("workflows.table.columns.created"), + meta: { title: t("workflows.table.columns.created") }, + cell: ({ row }) => {timeAgo(row.original.created_at, t)}, }, ], - [], + [t], ); + const statusLabels = Object.fromEntries(RUN_STATUS_OPTIONS.map((status) => [status, t(STATUS_LABEL_KEYS[status])])); + return (
{/* page header */}
-
Workflow Runs
-
- Durable state tracking for agents and automated workflows -
+
{t("workflows.header.title")}
+
{t("workflows.header.subtitle")}
= ({ accessToken }) => { columns={columns} getRowId={(run) => run.run_id} isLoading={loadingRuns} - loadingMessage="Loading workflow runs…" - noDataMessage={
No workflow runs yet
} + loadingMessage={t("workflows.table.loading")} + noDataMessage={ +
{t("workflows.table.empty")}
+ } paginationMode="client" pageSizeOptions={[50, 100]} filterMode="client" @@ -537,7 +548,7 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { table={table} searchValue={globalFilter} onSearchChange={setGlobalFilter} - searchPlaceholder="Search runs…" + searchPlaceholder={t("workflows.table.searchPlaceholder")} onRefresh={fetchRuns} isRefreshing={loadingRuns} onOpenFilters={() => setFiltersOpen(true)} @@ -546,35 +557,35 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { table={table} open={filtersOpen} onOpenChange={setFiltersOpen} - title="Filters" - description="Narrow down workflow runs" + title={t("workflows.filters.title")} + description={t("workflows.filters.description")} > {({ get, set }) => ( <> - + - + set("workflow_type", event.target.value)} - placeholder="Filter by type…" + placeholder={t("workflows.filters.typePlaceholder")} /> @@ -590,10 +601,8 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { showCloseButton={false} className="overflow-y-auto p-0 data-[side=right]:w-full data-[side=right]:sm:max-w-[680px]" > - Workflow run details - - Metadata, timeline and messages for the selected workflow run - + {t("workflows.drawer.srTitle")} + {t("workflows.drawer.srDescription")} {!selectedRun ? null : loadingDetail ? (
@@ -609,11 +618,11 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { onClick={() => setDrawerOpen(false)} > - close + {t("workflows.drawer.close")}
@@ -623,19 +632,17 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { {/* collapsible sections */}
- {events.length} {events.length === 1 ? "event" : "events"} - - } + title={t("workflows.drawer.timeline")} + meta={t("workflows.drawer.eventCount", { count: events.length })} defaultOpen > - + {messages.length === 0 ? ( -
No messages
+
+ {t("workflows.drawer.noMessages")} +
) : (
{messages.map((msg) => ( diff --git a/ui/litellm-dashboard/src/app/layout.tsx b/ui/litellm-dashboard/src/app/layout.tsx index 3a9d130839c..a8ff93f69f6 100644 --- a/ui/litellm-dashboard/src/app/layout.tsx +++ b/ui/litellm-dashboard/src/app/layout.tsx @@ -10,6 +10,7 @@ import ReactQueryProvider from "@/contexts/ReactQueryProvider"; import HotkeysProvider from "@/contexts/HotkeysProvider"; import { TerminologyProvider } from "@/contexts/TerminologyContext"; import { Toaster } from "@/components/ui/sonner"; +import I18nProvider from "@/i18n/I18nProvider"; const inter = Inter({ subsets: ["latin"] }); @@ -29,18 +30,20 @@ export default function RootLayout({ // cannot predict; suppressHydrationWarning confines that mismatch to this element. - - - - - - {children} - - - - - - + + + + + + + {children} + + + + + + + ); diff --git a/ui/litellm-dashboard/src/i18n/I18nProvider.tsx b/ui/litellm-dashboard/src/i18n/I18nProvider.tsx new file mode 100644 index 00000000000..f0ecdfe2568 --- /dev/null +++ b/ui/litellm-dashboard/src/i18n/I18nProvider.tsx @@ -0,0 +1,9 @@ +"use client"; + +import { I18nextProvider } from "react-i18next"; + +import i18n from "./i18n"; + +export default function I18nProvider({ children }: { children: React.ReactNode }) { + return {children}; +} diff --git a/ui/litellm-dashboard/src/i18n/i18n.test-d.ts b/ui/litellm-dashboard/src/i18n/i18n.test-d.ts new file mode 100644 index 00000000000..baa8c757311 --- /dev/null +++ b/ui/litellm-dashboard/src/i18n/i18n.test-d.ts @@ -0,0 +1,10 @@ +import type { TFunction } from "i18next"; +import { expectTypeOf, test } from "vitest"; + +test("translation keys come from the English message file", () => { + const t = {} as TFunction; + + expectTypeOf(t("workflows.drawer.eventCount", { count: 2 })).toBeString(); + // @ts-expect-error + t("workflows.header.titel"); +}); diff --git a/ui/litellm-dashboard/src/i18n/i18n.ts b/ui/litellm-dashboard/src/i18n/i18n.ts new file mode 100644 index 00000000000..0e9c364a0b3 --- /dev/null +++ b/ui/litellm-dashboard/src/i18n/i18n.ts @@ -0,0 +1,23 @@ +import i18n, { type InitOptions } from "i18next"; +import { initReactI18next } from "react-i18next"; + +import en from "./locales/en.json"; + +declare module "i18next" { + interface CustomTypeOptions { + defaultNS: "translation"; + resources: { translation: typeof en }; + } +} + +const options: InitOptions = { + lng: "en", + fallbackLng: "en", + interpolation: { escapeValue: false }, + resources: { en: { translation: en } }, + initAsync: false, +}; + +void i18n.use(initReactI18next).init(options); + +export default i18n; diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json new file mode 100644 index 00000000000..9a65e074b25 --- /dev/null +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -0,0 +1,76 @@ +{ + "workflows": { + "header": { + "title": "Workflow Runs", + "subtitle": "Durable state tracking for agents and automated workflows" + }, + "table": { + "columns": { + "run": "Run", + "type": "Type", + "status": "Status", + "created": "Created" + }, + "loading": "Loading workflow runs…", + "empty": "No workflow runs yet", + "searchPlaceholder": "Search runs…" + }, + "filters": { + "title": "Filters", + "description": "Narrow down workflow runs", + "status": "Status", + "type": "Type", + "allStatuses": "All statuses", + "typePlaceholder": "Filter by type…" + }, + "status": { + "pending": "Pending", + "running": "Running", + "paused": "Paused", + "completed": "Completed", + "failed": "Failed" + }, + "drawer": { + "srTitle": "Workflow run details", + "srDescription": "Metadata, timeline and messages for the selected workflow run", + "close": "close", + "refresh": "Refresh", + "timeline": "Timeline", + "eventCount_one": "{{count}} event", + "eventCount_other": "{{count}} events", + "messages": "Messages", + "noMessages": "No messages", + "noEvents": "No events recorded" + }, + "metadata": { + "status": "status", + "created": "created", + "pr": "pr", + "state": "state", + "worktree": "worktree", + "grillSession": "grill session", + "session": "session" + }, + "truncate": { + "more": "more", + "less": "less" + }, + "tooltip": { + "type": "type: ", + "step": "step: ", + "seq": "seq: ", + "time": "time: ", + "data": "data: " + }, + "timeAgo": { + "seconds": "{{count}}s ago", + "minutes": "{{count}}m ago", + "hours": "{{count}}h ago", + "days": "{{count}}d ago" + }, + "duration": { + "ms": "{{count}}ms", + "seconds": "{{value}}s" + } + } +} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6c4c5ba3b11..d542880f868 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -5362,6 +5362,32 @@ export interface paths { patch: operations["assemblyai_proxy_route_eu_assemblyai__endpoint__patch"]; trace?: never; }; + "/eu_jev/v1/systemone": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Eu Jev Proxy Route + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and + * `stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + post: operations["eu_jev_proxy_route_eu_jev_v1_systemone_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/extraction": { parameters: { query?: never; @@ -32287,6 +32313,50 @@ export interface components { [key: string]: unknown; }; }; + /** EuJevDecisionResponse */ + EuJevDecisionResponse: { + /** + * Answers + * @description One answer per question, under the same names + */ + answers: { + [key: string]: components["schemas"]["ChoiceAnswer"] | components["schemas"]["NoulAnswer"] | components["schemas"]["ScoreAnswer"]; + }; + meta: components["schemas"]["EuJevMeta"]; + /** + * Model + * @description The release that answered, such as eu-jeff-1.0.0 + */ + model: string; + usage: components["schemas"]["EuJevUsage"]; + }; + /** EuJevMeta */ + EuJevMeta: { + /** + * Cost Eur + * @description The exact charge in euros, on authenticated requests + */ + cost_eur?: string | null; + /** + * Latency Ms + * @description Server-side time for token quoting, credit reservation and inference + */ + latency_ms: number; + /** + * Mode + * @constant + */ + mode: "live"; + /** Request Id */ + request_id: string; + }; + /** EuJevUsage */ + EuJevUsage: { + /** Input Tokens */ + input_tokens: number; + /** Output Tokens */ + output_tokens: number; + }; /** * Eval * @description Represents an evaluation from the OpenAI Evals API @@ -59476,6 +59546,144 @@ export interface operations { }; }; }; + eu_jev_proxy_route_eu_jev_v1_systemone_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EuJevDecisionResponse"]; + }; + }; + }; + }; extraction_extraction_post: { parameters: { query?: never; diff --git a/ui/litellm-dashboard/tests/setupTests.ts b/ui/litellm-dashboard/tests/setupTests.ts index fe637e0473a..2b6b4c7484c 100644 --- a/ui/litellm-dashboard/tests/setupTests.ts +++ b/ui/litellm-dashboard/tests/setupTests.ts @@ -2,6 +2,8 @@ import "@testing-library/jest-dom"; import { cleanup } from "@testing-library/react"; import { afterEach, vi } from "vitest"; +import "@/i18n/i18n"; + const ensureTestLocalStorage = () => { if (typeof window === "undefined" || typeof window.Storage === "undefined") { return; diff --git a/uv.lock b/uv.lock index 5be97af9ed2..b3e8ddc492c 100644 --- a/uv.lock +++ b/uv.lock @@ -5478,12 +5478,12 @@ provides-extras = ["test"] [[package]] name = "litellm-enterprise" -version = "0.1.80" +version = "0.1.81" source = { editable = "enterprise" } [[package]] name = "litellm-proxy-extras" -version = "0.4.111" +version = "0.4.112" source = { editable = "litellm-proxy-extras" } dependencies = [ { name = "psycopg" },