diff --git a/litellm/images/main.py b/litellm/images/main.py index 6f1adaccab1..da160639721 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -56,6 +56,7 @@ from litellm.types.llms.openai import ImageGenerationRequestQuality from litellm.types.router import CustomPricingLiteLLMParams, GenericLiteLLMParams from litellm.types.utils import ( LITELLM_IMAGE_VARIATION_PROVIDERS, + DynamicPromptManagementParamLiteral, LlmProviders, ProviderRoutingHints, ) @@ -336,7 +337,11 @@ def image_generation( ] non_default_params: Final = filter_out_litellm_params( kwargs, - excluding=[*openai_params, "extra_headers", "headers"] if chatgpt_params is not None else openai_params, + excluding=( + *openai_params, + *(("extra_headers", "headers") if chatgpt_params is not None else ()), + *DynamicPromptManagementParamLiteral.list_all_params(), + ), ) image_generation_config: BaseImageGenerationConfig | None = None @@ -905,7 +910,9 @@ def image_edit( "style", "async_call", ] - non_default_params: Final = filter_out_litellm_params(kwargs, excluding=openai_params) + non_default_params: Final = filter_out_litellm_params( + kwargs, excluding=(*openai_params, *DynamicPromptManagementParamLiteral.list_all_params()) + ) litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None) model_info: Final = kwargs.get("model_info", None) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index ec0e92afa0e..99ae21e929c 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -19,6 +19,7 @@ from litellm._logging import verbose_logger from litellm.constants import KUBERNETES_POD_ROUTING_KEY from litellm.litellm_core_utils.hidden_params import set_extra_attribute from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id +from litellm.types.litellm_params import MCP_HANDLER_KWARG_NAMES from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason if TYPE_CHECKING: @@ -1023,36 +1024,8 @@ def _filter_exceptions_from_params(data: object, max_depth: int) -> object: return data -def filter_internal_params(data: dict, additional_internal_params: set | None = None) -> dict: - """ - Filter out LiteLLM internal parameters that shouldn't be sent to provider APIs. - - This removes internal/MCP-related parameters that are used by LiteLLM internally - but should not be included in API requests to providers. - - Args: - data: Dictionary of parameters to filter - additional_internal_params: Optional set of additional internal parameter names to filter - - Returns: - Filtered dictionary with internal parameters removed - """ - if not isinstance(data, dict): - return data - - # Known internal parameters that should never be sent to provider APIs - internal_params: Final = { - "skip_mcp_handler", - "mcp_handler_context", - "_skip_mcp_handler", - } - - # Add any additional internal params if provided - if additional_internal_params: - internal_params.update(additional_internal_params) - - # Filter out internal parameters - return {k: v for k, v in data.items() if k not in internal_params} +def filter_internal_params(data: Mapping[str, _Value]) -> dict[str, _Value]: + return {k: v for k, v in data.items() if k not in MCP_HANDLER_KWARG_NAMES} def redact_nested_match_and_regex_keys( diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 107db751036..fb12760eeed 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -1,6 +1,7 @@ import reprlib from collections.abc import Mapping from dataclasses import dataclass, fields +from types import MappingProxyType from typing import Final, cast from pydantic import TypeAdapter, ValidationError @@ -116,7 +117,12 @@ SUPPLEMENTAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS | {"provider_specific_hea _CONTROL_OPTIONS: Final = TypeAdapter(ControlOptions) _CONTROL_OPTION_NAMES: Final = tuple(field.name for field in fields(ControlOptions)) _MAX_SHOWN_INT_BITS: Final = 64 -_EXPECTED: Final = f"expected a positive integer of at most {MAX_CONTROL_INT_DIGITS} digits" +_EXPECTED: Final = MappingProxyType( + { + "stream_chunk_size": f"expected a positive integer of at most {MAX_CONTROL_INT_DIGITS} digits", + "fake_stream": "expected true or false", + } +) class _BoundedRepr(reprlib.Repr): @@ -142,10 +148,19 @@ def parse_control_options(kwargs: Mapping[str, object]) -> ControlOptions | Inva except ValidationError as e: param: Final = str(e.errors(include_url=False)[0]["loc"][0]) return InvalidControlOption( - param=param, message=f"Invalid {param}={_BOUNDED_REPR.repr(given[param])}: {_EXPECTED}" + param=param, message=f"Invalid {param}={_BOUNDED_REPR.repr(given[param])}: {_EXPECTED[param]}" ) +def valid_control_options(kwargs: Mapping[str, object]) -> ControlOptions: + valid: Final = { + name: kwargs[name] + for name in _CONTROL_OPTION_NAMES + if name in kwargs and isinstance(parse_control_options({name: kwargs[name]}), ControlOptions) + } + return _CONTROL_OPTIONS.validate_python(valid) + + def stored_control_options(litellm_params: Mapping[str, object]) -> ControlOptions: control: Final = litellm_params.get(CONTROL_OPTIONS_KEY) return control if isinstance(control, ControlOptions) else ControlOptions() diff --git a/litellm/llms/base_llm/base_model_iterator.py b/litellm/llms/base_llm/base_model_iterator.py index a705cfe20ad..d5161cba699 100644 --- a/litellm/llms/base_llm/base_model_iterator.py +++ b/litellm/llms/base_llm/base_model_iterator.py @@ -7,6 +7,7 @@ from typing_extensions import TypedDict import litellm from litellm.constants import STREAM_SSE_DONE_STRING +from litellm.litellm_core_utils.hidden_params import set_extra_attribute, stored_attribute if TYPE_CHECKING: import httpx @@ -76,6 +77,7 @@ def convert_model_response_to_streaming( index=choice.index, delta=_streaming_delta(cast(Choices, choice).message), finish_reason=choice.finish_reason, + logprobs=stored_attribute(choice, "logprobs"), ) ) processed_chunk: Final = ModelResponseStream( @@ -83,8 +85,12 @@ def convert_model_response_to_streaming( object="chat.completion.chunk", created=model_response.created, model=model_response.model, + system_fingerprint=model_response.system_fingerprint, choices=streaming_choices, ) + service_tier: Final = stored_attribute(model_response, "service_tier") + if service_tier is not None: + set_extra_attribute(processed_chunk, "service_tier", service_tier) # Carry usage onto the streaming chunk so fake-streamed responses # (e.g. Vertex AI Gemma :predict) still report token counts. usage: Final = getattr(model_response, "usage", None) diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index a6f9192339f..becfd605644 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -299,10 +299,10 @@ class BedrockConverseLLM(BaseAWSLLM): api_key: str | None = None, ): ## SETUP ## - stream: Final = optional_params.pop("stream", None) - stream_chunk_size: Final = stored_control_options(litellm_params).stream_chunk_size if stream is True else None + stream: Final = optional_params.pop("stream", None) is True + stream_chunk_size: Final = stored_control_options(litellm_params).stream_chunk_size if stream else None unencoded_model_id: Final = optional_params.pop("model_id", None) - fake_stream = optional_params.pop("fake_stream", False) + requested_fake_stream: Final = optional_params.pop("fake_stream", False) is True json_mode: Final = optional_params.get("json_mode") is True if unencoded_model_id is not None: modelId = self.encode_model_id(model_id=unencoded_model_id) @@ -331,8 +331,8 @@ class BedrockConverseLLM(BaseAWSLLM): if _region_from_model is not None and "aws_region_name" not in optional_params: optional_params["aws_region_name"] = _region_from_model - fake_stream = litellm.AmazonConverseConfig().should_fake_stream( - fake_stream=fake_stream, + fake_stream: Final = litellm.AmazonConverseConfig().should_fake_stream( + fake_stream=requested_fake_stream, model=model, stream=stream, custom_llm_provider="bedrock", @@ -363,7 +363,7 @@ class BedrockConverseLLM(BaseAWSLLM): aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, aws_region_name=aws_region_name, ) - if (stream is not None and stream is True) and not fake_stream: + if stream and not fake_stream: endpoint_url = join_api_base(endpoint_url, f"model/{modelId}/converse-stream") proxy_endpoint_url = join_api_base(proxy_endpoint_url, f"model/{modelId}/converse-stream") else: @@ -463,7 +463,7 @@ class BedrockConverseLLM(BaseAWSLLM): else: client = client - if stream is not None and stream is True: + if stream: completion_stream, response_headers = make_sync_call( client=(client if client is not None and isinstance(client, HTTPHandler) else None), api_base=proxy_endpoint_url, @@ -513,7 +513,7 @@ class BedrockConverseLLM(BaseAWSLLM): model=model, response=response, model_response=model_response, - stream=stream if isinstance(stream, bool) else False, + stream=stream, logging_obj=logging_obj, api_key="", data=data, diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index f5e55eea141..454a1fafbf5 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -32,6 +32,7 @@ from litellm.llms.custom_httpx.http_handler import ( HTTPHandler, get_httpx_client, ) +from litellm.types.litellm_params import MCP_HANDLER_KWARG_NAMES from litellm.types.llms.bedrock import GuardrailConfigBlock from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage @@ -226,8 +227,11 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): provider=provider, custom_prompt_dict=custom_prompt_dict, ) - inference_params = copy.deepcopy(optional_params) - inference_params = {k: v for k, v in inference_params.items() if k not in self.aws_authentication_params} + inference_params: Final = { + k: v + for k, v in copy.deepcopy(optional_params).items() + if k not in self.aws_authentication_params and k not in MCP_HANDLER_KWARG_NAMES + } request_data: dict = {} if provider == "cohere": if model.startswith("cohere.command-r"): diff --git a/litellm/main.py b/litellm/main.py index d87b5a5e900..f093a93d205 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -19,14 +19,14 @@ import random import sys import time import traceback -from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Mapping, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterable, Mapping, Sequence from concurrent import futures from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from copy import deepcopy from dataclasses import dataclass from functools import partial from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args +from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, TypeAlias, Union, cast, get_args from litellm._internal_context import is_resent_failure from litellm._logging import redact_string @@ -100,6 +100,7 @@ from litellm.litellm_core_utils.get_litellm_params import ( SUPPLEMENTAL_KWARGS_KEYS, InvalidControlOption, parse_control_options, + valid_control_options, with_control_options, ) from litellm.litellm_core_utils.get_llm_provider_logic import ( @@ -141,6 +142,7 @@ from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig from litellm.llms.base_llm.base_model_iterator import ( + MockResponseIterator, convert_model_response_to_streaming, ) from litellm.llms.base_llm.chat.transformation import with_attribution_headers @@ -180,6 +182,7 @@ from litellm.types.router import GenericLiteLLMParams, validate_litellm_params_s from litellm.types.utils import ( AgenticLoopParams, BaseUrlRoutingHint, + DynamicPromptManagementParamLiteral, ModelResponseStream, ProviderRoutingHints, RawRequestTypedDict, @@ -415,7 +418,7 @@ def _resolve_control_options(kwargs: Mapping[str, object], model: str) -> Contro return control case InvalidControlOption(param=param, message=message): if litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True: - return ControlOptions() + return valid_control_options(kwargs) raise litellm.BadRequestError(message=message, model=model, llm_provider=None, body={"param": param}) case _: return assert_never(control) @@ -1418,10 +1421,11 @@ def _completion_fakes_stream( provider_config: BaseConfig | None, optional_params: Mapping[str, object], *, + fake_stream_requested: bool, base_model: str | None, bridges_to_responses_api: bool, ) -> bool: - if optional_params.get("fake_stream") is True: + if fake_stream_requested or optional_params.get("fake_stream") is True: return True if custom_llm_provider is None: return False @@ -1447,6 +1451,60 @@ def _completion_fakes_stream( ) +def _replayed_stream( + response: ModelResponse, logging_obj: LiteLLMLoggingObj, stream_options: Mapping[str, object] | None +) -> CustomStreamWrapper: + hidden_params: Final = get_hidden_params(response) + provider_headers: Final = hidden_params.get("additional_headers") if hidden_params is not None else None + return CustomStreamWrapper( + completion_stream=MockResponseIterator(model_response=response), + model=response.model, + logging_obj=logging_obj, + stream_options=stream_options, + _response_headers=( + _RESPONSE_HEADERS_ADAPTER.validate_python(provider_headers) + if isinstance(provider_headers, Mapping) + else None + ), + ) + + +_DispatchedAnswer: TypeAlias = ModelResponse | TextCompletionResponse | CustomStreamWrapper + + +def _replayed_answer( + done: _DispatchedAnswer, replay: Callable[[ModelResponse], CustomStreamWrapper] +) -> ModelResponse | CustomStreamWrapper: + match done: + case ModelResponse(): + return replay(done) + case TextCompletionResponse(): + return replay( + litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( + response_object=done, model_response_object=ModelResponse() + ) + ) + case CustomStreamWrapper(): + return done + + +async def _replayed_when_done( + pending: Awaitable[_DispatchedAnswer], replay: Callable[[ModelResponse], CustomStreamWrapper] +) -> ModelResponse | CustomStreamWrapper: + return _replayed_answer(await pending, replay) + + +def _replayed_as_stream( + dispatched: Awaitable[_DispatchedAnswer] | _DispatchedAnswer, + replay: Callable[[ModelResponse], CustomStreamWrapper] | None, +) -> Awaitable[_DispatchedAnswer] | _DispatchedAnswer: + if replay is None: + return dispatched + if isinstance(dispatched, (ModelResponse, TextCompletionResponse, CustomStreamWrapper)): + return _replayed_answer(dispatched, replay) + return _replayed_when_done(dispatched, replay) + + class _ProxyAuthHeadersProvider(Protocol): def get_auth_headers(self) -> Mapping[str, str]: ... @@ -4391,6 +4449,7 @@ _ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str]) _OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object]) _OLLAMA_EMBEDDING_HEADERS_ADAPTER: Final = TypeAdapter(dict[str, str]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_RESPONSE_HEADERS_ADAPTER: Final = TypeAdapter(dict[str, object]) _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints) _BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint) @@ -5722,6 +5781,8 @@ def completion( skip_responses_api_bridge: Final = kwargs.pop("_skip_responses_api_bridge", False) control_options: Final = _resolve_control_options(kwargs, model) + replays_stream: Final = stream is True and control_options.fake_stream is True + dispatch_stream: Final = False if replays_stream else stream skip_mcp_handler: Final = kwargs.pop("_skip_mcp_handler", False) if not skip_mcp_handler and tools and may_name_litellm_mcp_gateway(tools): # pyright: ignore[reportUnknownArgumentType] # untyped tools list from litellm.responses.mcp.chat_completions_handler import acompletion_with_mcp @@ -6115,8 +6176,8 @@ def completion( "temperature": temperature, "top_p": top_p, "n": n, - "stream": stream, - "stream_options": stream_options, + "stream": dispatch_stream, + "stream_options": None if replays_stream else stream_options, "stop": stop, "max_tokens": max_tokens, "max_completion_tokens": max_completion_tokens, @@ -6188,6 +6249,7 @@ def completion( model, custom_llm_provider, provider_config, + fake_stream_requested=control_options.fake_stream is True, base_model=deployment_base_model, bridges_to_responses_api=bridges_to_responses_api is True, ) @@ -6427,6 +6489,9 @@ def completion( # detection when the deployment name differs from the model name. _azure_detection_model: Final = base_model or model + replay_as_stream: Final = ( + partial(_replayed_stream, logging_obj=logging, stream_options=stream_options) if replays_stream else None + ) if responses_api_model_info.get("mode") == "responses" and not skip_responses_api_bridge: from litellm.completion_extras import responses_api_bridge @@ -6444,7 +6509,7 @@ def completion( additional_drop_params=routing_kwargs.get("additional_drop_params"), ) reasoning_drop_warnings.log_unless_dropped_effort_sent(bridge_optional_params) - return responses_api_bridge.completion( # pyright: ignore[reportReturnType] # bridge returns a coroutine on the acompletion path; awaited by the async caller + bridged: Final = responses_api_bridge.completion( model=model, messages=messages, headers=headers, @@ -6459,9 +6524,12 @@ def completion( client=client, # pass AsyncOpenAI, OpenAI client custom_llm_provider=custom_llm_provider, encoding=_get_encoding(), - stream=stream, + stream=dispatch_stream, cache=kwargs.get("cache"), ) + return _replayed_as_stream( # pyright: ignore[reportReturnType] # bridge returns a coroutine on the acompletion path; awaited by the async caller + bridged, replay_as_stream + ) reasoning_drop_warnings.log_unless_dropped_effort_sent(optional_params) if (custom_llm_provider == "openai" and OpenAIGPT5Config.is_model_gpt_5_model(model)) or ( @@ -6500,7 +6568,7 @@ def completion( _OPTIONAL_PARAMS_ADAPTER.validate_python({**optional_param_args, **provider_params}) ), shared_session=shared_session, - stream=stream, + stream=dispatch_stream, temperature=temperature, text_completion=text_completion, timeout=timeout, @@ -6728,7 +6796,9 @@ def completion( else: raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) - return response + return _replayed_as_stream( # pyright: ignore[reportReturnType] # a dispatch returns a coroutine on the acompletion path; awaited by the async caller + response, replay_as_stream + ) except Exception as e: reasoning_drop_warnings.log_unless_dropped_effort_sent(MappingProxyType({})) ## Map to OpenAI Exception @@ -7110,7 +7180,9 @@ def embedding( "encoding_format", ] default_params: Final = [*openai_params, "aembedding", "extra_headers"] - non_default_params: Final = filter_out_litellm_params(kwargs, excluding=default_params) + non_default_params: Final = filter_out_litellm_params( + kwargs, excluding=(*default_params, *DynamicPromptManagementParamLiteral.list_all_params()) + ) litellm_params_dict: Final = get_litellm_params(**kwargs) diff --git a/litellm/text_completions/streaming_iterator.py b/litellm/text_completions/streaming_iterator.py index 517a499ec8c..7158e418542 100644 --- a/litellm/text_completions/streaming_iterator.py +++ b/litellm/text_completions/streaming_iterator.py @@ -10,6 +10,19 @@ class _SupportsGet(Protocol): def get(self, key: str, /) -> object: ... +def _text_choice(chat_choice: object) -> TextChoices: + choice_fields: Final = decoded_json_object( + {field: subscript(chat_choice, field) for field in VLLM_CHOICE_FIELDS if json_contains(chat_choice, field)} + ) + text_choices: Final = TextChoices(index=0, **choice_fields) + delta: Final = subscript(chat_choice, "delta") + text_choices["text"] = subscript(delta, "content") + text_choices["reasoning_content"] = require_attributes(delta, _SupportsGet, "get").get("reasoning_content") + text_choices["index"] = subscript(chat_choice, "index") + text_choices["finish_reason"] = subscript(chat_choice, "finish_reason") + return text_choices + + class TextCompletionStreamWrapper: def __init__( self, @@ -46,26 +59,7 @@ class TextCompletionStreamWrapper: if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices raise Exception - if chunk["choices"]: - chat_choice: Final = subscript(subscript(chunk, "choices"), 0) - choice_fields: Final = decoded_json_object( - { - field: subscript(chat_choice, field) - for field in VLLM_CHOICE_FIELDS - if json_contains(chat_choice, field) - } - ) - text_choices: Final = TextChoices(index=0, **choice_fields) - delta: Final = subscript(chat_choice, "delta") - text_choices["text"] = subscript(delta, "content") - text_choices["reasoning_content"] = require_attributes(delta, _SupportsGet, "get").get( - "reasoning_content" - ) - text_choices["index"] = subscript(chat_choice, "index") - text_choices["finish_reason"] = subscript(chat_choice, "finish_reason") - response["choices"] = [text_choices] - else: - response["choices"] = [] + response["choices"] = [_text_choice(chat_choice) for chat_choice in chunk.choices] if self.include_usage: response.usage = chunk.get("usage", None) diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 26c5bd4237c..183b6871258 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -310,6 +310,10 @@ def _int_from_decimal_string(value: object) -> object: return value +def _bool_from_true_false_string(value: object) -> object: + return value == "true" if value in ("true", "false") else value + + @pydantic_dataclass(frozen=True, slots=True, kw_only=True) class ControlOptions: stream_chunk_size: ( @@ -320,6 +324,7 @@ class ControlOptions: ] | None ) = None + fake_stream: Annotated[bool, BeforeValidator(_bool_from_true_false_string), Field(strict=True)] | None = None @dataclass(frozen=True, slots=True, kw_only=True) @@ -358,6 +363,13 @@ class CallState: context_compaction_state: "CompactionState | None" = field(default=None, metadata=wire("_context_compaction_state")) +@dataclass(frozen=True, slots=True, kw_only=True) +class McpHandlerState: + recursion_guard: bool | None = field(default=None, metadata=wire("_skip_mcp_handler")) + skip_mcp_handler: bool | None = None + mcp_handler_context: Mapping[str, object] | None = None + + @dataclass(frozen=True, slots=True, kw_only=True) class AgenticLoopState: headroom_context: "HeadroomRequestContext | None" = field(default=None, metadata=wire("_headroom_request_context")) @@ -438,6 +450,7 @@ class EntrypointState: class InternalState: call: CallState agentic_loop: AgenticLoopState + mcp_handler: McpHandlerState router: RouterState proxy: ProxyRequestState entrypoint: EntrypointState @@ -465,3 +478,4 @@ def owned_wire_names(root: type) -> tuple[str, ...]: OWNED_KWARG_NAMES: Final = tuple(chain.from_iterable(owned_wire_names(root) for root in LITELLM_OWNED_ROOTS)) AGENTIC_LOOP_KWARG_NAMES: Final = (*wire_names(AgenticLoopState), *wire_names(AgenticLoopOptions)) BEDROCK_BATCH_KWARG_NAMES: Final = wire_names(BedrockBatchConnection) +MCP_HANDLER_KWARG_NAMES: Final = frozenset(wire_names(McpHandlerState)) diff --git a/tests/integration/providers/test_internal_params_wire.py b/tests/integration/providers/test_internal_params_wire.py index f9f5d1e5478..789f8442f09 100644 --- a/tests/integration/providers/test_internal_params_wire.py +++ b/tests/integration/providers/test_internal_params_wire.py @@ -4,15 +4,29 @@ import json import os import struct import zlib -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterator, Mapping +from contextlib import suppress +from dataclasses import dataclass from pathlib import Path -from typing import Final +from types import MappingProxyType +from typing import Final, get_args +from urllib.parse import parse_qs +import openai import pytest from integration._support.upstream import INTERNAL_FIELDS from integration._support.wire import Reply, Request, wire_server +from pydantic import TypeAdapter import litellm +from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL, BEDROCK_INVOKE_PROVIDERS_LITERAL +from litellm.exceptions import LiteLLMUnknownProvider +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.common_utils import BedrockModelInfo, BedrockRoute +from litellm.types import litellm_params +from litellm.types.litellm_params import OWNED_KWARG_NAMES, wire_names +from litellm.types.utils import LlmProviders from tests._support.stream_chunk_size import keys_at_every_depth TEXT: Final = "wire control" @@ -313,3 +327,480 @@ async def test_internal_params_never_reach_provider_body( assert not INTERNAL_FIELDS.intersection(keys) assert not frozenset(key for key in keys if key.startswith("_litellm_")), keys assert _custom_key(body, provider) == 1 + + +WALK_TEXT: Final = "walk every provider" +UNDECLARED_SENTINEL: Final = "_litellm_undeclared_sentinel" + + +def _ignore_log_details(_details: Mapping[str, object]) -> None: + return None + + +WALK_SEEDS: Final[Mapping[str, object]] = MappingProxyType( + { + "aws_batch_role_arn": "arn:aws:iam::000000000000:role/wire", + "aws_bedrock_control_plane_endpoint": "https://bedrock.wire.invalid", + "s3_bucket_name": "wire-bucket", + "s3_region_name": "us-east-1", + "s3_endpoint_url": "https://s3.wire.invalid", + "s3_output_bucket_name": "wire-output", + "s3_bucket_owner": "000000000000", + "s3_access_key_id": "wire-access-key", + "s3_secret_access_key": "wire-secret-key", + "s3_encryption_key_id": "wire-kms-key", + "bedrock_tags": [{"key": "wire", "value": "walk"}], + "model_info": {"id": "wire-deployment"}, + "rpm": 7, + "tpm": 7, + "itpm": 7, + "otpm": 7, + "rpd": 7, + "tpd": 7, + "daily_reset_time": "00:00", + "kubernetes_pod_discovery": True, + "default_api_key_rpm_limit": 7, + "default_api_key_tpm_limit": 7, + "max_parallel_requests": 1, + "weight": 1, + "order": 1, + "tag_regex": ["^wire$"], + "max_file_size_mb": 1.0, + "auto_router_config_path": "wire-router.json", + "auto_router_config": "wire-router", + "auto_router_default_model": "wire-default", + "auto_router_embedding_model": "wire-embedding", + "auto_router_max_input_chars": 64, + "auto_router_routing_compression": "wire", + "auto_router_model_compression": "wire", + "complexity_router_config": {"wire": 1}, + "complexity_router_default_model": "wire-default", + "adaptive_router_config": {"wire": 1}, + "adaptive_router_default_model": "wire-default", + "quality_router_config": {"wire": 1}, + "quality_router_default_model": "wire-default", + "switchyard_router_config": {"wire": 1}, + "cost_per_query": 0.5, + "max_budget": 100.0, + "budget_duration": "30d", + "id": "wire-id", + "metadata": {"wire": 1}, + "litellm_metadata": {"wire": 1}, + "tags": ["wire"], + "litellm_trace_id": "wire-trace", + "litellm_session_id": "wire-session", + "litellm_request_debug": False, + "logger_fn": _ignore_log_details, + "verbose": False, + "no-log": False, + "log_client_error_tracebacks": False, + "max_agentic_loops": 1, + "merge_reasoning_content_in_choices": False, + "thought_signature_in_tool_call_id": False, + "enable_json_schema_validation": False, + "complete_response": False, + "keepalive_seconds": 30.0, + "allow_client_keepalive_override": False, + "stream_chunk_size": 64, + "litellm_call_id": "wire-call", + "completion_call_id": "wire-completion", + "preset_cache_key": "wire-cache-key", + "cache_key": "wire-cache-key", + "fake_stream": False, + "_skip_mcp_handler": True, + "skip_mcp_handler": True, + "mcp_handler_context": {"server": "wire"}, + "_encrypted_content_affinity_pinned": True, + "fallback_depth": 1, + "max_fallbacks": 1, + UNDECLARED_SENTINEL: "internal", + } +) +OWNED_LEAVES_NOT_SEEDED: Final = MappingProxyType( + { + litellm_params.ProviderConnection: "the route needs its own connection settings to send anything", + litellm_params.AnthropicFederationConnection: "switches the route to workload identity federation", + litellm_params.OpenAIFederationConnection: "switches the route to workload identity federation", + litellm_params.DispatchOptions: "chooses which provider route handles the call", + litellm_params.RoutingOptions: "retries and fallbacks change which requests are sent", + litellm_params.CachingOptions: "turns response caching on or off", + litellm_params.GuardrailOptions: "runs the named guardrails before the call", + litellm_params.PromptOptions: "changes how the prompt is built or loads a managed prompt", + litellm_params.MockOptions: "answers without sending a request", + litellm_params.AgenticLoopState: "drives the agentic loop between follow-up calls", + litellm_params.ProxyRequestState: "set by the proxy for its own request handling", + litellm_params.EntrypointState: "picks the sync or async entry point", + } +) +OWNED_NAMES_NOT_SEEDED: Final = MappingProxyType( + { + "silent_model": "sends a second request to another model", + "base_model": "stands in for the model name when a provider config resolves the model", + "model_alias_map": "renames the model before the route is chosen", + "data_residency": "picks a regional endpoint", + "litellm_logging_obj": "must be the call's own logging object", + "stream_response": "holds a response object, not request input", + "_context_compaction_state": "holds compaction state, not request input", + "_router_weights": "holds router weight state, not request input", + "attempted_targets": "holds the router's fallback bookkeeping, not request input", + } +) +EMBEDDING_WALK_SEEDS: Final[Mapping[str, object]] = MappingProxyType( + { + **WALK_SEEDS, + "cache_control_injection_points": [{"location": "message", "role": "user"}], + "knowledge_bases": ["kb_wire"], + "vector_store_ids": ["vs_wire"], + } +) +WALK_REJECTION: Final = Reply( + status=400, + body=json.dumps({"error": {"message": "rejected by the wire", "type": "invalid_request_error"}}).encode(), +) +SYNTHETIC_KEY: Final = MappingProxyType({"api_key": "synthetic-key"}) +AZURE_CONNECTION: Final = MappingProxyType({**SYNTHETIC_KEY, "api_version": "2025-01-01-preview"}) +AWS_CONNECTION: Final = MappingProxyType( + {"aws_access_key_id": "fake", "aws_secret_access_key": "fake", "aws_region_name": "us-east-1"} +) +AWS_BASE_PARAMS: Final = ("api_base", "aws_bedrock_runtime_endpoint") +ONE_PIXEL_GIF: Final = "data:image/gif;base64,R0lGODlhAQABAAAAACw=" + + +@dataclass(frozen=True, slots=True) +class Route: + model: str + connection: Mapping[str, object] = SYNTHETIC_KEY + base_params: tuple[str, ...] = ("api_base",) + base_suffix: str = "" + request: Mapping[str, object] = MappingProxyType({}) + protocol_fields: frozenset[str] = frozenset() + streams: bool = True + + def parameters(self, wire_url: str) -> dict[str, object]: + return { + "model": self.model, + **self.connection, + **dict.fromkeys(self.base_params, wire_url + self.base_suffix), + **self.request, + } + + +def _bedrock(model: str, **connection: object) -> Route: + return Route(f"bedrock/{model}", MappingProxyType({**AWS_CONNECTION, **connection}), AWS_BASE_PARAMS) + + +IMPORTED_MODEL: Final = "arn:aws:bedrock:us-east-1:000000000000:imported-model/wire" +BEDROCK_INVOKE_MODELS: Final = MappingProxyType( + { + "anthropic": "anthropic.claude-haiku-4-5-20251001-v1:0", + "cohere": "cohere.command-r-plus-v1:0", + "mistral": "mistral.mistral-large-2402-v1:0", + "amazon": "amazon.titan-text-express-v1", + "meta": "meta.llama3-1-70b-instruct-v1:0", + "llama": f"llama/{IMPORTED_MODEL}", + "ai21": "ai21.jamba-1-5-large-v1:0", + "nova": "amazon.nova-pro-v1:0", + "deepseek_r1": f"deepseek_r1/{IMPORTED_MODEL}", + "qwen3": f"qwen3/{IMPORTED_MODEL}", + "qwen2": f"qwen2/{IMPORTED_MODEL}", + "twelvelabs": "twelvelabs.pegasus-1-2-v1:0", + "openai": "openai.gpt-oss-120b-1:0", + "moonshot": "moonshot.kimi-k2-thinking", + } +) +BEDROCK_INVOKE_FAMILIES_WITHOUT_CHAT: Final = MappingProxyType({"stability": "generates images only"}) +BEDROCK_ROUTES: Final = MappingProxyType( + { + "converse": _bedrock("converse/anthropic.claude-haiku-4-5-20251001-v1:0"), + "converse_like": _bedrock("converse_like/anthropic.claude-haiku-4-5-20251001-v1:0"), + "openai": _bedrock("openai/wire-model"), + "chat_completions": _bedrock("chat_completions/openai.gpt-oss-120b-1:0"), + "mantle": _bedrock("mantle/openai.gpt-oss-120b"), + "claude_platform": _bedrock("claude_platform/claude-haiku-4-5", workspace_id="wrk_wire"), + "agent": _bedrock("agent/AGENTID/ALIASID"), + "agentcore": _bedrock("agentcore/arn:aws:bedrock-agentcore:us-east-1:000000000000:runtime/wire"), + **{f"invoke/{family}": _bedrock(f"invoke/{model}") for family, model in BEDROCK_INVOKE_MODELS.items()}, + } +) +BEDROCK_ROUTES_WITHOUT_CHAT: Final = MappingProxyType({"async_invoke": "starts an async job instead of a chat call"}) + +API_BASE_CHAT_PROVIDERS: Final = ( + "abliteration", "ai21", "ai21_chat", "aiand", "aiml", "aiohttp_openai", "amazon_nova", "anthropic", + "anthropic_text", "apertis", "assemblyai", "azure_ai", "baseten", "bigmodel", "cerebras", "charity_engine", + "chatgpt", "chutes", "clf_ai_gateway", "cloudflare", "codestral", "cognition", "cohere", "cohere_chat", + "cometapi", "compactifai", "coralbricks", "cortecs", "custom", "custom_openai", "darkbloom", "dashscope", + "databricks", "databricks_agent", "datarobot", "deepinfra", "deepseek", "docker_model_runner", "edenai", + "empiriolabs", "empower", "featherless_ai", "fireworks_ai", "friendliai", "galadriel", "gemini", "github", + "groq", "helicone", "heroku", "hosted_vllm", "hostinger_router", "huggingface", "hyperbolic", "inception", + "inferenceapis", "kyma", "lambda_ai", "langflow", "langgraph", "lemonade", "libertai", "litellm_proxy", + "llamafile", "lm_studio", "maritalk", "meta", "meta_llama", "minimax", "mistral", "modelscope", "moonshot", + "morph", "nadir", "nano-gpt", "nebius", "neosantara", "novita", "nscale", "nvidia_nim", "ollama", "ollama_chat", + "oobabooga", "openai", "opencode_go", "openinfer", "openrouter", "ovhcloud", "parasail", "perplexity", "petals", + "pinstripes", "poe", "prism", "publicai", "qwen_ai_platform", "qwencloud", "refuseless", "reka", "sail", + "sambanova", "scaleway", "scx-ai", "snowflake", "streamlake", "synthetic", "tencent", "tensormesh", + "text-completion-codestral", "text-completion-inception", "text-completion-openai", "together_ai", "v0", + "vercel_ai_gateway", "volcengine", "wally", "wandb", "xai", "xiaomi_mimo", "xinference", "zai", "zerogpu", +) # fmt: skip +CHAT_ROUTES: Final = MappingProxyType( + { + **{provider: Route(f"{provider}/wire-model") for provider in API_BASE_CHAT_PROVIDERS}, + "a2a": Route("a2a/wire-model", protocol_fields=frozenset({"id"})), + "azure": Route("azure/wire-model", AZURE_CONNECTION), + "azure_text": Route("azure_text/wire-model", AZURE_CONNECTION), + "bedrock_mantle": Route("bedrock_mantle/wire-model", AWS_CONNECTION, AWS_BASE_PARAMS), + "clarifai": Route("clarifai/user.app.wire-model"), + "fal_ai": Route( + "fal_ai/wire-model", + request=MappingProxyType( + { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": WALK_TEXT}, + {"type": "image_url", "image_url": {"url": ONE_PIXEL_GIF}}, + ], + } + ] + } + ), + streams=False, + ), + "gradient_ai": Route("gradient_ai/wire-model"), + "nlp_cloud": Route("nlp_cloud/chatdolphin", base_suffix="/"), + "predibase": Route("predibase/wire-model", MappingProxyType({**SYNTHETIC_KEY, "predibase_tenant_id": "wire"})), + "ragflow": Route("ragflow/chat/wire-chat/wire-model"), + "scaledown": Route("scaledown/summarize"), + "triton": Route("triton/wire-model", base_suffix="/generate"), + **{f"bedrock/{route}": target for route, target in BEDROCK_ROUTES.items()}, + } +) + +VENDOR_HOST: Final = "calls the vendor host even when api_base is set" +VENDOR_AUTH: Final = "needs vendor credentials or a token exchange the wire cannot fake" +DEVICE_LOGIN: Final = "needs an interactive device-code login" +CHAT_PROVIDERS_OFF_THE_WIRE: Final = MappingProxyType( + { + **dict.fromkeys(("bytez", "replicate", "sagemaker", "sagemaker_chat", "sagemaker_nova"), VENDOR_HOST), + **dict.fromkeys( + ("gdc", "gigachat", "microsoft_365_copilot", "oci", "sap", "vertex_ai", "vertex_ai_beta", "watsonx", + "watsonx_text"), + VENDOR_AUTH, + ), + "github_copilot": DEVICE_LOGIN, + "vllm": "runs the local vllm package instead of calling a server", + } +) # fmt: skip + +EMBEDDING_ROUTES: Final = MappingProxyType( + { + **{ + provider: Route(f"{provider}/wire-model") + for provider in ( + "azure_ai", "cohere", "cohere_chat", "cometapi", "dashscope", "databricks", "edenai", "empiriolabs", + "fireworks_ai", "gemini", "hosted_vllm", "hostinger_router", "infinity", "jina_ai", "litellm_proxy", + "llamafile", "lm_studio", "mistral", "nebius", "nvidia_nim", "ollama", "oobabooga", "openai", + "openai_like", "openrouter", "ovhcloud", "perplexity", "qwen_ai_platform", "qwencloud", "sambanova", + "snowflake", "together_ai", "triton", "vercel_ai_gateway", "volcengine", "voyage", "wandb", + "xinference", + ) + }, + "azure": Route("azure/wire-model", AZURE_CONNECTION), + "huggingface": Route("huggingface/wire-model", request=MappingProxyType({"input_type": "feature-extraction"})), + "bedrock/amazon": _bedrock("amazon.titan-embed-image-v1"), + "bedrock/cohere": _bedrock("cohere.embed-english-v3"), + "bedrock/nova": _bedrock("amazon.nova-2-multimodal-embeddings-v1:0"), + "bedrock/twelvelabs": _bedrock("twelvelabs.marengo-embed-2-7-v1:0"), + } +) # fmt: skip +EMBEDDING_PROVIDERS_OFF_THE_WIRE: Final = MappingProxyType( + { + "sagemaker": VENDOR_HOST, + **dict.fromkeys(("gigachat", "oci", "sap", "vertex_ai", "watsonx"), VENDOR_AUTH), + **dict.fromkeys(("chatgpt", "github_copilot"), DEVICE_LOGIN), + } +) +REFUSALS: Final = (openai.OpenAIError, BaseLLMException, ValueError) + +CHAT_ROUTE_BUGS: Final = MappingProxyType( + { + "gradient_ai": "BUG: rejects the max_retries that completion() derives from num_retries, before sending", + "huggingface": "BUG: sends the max_retries that completion() derives from num_retries in the request body", + "ragflow": "BUG: sends the max_retries that completion() derives from num_retries in the request body", + } +) +JSON_BODY: Final = TypeAdapter(dict[str, object]) + + +def _provider(route_id: str) -> str: + return route_id.split("/", 1)[0] + + +def _chat_cases() -> Iterator[tuple[str, bool]]: + for route_id, route in CHAT_ROUTES.items(): + yield route_id, False + if route.streams: + yield route_id, True + + +def _unwalked(routes: Mapping[str, Route], off_the_wire: Mapping[str, str]) -> tuple[str, ...]: + walked: Final = frozenset(_provider(route_id) for route_id in routes) + return tuple(provider.value for provider in LlmProviders if provider.value not in walked | frozenset(off_the_wire)) + + +def _unrouted_model(provider: str) -> str: + return CHAT_ROUTES[provider].model if provider in CHAT_ROUTES else f"{provider}/wire-model" + + +def _complete(parameters: Mapping[str, object]) -> object: + return litellm.completion(**{"messages": [{"role": "user", "content": WALK_TEXT}], **parameters}) + + +def _embed(parameters: Mapping[str, object]) -> object: + return litellm.embedding(**{"input": [WALK_TEXT], **parameters}) + + +def _unrouted_parameters(provider: str, wire_url: str) -> dict[str, object]: + return {"model": _unrouted_model(provider), **SYNTHETIC_KEY, "api_base": wire_url, "num_retries": 0, "timeout": 5} + + +def _sent_body(request: Request) -> Mapping[str, object]: + if request.headers.get("content-type", "").startswith("application/x-www-form-urlencoded"): + fields: Final = parse_qs(request.body.decode(), keep_blank_values=True, strict_parsing=True) + return {name: values[0] for name, values in fields.items()} + return JSON_BODY.validate_json(request.body) + + +def _walked_bodies( + route: Route, seeds: Mapping[str, object], send: Callable[[dict[str, object]], object] +) -> tuple[Mapping[str, object], ...]: + with wire_server(lambda _request: WALK_REJECTION) as wire: + with suppress(openai.OpenAIError, BaseLLMException): + result: Final = send({**seeds, **route.parameters(wire.url), "num_retries": 0, "timeout": 5}) + if isinstance(result, Iterator): + for _chunk in result: + pass + requests: Final = wire.drain() + assert requests, f"{route.model} sent nothing to the wire" + return tuple(_sent_body(request) for request in requests if request.body) + + +def _assert_only_provider_fields( + bodies: tuple[Mapping[str, object], ...], seeds: Mapping[str, object], route: Route +) -> None: + assert any(WALK_TEXT in json.dumps(body) for body in bodies), bodies + for body in bodies: + assert not (frozenset(seeds) - route.protocol_fields) & keys_at_every_depth(body), body + assert not (frozenset(OWNED_KWARG_NAMES) - route.protocol_fields) & frozenset(body), body + assert not tuple(key for key in body if key.startswith("_")), body + + +def test_the_form_body_parser_keeps_a_blank_leaked_field() -> None: + form: Final = Request( + method="POST", + target="/", + headers={"content-type": "application/x-www-form-urlencoded"}, + body=b"inputs=walk+every+provider&skip_mcp_handler=", + ) + + with pytest.raises(AssertionError): + _assert_only_provider_fields((_sent_body(form),), WALK_SEEDS, Route("petals/wire-model")) + + +def test_every_owned_name_is_seeded_or_has_a_reason_it_is_not() -> None: + seeded: Final = frozenset(WALK_SEEDS) - {UNDECLARED_SENTINEL} + unseeded: Final = frozenset(OWNED_NAMES_NOT_SEEDED).union(*map(wire_names, OWNED_LEAVES_NOT_SEEDED)) + + assert seeded.isdisjoint(unseeded), seeded & unseeded + assert seeded | unseeded == frozenset(OWNED_KWARG_NAMES), frozenset(OWNED_KWARG_NAMES) ^ (seeded | unseeded) + + +@pytest.mark.allow_hosts(["127.0.0.1"]) +@pytest.mark.parametrize(("route_id", "stream"), tuple(_chat_cases())) +def test_every_chat_route_keeps_internal_params_out_of_the_provider_body( + provider_wire_environment: None, route_id: str, stream: bool +) -> None: + if route_id in CHAT_ROUTE_BUGS: + pytest.skip(CHAT_ROUTE_BUGS[route_id]) + + bodies: Final = _walked_bodies( + CHAT_ROUTES[route_id], WALK_SEEDS, lambda parameters: _complete({**parameters, "stream": stream}) + ) + + _assert_only_provider_fields(bodies, WALK_SEEDS, CHAT_ROUTES[route_id]) + + +@pytest.mark.allow_hosts(["127.0.0.1"]) +@pytest.mark.parametrize("route_id", EMBEDDING_ROUTES) +def test_every_embedding_route_keeps_internal_params_out_of_the_provider_body( + provider_wire_environment: None, route_id: str +) -> None: + bodies: Final = _walked_bodies(EMBEDDING_ROUTES[route_id], EMBEDDING_WALK_SEEDS, _embed) + + _assert_only_provider_fields(bodies, EMBEDDING_WALK_SEEDS, EMBEDDING_ROUTES[route_id]) + + +@pytest.mark.allow_hosts(["127.0.0.1"]) +@pytest.mark.parametrize("provider", CHAT_PROVIDERS_OFF_THE_WIRE) +def test_a_chat_provider_kept_off_the_wire_sends_nothing_to_it(provider_wire_environment: None, provider: str) -> None: + with wire_server(lambda _request: WALK_REJECTION) as wire: + with suppress(*REFUSALS): + _complete(_unrouted_parameters(provider, wire.url)) + sent: Final = wire.drain() + + assert not sent, f"{provider} reaches the wire, so walk it in CHAT_ROUTES instead" + + +@pytest.mark.allow_hosts(["127.0.0.1"]) +@pytest.mark.parametrize("provider", _unwalked(CHAT_ROUTES, CHAT_PROVIDERS_OFF_THE_WIRE)) +def test_a_provider_without_a_chat_route_has_no_chat_handler(provider_wire_environment: None, provider: str) -> None: + with wire_server(lambda _request: WALK_REJECTION) as wire: + with pytest.raises(LiteLLMUnknownProvider): + _complete(_unrouted_parameters(provider, wire.url)) + sent: Final = wire.drain() + + assert not sent, sent + + +@pytest.mark.allow_hosts(["127.0.0.1"]) +@pytest.mark.parametrize("provider", EMBEDDING_PROVIDERS_OFF_THE_WIRE) +def test_an_embedding_provider_kept_off_the_wire_sends_nothing_to_it( + provider_wire_environment: None, provider: str +) -> None: + with wire_server(lambda _request: WALK_REJECTION) as wire: + with suppress(*REFUSALS): + _embed(_unrouted_parameters(provider, wire.url)) + sent: Final = wire.drain() + + assert not sent, f"{provider} reaches the wire, so walk it in EMBEDDING_ROUTES instead" + + +@pytest.mark.allow_hosts(["127.0.0.1"]) +@pytest.mark.parametrize("provider", _unwalked(EMBEDDING_ROUTES, EMBEDDING_PROVIDERS_OFF_THE_WIRE)) +def test_a_provider_without_an_embedding_route_has_no_embedding_handler( + provider_wire_environment: None, provider: str +) -> None: + with wire_server(lambda _request: WALK_REJECTION) as wire: + with pytest.raises(LiteLLMUnknownProvider): + _embed(_unrouted_parameters(provider, wire.url)) + sent: Final = wire.drain() + + assert not sent, sent + + +def test_every_bedrock_route_and_invoke_family_is_walked() -> None: + routes: Final = { + route: BedrockModelInfo.get_bedrock_route(target.model) for route, target in BEDROCK_ROUTES.items() + } + families: Final = { + family: BaseAWSLLM.get_bedrock_invoke_provider(model=f"invoke/{model}") + for family, model in BEDROCK_INVOKE_MODELS.items() + } + + assert routes == {route: route.split("/", 1)[0] for route in BEDROCK_ROUTES} + assert families == {family: family for family in BEDROCK_INVOKE_MODELS} + assert {*routes.values(), *BEDROCK_ROUTES_WITHOUT_CHAT} == frozenset(get_args(BedrockRoute)) + assert {*families, *BEDROCK_INVOKE_FAMILIES_WITHOUT_CHAT} == frozenset(get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL)) + assert {route_id.split("/", 1)[1] for route_id in EMBEDDING_ROUTES if route_id.startswith("bedrock/")} == frozenset( + get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL) + ) diff --git a/tests/unit/caching/test_caching.py b/tests/unit/caching/test_caching.py index bd14b505323..377601c1dff 100644 --- a/tests/unit/caching/test_caching.py +++ b/tests/unit/caching/test_caching.py @@ -474,6 +474,8 @@ def test_provider_specific_cache_key_ignores_litellm_owned_kwargs(monkeypatch: p assert cache.get_cache_key(**request, _litellm_control={"stream_chunk_size": 64}) == base_key assert cache.get_cache_key(**request, litellm_trace_id="trace-1") == base_key + assert cache.get_cache_key(**request, _skip_mcp_handler=True) == base_key + assert cache.get_cache_key(**request, fake_stream=True) == base_key assert cache.get_cache_key(**{**request, "top_k": 6}) != base_key diff --git a/tests/unit/images/test_main.py b/tests/unit/images/test_main.py index 2e31c1b67a2..c334902a117 100644 --- a/tests/unit/images/test_main.py +++ b/tests/unit/images/test_main.py @@ -252,6 +252,54 @@ def test_image_generation_keeps_an_internal_prefixed_kwarg_out_of_the_provider_r assert sent["prompt"] == "a red circle" +@pytest.mark.parametrize( + ("model", "answer"), + [ + ("bedrock/stability.sd3-5-large-v1:0", {"images": ["aW1n"], "seeds": [1], "finish_reasons": [None]}), + ("bedrock/amazon.nova-canvas-v1:0", {"images": ["aW1n"]}), + ], +) +def test_image_generation_keeps_deployment_prompt_management_params_out_of_the_bedrock_body( + respx_mock: respx.MockRouter, model: str, answer: dict[str, JsonValue] +) -> None: + route: Final = respx_mock.post(url__regex=r"https://bedrock-test\.invalid/.*").mock( + return_value=httpx.Response(200, json=answer) + ) + + litellm.image_generation( + model=model, + prompt="a red circle", + api_key="test-bearer", + aws_region_name="us-west-2", + aws_bedrock_runtime_endpoint="https://bedrock-test.invalid", + cache_control_injection_points=[{"location": "message", "role": "user"}], + ) + + sent: Final = route.calls.last.request.content.decode() + assert "cache_control_injection_points" not in sent, sent + assert "a red circle" in sent, sent + + +def test_image_edit_keeps_deployment_prompt_management_params_out_of_the_provider_body( + respx_mock: respx.MockRouter, +) -> None: + edits_route: Final = respx_mock.post("https://api.openai.com/v1/images/edits").mock( + return_value=httpx.Response(status_code=200, json={"created": 1792551838, "data": [{"b64_json": "aW1n"}]}) + ) + + litellm.image_edit( + model="openai/gpt-image-2.5-sunburst", + image=b"\x89PNG\r\n\x1a\nfakepng", + prompt="make the circle blue", + api_key="sk-test", + cache_control_injection_points=[{"location": "message", "role": "user"}], + ) + + sent: Final = edits_route.calls.last.request.content + assert b"cache_control_injection_points" not in sent, sent + assert b"make the circle blue" in sent, sent + + @pytest.mark.asyncio async def test_aimage_generation_routes_a_bare_model_name_to_its_explicit_provider( respx_mock: respx.MockRouter, diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index fa75e7f9a11..244ce0872b6 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.core_helpers import ( drop_params_env_flag, env_flag, drop_params_flag, + filter_internal_params, get_litellm_metadata_from_kwargs, get_or_create_metadata_bucket, get_parent_otel_span_from_kwargs, @@ -206,6 +207,27 @@ class TestGetOrCreateMetadataBucket: assert bucket is request_data["litellm_metadata"] +def test_filter_internal_params_drops_only_the_mcp_handler_keys() -> None: + rest_of_the_call: Final = { + "api_key": "sk-test", + "guardrails": ["default"], + "metadata": {"team": "search"}, + "stream_chunk_size": 64, + "temperature": 0.2, + } + + filtered: Final = filter_internal_params( + { + **rest_of_the_call, + "_skip_mcp_handler": True, + "skip_mcp_handler": True, + "mcp_handler_context": {"server": "docs"}, + } + ) + + assert filtered == rest_of_the_call + + def test_reconstruct_model_name_prefers_deployment_value(): """Ensure deployment metadata wins when reconstructing the model name.""" diff --git a/tests/unit/litellm_core_utils/test_get_litellm_params.py b/tests/unit/litellm_core_utils/test_get_litellm_params.py index a1184039102..1dc503ffac7 100644 --- a/tests/unit/litellm_core_utils/test_get_litellm_params.py +++ b/tests/unit/litellm_core_utils/test_get_litellm_params.py @@ -200,6 +200,10 @@ class TestGetLitellmParamsKwargsExtraction: ({"stream_chunk_size": "64"}, ControlOptions(stream_chunk_size=64)), ({"stream_chunk_size": None}, ControlOptions()), ({"temperature": 0.2}, ControlOptions()), + ({"fake_stream": True}, ControlOptions(fake_stream=True)), + ({"fake_stream": "true", "stream_chunk_size": 64}, ControlOptions(stream_chunk_size=64, fake_stream=True)), + ({"fake_stream": False}, ControlOptions(fake_stream=False)), + ({"fake_stream": "false"}, ControlOptions(fake_stream=False)), ], ) def test_control_options_are_read_from_the_request_kwargs(kwargs: dict[str, object], expected: ControlOptions) -> None: @@ -236,6 +240,13 @@ def test_control_options_reject_a_stream_chunk_size_that_is_not_a_positive_int(r ) +@pytest.mark.parametrize("raw", ["maybe", "yes", "1", "True", 1, 0, 1.0]) +def test_control_options_reject_a_fake_stream_that_is_not_true_or_false(raw: object) -> None: + assert parse_control_options({"fake_stream": raw}) == InvalidControlOption( + param="fake_stream", message=f"Invalid fake_stream={raw!r}: expected true or false" + ) + + @pytest.mark.parametrize("raw", [10**18 - 1, "9" * 18], ids=["int", "digit_string"]) def test_control_options_accept_the_largest_18_digit_value(raw: object) -> None: assert parse_control_options({"stream_chunk_size": raw}) == ControlOptions(stream_chunk_size=10**18 - 1) diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py index dc84052b812..da5d7533513 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py @@ -48,6 +48,23 @@ def test_completion_keeps_stream_chunk_size_out_of_invoke_bodies(model: str) -> assert "stream_chunk_size" not in keys_at_every_depth(json.loads(request.content)), request.content +def test_transform_request_drops_internal_params() -> None: + """LiteLLM-internal MCP params (e.g. skip_mcp_handler) are control flags used + inside LiteLLM and are not valid Bedrock inference parameters. Leaking them + into the provider request body makes Bedrock reject the request. The Converse + path already filters them via filter_internal_params; the invoke path must do + the same.""" + request_body: Final = AmazonInvokeConfig().transform_request( + model="mistral.mistral-7b-instruct-v0:2", + messages=[{"role": "user", "content": "hi"}], + optional_params={"skip_mcp_handler": True, "max_tokens": 10}, + litellm_params={}, + headers={}, + ) + + assert "skip_mcp_handler" not in json.dumps(request_body) + + def test_validate_environment_maps_guardrail_config_to_invoke_headers(): """The InvokeModel API takes the guardrail identifier/version/trace as X-Amzn-Bedrock-* request headers, unlike Converse which takes them in the diff --git a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py index 5a5653019d9..793b85afd90 100644 --- a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py +++ b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py @@ -342,12 +342,14 @@ _FAKE_STREAMED_REQUESTS: Final = pytest.mark.parametrize( {"response_format": {"type": "json_schema", "json_schema": {"name": "answer", "schema": {"type": "object"}}}}, ), ("bedrock/invoke/ai21.j2-ultra-v1", {}), + ("bedrock/converse/meta.llama3-3-70b-instruct-v1:0", {"fake_stream": True}), ], ids=[ "model_without_streaming", "request_shape_without_streaming", "json_schema_without_native_structured_output", "invoke_route_without_streaming", + "caller_fake_stream", ], ) diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index b86ca7483f6..695ff03213f 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -181,6 +181,37 @@ async def test_get_search_preserves_tinyfish_http_error_formatting(is_async: boo assert error.value.headers["retry-after"] == "7" +def test_a_caller_fake_stream_is_honored_and_kept_off_the_wire(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://api.mistral.ai/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-fake-stream", + "object": "chat.completion", + "created": 1, + "model": "mistral-small-latest", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + ) + + chunks: Final = list( + litellm.completion( + model="mistral/mistral-small-latest", + api_key="fake-mistral-key", + messages=[{"role": "user", "content": "hi"}], + stream=True, + fake_stream=True, + ) + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert "fake_stream" not in sent, sent + assert sent.get("stream") is not True, sent + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "ok" + + def test_prepare_fake_stream_request(): # Initialize the BaseLLMHTTPHandler handler = BaseLLMHTTPHandler() diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index ed6a03afe5c..f903111f595 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -13,6 +13,7 @@ from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from datetime import datetime from importlib import import_module +from itertools import chain from pathlib import Path from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -47,11 +48,13 @@ from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kw from litellm.litellm_core_utils.get_litellm_params import stored_control_options from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.litellm_params import ControlOptions from litellm.types.llms.openai import AllMessageValues, HttpxBinaryResponseContent from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import ( + ChoiceLogprobs, Delta, ModelResponseStream, StandardCallbackDynamicParams, @@ -1115,6 +1118,68 @@ def test_embedding_keeps_an_internal_prefixed_kwarg_out_of_the_provider_request( assert sent["model"] == "text-embedding-3-small" +def test_embedding_keeps_deployment_prompt_management_params_out_of_the_bedrock_body( + respx_mock: respx.MockRouter, +) -> None: + mock_route: Final = respx_mock.post( + "https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-embed-image-v1/invoke" + ).mock(return_value=httpx.Response(status_code=200, json={"embedding": [0.1, 0.2], "inputTextTokenCount": 1})) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "titan-image", + "litellm_params": { + "model": "bedrock/amazon.titan-embed-image-v1", + "api_key": "fake_bedrock_api_key", + "aws_region_name": "us-east-1", + "cache_control_injection_points": [{"location": "message", "role": "user"}], + "knowledge_bases": ["kb-1"], + "vector_store_ids": [], + }, + } + ] + ) + + response: Final = router.embedding(model="titan-image", input=["Help me !"]) + + assert json.loads(mock_route.calls.last.request.content) == {"inputText": "Help me !"} + assert response.data[0]["embedding"] == [0.1, 0.2] + + +def test_embedding_drops_root_prompt_management_params_but_keeps_explicit_extra_body( + respx_mock: respx.MockRouter, +) -> None: + api_base: Final = "http://localhost:12346/v1" + mock_route: Final = respx_mock.post(url__regex=rf"{api_base}/embeddings.*").mock( + return_value=httpx.Response( + status_code=200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + }, + ) + ) + + litellm.embedding( + model="openai/text-embedding-3-small", + input="hi", + api_base=api_base, + api_key="fake_openai_api_key", + cache_control_injection_points=[{"location": "message", "role": "user"}], + knowledge_bases=["kb-1"], + vector_store_ids=["root-store"], + extra_body={"vector_store_ids": ["caller-store"]}, + ) + + sent: Final = json.loads(mock_route.calls.last.request.content) + assert "cache_control_injection_points" not in sent, sent + assert "knowledge_bases" not in sent, sent + assert sent["vector_store_ids"] == ["caller-store"] + assert sent["input"] == "hi" + + def test_custom_provider_with_extra_headers(): with patch.object(litellm.llms.custom_httpx.http_handler.HTTPHandler, "post") as mock_post: @@ -7131,6 +7196,28 @@ def test_drop_params_drops_an_invalid_stream_chunk_size_instead_of_rejecting_it( assert stored_control_options(logging_obj.litellm_params) == ControlOptions() +@pytest.mark.parametrize( + ("given", "kept"), + [ + ({"stream_chunk_size": "sixty-four", "fake_stream": True}, ControlOptions(fake_stream=True)), + ({"stream_chunk_size": 64, "fake_stream": "maybe"}, ControlOptions(stream_chunk_size=64)), + ], +) +def test_drop_params_drops_only_the_invalid_control_option( + given: Mapping[str, object], kept: ControlOptions +) -> None: + logging_obj: Final = _completion_logging_obj(f"drop-only-invalid-{kept}") + litellm.completion( + model="openai/gpt-4.1-mini", + messages=[{"role": "user", "content": "hi"}], + drop_params=True, + mock_response="hi", + litellm_logging_obj=logging_obj, + **given, + ) + assert stored_control_options(logging_obj.litellm_params) == kept + + def test_drop_params_keeps_a_dropped_stream_chunk_size_out_of_the_provider_request( respx_mock: respx.MockRouter, ) -> None: @@ -7164,6 +7251,294 @@ def test_drop_params_keeps_a_dropped_stream_chunk_size_out_of_the_provider_reque assert sent["model"] == "gpt-4.1-mini" +_ROUTES_WITHOUT_A_NATIVE_FAKE_STREAM: Final = pytest.mark.parametrize( + ("model", "url", "answer"), + [ + ( + "openai/gpt-5.6", + "https://api.openai.com/v1/chat/completions", + { + "id": "chatcmpl-replayed", + "object": "chat.completion", + "created": 1712697600, + "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ), + ( + "anthropic/claude-sonnet-5-5", + "https://api.anthropic.com/v1/messages", + { + "id": "msg_replayed", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ), + ], + ids=["openai_sdk", "anthropic"], +) + + +async def _fake_streamed(model: str, asynchronous: bool) -> tuple[CustomStreamWrapper, list[ModelResponseStream]]: + if asynchronous: + streamed: Final = await litellm.acompletion( + model=model, + messages=[{"role": "user", "content": "hi"}], + api_key="fake-key", + stream=True, + stream_options={"include_usage": True}, + fake_stream=True, + ) + assert isinstance(streamed, CustomStreamWrapper), streamed + return streamed, [chunk async for chunk in streamed] + replayed: Final = litellm.completion( + model=model, + messages=[{"role": "user", "content": "hi"}], + api_key="fake-key", + stream=True, + stream_options={"include_usage": True}, + fake_stream=True, + ) + assert isinstance(replayed, CustomStreamWrapper), replayed + return replayed, list(replayed) + + +async def _fake_streamed_chunks(model: str, asynchronous: bool) -> list[ModelResponseStream]: + _, chunks = await _fake_streamed(model, asynchronous) + return chunks + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@_ROUTES_WITHOUT_A_NATIVE_FAKE_STREAM +async def test_a_caller_fake_stream_sends_one_plain_request_and_replays_the_answer_as_a_stream( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + asynchronous: bool, + model: str, + url: str, + answer: Mapping[str, object], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post(url).mock(return_value=httpx.Response(200, json=answer)) + + chunks: Final = await _fake_streamed_chunks(model, asynchronous) + + assert route.call_count == 1 + sent: Final = json.loads(route.calls.last.request.content) + assert sent.get("stream") is not True and "stream_options" not in sent and "fake_stream" not in sent, sent + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) == "ok" + assert chunks[-1].usage is not None and chunks[-1].usage.total_tokens == 2, chunks[-1] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_a_replayed_fake_stream_keeps_the_token_logprobs_of_the_plain_answer( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, asynchronous: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + logprobs: Final = {"content": [{"token": "ok", "logprob": -0.25, "bytes": [111, 107], "top_logprobs": []}]} + respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-replayed", + "object": "chat.completion", + "created": 1712697600, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "logprobs": logprobs, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + ) + + chunks: Final = await _fake_streamed_chunks("openai/gpt-5.6", asynchronous) + + replayed: Final = [chunk.choices[0].logprobs for chunk in chunks if chunk.choices and chunk.choices[0].logprobs] + assert [ChoiceLogprobs.model_validate(item).model_dump(exclude_none=True) for item in replayed] == [logprobs], chunks + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_a_replayed_fake_stream_keeps_the_provider_headers_and_served_tier_of_the_plain_answer( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, asynchronous: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + headers={"x-request-id": "req_replayed", "x-ratelimit-remaining-requests": "99"}, + json={ + "id": "chatcmpl-replayed", + "object": "chat.completion", + "created": 1712697600, + "model": "gpt-5.6", + "service_tier": "flex", + "system_fingerprint": "fp_replayed", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + ) + + stream, chunks = await _fake_streamed("openai/gpt-5.6", asynchronous) + + headers: Final = stream._hidden_params["additional_headers"] + assert headers["llm_provider-x-request-id"] == "req_replayed", headers + assert headers["x-ratelimit-remaining-requests"] == "99", headers + assert {getattr(chunk, "service_tier", None) for chunk in chunks} == {"flex"}, chunks + assert {chunk.system_fingerprint for chunk in chunks} == {"fp_replayed"}, chunks + + +_OPENAI_TEXT_ANSWER: Final = { + "id": "cmpl-replayed", + "object": "text_completion", + "created": 1712697600, + "model": "gpt-3.5-turbo-instruct", + "choices": [{"text": "ok", "index": 0, "logprobs": None, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +async def _fake_streamed_text(entrypoint: str, model: str) -> str: + request: Final = { + "model": model, + "api_key": "fake-key", + "stream": True, + "stream_options": {"include_usage": True}, + "fake_stream": True, + } + match entrypoint: + case "text_completion": + replayed_text: Final = litellm.text_completion(prompt="hi", **request) + return "".join(chunk.choices[0].text or "" for chunk in replayed_text if chunk.choices) + case "atext_completion": + streamed: Final = await litellm.atext_completion(prompt="hi", **request) + return "".join([chunk.choices[0].text or "" async for chunk in streamed if chunk.choices]) + case _: + replayed: Final = litellm.completion(messages=[{"role": "user", "content": "hi"}], **request) + return "".join(chunk.choices[0].delta.content or "" for chunk in replayed if chunk.choices) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "entrypoint, model, url, answer", + [ + ( + "text_completion", + "text-completion-openai/gpt-3.5-turbo-instruct", + "https://api.openai.com/v1/completions", + _OPENAI_TEXT_ANSWER, + ), + ( + "atext_completion", + "text-completion-openai/gpt-3.5-turbo-instruct", + "https://api.openai.com/v1/completions", + _OPENAI_TEXT_ANSWER, + ), + ( + "completion", + "text-completion-codestral/codestral-latest", + "https://codestral.mistral.ai/v1/fim/completions", + { + "id": "cmpl-replayed", + "object": "chat.completion", + "created": 1712697600, + "model": "codestral-latest", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ), + ], + ids=["text_completion", "atext_completion", "codestral_completion"], +) +async def test_a_text_completion_fake_stream_replays_the_plain_answer_as_a_stream( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + entrypoint: str, + model: str, + url: str, + answer: Mapping[str, object], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post(url).mock(return_value=httpx.Response(200, json=answer)) + + text: Final = await _fake_streamed_text(entrypoint, model) + + assert text == "ok" + assert route.call_count == 1 + sent: Final = json.loads(route.calls.last.request.content) + assert sent.get("stream") is not True and "fake_stream" not in sent, sent + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model, url, answer", + [ + ( + "text-completion-openai/gpt-3.5-turbo-instruct", + "https://api.openai.com/v1/completions", + { + **_OPENAI_TEXT_ANSWER, + "choices": [ + {"text": "first", "index": 0, "logprobs": None, "finish_reason": "stop"}, + {"text": "second", "index": 1, "logprobs": None, "finish_reason": "stop"}, + ], + }, + ), + ( + "openai/gpt-5.6", + "https://api.openai.com/v1/chat/completions", + { + "id": "chatcmpl-replayed", + "object": "chat.completion", + "created": 1712697600, + "model": "gpt-5.6", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "first"}, "finish_reason": "stop"}, + {"index": 1, "message": {"role": "assistant", "content": "second"}, "finish_reason": "stop"}, + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + }, + ), + ], + ids=["text_endpoint", "chat_endpoint"], +) +async def test_a_fake_streamed_text_completion_keeps_every_choice( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + model: str, + url: str, + answer: Mapping[str, object], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post(url).mock(return_value=httpx.Response(200, json=answer)) + + streamed: Final = await litellm.atext_completion( + model=model, api_key="fake-key", prompt="hi", n=2, stream=True, fake_stream=True + ) + chunks: Final = [chunk async for chunk in streamed] + choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in chunks)) + + assert {index: "".join(c.text or "" for c in choices if c.index == index) for index in (0, 1)} == { + 0: "first", + 1: "second", + } + + @pytest.mark.asyncio async def test_global_drop_params_drops_an_invalid_stream_chunk_size_on_acompletion( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index 044005620fd..59798610230 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -231,6 +231,7 @@ OPTION_NAMES: Final = ( "enable_json_schema_validation", "complete_response", "stream_chunk_size", + "fake_stream", "keepalive_seconds", "allow_client_keepalive_override", "mock_response", @@ -266,6 +267,9 @@ INTERNAL_STATE_NAMES: Final = ( "stream_response", "_context_compaction_state", *AGENTIC_LOOP_STATE_NAMES, + "_skip_mcp_handler", + "skip_mcp_handler", + "mcp_handler_context", "_router_weights", "_encrypted_content_affinity_pinned", "fallback_depth", @@ -569,7 +573,7 @@ LEAF_SAMPLES: Final[Mapping[type, Mapping[str, object]]] = { litellm_params.GuardrailOptions: {"guardrails": ("default",)}, litellm_params.PromptOptions: {"prompt_id": "prompt", "prompt_variables": {"name": "value"}}, litellm_params.ResponseOptions: {"keepalive_seconds": 1.5}, - litellm_params.ControlOptions: {"stream_chunk_size": 64}, + litellm_params.ControlOptions: {"stream_chunk_size": 64, "fake_stream": True}, litellm_params.MockOptions: {"mock_timeout": True}, litellm_params.CallState: { "completion_call_id": "call", @@ -577,6 +581,7 @@ LEAF_SAMPLES: Final[Mapping[type, Mapping[str, object]]] = { "data_residency": "us", }, litellm_params.AgenticLoopState: {"api_surface": "chat_completions", "depth": 1}, + litellm_params.McpHandlerState: {"recursion_guard": True, "mcp_handler_context": {"server": "docs"}}, litellm_params.RouterState: {"fallback_depth": 1}, litellm_params.ProxyRequestState: { "proxy_server_request": {"path": "/chat/completions"}, @@ -605,6 +610,7 @@ LEAF_BAD_SAMPLES: Final[Mapping[type, Mapping[str, object]]] = { litellm_params.MockOptions: {"mock_timeout": "true"}, litellm_params.CallState: {"completion_call_id": 1}, litellm_params.AgenticLoopState: {"depth": "1"}, + litellm_params.McpHandlerState: {"recursion_guard": "true"}, litellm_params.RouterState: {"fallback_depth": "1"}, litellm_params.ProxyRequestState: {"proxy_server_request": "request"}, litellm_params.EntrypointState: {"acompletion": "true"},