diff --git a/litellm/constants.py b/litellm/constants.py index a5be2f6568d..dac15c01fbf 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1610,6 +1610,7 @@ ALLOWED_VERTEX_AI_PASSTHROUGH_HEADERS: Final = { # e.g., 'x-pass-anthropic-beta: value' becomes 'anthropic-beta: value' # Works for all LLM pass-through endpoints (Vertex AI, Anthropic, Bedrock, etc.) PASS_THROUGH_HEADER_PREFIX: Final = "x-pass-" +INTERNAL_KWARG_PREFIX: Final = "_litellm_" AZURE_SPEECH_CUSTOM_LLM_PROVIDER: Final = "azure_speech" AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX: Final = "/azure_speech" diff --git a/litellm/images/main.py b/litellm/images/main.py index 1f722eb752a..5ca8a726a69 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -52,7 +52,7 @@ from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( LITELLM_IMAGE_VARIATION_PROVIDERS, LlmProviders, - all_litellm_params, + is_litellm_owned_kwarg, ) from litellm.utils import ( ImageResponse, @@ -249,11 +249,9 @@ def image_generation( "size", "style", ] - litellm_params: Final = all_litellm_params - default_params: Final = openai_params + litellm_params non_default_params: Final = { - k: v for k, v in kwargs.items() if k not in default_params - } # model-specific params - pass them straight to the model/provider + k: v for k, v in kwargs.items() if k not in openai_params and not is_litellm_owned_kwarg(k) + } image_generation_config: BaseImageGenerationConfig | None = None if custom_llm_provider is not None and custom_llm_provider in LlmProviders._member_map_.values(): @@ -757,11 +755,9 @@ def image_edit( "style", "async_call", ] - litellm_params_list: Final = all_litellm_params - default_params: Final = openai_params + litellm_params_list non_default_params: Final = { - k: v for k, v in kwargs.items() if k not in default_params - } # model-specific params - pass them straight to the model/provider + k: v for k, v in kwargs.items() if k not in openai_params and not is_litellm_owned_kwarg(k) + } litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None) model_info: Final = kwargs.get("model_info", None) diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index ce4c955a884..fdc8e34ed3d 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -58,7 +58,7 @@ from litellm.types.llms.openai import ( OpenAIFileObject, PathLike, ) -from litellm.types.utils import ExtractedFileData, LlmProviders, SpecialEnums, all_litellm_params +from litellm.types.utils import ExtractedFileData, LlmProviders, SpecialEnums, is_litellm_owned_kwarg from litellm.utils import get_llm_provider, get_optional_params from ..base_aws_llm import BaseAWSLLM @@ -907,7 +907,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): { k: v for k, v in optional_params.items() - if k not in all_litellm_params or k in _LITELLM_PARAMS_THE_MAPPER_TAKES + if not is_litellm_owned_kwarg(k) or k in _LITELLM_PARAMS_THE_MAPPER_TAKES } ), ) diff --git a/litellm/llms/elevenlabs/text_to_speech/transformation.py b/litellm/llms/elevenlabs/text_to_speech/transformation.py index 3cf9a983efe..eb93543df46 100644 --- a/litellm/llms/elevenlabs/text_to_speech/transformation.py +++ b/litellm/llms/elevenlabs/text_to_speech/transformation.py @@ -18,7 +18,7 @@ from litellm.llms.base_llm.text_to_speech.transformation import ( TextToSpeechRequestData, ) from litellm.secret_managers.main import get_secret_str -from litellm.types.utils import all_litellm_params +from litellm.types.utils import is_litellm_owned_kwarg from ..common_utils import ElevenLabsException @@ -241,7 +241,7 @@ class ElevenLabsTextToSpeechConfig(BaseTextToSpeechConfig): continue mapped_params[key] = value - reserved_kwarg_keys: Final = set(all_litellm_params) | { + reserved_kwarg_keys: Final = { self.ELEVENLABS_QUERY_PARAMS_KEY, self.ELEVENLABS_VOICE_ID_KEY, "voice", @@ -260,7 +260,7 @@ class ElevenLabsTextToSpeechConfig(BaseTextToSpeechConfig): mapped_params[key] = value for key in list(kwargs.keys()): - if key in reserved_kwarg_keys: + if key in reserved_kwarg_keys or is_litellm_owned_kwarg(key): continue value = kwargs[key] if value is None: diff --git a/litellm/main.py b/litellm/main.py index 12854db15d0..8c2afe4429a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -284,7 +284,7 @@ from .types.utils import ( LlmProviders, PromptTokensDetails, ProviderSpecificHeader, - all_litellm_params, + is_litellm_owned_kwarg, ) ####### ENVIRONMENT VARIABLES ################### @@ -6351,15 +6351,10 @@ def embedding( "max_retries", "encoding_format", ] - litellm_params: Final = [ - "aembedding", - "extra_headers", - ] + all_litellm_params - - default_params: Final = openai_params + litellm_params + default_params: Final = [*openai_params, "aembedding", "extra_headers"] non_default_params: Final = { - k: v for k, v in kwargs.items() if k not in default_params - } # model-specific params - pass them straight to the model/provider + k: v for k, v in kwargs.items() if k not in default_params and not is_litellm_owned_kwarg(k) + } model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( model=model, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 749ef229fbe..f8b57139b37 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -48,6 +48,7 @@ from typing_extensions import NotRequired, ReadOnly, Required, TypedDict from litellm._logging import verbose_logger from litellm._uuid import uuid +from litellm.constants import INTERNAL_KWARG_PREFIX from litellm.types.llms.base import ( BaseLiteLLMOpenAIResponseObject, CachedTokensDetails, @@ -3937,6 +3938,10 @@ all_litellm_params = [ # rebind-ok: two star imports in litellm/__init__.py re- ] +def is_litellm_owned_kwarg(name: str) -> bool: + return name in all_litellm_params or name.startswith(INTERNAL_KWARG_PREFIX) + + class KeyGenerationConfig(TypedDict, total=False): required_params: list[str] # specify params that must be present in the key generation request diff --git a/litellm/utils.py b/litellm/utils.py index e5eea562c11..092fe936cf9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -257,7 +257,7 @@ from litellm.types.utils import ( TextCompletionResponse, TranscriptionResponse, Usage, - all_litellm_params, + is_litellm_owned_kwarg, ) _CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes} @@ -4161,26 +4161,8 @@ def _remove_unsupported_params(non_default_params: dict, supported_openai_params return non_default_params -def filter_out_litellm_params(kwargs: dict) -> dict: - """ - Filter out LiteLLM internal parameters from kwargs dict. - - Returns a new dict containing only non-LiteLLM parameters that should be - passed to external provider APIs. - - Args: - kwargs: Dictionary that may contain LiteLLM internal parameters - - Returns: - Dictionary with LiteLLM internal parameters filtered out - - Example: - >>> kwargs = {"query": "test", "shared_session": session_obj, "metadata": {}} - >>> filtered = filter_out_litellm_params(kwargs) - >>> # filtered = {"query": "test"} - """ - - return {key: value for key, value in kwargs.items() if key not in all_litellm_params} +def filter_out_litellm_params(kwargs: Mapping[str, object]) -> dict: + return {key: value for key, value in kwargs.items() if not is_litellm_owned_kwarg(key)} def _provider_supports_vertex_params(custom_llm_provider: str) -> bool: @@ -10152,10 +10134,9 @@ def get_standard_openai_params(params: Mapping[str, object]) -> dict: def get_non_default_completion_params(kwargs: Mapping[str, object]) -> dict: openai_params: Final = litellm.OPENAI_CHAT_COMPLETION_PARAMS - default_params: Final = openai_params + all_litellm_params non_default_params: Final = { - k: v for k, v in kwargs.items() if k not in default_params - } # model-specific params - pass them straight to the model/provider + k: v for k, v in kwargs.items() if k not in openai_params and not is_litellm_owned_kwarg(k) + } return non_default_params @@ -10203,11 +10184,12 @@ def strip_reasoning_summary_aliases_from_optional_params( return op, rs_val -def get_non_default_transcription_params(kwargs: dict) -> dict: +def get_non_default_transcription_params(kwargs: Mapping[str, object]) -> dict: from litellm.constants import OPENAI_TRANSCRIPTION_PARAMS - default_params: Final = OPENAI_TRANSCRIPTION_PARAMS + all_litellm_params - non_default_params: Final = {k: v for k, v in kwargs.items() if k not in default_params} + non_default_params: Final = { + k: v for k, v in kwargs.items() if k not in OPENAI_TRANSCRIPTION_PARAMS and not is_litellm_owned_kwarg(k) + } return non_default_params diff --git a/tests/integration/providers/test_stream_chunk_size_wire.py b/tests/integration/providers/test_internal_params_wire.py similarity index 98% rename from tests/integration/providers/test_stream_chunk_size_wire.py rename to tests/integration/providers/test_internal_params_wire.py index 3681da0e3d4..17b0fc9d815 100644 --- a/tests/integration/providers/test_stream_chunk_size_wire.py +++ b/tests/integration/providers/test_internal_params_wire.py @@ -276,7 +276,7 @@ def provider_wire_environment(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) - @pytest.mark.parametrize("provider", PROVIDERS) @pytest.mark.parametrize("asynchronous", [False, True]) @pytest.mark.parametrize("stream", [False, True]) -async def test_stream_chunk_size_never_reaches_provider_body( +async def test_internal_params_never_reach_provider_body( monkeypatch: pytest.MonkeyPatch, provider_wire_environment: None, provider: str, @@ -289,6 +289,7 @@ async def test_stream_chunk_size_never_reaches_provider_body( **_request_parameters(provider, wire.url), "stream": stream, "stream_chunk_size": 64, + "_litellm_undeclared_sentinel": "internal", "extra_body": {"custom_provider_key": 1}, "max_tokens": 16, "timeout": 5, @@ -313,4 +314,5 @@ async def test_stream_chunk_size_never_reaches_provider_body( keys: Final = keys_at_every_depth(body) assert "stream_chunk_size" not in keys assert not INTERNAL_FIELDS.intersection(keys) + assert not frozenset(key for key in keys if key.startswith("_litellm_")), keys assert _custom_key(body, provider) == 1 diff --git a/tests/unit/images/test_image_edit_extra_params.py b/tests/unit/images/test_image_edit_extra_params.py index 088faafa9f3..c3b0a5d2828 100644 --- a/tests/unit/images/test_image_edit_extra_params.py +++ b/tests/unit/images/test_image_edit_extra_params.py @@ -58,6 +58,26 @@ def test_image_edit_forwards_provider_params_and_extra_body(): assert response.data +def test_image_edit_keeps_an_internal_prefixed_kwarg_out_of_the_provider_request(): + captured = {} + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_image_edit_request(captured)))) + + litellm.image_edit( + model="openai/gpt-image-1", + image=PNG_BYTES, + prompt="add a hat", + api_key="sk-test", + api_base="https://edit.example/v1", + client=client, + seed=42, + _litellm_undeclared_sentinel="internal", + ) + + fields = _multipart_text_fields(captured["content_type"], captured["body"]) + assert "_litellm_undeclared_sentinel" not in fields + assert fields["seed"] == "42" + + def test_image_edit_extra_body_takes_precedence_over_kwargs(): captured = {} client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_image_edit_request(captured)))) diff --git a/tests/unit/images/test_main.py b/tests/unit/images/test_main.py new file mode 100644 index 00000000000..d65e5d929b5 --- /dev/null +++ b/tests/unit/images/test_main.py @@ -0,0 +1,29 @@ +import json +from typing import Final + +import httpx +import respx + +import litellm + + +def test_image_generation_keeps_an_internal_prefixed_kwarg_out_of_the_provider_request( + respx_mock: respx.MockRouter, +) -> None: + api_base: Final = "http://localhost:12346/v1" + mock_route: Final = respx_mock.post(url__regex=rf"{api_base}/images/generations.*").mock( + return_value=httpx.Response(status_code=200, json={"created": 1712697600, "data": [{"b64_json": "aW1n"}]}) + ) + + litellm.image_generation( + model="openai/gpt-image-1", + prompt="a red circle", + api_base=api_base, + api_key="fake_openai_api_key", + _litellm_undeclared_sentinel="internal", + ) + + assert mock_route.called + sent: Final = json.loads(respx_mock.calls[0].request.content) + assert "_litellm_undeclared_sentinel" not in sent, sent + assert sent["prompt"] == "a red circle" diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index b2ce4ab2dde..12275df404f 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -84,6 +84,29 @@ class TestBedrockFilesTransformation: "max_tokens" in model_input ), f"Record {i+1} should have max_tokens" + def test_batch_keeps_an_internal_prefixed_key_out_of_the_bedrock_model_input(self): + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + result: Final = BedrockFilesConfig()._transform_openai_jsonl_content_to_bedrock_jsonl_content( + [ + { + "custom_id": "internal-key-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 10, + "_litellm_undeclared_sentinel": "internal", + }, + } + ] + ) + + model_input: Final = json.dumps(result[0]["modelInput"]) + assert "_litellm_undeclared_sentinel" not in model_input, model_input + assert result[0]["modelInput"]["max_tokens"] == 10 + def test_nova_text_only_uses_converse_format(self): """ Test that Nova models produce Converse API format in batch modelInput. diff --git a/tests/unit/llms/elevenlabs/test_elevenlabs_text_to_speech_transformation.py b/tests/unit/llms/elevenlabs/test_elevenlabs_text_to_speech_transformation.py index 54e689dea6b..d05371d7df9 100644 --- a/tests/unit/llms/elevenlabs/test_elevenlabs_text_to_speech_transformation.py +++ b/tests/unit/llms/elevenlabs/test_elevenlabs_text_to_speech_transformation.py @@ -1,5 +1,11 @@ -import pytest +import json +from typing import Final +import httpx +import pytest +import respx + +import litellm from litellm.llms.elevenlabs.text_to_speech.transformation import ( ElevenLabsTextToSpeechConfig, ) @@ -16,10 +22,7 @@ def test_should_encode_elevenlabs_voice_id_path_segment(): }, ) - assert ( - url - == "https://api.elevenlabs.io/v1/text-to-speech/voice%2F..%2F..%2Fmodels%3Fx%3D1%23frag" - ) + assert url == "https://api.elevenlabs.io/v1/text-to-speech/voice%2F..%2F..%2Fmodels%3Fx%3D1%23frag" def test_should_reject_dot_segment_elevenlabs_voice_id(): @@ -31,3 +34,24 @@ def test_should_reject_dot_segment_elevenlabs_voice_id(): api_base="https://api.elevenlabs.io", litellm_params={config.ELEVENLABS_VOICE_ID_KEY: ".."}, ) + + +def test_speech_keeps_an_internal_prefixed_kwarg_out_of_the_elevenlabs_request(respx_mock: respx.MockRouter) -> None: + api_base: Final = "http://localhost:12346" + mock_route: Final = respx_mock.post(url__regex=rf"{api_base}/v1/text-to-speech/.*").mock( + return_value=httpx.Response(status_code=200, content=b"audio", headers={"content-type": "audio/mpeg"}) + ) + + litellm.speech( + model="elevenlabs/eleven_multilingual_v2", + input="hi", + voice="21m00Tcm4TlvDq8ikWAM", + api_base=api_base, + api_key="fake_elevenlabs_api_key", + _litellm_undeclared_sentinel="internal", + ) + + assert mock_route.called + sent: Final = json.loads(respx_mock.calls[0].request.content) + assert "_litellm_undeclared_sentinel" not in sent, sent + assert sent["text"] == "hi" diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 57200a79a8c..7bef35d8559 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -395,6 +395,34 @@ def test_completion_strips_eager_input_streaming_before_openai(respx_mock: respx assert sent_tool["function"]["name"] == "write_file" +def test_embedding_keeps_an_internal_prefixed_kwarg_out_of_the_provider_request(respx_mock: respx.MockRouter) -> None: + api_base: Final = "http://localhost:12346/v1" + mock_route: Final = respx_mock.post(url__regex=rf"{api_base}/embeddings.*").mock( + return_value=httpx.Response( + status_code=200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + }, + ) + ) + + litellm.embedding( + model="openai/text-embedding-3-small", + input="hi", + api_base=api_base, + api_key="fake_openai_api_key", + _litellm_undeclared_sentinel="internal", + ) + + assert mock_route.called + sent: Final = json.loads(respx_mock.calls[0].request.content) + assert "_litellm_undeclared_sentinel" not in sent, sent + assert sent["model"] == "text-embedding-3-small" + + def test_custom_provider_with_extra_headers(): with patch.object( diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index e421321aaaa..33467a78aa8 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -262,7 +262,7 @@ OWNED_NAMES: Final = ( *PRICING_NAMES, ) -Classifier: TypeAlias = Callable[[dict[str, object]], dict[str, object]] # mutable-ok: classifiers use dict +Classifier: TypeAlias = Callable[[Mapping[str, object]], Mapping[str, object]] CLASSIFIERS: Final[Mapping[str, Classifier]] = MappingProxyType( { # pyright: ignore[reportUnknownArgumentType] # untyped legacy classifiers @@ -279,18 +279,31 @@ def test_owned_name_is_kept_out_of_provider_params(name: str, classifier_name: s provider_value: Final = object() classify: Final = CLASSIFIERS[classifier_name] - result: Final = classify({name: object(), PROVIDER_KNOB: provider_value}) # mutable-ok: classifiers take a dict + result: Final = classify(MappingProxyType({name: object(), PROVIDER_KNOB: provider_value})) assert result == MappingProxyType({PROVIDER_KNOB: provider_value}) assert result[PROVIDER_KNOB] is provider_value def test_a_name_no_object_declares_reaches_the_provider() -> None: - result: Final = CLASSIFIERS["completion"]({PROVIDER_KNOB: 1}) # mutable-ok: classifier input type + result: Final = CLASSIFIERS["completion"](MappingProxyType({PROVIDER_KNOB: 1})) assert result == MappingProxyType({PROVIDER_KNOB: 1}) +@pytest.mark.parametrize("classifier_name", CLASSIFIERS) +def test_an_undeclared_internal_prefixed_name_is_kept_out_of_provider_params(classifier_name: str) -> None: + undeclared: Final = "_litellm_never_declared_anywhere" + lookalike: Final = "provider_litellm_knob" + assert undeclared not in all_litellm_params + + result: Final = CLASSIFIERS[classifier_name]( + MappingProxyType({undeclared: object(), PROVIDER_KNOB: 1, lookalike: 2}) + ) + + assert result == MappingProxyType({PROVIDER_KNOB: 1, lookalike: 2}) + + def _cache_key_for_model_group(cache: Cache, model_group: str, options: CachingOptions) -> str: return cache.get_cache_key( # pyright: ignore[reportUnknownMemberType] # untyped legacy key builder model=model_group, @@ -421,9 +434,7 @@ CARRIED_PARAMS: Final = tuple( def test_every_param_get_litellm_params_carries_is_kept_out_of_provider_params(name: str) -> None: provider_value: Final = object() - result: Final = CLASSIFIERS["completion"]( - {name: object(), PROVIDER_KNOB: provider_value} # mutable-ok: classifier input type - ) + result: Final = CLASSIFIERS["completion"](MappingProxyType({name: object(), PROVIDER_KNOB: provider_value})) assert result == MappingProxyType({PROVIDER_KNOB: provider_value})