diff --git a/litellm-rust/crates/python-bridge/src/routes/inference.rs b/litellm-rust/crates/python-bridge/src/routes/inference.rs index bc442d1cf36..a5be7f51828 100644 --- a/litellm-rust/crates/python-bridge/src/routes/inference.rs +++ b/litellm-rust/crates/python-bridge/src/routes/inference.rs @@ -21,10 +21,7 @@ use serde_json::{Map, Value}; use super::{NativeCall, RequestView}; use crate::{ errors::{RustUpstreamError, route_error_to_pyerr}, - marshal::{ - RouteOptions, optional_timeout, project_optional_fields, public_response, - python_timeout_seconds, - }, + marshal::{RouteOptions, optional_timeout, public_response, python_timeout_seconds}, }; pub(super) struct InferenceHost { @@ -115,10 +112,7 @@ impl InferenceHost { py: Python<'_>, arguments: &Bound<'_, PyDict>, ) -> PyResult> { - let names: Vec = py.import(self.module)?.getattr("PARAMETERS")?.extract()?; - project_optional_fields(names.iter().map(String::as_str), |name| { - self.argument(py, arguments, name) - }) + self.view.body_params(py, arguments, self.module) } pub fn response(&self, py: Python<'_>, response: &impl Serialize) -> PyResult> { diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index 1add8fa485f..5e445fcaa7d 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -29,31 +29,6 @@ use crate::{ const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.messages"; const REQUEST_ERROR_MARKER: &str = "messages_request_error"; -const BODY_FIELDS: [&str; 22] = [ - "max_tokens", - "metadata", - "stop_sequences", - "stream", - "system", - "temperature", - "thinking", - "tool_choice", - "tools", - "top_k", - "inference_geo", - "top_p", - "mcp_servers", - "context_management", - "compaction", - "container", - "output_format", - "speed", - "output_config", - "cache_control", - "reasoning_effort", - "safeguards", -]; - fn merge_headers( forwarded: Option>, extra_headers: Option>, @@ -143,7 +118,7 @@ impl MessagesPythonHost { let model = string("model")?.ok_or_else(|| PyValueError::new_err("model is required"))?; let messages = argument("messages")?.ok_or_else(|| PyValueError::new_err("messages is required"))?; - let fields = project_optional_fields(BODY_FIELDS, argument)?; + let fields = self.view.body_params(py, arguments, ROUTE_HOST_MODULE)?; let body = [ ("model".to_string(), Value::String(model.clone())), ("messages".to_string(), from_py(&messages)?), diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs index 444ef4f01b7..f51c755c33e 100644 --- a/litellm-rust/crates/python-bridge/src/routes/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -13,6 +13,7 @@ use litellm_callbacks_legacy_python::{LegacyLogging, LoggingOperation, PublicCal use litellm_host::{call::HostedCompletion, machine::Machine, protocol::Protocol}; use litellm_host_python::{ CallOptions, HookChain, PythonBinding, PythonCallHooks, PythonHostCalls, effective_py_args, + from_py, }; use litellm_host_python::{Error as HostError, present}; use pyo3::{ @@ -20,6 +21,7 @@ use pyo3::{ prelude::*, types::{PyDict, PyMapping, PyTuple}, }; +use serde_json::{Map, Value}; /// The public call as Python bound it: `base` holds the positional arguments by name plus /// the signature defaults, `kwargs` the caller's keyword dict. @@ -92,6 +94,22 @@ impl RequestView { present(arguments, self.request(py)?, name) } + /// The body params `module.body_params` keeps from the keywords the call would send, so + /// each route's Python host decides which params reach the provider. + pub(crate) fn body_params( + &self, + py: Python<'_>, + arguments: &Bound<'_, PyDict>, + module: &str, + ) -> PyResult> { + let request = effective_py_args(self.request(py)?, arguments)?; + from_py( + &py.import(module)? + .getattr("body_params")? + .call1((request,))?, + ) + } + pub(crate) fn close(&mut self) { self.0 = None; } diff --git a/litellm/rust_bridge/host/routes/chat_completions.py b/litellm/rust_bridge/host/routes/chat_completions.py index d600495c919..5fcb2edf4f7 100644 --- a/litellm/rust_bridge/host/routes/chat_completions.py +++ b/litellm/rust_bridge/host/routes/chat_completions.py @@ -6,7 +6,7 @@ from typing import Final import litellm from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS from litellm.rust_bridge import failures -from litellm.rust_bridge.public_call import optional_str +from litellm.rust_bridge.public_call import known_params, optional_str from litellm.types.utils import ModelResponse _TRANSPORT_PARAMETERS: Final = frozenset( @@ -27,6 +27,10 @@ _TRANSPORT_PARAMETERS: Final = frozenset( PARAMETERS: Final = tuple(name for name in OPENAI_CHAT_COMPLETION_PARAMS if name not in _TRANSPORT_PARAMETERS) +def body_params(request: Mapping[str, object]) -> Mapping[str, object]: + return known_params(PARAMETERS, request) + + def connection_defaults(provider: str) -> tuple[str | None, str | None]: if provider == "anthropic": return litellm.anthropic_key or litellm.api_key, litellm.api_base diff --git a/litellm/rust_bridge/host/routes/messages.py b/litellm/rust_bridge/host/routes/messages.py index 5dfd3bfd530..1024141d5a2 100644 --- a/litellm/rust_bridge/host/routes/messages.py +++ b/litellm/rust_bridge/host/routes/messages.py @@ -12,10 +12,36 @@ from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout from litellm.llms.anthropic.pass_through.utils import is_reasoning_auto_summary_enabled from litellm.rust_bridge import failures -from litellm.rust_bridge.public_call import optional_str +from litellm.rust_bridge.public_call import known_params, optional_str from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse _DROP_PATHS: Final = TypeAdapter(list[object]) +_BODY_FIELDS: Final = frozenset( + { + "max_tokens", + "metadata", + "stop_sequences", + "stream", + "system", + "temperature", + "thinking", + "tool_choice", + "tools", + "top_k", + "inference_geo", + "top_p", + "mcp_servers", + "context_management", + "compaction", + "container", + "output_format", + "speed", + "output_config", + "cache_control", + "reasoning_effort", + "safeguards", + } +) @dataclass(frozen=True, slots=True) @@ -33,6 +59,10 @@ def response(value: Mapping[str, object]) -> AnthropicMessagesResponse: ) +def body_params(request: Mapping[str, object]) -> Mapping[str, object]: + return known_params(_BODY_FIELDS, request) + + def stream_hidden_params(headers: Sequence[tuple[str, str]]) -> Mapping[str, object]: from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( anthropic_messages_stream_hidden_params, diff --git a/litellm/rust_bridge/host/routes/responses.py b/litellm/rust_bridge/host/routes/responses.py index 218a866cf46..470a89a0e05 100644 --- a/litellm/rust_bridge/host/routes/responses.py +++ b/litellm/rust_bridge/host/routes/responses.py @@ -6,12 +6,16 @@ from typing import Final import litellm from litellm import get_llm_provider from litellm.rust_bridge import failures -from litellm.rust_bridge.public_call import inference_decline_reason, optional_str +from litellm.rust_bridge.public_call import inference_decline_reason, known_params, optional_str from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse PARAMETERS: Final = tuple(ResponsesAPIOptionalRequestParams.__annotations__) +def body_params(request: Mapping[str, object]) -> Mapping[str, object]: + return known_params(PARAMETERS, request) + + def connection_defaults(provider: str) -> tuple[str | None, str | None]: provider_key: Final = litellm.openai_key if provider == "openai" else None return litellm.api_key or provider_key, litellm.api_base diff --git a/litellm/rust_bridge/public_call.py b/litellm/rust_bridge/public_call.py index e5700be89ad..332421c68cc 100644 --- a/litellm/rust_bridge/public_call.py +++ b/litellm/rust_bridge/public_call.py @@ -3,7 +3,7 @@ from __future__ import annotations import inspect -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Collection, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import Final, TypeVar, cast # noqa: TID251 # narrows caller-owned containers without copying them @@ -71,6 +71,10 @@ def optional_sequence(value: object) -> Sequence[object] | None: return cast("Sequence[object]", value) # cast-ok: the same caller-owned object is handed on unchanged +def known_params(names: Collection[str], request: Mapping[str, object]) -> Mapping[str, object]: + return {name: value for name, value in request.items() if name in names and value is not None} + + def inference_decline_reason(parameters: tuple[str, ...], kwargs: Mapping[str, object]) -> str | None: if litellm.drop_params or litellm.modify_params: return "native inference does not implement the configured parameter rewrites" diff --git a/tests/unit/rust_bridge/host/routes/test_chat_completions.py b/tests/unit/rust_bridge/host/routes/test_chat_completions.py index 3496791f5a0..ea715a6eb22 100644 --- a/tests/unit/rust_bridge/host/routes/test_chat_completions.py +++ b/tests/unit/rust_bridge/host/routes/test_chat_completions.py @@ -4,7 +4,7 @@ from typing import Final import pytest import litellm -from litellm.rust_bridge.host.routes.chat_completions import connection_defaults, response +from litellm.rust_bridge.host.routes.chat_completions import body_params, connection_defaults, response from litellm.types.utils import ModelResponse @@ -57,3 +57,20 @@ def test_connection_defaults_preserve_provider_precedence( monkeypatch.setattr(litellm, "anthropic_key", provider_key) monkeypatch.setattr(litellm, "api_base", "https://configured.invalid") assert connection_defaults(provider) == (expected_key, expected_base) + + +def test_body_params_keep_request_params_and_leave_out_transport_and_unset_ones() -> None: + body: Final = body_params( + MappingProxyType( + { + "model": "anthropic/claude-sonnet-5-5", + "temperature": 0.2, + "max_tokens": None, + "api_key": "sk-secret", + "api_base": "https://example.invalid", + "timeout": 30, + } + ) + ) + + assert body == {"temperature": 0.2}