feat(messages): apply modify_params rewrites on rust

This commit is contained in:
narmstrong-berry 2026-11-10 20:11:30 +00:00
parent 6e99bb1f3d
commit 12c6283fd3
No known key found for this signature in database
12 changed files with 378 additions and 80 deletions

View file

@ -51,15 +51,21 @@ fn selected_cache<'py>(
kwargs: &Bound<'py, PyDict>,
call_type: &str,
) -> PyResult<Option<Bound<'py, PyAny>>> {
if kwargs
.get_item("caching")?
.is_some_and(|value| value.is(pyo3::types::PyBool::new(py, false)))
{
return Ok(None);
}
configured_cache(py, call_type)
}
fn configured_cache<'py>(py: Python<'py>, call_type: &str) -> PyResult<Option<Bound<'py, PyAny>>> {
let configured = py
.import(CACHE_HOST_MODULE)?
.getattr("configured_cache")?
.call0()?;
if configured.is_none()
|| kwargs
.get_item("caching")?
.is_some_and(|value| value.is(pyo3::types::PyBool::new(py, false)))
{
if configured.is_none() {
return Ok(None);
}
let supported = configured.getattr("supported_call_types")?;
@ -94,7 +100,7 @@ pub(crate) fn configure(
) -> PyResult<Selection> {
let call_type: String = function.getattr("__name__")?.extract()?;
let selected = if python.asynchronous() {
selected_cache(py, arguments, &call_type)?
configured_cache(py, &call_type)?
} else {
None
};
@ -107,6 +113,9 @@ pub(crate) fn configure(
let enabled = cache
.call_method("should_use_cache", (), Some(arguments))?
.extract::<bool>()?;
let skips_read = arguments
.get_item("caching")?
.is_some_and(|value| !value.is_none() && !value.is(pyo3::types::PyBool::new(py, true)));
let controls = arguments
.get_item("cache")?
.filter(|value| !value.is_none());
@ -122,7 +131,7 @@ pub(crate) fn configure(
};
let options = CacheOptions {
policy: CachePolicy {
no_cache: boolean("no-cache")?,
no_cache: boolean("no-cache")? || skips_read,
no_store: boolean("no-store")?,
..CachePolicy::default()
},

View file

@ -28,7 +28,7 @@ use crate::{
routes::RequestView,
};
const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.messages";
pub(super) const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.messages";
fn merge_headers(
forwarded: Option<Map<String, Value>>,

View file

@ -1,7 +1,9 @@
mod host;
mod modify_params;
use host::MessagesPythonHost;
use litellm_callbacks_legacy_python::LoggingOperation;
use modify_params::ModifyParams;
use pyo3::prelude::*;
use super::NativeCall;
@ -9,7 +11,8 @@ use super::NativeCall;
fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyResult<Py<PyAny>> {
let (arguments, hooks) =
crate::routes::call_hooks(py, LoggingOperation::Messages, &call, asynchronous)?;
crate::routes::run_public_call(
let modify_params = ModifyParams::new(&call.args, &call.kwargs)?;
crate::routes::run_public_call_after_policy(
py,
arguments,
move |py, arguments, request| {
@ -46,6 +49,7 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR
},
MessagesPythonHost::new(call.view()?, asynchronous),
hooks,
modify_params,
asynchronous,
)
}

View file

@ -0,0 +1,152 @@
use litellm_host::hooks::CallHooks;
use litellm_host_python::{PythonOwned, PythonRuntime};
use pyo3::{
exceptions::PyException,
gc::{PyTraverseError, PyVisit},
prelude::*,
types::{PyDict, PyTuple},
};
use super::host::ROUTE_HOST_MODULE;
pub(super) struct ModifyParams {
call: Option<(Py<PyAny>, Py<PyTuple>)>,
}
impl ModifyParams {
pub(super) fn new(args: &Bound<'_, PyTuple>, kwargs: &Bound<'_, PyDict>) -> PyResult<Self> {
let model = match args.get_item(0) {
Ok(model) => model,
Err(_) => kwargs
.get_item("model")?
.unwrap_or_else(|| args.py().None().into_bound(args.py())),
};
Ok(Self {
call: Some((model.unbind(), args.clone().unbind())),
})
}
}
impl CallHooks<PythonRuntime> for ModifyParams {
fn arguments_prepared(&mut self, py: Python<'_>, arguments: &Py<PyDict>) -> PyResult<()> {
let Some((model, args)) = &self.call else {
return Ok(());
};
let clamped = py
.import(ROUTE_HOST_MODULE)
.and_then(|module| module.getattr("modify_max_tokens"))
.and_then(|clamp| clamp.call1((model, args, arguments)));
match clamped {
Err(error) if !error.is_instance_of::<PyException>(py) => Err(error),
_ => Ok(()),
}
}
}
impl PythonOwned for ModifyParams {
fn close(&mut self, _: Python<'_>) {
self.call = None;
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
if let Some((model, args)) = &self.call {
visit.call(model)?;
visit.call(args)?;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use pyo3::{
prelude::*,
types::{PyDict, PyTuple},
};
use rstest::rstest;
use super::{ModifyParams, ROUTE_HOST_MODULE};
use litellm_host::hooks::CallHooks;
#[rstest]
#[case::keyword_model(
"()",
"{'model': 'anthropic/claude-sonnet-4-5'}",
"'anthropic/claude-sonnet-4-5'"
)]
#[case::first_positional_wins("(64, [])", "{'model': 'anthropic/claude-sonnet-4-5'}", "64")]
#[case::no_model("()", "{}", "None")]
fn reads_the_model_the_client_wrapper_reads(
#[case] args: &str,
#[case] kwargs: &str,
#[case] expected: &str,
) {
Python::initialize();
Python::attach(|py| {
let eval = |source: &str| {
py.eval(&std::ffi::CString::new(source).unwrap(), None, None)
.unwrap()
};
let args = eval(args).cast_into::<PyTuple>().unwrap();
let kwargs = eval(kwargs).cast_into::<PyDict>().unwrap();
let hook = ModifyParams::new(&args, &kwargs).unwrap();
let (model, captured) = hook.call.as_ref().unwrap();
assert!(model.bind(py).eq(eval(expected)).unwrap());
assert!(captured.bind(py).is(&args));
});
}
#[rstest]
#[case::exception_leaves_max_tokens("ValueError", true)]
#[case::base_exception_propagates("KeyboardInterrupt", false)]
fn a_failing_clamp_behaves_like_the_client_wrappers_swallowed_try(
#[case] raised: &str,
#[case] swallowed: bool,
) {
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
locals.set_item("name", ROUTE_HOST_MODULE).unwrap();
locals.set_item("raised", raised).unwrap();
py.run(
c"
import builtins, sys, types
for parent in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.host', 'litellm.rust_bridge.host.routes'):
sys.modules.setdefault(parent, types.ModuleType(parent))
saved = sys.modules.get(name)
stub = types.ModuleType(name)
def modify_max_tokens(model, args, kwargs):
kwargs['max_tokens'] = 1
raise getattr(builtins, raised)()
stub.modify_max_tokens = modify_max_tokens
sys.modules[name] = stub
",
Some(&locals),
Some(&locals),
)
.unwrap();
let args = PyTuple::empty(py);
let kwargs = PyDict::new(py);
kwargs
.set_item("model", "anthropic/claude-sonnet-4-5")
.unwrap();
let mut hook = ModifyParams::new(&args, &kwargs).unwrap();
let result = hook.arguments_prepared(py, &kwargs.clone().unbind());
py.run(
c"
if saved is None:
sys.modules.pop(name, None)
else:
sys.modules[name] = saved
",
Some(&locals),
Some(&locals),
)
.unwrap();
assert_eq!(result.is_ok(), swallowed);
if let Err(error) = result {
assert!(error.is_instance_of::<pyo3::exceptions::PyKeyboardInterrupt>(py));
}
});
}
}

View file

@ -166,6 +166,38 @@ fn run_public_call<H, M>(
hooks: impl PythonCallHooks + 'static,
asynchronous: bool,
) -> PyResult<Py<PyAny>>
where
H: PythonBinding + PythonHostCalls<H::Protocol> + 'static,
M: Machine<Protocol = H::Protocol> + 'static,
M::Complete: Into<HostedCompletion<<H::Protocol as Protocol>::Response>>,
{
run_public_call_after_policy(
py,
arguments,
start,
host,
hooks,
HookChain::new(),
asynchronous,
)
}
fn run_public_call_after_policy<H, M>(
py: Python<'_>,
arguments: Py<PyDict>,
start: impl FnOnce(
Python<'_>,
&Bound<'_, PyDict>,
<H::Protocol as Protocol>::Request,
) -> PyResult<M>
+ Send
+ Sync
+ 'static,
host: H,
hooks: impl PythonCallHooks + 'static,
after_policy: impl PythonCallHooks + 'static,
asynchronous: bool,
) -> PyResult<Py<PyAny>>
where
H: PythonBinding + PythonHostCalls<H::Protocol> + 'static,
M: Machine<Protocol = H::Protocol> + 'static,
@ -179,7 +211,8 @@ where
host,
HookChain::new()
.with(hooks)
.with(crate::preflight::SdkPolicy),
.with(crate::preflight::SdkPolicy)
.with(after_policy),
arguments,
CallOptions::new(asynchronous, execution_steps),
)

View file

@ -139,7 +139,6 @@ _FEATURES: Final[detectors.Detectors] = MappingProxyType(
{
**{feature: detect for feature, detect in detectors.COMMON.items() if feature not in _NATIVE_COMMON_FEATURES},
Feature.SAFEGUARDS_CLASSIFIER: route_detectors.safeguards_classifier,
Feature.MODIFY_PARAMS: detectors.modify_params,
Feature.WORKLOAD_IDENTITY: route_detectors.anthropic_workload_identity,
}
)

View file

@ -169,7 +169,6 @@ MESSAGES_PYTHON_FEATURES: Final = (COMMON_PYTHON_FEATURES - {Feature.TIMEOUT_VAR
Feature.PRE_REQUEST_HOOK,
Feature.WEBSEARCH_INTERCEPTION,
Feature.MESSAGES_INTERCEPTOR,
Feature.MODIFY_PARAMS,
Feature.WORKLOAD_IDENTITY,
Feature.STRICT_TOOLS_RETRY,
}

View file

@ -53,7 +53,8 @@ from litellm.rust_bridge.host import callbacks_legacy_python, http_transport
from litellm.rust_bridge.public_call import NativeCall, known_params, optional_str
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ProviderSpecificHeader
from litellm.types.utils import CallTypes, ProviderSpecificHeader
from litellm.utils import clamp_max_tokens
_METADATA: Final = TypeAdapter(dict[str, object])
_BODY_FIELDS: Final = frozenset(
@ -114,6 +115,14 @@ def inferred_provider(model: str, api_base: str | None) -> str | None:
return provider
def modify_max_tokens(
model: object,
args: tuple[object, ...],
kwargs: dict[str, object], # mutable-ok: the call's keyword view, rewritten in place as `@client` does
) -> None:
clamp_max_tokens(model, args, kwargs, CallTypes.anthropic_messages.value)
def _body_fields(request: Mapping[str, object]) -> Mapping[str, object]:
params: Final = known_params(_BODY_FIELDS.difference(handler.additional_drop_paths(request)), request)
if deployment_reasoning_effort_levels(request.get("model_info")) is None:

View file

@ -2066,6 +2066,39 @@ def _is_litellm_router_call(kwargs: Mapping[str, object]) -> bool:
return any(isinstance(bucket, Mapping) and "model_group" in bucket for bucket in metadata_buckets)
def clamp_max_tokens(
model: object,
args: tuple[object, ...],
kwargs: dict[str, object], # mutable-ok: the @client wrapper rewrites the call's own kwargs
call_type: str,
) -> None:
if (
kwargs.get("max_tokens", None) is not None
and model is not None
and litellm.modify_params is True
and call_type in (CallTypes.acompletion.value, CallTypes.completion.value, CallTypes.anthropic_messages.value)
):
try:
base_model = model
if kwargs.get("hf_model_name", None) is not None:
base_model = f"huggingface/{kwargs.get('hf_model_name')}"
messages = None
if len(args) > 1:
messages = args[1]
elif kwargs.get("messages", None):
messages = kwargs["messages"]
kwargs["max_tokens"] = _get_modified_max_tokens()(
model=model,
base_model=base_model,
messages=messages,
user_max_tokens=kwargs.get("max_tokens"),
buffer_num=None,
buffer_perc=None,
)
except Exception as e:
print_verbose(f"Error while checking max token limit: {e}")
def client(original_function):
from litellm.litellm_core_utils.core_helpers import max_retries_per_request_hit
@ -2182,38 +2215,7 @@ def client(original_function):
verbose_logger.debug("Cache hit!")
return caching_handler_response.cached_result
# CHECK MAX TOKENS
if (
kwargs.get("max_tokens", None) is not None
and model is not None
and litellm.modify_params is True # user is okay with params being modified
and (
call_type == CallTypes.acompletion.value
or call_type == CallTypes.completion.value
or call_type == CallTypes.anthropic_messages.value
)
):
try:
base_model = model
if kwargs.get("hf_model_name", None) is not None:
base_model = f"huggingface/{kwargs.get('hf_model_name')}"
messages = None
if len(args) > 1:
messages = args[1]
elif kwargs.get("messages", None):
messages = kwargs["messages"]
user_max_tokens: Final = kwargs.get("max_tokens")
modified_max_tokens: Final = _get_modified_max_tokens()(
model=model,
base_model=base_model,
messages=messages,
user_max_tokens=user_max_tokens,
buffer_num=None,
buffer_perc=None,
)
kwargs["max_tokens"] = modified_max_tokens
except Exception as e:
print_verbose(f"Error while checking max token limit: {e}")
clamp_max_tokens(model, args, kwargs, call_type)
# MODEL CALL
result = original_function(*args, **kwargs)
end_time = datetime.datetime.now()
@ -2478,38 +2480,7 @@ def client(original_function):
if _llm_caching_handler.preset_cache_key is not None:
logging_obj.litellm_params["preset_cache_key"] = _llm_caching_handler.preset_cache_key
# CHECK MAX TOKENS
if (
kwargs.get("max_tokens", None) is not None
and model is not None
and litellm.modify_params is True # user is okay with params being modified
and (
call_type == CallTypes.acompletion.value
or call_type == CallTypes.completion.value
or call_type == CallTypes.anthropic_messages.value
)
):
try:
base_model = model
if kwargs.get("hf_model_name", None) is not None:
base_model = f"huggingface/{kwargs.get('hf_model_name')}"
messages = None
if len(args) > 1:
messages = args[1]
elif kwargs.get("messages", None):
messages = kwargs["messages"]
user_max_tokens: Final = kwargs.get("max_tokens")
modified_max_tokens: Final = _get_modified_max_tokens()(
model=model,
base_model=base_model,
messages=messages,
user_max_tokens=user_max_tokens,
buffer_num=None,
buffer_perc=None,
)
kwargs["max_tokens"] = modified_max_tokens
except Exception as e:
print_verbose(f"Error while checking max token limit: {e}")
clamp_max_tokens(model, args, kwargs, call_type)
# MODEL CALL
call_kwargs: Final = (

View file

@ -166,6 +166,7 @@ THINKING_REPLAY: Final[tuple[JsonValue, ...]] = (
{"role": "user", "content": "And Germany?"},
)
_MODIFY_PARAMS: Final[Mapping[str, object]] = MappingProxyType({"modify_params": True})
NO_INPUT: Final[Mapping[str, JsonValue]] = MappingProxyType({})
PNG_PIXEL: Final = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
@ -329,7 +330,7 @@ class Case:
gaps: Mapping[str, str | None] = field(default_factory=lambda: MappingProxyType({}))
def params(self) -> Iterator[object]:
for aspect in ASPECTS:
for aspect in (*ASPECTS, "cached") if self.cache is not None else ASPECTS:
gap = {**KNOWN_GAPS.get(self.name, {}), **self.gaps}.get(aspect)
marks = () if not gap else (pytest.mark.xfail(reason=gap, strict=True),)
yield pytest.param(self, aspect, id=f"{self.name}-{aspect}", marks=marks)
@ -911,7 +912,58 @@ FALLBACK_CASES: Final = (
_ok(),
),
),
Case("modify_params", _kw(max_tokens=1), settings=MappingProxyType({"modify_params": True})),
Case("modify_params", _kw(max_tokens=1), settings=_MODIFY_PARAMS),
Case("modify_params_clamps_max_tokens", _kw(max_tokens=200_000), settings=_MODIFY_PARAMS),
Case(
"modify_params_clamps_streamed_max_tokens",
_kw(stream=True, max_tokens=200_000),
upstream=(_stream(TEXT_STREAM),),
settings=_MODIFY_PARAMS,
),
Case(
"modify_params_clamps_tool_turns",
_kw(messages=list(TOOL_TURNS), tools=[dict(WEATHER_TOOL)], max_tokens=200_000),
upstream=(_ok(TOOL_USE_RESPONSE),),
settings=_MODIFY_PARAMS,
),
Case(
"modify_params_clamps_thinking",
_kw(thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=200_000),
upstream=(_ok(THINKING_RESPONSE),),
settings=_MODIFY_PARAMS,
),
Case(
"modify_params_image_content",
_kw(
messages=[
{
"role": "user",
"content": [
{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG_PIXEL}},
{"type": "text", "text": "What is this?"},
],
}
],
max_tokens=200_000,
),
settings=_MODIFY_PARAMS,
),
Case(
"modify_params_unknown_model",
_kw(max_tokens=200_000),
model="anthropic/claude-parity-unknown",
settings=_MODIFY_PARAMS,
),
Case("modify_params_hf_model_name", _kw(max_tokens=200_000, hf_model_name="gpt2"), settings=_MODIFY_PARAMS),
Case(
"modify_params_cache_write_without_read",
_kw(max_tokens=200_000, cache={"no-cache": True}),
upstream=(_ok(), _ok()),
settings=_MODIFY_PARAMS,
cache=Cache,
),
Case("cache_write_without_read", _kw(cache={"no-cache": True}), upstream=(_ok(), _ok()), cache=Cache),
Case("modify_params_sync", _kw(max_tokens=200_000), sync=True, settings=_MODIFY_PARAMS),
Case("request_timeout_kwarg", _kw(request_timeout=30)),
Case("stream_timeout_kwarg", _kw(stream=True, stream_timeout=30), upstream=(_stream(TEXT_STREAM),)),
)
@ -1047,6 +1099,7 @@ class Leg:
hidden_params: Mapping[str, object]
logged: Mapping[str, object]
callback_kwargs: Mapping[str, object]
cached: tuple[str, ...] = ()
@dataclass(slots=True)
@ -1325,6 +1378,7 @@ async def _leg(case: Case, server: RecordingServer, probe: FallbackProbe | None)
hidden_params=hidden,
logged=logged,
callback_kwargs=callback_kwargs,
cached=() if litellm.cache is None else tuple(sorted(litellm.cache.cache.cache_dict)),
)
@ -1371,6 +1425,8 @@ async def test_native_messages_match_python(
assert native.hidden_params == python.hidden_params
case "logged":
assert native.logged == python.logged
case "cached":
assert native.cached == python.cached
case _:
assert native.callback_kwargs.get(aspect) == python.callback_kwargs.get(aspect)

View file

@ -17,6 +17,7 @@ from litellm.messages.dispatch import (
_DISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch
)
from litellm.proxy import proxy_server
from litellm.rust_bridge import catalog
from litellm.rust_bridge.bindings import NativeBinding
from litellm.rust_bridge.catalog import MESSAGES_PYTHON_FEATURES, RULES, Feature, Route, RouteRule, Rules, Target
from litellm.rust_bridge.configuration import Rollout
@ -477,7 +478,6 @@ STRICT_TOOL: Final = {"name": "a", "input_schema": {"type": "object"}, "strict":
(Feature.CALLER_CLIENT, "anthropic/claude-sonnet-4-5", {"client": object()}, {}),
(Feature.AGENTIC_LOOP_HOOK, "anthropic/claude-sonnet-4-5", {}, {"callbacks": [_Agentic()]}),
(Feature.TLS_POLICY, "anthropic/claude-sonnet-4-5", {}, {"ssl_security_level": "DEFAULT@SECLEVEL=1"}),
(Feature.MODIFY_PARAMS, "anthropic/claude-sonnet-4-5", {}, {"modify_params": True}),
(
Feature.WORKLOAD_IDENTITY,
"anthropic/claude-sonnet-4-5",
@ -676,6 +676,46 @@ async def test_a_configured_response_cache_stays_native(
assert served is native_result
@pytest.mark.asyncio
async def test_modify_params_stays_native(monkeypatch: pytest.MonkeyPatch) -> None:
rules: Final[Rules] = (
RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=catalog.MESSAGES_PYTHON_FEATURES),
RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED),
)
native_result: Final = response()
monkeypatch.setattr(litellm, "callbacks", [])
monkeypatch.setattr(litellm, "modify_params", True)
def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: rejected fallback
pytest.fail("modify_params must not fall back to Python")
async def apython(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub
pytest.fail("modify_params must not fall back to Python")
async def native(request: NativeCall) -> AnthropicMessagesResponse:
return native_result
call_args: Final = (200_000, MESSAGES, "anthropic/claude-sonnet-4-5")
served: Final = _DISPATCH.run(
call_args,
{},
python=python,
binding=messages_binding(lambda request: native_result),
native=lambda hook, request, args, kwargs: hook(request),
rules=rules,
)
aserved: Final = await _ADISPATCH.arun(
call_args,
{},
python=apython,
binding=amessages_binding(native),
native=lambda hook, request, args, kwargs: hook(request),
rules=rules,
)
assert (served, aserved) == (native_result, native_result)
ASYNC_ONLY_FEATURES: Final = (
pytest.param(Feature.PRE_REQUEST_HOOK, {}, {"callbacks": [_Rewrite()]}, id="pre-request-hook"),
pytest.param(

View file

@ -627,3 +627,29 @@ def test_body_params_prepare_prompt_caching_as_the_entry_that_ran(
)
assert fields["system"] == expected_system
@pytest.mark.parametrize(
("modify_params", "args", "kwargs", "expected"),
[
pytest.param(True, (), {"max_tokens": 200_000}, 64_000, id="clamped-to-model-output-limit"),
pytest.param(True, (), {"max_tokens": 1}, 1, id="under-the-limit"),
pytest.param(False, (), {"max_tokens": 200_000}, 200_000, id="modify-params-off"),
pytest.param("true", (), {"max_tokens": 200_000}, 200_000, id="truthy-but-not-true"),
pytest.param(True, (200_000,), {"max_tokens": 200_000}, 200_000, id="positional-model-is-not-a-model"),
],
)
def test_modify_max_tokens_clamps_the_keyword_view_as_the_client_wrapper_does(
monkeypatch: pytest.MonkeyPatch,
modify_params: object,
args: tuple[object, ...],
kwargs: dict[str, object],
expected: int,
) -> None:
monkeypatch.setattr(litellm, "modify_params", modify_params)
model: Final = args[0] if args else "anthropic/claude-sonnet-4-5"
view: Final = {**kwargs, "messages": [{"role": "user", "content": "hi"}]}
messages_host.modify_max_tokens(model, args, view)
assert view["max_tokens"] == expected