mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
fix(hosted_vllm): isolate forwarded reasoning in response caches
This commit is contained in:
jibanez-staticduo
parent
30bee8adc9
commit
8a82ea72ac
4 changed files with 164 additions and 39 deletions
|
|
@ -366,6 +366,13 @@ class Cache:
|
|||
param_value = kwargs[param]
|
||||
cache_key += f"{param}: {param_value}"
|
||||
|
||||
nested_litellm_params: Final = kwargs.get("litellm_params") or {}
|
||||
forward_reasoning_content: Final = kwargs.get(
|
||||
"forward_reasoning_content", nested_litellm_params.get("forward_reasoning_content")
|
||||
)
|
||||
if forward_reasoning_content is True:
|
||||
cache_key += "forward_reasoning_content: True"
|
||||
|
||||
if is_semantic_cache:
|
||||
cache_key += self._get_semantic_cache_tenant_scope(kwargs)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,39 +0,0 @@
|
|||
# Forwarding assistant reasoning to hosted vLLM
|
||||
|
||||
Some vLLM backends and chat templates accept previous assistant reasoning alongside content and tool calls. Set `forward_reasoning_content: true` on an individual model entry to forward the `reasoning_content` field supplied by the client
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: reasoning-history
|
||||
litellm_params:
|
||||
model: hosted_vllm/your-served-model
|
||||
api_base: http://localhost:8000/v1
|
||||
forward_reasoning_content: true
|
||||
- model_name: default-history
|
||||
litellm_params:
|
||||
model: hosted_vllm/your-served-model
|
||||
api_base: http://localhost:8000/v1
|
||||
forward_reasoning_content: false
|
||||
```
|
||||
|
||||
The default is false. Omitting the option or setting it to false retains the existing removal of assistant `reasoning_content`. The option is local to each request, applies only to `hosted_vllm`, and is not sent to the backend. It can also be passed to `litellm.completion` or `litellm.acompletion`
|
||||
|
||||
When enabled, LiteLLM forwards the supplied field without trimming it or inserting it into visible `content`. Content and tool calls keep their existing transformations. It does not reconstruct missing reasoning or convert Anthropic `thinking_blocks`, signatures, or redacted thinking. Their existing handling is unchanged
|
||||
|
||||
For Responses requests routed through Chat Completions, also set `use_chat_completions_api: true`. The option preserves the `reasoning_content` produced by that bridge from supported Responses reasoning input. It does not change native vLLM Responses requests or make opaque encrypted reasoning portable
|
||||
|
||||
## Backend compatibility
|
||||
|
||||
Enable this only after checking both your vLLM request parser and model chat template. A backend returning reasoning in its responses does not necessarily accept reasoning in previous assistant messages
|
||||
|
||||
The vLLM v0.12.0 and v0.13.0 chat parsers accept `reasoning` and `reasoning_content` and expose both names to the template. Newer vLLM versions may normalize the deprecated input name `reasoning_content` to `reasoning` before rendering. LiteLLM forwards its existing `reasoning_content` field, without adding a duplicate `reasoning` field or selecting behavior by model name. Older releases, vendor forks and custom templates need separate verification
|
||||
|
||||
Sources: [vLLM v0.12.0 chat parser](https://github.com/vllm-project/vllm/blob/v0.12.0/vllm/entrypoints/chat_utils.py#L1532), [vLLM v0.13.0 chat parser](https://github.com/vllm-project/vllm/blob/v0.13.0/vllm/entrypoints/chat_utils.py)
|
||||
|
||||
## Template policy and performance
|
||||
|
||||
`forward_reasoning_content` controls transport. A model-specific setting such as `chat_template_kwargs.preserve_thinking` controls which received history its template uses. In the Qwen3.8-Flash-Next template, `preserve_thinking: false` removes reasoning from earlier user turns but still retains reasoning within the current tool sequence. It does not mean that every assistant reasoning field should be deleted
|
||||
|
||||
See [Qwen3.8-Flash-Next preserved thinking](https://huggingface.co/Qwen/Qwen3.8-Flash-Next#disable-preserved-thinking)
|
||||
|
||||
Forwarding history can change the rendered prompt and token prefix. Boundary whitespace normalization may have no effect when the template trims that field. Validate rendered tokens with the actual tokenizer and template before drawing cache conclusions. This option provides no measured latency, cache-hit or reasoning-quality guarantee
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import asyncio
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from typing import Final
|
||||
|
|
@ -8,6 +9,8 @@ import respx
|
|||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.caching.caching import Cache
|
||||
from litellm.caching.caching_handler import _PENDING_CACHE_WRITES
|
||||
|
||||
URL: Final = "https://reasoning-test.invalid/v1/chat/completions"
|
||||
MODEL: Final = "hosted_vllm/reasoning-test"
|
||||
|
|
@ -117,3 +120,111 @@ async def test_router_aliases_isolate_reasoning_flag_on_same_backend(async_mode:
|
|||
assert messages == original_messages
|
||||
assert route.call_count == 4
|
||||
assert model_list == original_models
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"])
|
||||
@pytest.mark.parametrize("surface", ["sdk", "router", "responses"])
|
||||
async def test_local_cache_separates_forwarded_reasoning_history(
|
||||
async_mode: bool, surface: str, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "cache", Cache(type="local", namespace="reasoning-cache-test"))
|
||||
messages: Final = _messages()
|
||||
original_messages: Final = deepcopy(messages)
|
||||
input_items: Final = [
|
||||
{"role": "user", "content": "Compare both records"},
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": "rs_previous",
|
||||
"summary": [],
|
||||
"content": [{"type": "reasoning_text", "text": REASONING}],
|
||||
},
|
||||
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
|
||||
{"type": "function_call", "call_id": "call_2", "name": "lookup", "arguments": "{}"},
|
||||
{"type": "function_call_output", "call_id": "call_1", "output": "first record"},
|
||||
{"type": "function_call_output", "call_id": "call_2", "output": "second record"},
|
||||
]
|
||||
original_input: Final = deepcopy(input_items)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": alias,
|
||||
"litellm_params": {
|
||||
"model": MODEL,
|
||||
"api_base": URL.removesuffix("/chat/completions"),
|
||||
"api_key": "test-key",
|
||||
**({} if enabled is None else {"forward_reasoning_content": enabled}),
|
||||
},
|
||||
}
|
||||
for alias, enabled in (("default", None), ("disabled", False), ("enabled", True))
|
||||
],
|
||||
cache_responses=True,
|
||||
caching_groups=[("default", "disabled", "enabled")],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
def backend(request: httpx.Request) -> httpx.Response:
|
||||
body: Final = json.loads(request.content)
|
||||
assert "forward_reasoning_content" not in body
|
||||
enabled: Final = body["messages"][1].get("reasoning_content") == REASONING
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "provider-cache-reasoning",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "reasoning-test",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "forwarded" if enabled else "omitted"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12},
|
||||
},
|
||||
)
|
||||
|
||||
with respx.mock(assert_all_called=True) as mock:
|
||||
route: Final = mock.post(URL).mock(side_effect=backend)
|
||||
for alias, forward, expected_calls in (
|
||||
("default", None, 1),
|
||||
("disabled", False, 1),
|
||||
("enabled", True, 2),
|
||||
("disabled", False, 2),
|
||||
("enabled", True, 2),
|
||||
):
|
||||
kwargs: Final = {
|
||||
"model": MODEL,
|
||||
"api_base": URL.removesuffix("/chat/completions"),
|
||||
"api_key": "test-key",
|
||||
"caching": True,
|
||||
**({} if forward is None else {"forward_reasoning_content": forward}),
|
||||
}
|
||||
if surface == "router":
|
||||
response = (
|
||||
await router.acompletion(model=alias, messages=messages)
|
||||
if async_mode
|
||||
else router.completion(model=alias, messages=messages)
|
||||
)
|
||||
elif surface == "responses":
|
||||
response = (
|
||||
await litellm.aresponses(input=input_items, use_chat_completions_api=True, **kwargs)
|
||||
if async_mode
|
||||
else litellm.responses(input=input_items, use_chat_completions_api=True, **kwargs)
|
||||
)
|
||||
else:
|
||||
response = (
|
||||
await litellm.acompletion(messages=messages, **kwargs)
|
||||
if async_mode
|
||||
else litellm.completion(messages=messages, **kwargs)
|
||||
)
|
||||
await asyncio.gather(*tuple(_PENDING_CACHE_WRITES))
|
||||
content: Final = (
|
||||
response.output[0].content[0].text if surface == "responses" else response.choices[0].message.content
|
||||
)
|
||||
assert content == ("forwarded" if forward is True else "omitted")
|
||||
assert route.call_count == expected_calls
|
||||
assert messages == original_messages
|
||||
assert input_items == original_input
|
||||
|
|
|
|||
|
|
@ -90,6 +90,52 @@ def _semantic_cache(**cache_kwargs):
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("semantic", [False, True])
|
||||
def test_reasoning_forwarding_cache_scope_preserves_groups_namespace_and_tenant(semantic):
|
||||
cache = _semantic_cache(namespace="reasoning-test") if semantic else Cache(type="local", namespace="reasoning-test")
|
||||
|
||||
def key(alias="first", tenant="tenant-a", namespace="reasoning-test", nested=False, forward=None, prompt="hi"):
|
||||
return cache.get_cache_key(
|
||||
model="hosted_vllm/reasoning-test",
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
metadata={"model_group": alias, "caching_groups": [("first", "second")], "user_api_key": tenant},
|
||||
cache={"namespace": namespace},
|
||||
**(
|
||||
{"litellm_params": {"forward_reasoning_content": forward}}
|
||||
if nested
|
||||
else {"forward_reasoning_content": forward}
|
||||
),
|
||||
)
|
||||
|
||||
default = key()
|
||||
enabled = key(forward=True)
|
||||
assert default == key(forward=False) == key(nested=True, forward=False)
|
||||
assert enabled != default
|
||||
assert enabled == key(nested=True, forward=True) == key(alias="second", forward=True)
|
||||
assert enabled.startswith("reasoning-test:")
|
||||
assert enabled != key(namespace="other-namespace", forward=True)
|
||||
if semantic:
|
||||
assert enabled != key(tenant="tenant-b", forward=True)
|
||||
assert enabled == key(prompt="hello", forward=True)
|
||||
else:
|
||||
assert enabled != key(prompt="hello", forward=True)
|
||||
|
||||
|
||||
def test_reasoning_forwarding_cache_key_preserves_legacy_disabled_key_and_top_level_precedence():
|
||||
cache = Cache(type="local", namespace="reasoning-cache-test")
|
||||
request = {"model": "hosted_vllm/reasoning-test", "messages": [{"role": "user", "content": "hi"}]}
|
||||
legacy = "reasoning-cache-test:fca1120c8360f4b9ca0cd9b52f981f290a6eec25a8c6256033a81edcc713618c"
|
||||
assert cache.get_cache_key(**request) == legacy
|
||||
assert cache.get_cache_key(**request, forward_reasoning_content=False) == legacy
|
||||
assert (
|
||||
cache.get_cache_key(
|
||||
**request, forward_reasoning_content=False, litellm_params={"forward_reasoning_content": True}
|
||||
)
|
||||
== legacy
|
||||
)
|
||||
assert cache.get_cache_key(**request, litellm_params={"forward_reasoning_content": True}) != legacy
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cache_type",
|
||||
[LiteLLMCacheType.REDIS_SEMANTIC, LiteLLMCacheType.VALKEY_SEMANTIC],
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue