fix(caching): keep the routed model group on adapter cache hits

A generateContent adapter request through the proxy shares one logging
object that is created before the router adds model_group and the
deployment to litellm_metadata. On a cache hit the inner acompletion
never refreshes it, so the spend row and callbacks logged an empty model
group and a model with no provider prefix. Carry the request's
litellm_metadata into the cache-hit environment update.
This commit is contained in:
mateo-berri 2026-11-07 02:29:09 -08:00
parent 3a7fc2a926
commit 94539aa5e6
2 changed files with 54 additions and 0 deletions

View file

@ -1387,6 +1387,7 @@ class LLMCachingHandler:
"proxy_server_request": kwargs.get("proxy_server_request", None),
"stream_response": kwargs.get("stream_response", {}),
"custom_llm_provider": custom_llm_provider,
**({"litellm_metadata": kwargs["litellm_metadata"]} if kwargs.get("litellm_metadata") is not None else {}),
}
if litellm.cache is not None:

View file

@ -4,6 +4,7 @@ Test to verify the Google GenAI generate_content handler functionality
"""
import asyncio
import datetime
import json
import logging
import threading
@ -415,6 +416,16 @@ class _SuccessPayloads(CustomLogger):
self.model_groups.append(kwargs["standard_logging_object"]["model_group"])
class _SuccessRows(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.rows: list[tuple[object, str, str]] = []
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
payload: Final = kwargs["standard_logging_object"]
self.rows.append((payload["cache_hit"], payload["model_group"], payload["model"]))
@pytest.fixture
def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
@ -740,6 +751,48 @@ async def test_async_generate_content_adapter_cache_hit_reaches_callbacks_as_a_c
assert _litellm_error_messages(_litellm_errors) == []
@pytest.mark.asyncio
async def test_generate_content_adapter_cache_hit_on_a_proxy_logging_object_logs_the_routed_model(
_httpx_transport: None, monkeypatch: pytest.MonkeyPatch
):
from litellm.litellm_core_utils.rules import Rules
from litellm.utils import function_setup
logged: Final = _SuccessRows()
monkeypatch.setattr(litellm, "callbacks", [logged])
monkeypatch.setattr(litellm, "cache", litellm.Cache(type="local"))
router: Final = _adapter_router()
contents: Final = [{"role": "user", "parts": [{"text": "Hello"}]}]
async def call(call_id: str, logged_count: int) -> None:
logging_obj, _ = function_setup(
"agenerate_content",
Rules(),
datetime.datetime.now(),
model="grok-group",
contents=contents,
litellm_call_id=call_id,
)
await router.agenerate_content(
model="grok-group",
contents=contents,
caching=True,
litellm_call_id=call_id,
litellm_logging_obj=logging_obj,
)
await await_until(lambda: len(logged.rows) == logged_count, message="the call's success callback ran")
with respx.mock() as upstream:
upstream.post("http://upstream.test/v1/chat/completions").mock(return_value=_upstream_reply(stream=False))
await call("miss", 1)
await call("hit", 2)
miss, hit = logged.rows
assert (miss[0] is True, hit[0] is True) == (False, True)
assert miss[1] == "grok-group"
assert hit[1:] == miss[1:]
@pytest.mark.parametrize(("entrypoint", "stream"), [("agenerate_content", False), ("agenerate_content_stream", True)])
@pytest.mark.asyncio
async def test_generate_content_adapter_logs_the_whole_conversation(