From 94539aa5e6dcbe72c55cb3e63075b09fcef8eaa5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:29:09 -0800 Subject: [PATCH] fix(caching): keep the routed model group on adapter cache hits A generateContent adapter request through the proxy shares one logging object that is created before the router adds model_group and the deployment to litellm_metadata. On a cache hit the inner acompletion never refreshes it, so the spend row and callbacks logged an empty model group and a model with no provider prefix. Carry the request's litellm_metadata into the cache-hit environment update. --- litellm/caching/caching_handler.py | 1 + .../google_genai/test_google_genai_handler.py | 53 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 424c3497017..38bf044d396 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -1387,6 +1387,7 @@ class LLMCachingHandler: "proxy_server_request": kwargs.get("proxy_server_request", None), "stream_response": kwargs.get("stream_response", {}), "custom_llm_provider": custom_llm_provider, + **({"litellm_metadata": kwargs["litellm_metadata"]} if kwargs.get("litellm_metadata") is not None else {}), } if litellm.cache is not None: diff --git a/tests/unit/google_genai/test_google_genai_handler.py b/tests/unit/google_genai/test_google_genai_handler.py index e2da6662693..b518cfffa68 100644 --- a/tests/unit/google_genai/test_google_genai_handler.py +++ b/tests/unit/google_genai/test_google_genai_handler.py @@ -4,6 +4,7 @@ Test to verify the Google GenAI generate_content handler functionality """ import asyncio +import datetime import json import logging import threading @@ -415,6 +416,16 @@ class _SuccessPayloads(CustomLogger): self.model_groups.append(kwargs["standard_logging_object"]["model_group"]) +class _SuccessRows(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.rows: list[tuple[object, str, str]] = [] + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + payload: Final = kwargs["standard_logging_object"] + self.rows.append((payload["cache_hit"], payload["model_group"], payload["model"])) + + @pytest.fixture def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) @@ -740,6 +751,48 @@ async def test_async_generate_content_adapter_cache_hit_reaches_callbacks_as_a_c assert _litellm_error_messages(_litellm_errors) == [] +@pytest.mark.asyncio +async def test_generate_content_adapter_cache_hit_on_a_proxy_logging_object_logs_the_routed_model( + _httpx_transport: None, monkeypatch: pytest.MonkeyPatch +): + from litellm.litellm_core_utils.rules import Rules + from litellm.utils import function_setup + + logged: Final = _SuccessRows() + monkeypatch.setattr(litellm, "callbacks", [logged]) + monkeypatch.setattr(litellm, "cache", litellm.Cache(type="local")) + router: Final = _adapter_router() + contents: Final = [{"role": "user", "parts": [{"text": "Hello"}]}] + + async def call(call_id: str, logged_count: int) -> None: + logging_obj, _ = function_setup( + "agenerate_content", + Rules(), + datetime.datetime.now(), + model="grok-group", + contents=contents, + litellm_call_id=call_id, + ) + await router.agenerate_content( + model="grok-group", + contents=contents, + caching=True, + litellm_call_id=call_id, + litellm_logging_obj=logging_obj, + ) + await await_until(lambda: len(logged.rows) == logged_count, message="the call's success callback ran") + + with respx.mock() as upstream: + upstream.post("http://upstream.test/v1/chat/completions").mock(return_value=_upstream_reply(stream=False)) + await call("miss", 1) + await call("hit", 2) + + miss, hit = logged.rows + assert (miss[0] is True, hit[0] is True) == (False, True) + assert miss[1] == "grok-group" + assert hit[1:] == miss[1:] + + @pytest.mark.parametrize(("entrypoint", "stream"), [("agenerate_content", False), ("agenerate_content_stream", True)]) @pytest.mark.asyncio async def test_generate_content_adapter_logs_the_whole_conversation(