From 3a7fc2a926c97daf28141e2ae6dfe0f16b56701e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:12:22 -0800 Subject: [PATCH] fix(logging): claim the async success log on cache hits An async cache hit inside the generateContent adapter queued its own success handler without claiming the async log, so the outer agenerate_content wrapper queued a second one with cache_hit unset. When an async logging hook was slow, the second handler reset cache_hit to None before callbacks read it and logged "httpx_response is None". The cache hit now claims the async log, the same way the wrapper does. (cherry picked from commit 62db373dd5d0fa9f6c5159c449341831ba5bcf1c) --- litellm/caching/caching_handler.py | 15 +++--- .../google_genai/test_google_genai_handler.py | 46 ++++++++++++++++++- 2 files changed, 52 insertions(+), 9 deletions(-) diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 74c45189622..424c3497017 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -823,14 +823,15 @@ class LLMCachingHandler: """ from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER - GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( - async_coroutine=logging_obj.async_success_handler( - result=cached_result, - start_time=start_time, - end_time=end_time, - cache_hit=cache_hit, + if logging_obj.claim_async_success_log(): + GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( + async_coroutine=logging_obj.async_success_handler( + result=cached_result, + start_time=start_time, + end_time=end_time, + cache_hit=cache_hit, + ) ) - ) logging_obj.handle_sync_success_callbacks_for_async_calls( result=cached_result, diff --git a/tests/unit/google_genai/test_google_genai_handler.py b/tests/unit/google_genai/test_google_genai_handler.py index 7c3319b283e..e2da6662693 100644 --- a/tests/unit/google_genai/test_google_genai_handler.py +++ b/tests/unit/google_genai/test_google_genai_handler.py @@ -393,6 +393,19 @@ class _CacheHitFlags(CustomLogger): self.cache_hits.append(kwargs.get("cache_hit")) +class _AsyncCacheHitFlags(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.cache_hits: list[object] = [] + + async def async_logging_hook(self, kwargs, result, call_type): + await asyncio.sleep(0.3) + return kwargs, result + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.cache_hits.append(kwargs.get("cache_hit")) + + class _SuccessPayloads(CustomLogger): def __init__(self) -> None: super().__init__() @@ -672,7 +685,7 @@ def test_sync_generate_content_adapter_success_is_logged_once_on_the_request( def test_sync_generate_content_adapter_cache_hit_reaches_callbacks_as_a_cache_hit( _httpx_transport: None, monkeypatch: pytest.MonkeyPatch ): - logged = _CacheHitFlags() + logged: Final = _CacheHitFlags() monkeypatch.setattr(litellm, "callbacks", [logged]) monkeypatch.setattr(litellm, "cache", litellm.Cache(type="local")) router: Final = _adapter_router() @@ -687,7 +700,7 @@ def test_sync_generate_content_adapter_cache_hit_reaches_callbacks_as_a_cache_hi wait_until(lambda: len(logged.cache_hits) == logged_count, message="the call's success callback ran") with respx.mock() as upstream: - route = upstream.post("http://upstream.test/v1/chat/completions").mock( + route: Final = upstream.post("http://upstream.test/v1/chat/completions").mock( return_value=_upstream_reply(stream=False) ) for index, prompt in enumerate(prompts): @@ -698,6 +711,35 @@ def test_sync_generate_content_adapter_cache_hit_reaches_callbacks_as_a_cache_hi assert [hit is True for hit in logged.cache_hits] == [False, True] * len(prompts) +@pytest.mark.asyncio +async def test_async_generate_content_adapter_cache_hit_reaches_callbacks_as_a_cache_hit( + _httpx_transport: None, monkeypatch: pytest.MonkeyPatch, _litellm_errors: pytest.LogCaptureFixture +): + logged: Final = _AsyncCacheHitFlags() + monkeypatch.setattr(litellm, "callbacks", [logged]) + monkeypatch.setattr(litellm, "cache", litellm.Cache(type="local")) + router: Final = _adapter_router() + prompts: Final = ("first", "second", "third") + + async def call(prompt: str, logged_count: int) -> None: + await router.agenerate_content( + model="grok-group", contents=[{"role": "user", "parts": [{"text": prompt}]}], caching=True + ) + await await_until(lambda: len(logged.cache_hits) == logged_count, message="the call's success callback ran") + + with respx.mock() as upstream: + route: Final = upstream.post("http://upstream.test/v1/chat/completions").mock( + return_value=_upstream_reply(stream=False) + ) + for index, prompt in enumerate(prompts): + await call(prompt, 2 * index + 1) + await call(prompt, 2 * index + 2) + + assert route.call_count == len(prompts) + assert [hit is True for hit in logged.cache_hits] == [False, True] * len(prompts) + assert _litellm_error_messages(_litellm_errors) == [] + + @pytest.mark.parametrize(("entrypoint", "stream"), [("agenerate_content", False), ("agenerate_content_stream", True)]) @pytest.mark.asyncio async def test_generate_content_adapter_logs_the_whole_conversation(