mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(logging): claim the async success log on cache hits
An async cache hit inside the generateContent adapter queued its own success handler without claiming the async log, so the outer agenerate_content wrapper queued a second one with cache_hit unset. When an async logging hook was slow, the second handler reset cache_hit to None before callbacks read it and logged "httpx_response is None". The cache hit now claims the async log, the same way the wrapper does.
(cherry picked from commit 62db373dd5)
This commit is contained in:
parent
9598dae8c9
commit
3a7fc2a926
2 changed files with 52 additions and 9 deletions
|
|
@ -823,14 +823,15 @@ class LLMCachingHandler:
|
|||
"""
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
|
||||
async_coroutine=logging_obj.async_success_handler(
|
||||
result=cached_result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
cache_hit=cache_hit,
|
||||
if logging_obj.claim_async_success_log():
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
|
||||
async_coroutine=logging_obj.async_success_handler(
|
||||
result=cached_result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
cache_hit=cache_hit,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
logging_obj.handle_sync_success_callbacks_for_async_calls(
|
||||
result=cached_result,
|
||||
|
|
|
|||
|
|
@ -393,6 +393,19 @@ class _CacheHitFlags(CustomLogger):
|
|||
self.cache_hits.append(kwargs.get("cache_hit"))
|
||||
|
||||
|
||||
class _AsyncCacheHitFlags(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.cache_hits: list[object] = []
|
||||
|
||||
async def async_logging_hook(self, kwargs, result, call_type):
|
||||
await asyncio.sleep(0.3)
|
||||
return kwargs, result
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
self.cache_hits.append(kwargs.get("cache_hit"))
|
||||
|
||||
|
||||
class _SuccessPayloads(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
|
@ -672,7 +685,7 @@ def test_sync_generate_content_adapter_success_is_logged_once_on_the_request(
|
|||
def test_sync_generate_content_adapter_cache_hit_reaches_callbacks_as_a_cache_hit(
|
||||
_httpx_transport: None, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
logged = _CacheHitFlags()
|
||||
logged: Final = _CacheHitFlags()
|
||||
monkeypatch.setattr(litellm, "callbacks", [logged])
|
||||
monkeypatch.setattr(litellm, "cache", litellm.Cache(type="local"))
|
||||
router: Final = _adapter_router()
|
||||
|
|
@ -687,7 +700,7 @@ def test_sync_generate_content_adapter_cache_hit_reaches_callbacks_as_a_cache_hi
|
|||
wait_until(lambda: len(logged.cache_hits) == logged_count, message="the call's success callback ran")
|
||||
|
||||
with respx.mock() as upstream:
|
||||
route = upstream.post("http://upstream.test/v1/chat/completions").mock(
|
||||
route: Final = upstream.post("http://upstream.test/v1/chat/completions").mock(
|
||||
return_value=_upstream_reply(stream=False)
|
||||
)
|
||||
for index, prompt in enumerate(prompts):
|
||||
|
|
@ -698,6 +711,35 @@ def test_sync_generate_content_adapter_cache_hit_reaches_callbacks_as_a_cache_hi
|
|||
assert [hit is True for hit in logged.cache_hits] == [False, True] * len(prompts)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_generate_content_adapter_cache_hit_reaches_callbacks_as_a_cache_hit(
|
||||
_httpx_transport: None, monkeypatch: pytest.MonkeyPatch, _litellm_errors: pytest.LogCaptureFixture
|
||||
):
|
||||
logged: Final = _AsyncCacheHitFlags()
|
||||
monkeypatch.setattr(litellm, "callbacks", [logged])
|
||||
monkeypatch.setattr(litellm, "cache", litellm.Cache(type="local"))
|
||||
router: Final = _adapter_router()
|
||||
prompts: Final = ("first", "second", "third")
|
||||
|
||||
async def call(prompt: str, logged_count: int) -> None:
|
||||
await router.agenerate_content(
|
||||
model="grok-group", contents=[{"role": "user", "parts": [{"text": prompt}]}], caching=True
|
||||
)
|
||||
await await_until(lambda: len(logged.cache_hits) == logged_count, message="the call's success callback ran")
|
||||
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("http://upstream.test/v1/chat/completions").mock(
|
||||
return_value=_upstream_reply(stream=False)
|
||||
)
|
||||
for index, prompt in enumerate(prompts):
|
||||
await call(prompt, 2 * index + 1)
|
||||
await call(prompt, 2 * index + 2)
|
||||
|
||||
assert route.call_count == len(prompts)
|
||||
assert [hit is True for hit in logged.cache_hits] == [False, True] * len(prompts)
|
||||
assert _litellm_error_messages(_litellm_errors) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("entrypoint", "stream"), [("agenerate_content", False), ("agenerate_content_stream", True)])
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_content_adapter_logs_the_whole_conversation(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue