mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-14 09:03:16 +00:00
fix(logging): port the rest of #43667's internal-call gate
An internal sub-call under an async parent no longer schedules sync success callbacks of its own, and a cache hit inside an internal sub-call fires no callbacks, since the parent call that folds it in logs it. This matches main
This commit is contained in:
parent
c7db063887
commit
53fb0464dd
3 changed files with 68 additions and 9 deletions
|
|
@ -822,7 +822,12 @@ class LLMCachingHandler:
|
|||
start_time (datetime): The start time of the operation.
|
||||
end_time (datetime): The end time of the operation.
|
||||
cache_hit (bool): Whether it was a cache hit.
|
||||
|
||||
An internal sub-call's hit fires nothing, since the parent call that folds it in logs it.
|
||||
"""
|
||||
if is_internal_call.get():
|
||||
return
|
||||
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
|
||||
|
|
|
|||
|
|
@ -1375,15 +1375,16 @@ def _dispatch_success_logging(
|
|||
is_completion_with_fallbacks: bool,
|
||||
is_litellm_internal_call: bool,
|
||||
) -> None:
|
||||
if not is_litellm_internal_call:
|
||||
_schedule_async_success_logging(
|
||||
logging_obj=logging_obj,
|
||||
result=result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
is_completion_with_fallbacks=is_completion_with_fallbacks,
|
||||
)
|
||||
if is_litellm_internal_call:
|
||||
return
|
||||
|
||||
_schedule_async_success_logging(
|
||||
logging_obj=logging_obj,
|
||||
result=result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
is_completion_with_fallbacks=is_completion_with_fallbacks,
|
||||
)
|
||||
logging_obj.handle_sync_success_callbacks_for_async_calls(
|
||||
result=result,
|
||||
start_time=start_time,
|
||||
|
|
|
|||
|
|
@ -54,7 +54,8 @@ import json
|
|||
import httpx
|
||||
import respx
|
||||
from fastapi.testclient import TestClient
|
||||
from litellm._internal_context import current_service_target, in_post_response_phase
|
||||
from contextlib import nullcontext
|
||||
from litellm._internal_context import current_service_target, in_post_response_phase, internal_sub_call
|
||||
from litellm.caching.caching_handler import _PENDING_CACHE_WRITES
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
|
||||
|
|
@ -3088,3 +3089,55 @@ async def test_async_get_cache_answers_400_for_a_single_object_embedding_input()
|
|||
call_type=CallTypes.aembedding.value,
|
||||
kwargs={"model": model, "custom_llm_provider": "gemini", "input": clip_block, "caching": True},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("internal", [False, True])
|
||||
async def test_a_cache_hit_inside_an_internal_sub_call_fires_no_callbacks_of_its_own(
|
||||
monkeypatch: pytest.MonkeyPatch, internal: bool
|
||||
):
|
||||
cache: Final = Cache(type="local")
|
||||
monkeypatch.setattr(litellm, "cache", cache)
|
||||
kwargs: Final = {"model": "gpt-5.4", "messages": [{"role": "user", "content": "hello"}], "caching": True}
|
||||
cache.add_cache(
|
||||
result=litellm.ModelResponse(
|
||||
model="gpt-5.4",
|
||||
choices=[litellm.Choices(message=litellm.Message(role="assistant", content="hi"))],
|
||||
usage=litellm.Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500),
|
||||
),
|
||||
**kwargs,
|
||||
)
|
||||
async_logging: Final = _build_logging_obj(CallTypes.acompletion.value, stream=False)
|
||||
async_logging.async_success_handler = AsyncMock()
|
||||
async_logging.handle_sync_success_callbacks_for_async_calls = MagicMock()
|
||||
sync_logging: Final = _build_logging_obj(CallTypes.completion.value, stream=False)
|
||||
sync_logging.handle_sync_cache_hit_success_callbacks = MagicMock()
|
||||
|
||||
with internal_sub_call() if internal else nullcontext():
|
||||
async_hit: Final = await LLMCachingHandler(
|
||||
original_function=litellm.acompletion, request_kwargs=kwargs, start_time=_FIXED_START
|
||||
).async_get_cache(
|
||||
model="gpt-5.4",
|
||||
original_function=litellm.acompletion,
|
||||
logging_obj=async_logging,
|
||||
start_time=_FIXED_START,
|
||||
call_type=CallTypes.acompletion.value,
|
||||
kwargs=kwargs,
|
||||
args=(),
|
||||
)
|
||||
sync_hit: Final = LLMCachingHandler(
|
||||
original_function=completion, request_kwargs=kwargs, start_time=_FIXED_START
|
||||
).sync_get_cache(
|
||||
model="gpt-5.4",
|
||||
original_function=completion,
|
||||
logging_obj=sync_logging,
|
||||
start_time=_FIXED_START,
|
||||
call_type=CallTypes.completion.value,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
assert async_hit is not None and async_hit.cached_result is not None
|
||||
assert sync_hit.cached_result is not None
|
||||
assert async_logging.async_success_handler.called is not internal
|
||||
assert async_logging.handle_sync_success_callbacks_for_async_calls.called is not internal
|
||||
assert sync_logging.handle_sync_cache_hit_success_callbacks.called is not internal
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue