scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-05 02:41:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 48

perf: optimize wrapper_async hot path with CallTypes caching and reduced lookups

Browse source 
- Pre-compute CallTypes enum values as module-level frozenset and dict map,
  replacing per-request list comprehension (133µs → 0.4µs/call)
- Guard debug f-string with _is_debugging_on() to skip evaluation when off
- Cache update_response_metadata getattr lookup once per call instead of twice
This commit is contained in:
Ryan Crabbe 2026-01-31 10:18:12 -08:00 • committed by Alexsander Hamir
parent 534fa9f4c0
commit cbc366f0d7
1 changed files with 13 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
litellm/utils.py
View file

@ -193,6 +193,9 @@ from litellm.types.utils import (
all_litellm_params,
)
_CALL_TYPE_VALUES: frozenset = frozenset(ct.value for ct in CallTypes)
_CALL_TYPE_ENUM_MAP: dict = {ct.value: ct for ct in CallTypes}
# +-----------------------------------------------+
# | |
# | Give Feedback / Get Help |
@ -1686,6 +1689,7 @@ def client(original_function): # noqa: PLR0915
print_args_passed_to_litellm(original_function, args, kwargs)
start_time = datetime.datetime.now()
result = None
_update_response_metadata = getattr(sys.modules[__name__], "update_response_metadata")
logging_obj: Optional[LiteLLMLoggingObject] = kwargs.get(
"litellm_logging_obj", None
)
@ -1726,9 +1730,10 @@ def client(original_function): # noqa: PLR0915
)
# [OPTIONAL] CHECK CACHE
print_verbose(
f"ASYNC kwargs[caching]: {kwargs.get('caching', False)}; litellm.cache: {litellm.cache}; kwargs.get('cache'): {kwargs.get('cache', None)}"
)
if _is_debugging_on():
print_verbose(
f"ASYNC kwargs[caching]: {kwargs.get('caching', False)}; litellm.cache: {litellm.cache}; kwargs.get('cache'): {kwargs.get('cache', None)}"
)
_caching_handler_response: "Optional[CachingHandlerResponse]" = (
await _llm_caching_handler._async_get_cache(
model=model or "",
@ -1804,8 +1809,7 @@ def client(original_function): # noqa: PLR0915
chunks, messages=kwargs.get("messages", None)
)
else:
update_response_metadata = getattr(sys.modules[__name__], 'update_response_metadata')
update_response_metadata(
_update_response_metadata(
result=result,
logging_obj=logging_obj,
model=model,
@ -1825,11 +1829,12 @@ def client(original_function): # noqa: PLR0915
rules_obj=rules_obj,
)
# Only run if call_type is a valid value in CallTypes
if call_type in [ct.value for ct in CallTypes]:
_call_type_enum = _CALL_TYPE_ENUM_MAP.get(call_type)
if _call_type_enum is not None:
result = await async_post_call_success_deployment_hook(
request_data=kwargs,
response=result,
call_type=CallTypes(call_type),
call_type=_call_type_enum,
)
## Add response to cache
@ -1869,8 +1874,7 @@ def client(original_function): # noqa: PLR0915
end_time=end_time,
)
update_response_metadata = getattr(sys.modules[__name__], 'update_response_metadata')
update_response_metadata(
_update_response_metadata(
result=result,
logging_obj=logging_obj,
model=model,