scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 109

[Performance] Performance improvements for /v1/messages route (#11421)

Browse source 
* fix: perf anthropic /v1/messages

* fix: perf anthropic /v1/messages

* fix: linting checks

* fix: linting checks
This commit is contained in:
Ishaan Jaff 2025-06-04 18:47:53 -07:00
parent cde397a42f
commit bec0f2581b
2 changed files with 13 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py
View file

@ -209,10 +209,6 @@ class AnthropicPassthroughLoggingHandler:
if transformed_openai_chunk is not None:
all_openai_chunks.append(transformed_openai_chunk)
verbose_proxy_logger.debug(
"all openai chunks= %s",
json.dumps(all_openai_chunks, indent=4, default=str),
)
except (StopIteration, StopAsyncIteration):
break
complete_streaming_response = litellm.stream_chunk_builder(

23
litellm/proxy/pass_through_endpoints/streaming_handler.py
View file

@ -1,5 +1,4 @@
import asyncio
import threading
from datetime import datetime
from typing import List, Optional
@ -7,6 +6,7 @@ import httpx
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.proxy._types import PassThroughEndpointLoggingResultValues
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
from litellm.types.utils import StandardPassThroughResponseObject
@ -122,15 +122,7 @@ class PassThroughStreamingHandler:
standard_logging_response_object = StandardPassThroughResponseObject(
response=f"cannot parse chunks to standard response object. Chunks={all_chunks}"
)
threading.Thread(
target=litellm_logging_obj.success_handler,
args=(
standard_logging_response_object,
start_time,
end_time,
False,
),
).start()
await litellm_logging_obj.async_success_handler(
result=standard_logging_response_object,
start_time=start_time,
@ -138,6 +130,17 @@ class PassThroughStreamingHandler:
cache_hit=False,
**kwargs,
)
if litellm_logging_obj._should_run_sync_callbacks_for_async_calls() is False:
return
executor.submit(
litellm_logging_obj.success_handler,
result=standard_logging_response_object,
end_time=end_time,
cache_hit=False,
start_time=start_time,
**kwargs,
)
@staticmethod
def _convert_raw_bytes_to_str_lines(raw_bytes: List[bytes]) -> List[str]: