scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-14 04:09:09 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 97

refactor(passthrough): build pass-through success handler in proxy layer

Browse source 
Part of LIT-9423
This commit is contained in:
nate-berri 2026-10-10 08:14:26 +00:00
parent 5282aed09b
commit 47f642ff6c
No known key found for this signature in database
4 changed files with 13 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
litellm/google_genai/streaming_iterator.py
View file

@ -4,9 +4,6 @@ from typing import TYPE_CHECKING, Any, Final
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy.pass_through_endpoints.success_handler import (
PassThroughEndpointLogging,
)
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
if TYPE_CHECKING:
@ -16,8 +13,6 @@ if TYPE_CHECKING:
else:
BaseGoogleGenAIGenerateContentConfig = Any
GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ: Final = PassThroughEndpointLogging()
def _encode_google_genai_sse_event(event_lines: list[str]) -> bytes:
return ("\n".join(event_lines) + "\n\n").encode("utf-8")
@ -87,6 +82,9 @@ class BaseGoogleGenAIGenerateContentStreamingIterator:
from litellm.proxy.pass_through_endpoints.streaming_handler import (
PassThroughStreamingHandler,
)
from litellm.proxy.pass_through_endpoints.success_handler import (
GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ,
)
end_time: Final = datetime.now()
asyncio.create_task(

11
litellm/llms/anthropic/pass_through/messages/streaming_iterator.py
View file

@ -17,15 +17,10 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.anthropic.common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP
from litellm.llms.anthropic.pass_through.messages.utils import INCOMPLETE_STREAM_ERROR_MESSAGE
from litellm.proxy.pass_through_endpoints.success_handler import (
PassThroughEndpointLogging,
)
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ: Final = PassThroughEndpointLogging()
_UPSTREAM_PUMP_TASKS: Final[set[asyncio.Task[None]]] = set() # mutable-ok: stdlib strong-ref set for pump tasks
_DETACHED_STREAM_DRAINS: Final[set[asyncio.Task[None]]] = set() # mutable-ok: bounded strong-ref set, detached drains
@ -414,6 +409,9 @@ class BaseAnthropicMessagesStreamingIterator:
from litellm.proxy.pass_through_endpoints.streaming_handler import (
PassThroughStreamingHandler,
)
from litellm.proxy.pass_through_endpoints.success_handler import (
GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ,
)
end_time: Final = datetime.now()
# Set completion_start_time so TTFT is calculated from the first
@ -452,6 +450,9 @@ class BaseAnthropicMessagesStreamingIterator:
from litellm.proxy.pass_through_endpoints.streaming_handler import (
PassThroughStreamingHandler,
)
from litellm.proxy.pass_through_endpoints.success_handler import (
GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ,
)
# Use the existing streaming handler for Anthropic
return PassThroughStreamingHandler.chunk_processor(

3
litellm/proxy/pass_through_endpoints/success_handler.py
View file

@ -677,3 +677,6 @@ class PassThroughEndpointLogging:
logging_obj.model_call_details["response_cost"] = passthrough_logging_payload.get("cost_per_request")
return kwargs
GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ: Final = PassThroughEndpointLogging()

4
litellm/rust_bridge/callbacks_legacy_python.py
View file

@ -360,10 +360,8 @@ def stream_success(
end: datetime.datetime,
first_chunk: datetime.datetime | None,
) -> None:
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ,
)
from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler
from litellm.proxy.pass_through_endpoints.success_handler import GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
if first_chunk is not None: