From 47f642ff6cc0e31507a565000b3031c718f5f29c Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 08:14:26 +0000 Subject: [PATCH] refactor(passthrough): build pass-through success handler in proxy layer Part of LIT-9423 --- litellm/google_genai/streaming_iterator.py | 8 +++----- .../pass_through/messages/streaming_iterator.py | 11 ++++++----- .../proxy/pass_through_endpoints/success_handler.py | 3 +++ litellm/rust_bridge/callbacks_legacy_python.py | 4 +--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/litellm/google_genai/streaming_iterator.py b/litellm/google_genai/streaming_iterator.py index 64508ec93a6..9520aa4f657 100644 --- a/litellm/google_genai/streaming_iterator.py +++ b/litellm/google_genai/streaming_iterator.py @@ -4,9 +4,6 @@ from typing import TYPE_CHECKING, Any, Final import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.proxy.pass_through_endpoints.success_handler import ( - PassThroughEndpointLogging, -) from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType if TYPE_CHECKING: @@ -16,8 +13,6 @@ if TYPE_CHECKING: else: BaseGoogleGenAIGenerateContentConfig = Any -GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ: Final = PassThroughEndpointLogging() - def _encode_google_genai_sse_event(event_lines: list[str]) -> bytes: return ("\n".join(event_lines) + "\n\n").encode("utf-8") @@ -87,6 +82,9 @@ class BaseGoogleGenAIGenerateContentStreamingIterator: from litellm.proxy.pass_through_endpoints.streaming_handler import ( PassThroughStreamingHandler, ) + from litellm.proxy.pass_through_endpoints.success_handler import ( + GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ, + ) end_time: Final = datetime.now() asyncio.create_task( diff --git a/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py b/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py index f37e5190850..547dd098b55 100644 --- a/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py +++ b/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py @@ -17,15 +17,10 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.anthropic.common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP from litellm.llms.anthropic.pass_through.messages.utils import INCOMPLETE_STREAM_ERROR_MESSAGE -from litellm.proxy.pass_through_endpoints.success_handler import ( - PassThroughEndpointLogging, -) from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType from litellm.types.utils import GenericStreamingChunk, ModelResponseStream -GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ: Final = PassThroughEndpointLogging() - _UPSTREAM_PUMP_TASKS: Final[set[asyncio.Task[None]]] = set() # mutable-ok: stdlib strong-ref set for pump tasks _DETACHED_STREAM_DRAINS: Final[set[asyncio.Task[None]]] = set() # mutable-ok: bounded strong-ref set, detached drains @@ -414,6 +409,9 @@ class BaseAnthropicMessagesStreamingIterator: from litellm.proxy.pass_through_endpoints.streaming_handler import ( PassThroughStreamingHandler, ) + from litellm.proxy.pass_through_endpoints.success_handler import ( + GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ, + ) end_time: Final = datetime.now() # Set completion_start_time so TTFT is calculated from the first @@ -452,6 +450,9 @@ class BaseAnthropicMessagesStreamingIterator: from litellm.proxy.pass_through_endpoints.streaming_handler import ( PassThroughStreamingHandler, ) + from litellm.proxy.pass_through_endpoints.success_handler import ( + GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ, + ) # Use the existing streaming handler for Anthropic return PassThroughStreamingHandler.chunk_processor( diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 9c6c3111209..9362d192f1b 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -677,3 +677,6 @@ class PassThroughEndpointLogging: logging_obj.model_call_details["response_cost"] = passthrough_logging_payload.get("cost_per_request") return kwargs + + +GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ: Final = PassThroughEndpointLogging() diff --git a/litellm/rust_bridge/callbacks_legacy_python.py b/litellm/rust_bridge/callbacks_legacy_python.py index bfd16b29bf6..151502b77ed 100644 --- a/litellm/rust_bridge/callbacks_legacy_python.py +++ b/litellm/rust_bridge/callbacks_legacy_python.py @@ -360,10 +360,8 @@ def stream_success( end: datetime.datetime, first_chunk: datetime.datetime | None, ) -> None: - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( - GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ, - ) from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler + from litellm.proxy.pass_through_endpoints.success_handler import GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType if first_chunk is not None: