From a545e6ab6d4a577a48acbc3c5e442b20990a28f1 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Sun, 20 Sep 2026 18:29:31 -0700 Subject: [PATCH] refactor(rust): scaffold streaming translation contracts --- .../handler.rs | 22 +++++ .../litellm_completion_transformation/mod.rs | 2 + .../streaming_iterator.rs | 97 +++++++++++++++++++ litellm-rust/crates/core/src/responses/mod.rs | 2 + .../core/src/responses/streaming_iterator.rs | 89 +++++++++++++++++ .../crates/llms/src/anthropic/chat/handler.rs | 25 +++++ .../experimental_pass_through/adapters/mod.rs | 1 + .../adapters/streaming_iterator.rs | 54 +++++++++++ .../experimental_pass_through/mod.rs | 1 + .../llms/src/base_llm/base_model_iterator.rs | 71 ++++++++++++++ .../crates/types/src/responses/mod.rs | 1 + .../crates/types/src/responses/streaming.rs | 16 +++ 12 files changed, 381 insertions(+) create mode 100644 litellm-rust/crates/core/src/responses/litellm_completion_transformation/handler.rs create mode 100644 litellm-rust/crates/core/src/responses/litellm_completion_transformation/mod.rs create mode 100644 litellm-rust/crates/core/src/responses/litellm_completion_transformation/streaming_iterator.rs create mode 100644 litellm-rust/crates/core/src/responses/streaming_iterator.rs create mode 100644 litellm-rust/crates/llms/src/anthropic/experimental_pass_through/adapters/mod.rs create mode 100644 litellm-rust/crates/llms/src/anthropic/experimental_pass_through/adapters/streaming_iterator.rs create mode 100644 litellm-rust/crates/types/src/responses/streaming.rs diff --git a/litellm-rust/crates/core/src/responses/litellm_completion_transformation/handler.rs b/litellm-rust/crates/core/src/responses/litellm_completion_transformation/handler.rs new file mode 100644 index 00000000000..4ae29054ca3 --- /dev/null +++ b/litellm-rust/crates/core/src/responses/litellm_completion_transformation/handler.rs @@ -0,0 +1,22 @@ +use futures_util::Stream; +use litellm_llms::base_llm::base_model_iterator::TransformedStream; +use litellm_types::utils::ChatCompletionChunk; + +use super::streaming_iterator::LiteLlmCompletionStreamingIterator; +use crate::responses::streaming_iterator::{ResponsesApiStreamingIterator, ResponsesStreamError}; + +pub struct LiteLlmCompletionTransformationHandler; + +impl LiteLlmCompletionTransformationHandler { + pub fn streaming_response( + _chat_stream: S, + _adapter: LiteLlmCompletionStreamingIterator, + ) -> ResponsesApiStreamingIterator> + where + S: Stream>, + { + todo!( + "Compose the transformed Chat source inside the same driver used for native Responses events" + ) + } +} diff --git a/litellm-rust/crates/core/src/responses/litellm_completion_transformation/mod.rs b/litellm-rust/crates/core/src/responses/litellm_completion_transformation/mod.rs new file mode 100644 index 00000000000..8671b136bb6 --- /dev/null +++ b/litellm-rust/crates/core/src/responses/litellm_completion_transformation/mod.rs @@ -0,0 +1,2 @@ +pub mod handler; +pub mod streaming_iterator; diff --git a/litellm-rust/crates/core/src/responses/litellm_completion_transformation/streaming_iterator.rs b/litellm-rust/crates/core/src/responses/litellm_completion_transformation/streaming_iterator.rs new file mode 100644 index 00000000000..f8700114fb3 --- /dev/null +++ b/litellm-rust/crates/core/src/responses/litellm_completion_transformation/streaming_iterator.rs @@ -0,0 +1,97 @@ +use litellm_llms::base_llm::base_model_iterator::{ + ChatCompletions, ResponsesApi, StreamTransformation, +}; +use litellm_types::{ + llms::openai::ChatCompletionToolCallChunk, + responses::streaming::{ + ResponseOutputItem, ResponsesEvent, ResponsesRequest, ResponsesResponse, + }, + utils::ChatCompletionChunk, +}; + +use crate::responses::streaming_iterator::ResponsesStreamError; + +pub struct LiteLlmCompletionStreamingIterator { + // TODO: Port per-call item/tool associations, request options, bounded accumulators, and reasoning state from Python + _state: (), +} + +impl LiteLlmCompletionStreamingIterator { + pub fn new(_model: String, _responses_api_request: ResponsesRequest) -> Self { + todo!( + "Retain bridged input/options and provider identity for response construction and ID encoding" + ) + } + + pub fn adopt_response_id_from_chunk(&mut self, _chunk: &ChatCompletionChunk) { + todo!( + "Prime the stable response ID before created/in_progress events, retaining the first chunk" + ) + } + + pub fn queue_tool_call_delta_events( + &mut self, + _tool_calls: &[ChatCompletionToolCallChunk], + ) -> Result<(), ResponsesStreamError> { + todo!( + "Associate fragmented tool indexes and call IDs; preserve custom/namespaced tools, arguments, item IDs and output indexes" + ) + } + + pub fn output_with_streamed_item_ids( + &self, + _response: &ResponsesResponse, + ) -> Box<[ResponseOutputItem]> { + todo!("Reuse streamed message, reasoning and tool item IDs in terminal output") + } + + pub fn transform_chat_completion_chunk_to_response_api_chunk( + &mut self, + _chunk: ChatCompletionChunk, + ) -> Result, ResponsesStreamError> { + todo!( + "Convert text, tools, reasoning, annotations and provider web-search fields while preserving sequence numbers" + ) + } + + pub fn return_default_initial_events(&mut self) -> Vec { + todo!( + "Emit created and in_progress once after response-ID priming; defer output items until their content is known" + ) + } + + pub fn return_default_done_events( + &mut self, + ) -> Result, ResponsesStreamError> { + todo!("Close text, content, reasoning and tool items in order before the terminal response") + } + + pub fn emit_response_completed_event( + &mut self, + ) -> Result { + todo!( + "Build the final response from accumulated chunks and late usage, retaining encoded IDs and request options" + ) + } +} + +impl StreamTransformation for LiteLlmCompletionStreamingIterator { + type Caller = ResponsesApi; + type Upstream = ChatCompletions; + type Error = ResponsesStreamError; + + fn transform_event( + &mut self, + _chunk: ChatCompletionChunk, + ) -> Result, Self::Error> { + todo!( + "Port conversion from both __next__ and __anext__: prime IDs, emit initial events, merge provider fields, reserve indexes, and drain ordered tool/reasoning events" + ) + } + + fn finish(self) -> Result, Self::Error> { + todo!( + "Require observed upstream termination, include late usage, close pending items, and preserve failure/incomplete/cancellation instead of treating EOF as success" + ) + } +} diff --git a/litellm-rust/crates/core/src/responses/mod.rs b/litellm-rust/crates/core/src/responses/mod.rs index bc0f71896e5..6c175a3f1f5 100644 --- a/litellm-rust/crates/core/src/responses/mod.rs +++ b/litellm-rust/crates/core/src/responses/mod.rs @@ -1,3 +1,5 @@ mod error; pub use error::Error; +pub mod litellm_completion_transformation; +pub mod streaming_iterator; pub mod websocket; diff --git a/litellm-rust/crates/core/src/responses/streaming_iterator.rs b/litellm-rust/crates/core/src/responses/streaming_iterator.rs new file mode 100644 index 00000000000..8bccc52a73a --- /dev/null +++ b/litellm-rust/crates/core/src/responses/streaming_iterator.rs @@ -0,0 +1,89 @@ +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +use futures_util::Stream; +use litellm_types::responses::streaming::{ResponsesEvent, ResponsesResponse}; + +pub enum ResponsesStreamError { + Upstream(String), + InvalidEvent(String), + UnexpectedEof, + BufferLimitExceeded, + Cancelled, + DeadlineExceeded, +} + +pub enum ResponsesStreamState { + Pending, + Active, + Terminal(ResponsesResponse), + Failed(ResponsesStreamError), +} + +pub struct ResponsesApiStreamingIterator { + source: S, + state: ResponsesStreamState, +} + +impl ResponsesApiStreamingIterator { + pub fn new(_source: S) -> Self { + todo!("Compose BaseResponsesAPIStreamingIterator lifecycle state with a typed event source") + } + + pub fn process_chunk( + &mut self, + _event: ResponsesEvent, + ) -> Result, ResponsesStreamError> { + todo!( + "Port base _process_chunk after provider decoding: response IDs, completed/failed/incomplete outcomes, usage, and error events" + ) + } + + pub fn get_completed_response_object(&self) -> Option<&ResponsesResponse> { + todo!("Expose the completed response retained by the shared lifecycle") + } + + pub fn record_failed_response_usage(&mut self, _response: &ResponsesResponse) { + todo!("Retain billable usage on failed and incomplete responses for host accounting") + } +} + +impl Stream for ResponsesApiStreamingIterator +where + S: Stream>, +{ + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + todo!( + "Port __anext__: read typed events, enforce deadlines and terminal validation, and coordinate delivery, cache, logging and post-stream hooks through route-machine host operations" + ) + } +} + +pub struct NativeResponsesStream { + source: S, +} + +impl NativeResponsesStream { + pub fn new(_source: S) -> Self { + todo!( + "Accept canonical Responses events from provider decoding, independent of transport framing" + ) + } +} + +impl Stream for NativeResponsesStream +where + S: Stream>, +{ + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + todo!( + "Forward native typed events with backpressure and cancellation to the shared Responses driver" + ) + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/chat/handler.rs b/litellm-rust/crates/llms/src/anthropic/chat/handler.rs index a80cfbf28bd..c9686bddf2d 100644 --- a/litellm-rust/crates/llms/src/anthropic/chat/handler.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/handler.rs @@ -14,6 +14,10 @@ use crate::{ base_llm::{base_model_iterator::StreamTransformer, chat::transformation::Error}, }; +use crate::base_llm::base_model_iterator::{ + AnthropicMessagesApi, ChatCompletions, StreamTransformation, +}; + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum AnthropicJsonChunkType { ValidJson, @@ -163,3 +167,24 @@ impl StreamTransformer for AnthropicChatCompletionsStreamTransformer { todo!() } } + +impl StreamTransformation for AnthropicChatCompletionsStreamTransformer { + type Caller = ChatCompletions; + type Upstream = AnthropicMessagesApi; + type Error = Error; + + fn transform_event( + &mut self, + _event: AnthropicMessagesStreamEvent, + ) -> Result, Error> { + todo!( + "Port ModelResponseIterator conversion with indexed tools, thinking/signatures, JSON mode, stop reasons, and late usage" + ) + } + + fn finish(self) -> Result, Error> { + todo!( + "Require message_stop, validate partial tool JSON, and preserve failed or cancelled termination" + ) + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/adapters/mod.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/adapters/mod.rs new file mode 100644 index 00000000000..0a64bfe97d9 --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/adapters/mod.rs @@ -0,0 +1 @@ +pub mod streaming_iterator; diff --git a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/adapters/streaming_iterator.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/adapters/streaming_iterator.rs new file mode 100644 index 00000000000..e22f1b36d72 --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/adapters/streaming_iterator.rs @@ -0,0 +1,54 @@ +use litellm_types::utils::ChatCompletionChunk; + +use crate::{ + anthropic::experimental_pass_through::messages::streaming_iterator::AnthropicMessagesStreamEvent, + base_llm::{ + base_model_iterator::{AnthropicMessagesApi, ChatCompletions, StreamTransformation}, + chat::transformation::Error, + }, +}; + +pub struct AnthropicStreamWrapper { + // TODO: Port block indexes, held stop/usage chunks, thinking signatures and compaction state from Python + _state: (), +} + +impl AnthropicStreamWrapper { + pub fn new(_model: String) -> Self { + todo!("Initialize one Chat-to-Messages conversion without owning upstream reads") + } + + pub fn handle_choiceless_chunk(&mut self, _chunk: &ChatCompletionChunk) -> Result<(), Error> { + todo!("Preserve late usage and provider fields even when choices are empty") + } + + pub fn merge_usage_into_held_stop_reason_chunk( + &mut self, + _chunk: &ChatCompletionChunk, + ) -> Result { + todo!( + "Merge final usage, cache usage, iterations and context management before message_delta and message_stop" + ) + } +} + +impl StreamTransformation for AnthropicStreamWrapper { + type Caller = AnthropicMessagesApi; + type Upstream = ChatCompletions; + type Error = Error; + + fn transform_event( + &mut self, + _chunk: ChatCompletionChunk, + ) -> Result, Error> { + todo!( + "Port _CombinedChunkSplitter and AnthropicStreamWrapper: ordered thinking/text/tools, block starts/stops, signatures, refusal and compaction; retain usage exactly once" + ) + } + + fn finish(self) -> Result, Error> { + todo!( + "Validate upstream termination and close held blocks/stops after late usage; reject truncated streams" + ) + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/mod.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/mod.rs index ba63992f3cb..1b0d2d351d8 100644 --- a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/mod.rs +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/mod.rs @@ -1 +1,2 @@ +pub mod adapters; pub mod messages; diff --git a/litellm-rust/crates/llms/src/base_llm/base_model_iterator.rs b/litellm-rust/crates/llms/src/base_llm/base_model_iterator.rs index 928ef80b29a..3289cd1afc7 100644 --- a/litellm-rust/crates/llms/src/base_llm/base_model_iterator.rs +++ b/litellm-rust/crates/llms/src/base_llm/base_model_iterator.rs @@ -1,3 +1,74 @@ +use std::{ + collections::VecDeque, + pin::Pin, + task::{Context, Poll}, +}; + +use futures_util::Stream; +use litellm_types::{responses::streaming::ResponsesEvent, utils::ChatCompletionChunk}; + +use crate::anthropic::experimental_pass_through::messages::streaming_iterator::AnthropicMessagesStreamEvent; + +pub trait ServerStreamingContract { + type Event; +} + +pub struct ChatCompletions; +pub struct ResponsesApi; +pub struct AnthropicMessagesApi; + +impl ServerStreamingContract for ChatCompletions { + type Event = ChatCompletionChunk; +} + +impl ServerStreamingContract for ResponsesApi { + type Event = ResponsesEvent; +} + +impl ServerStreamingContract for AnthropicMessagesApi { + type Event = AnthropicMessagesStreamEvent; +} + +pub trait StreamTransformation: Sized { + type Caller: ServerStreamingContract; + type Upstream: ServerStreamingContract; + type Error; + + fn transform_event( + &mut self, + event: ::Event, + ) -> Result::Event>, Self::Error>; + + fn finish(self) -> Result::Event>, Self::Error>; +} + +pub struct TransformedStream { + upstream: S, + transformation: Option, + pending: VecDeque<::Event>, + finished: bool, +} + +impl TransformedStream { + pub fn new(_upstream: S, _transformation: T) -> Self { + todo!("Retain one call's conversion state and a bounded pending-event queue") + } +} + +impl Stream for TransformedStream +where + T: StreamTransformation, + S: Stream::Event, T::Error>>, +{ + type Item = Result<::Event, T::Error>; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + todo!( + "Drain pending events before polling upstream; preserve backpressure, finish once on EOF, and propagate cancellation and errors without synthesizing success" + ) + } +} + pub trait StreamTransformer { type Input; type Output; diff --git a/litellm-rust/crates/types/src/responses/mod.rs b/litellm-rust/crates/types/src/responses/mod.rs index 02493c5f6ed..85703ef8953 100644 --- a/litellm-rust/crates/types/src/responses/mod.rs +++ b/litellm-rust/crates/types/src/responses/mod.rs @@ -1 +1,2 @@ +pub mod streaming; pub mod streaming_websocket; diff --git a/litellm-rust/crates/types/src/responses/streaming.rs b/litellm-rust/crates/types/src/responses/streaming.rs new file mode 100644 index 00000000000..092f82356c5 --- /dev/null +++ b/litellm-rust/crates/types/src/responses/streaming.rs @@ -0,0 +1,16 @@ +// TODO: Port the complete Python Responses models before enabling these streaming contracts +pub struct ResponsesResponse { + _schema: (), +} + +pub struct ResponseOutputItem { + _schema: (), +} + +pub struct ResponsesRequest { + _schema: (), +} + +pub struct ResponsesEvent { + _schema: (), +}