refactor(rust): scaffold streaming translation contracts

This commit is contained in:
Yujong Lee 2026-09-20 18:29:31 -07:00
parent 6814373a48
commit a545e6ab6d
12 changed files with 381 additions and 0 deletions

View file

@ -0,0 +1,22 @@
use futures_util::Stream;
use litellm_llms::base_llm::base_model_iterator::TransformedStream;
use litellm_types::utils::ChatCompletionChunk;
use super::streaming_iterator::LiteLlmCompletionStreamingIterator;
use crate::responses::streaming_iterator::{ResponsesApiStreamingIterator, ResponsesStreamError};
pub struct LiteLlmCompletionTransformationHandler;
impl LiteLlmCompletionTransformationHandler {
pub fn streaming_response<S>(
_chat_stream: S,
_adapter: LiteLlmCompletionStreamingIterator,
) -> ResponsesApiStreamingIterator<TransformedStream<S, LiteLlmCompletionStreamingIterator>>
where
S: Stream<Item = Result<ChatCompletionChunk, ResponsesStreamError>>,
{
todo!(
"Compose the transformed Chat source inside the same driver used for native Responses events"
)
}
}

View file

@ -0,0 +1,2 @@
pub mod handler;
pub mod streaming_iterator;

View file

@ -0,0 +1,97 @@
use litellm_llms::base_llm::base_model_iterator::{
ChatCompletions, ResponsesApi, StreamTransformation,
};
use litellm_types::{
llms::openai::ChatCompletionToolCallChunk,
responses::streaming::{
ResponseOutputItem, ResponsesEvent, ResponsesRequest, ResponsesResponse,
},
utils::ChatCompletionChunk,
};
use crate::responses::streaming_iterator::ResponsesStreamError;
pub struct LiteLlmCompletionStreamingIterator {
// TODO: Port per-call item/tool associations, request options, bounded accumulators, and reasoning state from Python
_state: (),
}
impl LiteLlmCompletionStreamingIterator {
pub fn new(_model: String, _responses_api_request: ResponsesRequest) -> Self {
todo!(
"Retain bridged input/options and provider identity for response construction and ID encoding"
)
}
pub fn adopt_response_id_from_chunk(&mut self, _chunk: &ChatCompletionChunk) {
todo!(
"Prime the stable response ID before created/in_progress events, retaining the first chunk"
)
}
pub fn queue_tool_call_delta_events(
&mut self,
_tool_calls: &[ChatCompletionToolCallChunk],
) -> Result<(), ResponsesStreamError> {
todo!(
"Associate fragmented tool indexes and call IDs; preserve custom/namespaced tools, arguments, item IDs and output indexes"
)
}
pub fn output_with_streamed_item_ids(
&self,
_response: &ResponsesResponse,
) -> Box<[ResponseOutputItem]> {
todo!("Reuse streamed message, reasoning and tool item IDs in terminal output")
}
pub fn transform_chat_completion_chunk_to_response_api_chunk(
&mut self,
_chunk: ChatCompletionChunk,
) -> Result<Vec<ResponsesEvent>, ResponsesStreamError> {
todo!(
"Convert text, tools, reasoning, annotations and provider web-search fields while preserving sequence numbers"
)
}
pub fn return_default_initial_events(&mut self) -> Vec<ResponsesEvent> {
todo!(
"Emit created and in_progress once after response-ID priming; defer output items until their content is known"
)
}
pub fn return_default_done_events(
&mut self,
) -> Result<Vec<ResponsesEvent>, ResponsesStreamError> {
todo!("Close text, content, reasoning and tool items in order before the terminal response")
}
pub fn emit_response_completed_event(
&mut self,
) -> Result<ResponsesEvent, ResponsesStreamError> {
todo!(
"Build the final response from accumulated chunks and late usage, retaining encoded IDs and request options"
)
}
}
impl StreamTransformation for LiteLlmCompletionStreamingIterator {
type Caller = ResponsesApi;
type Upstream = ChatCompletions;
type Error = ResponsesStreamError;
fn transform_event(
&mut self,
_chunk: ChatCompletionChunk,
) -> Result<Vec<ResponsesEvent>, Self::Error> {
todo!(
"Port conversion from both __next__ and __anext__: prime IDs, emit initial events, merge provider fields, reserve indexes, and drain ordered tool/reasoning events"
)
}
fn finish(self) -> Result<Vec<ResponsesEvent>, Self::Error> {
todo!(
"Require observed upstream termination, include late usage, close pending items, and preserve failure/incomplete/cancellation instead of treating EOF as success"
)
}
}

View file

@ -1,3 +1,5 @@
mod error;
pub use error::Error;
pub mod litellm_completion_transformation;
pub mod streaming_iterator;
pub mod websocket;

View file

@ -0,0 +1,89 @@
use std::{
pin::Pin,
task::{Context, Poll},
};
use futures_util::Stream;
use litellm_types::responses::streaming::{ResponsesEvent, ResponsesResponse};
pub enum ResponsesStreamError {
Upstream(String),
InvalidEvent(String),
UnexpectedEof,
BufferLimitExceeded,
Cancelled,
DeadlineExceeded,
}
pub enum ResponsesStreamState {
Pending,
Active,
Terminal(ResponsesResponse),
Failed(ResponsesStreamError),
}
pub struct ResponsesApiStreamingIterator<S> {
source: S,
state: ResponsesStreamState,
}
impl<S> ResponsesApiStreamingIterator<S> {
pub fn new(_source: S) -> Self {
todo!("Compose BaseResponsesAPIStreamingIterator lifecycle state with a typed event source")
}
pub fn process_chunk(
&mut self,
_event: ResponsesEvent,
) -> Result<Option<ResponsesEvent>, ResponsesStreamError> {
todo!(
"Port base _process_chunk after provider decoding: response IDs, completed/failed/incomplete outcomes, usage, and error events"
)
}
pub fn get_completed_response_object(&self) -> Option<&ResponsesResponse> {
todo!("Expose the completed response retained by the shared lifecycle")
}
pub fn record_failed_response_usage(&mut self, _response: &ResponsesResponse) {
todo!("Retain billable usage on failed and incomplete responses for host accounting")
}
}
impl<S> Stream for ResponsesApiStreamingIterator<S>
where
S: Stream<Item = Result<ResponsesEvent, ResponsesStreamError>>,
{
type Item = Result<ResponsesEvent, ResponsesStreamError>;
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
todo!(
"Port __anext__: read typed events, enforce deadlines and terminal validation, and coordinate delivery, cache, logging and post-stream hooks through route-machine host operations"
)
}
}
pub struct NativeResponsesStream<S> {
source: S,
}
impl<S> NativeResponsesStream<S> {
pub fn new(_source: S) -> Self {
todo!(
"Accept canonical Responses events from provider decoding, independent of transport framing"
)
}
}
impl<S> Stream for NativeResponsesStream<S>
where
S: Stream<Item = Result<ResponsesEvent, ResponsesStreamError>>,
{
type Item = Result<ResponsesEvent, ResponsesStreamError>;
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
todo!(
"Forward native typed events with backpressure and cancellation to the shared Responses driver"
)
}
}

View file

@ -14,6 +14,10 @@ use crate::{
base_llm::{base_model_iterator::StreamTransformer, chat::transformation::Error},
};
use crate::base_llm::base_model_iterator::{
AnthropicMessagesApi, ChatCompletions, StreamTransformation,
};
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum AnthropicJsonChunkType {
ValidJson,
@ -163,3 +167,24 @@ impl StreamTransformer for AnthropicChatCompletionsStreamTransformer {
todo!()
}
}
impl StreamTransformation for AnthropicChatCompletionsStreamTransformer {
type Caller = ChatCompletions;
type Upstream = AnthropicMessagesApi;
type Error = Error;
fn transform_event(
&mut self,
_event: AnthropicMessagesStreamEvent,
) -> Result<Vec<ChatCompletionChunk>, Error> {
todo!(
"Port ModelResponseIterator conversion with indexed tools, thinking/signatures, JSON mode, stop reasons, and late usage"
)
}
fn finish(self) -> Result<Vec<ChatCompletionChunk>, Error> {
todo!(
"Require message_stop, validate partial tool JSON, and preserve failed or cancelled termination"
)
}
}

View file

@ -0,0 +1 @@
pub mod streaming_iterator;

View file

@ -0,0 +1,54 @@
use litellm_types::utils::ChatCompletionChunk;
use crate::{
anthropic::experimental_pass_through::messages::streaming_iterator::AnthropicMessagesStreamEvent,
base_llm::{
base_model_iterator::{AnthropicMessagesApi, ChatCompletions, StreamTransformation},
chat::transformation::Error,
},
};
pub struct AnthropicStreamWrapper {
// TODO: Port block indexes, held stop/usage chunks, thinking signatures and compaction state from Python
_state: (),
}
impl AnthropicStreamWrapper {
pub fn new(_model: String) -> Self {
todo!("Initialize one Chat-to-Messages conversion without owning upstream reads")
}
pub fn handle_choiceless_chunk(&mut self, _chunk: &ChatCompletionChunk) -> Result<(), Error> {
todo!("Preserve late usage and provider fields even when choices are empty")
}
pub fn merge_usage_into_held_stop_reason_chunk(
&mut self,
_chunk: &ChatCompletionChunk,
) -> Result<AnthropicMessagesStreamEvent, Error> {
todo!(
"Merge final usage, cache usage, iterations and context management before message_delta and message_stop"
)
}
}
impl StreamTransformation for AnthropicStreamWrapper {
type Caller = AnthropicMessagesApi;
type Upstream = ChatCompletions;
type Error = Error;
fn transform_event(
&mut self,
_chunk: ChatCompletionChunk,
) -> Result<Vec<AnthropicMessagesStreamEvent>, Error> {
todo!(
"Port _CombinedChunkSplitter and AnthropicStreamWrapper: ordered thinking/text/tools, block starts/stops, signatures, refusal and compaction; retain usage exactly once"
)
}
fn finish(self) -> Result<Vec<AnthropicMessagesStreamEvent>, Error> {
todo!(
"Validate upstream termination and close held blocks/stops after late usage; reject truncated streams"
)
}
}

View file

@ -1 +1,2 @@
pub mod adapters;
pub mod messages;

View file

@ -1,3 +1,74 @@
use std::{
collections::VecDeque,
pin::Pin,
task::{Context, Poll},
};
use futures_util::Stream;
use litellm_types::{responses::streaming::ResponsesEvent, utils::ChatCompletionChunk};
use crate::anthropic::experimental_pass_through::messages::streaming_iterator::AnthropicMessagesStreamEvent;
pub trait ServerStreamingContract {
type Event;
}
pub struct ChatCompletions;
pub struct ResponsesApi;
pub struct AnthropicMessagesApi;
impl ServerStreamingContract for ChatCompletions {
type Event = ChatCompletionChunk;
}
impl ServerStreamingContract for ResponsesApi {
type Event = ResponsesEvent;
}
impl ServerStreamingContract for AnthropicMessagesApi {
type Event = AnthropicMessagesStreamEvent;
}
pub trait StreamTransformation: Sized {
type Caller: ServerStreamingContract;
type Upstream: ServerStreamingContract;
type Error;
fn transform_event(
&mut self,
event: <Self::Upstream as ServerStreamingContract>::Event,
) -> Result<Vec<<Self::Caller as ServerStreamingContract>::Event>, Self::Error>;
fn finish(self) -> Result<Vec<<Self::Caller as ServerStreamingContract>::Event>, Self::Error>;
}
pub struct TransformedStream<S, T: StreamTransformation> {
upstream: S,
transformation: Option<T>,
pending: VecDeque<<T::Caller as ServerStreamingContract>::Event>,
finished: bool,
}
impl<S, T: StreamTransformation> TransformedStream<S, T> {
pub fn new(_upstream: S, _transformation: T) -> Self {
todo!("Retain one call's conversion state and a bounded pending-event queue")
}
}
impl<S, T> Stream for TransformedStream<S, T>
where
T: StreamTransformation,
S: Stream<Item = Result<<T::Upstream as ServerStreamingContract>::Event, T::Error>>,
{
type Item = Result<<T::Caller as ServerStreamingContract>::Event, T::Error>;
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
todo!(
"Drain pending events before polling upstream; preserve backpressure, finish once on EOF, and propagate cancellation and errors without synthesizing success"
)
}
}
pub trait StreamTransformer {
type Input;
type Output;

View file

@ -1 +1,2 @@
pub mod streaming;
pub mod streaming_websocket;

View file

@ -0,0 +1,16 @@
// TODO: Port the complete Python Responses models before enabling these streaming contracts
pub struct ResponsesResponse {
_schema: (),
}
pub struct ResponseOutputItem {
_schema: (),
}
pub struct ResponsesRequest {
_schema: (),
}
pub struct ResponsesEvent {
_schema: (),
}