mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
refactor(rust): scaffold streaming translation contracts
This commit is contained in:
parent
6814373a48
commit
a545e6ab6d
12 changed files with 381 additions and 0 deletions
|
|
@ -0,0 +1,22 @@
|
|||
use futures_util::Stream;
|
||||
use litellm_llms::base_llm::base_model_iterator::TransformedStream;
|
||||
use litellm_types::utils::ChatCompletionChunk;
|
||||
|
||||
use super::streaming_iterator::LiteLlmCompletionStreamingIterator;
|
||||
use crate::responses::streaming_iterator::{ResponsesApiStreamingIterator, ResponsesStreamError};
|
||||
|
||||
pub struct LiteLlmCompletionTransformationHandler;
|
||||
|
||||
impl LiteLlmCompletionTransformationHandler {
|
||||
pub fn streaming_response<S>(
|
||||
_chat_stream: S,
|
||||
_adapter: LiteLlmCompletionStreamingIterator,
|
||||
) -> ResponsesApiStreamingIterator<TransformedStream<S, LiteLlmCompletionStreamingIterator>>
|
||||
where
|
||||
S: Stream<Item = Result<ChatCompletionChunk, ResponsesStreamError>>,
|
||||
{
|
||||
todo!(
|
||||
"Compose the transformed Chat source inside the same driver used for native Responses events"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
pub mod handler;
|
||||
pub mod streaming_iterator;
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
use litellm_llms::base_llm::base_model_iterator::{
|
||||
ChatCompletions, ResponsesApi, StreamTransformation,
|
||||
};
|
||||
use litellm_types::{
|
||||
llms::openai::ChatCompletionToolCallChunk,
|
||||
responses::streaming::{
|
||||
ResponseOutputItem, ResponsesEvent, ResponsesRequest, ResponsesResponse,
|
||||
},
|
||||
utils::ChatCompletionChunk,
|
||||
};
|
||||
|
||||
use crate::responses::streaming_iterator::ResponsesStreamError;
|
||||
|
||||
pub struct LiteLlmCompletionStreamingIterator {
|
||||
// TODO: Port per-call item/tool associations, request options, bounded accumulators, and reasoning state from Python
|
||||
_state: (),
|
||||
}
|
||||
|
||||
impl LiteLlmCompletionStreamingIterator {
|
||||
pub fn new(_model: String, _responses_api_request: ResponsesRequest) -> Self {
|
||||
todo!(
|
||||
"Retain bridged input/options and provider identity for response construction and ID encoding"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn adopt_response_id_from_chunk(&mut self, _chunk: &ChatCompletionChunk) {
|
||||
todo!(
|
||||
"Prime the stable response ID before created/in_progress events, retaining the first chunk"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn queue_tool_call_delta_events(
|
||||
&mut self,
|
||||
_tool_calls: &[ChatCompletionToolCallChunk],
|
||||
) -> Result<(), ResponsesStreamError> {
|
||||
todo!(
|
||||
"Associate fragmented tool indexes and call IDs; preserve custom/namespaced tools, arguments, item IDs and output indexes"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn output_with_streamed_item_ids(
|
||||
&self,
|
||||
_response: &ResponsesResponse,
|
||||
) -> Box<[ResponseOutputItem]> {
|
||||
todo!("Reuse streamed message, reasoning and tool item IDs in terminal output")
|
||||
}
|
||||
|
||||
pub fn transform_chat_completion_chunk_to_response_api_chunk(
|
||||
&mut self,
|
||||
_chunk: ChatCompletionChunk,
|
||||
) -> Result<Vec<ResponsesEvent>, ResponsesStreamError> {
|
||||
todo!(
|
||||
"Convert text, tools, reasoning, annotations and provider web-search fields while preserving sequence numbers"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn return_default_initial_events(&mut self) -> Vec<ResponsesEvent> {
|
||||
todo!(
|
||||
"Emit created and in_progress once after response-ID priming; defer output items until their content is known"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn return_default_done_events(
|
||||
&mut self,
|
||||
) -> Result<Vec<ResponsesEvent>, ResponsesStreamError> {
|
||||
todo!("Close text, content, reasoning and tool items in order before the terminal response")
|
||||
}
|
||||
|
||||
pub fn emit_response_completed_event(
|
||||
&mut self,
|
||||
) -> Result<ResponsesEvent, ResponsesStreamError> {
|
||||
todo!(
|
||||
"Build the final response from accumulated chunks and late usage, retaining encoded IDs and request options"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl StreamTransformation for LiteLlmCompletionStreamingIterator {
|
||||
type Caller = ResponsesApi;
|
||||
type Upstream = ChatCompletions;
|
||||
type Error = ResponsesStreamError;
|
||||
|
||||
fn transform_event(
|
||||
&mut self,
|
||||
_chunk: ChatCompletionChunk,
|
||||
) -> Result<Vec<ResponsesEvent>, Self::Error> {
|
||||
todo!(
|
||||
"Port conversion from both __next__ and __anext__: prime IDs, emit initial events, merge provider fields, reserve indexes, and drain ordered tool/reasoning events"
|
||||
)
|
||||
}
|
||||
|
||||
fn finish(self) -> Result<Vec<ResponsesEvent>, Self::Error> {
|
||||
todo!(
|
||||
"Require observed upstream termination, include late usage, close pending items, and preserve failure/incomplete/cancellation instead of treating EOF as success"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
mod error;
|
||||
pub use error::Error;
|
||||
pub mod litellm_completion_transformation;
|
||||
pub mod streaming_iterator;
|
||||
pub mod websocket;
|
||||
|
|
|
|||
89
litellm-rust/crates/core/src/responses/streaming_iterator.rs
Normal file
89
litellm-rust/crates/core/src/responses/streaming_iterator.rs
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
use std::{
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
|
||||
use futures_util::Stream;
|
||||
use litellm_types::responses::streaming::{ResponsesEvent, ResponsesResponse};
|
||||
|
||||
pub enum ResponsesStreamError {
|
||||
Upstream(String),
|
||||
InvalidEvent(String),
|
||||
UnexpectedEof,
|
||||
BufferLimitExceeded,
|
||||
Cancelled,
|
||||
DeadlineExceeded,
|
||||
}
|
||||
|
||||
pub enum ResponsesStreamState {
|
||||
Pending,
|
||||
Active,
|
||||
Terminal(ResponsesResponse),
|
||||
Failed(ResponsesStreamError),
|
||||
}
|
||||
|
||||
pub struct ResponsesApiStreamingIterator<S> {
|
||||
source: S,
|
||||
state: ResponsesStreamState,
|
||||
}
|
||||
|
||||
impl<S> ResponsesApiStreamingIterator<S> {
|
||||
pub fn new(_source: S) -> Self {
|
||||
todo!("Compose BaseResponsesAPIStreamingIterator lifecycle state with a typed event source")
|
||||
}
|
||||
|
||||
pub fn process_chunk(
|
||||
&mut self,
|
||||
_event: ResponsesEvent,
|
||||
) -> Result<Option<ResponsesEvent>, ResponsesStreamError> {
|
||||
todo!(
|
||||
"Port base _process_chunk after provider decoding: response IDs, completed/failed/incomplete outcomes, usage, and error events"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn get_completed_response_object(&self) -> Option<&ResponsesResponse> {
|
||||
todo!("Expose the completed response retained by the shared lifecycle")
|
||||
}
|
||||
|
||||
pub fn record_failed_response_usage(&mut self, _response: &ResponsesResponse) {
|
||||
todo!("Retain billable usage on failed and incomplete responses for host accounting")
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> Stream for ResponsesApiStreamingIterator<S>
|
||||
where
|
||||
S: Stream<Item = Result<ResponsesEvent, ResponsesStreamError>>,
|
||||
{
|
||||
type Item = Result<ResponsesEvent, ResponsesStreamError>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
todo!(
|
||||
"Port __anext__: read typed events, enforce deadlines and terminal validation, and coordinate delivery, cache, logging and post-stream hooks through route-machine host operations"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct NativeResponsesStream<S> {
|
||||
source: S,
|
||||
}
|
||||
|
||||
impl<S> NativeResponsesStream<S> {
|
||||
pub fn new(_source: S) -> Self {
|
||||
todo!(
|
||||
"Accept canonical Responses events from provider decoding, independent of transport framing"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> Stream for NativeResponsesStream<S>
|
||||
where
|
||||
S: Stream<Item = Result<ResponsesEvent, ResponsesStreamError>>,
|
||||
{
|
||||
type Item = Result<ResponsesEvent, ResponsesStreamError>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
todo!(
|
||||
"Forward native typed events with backpressure and cancellation to the shared Responses driver"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -14,6 +14,10 @@ use crate::{
|
|||
base_llm::{base_model_iterator::StreamTransformer, chat::transformation::Error},
|
||||
};
|
||||
|
||||
use crate::base_llm::base_model_iterator::{
|
||||
AnthropicMessagesApi, ChatCompletions, StreamTransformation,
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum AnthropicJsonChunkType {
|
||||
ValidJson,
|
||||
|
|
@ -163,3 +167,24 @@ impl StreamTransformer for AnthropicChatCompletionsStreamTransformer {
|
|||
todo!()
|
||||
}
|
||||
}
|
||||
|
||||
impl StreamTransformation for AnthropicChatCompletionsStreamTransformer {
|
||||
type Caller = ChatCompletions;
|
||||
type Upstream = AnthropicMessagesApi;
|
||||
type Error = Error;
|
||||
|
||||
fn transform_event(
|
||||
&mut self,
|
||||
_event: AnthropicMessagesStreamEvent,
|
||||
) -> Result<Vec<ChatCompletionChunk>, Error> {
|
||||
todo!(
|
||||
"Port ModelResponseIterator conversion with indexed tools, thinking/signatures, JSON mode, stop reasons, and late usage"
|
||||
)
|
||||
}
|
||||
|
||||
fn finish(self) -> Result<Vec<ChatCompletionChunk>, Error> {
|
||||
todo!(
|
||||
"Require message_stop, validate partial tool JSON, and preserve failed or cancelled termination"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
pub mod streaming_iterator;
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
use litellm_types::utils::ChatCompletionChunk;
|
||||
|
||||
use crate::{
|
||||
anthropic::experimental_pass_through::messages::streaming_iterator::AnthropicMessagesStreamEvent,
|
||||
base_llm::{
|
||||
base_model_iterator::{AnthropicMessagesApi, ChatCompletions, StreamTransformation},
|
||||
chat::transformation::Error,
|
||||
},
|
||||
};
|
||||
|
||||
pub struct AnthropicStreamWrapper {
|
||||
// TODO: Port block indexes, held stop/usage chunks, thinking signatures and compaction state from Python
|
||||
_state: (),
|
||||
}
|
||||
|
||||
impl AnthropicStreamWrapper {
|
||||
pub fn new(_model: String) -> Self {
|
||||
todo!("Initialize one Chat-to-Messages conversion without owning upstream reads")
|
||||
}
|
||||
|
||||
pub fn handle_choiceless_chunk(&mut self, _chunk: &ChatCompletionChunk) -> Result<(), Error> {
|
||||
todo!("Preserve late usage and provider fields even when choices are empty")
|
||||
}
|
||||
|
||||
pub fn merge_usage_into_held_stop_reason_chunk(
|
||||
&mut self,
|
||||
_chunk: &ChatCompletionChunk,
|
||||
) -> Result<AnthropicMessagesStreamEvent, Error> {
|
||||
todo!(
|
||||
"Merge final usage, cache usage, iterations and context management before message_delta and message_stop"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl StreamTransformation for AnthropicStreamWrapper {
|
||||
type Caller = AnthropicMessagesApi;
|
||||
type Upstream = ChatCompletions;
|
||||
type Error = Error;
|
||||
|
||||
fn transform_event(
|
||||
&mut self,
|
||||
_chunk: ChatCompletionChunk,
|
||||
) -> Result<Vec<AnthropicMessagesStreamEvent>, Error> {
|
||||
todo!(
|
||||
"Port _CombinedChunkSplitter and AnthropicStreamWrapper: ordered thinking/text/tools, block starts/stops, signatures, refusal and compaction; retain usage exactly once"
|
||||
)
|
||||
}
|
||||
|
||||
fn finish(self) -> Result<Vec<AnthropicMessagesStreamEvent>, Error> {
|
||||
todo!(
|
||||
"Validate upstream termination and close held blocks/stops after late usage; reject truncated streams"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -1 +1,2 @@
|
|||
pub mod adapters;
|
||||
pub mod messages;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,74 @@
|
|||
use std::{
|
||||
collections::VecDeque,
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
|
||||
use futures_util::Stream;
|
||||
use litellm_types::{responses::streaming::ResponsesEvent, utils::ChatCompletionChunk};
|
||||
|
||||
use crate::anthropic::experimental_pass_through::messages::streaming_iterator::AnthropicMessagesStreamEvent;
|
||||
|
||||
pub trait ServerStreamingContract {
|
||||
type Event;
|
||||
}
|
||||
|
||||
pub struct ChatCompletions;
|
||||
pub struct ResponsesApi;
|
||||
pub struct AnthropicMessagesApi;
|
||||
|
||||
impl ServerStreamingContract for ChatCompletions {
|
||||
type Event = ChatCompletionChunk;
|
||||
}
|
||||
|
||||
impl ServerStreamingContract for ResponsesApi {
|
||||
type Event = ResponsesEvent;
|
||||
}
|
||||
|
||||
impl ServerStreamingContract for AnthropicMessagesApi {
|
||||
type Event = AnthropicMessagesStreamEvent;
|
||||
}
|
||||
|
||||
pub trait StreamTransformation: Sized {
|
||||
type Caller: ServerStreamingContract;
|
||||
type Upstream: ServerStreamingContract;
|
||||
type Error;
|
||||
|
||||
fn transform_event(
|
||||
&mut self,
|
||||
event: <Self::Upstream as ServerStreamingContract>::Event,
|
||||
) -> Result<Vec<<Self::Caller as ServerStreamingContract>::Event>, Self::Error>;
|
||||
|
||||
fn finish(self) -> Result<Vec<<Self::Caller as ServerStreamingContract>::Event>, Self::Error>;
|
||||
}
|
||||
|
||||
pub struct TransformedStream<S, T: StreamTransformation> {
|
||||
upstream: S,
|
||||
transformation: Option<T>,
|
||||
pending: VecDeque<<T::Caller as ServerStreamingContract>::Event>,
|
||||
finished: bool,
|
||||
}
|
||||
|
||||
impl<S, T: StreamTransformation> TransformedStream<S, T> {
|
||||
pub fn new(_upstream: S, _transformation: T) -> Self {
|
||||
todo!("Retain one call's conversion state and a bounded pending-event queue")
|
||||
}
|
||||
}
|
||||
|
||||
impl<S, T> Stream for TransformedStream<S, T>
|
||||
where
|
||||
T: StreamTransformation,
|
||||
S: Stream<Item = Result<<T::Upstream as ServerStreamingContract>::Event, T::Error>>,
|
||||
{
|
||||
type Item = Result<<T::Caller as ServerStreamingContract>::Event, T::Error>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
todo!(
|
||||
"Drain pending events before polling upstream; preserve backpressure, finish once on EOF, and propagate cancellation and errors without synthesizing success"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub trait StreamTransformer {
|
||||
type Input;
|
||||
type Output;
|
||||
|
|
|
|||
|
|
@ -1 +1,2 @@
|
|||
pub mod streaming;
|
||||
pub mod streaming_websocket;
|
||||
|
|
|
|||
16
litellm-rust/crates/types/src/responses/streaming.rs
Normal file
16
litellm-rust/crates/types/src/responses/streaming.rs
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
// TODO: Port the complete Python Responses models before enabling these streaming contracts
|
||||
pub struct ResponsesResponse {
|
||||
_schema: (),
|
||||
}
|
||||
|
||||
pub struct ResponseOutputItem {
|
||||
_schema: (),
|
||||
}
|
||||
|
||||
pub struct ResponsesRequest {
|
||||
_schema: (),
|
||||
}
|
||||
|
||||
pub struct ResponsesEvent {
|
||||
_schema: (),
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue