diff --git a/litellm-rust/crates/core/src/chat_completions/mod.rs b/litellm-rust/crates/core/src/chat_completions/mod.rs index 81d35044d08..24113b4d5a2 100644 --- a/litellm-rust/crates/core/src/chat_completions/mod.rs +++ b/litellm-rust/crates/core/src/chat_completions/mod.rs @@ -7,6 +7,7 @@ //! calls the provider, and returns a typed OpenAI-shaped response. mod error; +pub mod streaming; pub mod types; pub use error::Error; mod client; diff --git a/litellm-rust/crates/core/src/chat_completions/streaming.rs b/litellm-rust/crates/core/src/chat_completions/streaming.rs new file mode 100644 index 00000000000..05345cfd2cc --- /dev/null +++ b/litellm-rust/crates/core/src/chat_completions/streaming.rs @@ -0,0 +1,29 @@ +use litellm_host::machine::RouteMachine; +use litellm_llms::base_llm::base_model_iterator::ChatCompletions; + +use crate::streaming::{StreamingCall, StreamingRoute}; + +pub struct ChatCompletionsCall { + pub model: String, + pub messages: Box<[litellm_types::llms::openai::ChatMessage]>, + pub optional_params: std::collections::BTreeMap, + pub custom_llm_provider: Option, + pub api_key: Option, + pub api_base: Option, + pub extra_headers: Box<[(String, String)]>, + pub timeout: Option, +} + +pub struct Streaming; + +impl StreamingCall for Streaming { + type Contract = ChatCompletions; + type Request = ChatCompletionsCall; + type Response = litellm_types::utils::ChatCompletionsResponse; +} + +pub fn chat_completions_stream_machine() -> RouteMachine> { + todo!( + "Select a typed native or translated stream and preserve existing provider acceptance gates until the streaming path is implemented" + ) +} diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs index afe5ea595aa..055aa986f15 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/core/src/lib.rs @@ -6,5 +6,6 @@ pub mod messages; pub mod ocr; mod outbound; pub mod responses; +pub mod streaming; pub use error::Error; diff --git a/litellm-rust/crates/core/src/messages/mod.rs b/litellm-rust/crates/core/src/messages/mod.rs index 289f79109dd..cf10da0c15d 100644 --- a/litellm-rust/crates/core/src/messages/mod.rs +++ b/litellm-rust/crates/core/src/messages/mod.rs @@ -5,6 +5,7 @@ //! it in process for a caller that already holds the request and wants the message. mod error; +pub mod streaming; pub mod types; pub use error::Error; mod client; diff --git a/litellm-rust/crates/core/src/messages/streaming.rs b/litellm-rust/crates/core/src/messages/streaming.rs new file mode 100644 index 00000000000..113050237ac --- /dev/null +++ b/litellm-rust/crates/core/src/messages/streaming.rs @@ -0,0 +1,37 @@ +use futures_util::Stream; +use litellm_host::machine::RouteMachine; +use litellm_llms::{ + anthropic::experimental_pass_through::responses_adapters::streaming_iterator::AnthropicResponsesStreamWrapper, + base_llm::base_model_iterator::{ + AnthropicMessagesApi, StreamError, StreamPolicy, TransformedStream, + }, +}; +use litellm_types::responses::streaming::ResponsesEvent; + +use crate::streaming::{StreamingCall, StreamingRoute}; + +pub struct Streaming; + +impl StreamingCall for Streaming { + type Contract = AnthropicMessagesApi; + type Request = super::route::MessagesCall; + type Response = + litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse; +} + +pub fn messages_stream_machine() -> RouteMachine> { + todo!( + "Select a typed native or translated stream and preserve existing provider acceptance gates until the streaming path is implemented" + ) +} + +pub fn messages_from_responses( + _source: S, + _adapter: AnthropicResponsesStreamWrapper, + _policy: StreamPolicy, +) -> TransformedStream +where + S: Stream>, +{ + todo!("Compose canonical Responses events with the Messages bridge before typed host delivery") +} diff --git a/litellm-rust/crates/core/src/responses/litellm_completion_transformation/handler.rs b/litellm-rust/crates/core/src/responses/litellm_completion_transformation/handler.rs index 4ae29054ca3..2b0c9b4ffc4 100644 --- a/litellm-rust/crates/core/src/responses/litellm_completion_transformation/handler.rs +++ b/litellm-rust/crates/core/src/responses/litellm_completion_transformation/handler.rs @@ -1,9 +1,9 @@ use futures_util::Stream; -use litellm_llms::base_llm::base_model_iterator::TransformedStream; +use litellm_llms::base_llm::base_model_iterator::{StreamError, StreamPolicy, TransformedStream}; use litellm_types::utils::ChatCompletionChunk; use super::streaming_iterator::LiteLlmCompletionStreamingIterator; -use crate::responses::streaming_iterator::{ResponsesApiStreamingIterator, ResponsesStreamError}; +use crate::responses::streaming_iterator::ResponsesApiStreamingIterator; pub struct LiteLlmCompletionTransformationHandler; @@ -11,9 +11,10 @@ impl LiteLlmCompletionTransformationHandler { pub fn streaming_response( _chat_stream: S, _adapter: LiteLlmCompletionStreamingIterator, + _policy: StreamPolicy, ) -> ResponsesApiStreamingIterator> where - S: Stream>, + S: Stream>, { todo!( "Compose the transformed Chat source inside the same driver used for native Responses events" diff --git a/litellm-rust/crates/core/src/responses/litellm_completion_transformation/streaming_iterator.rs b/litellm-rust/crates/core/src/responses/litellm_completion_transformation/streaming_iterator.rs index f8700114fb3..38f24c0cbc1 100644 --- a/litellm-rust/crates/core/src/responses/litellm_completion_transformation/streaming_iterator.rs +++ b/litellm-rust/crates/core/src/responses/litellm_completion_transformation/streaming_iterator.rs @@ -1,5 +1,5 @@ use litellm_llms::base_llm::base_model_iterator::{ - ChatCompletions, ResponsesApi, StreamTransformation, + ChatCompletions, ResponsesApi, StreamError, StreamTransformation, }; use litellm_types::{ llms::openai::ChatCompletionToolCallChunk, @@ -9,8 +9,6 @@ use litellm_types::{ utils::ChatCompletionChunk, }; -use crate::responses::streaming_iterator::ResponsesStreamError; - pub struct LiteLlmCompletionStreamingIterator { // TODO: Port per-call item/tool associations, request options, bounded accumulators, and reasoning state from Python _state: (), @@ -32,7 +30,7 @@ impl LiteLlmCompletionStreamingIterator { pub fn queue_tool_call_delta_events( &mut self, _tool_calls: &[ChatCompletionToolCallChunk], - ) -> Result<(), ResponsesStreamError> { + ) -> Result<(), StreamError> { todo!( "Associate fragmented tool indexes and call IDs; preserve custom/namespaced tools, arguments, item IDs and output indexes" ) @@ -48,7 +46,7 @@ impl LiteLlmCompletionStreamingIterator { pub fn transform_chat_completion_chunk_to_response_api_chunk( &mut self, _chunk: ChatCompletionChunk, - ) -> Result, ResponsesStreamError> { + ) -> Result, StreamError> { todo!( "Convert text, tools, reasoning, annotations and provider web-search fields while preserving sequence numbers" ) @@ -60,15 +58,11 @@ impl LiteLlmCompletionStreamingIterator { ) } - pub fn return_default_done_events( - &mut self, - ) -> Result, ResponsesStreamError> { + pub fn return_default_done_events(&mut self) -> Result, StreamError> { todo!("Close text, content, reasoning and tool items in order before the terminal response") } - pub fn emit_response_completed_event( - &mut self, - ) -> Result { + pub fn emit_response_completed_event(&mut self) -> Result { todo!( "Build the final response from accumulated chunks and late usage, retaining encoded IDs and request options" ) @@ -78,7 +72,7 @@ impl LiteLlmCompletionStreamingIterator { impl StreamTransformation for LiteLlmCompletionStreamingIterator { type Caller = ResponsesApi; type Upstream = ChatCompletions; - type Error = ResponsesStreamError; + type Error = StreamError; fn transform_event( &mut self, diff --git a/litellm-rust/crates/core/src/responses/mod.rs b/litellm-rust/crates/core/src/responses/mod.rs index 6c175a3f1f5..73b1334f870 100644 --- a/litellm-rust/crates/core/src/responses/mod.rs +++ b/litellm-rust/crates/core/src/responses/mod.rs @@ -1,5 +1,15 @@ mod error; pub use error::Error; pub mod litellm_completion_transformation; +pub mod route; pub mod streaming_iterator; pub mod websocket; +pub mod websocket_session; + +use litellm_types::responses::streaming::ResponsesResponse; + +pub async fn responses(_call: route::ResponsesCall) -> Result { + todo!( + "Resolve native or bridged provider, prepare/send HTTP request, and normalize the response through route host operations" + ) +} diff --git a/litellm-rust/crates/core/src/responses/route.rs b/litellm-rust/crates/core/src/responses/route.rs new file mode 100644 index 00000000000..955af38a60c --- /dev/null +++ b/litellm-rust/crates/core/src/responses/route.rs @@ -0,0 +1,31 @@ +use std::time::Duration; + +use litellm_auth::SecretValue; +use litellm_host::machine::RouteMachine; +use litellm_llms::base_llm::base_model_iterator::ResponsesApi; +use litellm_types::responses::streaming::{ResponsesRequest, ResponsesResponse}; + +use crate::streaming::{StreamingCall, StreamingRoute}; + +pub struct ResponsesCall { + pub request: ResponsesRequest, + pub custom_llm_provider: Option, + pub api_key: Option, + pub api_base: Option, + pub extra_headers: Box<[(String, String)]>, + pub timeout: Option, +} + +pub struct Responses; + +impl StreamingCall for Responses { + type Contract = ResponsesApi; + type Request = ResponsesCall; + type Response = ResponsesResponse; +} + +pub fn responses_machine() -> RouteMachine> { + todo!( + "Compose native Responses or Chat-to-Responses conversion with the shared Responses driver and host operations" + ) +} diff --git a/litellm-rust/crates/core/src/responses/streaming_iterator.rs b/litellm-rust/crates/core/src/responses/streaming_iterator.rs index 8bccc52a73a..11a8f067397 100644 --- a/litellm-rust/crates/core/src/responses/streaming_iterator.rs +++ b/litellm-rust/crates/core/src/responses/streaming_iterator.rs @@ -1,3 +1,6 @@ +use litellm_llms::base_llm::base_model_iterator::{ + BufferedSource, StreamError, StreamOutcome, StreamPolicy, +}; use std::{ pin::Pin, task::{Context, Poll}, @@ -6,36 +9,28 @@ use std::{ use futures_util::Stream; use litellm_types::responses::streaming::{ResponsesEvent, ResponsesResponse}; -pub enum ResponsesStreamError { - Upstream(String), - InvalidEvent(String), - UnexpectedEof, - BufferLimitExceeded, - Cancelled, - DeadlineExceeded, -} - pub enum ResponsesStreamState { Pending, Active, - Terminal(ResponsesResponse), - Failed(ResponsesStreamError), + Terminal(StreamOutcome), + Failed(StreamError), } pub struct ResponsesApiStreamingIterator { source: S, state: ResponsesStreamState, + policy: StreamPolicy, } impl ResponsesApiStreamingIterator { - pub fn new(_source: S) -> Self { + pub fn new(_source: S, _policy: StreamPolicy) -> Self { todo!("Compose BaseResponsesAPIStreamingIterator lifecycle state with a typed event source") } pub fn process_chunk( &mut self, _event: ResponsesEvent, - ) -> Result, ResponsesStreamError> { + ) -> Result, StreamError> { todo!( "Port base _process_chunk after provider decoding: response IDs, completed/failed/incomplete outcomes, usage, and error events" ) @@ -50,11 +45,39 @@ impl ResponsesApiStreamingIterator { } } +pub struct BufferedResponsesStream { + _response: ResponsesResponse, + _source: BufferedSource, + _policy: StreamPolicy, +} + +impl BufferedResponsesStream { + pub fn new( + _response: ResponsesResponse, + _source: BufferedSource, + _policy: StreamPolicy, + ) -> Self { + todo!( + "Port cached/mock/simulated response event generation with explicit buffered delivery and stable item IDs" + ) + } +} + +impl Stream for BufferedResponsesStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + todo!( + "Emit bounded ordered lifecycle/content/tool events from the retained response, preserving its terminal status and usage" + ) + } +} + impl Stream for ResponsesApiStreamingIterator where - S: Stream>, + S: Stream>, { - type Item = Result; + type Item = Result; fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { todo!( @@ -77,9 +100,9 @@ impl NativeResponsesStream { impl Stream for NativeResponsesStream where - S: Stream>, + S: Stream>, { - type Item = Result; + type Item = Result; fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { todo!( diff --git a/litellm-rust/crates/core/src/responses/websocket_session.rs b/litellm-rust/crates/core/src/responses/websocket_session.rs new file mode 100644 index 00000000000..9f6ed6a3287 --- /dev/null +++ b/litellm-rust/crates/core/src/responses/websocket_session.rs @@ -0,0 +1,356 @@ +use std::collections::BTreeMap; + +use litellm_host::{ + machine::{HostChannel, RouteMachine}, + route::Route, +}; +use litellm_llms::base_llm::base_model_iterator::{StreamError, StreamOutcome, StreamPolicy}; +use litellm_types::responses::streaming::{ + ResponseInputItem, ResponsesError, ResponsesEvent, ResponsesInput, ResponsesRequest, + ResponsesResponse, +}; +use litellm_types::responses::streaming_websocket::ResponsesWebSocketRequestDefaults; + +use super::{route::ResponsesCall, websocket::ResponsesWebSocketConnection}; + +pub struct ResponseCreate { + pub request: ResponsesRequest, + pub generate: Option, +} + +pub enum ClientFrame { + Create(ResponseCreate), + Closed, +} + +pub struct ResponsesWebSocketConfig { + pub call: ResponsesCall, + pub authorized_model: String, + pub first_message: Option, + pub request_defaults: ResponsesWebSocketRequestDefaults, + pub stream_policy: StreamPolicy, + pub max_history_bytes: usize, + pub max_history_turns: usize, +} + +pub enum ResponsesWebSocketState { + Idle, + Warmup { response_id: String }, + Active { response_id: Option }, + Closed, + Failed(StreamError), +} + +pub enum WebSocketOutcome { + Closed, + Cancelled, + Failed(StreamError), +} + +pub enum InputPolicyResult { + Approved(ResponsesRequest), + Rejected(ResponsesError), +} + +pub enum WebSocketOp { + ReceiveClientFrame, + SendClientFrame(String), + ApplyInputPolicy { + request: ResponsesRequest, + authorized_model: String, + }, + EnforceQuota { + request: ResponsesRequest, + }, + ApplyOutputPolicy(ResponsesEvent), + LoadHistory { + response_id: String, + }, + StoreHistory { + response_id: String, + items: Box<[ResponseInputItem]>, + }, + AccountTurn(StreamOutcome), + PostStreamingHooks, +} + +pub enum WebSocketOpResult { + ClientFrame(Option), + InputPolicy(InputPolicyResult), + Quota(Result<(), ResponsesError>), + OutputEvent(ResponsesEvent), + History(Box<[ResponseInputItem]>), + Acknowledged, +} + +pub struct ResponsesWebSocket; + +impl Route for ResponsesWebSocket { + type Response = WebSocketOutcome; + type Error = StreamError; + type Op = WebSocketOp; + type OpResult = WebSocketOpResult; + type Chunk = std::convert::Infallible; + type StreamHead = (); +} + +pub enum ResponsesWebSocketMode { + Native(ResponsesWebSocketConnection), + Managed, +} + +pub fn responses_websocket_machine( + _config: ResponsesWebSocketConfig, + _mode: ResponsesWebSocketMode, +) -> RouteMachine { + todo!( + "Select native or managed session driver; keep client socket I/O, policies, storage and callback execution behind host operations" + ) +} + +pub fn parse_message(_raw_message: &str) -> Result { + todo!( + "Validate response.create in flat or nested form, including generate=false and previous_response_id; reject unsupported client events" + ) +} + +pub struct ResponsesWebSocketStreaming { + _backend_ws: ResponsesWebSocketConnection, + _config: ResponsesWebSocketConfig, + _state: ResponsesWebSocketState, + _terminal: Option>, +} + +impl ResponsesWebSocketStreaming { + pub fn new( + _backend_ws: ResponsesWebSocketConnection, + _config: ResponsesWebSocketConfig, + ) -> Self { + todo!( + "Initialize a native session with first-frame handling and one active turn per connection" + ) + } + + pub fn enforce_authorized_model(&self, _request: &ResponsesRequest) -> Result<(), StreamError> { + todo!("Check the authorized model on every response.create, including nested requests") + } + + pub fn with_request_defaults( + &self, + _request: ResponsesRequest, + ) -> Result { + todo!( + "Merge fill_missing defaults, then per-turn parameters, then deployment overrides in Python's precedence order" + ) + } + + pub fn collect_input_from_client_event( + &mut self, + _request: &ResponsesRequest, + ) -> Result<(), StreamError> { + todo!("Accumulate per-turn input for accounting within session limits") + } + + pub fn store_event(&mut self, _event: &ResponsesEvent) -> Result<(), StreamError> { + todo!( + "Retain required lifecycle/usage events within bounds, resetting accumulation between turns" + ) + } + + pub async fn enforce_or_reject_frame( + &mut self, + _host: &HostChannel, + _frame: ResponseCreate, + ) -> Result { + todo!( + "Enforce model, request defaults, host input masking and per-turn quota before forwarding" + ) + } + + pub async fn backend_to_client( + &mut self, + _host: &HostChannel, + ) -> Result<(), StreamError> { + todo!( + "Decode backend events, apply host output policies, forward frames, and account completed/failed/incomplete turns exactly once" + ) + } + + pub async fn client_to_backend( + &mut self, + _host: &HostChannel, + ) -> Result<(), StreamError> { + todo!( + "Read first/subsequent frames through host operations and validate each turn before upstream send" + ) + } + + pub async fn log_messages( + &mut self, + _host: &HostChannel, + ) -> Result<(), StreamError> { + todo!( + "Yield turn accounting and post-stream hooks without running callbacks or writing storage in core" + ) + } + + pub async fn record_usage_for_failure( + &mut self, + _host: &HostChannel, + ) -> Result<(), StreamError> { + todo!( + "Preserve observed usage after failures, disconnects and cancellation without double accounting" + ) + } + + pub async fn bidirectional_forward( + self, + _host: &HostChannel, + ) -> Result { + todo!( + "Split independent read/write ownership, coordinate both directions with backpressure, cancel peers and close upstream on exit; created is not turn completion" + ) + } +} + +pub struct ManagedResponsesWebSocketHandler { + _config: ResponsesWebSocketConfig, + _state: ResponsesWebSocketState, + _session_history: BTreeMap>, +} + +impl ManagedResponsesWebSocketHandler { + pub fn new(_config: ResponsesWebSocketConfig) -> Self { + todo!("Initialize bounded per-connection history and sequential HTTP-backed turns") + } + + pub fn serialize_chunk(_event: &ResponsesEvent) -> Result { + todo!("Use the canonical Responses event encoder shared with SSE") + } + + pub fn get_history_messages( + &self, + _previous_response_id: &str, + ) -> Option<&[ResponseInputItem]> { + todo!( + "Look up decoded IDs in session history before requesting persistent history from the host" + ) + } + + pub fn store_history( + &mut self, + _response_id: String, + _items: Box<[ResponseInputItem]>, + ) -> Result<(), StreamError> { + todo!( + "Enforce history byte/turn limits and preserve follow-up availability before asynchronous host persistence" + ) + } + + pub fn input_to_messages(_input: ResponsesInput) -> Box<[ResponseInputItem]> { + todo!( + "Normalize text and typed input items without losing tools, reasoning or multimodal content" + ) + } + + pub fn is_warmup_frame(_frame: &ResponseCreate) -> bool { + todo!("Recognize generate=false without issuing a provider generation request") + } + + pub fn is_warmup_response_id(_response_id: &str) -> bool { + todo!("Recognize synthetic warmup IDs so they are not replayed as provider response IDs") + } + + pub fn build_warmup_response(&self, _frame: &ResponseCreate) -> ResponsesResponse { + todo!("Construct a synthetic warmup acknowledgement using the request's model and options") + } + + pub fn build_base_call_kwargs(_frame: ResponseCreate) -> ResponsesRequest { + todo!("Extract supported Responses parameters while excluding WebSocket-only fields") + } + + pub fn apply_history( + &self, + _request: ResponsesRequest, + _prior_history: &[ResponseInputItem], + ) -> Result { + todo!( + "Prepend session or host history without duplicating input or forwarding synthetic warmup IDs" + ) + } + + pub fn same_provider(&self, _model: &str) -> Result { + todo!( + "Resolve the turn provider before deciding whether connection credentials can be reused" + ) + } + + pub fn inject_credentials( + &self, + _request: ResponsesRequest, + ) -> Result { + todo!( + "Reuse connection credentials only for the same provider and preserve per-turn routing" + ) + } + + pub async fn send_error( + &self, + _host: &HostChannel, + _error: ResponsesError, + ) -> Result<(), StreamError> { + todo!( + "Send a structured error frame through the host, preserving session versus turn failure" + ) + } + + pub async fn send_warmup_ack( + &mut self, + _host: &HostChannel, + _frame: ResponseCreate, + ) -> Result<(), StreamError> { + todo!( + "Send warmup acknowledgement without generation, usage fabrication or history pollution" + ) + } + + pub async fn stream_and_forward( + &mut self, + _host: &HostChannel, + _call: ResponsesCall, + ) -> Result, StreamError> { + todo!( + "Drive the HTTP Responses route and forward canonical events with host output policy, backpressure, cancellation and terminal validation" + ) + } + + pub async fn save_turn_history( + &mut self, + _host: &HostChannel, + _request: ResponsesRequest, + _response: ResponsesResponse, + ) -> Result<(), StreamError> { + todo!( + "Save normalized input and output under the decoded response ID, then yield host persistence" + ) + } + + pub async fn process_response_create( + &mut self, + _host: &HostChannel, + _frame: ResponseCreate, + ) -> Result<(), StreamError> { + todo!( + "Coordinate authorization/quota, warmup, history, provider resolution, HTTP streaming and accounting for one turn" + ) + } + + pub async fn run( + self, + _host: &HostChannel, + ) -> Result { + todo!( + "Process sequential response.create turns until disconnect; cancel active generation and retain observed usage on exit" + ) + } +} diff --git a/litellm-rust/crates/core/src/streaming.rs b/litellm-rust/crates/core/src/streaming.rs new file mode 100644 index 00000000000..5d0ab3e1c1b --- /dev/null +++ b/litellm-rust/crates/core/src/streaming.rs @@ -0,0 +1,84 @@ +use std::marker::PhantomData; + +use litellm_host::{ + machine::{HostChannel, RouteMachine}, + route::Route, +}; +use litellm_llms::base_llm::base_model_iterator::{ + ApiDialect, DeliveryMode, ServerStreamingContract, StreamError, StreamOutcome, StreamPolicy, +}; + +pub trait StreamingCall: Send + Sync + 'static { + type Contract: ServerStreamingContract; + type Request: Send + 'static; + type Response: Send + 'static; +} + +pub struct StreamingRoute(PhantomData); + +pub struct StreamHead { + pub dialect: ApiDialect, + pub delivery: DeliveryMode, +} + +pub enum StreamOp { + ProjectRequest, + LoadCachedResponse { + key: String, + }, + PostStreamEvent { + event: ::Event, + }, + StoreResponse { + key: String, + response: C::Response, + }, + AccountOutcome { + outcome: StreamOutcome, + }, + PostStreamingHooks, +} + +pub enum StreamOpResult { + Request { + request: C::Request, + policy: StreamPolicy, + }, + CachedResponse(Option), + StoredResponse(C::Response), + AccountedOutcome(StreamOutcome), + Event(::Event), + Acknowledged, +} + +impl Route for StreamingRoute { + type Response = StreamOutcome; + type Error = StreamError; + type Op = StreamOp; + type OpResult = StreamOpResult; + type Chunk = ::Event; + type StreamHead = StreamHead; +} + +pub fn streaming_machine() -> RouteMachine> { + todo!( + "Project request, preserve acceptance gates, select native/bridged/buffered source, then drive host delivery and terminal accounting" + ) +} + +pub async fn deliver_stream( + _host: &HostChannel>, + _source: S, + _head: StreamHead, + _policy: StreamPolicy, +) -> Result, StreamError> +where + C: StreamingCall, + S: futures_util::Stream< + Item = Result<::Event, StreamError>, + > + Send, +{ + todo!( + "Honor host demand, bound buffers, validate the terminator, retain late/failed usage, and emit cache/hooks/accounting once; detachment is cancellation" + ) +} diff --git a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/mod.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/mod.rs index 1b0d2d351d8..b9f4e9bb4bc 100644 --- a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/mod.rs +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/mod.rs @@ -1,2 +1,3 @@ pub mod adapters; pub mod messages; +pub mod responses_adapters; diff --git a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/responses_adapters/mod.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/responses_adapters/mod.rs new file mode 100644 index 00000000000..0a64bfe97d9 --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/responses_adapters/mod.rs @@ -0,0 +1 @@ +pub mod streaming_iterator; diff --git a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.rs new file mode 100644 index 00000000000..58d48222999 --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.rs @@ -0,0 +1,80 @@ +use std::collections::{BTreeMap, VecDeque}; + +use litellm_types::responses::streaming::{ResponseOutputItem, ResponsesEvent}; + +use crate::{ + anthropic::experimental_pass_through::messages::streaming_iterator::{ + AnthropicContentBlock, AnthropicMessagesStreamEvent, + }, + base_llm::base_model_iterator::{ + AnthropicMessagesApi, ResponsesApi, StreamError, StreamPolicy, StreamTransformation, + }, +}; + +pub struct AnthropicResponsesStreamWrapper { + _model: String, + _message_id: String, + _current_block_index: Option, + _item_id_to_block_index: BTreeMap, + _pending_tool_ids: BTreeMap, + _sent_message_start: bool, + _sent_message_stop: bool, + _chunk_queue: VecDeque, + _refusal_text: String, + _policy: StreamPolicy, +} + +impl AnthropicResponsesStreamWrapper { + pub fn new(_model: String, _policy: StreamPolicy) -> Self { + todo!( + "Initialize one Responses-to-Messages conversion and record its message ID through the host" + ) + } + + pub fn make_message_start(&self) -> AnthropicMessagesStreamEvent { + todo!("Emit message_start once with model, stable ID and initial usage") + } + + pub fn next_block_index(&mut self) -> Result { + todo!("Allocate monotonically increasing content block indexes within stream limits") + } + + pub fn open_block( + &mut self, + _item_id: Option<&str>, + _content_block: AnthropicContentBlock, + ) -> Result { + todo!("Associate item ID and block index, then queue content_block_start before deltas") + } + + pub fn close_reasoning_item(&mut self, _item: &ResponseOutputItem) -> Result<(), StreamError> { + todo!("Preserve encrypted signatures or redacted thinking before content_block_stop") + } + + pub fn process_event(&mut self, _event: ResponsesEvent) -> Result<(), StreamError> { + todo!( + "Port _process_event: text/refusal, reasoning separators, tools and missing starts; retain failed/incomplete outcomes and terminal usage" + ) + } +} + +impl StreamTransformation for AnthropicResponsesStreamWrapper { + type Caller = AnthropicMessagesApi; + type Upstream = ResponsesApi; + type Error = StreamError; + + fn transform_event( + &mut self, + _event: ResponsesEvent, + ) -> Result, StreamError> { + todo!( + "Process one canonical Responses event and drain ordered Messages events within the queue limit" + ) + } + + fn finish(self) -> Result, StreamError> { + todo!( + "Require an observed terminal response, close outstanding blocks, emit final usage/stop once, and reject truncation" + ) + } +} diff --git a/litellm-rust/crates/llms/src/base_llm/base_model_iterator.rs b/litellm-rust/crates/llms/src/base_llm/base_model_iterator.rs index 3289cd1afc7..5939134e49e 100644 --- a/litellm-rust/crates/llms/src/base_llm/base_model_iterator.rs +++ b/litellm-rust/crates/llms/src/base_llm/base_model_iterator.rs @@ -17,6 +17,67 @@ pub struct ChatCompletions; pub struct ResponsesApi; pub struct AnthropicMessagesApi; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ApiDialect { + ChatCompletions, + Responses, + AnthropicMessages, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BufferedSource { + Cache, + NonStreamingResponse, + Mock, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DeliveryMode { + Live, + Buffered(BufferedSource), +} + +#[derive(Clone, Copy, Debug)] +pub struct StreamPolicy { + pub max_buffer_bytes: usize, + pub max_pending_events: usize, + pub max_tool_argument_bytes: usize, + pub deadline: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StreamError { + Upstream(String), + InvalidEvent(String), + UnexpectedEof, + BufferLimitExceeded, + Cancelled, + DeadlineExceeded, + Host(String), +} + +pub enum StreamOutcome { + Completed(Response), + Incomplete(Response), + Failed { + error: StreamError, + response: Option, + }, + Cancelled { + response: Option, + }, +} + +pub struct MatchingDialectRelay { + _dialect: ApiDialect, +} + +impl MatchingDialectRelay { + pub fn new(_caller: ApiDialect, _upstream: ApiDialect) -> Result { + todo!("Reject raw relay unless caller and upstream use the same semantic dialect") + } +} + impl ServerStreamingContract for ChatCompletions { type Event = ChatCompletionChunk; } @@ -47,10 +108,11 @@ pub struct TransformedStream { transformation: Option, pending: VecDeque<::Event>, finished: bool, + policy: StreamPolicy, } impl TransformedStream { - pub fn new(_upstream: S, _transformation: T) -> Self { + pub fn new(_upstream: S, _transformation: T, _policy: StreamPolicy) -> Self { todo!("Retain one call's conversion state and a bounded pending-event queue") } } diff --git a/litellm-rust/crates/llms/src/base_llm/responses/mod.rs b/litellm-rust/crates/llms/src/base_llm/responses/mod.rs index f239b6921fa..33997505d13 100644 --- a/litellm-rust/crates/llms/src/base_llm/responses/mod.rs +++ b/litellm-rust/crates/llms/src/base_llm/responses/mod.rs @@ -1 +1,3 @@ pub mod transformation; + +pub mod streaming; diff --git a/litellm-rust/crates/llms/src/base_llm/responses/streaming.rs b/litellm-rust/crates/llms/src/base_llm/responses/streaming.rs new file mode 100644 index 00000000000..dc501b9a78b --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/responses/streaming.rs @@ -0,0 +1,37 @@ +use litellm_framing::sse::SseFrame; +use litellm_types::responses::streaming::{ResponsesEvent, ResponsesResponse}; + +use crate::base_llm::base_model_iterator::{StreamError, StreamOutcome, StreamPolicy}; + +pub struct ResponsesSseDecoder { + _policy: StreamPolicy, + _terminal: Option>, +} + +impl ResponsesSseDecoder { + pub fn new(_policy: StreamPolicy) -> Self { + todo!( + "Initialize semantic decoding downstream of SseFramer, which owns split UTF-8 and frame assembly" + ) + } + + pub fn decode(&mut self, _frame: SseFrame) -> Result, StreamError> { + todo!( + "Decode canonical events, skip SSE control frames, handle DONE, preserve extensions and usage, and enforce event/buffer limits" + ) + } + + pub fn finish(self) -> Result, StreamError> { + todo!( + "Require a terminal Responses event; EOF or DONE alone cannot fabricate completed output" + ) + } +} + +pub fn decode_response(_body: &[u8]) -> Result { + todo!("Validate the full wire schema and retain provider extensions in the typed response") +} + +pub fn encode_event(_event: &ResponsesEvent) -> Result { + todo!("Serialize canonical event names and payload fields for SSE or WebSocket delivery") +} diff --git a/litellm-rust/crates/types/src/responses/streaming.rs b/litellm-rust/crates/types/src/responses/streaming.rs index 092f82356c5..88ddba13ee0 100644 --- a/litellm-rust/crates/types/src/responses/streaming.rs +++ b/litellm-rust/crates/types/src/responses/streaming.rs @@ -1,16 +1,325 @@ -// TODO: Port the complete Python Responses models before enabling these streaming contracts -pub struct ResponsesResponse { - _schema: (), +use std::collections::BTreeMap; + +use serde_json::Value; + +#[derive(Clone, Debug, PartialEq)] +pub struct ProviderFields(pub BTreeMap); + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ResponseStatus { + Queued, + InProgress, + Completed, + Failed, + Incomplete, + Cancelled, } -pub struct ResponseOutputItem { - _schema: (), +#[derive(Clone, Debug, PartialEq)] +pub struct ResponsesUsage { + pub input_tokens: u64, + pub output_tokens: u64, + pub total_tokens: u64, + pub cached_tokens: Option, + pub reasoning_tokens: Option, + pub input_token_details: ProviderFields, + pub output_token_details: ProviderFields, + pub extensions: ProviderFields, } +#[derive(Clone, Debug, PartialEq)] +pub struct ResponsesError { + pub code: Option, + pub message: String, + pub param: Option, + pub extensions: ProviderFields, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ResponseAnnotation { + pub kind: String, + pub start_index: Option, + pub end_index: Option, + pub url: Option, + pub title: Option, + pub extensions: ProviderFields, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum ResponseContentPart { + OutputText { + text: String, + annotations: Box<[ResponseAnnotation]>, + }, + Refusal { + refusal: String, + }, + SummaryText { + text: String, + }, + Extension { + kind: String, + fields: ProviderFields, + }, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum ResponseOutputItem { + Message { + id: String, + status: ResponseStatus, + role: String, + content: Box<[ResponseContentPart]>, + phase: Option, + extensions: ProviderFields, + }, + FunctionCall { + id: String, + call_id: String, + name: String, + namespace: Option, + arguments: String, + status: ResponseStatus, + extensions: ProviderFields, + }, + CustomToolCall { + id: String, + call_id: String, + name: String, + input: String, + status: Option, + extensions: ProviderFields, + }, + Reasoning { + id: String, + summary: Box<[ResponseContentPart]>, + encrypted_content: Option, + extensions: ProviderFields, + }, + Extension { + kind: String, + id: Option, + fields: ProviderFields, + }, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum ResponseInputContent { + Text(String), + Image { + image_url: Option, + file_id: Option, + detail: Option, + }, + File { + file_id: Option, + file_data: Option, + filename: Option, + }, + Extension { + kind: String, + fields: ProviderFields, + }, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum ResponseInputItem { + Message { + role: String, + content: Box<[ResponseInputContent]>, + }, + FunctionCallOutput { + call_id: String, + output: String, + }, + CustomToolCallOutput { + call_id: String, + output: String, + }, + ItemReference { + id: String, + }, + Output(ResponseOutputItem), + Extension { + kind: String, + fields: ProviderFields, + }, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum ResponsesInput { + Text(String), + Items(Box<[ResponseInputItem]>), +} + +#[derive(Clone, Debug, PartialEq)] +pub enum ResponseTool { + Function { + name: String, + description: Option, + parameters: ProviderFields, + strict: Option, + }, + Custom { + name: String, + description: Option, + format: ProviderFields, + }, + Namespace { + name: String, + description: Option, + tools: Box<[ResponseTool]>, + }, + BuiltIn { + kind: String, + options: ProviderFields, + }, +} + +#[derive(Clone, Debug, PartialEq)] pub struct ResponsesRequest { - _schema: (), + pub model: Option, + pub input: Option, + pub previous_response_id: Option, + pub instructions: Option, + pub tools: Box<[ResponseTool]>, + pub parallel_tool_calls: Option, + pub max_output_tokens: Option, + pub store: Option, + pub background: Option, + pub stream: Option, + pub include: Box<[String]>, + pub metadata: BTreeMap, + // TODO: Type remaining Python request options before enabling wire decoding; preserve them losslessly meanwhile + pub extensions: ProviderFields, } -pub struct ResponsesEvent { - _schema: (), +#[derive(Clone, Debug, PartialEq)] +pub struct ResponsesResponse { + pub id: String, + pub created_at: u64, + pub model: Option, + pub status: Option, + pub output: Box<[ResponseOutputItem]>, + pub usage: Option, + pub error: Option, + pub incomplete_reason: Option, + pub previous_response_id: Option, + pub extensions: ProviderFields, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ResponsesEvent { + pub sequence_number: Option, + pub data: ResponsesEventData, + pub extensions: ProviderFields, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum ResponsesEventData { + Created(ResponsesResponse), + InProgress(ResponsesResponse), + Completed(ResponsesResponse), + Failed(ResponsesResponse), + Incomplete(ResponsesResponse), + OutputItemAdded { + output_index: u64, + item: ResponseOutputItem, + }, + OutputItemDone { + output_index: u64, + item: ResponseOutputItem, + }, + ContentPartAdded { + item_id: String, + output_index: u64, + content_index: u64, + part: ResponseContentPart, + }, + ContentPartDone { + item_id: String, + output_index: u64, + content_index: u64, + part: ResponseContentPart, + }, + OutputTextDelta { + item_id: String, + output_index: u64, + content_index: u64, + delta: String, + }, + OutputTextDone { + item_id: String, + output_index: u64, + content_index: u64, + text: String, + }, + AnnotationAdded { + item_id: String, + output_index: u64, + content_index: u64, + annotation_index: u64, + annotation: ResponseAnnotation, + }, + RefusalDelta { + item_id: String, + output_index: u64, + content_index: u64, + delta: String, + }, + RefusalDone { + item_id: String, + output_index: u64, + content_index: u64, + refusal: String, + }, + FunctionCallArgumentsDelta { + item_id: String, + output_index: u64, + delta: String, + }, + FunctionCallArgumentsDone { + item_id: String, + output_index: u64, + arguments: String, + }, + CustomToolCallInputDelta { + item_id: String, + output_index: u64, + delta: String, + }, + CustomToolCallInputDone { + item_id: String, + output_index: u64, + input: String, + }, + ReasoningSummaryPartAdded { + item_id: String, + output_index: u64, + summary_index: u64, + part: ResponseContentPart, + }, + ReasoningSummaryPartDone { + item_id: String, + output_index: u64, + summary_index: u64, + part: ResponseContentPart, + }, + ReasoningSummaryTextDelta { + item_id: String, + output_index: u64, + summary_index: u64, + delta: String, + }, + ReasoningSummaryTextDone { + item_id: String, + output_index: u64, + summary_index: u64, + text: String, + }, + Error(ResponsesError), + Extension { + kind: String, + fields: ProviderFields, + }, } diff --git a/litellm-rust/crates/types/src/responses/streaming_websocket.rs b/litellm-rust/crates/types/src/responses/streaming_websocket.rs index 4942309992e..bdfbdf621ab 100644 --- a/litellm-rust/crates/types/src/responses/streaming_websocket.rs +++ b/litellm-rust/crates/types/src/responses/streaming_websocket.rs @@ -1,6 +1,11 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::{Map, Value}; +pub struct ResponsesWebSocketRequestDefaults { + pub fill_missing: crate::responses::streaming::ProviderFields, + pub overrides: crate::responses::streaming::ProviderFields, +} + #[derive(Clone, Debug, PartialEq, Eq)] pub enum ResponsesWsEventType { ResponseCreate,