refactor(rust): scaffold response sessions and streaming routes

This commit is contained in:
Yujong Lee 2026-09-20 18:45:32 -07:00
parent a545e6ab6d
commit 39aa0a29d3
20 changed files with 1106 additions and 41 deletions

View file

@ -7,6 +7,7 @@
//! calls the provider, and returns a typed OpenAI-shaped response.
mod error;
pub mod streaming;
pub mod types;
pub use error::Error;
mod client;

View file

@ -0,0 +1,29 @@
use litellm_host::machine::RouteMachine;
use litellm_llms::base_llm::base_model_iterator::ChatCompletions;
use crate::streaming::{StreamingCall, StreamingRoute};
pub struct ChatCompletionsCall {
pub model: String,
pub messages: Box<[litellm_types::llms::openai::ChatMessage]>,
pub optional_params: std::collections::BTreeMap<String, serde_json::Value>,
pub custom_llm_provider: Option<String>,
pub api_key: Option<litellm_auth::SecretValue>,
pub api_base: Option<String>,
pub extra_headers: Box<[(String, String)]>,
pub timeout: Option<std::time::Duration>,
}
pub struct Streaming;
impl StreamingCall for Streaming {
type Contract = ChatCompletions;
type Request = ChatCompletionsCall;
type Response = litellm_types::utils::ChatCompletionsResponse;
}
pub fn chat_completions_stream_machine() -> RouteMachine<StreamingRoute<Streaming>> {
todo!(
"Select a typed native or translated stream and preserve existing provider acceptance gates until the streaming path is implemented"
)
}

View file

@ -6,5 +6,6 @@ pub mod messages;
pub mod ocr;
mod outbound;
pub mod responses;
pub mod streaming;
pub use error::Error;

View file

@ -5,6 +5,7 @@
//! it in process for a caller that already holds the request and wants the message.
mod error;
pub mod streaming;
pub mod types;
pub use error::Error;
mod client;

View file

@ -0,0 +1,37 @@
use futures_util::Stream;
use litellm_host::machine::RouteMachine;
use litellm_llms::{
anthropic::experimental_pass_through::responses_adapters::streaming_iterator::AnthropicResponsesStreamWrapper,
base_llm::base_model_iterator::{
AnthropicMessagesApi, StreamError, StreamPolicy, TransformedStream,
},
};
use litellm_types::responses::streaming::ResponsesEvent;
use crate::streaming::{StreamingCall, StreamingRoute};
pub struct Streaming;
impl StreamingCall for Streaming {
type Contract = AnthropicMessagesApi;
type Request = super::route::MessagesCall;
type Response =
litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse;
}
pub fn messages_stream_machine() -> RouteMachine<StreamingRoute<Streaming>> {
todo!(
"Select a typed native or translated stream and preserve existing provider acceptance gates until the streaming path is implemented"
)
}
pub fn messages_from_responses<S>(
_source: S,
_adapter: AnthropicResponsesStreamWrapper,
_policy: StreamPolicy,
) -> TransformedStream<S, AnthropicResponsesStreamWrapper>
where
S: Stream<Item = Result<ResponsesEvent, StreamError>>,
{
todo!("Compose canonical Responses events with the Messages bridge before typed host delivery")
}

View file

@ -1,9 +1,9 @@
use futures_util::Stream;
use litellm_llms::base_llm::base_model_iterator::TransformedStream;
use litellm_llms::base_llm::base_model_iterator::{StreamError, StreamPolicy, TransformedStream};
use litellm_types::utils::ChatCompletionChunk;
use super::streaming_iterator::LiteLlmCompletionStreamingIterator;
use crate::responses::streaming_iterator::{ResponsesApiStreamingIterator, ResponsesStreamError};
use crate::responses::streaming_iterator::ResponsesApiStreamingIterator;
pub struct LiteLlmCompletionTransformationHandler;
@ -11,9 +11,10 @@ impl LiteLlmCompletionTransformationHandler {
pub fn streaming_response<S>(
_chat_stream: S,
_adapter: LiteLlmCompletionStreamingIterator,
_policy: StreamPolicy,
) -> ResponsesApiStreamingIterator<TransformedStream<S, LiteLlmCompletionStreamingIterator>>
where
S: Stream<Item = Result<ChatCompletionChunk, ResponsesStreamError>>,
S: Stream<Item = Result<ChatCompletionChunk, StreamError>>,
{
todo!(
"Compose the transformed Chat source inside the same driver used for native Responses events"

View file

@ -1,5 +1,5 @@
use litellm_llms::base_llm::base_model_iterator::{
ChatCompletions, ResponsesApi, StreamTransformation,
ChatCompletions, ResponsesApi, StreamError, StreamTransformation,
};
use litellm_types::{
llms::openai::ChatCompletionToolCallChunk,
@ -9,8 +9,6 @@ use litellm_types::{
utils::ChatCompletionChunk,
};
use crate::responses::streaming_iterator::ResponsesStreamError;
pub struct LiteLlmCompletionStreamingIterator {
// TODO: Port per-call item/tool associations, request options, bounded accumulators, and reasoning state from Python
_state: (),
@ -32,7 +30,7 @@ impl LiteLlmCompletionStreamingIterator {
pub fn queue_tool_call_delta_events(
&mut self,
_tool_calls: &[ChatCompletionToolCallChunk],
) -> Result<(), ResponsesStreamError> {
) -> Result<(), StreamError> {
todo!(
"Associate fragmented tool indexes and call IDs; preserve custom/namespaced tools, arguments, item IDs and output indexes"
)
@ -48,7 +46,7 @@ impl LiteLlmCompletionStreamingIterator {
pub fn transform_chat_completion_chunk_to_response_api_chunk(
&mut self,
_chunk: ChatCompletionChunk,
) -> Result<Vec<ResponsesEvent>, ResponsesStreamError> {
) -> Result<Vec<ResponsesEvent>, StreamError> {
todo!(
"Convert text, tools, reasoning, annotations and provider web-search fields while preserving sequence numbers"
)
@ -60,15 +58,11 @@ impl LiteLlmCompletionStreamingIterator {
)
}
pub fn return_default_done_events(
&mut self,
) -> Result<Vec<ResponsesEvent>, ResponsesStreamError> {
pub fn return_default_done_events(&mut self) -> Result<Vec<ResponsesEvent>, StreamError> {
todo!("Close text, content, reasoning and tool items in order before the terminal response")
}
pub fn emit_response_completed_event(
&mut self,
) -> Result<ResponsesEvent, ResponsesStreamError> {
pub fn emit_response_completed_event(&mut self) -> Result<ResponsesEvent, StreamError> {
todo!(
"Build the final response from accumulated chunks and late usage, retaining encoded IDs and request options"
)
@ -78,7 +72,7 @@ impl LiteLlmCompletionStreamingIterator {
impl StreamTransformation for LiteLlmCompletionStreamingIterator {
type Caller = ResponsesApi;
type Upstream = ChatCompletions;
type Error = ResponsesStreamError;
type Error = StreamError;
fn transform_event(
&mut self,

View file

@ -1,5 +1,15 @@
mod error;
pub use error::Error;
pub mod litellm_completion_transformation;
pub mod route;
pub mod streaming_iterator;
pub mod websocket;
pub mod websocket_session;
use litellm_types::responses::streaming::ResponsesResponse;
pub async fn responses(_call: route::ResponsesCall) -> Result<ResponsesResponse, Error> {
todo!(
"Resolve native or bridged provider, prepare/send HTTP request, and normalize the response through route host operations"
)
}

View file

@ -0,0 +1,31 @@
use std::time::Duration;
use litellm_auth::SecretValue;
use litellm_host::machine::RouteMachine;
use litellm_llms::base_llm::base_model_iterator::ResponsesApi;
use litellm_types::responses::streaming::{ResponsesRequest, ResponsesResponse};
use crate::streaming::{StreamingCall, StreamingRoute};
pub struct ResponsesCall {
pub request: ResponsesRequest,
pub custom_llm_provider: Option<String>,
pub api_key: Option<SecretValue>,
pub api_base: Option<String>,
pub extra_headers: Box<[(String, String)]>,
pub timeout: Option<Duration>,
}
pub struct Responses;
impl StreamingCall for Responses {
type Contract = ResponsesApi;
type Request = ResponsesCall;
type Response = ResponsesResponse;
}
pub fn responses_machine() -> RouteMachine<StreamingRoute<Responses>> {
todo!(
"Compose native Responses or Chat-to-Responses conversion with the shared Responses driver and host operations"
)
}

View file

@ -1,3 +1,6 @@
use litellm_llms::base_llm::base_model_iterator::{
BufferedSource, StreamError, StreamOutcome, StreamPolicy,
};
use std::{
pin::Pin,
task::{Context, Poll},
@ -6,36 +9,28 @@ use std::{
use futures_util::Stream;
use litellm_types::responses::streaming::{ResponsesEvent, ResponsesResponse};
pub enum ResponsesStreamError {
Upstream(String),
InvalidEvent(String),
UnexpectedEof,
BufferLimitExceeded,
Cancelled,
DeadlineExceeded,
}
pub enum ResponsesStreamState {
Pending,
Active,
Terminal(ResponsesResponse),
Failed(ResponsesStreamError),
Terminal(StreamOutcome<ResponsesResponse>),
Failed(StreamError),
}
pub struct ResponsesApiStreamingIterator<S> {
source: S,
state: ResponsesStreamState,
policy: StreamPolicy,
}
impl<S> ResponsesApiStreamingIterator<S> {
pub fn new(_source: S) -> Self {
pub fn new(_source: S, _policy: StreamPolicy) -> Self {
todo!("Compose BaseResponsesAPIStreamingIterator lifecycle state with a typed event source")
}
pub fn process_chunk(
&mut self,
_event: ResponsesEvent,
) -> Result<Option<ResponsesEvent>, ResponsesStreamError> {
) -> Result<Option<ResponsesEvent>, StreamError> {
todo!(
"Port base _process_chunk after provider decoding: response IDs, completed/failed/incomplete outcomes, usage, and error events"
)
@ -50,11 +45,39 @@ impl<S> ResponsesApiStreamingIterator<S> {
}
}
pub struct BufferedResponsesStream {
_response: ResponsesResponse,
_source: BufferedSource,
_policy: StreamPolicy,
}
impl BufferedResponsesStream {
pub fn new(
_response: ResponsesResponse,
_source: BufferedSource,
_policy: StreamPolicy,
) -> Self {
todo!(
"Port cached/mock/simulated response event generation with explicit buffered delivery and stable item IDs"
)
}
}
impl Stream for BufferedResponsesStream {
type Item = Result<ResponsesEvent, StreamError>;
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
todo!(
"Emit bounded ordered lifecycle/content/tool events from the retained response, preserving its terminal status and usage"
)
}
}
impl<S> Stream for ResponsesApiStreamingIterator<S>
where
S: Stream<Item = Result<ResponsesEvent, ResponsesStreamError>>,
S: Stream<Item = Result<ResponsesEvent, StreamError>>,
{
type Item = Result<ResponsesEvent, ResponsesStreamError>;
type Item = Result<ResponsesEvent, StreamError>;
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
todo!(
@ -77,9 +100,9 @@ impl<S> NativeResponsesStream<S> {
impl<S> Stream for NativeResponsesStream<S>
where
S: Stream<Item = Result<ResponsesEvent, ResponsesStreamError>>,
S: Stream<Item = Result<ResponsesEvent, StreamError>>,
{
type Item = Result<ResponsesEvent, ResponsesStreamError>;
type Item = Result<ResponsesEvent, StreamError>;
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
todo!(

View file

@ -0,0 +1,356 @@
use std::collections::BTreeMap;
use litellm_host::{
machine::{HostChannel, RouteMachine},
route::Route,
};
use litellm_llms::base_llm::base_model_iterator::{StreamError, StreamOutcome, StreamPolicy};
use litellm_types::responses::streaming::{
ResponseInputItem, ResponsesError, ResponsesEvent, ResponsesInput, ResponsesRequest,
ResponsesResponse,
};
use litellm_types::responses::streaming_websocket::ResponsesWebSocketRequestDefaults;
use super::{route::ResponsesCall, websocket::ResponsesWebSocketConnection};
pub struct ResponseCreate {
pub request: ResponsesRequest,
pub generate: Option<bool>,
}
pub enum ClientFrame {
Create(ResponseCreate),
Closed,
}
pub struct ResponsesWebSocketConfig {
pub call: ResponsesCall,
pub authorized_model: String,
pub first_message: Option<String>,
pub request_defaults: ResponsesWebSocketRequestDefaults,
pub stream_policy: StreamPolicy,
pub max_history_bytes: usize,
pub max_history_turns: usize,
}
pub enum ResponsesWebSocketState {
Idle,
Warmup { response_id: String },
Active { response_id: Option<String> },
Closed,
Failed(StreamError),
}
pub enum WebSocketOutcome {
Closed,
Cancelled,
Failed(StreamError),
}
pub enum InputPolicyResult {
Approved(ResponsesRequest),
Rejected(ResponsesError),
}
pub enum WebSocketOp {
ReceiveClientFrame,
SendClientFrame(String),
ApplyInputPolicy {
request: ResponsesRequest,
authorized_model: String,
},
EnforceQuota {
request: ResponsesRequest,
},
ApplyOutputPolicy(ResponsesEvent),
LoadHistory {
response_id: String,
},
StoreHistory {
response_id: String,
items: Box<[ResponseInputItem]>,
},
AccountTurn(StreamOutcome<ResponsesResponse>),
PostStreamingHooks,
}
pub enum WebSocketOpResult {
ClientFrame(Option<String>),
InputPolicy(InputPolicyResult),
Quota(Result<(), ResponsesError>),
OutputEvent(ResponsesEvent),
History(Box<[ResponseInputItem]>),
Acknowledged,
}
pub struct ResponsesWebSocket;
impl Route for ResponsesWebSocket {
type Response = WebSocketOutcome;
type Error = StreamError;
type Op = WebSocketOp;
type OpResult = WebSocketOpResult;
type Chunk = std::convert::Infallible;
type StreamHead = ();
}
pub enum ResponsesWebSocketMode {
Native(ResponsesWebSocketConnection),
Managed,
}
pub fn responses_websocket_machine(
_config: ResponsesWebSocketConfig,
_mode: ResponsesWebSocketMode,
) -> RouteMachine<ResponsesWebSocket> {
todo!(
"Select native or managed session driver; keep client socket I/O, policies, storage and callback execution behind host operations"
)
}
pub fn parse_message(_raw_message: &str) -> Result<ClientFrame, StreamError> {
todo!(
"Validate response.create in flat or nested form, including generate=false and previous_response_id; reject unsupported client events"
)
}
pub struct ResponsesWebSocketStreaming {
_backend_ws: ResponsesWebSocketConnection,
_config: ResponsesWebSocketConfig,
_state: ResponsesWebSocketState,
_terminal: Option<StreamOutcome<ResponsesResponse>>,
}
impl ResponsesWebSocketStreaming {
pub fn new(
_backend_ws: ResponsesWebSocketConnection,
_config: ResponsesWebSocketConfig,
) -> Self {
todo!(
"Initialize a native session with first-frame handling and one active turn per connection"
)
}
pub fn enforce_authorized_model(&self, _request: &ResponsesRequest) -> Result<(), StreamError> {
todo!("Check the authorized model on every response.create, including nested requests")
}
pub fn with_request_defaults(
&self,
_request: ResponsesRequest,
) -> Result<ResponsesRequest, StreamError> {
todo!(
"Merge fill_missing defaults, then per-turn parameters, then deployment overrides in Python's precedence order"
)
}
pub fn collect_input_from_client_event(
&mut self,
_request: &ResponsesRequest,
) -> Result<(), StreamError> {
todo!("Accumulate per-turn input for accounting within session limits")
}
pub fn store_event(&mut self, _event: &ResponsesEvent) -> Result<(), StreamError> {
todo!(
"Retain required lifecycle/usage events within bounds, resetting accumulation between turns"
)
}
pub async fn enforce_or_reject_frame(
&mut self,
_host: &HostChannel<ResponsesWebSocket>,
_frame: ResponseCreate,
) -> Result<InputPolicyResult, StreamError> {
todo!(
"Enforce model, request defaults, host input masking and per-turn quota before forwarding"
)
}
pub async fn backend_to_client(
&mut self,
_host: &HostChannel<ResponsesWebSocket>,
) -> Result<(), StreamError> {
todo!(
"Decode backend events, apply host output policies, forward frames, and account completed/failed/incomplete turns exactly once"
)
}
pub async fn client_to_backend(
&mut self,
_host: &HostChannel<ResponsesWebSocket>,
) -> Result<(), StreamError> {
todo!(
"Read first/subsequent frames through host operations and validate each turn before upstream send"
)
}
pub async fn log_messages(
&mut self,
_host: &HostChannel<ResponsesWebSocket>,
) -> Result<(), StreamError> {
todo!(
"Yield turn accounting and post-stream hooks without running callbacks or writing storage in core"
)
}
pub async fn record_usage_for_failure(
&mut self,
_host: &HostChannel<ResponsesWebSocket>,
) -> Result<(), StreamError> {
todo!(
"Preserve observed usage after failures, disconnects and cancellation without double accounting"
)
}
pub async fn bidirectional_forward(
self,
_host: &HostChannel<ResponsesWebSocket>,
) -> Result<WebSocketOutcome, StreamError> {
todo!(
"Split independent read/write ownership, coordinate both directions with backpressure, cancel peers and close upstream on exit; created is not turn completion"
)
}
}
pub struct ManagedResponsesWebSocketHandler {
_config: ResponsesWebSocketConfig,
_state: ResponsesWebSocketState,
_session_history: BTreeMap<String, Box<[ResponseInputItem]>>,
}
impl ManagedResponsesWebSocketHandler {
pub fn new(_config: ResponsesWebSocketConfig) -> Self {
todo!("Initialize bounded per-connection history and sequential HTTP-backed turns")
}
pub fn serialize_chunk(_event: &ResponsesEvent) -> Result<String, StreamError> {
todo!("Use the canonical Responses event encoder shared with SSE")
}
pub fn get_history_messages(
&self,
_previous_response_id: &str,
) -> Option<&[ResponseInputItem]> {
todo!(
"Look up decoded IDs in session history before requesting persistent history from the host"
)
}
pub fn store_history(
&mut self,
_response_id: String,
_items: Box<[ResponseInputItem]>,
) -> Result<(), StreamError> {
todo!(
"Enforce history byte/turn limits and preserve follow-up availability before asynchronous host persistence"
)
}
pub fn input_to_messages(_input: ResponsesInput) -> Box<[ResponseInputItem]> {
todo!(
"Normalize text and typed input items without losing tools, reasoning or multimodal content"
)
}
pub fn is_warmup_frame(_frame: &ResponseCreate) -> bool {
todo!("Recognize generate=false without issuing a provider generation request")
}
pub fn is_warmup_response_id(_response_id: &str) -> bool {
todo!("Recognize synthetic warmup IDs so they are not replayed as provider response IDs")
}
pub fn build_warmup_response(&self, _frame: &ResponseCreate) -> ResponsesResponse {
todo!("Construct a synthetic warmup acknowledgement using the request's model and options")
}
pub fn build_base_call_kwargs(_frame: ResponseCreate) -> ResponsesRequest {
todo!("Extract supported Responses parameters while excluding WebSocket-only fields")
}
pub fn apply_history(
&self,
_request: ResponsesRequest,
_prior_history: &[ResponseInputItem],
) -> Result<ResponsesRequest, StreamError> {
todo!(
"Prepend session or host history without duplicating input or forwarding synthetic warmup IDs"
)
}
pub fn same_provider(&self, _model: &str) -> Result<bool, StreamError> {
todo!(
"Resolve the turn provider before deciding whether connection credentials can be reused"
)
}
pub fn inject_credentials(
&self,
_request: ResponsesRequest,
) -> Result<ResponsesCall, StreamError> {
todo!(
"Reuse connection credentials only for the same provider and preserve per-turn routing"
)
}
pub async fn send_error(
&self,
_host: &HostChannel<ResponsesWebSocket>,
_error: ResponsesError,
) -> Result<(), StreamError> {
todo!(
"Send a structured error frame through the host, preserving session versus turn failure"
)
}
pub async fn send_warmup_ack(
&mut self,
_host: &HostChannel<ResponsesWebSocket>,
_frame: ResponseCreate,
) -> Result<(), StreamError> {
todo!(
"Send warmup acknowledgement without generation, usage fabrication or history pollution"
)
}
pub async fn stream_and_forward(
&mut self,
_host: &HostChannel<ResponsesWebSocket>,
_call: ResponsesCall,
) -> Result<StreamOutcome<ResponsesResponse>, StreamError> {
todo!(
"Drive the HTTP Responses route and forward canonical events with host output policy, backpressure, cancellation and terminal validation"
)
}
pub async fn save_turn_history(
&mut self,
_host: &HostChannel<ResponsesWebSocket>,
_request: ResponsesRequest,
_response: ResponsesResponse,
) -> Result<(), StreamError> {
todo!(
"Save normalized input and output under the decoded response ID, then yield host persistence"
)
}
pub async fn process_response_create(
&mut self,
_host: &HostChannel<ResponsesWebSocket>,
_frame: ResponseCreate,
) -> Result<(), StreamError> {
todo!(
"Coordinate authorization/quota, warmup, history, provider resolution, HTTP streaming and accounting for one turn"
)
}
pub async fn run(
self,
_host: &HostChannel<ResponsesWebSocket>,
) -> Result<WebSocketOutcome, StreamError> {
todo!(
"Process sequential response.create turns until disconnect; cancel active generation and retain observed usage on exit"
)
}
}

View file

@ -0,0 +1,84 @@
use std::marker::PhantomData;
use litellm_host::{
machine::{HostChannel, RouteMachine},
route::Route,
};
use litellm_llms::base_llm::base_model_iterator::{
ApiDialect, DeliveryMode, ServerStreamingContract, StreamError, StreamOutcome, StreamPolicy,
};
pub trait StreamingCall: Send + Sync + 'static {
type Contract: ServerStreamingContract<Event: Send + 'static>;
type Request: Send + 'static;
type Response: Send + 'static;
}
pub struct StreamingRoute<C: StreamingCall>(PhantomData<C>);
pub struct StreamHead {
pub dialect: ApiDialect,
pub delivery: DeliveryMode,
}
pub enum StreamOp<C: StreamingCall> {
ProjectRequest,
LoadCachedResponse {
key: String,
},
PostStreamEvent {
event: <C::Contract as ServerStreamingContract>::Event,
},
StoreResponse {
key: String,
response: C::Response,
},
AccountOutcome {
outcome: StreamOutcome<C::Response>,
},
PostStreamingHooks,
}
pub enum StreamOpResult<C: StreamingCall> {
Request {
request: C::Request,
policy: StreamPolicy,
},
CachedResponse(Option<C::Response>),
StoredResponse(C::Response),
AccountedOutcome(StreamOutcome<C::Response>),
Event(<C::Contract as ServerStreamingContract>::Event),
Acknowledged,
}
impl<C: StreamingCall> Route for StreamingRoute<C> {
type Response = StreamOutcome<C::Response>;
type Error = StreamError;
type Op = StreamOp<C>;
type OpResult = StreamOpResult<C>;
type Chunk = <C::Contract as ServerStreamingContract>::Event;
type StreamHead = StreamHead;
}
pub fn streaming_machine<C: StreamingCall>() -> RouteMachine<StreamingRoute<C>> {
todo!(
"Project request, preserve acceptance gates, select native/bridged/buffered source, then drive host delivery and terminal accounting"
)
}
pub async fn deliver_stream<C, S>(
_host: &HostChannel<StreamingRoute<C>>,
_source: S,
_head: StreamHead,
_policy: StreamPolicy,
) -> Result<StreamOutcome<C::Response>, StreamError>
where
C: StreamingCall,
S: futures_util::Stream<
Item = Result<<C::Contract as ServerStreamingContract>::Event, StreamError>,
> + Send,
{
todo!(
"Honor host demand, bound buffers, validate the terminator, retain late/failed usage, and emit cache/hooks/accounting once; detachment is cancellation"
)
}

View file

@ -1,2 +1,3 @@
pub mod adapters;
pub mod messages;
pub mod responses_adapters;

View file

@ -0,0 +1 @@
pub mod streaming_iterator;

View file

@ -0,0 +1,80 @@
use std::collections::{BTreeMap, VecDeque};
use litellm_types::responses::streaming::{ResponseOutputItem, ResponsesEvent};
use crate::{
anthropic::experimental_pass_through::messages::streaming_iterator::{
AnthropicContentBlock, AnthropicMessagesStreamEvent,
},
base_llm::base_model_iterator::{
AnthropicMessagesApi, ResponsesApi, StreamError, StreamPolicy, StreamTransformation,
},
};
pub struct AnthropicResponsesStreamWrapper {
_model: String,
_message_id: String,
_current_block_index: Option<u64>,
_item_id_to_block_index: BTreeMap<String, u64>,
_pending_tool_ids: BTreeMap<String, String>,
_sent_message_start: bool,
_sent_message_stop: bool,
_chunk_queue: VecDeque<AnthropicMessagesStreamEvent>,
_refusal_text: String,
_policy: StreamPolicy,
}
impl AnthropicResponsesStreamWrapper {
pub fn new(_model: String, _policy: StreamPolicy) -> Self {
todo!(
"Initialize one Responses-to-Messages conversion and record its message ID through the host"
)
}
pub fn make_message_start(&self) -> AnthropicMessagesStreamEvent {
todo!("Emit message_start once with model, stable ID and initial usage")
}
pub fn next_block_index(&mut self) -> Result<u64, StreamError> {
todo!("Allocate monotonically increasing content block indexes within stream limits")
}
pub fn open_block(
&mut self,
_item_id: Option<&str>,
_content_block: AnthropicContentBlock,
) -> Result<u64, StreamError> {
todo!("Associate item ID and block index, then queue content_block_start before deltas")
}
pub fn close_reasoning_item(&mut self, _item: &ResponseOutputItem) -> Result<(), StreamError> {
todo!("Preserve encrypted signatures or redacted thinking before content_block_stop")
}
pub fn process_event(&mut self, _event: ResponsesEvent) -> Result<(), StreamError> {
todo!(
"Port _process_event: text/refusal, reasoning separators, tools and missing starts; retain failed/incomplete outcomes and terminal usage"
)
}
}
impl StreamTransformation for AnthropicResponsesStreamWrapper {
type Caller = AnthropicMessagesApi;
type Upstream = ResponsesApi;
type Error = StreamError;
fn transform_event(
&mut self,
_event: ResponsesEvent,
) -> Result<Vec<AnthropicMessagesStreamEvent>, StreamError> {
todo!(
"Process one canonical Responses event and drain ordered Messages events within the queue limit"
)
}
fn finish(self) -> Result<Vec<AnthropicMessagesStreamEvent>, StreamError> {
todo!(
"Require an observed terminal response, close outstanding blocks, emit final usage/stop once, and reject truncation"
)
}
}

View file

@ -17,6 +17,67 @@ pub struct ChatCompletions;
pub struct ResponsesApi;
pub struct AnthropicMessagesApi;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ApiDialect {
ChatCompletions,
Responses,
AnthropicMessages,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum BufferedSource {
Cache,
NonStreamingResponse,
Mock,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum DeliveryMode {
Live,
Buffered(BufferedSource),
}
#[derive(Clone, Copy, Debug)]
pub struct StreamPolicy {
pub max_buffer_bytes: usize,
pub max_pending_events: usize,
pub max_tool_argument_bytes: usize,
pub deadline: Option<std::time::Instant>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum StreamError {
Upstream(String),
InvalidEvent(String),
UnexpectedEof,
BufferLimitExceeded,
Cancelled,
DeadlineExceeded,
Host(String),
}
pub enum StreamOutcome<Response> {
Completed(Response),
Incomplete(Response),
Failed {
error: StreamError,
response: Option<Response>,
},
Cancelled {
response: Option<Response>,
},
}
pub struct MatchingDialectRelay {
_dialect: ApiDialect,
}
impl MatchingDialectRelay {
pub fn new(_caller: ApiDialect, _upstream: ApiDialect) -> Result<Self, StreamError> {
todo!("Reject raw relay unless caller and upstream use the same semantic dialect")
}
}
impl ServerStreamingContract for ChatCompletions {
type Event = ChatCompletionChunk;
}
@ -47,10 +108,11 @@ pub struct TransformedStream<S, T: StreamTransformation> {
transformation: Option<T>,
pending: VecDeque<<T::Caller as ServerStreamingContract>::Event>,
finished: bool,
policy: StreamPolicy,
}
impl<S, T: StreamTransformation> TransformedStream<S, T> {
pub fn new(_upstream: S, _transformation: T) -> Self {
pub fn new(_upstream: S, _transformation: T, _policy: StreamPolicy) -> Self {
todo!("Retain one call's conversion state and a bounded pending-event queue")
}
}

View file

@ -1 +1,3 @@
pub mod transformation;
pub mod streaming;

View file

@ -0,0 +1,37 @@
use litellm_framing::sse::SseFrame;
use litellm_types::responses::streaming::{ResponsesEvent, ResponsesResponse};
use crate::base_llm::base_model_iterator::{StreamError, StreamOutcome, StreamPolicy};
pub struct ResponsesSseDecoder {
_policy: StreamPolicy,
_terminal: Option<StreamOutcome<ResponsesResponse>>,
}
impl ResponsesSseDecoder {
pub fn new(_policy: StreamPolicy) -> Self {
todo!(
"Initialize semantic decoding downstream of SseFramer, which owns split UTF-8 and frame assembly"
)
}
pub fn decode(&mut self, _frame: SseFrame) -> Result<Vec<ResponsesEvent>, StreamError> {
todo!(
"Decode canonical events, skip SSE control frames, handle DONE, preserve extensions and usage, and enforce event/buffer limits"
)
}
pub fn finish(self) -> Result<StreamOutcome<ResponsesResponse>, StreamError> {
todo!(
"Require a terminal Responses event; EOF or DONE alone cannot fabricate completed output"
)
}
}
pub fn decode_response(_body: &[u8]) -> Result<ResponsesResponse, StreamError> {
todo!("Validate the full wire schema and retain provider extensions in the typed response")
}
pub fn encode_event(_event: &ResponsesEvent) -> Result<String, StreamError> {
todo!("Serialize canonical event names and payload fields for SSE or WebSocket delivery")
}

View file

@ -1,16 +1,325 @@
// TODO: Port the complete Python Responses models before enabling these streaming contracts
pub struct ResponsesResponse {
_schema: (),
use std::collections::BTreeMap;
use serde_json::Value;
#[derive(Clone, Debug, PartialEq)]
pub struct ProviderFields(pub BTreeMap<String, Value>);
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ResponseStatus {
Queued,
InProgress,
Completed,
Failed,
Incomplete,
Cancelled,
}
pub struct ResponseOutputItem {
_schema: (),
#[derive(Clone, Debug, PartialEq)]
pub struct ResponsesUsage {
pub input_tokens: u64,
pub output_tokens: u64,
pub total_tokens: u64,
pub cached_tokens: Option<u64>,
pub reasoning_tokens: Option<u64>,
pub input_token_details: ProviderFields,
pub output_token_details: ProviderFields,
pub extensions: ProviderFields,
}
#[derive(Clone, Debug, PartialEq)]
pub struct ResponsesError {
pub code: Option<String>,
pub message: String,
pub param: Option<String>,
pub extensions: ProviderFields,
}
#[derive(Clone, Debug, PartialEq)]
pub struct ResponseAnnotation {
pub kind: String,
pub start_index: Option<u64>,
pub end_index: Option<u64>,
pub url: Option<String>,
pub title: Option<String>,
pub extensions: ProviderFields,
}
#[derive(Clone, Debug, PartialEq)]
pub enum ResponseContentPart {
OutputText {
text: String,
annotations: Box<[ResponseAnnotation]>,
},
Refusal {
refusal: String,
},
SummaryText {
text: String,
},
Extension {
kind: String,
fields: ProviderFields,
},
}
#[derive(Clone, Debug, PartialEq)]
pub enum ResponseOutputItem {
Message {
id: String,
status: ResponseStatus,
role: String,
content: Box<[ResponseContentPart]>,
phase: Option<String>,
extensions: ProviderFields,
},
FunctionCall {
id: String,
call_id: String,
name: String,
namespace: Option<String>,
arguments: String,
status: ResponseStatus,
extensions: ProviderFields,
},
CustomToolCall {
id: String,
call_id: String,
name: String,
input: String,
status: Option<ResponseStatus>,
extensions: ProviderFields,
},
Reasoning {
id: String,
summary: Box<[ResponseContentPart]>,
encrypted_content: Option<String>,
extensions: ProviderFields,
},
Extension {
kind: String,
id: Option<String>,
fields: ProviderFields,
},
}
#[derive(Clone, Debug, PartialEq)]
pub enum ResponseInputContent {
Text(String),
Image {
image_url: Option<String>,
file_id: Option<String>,
detail: Option<String>,
},
File {
file_id: Option<String>,
file_data: Option<String>,
filename: Option<String>,
},
Extension {
kind: String,
fields: ProviderFields,
},
}
#[derive(Clone, Debug, PartialEq)]
pub enum ResponseInputItem {
Message {
role: String,
content: Box<[ResponseInputContent]>,
},
FunctionCallOutput {
call_id: String,
output: String,
},
CustomToolCallOutput {
call_id: String,
output: String,
},
ItemReference {
id: String,
},
Output(ResponseOutputItem),
Extension {
kind: String,
fields: ProviderFields,
},
}
#[derive(Clone, Debug, PartialEq)]
pub enum ResponsesInput {
Text(String),
Items(Box<[ResponseInputItem]>),
}
#[derive(Clone, Debug, PartialEq)]
pub enum ResponseTool {
Function {
name: String,
description: Option<String>,
parameters: ProviderFields,
strict: Option<bool>,
},
Custom {
name: String,
description: Option<String>,
format: ProviderFields,
},
Namespace {
name: String,
description: Option<String>,
tools: Box<[ResponseTool]>,
},
BuiltIn {
kind: String,
options: ProviderFields,
},
}
#[derive(Clone, Debug, PartialEq)]
pub struct ResponsesRequest {
_schema: (),
pub model: Option<String>,
pub input: Option<ResponsesInput>,
pub previous_response_id: Option<String>,
pub instructions: Option<String>,
pub tools: Box<[ResponseTool]>,
pub parallel_tool_calls: Option<bool>,
pub max_output_tokens: Option<u64>,
pub store: Option<bool>,
pub background: Option<bool>,
pub stream: Option<bool>,
pub include: Box<[String]>,
pub metadata: BTreeMap<String, String>,
// TODO: Type remaining Python request options before enabling wire decoding; preserve them losslessly meanwhile
pub extensions: ProviderFields,
}
pub struct ResponsesEvent {
_schema: (),
#[derive(Clone, Debug, PartialEq)]
pub struct ResponsesResponse {
pub id: String,
pub created_at: u64,
pub model: Option<String>,
pub status: Option<ResponseStatus>,
pub output: Box<[ResponseOutputItem]>,
pub usage: Option<ResponsesUsage>,
pub error: Option<ResponsesError>,
pub incomplete_reason: Option<String>,
pub previous_response_id: Option<String>,
pub extensions: ProviderFields,
}
#[derive(Clone, Debug, PartialEq)]
pub struct ResponsesEvent {
pub sequence_number: Option<u64>,
pub data: ResponsesEventData,
pub extensions: ProviderFields,
}
#[derive(Clone, Debug, PartialEq)]
pub enum ResponsesEventData {
Created(ResponsesResponse),
InProgress(ResponsesResponse),
Completed(ResponsesResponse),
Failed(ResponsesResponse),
Incomplete(ResponsesResponse),
OutputItemAdded {
output_index: u64,
item: ResponseOutputItem,
},
OutputItemDone {
output_index: u64,
item: ResponseOutputItem,
},
ContentPartAdded {
item_id: String,
output_index: u64,
content_index: u64,
part: ResponseContentPart,
},
ContentPartDone {
item_id: String,
output_index: u64,
content_index: u64,
part: ResponseContentPart,
},
OutputTextDelta {
item_id: String,
output_index: u64,
content_index: u64,
delta: String,
},
OutputTextDone {
item_id: String,
output_index: u64,
content_index: u64,
text: String,
},
AnnotationAdded {
item_id: String,
output_index: u64,
content_index: u64,
annotation_index: u64,
annotation: ResponseAnnotation,
},
RefusalDelta {
item_id: String,
output_index: u64,
content_index: u64,
delta: String,
},
RefusalDone {
item_id: String,
output_index: u64,
content_index: u64,
refusal: String,
},
FunctionCallArgumentsDelta {
item_id: String,
output_index: u64,
delta: String,
},
FunctionCallArgumentsDone {
item_id: String,
output_index: u64,
arguments: String,
},
CustomToolCallInputDelta {
item_id: String,
output_index: u64,
delta: String,
},
CustomToolCallInputDone {
item_id: String,
output_index: u64,
input: String,
},
ReasoningSummaryPartAdded {
item_id: String,
output_index: u64,
summary_index: u64,
part: ResponseContentPart,
},
ReasoningSummaryPartDone {
item_id: String,
output_index: u64,
summary_index: u64,
part: ResponseContentPart,
},
ReasoningSummaryTextDelta {
item_id: String,
output_index: u64,
summary_index: u64,
delta: String,
},
ReasoningSummaryTextDone {
item_id: String,
output_index: u64,
summary_index: u64,
text: String,
},
Error(ResponsesError),
Extension {
kind: String,
fields: ProviderFields,
},
}

View file

@ -1,6 +1,11 @@
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_json::{Map, Value};
pub struct ResponsesWebSocketRequestDefaults {
pub fill_missing: crate::responses::streaming::ProviderFields,
pub overrides: crate::responses::streaming::ProviderFields,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ResponsesWsEventType {
ResponseCreate,