mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
refactor(rust): scaffold response sessions and streaming routes
This commit is contained in:
parent
a545e6ab6d
commit
39aa0a29d3
20 changed files with 1106 additions and 41 deletions
|
|
@ -7,6 +7,7 @@
|
|||
//! calls the provider, and returns a typed OpenAI-shaped response.
|
||||
|
||||
mod error;
|
||||
pub mod streaming;
|
||||
pub mod types;
|
||||
pub use error::Error;
|
||||
mod client;
|
||||
|
|
|
|||
29
litellm-rust/crates/core/src/chat_completions/streaming.rs
Normal file
29
litellm-rust/crates/core/src/chat_completions/streaming.rs
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
use litellm_host::machine::RouteMachine;
|
||||
use litellm_llms::base_llm::base_model_iterator::ChatCompletions;
|
||||
|
||||
use crate::streaming::{StreamingCall, StreamingRoute};
|
||||
|
||||
pub struct ChatCompletionsCall {
|
||||
pub model: String,
|
||||
pub messages: Box<[litellm_types::llms::openai::ChatMessage]>,
|
||||
pub optional_params: std::collections::BTreeMap<String, serde_json::Value>,
|
||||
pub custom_llm_provider: Option<String>,
|
||||
pub api_key: Option<litellm_auth::SecretValue>,
|
||||
pub api_base: Option<String>,
|
||||
pub extra_headers: Box<[(String, String)]>,
|
||||
pub timeout: Option<std::time::Duration>,
|
||||
}
|
||||
|
||||
pub struct Streaming;
|
||||
|
||||
impl StreamingCall for Streaming {
|
||||
type Contract = ChatCompletions;
|
||||
type Request = ChatCompletionsCall;
|
||||
type Response = litellm_types::utils::ChatCompletionsResponse;
|
||||
}
|
||||
|
||||
pub fn chat_completions_stream_machine() -> RouteMachine<StreamingRoute<Streaming>> {
|
||||
todo!(
|
||||
"Select a typed native or translated stream and preserve existing provider acceptance gates until the streaming path is implemented"
|
||||
)
|
||||
}
|
||||
|
|
@ -6,5 +6,6 @@ pub mod messages;
|
|||
pub mod ocr;
|
||||
mod outbound;
|
||||
pub mod responses;
|
||||
pub mod streaming;
|
||||
|
||||
pub use error::Error;
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
//! it in process for a caller that already holds the request and wants the message.
|
||||
|
||||
mod error;
|
||||
pub mod streaming;
|
||||
pub mod types;
|
||||
pub use error::Error;
|
||||
mod client;
|
||||
|
|
|
|||
37
litellm-rust/crates/core/src/messages/streaming.rs
Normal file
37
litellm-rust/crates/core/src/messages/streaming.rs
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
use futures_util::Stream;
|
||||
use litellm_host::machine::RouteMachine;
|
||||
use litellm_llms::{
|
||||
anthropic::experimental_pass_through::responses_adapters::streaming_iterator::AnthropicResponsesStreamWrapper,
|
||||
base_llm::base_model_iterator::{
|
||||
AnthropicMessagesApi, StreamError, StreamPolicy, TransformedStream,
|
||||
},
|
||||
};
|
||||
use litellm_types::responses::streaming::ResponsesEvent;
|
||||
|
||||
use crate::streaming::{StreamingCall, StreamingRoute};
|
||||
|
||||
pub struct Streaming;
|
||||
|
||||
impl StreamingCall for Streaming {
|
||||
type Contract = AnthropicMessagesApi;
|
||||
type Request = super::route::MessagesCall;
|
||||
type Response =
|
||||
litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse;
|
||||
}
|
||||
|
||||
pub fn messages_stream_machine() -> RouteMachine<StreamingRoute<Streaming>> {
|
||||
todo!(
|
||||
"Select a typed native or translated stream and preserve existing provider acceptance gates until the streaming path is implemented"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn messages_from_responses<S>(
|
||||
_source: S,
|
||||
_adapter: AnthropicResponsesStreamWrapper,
|
||||
_policy: StreamPolicy,
|
||||
) -> TransformedStream<S, AnthropicResponsesStreamWrapper>
|
||||
where
|
||||
S: Stream<Item = Result<ResponsesEvent, StreamError>>,
|
||||
{
|
||||
todo!("Compose canonical Responses events with the Messages bridge before typed host delivery")
|
||||
}
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
use futures_util::Stream;
|
||||
use litellm_llms::base_llm::base_model_iterator::TransformedStream;
|
||||
use litellm_llms::base_llm::base_model_iterator::{StreamError, StreamPolicy, TransformedStream};
|
||||
use litellm_types::utils::ChatCompletionChunk;
|
||||
|
||||
use super::streaming_iterator::LiteLlmCompletionStreamingIterator;
|
||||
use crate::responses::streaming_iterator::{ResponsesApiStreamingIterator, ResponsesStreamError};
|
||||
use crate::responses::streaming_iterator::ResponsesApiStreamingIterator;
|
||||
|
||||
pub struct LiteLlmCompletionTransformationHandler;
|
||||
|
||||
|
|
@ -11,9 +11,10 @@ impl LiteLlmCompletionTransformationHandler {
|
|||
pub fn streaming_response<S>(
|
||||
_chat_stream: S,
|
||||
_adapter: LiteLlmCompletionStreamingIterator,
|
||||
_policy: StreamPolicy,
|
||||
) -> ResponsesApiStreamingIterator<TransformedStream<S, LiteLlmCompletionStreamingIterator>>
|
||||
where
|
||||
S: Stream<Item = Result<ChatCompletionChunk, ResponsesStreamError>>,
|
||||
S: Stream<Item = Result<ChatCompletionChunk, StreamError>>,
|
||||
{
|
||||
todo!(
|
||||
"Compose the transformed Chat source inside the same driver used for native Responses events"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use litellm_llms::base_llm::base_model_iterator::{
|
||||
ChatCompletions, ResponsesApi, StreamTransformation,
|
||||
ChatCompletions, ResponsesApi, StreamError, StreamTransformation,
|
||||
};
|
||||
use litellm_types::{
|
||||
llms::openai::ChatCompletionToolCallChunk,
|
||||
|
|
@ -9,8 +9,6 @@ use litellm_types::{
|
|||
utils::ChatCompletionChunk,
|
||||
};
|
||||
|
||||
use crate::responses::streaming_iterator::ResponsesStreamError;
|
||||
|
||||
pub struct LiteLlmCompletionStreamingIterator {
|
||||
// TODO: Port per-call item/tool associations, request options, bounded accumulators, and reasoning state from Python
|
||||
_state: (),
|
||||
|
|
@ -32,7 +30,7 @@ impl LiteLlmCompletionStreamingIterator {
|
|||
pub fn queue_tool_call_delta_events(
|
||||
&mut self,
|
||||
_tool_calls: &[ChatCompletionToolCallChunk],
|
||||
) -> Result<(), ResponsesStreamError> {
|
||||
) -> Result<(), StreamError> {
|
||||
todo!(
|
||||
"Associate fragmented tool indexes and call IDs; preserve custom/namespaced tools, arguments, item IDs and output indexes"
|
||||
)
|
||||
|
|
@ -48,7 +46,7 @@ impl LiteLlmCompletionStreamingIterator {
|
|||
pub fn transform_chat_completion_chunk_to_response_api_chunk(
|
||||
&mut self,
|
||||
_chunk: ChatCompletionChunk,
|
||||
) -> Result<Vec<ResponsesEvent>, ResponsesStreamError> {
|
||||
) -> Result<Vec<ResponsesEvent>, StreamError> {
|
||||
todo!(
|
||||
"Convert text, tools, reasoning, annotations and provider web-search fields while preserving sequence numbers"
|
||||
)
|
||||
|
|
@ -60,15 +58,11 @@ impl LiteLlmCompletionStreamingIterator {
|
|||
)
|
||||
}
|
||||
|
||||
pub fn return_default_done_events(
|
||||
&mut self,
|
||||
) -> Result<Vec<ResponsesEvent>, ResponsesStreamError> {
|
||||
pub fn return_default_done_events(&mut self) -> Result<Vec<ResponsesEvent>, StreamError> {
|
||||
todo!("Close text, content, reasoning and tool items in order before the terminal response")
|
||||
}
|
||||
|
||||
pub fn emit_response_completed_event(
|
||||
&mut self,
|
||||
) -> Result<ResponsesEvent, ResponsesStreamError> {
|
||||
pub fn emit_response_completed_event(&mut self) -> Result<ResponsesEvent, StreamError> {
|
||||
todo!(
|
||||
"Build the final response from accumulated chunks and late usage, retaining encoded IDs and request options"
|
||||
)
|
||||
|
|
@ -78,7 +72,7 @@ impl LiteLlmCompletionStreamingIterator {
|
|||
impl StreamTransformation for LiteLlmCompletionStreamingIterator {
|
||||
type Caller = ResponsesApi;
|
||||
type Upstream = ChatCompletions;
|
||||
type Error = ResponsesStreamError;
|
||||
type Error = StreamError;
|
||||
|
||||
fn transform_event(
|
||||
&mut self,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,15 @@
|
|||
mod error;
|
||||
pub use error::Error;
|
||||
pub mod litellm_completion_transformation;
|
||||
pub mod route;
|
||||
pub mod streaming_iterator;
|
||||
pub mod websocket;
|
||||
pub mod websocket_session;
|
||||
|
||||
use litellm_types::responses::streaming::ResponsesResponse;
|
||||
|
||||
pub async fn responses(_call: route::ResponsesCall) -> Result<ResponsesResponse, Error> {
|
||||
todo!(
|
||||
"Resolve native or bridged provider, prepare/send HTTP request, and normalize the response through route host operations"
|
||||
)
|
||||
}
|
||||
|
|
|
|||
31
litellm-rust/crates/core/src/responses/route.rs
Normal file
31
litellm-rust/crates/core/src/responses/route.rs
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use litellm_auth::SecretValue;
|
||||
use litellm_host::machine::RouteMachine;
|
||||
use litellm_llms::base_llm::base_model_iterator::ResponsesApi;
|
||||
use litellm_types::responses::streaming::{ResponsesRequest, ResponsesResponse};
|
||||
|
||||
use crate::streaming::{StreamingCall, StreamingRoute};
|
||||
|
||||
pub struct ResponsesCall {
|
||||
pub request: ResponsesRequest,
|
||||
pub custom_llm_provider: Option<String>,
|
||||
pub api_key: Option<SecretValue>,
|
||||
pub api_base: Option<String>,
|
||||
pub extra_headers: Box<[(String, String)]>,
|
||||
pub timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
pub struct Responses;
|
||||
|
||||
impl StreamingCall for Responses {
|
||||
type Contract = ResponsesApi;
|
||||
type Request = ResponsesCall;
|
||||
type Response = ResponsesResponse;
|
||||
}
|
||||
|
||||
pub fn responses_machine() -> RouteMachine<StreamingRoute<Responses>> {
|
||||
todo!(
|
||||
"Compose native Responses or Chat-to-Responses conversion with the shared Responses driver and host operations"
|
||||
)
|
||||
}
|
||||
|
|
@ -1,3 +1,6 @@
|
|||
use litellm_llms::base_llm::base_model_iterator::{
|
||||
BufferedSource, StreamError, StreamOutcome, StreamPolicy,
|
||||
};
|
||||
use std::{
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
|
|
@ -6,36 +9,28 @@ use std::{
|
|||
use futures_util::Stream;
|
||||
use litellm_types::responses::streaming::{ResponsesEvent, ResponsesResponse};
|
||||
|
||||
pub enum ResponsesStreamError {
|
||||
Upstream(String),
|
||||
InvalidEvent(String),
|
||||
UnexpectedEof,
|
||||
BufferLimitExceeded,
|
||||
Cancelled,
|
||||
DeadlineExceeded,
|
||||
}
|
||||
|
||||
pub enum ResponsesStreamState {
|
||||
Pending,
|
||||
Active,
|
||||
Terminal(ResponsesResponse),
|
||||
Failed(ResponsesStreamError),
|
||||
Terminal(StreamOutcome<ResponsesResponse>),
|
||||
Failed(StreamError),
|
||||
}
|
||||
|
||||
pub struct ResponsesApiStreamingIterator<S> {
|
||||
source: S,
|
||||
state: ResponsesStreamState,
|
||||
policy: StreamPolicy,
|
||||
}
|
||||
|
||||
impl<S> ResponsesApiStreamingIterator<S> {
|
||||
pub fn new(_source: S) -> Self {
|
||||
pub fn new(_source: S, _policy: StreamPolicy) -> Self {
|
||||
todo!("Compose BaseResponsesAPIStreamingIterator lifecycle state with a typed event source")
|
||||
}
|
||||
|
||||
pub fn process_chunk(
|
||||
&mut self,
|
||||
_event: ResponsesEvent,
|
||||
) -> Result<Option<ResponsesEvent>, ResponsesStreamError> {
|
||||
) -> Result<Option<ResponsesEvent>, StreamError> {
|
||||
todo!(
|
||||
"Port base _process_chunk after provider decoding: response IDs, completed/failed/incomplete outcomes, usage, and error events"
|
||||
)
|
||||
|
|
@ -50,11 +45,39 @@ impl<S> ResponsesApiStreamingIterator<S> {
|
|||
}
|
||||
}
|
||||
|
||||
pub struct BufferedResponsesStream {
|
||||
_response: ResponsesResponse,
|
||||
_source: BufferedSource,
|
||||
_policy: StreamPolicy,
|
||||
}
|
||||
|
||||
impl BufferedResponsesStream {
|
||||
pub fn new(
|
||||
_response: ResponsesResponse,
|
||||
_source: BufferedSource,
|
||||
_policy: StreamPolicy,
|
||||
) -> Self {
|
||||
todo!(
|
||||
"Port cached/mock/simulated response event generation with explicit buffered delivery and stable item IDs"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for BufferedResponsesStream {
|
||||
type Item = Result<ResponsesEvent, StreamError>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
todo!(
|
||||
"Emit bounded ordered lifecycle/content/tool events from the retained response, preserving its terminal status and usage"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> Stream for ResponsesApiStreamingIterator<S>
|
||||
where
|
||||
S: Stream<Item = Result<ResponsesEvent, ResponsesStreamError>>,
|
||||
S: Stream<Item = Result<ResponsesEvent, StreamError>>,
|
||||
{
|
||||
type Item = Result<ResponsesEvent, ResponsesStreamError>;
|
||||
type Item = Result<ResponsesEvent, StreamError>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
todo!(
|
||||
|
|
@ -77,9 +100,9 @@ impl<S> NativeResponsesStream<S> {
|
|||
|
||||
impl<S> Stream for NativeResponsesStream<S>
|
||||
where
|
||||
S: Stream<Item = Result<ResponsesEvent, ResponsesStreamError>>,
|
||||
S: Stream<Item = Result<ResponsesEvent, StreamError>>,
|
||||
{
|
||||
type Item = Result<ResponsesEvent, ResponsesStreamError>;
|
||||
type Item = Result<ResponsesEvent, StreamError>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
todo!(
|
||||
|
|
|
|||
356
litellm-rust/crates/core/src/responses/websocket_session.rs
Normal file
356
litellm-rust/crates/core/src/responses/websocket_session.rs
Normal file
|
|
@ -0,0 +1,356 @@
|
|||
use std::collections::BTreeMap;
|
||||
|
||||
use litellm_host::{
|
||||
machine::{HostChannel, RouteMachine},
|
||||
route::Route,
|
||||
};
|
||||
use litellm_llms::base_llm::base_model_iterator::{StreamError, StreamOutcome, StreamPolicy};
|
||||
use litellm_types::responses::streaming::{
|
||||
ResponseInputItem, ResponsesError, ResponsesEvent, ResponsesInput, ResponsesRequest,
|
||||
ResponsesResponse,
|
||||
};
|
||||
use litellm_types::responses::streaming_websocket::ResponsesWebSocketRequestDefaults;
|
||||
|
||||
use super::{route::ResponsesCall, websocket::ResponsesWebSocketConnection};
|
||||
|
||||
pub struct ResponseCreate {
|
||||
pub request: ResponsesRequest,
|
||||
pub generate: Option<bool>,
|
||||
}
|
||||
|
||||
pub enum ClientFrame {
|
||||
Create(ResponseCreate),
|
||||
Closed,
|
||||
}
|
||||
|
||||
pub struct ResponsesWebSocketConfig {
|
||||
pub call: ResponsesCall,
|
||||
pub authorized_model: String,
|
||||
pub first_message: Option<String>,
|
||||
pub request_defaults: ResponsesWebSocketRequestDefaults,
|
||||
pub stream_policy: StreamPolicy,
|
||||
pub max_history_bytes: usize,
|
||||
pub max_history_turns: usize,
|
||||
}
|
||||
|
||||
pub enum ResponsesWebSocketState {
|
||||
Idle,
|
||||
Warmup { response_id: String },
|
||||
Active { response_id: Option<String> },
|
||||
Closed,
|
||||
Failed(StreamError),
|
||||
}
|
||||
|
||||
pub enum WebSocketOutcome {
|
||||
Closed,
|
||||
Cancelled,
|
||||
Failed(StreamError),
|
||||
}
|
||||
|
||||
pub enum InputPolicyResult {
|
||||
Approved(ResponsesRequest),
|
||||
Rejected(ResponsesError),
|
||||
}
|
||||
|
||||
pub enum WebSocketOp {
|
||||
ReceiveClientFrame,
|
||||
SendClientFrame(String),
|
||||
ApplyInputPolicy {
|
||||
request: ResponsesRequest,
|
||||
authorized_model: String,
|
||||
},
|
||||
EnforceQuota {
|
||||
request: ResponsesRequest,
|
||||
},
|
||||
ApplyOutputPolicy(ResponsesEvent),
|
||||
LoadHistory {
|
||||
response_id: String,
|
||||
},
|
||||
StoreHistory {
|
||||
response_id: String,
|
||||
items: Box<[ResponseInputItem]>,
|
||||
},
|
||||
AccountTurn(StreamOutcome<ResponsesResponse>),
|
||||
PostStreamingHooks,
|
||||
}
|
||||
|
||||
pub enum WebSocketOpResult {
|
||||
ClientFrame(Option<String>),
|
||||
InputPolicy(InputPolicyResult),
|
||||
Quota(Result<(), ResponsesError>),
|
||||
OutputEvent(ResponsesEvent),
|
||||
History(Box<[ResponseInputItem]>),
|
||||
Acknowledged,
|
||||
}
|
||||
|
||||
pub struct ResponsesWebSocket;
|
||||
|
||||
impl Route for ResponsesWebSocket {
|
||||
type Response = WebSocketOutcome;
|
||||
type Error = StreamError;
|
||||
type Op = WebSocketOp;
|
||||
type OpResult = WebSocketOpResult;
|
||||
type Chunk = std::convert::Infallible;
|
||||
type StreamHead = ();
|
||||
}
|
||||
|
||||
pub enum ResponsesWebSocketMode {
|
||||
Native(ResponsesWebSocketConnection),
|
||||
Managed,
|
||||
}
|
||||
|
||||
pub fn responses_websocket_machine(
|
||||
_config: ResponsesWebSocketConfig,
|
||||
_mode: ResponsesWebSocketMode,
|
||||
) -> RouteMachine<ResponsesWebSocket> {
|
||||
todo!(
|
||||
"Select native or managed session driver; keep client socket I/O, policies, storage and callback execution behind host operations"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn parse_message(_raw_message: &str) -> Result<ClientFrame, StreamError> {
|
||||
todo!(
|
||||
"Validate response.create in flat or nested form, including generate=false and previous_response_id; reject unsupported client events"
|
||||
)
|
||||
}
|
||||
|
||||
pub struct ResponsesWebSocketStreaming {
|
||||
_backend_ws: ResponsesWebSocketConnection,
|
||||
_config: ResponsesWebSocketConfig,
|
||||
_state: ResponsesWebSocketState,
|
||||
_terminal: Option<StreamOutcome<ResponsesResponse>>,
|
||||
}
|
||||
|
||||
impl ResponsesWebSocketStreaming {
|
||||
pub fn new(
|
||||
_backend_ws: ResponsesWebSocketConnection,
|
||||
_config: ResponsesWebSocketConfig,
|
||||
) -> Self {
|
||||
todo!(
|
||||
"Initialize a native session with first-frame handling and one active turn per connection"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn enforce_authorized_model(&self, _request: &ResponsesRequest) -> Result<(), StreamError> {
|
||||
todo!("Check the authorized model on every response.create, including nested requests")
|
||||
}
|
||||
|
||||
pub fn with_request_defaults(
|
||||
&self,
|
||||
_request: ResponsesRequest,
|
||||
) -> Result<ResponsesRequest, StreamError> {
|
||||
todo!(
|
||||
"Merge fill_missing defaults, then per-turn parameters, then deployment overrides in Python's precedence order"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn collect_input_from_client_event(
|
||||
&mut self,
|
||||
_request: &ResponsesRequest,
|
||||
) -> Result<(), StreamError> {
|
||||
todo!("Accumulate per-turn input for accounting within session limits")
|
||||
}
|
||||
|
||||
pub fn store_event(&mut self, _event: &ResponsesEvent) -> Result<(), StreamError> {
|
||||
todo!(
|
||||
"Retain required lifecycle/usage events within bounds, resetting accumulation between turns"
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn enforce_or_reject_frame(
|
||||
&mut self,
|
||||
_host: &HostChannel<ResponsesWebSocket>,
|
||||
_frame: ResponseCreate,
|
||||
) -> Result<InputPolicyResult, StreamError> {
|
||||
todo!(
|
||||
"Enforce model, request defaults, host input masking and per-turn quota before forwarding"
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn backend_to_client(
|
||||
&mut self,
|
||||
_host: &HostChannel<ResponsesWebSocket>,
|
||||
) -> Result<(), StreamError> {
|
||||
todo!(
|
||||
"Decode backend events, apply host output policies, forward frames, and account completed/failed/incomplete turns exactly once"
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn client_to_backend(
|
||||
&mut self,
|
||||
_host: &HostChannel<ResponsesWebSocket>,
|
||||
) -> Result<(), StreamError> {
|
||||
todo!(
|
||||
"Read first/subsequent frames through host operations and validate each turn before upstream send"
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn log_messages(
|
||||
&mut self,
|
||||
_host: &HostChannel<ResponsesWebSocket>,
|
||||
) -> Result<(), StreamError> {
|
||||
todo!(
|
||||
"Yield turn accounting and post-stream hooks without running callbacks or writing storage in core"
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn record_usage_for_failure(
|
||||
&mut self,
|
||||
_host: &HostChannel<ResponsesWebSocket>,
|
||||
) -> Result<(), StreamError> {
|
||||
todo!(
|
||||
"Preserve observed usage after failures, disconnects and cancellation without double accounting"
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn bidirectional_forward(
|
||||
self,
|
||||
_host: &HostChannel<ResponsesWebSocket>,
|
||||
) -> Result<WebSocketOutcome, StreamError> {
|
||||
todo!(
|
||||
"Split independent read/write ownership, coordinate both directions with backpressure, cancel peers and close upstream on exit; created is not turn completion"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ManagedResponsesWebSocketHandler {
|
||||
_config: ResponsesWebSocketConfig,
|
||||
_state: ResponsesWebSocketState,
|
||||
_session_history: BTreeMap<String, Box<[ResponseInputItem]>>,
|
||||
}
|
||||
|
||||
impl ManagedResponsesWebSocketHandler {
|
||||
pub fn new(_config: ResponsesWebSocketConfig) -> Self {
|
||||
todo!("Initialize bounded per-connection history and sequential HTTP-backed turns")
|
||||
}
|
||||
|
||||
pub fn serialize_chunk(_event: &ResponsesEvent) -> Result<String, StreamError> {
|
||||
todo!("Use the canonical Responses event encoder shared with SSE")
|
||||
}
|
||||
|
||||
pub fn get_history_messages(
|
||||
&self,
|
||||
_previous_response_id: &str,
|
||||
) -> Option<&[ResponseInputItem]> {
|
||||
todo!(
|
||||
"Look up decoded IDs in session history before requesting persistent history from the host"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn store_history(
|
||||
&mut self,
|
||||
_response_id: String,
|
||||
_items: Box<[ResponseInputItem]>,
|
||||
) -> Result<(), StreamError> {
|
||||
todo!(
|
||||
"Enforce history byte/turn limits and preserve follow-up availability before asynchronous host persistence"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn input_to_messages(_input: ResponsesInput) -> Box<[ResponseInputItem]> {
|
||||
todo!(
|
||||
"Normalize text and typed input items without losing tools, reasoning or multimodal content"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn is_warmup_frame(_frame: &ResponseCreate) -> bool {
|
||||
todo!("Recognize generate=false without issuing a provider generation request")
|
||||
}
|
||||
|
||||
pub fn is_warmup_response_id(_response_id: &str) -> bool {
|
||||
todo!("Recognize synthetic warmup IDs so they are not replayed as provider response IDs")
|
||||
}
|
||||
|
||||
pub fn build_warmup_response(&self, _frame: &ResponseCreate) -> ResponsesResponse {
|
||||
todo!("Construct a synthetic warmup acknowledgement using the request's model and options")
|
||||
}
|
||||
|
||||
pub fn build_base_call_kwargs(_frame: ResponseCreate) -> ResponsesRequest {
|
||||
todo!("Extract supported Responses parameters while excluding WebSocket-only fields")
|
||||
}
|
||||
|
||||
pub fn apply_history(
|
||||
&self,
|
||||
_request: ResponsesRequest,
|
||||
_prior_history: &[ResponseInputItem],
|
||||
) -> Result<ResponsesRequest, StreamError> {
|
||||
todo!(
|
||||
"Prepend session or host history without duplicating input or forwarding synthetic warmup IDs"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn same_provider(&self, _model: &str) -> Result<bool, StreamError> {
|
||||
todo!(
|
||||
"Resolve the turn provider before deciding whether connection credentials can be reused"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn inject_credentials(
|
||||
&self,
|
||||
_request: ResponsesRequest,
|
||||
) -> Result<ResponsesCall, StreamError> {
|
||||
todo!(
|
||||
"Reuse connection credentials only for the same provider and preserve per-turn routing"
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn send_error(
|
||||
&self,
|
||||
_host: &HostChannel<ResponsesWebSocket>,
|
||||
_error: ResponsesError,
|
||||
) -> Result<(), StreamError> {
|
||||
todo!(
|
||||
"Send a structured error frame through the host, preserving session versus turn failure"
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn send_warmup_ack(
|
||||
&mut self,
|
||||
_host: &HostChannel<ResponsesWebSocket>,
|
||||
_frame: ResponseCreate,
|
||||
) -> Result<(), StreamError> {
|
||||
todo!(
|
||||
"Send warmup acknowledgement without generation, usage fabrication or history pollution"
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn stream_and_forward(
|
||||
&mut self,
|
||||
_host: &HostChannel<ResponsesWebSocket>,
|
||||
_call: ResponsesCall,
|
||||
) -> Result<StreamOutcome<ResponsesResponse>, StreamError> {
|
||||
todo!(
|
||||
"Drive the HTTP Responses route and forward canonical events with host output policy, backpressure, cancellation and terminal validation"
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn save_turn_history(
|
||||
&mut self,
|
||||
_host: &HostChannel<ResponsesWebSocket>,
|
||||
_request: ResponsesRequest,
|
||||
_response: ResponsesResponse,
|
||||
) -> Result<(), StreamError> {
|
||||
todo!(
|
||||
"Save normalized input and output under the decoded response ID, then yield host persistence"
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn process_response_create(
|
||||
&mut self,
|
||||
_host: &HostChannel<ResponsesWebSocket>,
|
||||
_frame: ResponseCreate,
|
||||
) -> Result<(), StreamError> {
|
||||
todo!(
|
||||
"Coordinate authorization/quota, warmup, history, provider resolution, HTTP streaming and accounting for one turn"
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn run(
|
||||
self,
|
||||
_host: &HostChannel<ResponsesWebSocket>,
|
||||
) -> Result<WebSocketOutcome, StreamError> {
|
||||
todo!(
|
||||
"Process sequential response.create turns until disconnect; cancel active generation and retain observed usage on exit"
|
||||
)
|
||||
}
|
||||
}
|
||||
84
litellm-rust/crates/core/src/streaming.rs
Normal file
84
litellm-rust/crates/core/src/streaming.rs
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
use std::marker::PhantomData;
|
||||
|
||||
use litellm_host::{
|
||||
machine::{HostChannel, RouteMachine},
|
||||
route::Route,
|
||||
};
|
||||
use litellm_llms::base_llm::base_model_iterator::{
|
||||
ApiDialect, DeliveryMode, ServerStreamingContract, StreamError, StreamOutcome, StreamPolicy,
|
||||
};
|
||||
|
||||
pub trait StreamingCall: Send + Sync + 'static {
|
||||
type Contract: ServerStreamingContract<Event: Send + 'static>;
|
||||
type Request: Send + 'static;
|
||||
type Response: Send + 'static;
|
||||
}
|
||||
|
||||
pub struct StreamingRoute<C: StreamingCall>(PhantomData<C>);
|
||||
|
||||
pub struct StreamHead {
|
||||
pub dialect: ApiDialect,
|
||||
pub delivery: DeliveryMode,
|
||||
}
|
||||
|
||||
pub enum StreamOp<C: StreamingCall> {
|
||||
ProjectRequest,
|
||||
LoadCachedResponse {
|
||||
key: String,
|
||||
},
|
||||
PostStreamEvent {
|
||||
event: <C::Contract as ServerStreamingContract>::Event,
|
||||
},
|
||||
StoreResponse {
|
||||
key: String,
|
||||
response: C::Response,
|
||||
},
|
||||
AccountOutcome {
|
||||
outcome: StreamOutcome<C::Response>,
|
||||
},
|
||||
PostStreamingHooks,
|
||||
}
|
||||
|
||||
pub enum StreamOpResult<C: StreamingCall> {
|
||||
Request {
|
||||
request: C::Request,
|
||||
policy: StreamPolicy,
|
||||
},
|
||||
CachedResponse(Option<C::Response>),
|
||||
StoredResponse(C::Response),
|
||||
AccountedOutcome(StreamOutcome<C::Response>),
|
||||
Event(<C::Contract as ServerStreamingContract>::Event),
|
||||
Acknowledged,
|
||||
}
|
||||
|
||||
impl<C: StreamingCall> Route for StreamingRoute<C> {
|
||||
type Response = StreamOutcome<C::Response>;
|
||||
type Error = StreamError;
|
||||
type Op = StreamOp<C>;
|
||||
type OpResult = StreamOpResult<C>;
|
||||
type Chunk = <C::Contract as ServerStreamingContract>::Event;
|
||||
type StreamHead = StreamHead;
|
||||
}
|
||||
|
||||
pub fn streaming_machine<C: StreamingCall>() -> RouteMachine<StreamingRoute<C>> {
|
||||
todo!(
|
||||
"Project request, preserve acceptance gates, select native/bridged/buffered source, then drive host delivery and terminal accounting"
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn deliver_stream<C, S>(
|
||||
_host: &HostChannel<StreamingRoute<C>>,
|
||||
_source: S,
|
||||
_head: StreamHead,
|
||||
_policy: StreamPolicy,
|
||||
) -> Result<StreamOutcome<C::Response>, StreamError>
|
||||
where
|
||||
C: StreamingCall,
|
||||
S: futures_util::Stream<
|
||||
Item = Result<<C::Contract as ServerStreamingContract>::Event, StreamError>,
|
||||
> + Send,
|
||||
{
|
||||
todo!(
|
||||
"Honor host demand, bound buffers, validate the terminator, retain late/failed usage, and emit cache/hooks/accounting once; detachment is cancellation"
|
||||
)
|
||||
}
|
||||
|
|
@ -1,2 +1,3 @@
|
|||
pub mod adapters;
|
||||
pub mod messages;
|
||||
pub mod responses_adapters;
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
pub mod streaming_iterator;
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
use std::collections::{BTreeMap, VecDeque};
|
||||
|
||||
use litellm_types::responses::streaming::{ResponseOutputItem, ResponsesEvent};
|
||||
|
||||
use crate::{
|
||||
anthropic::experimental_pass_through::messages::streaming_iterator::{
|
||||
AnthropicContentBlock, AnthropicMessagesStreamEvent,
|
||||
},
|
||||
base_llm::base_model_iterator::{
|
||||
AnthropicMessagesApi, ResponsesApi, StreamError, StreamPolicy, StreamTransformation,
|
||||
},
|
||||
};
|
||||
|
||||
pub struct AnthropicResponsesStreamWrapper {
|
||||
_model: String,
|
||||
_message_id: String,
|
||||
_current_block_index: Option<u64>,
|
||||
_item_id_to_block_index: BTreeMap<String, u64>,
|
||||
_pending_tool_ids: BTreeMap<String, String>,
|
||||
_sent_message_start: bool,
|
||||
_sent_message_stop: bool,
|
||||
_chunk_queue: VecDeque<AnthropicMessagesStreamEvent>,
|
||||
_refusal_text: String,
|
||||
_policy: StreamPolicy,
|
||||
}
|
||||
|
||||
impl AnthropicResponsesStreamWrapper {
|
||||
pub fn new(_model: String, _policy: StreamPolicy) -> Self {
|
||||
todo!(
|
||||
"Initialize one Responses-to-Messages conversion and record its message ID through the host"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn make_message_start(&self) -> AnthropicMessagesStreamEvent {
|
||||
todo!("Emit message_start once with model, stable ID and initial usage")
|
||||
}
|
||||
|
||||
pub fn next_block_index(&mut self) -> Result<u64, StreamError> {
|
||||
todo!("Allocate monotonically increasing content block indexes within stream limits")
|
||||
}
|
||||
|
||||
pub fn open_block(
|
||||
&mut self,
|
||||
_item_id: Option<&str>,
|
||||
_content_block: AnthropicContentBlock,
|
||||
) -> Result<u64, StreamError> {
|
||||
todo!("Associate item ID and block index, then queue content_block_start before deltas")
|
||||
}
|
||||
|
||||
pub fn close_reasoning_item(&mut self, _item: &ResponseOutputItem) -> Result<(), StreamError> {
|
||||
todo!("Preserve encrypted signatures or redacted thinking before content_block_stop")
|
||||
}
|
||||
|
||||
pub fn process_event(&mut self, _event: ResponsesEvent) -> Result<(), StreamError> {
|
||||
todo!(
|
||||
"Port _process_event: text/refusal, reasoning separators, tools and missing starts; retain failed/incomplete outcomes and terminal usage"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl StreamTransformation for AnthropicResponsesStreamWrapper {
|
||||
type Caller = AnthropicMessagesApi;
|
||||
type Upstream = ResponsesApi;
|
||||
type Error = StreamError;
|
||||
|
||||
fn transform_event(
|
||||
&mut self,
|
||||
_event: ResponsesEvent,
|
||||
) -> Result<Vec<AnthropicMessagesStreamEvent>, StreamError> {
|
||||
todo!(
|
||||
"Process one canonical Responses event and drain ordered Messages events within the queue limit"
|
||||
)
|
||||
}
|
||||
|
||||
fn finish(self) -> Result<Vec<AnthropicMessagesStreamEvent>, StreamError> {
|
||||
todo!(
|
||||
"Require an observed terminal response, close outstanding blocks, emit final usage/stop once, and reject truncation"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -17,6 +17,67 @@ pub struct ChatCompletions;
|
|||
pub struct ResponsesApi;
|
||||
pub struct AnthropicMessagesApi;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ApiDialect {
|
||||
ChatCompletions,
|
||||
Responses,
|
||||
AnthropicMessages,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum BufferedSource {
|
||||
Cache,
|
||||
NonStreamingResponse,
|
||||
Mock,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum DeliveryMode {
|
||||
Live,
|
||||
Buffered(BufferedSource),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct StreamPolicy {
|
||||
pub max_buffer_bytes: usize,
|
||||
pub max_pending_events: usize,
|
||||
pub max_tool_argument_bytes: usize,
|
||||
pub deadline: Option<std::time::Instant>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum StreamError {
|
||||
Upstream(String),
|
||||
InvalidEvent(String),
|
||||
UnexpectedEof,
|
||||
BufferLimitExceeded,
|
||||
Cancelled,
|
||||
DeadlineExceeded,
|
||||
Host(String),
|
||||
}
|
||||
|
||||
pub enum StreamOutcome<Response> {
|
||||
Completed(Response),
|
||||
Incomplete(Response),
|
||||
Failed {
|
||||
error: StreamError,
|
||||
response: Option<Response>,
|
||||
},
|
||||
Cancelled {
|
||||
response: Option<Response>,
|
||||
},
|
||||
}
|
||||
|
||||
pub struct MatchingDialectRelay {
|
||||
_dialect: ApiDialect,
|
||||
}
|
||||
|
||||
impl MatchingDialectRelay {
|
||||
pub fn new(_caller: ApiDialect, _upstream: ApiDialect) -> Result<Self, StreamError> {
|
||||
todo!("Reject raw relay unless caller and upstream use the same semantic dialect")
|
||||
}
|
||||
}
|
||||
|
||||
impl ServerStreamingContract for ChatCompletions {
|
||||
type Event = ChatCompletionChunk;
|
||||
}
|
||||
|
|
@ -47,10 +108,11 @@ pub struct TransformedStream<S, T: StreamTransformation> {
|
|||
transformation: Option<T>,
|
||||
pending: VecDeque<<T::Caller as ServerStreamingContract>::Event>,
|
||||
finished: bool,
|
||||
policy: StreamPolicy,
|
||||
}
|
||||
|
||||
impl<S, T: StreamTransformation> TransformedStream<S, T> {
|
||||
pub fn new(_upstream: S, _transformation: T) -> Self {
|
||||
pub fn new(_upstream: S, _transformation: T, _policy: StreamPolicy) -> Self {
|
||||
todo!("Retain one call's conversion state and a bounded pending-event queue")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1 +1,3 @@
|
|||
pub mod transformation;
|
||||
|
||||
pub mod streaming;
|
||||
|
|
|
|||
37
litellm-rust/crates/llms/src/base_llm/responses/streaming.rs
Normal file
37
litellm-rust/crates/llms/src/base_llm/responses/streaming.rs
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
use litellm_framing::sse::SseFrame;
|
||||
use litellm_types::responses::streaming::{ResponsesEvent, ResponsesResponse};
|
||||
|
||||
use crate::base_llm::base_model_iterator::{StreamError, StreamOutcome, StreamPolicy};
|
||||
|
||||
pub struct ResponsesSseDecoder {
|
||||
_policy: StreamPolicy,
|
||||
_terminal: Option<StreamOutcome<ResponsesResponse>>,
|
||||
}
|
||||
|
||||
impl ResponsesSseDecoder {
|
||||
pub fn new(_policy: StreamPolicy) -> Self {
|
||||
todo!(
|
||||
"Initialize semantic decoding downstream of SseFramer, which owns split UTF-8 and frame assembly"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn decode(&mut self, _frame: SseFrame) -> Result<Vec<ResponsesEvent>, StreamError> {
|
||||
todo!(
|
||||
"Decode canonical events, skip SSE control frames, handle DONE, preserve extensions and usage, and enforce event/buffer limits"
|
||||
)
|
||||
}
|
||||
|
||||
pub fn finish(self) -> Result<StreamOutcome<ResponsesResponse>, StreamError> {
|
||||
todo!(
|
||||
"Require a terminal Responses event; EOF or DONE alone cannot fabricate completed output"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decode_response(_body: &[u8]) -> Result<ResponsesResponse, StreamError> {
|
||||
todo!("Validate the full wire schema and retain provider extensions in the typed response")
|
||||
}
|
||||
|
||||
pub fn encode_event(_event: &ResponsesEvent) -> Result<String, StreamError> {
|
||||
todo!("Serialize canonical event names and payload fields for SSE or WebSocket delivery")
|
||||
}
|
||||
|
|
@ -1,16 +1,325 @@
|
|||
// TODO: Port the complete Python Responses models before enabling these streaming contracts
|
||||
pub struct ResponsesResponse {
|
||||
_schema: (),
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ProviderFields(pub BTreeMap<String, Value>);
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ResponseStatus {
|
||||
Queued,
|
||||
InProgress,
|
||||
Completed,
|
||||
Failed,
|
||||
Incomplete,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
pub struct ResponseOutputItem {
|
||||
_schema: (),
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ResponsesUsage {
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub input_token_details: ProviderFields,
|
||||
pub output_token_details: ProviderFields,
|
||||
pub extensions: ProviderFields,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ResponsesError {
|
||||
pub code: Option<String>,
|
||||
pub message: String,
|
||||
pub param: Option<String>,
|
||||
pub extensions: ProviderFields,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ResponseAnnotation {
|
||||
pub kind: String,
|
||||
pub start_index: Option<u64>,
|
||||
pub end_index: Option<u64>,
|
||||
pub url: Option<String>,
|
||||
pub title: Option<String>,
|
||||
pub extensions: ProviderFields,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum ResponseContentPart {
|
||||
OutputText {
|
||||
text: String,
|
||||
annotations: Box<[ResponseAnnotation]>,
|
||||
},
|
||||
Refusal {
|
||||
refusal: String,
|
||||
},
|
||||
SummaryText {
|
||||
text: String,
|
||||
},
|
||||
Extension {
|
||||
kind: String,
|
||||
fields: ProviderFields,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum ResponseOutputItem {
|
||||
Message {
|
||||
id: String,
|
||||
status: ResponseStatus,
|
||||
role: String,
|
||||
content: Box<[ResponseContentPart]>,
|
||||
phase: Option<String>,
|
||||
extensions: ProviderFields,
|
||||
},
|
||||
FunctionCall {
|
||||
id: String,
|
||||
call_id: String,
|
||||
name: String,
|
||||
namespace: Option<String>,
|
||||
arguments: String,
|
||||
status: ResponseStatus,
|
||||
extensions: ProviderFields,
|
||||
},
|
||||
CustomToolCall {
|
||||
id: String,
|
||||
call_id: String,
|
||||
name: String,
|
||||
input: String,
|
||||
status: Option<ResponseStatus>,
|
||||
extensions: ProviderFields,
|
||||
},
|
||||
Reasoning {
|
||||
id: String,
|
||||
summary: Box<[ResponseContentPart]>,
|
||||
encrypted_content: Option<String>,
|
||||
extensions: ProviderFields,
|
||||
},
|
||||
Extension {
|
||||
kind: String,
|
||||
id: Option<String>,
|
||||
fields: ProviderFields,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum ResponseInputContent {
|
||||
Text(String),
|
||||
Image {
|
||||
image_url: Option<String>,
|
||||
file_id: Option<String>,
|
||||
detail: Option<String>,
|
||||
},
|
||||
File {
|
||||
file_id: Option<String>,
|
||||
file_data: Option<String>,
|
||||
filename: Option<String>,
|
||||
},
|
||||
Extension {
|
||||
kind: String,
|
||||
fields: ProviderFields,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum ResponseInputItem {
|
||||
Message {
|
||||
role: String,
|
||||
content: Box<[ResponseInputContent]>,
|
||||
},
|
||||
FunctionCallOutput {
|
||||
call_id: String,
|
||||
output: String,
|
||||
},
|
||||
CustomToolCallOutput {
|
||||
call_id: String,
|
||||
output: String,
|
||||
},
|
||||
ItemReference {
|
||||
id: String,
|
||||
},
|
||||
Output(ResponseOutputItem),
|
||||
Extension {
|
||||
kind: String,
|
||||
fields: ProviderFields,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum ResponsesInput {
|
||||
Text(String),
|
||||
Items(Box<[ResponseInputItem]>),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum ResponseTool {
|
||||
Function {
|
||||
name: String,
|
||||
description: Option<String>,
|
||||
parameters: ProviderFields,
|
||||
strict: Option<bool>,
|
||||
},
|
||||
Custom {
|
||||
name: String,
|
||||
description: Option<String>,
|
||||
format: ProviderFields,
|
||||
},
|
||||
Namespace {
|
||||
name: String,
|
||||
description: Option<String>,
|
||||
tools: Box<[ResponseTool]>,
|
||||
},
|
||||
BuiltIn {
|
||||
kind: String,
|
||||
options: ProviderFields,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ResponsesRequest {
|
||||
_schema: (),
|
||||
pub model: Option<String>,
|
||||
pub input: Option<ResponsesInput>,
|
||||
pub previous_response_id: Option<String>,
|
||||
pub instructions: Option<String>,
|
||||
pub tools: Box<[ResponseTool]>,
|
||||
pub parallel_tool_calls: Option<bool>,
|
||||
pub max_output_tokens: Option<u64>,
|
||||
pub store: Option<bool>,
|
||||
pub background: Option<bool>,
|
||||
pub stream: Option<bool>,
|
||||
pub include: Box<[String]>,
|
||||
pub metadata: BTreeMap<String, String>,
|
||||
// TODO: Type remaining Python request options before enabling wire decoding; preserve them losslessly meanwhile
|
||||
pub extensions: ProviderFields,
|
||||
}
|
||||
|
||||
pub struct ResponsesEvent {
|
||||
_schema: (),
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ResponsesResponse {
|
||||
pub id: String,
|
||||
pub created_at: u64,
|
||||
pub model: Option<String>,
|
||||
pub status: Option<ResponseStatus>,
|
||||
pub output: Box<[ResponseOutputItem]>,
|
||||
pub usage: Option<ResponsesUsage>,
|
||||
pub error: Option<ResponsesError>,
|
||||
pub incomplete_reason: Option<String>,
|
||||
pub previous_response_id: Option<String>,
|
||||
pub extensions: ProviderFields,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ResponsesEvent {
|
||||
pub sequence_number: Option<u64>,
|
||||
pub data: ResponsesEventData,
|
||||
pub extensions: ProviderFields,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum ResponsesEventData {
|
||||
Created(ResponsesResponse),
|
||||
InProgress(ResponsesResponse),
|
||||
Completed(ResponsesResponse),
|
||||
Failed(ResponsesResponse),
|
||||
Incomplete(ResponsesResponse),
|
||||
OutputItemAdded {
|
||||
output_index: u64,
|
||||
item: ResponseOutputItem,
|
||||
},
|
||||
OutputItemDone {
|
||||
output_index: u64,
|
||||
item: ResponseOutputItem,
|
||||
},
|
||||
ContentPartAdded {
|
||||
item_id: String,
|
||||
output_index: u64,
|
||||
content_index: u64,
|
||||
part: ResponseContentPart,
|
||||
},
|
||||
ContentPartDone {
|
||||
item_id: String,
|
||||
output_index: u64,
|
||||
content_index: u64,
|
||||
part: ResponseContentPart,
|
||||
},
|
||||
OutputTextDelta {
|
||||
item_id: String,
|
||||
output_index: u64,
|
||||
content_index: u64,
|
||||
delta: String,
|
||||
},
|
||||
OutputTextDone {
|
||||
item_id: String,
|
||||
output_index: u64,
|
||||
content_index: u64,
|
||||
text: String,
|
||||
},
|
||||
AnnotationAdded {
|
||||
item_id: String,
|
||||
output_index: u64,
|
||||
content_index: u64,
|
||||
annotation_index: u64,
|
||||
annotation: ResponseAnnotation,
|
||||
},
|
||||
RefusalDelta {
|
||||
item_id: String,
|
||||
output_index: u64,
|
||||
content_index: u64,
|
||||
delta: String,
|
||||
},
|
||||
RefusalDone {
|
||||
item_id: String,
|
||||
output_index: u64,
|
||||
content_index: u64,
|
||||
refusal: String,
|
||||
},
|
||||
FunctionCallArgumentsDelta {
|
||||
item_id: String,
|
||||
output_index: u64,
|
||||
delta: String,
|
||||
},
|
||||
FunctionCallArgumentsDone {
|
||||
item_id: String,
|
||||
output_index: u64,
|
||||
arguments: String,
|
||||
},
|
||||
CustomToolCallInputDelta {
|
||||
item_id: String,
|
||||
output_index: u64,
|
||||
delta: String,
|
||||
},
|
||||
CustomToolCallInputDone {
|
||||
item_id: String,
|
||||
output_index: u64,
|
||||
input: String,
|
||||
},
|
||||
ReasoningSummaryPartAdded {
|
||||
item_id: String,
|
||||
output_index: u64,
|
||||
summary_index: u64,
|
||||
part: ResponseContentPart,
|
||||
},
|
||||
ReasoningSummaryPartDone {
|
||||
item_id: String,
|
||||
output_index: u64,
|
||||
summary_index: u64,
|
||||
part: ResponseContentPart,
|
||||
},
|
||||
ReasoningSummaryTextDelta {
|
||||
item_id: String,
|
||||
output_index: u64,
|
||||
summary_index: u64,
|
||||
delta: String,
|
||||
},
|
||||
ReasoningSummaryTextDone {
|
||||
item_id: String,
|
||||
output_index: u64,
|
||||
summary_index: u64,
|
||||
text: String,
|
||||
},
|
||||
Error(ResponsesError),
|
||||
Extension {
|
||||
kind: String,
|
||||
fields: ProviderFields,
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,11 @@
|
|||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
pub struct ResponsesWebSocketRequestDefaults {
|
||||
pub fill_missing: crate::responses::streaming::ProviderFields,
|
||||
pub overrides: crate::responses::streaming::ProviderFields,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ResponsesWsEventType {
|
||||
ResponseCreate,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue