mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-21 10:15:47 +00:00
* refactor(rust): extract litellm-host-native as the shared Rust host driver Move service and hook dispatch out of host-http into a Driver that owns the machine and Rust handlers, returning at completion or a stream boundary and holding the demand reply until the consumer advances. Move the in-process runner onto the same driver. host-http now layers encoding, SSE, body polling and lifecycle observation over it. host-python keeps driving litellm-host directly Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(rust): interrupt the machine when the in-process stream consumer fails Restores the pre-refactor interruption path for StreamConsumer errors via Driver::fail and ports the generic run lifecycle tests into host-native. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(rust): separate the machine contract from coroutine execution * auth update * refactor(rust): use standard flow control for host requests * style(rust): keep host driver imports formatted * chores * mostly relocation * refactor(rust): separate interceptors from queued observers * refactor(rust): centralize legacy callback mappings and lifecycle * docs: define Python host boundaries and migration plan * refactor: enforce Python host and bridge boundaries * refactor(rust): separate operations from callback composition * refactor(rust): compose SDK policy through call hooks --------- Co-authored-by: Yujong Lee <yujong@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
111 lines
3.7 KiB
Rust
111 lines
3.7 KiB
Rust
use litellm_host::observation::ObservationSender;
|
|
use std::{convert::Infallible, sync::Arc};
|
|
|
|
use axum::{body::Body, response::Response};
|
|
use bytes::Bytes;
|
|
use futures_util::{StreamExt, stream};
|
|
|
|
use litellm_host::{
|
|
call::{CallOutput, HostedCompletion, HostedMachine},
|
|
interceptors::Interceptors,
|
|
lifecycle::{observe_call, observe_unary},
|
|
machine::MachineFault,
|
|
protocol::Protocol,
|
|
};
|
|
use litellm_host_native::{Boundary, Driver, services::HostCallHandler};
|
|
|
|
use crate::{Error, ResponseEncoder, StreamEncoder};
|
|
|
|
type Output<E> = CallOutput<Response, http::Response<()>, Bytes, E>;
|
|
type HostedDriver<P, S, H> = Driver<HostedMachine<P>, S, H>;
|
|
|
|
pub async fn serve_unary<P, A, H, S>(
|
|
machine: HostedMachine<P>,
|
|
services: S,
|
|
interceptors: H,
|
|
encoder: A,
|
|
observers: Option<ObservationSender>,
|
|
) -> Result<Response, Error<P::Error>>
|
|
where
|
|
P: Protocol<Chunk = Infallible, StreamHead = Infallible>,
|
|
P::Error: From<MachineFault>,
|
|
H: Interceptors<P::Error>,
|
|
S: HostCallHandler<P>,
|
|
A: ResponseEncoder<Protocol = P>,
|
|
{
|
|
let mut driver = Driver::new(machine, services, interceptors);
|
|
observe_unary(observers, async move {
|
|
match driver.advance().await.map_err(Error::Call)? {
|
|
Boundary::Complete(HostedCompletion::Complete(value)) => {
|
|
encoder.encode_response(value).map_err(Error::Call)
|
|
}
|
|
_ => Err(Error::Protocol),
|
|
}
|
|
})
|
|
.await
|
|
}
|
|
|
|
pub async fn serve<P, A, H, S>(
|
|
machine: HostedMachine<P>,
|
|
services: S,
|
|
interceptors: H,
|
|
encoder: A,
|
|
observers: Option<ObservationSender>,
|
|
) -> Result<Response, Error<P::Error>>
|
|
where
|
|
P: Protocol,
|
|
P::Error: From<MachineFault>,
|
|
A: StreamEncoder<Protocol = P>,
|
|
H: Interceptors<P::Error> + 'static,
|
|
S: HostCallHandler<P> + 'static,
|
|
{
|
|
let encoder = Arc::new(encoder);
|
|
let driver = Driver::new(machine, services, interceptors);
|
|
match observe_call(observers, start(driver, encoder.clone())).await? {
|
|
CallOutput::Complete(response) => Ok(response),
|
|
CallOutput::Stream { head, chunks } => {
|
|
let body = chunks.map(move |chunk| {
|
|
Ok::<_, Infallible>(
|
|
chunk.unwrap_or_else(|error| encoder.encode_stream_error(error)),
|
|
)
|
|
});
|
|
Ok(head.map(|()| Body::from_stream(body)))
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn start<P, A, H, S>(
|
|
mut driver: HostedDriver<P, S, H>,
|
|
encoder: Arc<A>,
|
|
) -> Result<Output<Error<P::Error>>, Error<P::Error>>
|
|
where
|
|
P: Protocol,
|
|
P::Error: From<MachineFault>,
|
|
A: StreamEncoder<Protocol = P>,
|
|
H: Interceptors<P::Error> + 'static,
|
|
S: HostCallHandler<P> + 'static,
|
|
{
|
|
match driver.advance().await.map_err(Error::Call)? {
|
|
Boundary::Complete(HostedCompletion::Complete(value)) => encoder
|
|
.encode_response(value)
|
|
.map(CallOutput::Complete)
|
|
.map_err(Error::Call),
|
|
Boundary::Open(head) => {
|
|
let head = encoder.encode_stream_head(head).map_err(Error::Call)?;
|
|
let chunks =
|
|
stream::try_unfold((driver, encoder), |(mut driver, encoder)| async move {
|
|
match driver.advance().await.map_err(Error::Call)? {
|
|
Boundary::Chunk(chunk) => {
|
|
let bytes = encoder.encode_chunk(chunk).map_err(Error::Call)?;
|
|
Ok(Some((bytes, (driver, encoder))))
|
|
}
|
|
Boundary::Complete(HostedCompletion::StreamEnded) => Ok(None),
|
|
_ => Err(Error::Protocol),
|
|
}
|
|
})
|
|
.boxed();
|
|
Ok(CallOutput::Stream { head, chunks })
|
|
}
|
|
_ => Err(Error::Protocol),
|
|
}
|
|
}
|