litellm/litellm-rust/crates/host-http/src/driver.rs
devin-ai-integration[bot] e4190d86a6
refactor(rust): centralize host execution and compose callbacks (#43515)
* refactor(rust): extract litellm-host-native as the shared Rust host driver

Move service and hook dispatch out of host-http into a Driver that owns the
machine and Rust handlers, returning at completion or a stream boundary and
holding the demand reply until the consumer advances. Move the in-process
runner onto the same driver. host-http now layers encoding, SSE, body polling
and lifecycle observation over it. host-python keeps driving litellm-host
directly

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(rust): interrupt the machine when the in-process stream consumer fails

Restores the pre-refactor interruption path for StreamConsumer errors via
Driver::fail and ports the generic run lifecycle tests into host-native.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(rust): separate the machine contract from coroutine execution

* auth update

* refactor(rust): use standard flow control for host requests

* style(rust): keep host driver imports formatted

* chores

* mostly relocation

* refactor(rust): separate interceptors from queued observers

* refactor(rust): centralize legacy callback mappings and lifecycle

* docs: define Python host boundaries and migration plan

* refactor: enforce Python host and bridge boundaries

* refactor(rust): separate operations from callback composition

* refactor(rust): compose SDK policy through call hooks

---------

Co-authored-by: Yujong Lee <yujong@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-09-28 19:20:27 +00:00

111 lines
3.7 KiB
Rust

use litellm_host::observation::ObservationSender;
use std::{convert::Infallible, sync::Arc};
use axum::{body::Body, response::Response};
use bytes::Bytes;
use futures_util::{StreamExt, stream};
use litellm_host::{
call::{CallOutput, HostedCompletion, HostedMachine},
interceptors::Interceptors,
lifecycle::{observe_call, observe_unary},
machine::MachineFault,
protocol::Protocol,
};
use litellm_host_native::{Boundary, Driver, services::HostCallHandler};
use crate::{Error, ResponseEncoder, StreamEncoder};
type Output<E> = CallOutput<Response, http::Response<()>, Bytes, E>;
type HostedDriver<P, S, H> = Driver<HostedMachine<P>, S, H>;
pub async fn serve_unary<P, A, H, S>(
machine: HostedMachine<P>,
services: S,
interceptors: H,
encoder: A,
observers: Option<ObservationSender>,
) -> Result<Response, Error<P::Error>>
where
P: Protocol<Chunk = Infallible, StreamHead = Infallible>,
P::Error: From<MachineFault>,
H: Interceptors<P::Error>,
S: HostCallHandler<P>,
A: ResponseEncoder<Protocol = P>,
{
let mut driver = Driver::new(machine, services, interceptors);
observe_unary(observers, async move {
match driver.advance().await.map_err(Error::Call)? {
Boundary::Complete(HostedCompletion::Complete(value)) => {
encoder.encode_response(value).map_err(Error::Call)
}
_ => Err(Error::Protocol),
}
})
.await
}
pub async fn serve<P, A, H, S>(
machine: HostedMachine<P>,
services: S,
interceptors: H,
encoder: A,
observers: Option<ObservationSender>,
) -> Result<Response, Error<P::Error>>
where
P: Protocol,
P::Error: From<MachineFault>,
A: StreamEncoder<Protocol = P>,
H: Interceptors<P::Error> + 'static,
S: HostCallHandler<P> + 'static,
{
let encoder = Arc::new(encoder);
let driver = Driver::new(machine, services, interceptors);
match observe_call(observers, start(driver, encoder.clone())).await? {
CallOutput::Complete(response) => Ok(response),
CallOutput::Stream { head, chunks } => {
let body = chunks.map(move |chunk| {
Ok::<_, Infallible>(
chunk.unwrap_or_else(|error| encoder.encode_stream_error(error)),
)
});
Ok(head.map(|()| Body::from_stream(body)))
}
}
}
async fn start<P, A, H, S>(
mut driver: HostedDriver<P, S, H>,
encoder: Arc<A>,
) -> Result<Output<Error<P::Error>>, Error<P::Error>>
where
P: Protocol,
P::Error: From<MachineFault>,
A: StreamEncoder<Protocol = P>,
H: Interceptors<P::Error> + 'static,
S: HostCallHandler<P> + 'static,
{
match driver.advance().await.map_err(Error::Call)? {
Boundary::Complete(HostedCompletion::Complete(value)) => encoder
.encode_response(value)
.map(CallOutput::Complete)
.map_err(Error::Call),
Boundary::Open(head) => {
let head = encoder.encode_stream_head(head).map_err(Error::Call)?;
let chunks =
stream::try_unfold((driver, encoder), |(mut driver, encoder)| async move {
match driver.advance().await.map_err(Error::Call)? {
Boundary::Chunk(chunk) => {
let bytes = encoder.encode_chunk(chunk).map_err(Error::Call)?;
Ok(Some((bytes, (driver, encoder))))
}
Boundary::Complete(HostedCompletion::StreamEnded) => Ok(None),
_ => Err(Error::Protocol),
}
})
.boxed();
Ok(CallOutput::Stream { head, chunks })
}
_ => Err(Error::Protocol),
}
}