litellm/litellm-rust/crates/inference-chat/src/route.rs
yujonglee 0675c22d56
refactor(rust): inline chat provider resolution (#47218)
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-19 09:03:45 -07:00

46 lines
1.3 KiB
Rust

use std::convert::Infallible;
use litellm_host::{
call::{CallOutput, HostedMachine, hosted_call},
protocol::Protocol,
};
use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
use super::{ChatCompletionsRoute, Error, types::ChatCompletionsCall};
pub struct ChatCompletions;
impl Protocol for ChatCompletions {
type Response = ChatCompletionsResponse;
type Error = Error;
type Request = ChatCompletionsCall;
type HostCall = Infallible;
type Chunk = Infallible;
type StreamHead = Infallible;
}
impl ChatCompletionsRoute {
pub fn machine(
self,
call: ChatCompletionsCall,
options: impl Into<litellm_inference::CallOptions>,
) -> HostedMachine<ChatCompletions> {
let litellm_inference::CallOptions {
cache: cache_options,
observers,
} = options.into();
hosted_call(
call,
observers,
move |call, _, interceptors, observers| async move {
self.run(call, cache_options, &interceptors, observers.as_ref())
.await
.map(CallOutput::Complete)
},
)
}
}
impl litellm_inference::caching::Cachable for ChatCompletions {
const SURFACE: &'static str = "chat_completions";
}