mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-31 06:56:59 +00:00
5e38a08741
* refactor(cache): organize v2 cache as a package * docs: clarify experimental v2 guidance * fix(cache): verify cache-hit accounting and preserve logging metadata * refactor(cache): separate execution facts from host accounting * refactor(rust): build messages routes with named dependencies * wip * fix(cache): preserve facade policy and preflight fallback * refactor(cache): defer shared Python logging changes * test(gateway-inference): allow dead code in shared test helpers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cache): key prepared requests and honor facade controls * feat(cache): use Python caches from Rust Messages inference * refactor(cache): separate native and Python cache adapters * refactor(cache): enforce shared composition and adapter boundaries * fix(cache): let Python key delegated Rust Messages entries --------- Co-authored-by: Yujong Lee <yujong@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
79 lines
1.8 KiB
Rust
79 lines
1.8 KiB
Rust
use crate::{
|
|
interceptors::{RawResponse, RequestContext, WireRequest},
|
|
lifecycle::{CallEvent, Timing},
|
|
};
|
|
|
|
pub trait HookRuntime {
|
|
type Context<'a>;
|
|
type Arguments;
|
|
type Response;
|
|
type Chunk;
|
|
type Error;
|
|
type Step<H, T>;
|
|
|
|
fn ready<H, T>(value: T) -> Self::Step<H, T>;
|
|
}
|
|
|
|
pub type RuntimeCallEvent<'a, R> =
|
|
CallEvent<&'a <R as HookRuntime>::Response, &'a <R as HookRuntime>::Error, &'a RawResponse>;
|
|
|
|
pub trait CallHooks<R: HookRuntime>: Sized {
|
|
fn prepare_arguments(
|
|
&mut self,
|
|
_runtime: R::Context<'_>,
|
|
arguments: R::Arguments,
|
|
_started_at: f64,
|
|
) -> Result<R::Step<Self, R::Arguments>, R::Error> {
|
|
Ok(R::ready(arguments))
|
|
}
|
|
|
|
fn arguments_prepared(
|
|
&mut self,
|
|
_runtime: R::Context<'_>,
|
|
_arguments: &R::Arguments,
|
|
) -> Result<(), R::Error> {
|
|
Ok(())
|
|
}
|
|
|
|
fn before_provider_request(
|
|
&mut self,
|
|
_runtime: R::Context<'_>,
|
|
wire: Box<WireRequest>,
|
|
_context: &RequestContext,
|
|
) -> Result<R::Step<Self, Box<WireRequest>>, R::Error> {
|
|
Ok(R::ready(wire))
|
|
}
|
|
|
|
fn transform_response(
|
|
&mut self,
|
|
_runtime: R::Context<'_>,
|
|
response: R::Response,
|
|
_timing: Timing,
|
|
) -> Result<R::Step<Self, R::Response>, R::Error> {
|
|
Ok(R::ready(response))
|
|
}
|
|
|
|
fn on_event(
|
|
&mut self,
|
|
_runtime: R::Context<'_>,
|
|
_event: RuntimeCallEvent<'_, R>,
|
|
) -> Result<R::Step<Self, ()>, R::Error> {
|
|
Ok(R::ready(()))
|
|
}
|
|
|
|
fn on_stream_open(
|
|
&mut self,
|
|
_runtime: R::Context<'_>,
|
|
_head: &R::Response,
|
|
) -> Result<(), R::Error> {
|
|
Ok(())
|
|
}
|
|
|
|
fn on_stream_chunk(
|
|
&mut self,
|
|
_runtime: R::Context<'_>,
|
|
_chunk: &R::Chunk,
|
|
) -> Result<(), R::Error> {
|
|
Ok(())
|
|
}
|
|
}
|