This commit is contained in:
Yujong Lee 2026-09-19 21:00:51 -07:00
parent 57aa3f2895
commit e0ead49d96
63 changed files with 5269 additions and 632 deletions

View file

@ -2057,6 +2057,17 @@ dependencies = [
"thiserror 2.0.19",
]
[[package]]
name = "litellm-callbacks-v1-exthost"
version = "0.1.0"
dependencies = [
"litellm-callbacks-v1",
"litellm-host",
"serde",
"serde_json",
"thiserror 2.0.19",
]
[[package]]
name = "litellm-callbacks-v1-python"
version = "0.1.0"

View file

@ -14,6 +14,7 @@ litellm-host = { path = "crates/host" }
litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
litellm-callbacks-v1 = { path = "crates/callbacks-v1" }
litellm-callbacks-v1-python = { path = "crates/callbacks-v1-python" }
litellm-callbacks-v1-exthost = { path = "crates/callbacks-v1-exthost" }
litellm-framing = { path = "crates/framer" }
litellm-auth = { path = "crates/auth" }
litellm-auth-aws = { path = "crates/auth-aws" }

View file

@ -1,10 +1,10 @@
//! The Python contract of this crate: every function of
//! `litellm/rust_bridge/callbacks_legacy_python.py` that Rust calls, by name.
//! `litellm/rust_bridge/callbacks_legacy.py` that Rust calls, by name.
use pyo3::prelude::*;
use strum::{IntoStaticStr, VariantArray};
const MODULE: &str = "litellm.rust_bridge.callbacks_legacy_python";
const MODULE: &str = "litellm.rust_bridge.callbacks_legacy";
/// Every litellm Python internal the native call still borrows, grouped by the subsystem it
/// belongs to. Rust drives the call; these exist only so behaviour that Python owns today
@ -12,7 +12,7 @@ const MODULE: &str = "litellm.rust_bridge.callbacks_legacy_python";
/// A group is deleted once Rust owns that subsystem, so this enum only shrinks. Calling a
/// user's own callback is not borrowing and does not belong here.
///
/// `litellm/rust_bridge/callbacks_legacy_python.py` is the only Python module behind it, and
/// `litellm/rust_bridge/callbacks_legacy.py` is the only Python module behind it, and
/// `python_contract.json` pins each function's parameters on both sides.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum LegacyPython {

View file

@ -5,12 +5,12 @@ use pyo3::types::{PyDict, PyTuple};
use crate::{LegacyPythonLifecycle, LegacyPythonSurface, PublicCall};
/// The parameters of every `callbacks_legacy_python` function, as the real module declares them.
/// `tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py` pins this file to the Python
/// The parameters of every `callbacks.legacy` function, as the real module declares them.
/// `tests/test_litellm/rust_bridge/test_callbacks_legacy.py` pins this file to the Python
/// signatures, and [`namespace`] binds every fake call against it.
pub(crate) const PYTHON_CONTRACT: &str = include_str!("../python_contract.json");
/// Stand-ins for `callbacks_legacy_python`, the only Python module the crate calls. Tests
/// Stand-ins for `callbacks.legacy`, the only Python module the crate calls. Tests
/// share one interpreter and run concurrently, so each fake is installed idempotently and
/// forwards to the per-test `StubLogger` it is handed (directly, or as `kwargs['logger']`).
/// Every fake is bound against the contract first, so a call the real module would reject
@ -23,10 +23,10 @@ import sys
import traceback
import types
for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_legacy_python'):
for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_legacy'):
sys.modules.setdefault(name, types.ModuleType(name))
legacy = sys.modules['litellm.rust_bridge.callbacks_legacy_python']
legacy = sys.modules['litellm.rust_bridge.callbacks_legacy']
CONTRACT = json.loads(python_contract)

View file

@ -0,0 +1,18 @@
- Target invariants, not completion claims
- Keep this crate the out-of-process host of `litellm-callbacks-v1` and nothing else: the wire protocol to an extension host worker and the client that speaks it
- The protocol of a call (which envelope, to whom, how patches fold, patch validation) belongs to `litellm-callbacks-v1`'s `CallSession`; this crate sends what the session says and brings answers back to it. It never decides protocol and never validates a patch itself
- No PyO3 and no interpreter in this crate or in the process that uses it. The worker is `python -m litellm.callbacks_v1.host` (`litellm/callbacks_v1/host.py`), which loads callbacks with the SDK's own `register` and decorators; a callback file is the same file in the SDK and behind this host
- `callbacks-v1-python` is the sibling adapter for the SDK (in process, inline). The two share the session and nothing else
- Terms (contract terms are in `callbacks-v1/AGENTS.md`)
- Worker: the process that runs callbacks. Gateway: the process that owns calls and this crate
- Frame: a 4-byte big-endian length and one JSON object tagged by `type`. `ToWorker` is `event`, `intercept`, `flush`; `FromWorker` is `hello`, `patch`, `error`, `report`, `flushed`
- Hello: the worker's first frame, its `Subscription`s as data; `subscriber` in every later frame is an index into it
- Report: an observer's swallowed failure, told to the gateway for logging only
- The worker only answers. There is no frame by which it calls into the gateway; a host service (sending vendor HTTP, a secret by name) would be a new frame and a contract capability, never an escape hatch
- Everything from a worker is untrusted: frames are size-capped before they are read, unknown frame types and unknown fields are errors, and a patch is data until `Interception::patched` validates it
- This crate isolates nothing. What a worker can reach (environment, files, user, network) is what the process it runs in was given; the spawner clears the environment and picks the working directory, and the deployment (a sidecar, a sandbox runtime, another host) supplies the rest
- Observers are asynchronous to the call: `observe` writes and returns. Only `intercept` waits, and only it may carry a timeout or a failure policy
- An interceptor that raises is `ExtHostError::Interceptor`; whether that fails the call or skips the patch is the gateway's policy per subscriber, not the worker's
- `golden/` pins every frame for both sides: Rust serialises `to_worker` and parses `from_worker`; `tests/test_litellm/callbacks_v1/test_host.py` feeds the worker the first and expects the second. The frames embed the contract's own golden envelope and request, so the two golden directories cannot drift
- `Worker` is the blocking reference client: what a conforming gateway does, in the simplest form that can be tested end to end. The production host (async, supervised, a bounded queue per subscriber, restart, timeouts) replaces the client and keeps the protocol
- `tests/worker.rs` is end to end against a real Python worker and is `#[ignore]`d because it needs an interpreter that can import `litellm`; run it with `PYO3_PYTHON=... cargo test -p litellm-callbacks-v1-exthost -- --ignored`

View file

@ -0,0 +1,15 @@
[package]
name = "litellm-callbacks-v1-exthost"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
litellm-callbacks-v1.workspace = true
serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
[dev-dependencies]
litellm-host.workspace = true

View file

@ -0,0 +1,6 @@
{
"type": "error",
"id": 1,
"error_class": "builtins.PermissionError",
"message": "not this model"
}

View file

@ -0,0 +1,4 @@
{
"type": "flushed",
"id": 2
}

View file

@ -0,0 +1,21 @@
{
"type": "hello",
"schema": 1,
"subscriptions": [
{
"name": "recorder",
"events": [
"call.failed",
"call.succeeded"
],
"observes": true,
"intercepts": false
},
{
"name": "my_pkg.cb.patch",
"events": [],
"observes": false,
"intercepts": true
}
]
}

View file

@ -0,0 +1,20 @@
{
"type": "patch",
"id": 1,
"patch": {
"headers": {
"remove": [
"X-Trace"
],
"set": [
[
"X-Team",
"core"
]
]
},
"body": {
"input": "patched"
}
}
}

View file

@ -0,0 +1,5 @@
{
"type": "patch",
"id": 1,
"patch": null
}

View file

@ -0,0 +1,6 @@
{
"type": "report",
"subscriber": 0,
"event": "call.started",
"message": "boom"
}

View file

@ -0,0 +1,21 @@
{
"type": "event",
"subscriber": 0,
"envelope": {
"schema": 1,
"call_id": "call-123",
"call_type": "ocr",
"seq": 0,
"event": {
"type": "call.started",
"start_time": 10.0,
"asynchronous": true,
"metadata": {
"team": "core"
},
"metadata_dropped": [
"broken"
]
}
}
}

View file

@ -0,0 +1,4 @@
{
"type": "flush",
"id": 2
}

View file

@ -0,0 +1,22 @@
{
"type": "intercept",
"id": 1,
"subscriber": 1,
"request": {
"model": "model",
"custom_llm_provider": "provider",
"optional_params": {
"temperature": 0.2
},
"url": "https://provider.example/v1",
"headers": [
[
"Authorization",
"[REDACTED]"
]
],
"body": {
"input": "hello"
}
}
}

View file

@ -0,0 +1,15 @@
//! The out-of-process host of the v1 callback contract: the gateway's side of the socket to
//! an extension host worker (`python -m litellm.callbacks_v1.host`). The gateway never loads
//! a callback; the worker reports its subscriptions, and from then on this crate sends it
//! what a [`CallSession`](litellm_callbacks_v1::CallSession) says to deliver and brings
//! back interceptor patches for the session to validate.
//!
//! [`protocol`] is the wire, pinned by `golden/` on both sides. [`Worker`] is the blocking
//! reference client: what a conforming gateway does, in the simplest form that can be
//! tested end to end. The production host (async, supervised, queued) reuses the protocol
//! and replaces the client.
pub mod protocol;
mod worker;
pub use worker::{ExtHostError, Report, Worker};

View file

@ -0,0 +1,254 @@
//! Frames: a 4-byte big-endian length, then that many bytes of JSON, one object tagged by
//! `type`. The worker only answers; there is no frame by which it calls into the gateway.
//! Everything a worker sends is untrusted: sizes are capped, unknown frames and fields are
//! errors, and a patch is only data until the session validates it.
use std::io::{self, Read, Write};
use litellm_callbacks_v1::{Envelope, RequestFacts, Subscription, WirePatch};
use serde::{Deserialize, Serialize};
pub const MAX_FRAME_BYTES: usize = 64 * 1024 * 1024;
/// `subscriber` is an index into the hello's subscriptions.
#[derive(Debug, Serialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ToWorker<'a> {
/// Fire and forget: nothing an observer does comes back, except a [`FromWorker::Report`].
Event {
subscriber: usize,
envelope: &'a Envelope,
},
Intercept {
id: u64,
subscriber: usize,
request: &'a RequestFacts,
},
/// Answered by `Flushed` once every event sent before it has been handled.
Flush { id: u64 },
}
#[derive(Debug, PartialEq, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
pub enum FromWorker {
Hello {
schema: u32,
subscriptions: Vec<Subscription>,
},
/// `None` is an interceptor that returned nothing.
Patch {
id: u64,
patch: Option<WirePatch>,
},
Error {
id: u64,
error_class: String,
message: String,
},
/// An observer raised; the worker swallowed it, as the contract says.
Report {
subscriber: usize,
event: Option<String>,
message: String,
},
Flushed {
id: u64,
},
}
pub fn write_frame(writer: &mut impl Write, frame: &ToWorker<'_>) -> io::Result<()> {
let body = serde_json::to_vec(frame)?;
let length = u32::try_from(body.len())
.ok()
.filter(|_| body.len() <= MAX_FRAME_BYTES)
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "frame too large"))?;
writer.write_all(&length.to_be_bytes())?;
writer.write_all(&body)?;
writer.flush()
}
/// The next frame, or `None` when the worker closed the connection between frames.
pub fn read_frame(reader: &mut impl Read) -> io::Result<Option<FromWorker>> {
let mut header = [0u8; 4];
match reader.read_exact(&mut header) {
Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),
other => other?,
}
let length = u32::from_be_bytes(header) as usize;
if length > MAX_FRAME_BYTES {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("frame of {length} bytes exceeds {MAX_FRAME_BYTES}"),
));
}
let mut body = vec![0u8; length];
reader.read_exact(&mut body)?;
serde_json::from_slice(&body)
.map(Some)
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use litellm_callbacks_v1::{CallFacts, CallSession, EventKind, HeaderPatch};
use litellm_host::event::{RequestContext, WireRequest};
use serde_json::{Value, json};
use super::*;
fn subscriptions() -> Vec<Subscription> {
vec![
Subscription {
name: "recorder".to_string(),
events: BTreeSet::from([EventKind::CallFailed, EventKind::CallSucceeded]),
observes: true,
intercepts: false,
},
Subscription {
name: "my_pkg.cb.patch".to_string(),
events: BTreeSet::new(),
observes: false,
intercepts: true,
},
]
}
fn golden(text: &str) -> Value {
serde_json::from_str(text).unwrap()
}
/// The frames carry the contract's own golden envelope and request, so the two golden
/// directories cannot drift apart.
#[test]
fn frames_to_the_worker_match_golden() {
let facts = CallFacts {
start_time: 10.0,
asynchronous: true,
metadata: json!({"team": "core"}).as_object().unwrap().clone(),
metadata_dropped: vec!["broken".to_string()],
};
let (session, started) =
CallSession::start("call-123".to_string(), "ocr", subscriptions(), facts);
let interception = session.interception(
WireRequest {
url: "https://provider.example/v1".to_string(),
headers: vec![("Authorization".to_string(), "secret".to_string())],
body: json!({"input": "hello"}),
},
RequestContext {
model: "model".to_string(),
custom_llm_provider: "provider".to_string(),
optional_params: json!({"temperature": 0.2}),
secret_fields: Vec::new(),
api_key: None,
},
);
let turn = interception.turn().unwrap();
let frames = [
(
ToWorker::Event {
subscriber: 0,
envelope: &started.envelope,
},
include_str!("../golden/to_worker/event.json"),
),
(
ToWorker::Intercept {
id: 1,
subscriber: turn.subscriber,
request: &turn.request,
},
include_str!("../golden/to_worker/intercept.json"),
),
(
ToWorker::Flush { id: 2 },
include_str!("../golden/to_worker/flush.json"),
),
];
for (frame, expected) in frames {
assert_eq!(serde_json::to_value(&frame).unwrap(), golden(expected));
}
}
#[test]
fn frames_from_the_worker_parse_from_golden() {
let parsed = |text: &str| serde_json::from_str::<FromWorker>(text).unwrap();
assert_eq!(
parsed(include_str!("../golden/from_worker/hello.json")),
FromWorker::Hello {
schema: 1,
subscriptions: subscriptions()
}
);
assert_eq!(
parsed(include_str!("../golden/from_worker/patch.json")),
FromWorker::Patch {
id: 1,
patch: Some(WirePatch {
headers: HeaderPatch {
remove: vec!["X-Trace".to_string()],
set: vec![("X-Team".to_string(), "core".to_string())],
},
body: Some(json!({"input": "patched"})),
}),
}
);
assert_eq!(
parsed(include_str!("../golden/from_worker/patch_none.json")),
FromWorker::Patch { id: 1, patch: None }
);
assert_eq!(
parsed(include_str!("../golden/from_worker/error.json")),
FromWorker::Error {
id: 1,
error_class: "builtins.PermissionError".to_string(),
message: "not this model".to_string(),
}
);
assert_eq!(
parsed(include_str!("../golden/from_worker/report.json")),
FromWorker::Report {
subscriber: 0,
event: Some("call.started".to_string()),
message: "boom".to_string(),
}
);
assert_eq!(
parsed(include_str!("../golden/from_worker/flushed.json")),
FromWorker::Flushed { id: 2 }
);
}
fn framed(body: &[u8]) -> Vec<u8> {
[&(body.len() as u32).to_be_bytes()[..], body].concat()
}
#[test]
fn a_worker_cannot_say_anything_the_protocol_does_not_name() {
for body in [
&br#"{"type":"call_gateway","method":"get_secret"}"#[..],
br#"{"type":"flushed","id":1,"extra":true}"#,
br#"{"type":"patch","id":1,"patch":{"url":"https://evil.example"}}"#,
br#"[1,2,3]"#,
] {
let error = read_frame(&mut framed(body).as_slice()).unwrap_err();
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
}
}
#[test]
fn an_oversized_frame_is_refused_before_it_is_read() {
let header = ((MAX_FRAME_BYTES + 1) as u32).to_be_bytes();
let error = read_frame(&mut header.as_slice()).unwrap_err();
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
}
#[test]
fn a_closed_connection_between_frames_is_the_end_not_an_error() {
assert_eq!(read_frame(&mut [].as_slice()).unwrap(), None);
}
}

View file

@ -0,0 +1,214 @@
//! The blocking reference client of one worker. It starts the worker on a unix socket it
//! listens on (so nothing but a path is handed over, and an attached sidecar connects the
//! same way), reads the hello, and then does exactly what a [`CallSession`] says: sends an
//! emission's envelope to its observers and asks each interceptor for its patch in turn.
//! The session, not this client, decides the protocol and validates every patch.
use std::{
io,
os::unix::net::{UnixListener, UnixStream},
path::Path,
process::{Child, Command},
};
use litellm_callbacks_v1::{Emission, Interception, PatchError, SCHEMA_V1, Subscription};
use thiserror::Error;
use crate::protocol::{FromWorker, ToWorker, read_frame, write_frame};
#[derive(Debug, Error)]
pub enum ExtHostError {
#[error("extension host i/o: {0}")]
Io(#[from] io::Error),
#[error("extension host protocol: {0}")]
Protocol(String),
#[error("worker speaks schema {0}, this gateway speaks {SCHEMA_V1}")]
Schema(u32),
/// The interceptor raised. Whether that fails the call or skips the patch is the
/// gateway's policy for this subscriber, not the worker's.
#[error("callback {subscriber}: {error_class}: {message}")]
Interceptor {
subscriber: String,
error_class: String,
message: String,
},
#[error("callback {subscriber}: {error}")]
Patch {
subscriber: String,
error: PatchError,
},
}
/// An observer's swallowed failure, as the worker reported it.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Report {
pub subscriber: String,
pub event: Option<String>,
pub message: String,
}
pub struct Worker {
stream: UnixStream,
child: Option<Child>,
subscriptions: Vec<Subscription>,
reports: Vec<Report>,
next_id: u64,
}
impl Worker {
/// Starts `command` with `--connect <socket>` appended and waits for its hello. What
/// the worker can reach is whatever `command` was given: its environment, its working
/// directory, its user. This crate grants nothing.
pub fn spawn(mut command: Command, socket: &Path) -> Result<Self, ExtHostError> {
let listener = UnixListener::bind(socket)?;
let child = command.arg("--connect").arg(socket).spawn()?;
let (stream, _) = listener.accept()?;
Self::greeted(stream, Some(child))
}
/// Takes a worker that connected by itself, a sidecar for example.
pub fn attach(stream: UnixStream) -> Result<Self, ExtHostError> {
Self::greeted(stream, None)
}
fn greeted(mut stream: UnixStream, child: Option<Child>) -> Result<Self, ExtHostError> {
match read_frame(&mut stream)? {
Some(FromWorker::Hello {
schema: SCHEMA_V1,
subscriptions,
}) => Ok(Self {
stream,
child,
subscriptions,
reports: Vec::new(),
next_id: 0,
}),
Some(FromWorker::Hello { schema, .. }) => Err(ExtHostError::Schema(schema)),
other => Err(ExtHostError::Protocol(format!(
"expected hello, got {other:?}"
))),
}
}
/// What the worker's callbacks subscribed to: the session's subscriptions for every
/// call this worker serves.
pub fn subscriptions(&self) -> &[Subscription] {
&self.subscriptions
}
fn name(&self, subscriber: usize) -> String {
self.subscriptions
.get(subscriber)
.map_or_else(|| format!("#{subscriber}"), |found| found.name.clone())
}
fn id(&mut self) -> u64 {
self.next_id += 1;
self.next_id
}
/// Sends the envelope to each of its observers and returns at once: observers are
/// asynchronous to the call.
pub fn observe(&mut self, emission: &Emission) -> Result<(), ExtHostError> {
for &subscriber in &emission.observers {
write_frame(
&mut self.stream,
&ToWorker::Event {
subscriber,
envelope: &emission.envelope,
},
)?;
}
Ok(())
}
/// The next frame that is not a report; reports are kept for [`Worker::flush`].
fn answer(&mut self) -> Result<FromWorker, ExtHostError> {
loop {
match read_frame(&mut self.stream)? {
Some(FromWorker::Report {
subscriber,
event,
message,
}) => {
let subscriber = self.name(subscriber);
self.reports.push(Report {
subscriber,
event,
message,
});
}
Some(frame) => return Ok(frame),
None => return Err(ExtHostError::Protocol("worker closed".to_string())),
}
}
}
/// Asks every interceptor for its patch, in the session's order, and folds each into
/// the interception, which validates it.
pub fn intercept(&mut self, interception: Interception) -> Result<Interception, ExtHostError> {
let mut current = interception;
while let Some(turn) = current.turn() {
let id = self.id();
write_frame(
&mut self.stream,
&ToWorker::Intercept {
id,
subscriber: turn.subscriber,
request: &turn.request,
},
)?;
let subscriber = self.name(turn.subscriber);
current = match self.answer()? {
FromWorker::Patch {
id: answered,
patch,
} if answered == id => current
.patched(patch.unwrap_or_default())
.map_err(|error| ExtHostError::Patch { subscriber, error })?,
FromWorker::Error {
id: answered,
error_class,
message,
} if answered == id => {
return Err(ExtHostError::Interceptor {
subscriber,
error_class,
message,
});
}
other => {
return Err(ExtHostError::Protocol(format!(
"expected the answer to intercept {id}, got {other:?}"
)));
}
};
}
Ok(current)
}
/// Waits until the worker has handled every event sent so far, and returns what its
/// observers reported since the last flush.
pub fn flush(&mut self) -> Result<Vec<Report>, ExtHostError> {
let id = self.id();
write_frame(&mut self.stream, &ToWorker::Flush { id })?;
match self.answer()? {
FromWorker::Flushed { id: answered } if answered == id => {
Ok(std::mem::take(&mut self.reports))
}
other => Err(ExtHostError::Protocol(format!(
"expected flushed {id}, got {other:?}"
))),
}
}
}
impl Drop for Worker {
/// Closing the socket is how a worker is told to stop; it exits once its queues drain.
fn drop(&mut self) {
let _ = self.stream.shutdown(std::net::Shutdown::Both);
if let Some(child) = self.child.as_mut() {
let _ = child.wait();
}
}
}

View file

@ -0,0 +1,239 @@
//! End to end: a real `CallSession` driven through a real Python worker. The callback file
//! below is an ordinary SDK callback; nothing in it knows it runs out of process.
//!
//! Ignored by default because it needs an interpreter that can import `litellm`:
//!
//! PYO3_PYTHON=/path/to/.venv/bin/python cargo test -p litellm-callbacks-v1-exthost -- --ignored
use std::{
env, fs,
path::{Path, PathBuf},
process::Command,
};
use litellm_callbacks_v1::{CallFacts, CallSession, ErrorFacts, PatchError};
use litellm_callbacks_v1_exthost::{ExtHostError, Report, Worker};
use litellm_host::event::{FailureOrigin, RequestContext, Timing, WireRequest};
use serde_json::{Map, Value, json};
const CALLBACKS: &str = r#"
import json
import os
from litellm.callbacks_v1 import before_send, on_event
LOG = os.environ["CALLBACK_LOG"]
@on_event("call.started", "request.sending", "response.received", "call.succeeded", name="recorder")
async def record(event):
with open(LOG, "a") as log:
log.write(json.dumps([event["seq"], event["event"]["type"], event["event"].get("body")]) + "\n")
@on_event("call.started", name="flaky")
def flaky(event):
raise ValueError("boom")
@before_send
def tag(request):
return {"headers": {"set": [["X-Team", "core"]]}, "body": {**request["body"], "tagged": True}}
@before_send
async def guard(request):
if request["body"].get("input") == "steal":
return {"headers": {"set": [["Authorization", "stolen"]]}}
if request["body"].get("input") == "refuse":
raise PermissionError("not this input")
return None
"#;
const TIMING: Timing = Timing {
start_time: 10.0,
end_time: 12.5,
};
struct Scratch(PathBuf);
impl Scratch {
fn new(name: &str) -> Self {
let path = env::temp_dir().join(format!("exthost-{}-{name}", std::process::id()));
let _ = fs::remove_dir_all(&path);
fs::create_dir_all(path.join("modules")).unwrap();
fs::create_dir_all(path.join("cwd")).unwrap();
fs::write(path.join("modules/gateway_callbacks.py"), CALLBACKS).unwrap();
Self(path)
}
fn log(&self) -> Vec<Value> {
fs::read_to_string(self.0.join("log"))
.unwrap_or_default()
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect()
}
}
impl Drop for Scratch {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
/// The worker gets a cleared environment and an empty working directory: it sees what it
/// is given and nothing of the gateway's.
fn worker(scratch: &Scratch) -> Worker {
let python = env::var("PYO3_PYTHON").expect("PYO3_PYTHON names an interpreter with litellm");
let repository = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..");
let python_path = env::join_paths([repository, scratch.0.join("modules")]).unwrap();
let mut command = Command::new(python);
command
.args([
"-m",
"litellm.callbacks_v1.host",
"--load",
"gateway_callbacks",
])
.env_clear()
.env("PYTHONPATH", python_path)
.env("LITELLM_MODE", "PRODUCTION")
.env("CALLBACK_LOG", scratch.0.join("log"))
.current_dir(scratch.0.join("cwd"));
Worker::spawn(command, &scratch.0.join("socket")).unwrap()
}
fn wire(input: &str) -> WireRequest {
WireRequest {
url: "https://provider.example/v1".to_string(),
headers: vec![("Authorization".to_string(), "secret".to_string())],
body: json!({"input": input}),
}
}
fn context() -> RequestContext {
RequestContext {
model: "model".to_string(),
custom_llm_provider: "provider".to_string(),
optional_params: json!({}),
secret_fields: Vec::new(),
api_key: None,
}
}
fn start(worker: &mut Worker) -> CallSession {
let facts = CallFacts {
start_time: TIMING.start_time,
asynchronous: true,
metadata: Map::new(),
metadata_dropped: Vec::new(),
};
let (session, started) = CallSession::start(
"call-123".to_string(),
"ocr",
worker.subscriptions().to_vec(),
facts,
);
worker.observe(&started).unwrap();
session
}
#[test]
#[ignore = "needs PYO3_PYTHON to name an interpreter that can import litellm"]
fn a_call_reaches_python_callbacks_and_their_patches_reach_the_wire() {
let scratch = Scratch::new("call");
let mut worker = worker(&scratch);
let names: Vec<&str> = worker
.subscriptions()
.iter()
.map(|subscription| subscription.name.as_str())
.collect();
assert_eq!(
names,
[
"recorder",
"flaky",
"gateway_callbacks.tag",
"gateway_callbacks.guard"
]
);
let mut session = start(&mut worker);
let interception = worker
.intercept(session.interception(wire("hello"), context()))
.unwrap();
let (sending, sent) = session.request_sending(interception);
worker.observe(&sending).unwrap();
worker
.observe(&session.response_received("{}".to_string()))
.unwrap();
worker
.observe(&session.succeeded(TIMING, json!({"id": "response"}), None))
.unwrap();
assert_eq!(sent.body, json!({"input": "hello", "tagged": true}));
assert!(
sent.headers
.contains(&("X-Team".to_string(), "core".to_string()))
);
assert!(
sent.headers
.contains(&("Authorization".to_string(), "secret".to_string()))
);
assert_eq!(
worker.flush().unwrap(),
vec![Report {
subscriber: "flaky".to_string(),
event: Some("call.started".to_string()),
message: "boom".to_string(),
}]
);
assert_eq!(
scratch.log(),
vec![
json!([0, "call.started", null]),
json!([1, "request.sending", {"input": "hello", "tagged": true}]),
json!([2, "response.received", "{}"]),
json!([3, "call.succeeded", null]),
]
);
}
#[test]
#[ignore = "needs PYO3_PYTHON to name an interpreter that can import litellm"]
fn a_worker_cannot_patch_a_credential_and_its_refusal_is_the_gateways_to_act_on() {
let scratch = Scratch::new("refusals");
let mut worker = worker(&scratch);
let session = start(&mut worker);
let stolen = worker
.intercept(session.interception(wire("steal"), context()))
.unwrap_err();
assert!(matches!(
stolen,
ExtHostError::Patch { ref subscriber, error: PatchError::ProtectedHeader(_) }
if subscriber == "gateway_callbacks.guard"
));
let refused = worker
.intercept(session.interception(wire("refuse"), context()))
.unwrap_err();
assert!(matches!(
refused,
ExtHostError::Interceptor { ref error_class, ref message, .. }
if error_class == "builtins.PermissionError" && message == "not this input"
));
let failed = session.failed(
TIMING,
FailureOrigin::Call,
ErrorFacts {
class: "builtins.PermissionError".to_string(),
message: "not this input".to_string(),
status_code: None,
},
);
assert!(failed.observers.is_empty());
worker.flush().unwrap();
}

View file

@ -1,7 +1,9 @@
- Target invariants, not completion claims
- Keep this crate the Python runtime adapter for `litellm-callbacks-v1` and nothing else: the subscriber snapshot and one `PythonLifecycle` that delivers the core contract
- Envelopes, sequencing, redaction, handler selection and wire patch validation belong to `litellm-callbacks-v1`
- `litellm/rust_bridge/callbacks_v1_python.py` is the only Python module behind it; `python_contract.json` pins its functions on both sides
- Envelopes, sequencing, routing, the interceptor fold, redaction, handler selection and wire patch validation belong to `litellm-callbacks-v1`; this adapter drives its `CallSession` and adds only projection (Python arguments, response and exception into facts) and execution (calling handlers, `Await`/`resume`, cancellation, GC traversal)
- The snapshot splits each subscriber into the contract's `Subscription` and this crate's `Handlers`, kept at the same index
- `litellm/rust_bridge/callbacks_v1.py` is the only Python module Rust imports; `python_contract.json` pins its functions on both sides
- The typed authoring surface (callback protocols, envelope and patch types, the registry) is `litellm/callbacks_v1/__init__.py`; the bridge module re-exports `snapshot` from it and otherwise holds only marshalling
- Terms (contract terms are in `callbacks-v1/AGENTS.md`, lifecycle terms in `host-python/AGENTS.md`)
- Registry: the process-wide subscriber list; it lives in Python (`register`/`unregister`), never in Rust
- Snapshot: that list read once per call, with one observer and one interceptor handler already selected per subscriber for the call's mode (`subscribers.rs`)

View file

@ -1,23 +1,25 @@
//! The v1 lifecycle: each step builds one envelope, hands every subscribed observer a
//! fresh conversion of it, and, before the request is sent, folds every interceptor's
//! patch into the wire request.
//! The v1 lifecycle in CPython. The contract's [`CallSession`] decides which envelope a
//! step produces, who receives it and how patches fold into the wire; this adapter only
//! projects Python values into facts and runs the Python handlers: every observer of an
//! emission gets a fresh conversion of its envelope, inline, and every interceptor's
//! answer goes back to the session.
use litellm_callbacks_v1::{
CallFacts, Envelope, ErrorFacts, Event, RequestFacts, Sequencer, WirePatch, apply,
CallFacts, CallSession, Emission, Envelope, ErrorFacts, Interception, Subscription, WirePatch,
};
use litellm_host::event::{MachineEvent, RequestContext, WireRequest};
use litellm_host_python::{
LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, is_cancellation, missing_state, to_py,
};
use pyo3::exceptions::PyValueError;
use pyo3::gc::{PyTraverseError, PyVisit};
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyString};
use pyo3::{
exceptions::PyValueError,
gc::{PyTraverseError, PyVisit},
prelude::*,
types::{PyDict, PyString},
};
use serde_json::{Map, Value};
use crate::call::V1PythonSurface;
use crate::python::V1Python;
use crate::subscribers::Subscriber;
use crate::{call::V1PythonSurface, python::V1Python, subscribers::Handlers};
/// What a step hands back to the driver once every observer of its envelope has run.
enum Continuation {
@ -26,26 +28,29 @@ enum Continuation {
Wire(Box<WireRequest>),
}
/// The suspended step; `index` is the subscriber whose awaitable the driver is awaiting.
/// The suspended step, while the driver awaits one handler's awaitable.
enum Pending {
/// `position` is the place in `emission.observers` of the observer being awaited.
Observe {
envelope: Envelope,
index: usize,
emission: Emission,
position: usize,
then: Continuation,
},
/// `subscriber` is the interceptor whose turn it is.
Intercept {
wire: Box<WireRequest>,
context: RequestContext,
index: usize,
interception: Interception,
subscriber: usize,
},
}
pub struct V1PythonLifecycle {
surface: V1PythonSurface,
subscribers: Vec<Subscriber>,
/// Held until `begin` learns the call's identity and hands them to the session.
subscriptions: Vec<Subscription>,
/// The Python side of each subscription, at the same index.
handlers: Vec<Handlers>,
asynchronous: bool,
sequencer: Option<Sequencer>,
streamed: bool,
session: Option<CallSession>,
pending: Option<Pending>,
closed: bool,
}
@ -70,27 +75,21 @@ impl Continuation {
impl V1PythonLifecycle {
pub(crate) fn new(
surface: V1PythonSurface,
subscribers: Vec<Subscriber>,
subscriptions: Vec<Subscription>,
handlers: Vec<Handlers>,
asynchronous: bool,
) -> Self {
Self {
surface,
subscribers,
subscriptions,
handlers,
asynchronous,
sequencer: None,
streamed: false,
session: None,
pending: None,
closed: false,
}
}
fn envelope(&mut self, event: Event) -> PyResult<Envelope> {
self.sequencer
.as_mut()
.map(|sequencer| sequencer.envelope(event))
.ok_or_else(missing_state)
}
fn report(
&self,
py: Python<'_>,
@ -98,7 +97,7 @@ impl V1PythonLifecycle {
envelope: &Envelope,
error: PyErr,
) -> PyResult<()> {
let name = &self.subscribers[index].name;
let name = &self.handlers[index].name;
let event: &'static str = envelope.event.kind().into();
match V1Python::Report.call(py, (name, event, error.value(py))) {
Err(report_error) if is_cancellation(py, &report_error) => Err(report_error),
@ -106,86 +105,87 @@ impl V1PythonLifecycle {
}
}
/// Runs the emission's observers inline from `from`, which is one legal schedule of
/// the contract and nothing a callback may rely on.
fn observe(
&mut self,
py: Python<'_>,
envelope: Envelope,
emission: Emission,
from: usize,
then: Continuation,
) -> PyResult<LifecycleStep> {
for index in from..self.subscribers.len() {
let Some(handler) = self.subscribers[index]
.observer(envelope.event.kind())
for position in from..emission.observers.len() {
let index = emission.observers[position];
let Some(handler) = self.handlers[index]
.observe
.as_ref()
.map(|handler| handler.clone_ref(py))
else {
continue;
};
let outcome = to_py(py, &envelope).and_then(|argument| handler.call(py, argument));
let outcome =
to_py(py, &emission.envelope).and_then(|argument| handler.call(py, argument));
match outcome {
Ok(awaitable) if handler.is_async() => {
self.pending = Some(Pending::Observe {
envelope,
index,
emission,
position,
then,
});
return Ok(LifecycleStep::Await(awaitable.unbind()));
}
Ok(_) => {}
Err(error) if is_cancellation(py, &error) => return Err(error),
Err(error) => self.report(py, index, &envelope, error)?,
Err(error) => self.report(py, index, &emission.envelope, error)?,
}
}
Ok(then.into_step())
}
fn apply_patch(
fn patched(
&self,
py: Python<'_>,
index: usize,
wire: WireRequest,
interception: Interception,
subscriber: usize,
result: Py<PyAny>,
) -> PyResult<WireRequest> {
) -> PyResult<Interception> {
let patch = if result.bind(py).is_none() {
WirePatch::default()
} else {
from_py::<WirePatch>(result.bind(py))?
};
let name = &self.subscribers[index].name;
apply(wire, patch)
let name = &self.handlers[subscriber].name;
interception
.patched(patch)
.map_err(|error| PyValueError::new_err(format!("callback {name}: {error}")))
}
fn intercept(
&mut self,
py: Python<'_>,
wire: Box<WireRequest>,
context: RequestContext,
from: usize,
) -> PyResult<LifecycleStep> {
let mut current = wire;
for index in from..self.subscribers.len() {
let Some(handler) = self.subscribers[index]
fn intercept(&mut self, py: Python<'_>, interception: Interception) -> PyResult<LifecycleStep> {
let mut current = interception;
while let Some(turn) = current.turn() {
let Some(handler) = self.handlers[turn.subscriber]
.intercept
.as_ref()
.map(|handler| handler.clone_ref(py))
else {
continue;
return Err(missing_state());
};
let request = to_py(py, &RequestFacts::new(&current, &context))?;
let result = handler.call(py, request)?;
let result = handler.call(py, to_py(py, &turn.request)?)?;
if handler.is_async() {
self.pending = Some(Pending::Intercept {
wire: current,
context,
index,
interception: current,
subscriber: turn.subscriber,
});
return Ok(LifecycleStep::Await(result.unbind()));
}
current = Box::new(self.apply_patch(py, index, *current, result.unbind())?);
current = self.patched(py, current, turn.subscriber, result.unbind())?;
}
let envelope =
self.envelope(Event::RequestSending(RequestFacts::new(&current, &context)))?;
self.observe(py, envelope, 0, Continuation::Wire(current))
let (emission, wire) = self
.session
.as_mut()
.ok_or_else(missing_state)?
.request_sending(current);
self.observe(py, emission, 0, Continuation::Wire(Box::new(wire)))
}
fn metadata(arguments: &Bound<'_, PyDict>) -> PyResult<(Map<String, Value>, Vec<String>)> {
@ -246,14 +246,19 @@ impl PythonLifecycle for V1PythonLifecycle {
) -> PyResult<LifecycleStep> {
let call_id = Self::call_id(py, arguments.bind(py))?;
let (metadata, metadata_dropped) = Self::metadata(arguments.bind(py))?;
self.sequencer = Some(Sequencer::new(call_id, self.surface.call_type));
let envelope = self.envelope(Event::CallStarted(CallFacts {
start_time,
asynchronous: self.asynchronous,
metadata,
metadata_dropped,
}))?;
self.observe(py, envelope, 0, Continuation::Arguments(arguments))
let (session, emission) = CallSession::start(
call_id,
self.surface.call_type,
std::mem::take(&mut self.subscriptions),
CallFacts {
start_time,
asynchronous: self.asynchronous,
metadata,
metadata_dropped,
},
);
self.session = Some(session);
self.observe(py, emission, 0, Continuation::Arguments(arguments))
}
fn before_send(
@ -262,16 +267,23 @@ impl PythonLifecycle for V1PythonLifecycle {
wire: Box<WireRequest>,
context: &RequestContext,
) -> PyResult<LifecycleStep> {
self.intercept(py, wire, context.clone(), 0)
let interception = self
.session
.as_ref()
.ok_or_else(missing_state)?
.interception(*wire, context.clone());
self.intercept(py, interception)
}
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
match event {
LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => {
let envelope = self.envelope(Event::ResponseReceived {
body: raw.body.clone(),
})?;
self.observe(py, envelope, 0, Continuation::Done)
let emission = self
.session
.as_mut()
.ok_or_else(missing_state)?
.response_received(raw.body.clone());
self.observe(py, emission, 0, Continuation::Done)
}
LifecycleEvent::Succeeded { timing, response } => {
let projected = V1Python::ProjectResponse
@ -282,13 +294,9 @@ impl PythonLifecycle for V1PythonLifecycle {
Err(error) if is_cancellation(py, &error) => return Err(error),
Err(error) => (Value::Null, Some(error.to_string())),
};
let envelope = self.envelope(Event::CallSucceeded {
timing: timing.into(),
streamed: self.streamed,
response,
response_error,
})?;
self.observe(py, envelope, 0, Continuation::Done)
let session = self.session.take().ok_or_else(missing_state)?;
let emission = session.succeeded(timing, response, response_error);
self.observe(py, emission, 0, Continuation::Done)
}
LifecycleEvent::Failed {
timing,
@ -296,44 +304,41 @@ impl PythonLifecycle for V1PythonLifecycle {
error,
} => {
let facts = Self::error_facts(py, error)?;
let envelope = self.envelope(Event::CallFailed {
timing: timing.into(),
streamed: self.streamed,
origin: origin.into(),
error: facts,
})?;
self.observe(py, envelope, 0, Continuation::Done)
let session = self.session.take().ok_or_else(missing_state)?;
let emission = session.failed(timing, origin, facts);
self.observe(py, emission, 0, Continuation::Done)
}
}
}
fn opened(&mut self, _py: Python<'_>) -> PyResult<()> {
self.streamed = true;
if let Some(session) = self.session.as_mut() {
session.opened();
}
Ok(())
}
fn resume(&mut self, py: Python<'_>, result: PyResult<Py<PyAny>>) -> PyResult<LifecycleStep> {
match self.pending.take().ok_or_else(missing_state)? {
Pending::Observe {
envelope,
index,
emission,
position,
then,
} => {
if let Err(error) = result {
if is_cancellation(py, &error) {
return Err(error);
}
self.report(py, index, &envelope, error)?;
self.report(py, emission.observers[position], &emission.envelope, error)?;
}
self.observe(py, envelope, index + 1, then)
self.observe(py, emission, position + 1, then)
}
Pending::Intercept {
wire,
context,
index,
interception,
subscriber,
} => {
let patched = self.apply_patch(py, index, *wire, result?)?;
self.intercept(py, Box::new(patched), context, index + 1)
let patched = self.patched(py, interception, subscriber, result?)?;
self.intercept(py, patched)
}
}
}
@ -344,12 +349,14 @@ impl PythonLifecycle for V1PythonLifecycle {
}
self.closed = true;
self.pending = None;
self.subscribers.clear();
self.session = None;
self.subscriptions.clear();
self.handlers.clear();
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
for subscriber in &self.subscribers {
subscriber.traverse(visit)?;
for handlers in &self.handlers {
handlers.traverse(visit)?;
}
match &self.pending {
Some(Pending::Observe { then, .. }) => then.traverse(visit),

View file

@ -2,8 +2,7 @@
use pyo3::prelude::*;
use crate::adapter::V1PythonLifecycle;
use crate::subscribers::snapshot;
use crate::{adapter::V1PythonLifecycle, subscribers::snapshot};
#[derive(Clone, Copy, Debug)]
pub struct V1PythonSurface {
@ -19,7 +18,9 @@ impl V1PythonLifecycle {
surface: V1PythonSurface,
asynchronous: bool,
) -> PyResult<Option<Self>> {
let subscribers = snapshot(py, asynchronous)?;
Ok((!subscribers.is_empty()).then(|| Self::new(surface, subscribers, asynchronous)))
let (subscriptions, handlers): (Vec<_>, Vec<_>) =
snapshot(py, asynchronous)?.into_iter().unzip();
Ok((!subscriptions.is_empty())
.then(|| Self::new(surface, subscriptions, handlers, asynchronous)))
}
}

View file

@ -1,10 +1,10 @@
//! The Python contract of this crate: every function of
//! `litellm/rust_bridge/callbacks_v1_python.py` that Rust calls, by name.
//! `litellm/rust_bridge/callbacks_v1.py` that Rust calls, by name.
use pyo3::prelude::*;
use strum::{IntoStaticStr, VariantArray};
const MODULE: &str = "litellm.rust_bridge.callbacks_v1_python";
const MODULE: &str = "litellm.rust_bridge.callbacks_v1";
#[derive(Clone, Copy, Debug, IntoStaticStr, PartialEq, Eq, VariantArray)]
pub(crate) enum V1Python {

View file

@ -1,13 +1,19 @@
//! The per-call subscriber snapshot: what Python's `register` recorded, read once, with
//! one handler chosen per role for the call's execution mode. `register` has already
//! rejected everything rejectable, so this only parses.
//! rejected everything rejectable, so this only parses. Each subscriber splits in two: the
//! [`Subscription`] the contract's session routes by, and the [`Handlers`] only this host
//! can call.
use std::collections::BTreeSet;
use litellm_callbacks_v1::{EventKind, ExecutionMode, HandlerSelection, select_handler};
use pyo3::exceptions::PyValueError;
use pyo3::gc::{PyTraverseError, PyVisit};
use pyo3::prelude::*;
use litellm_callbacks_v1::{
EventKind, ExecutionMode, HandlerSelection, Subscription, select_handler,
};
use pyo3::{
exceptions::PyValueError,
gc::{PyTraverseError, PyVisit},
prelude::*,
};
use crate::python::V1Python;
@ -17,9 +23,9 @@ pub enum Handler {
Async(Py<PyAny>),
}
pub struct Subscriber {
/// The Python callables of one subscriber, at the index of its [`Subscription`].
pub struct Handlers {
pub name: String,
pub events: BTreeSet<EventKind>,
/// `on_event` or `async_on_event`: cannot affect the call.
pub observe: Option<Handler>,
/// `before_send` or `async_before_send`: returns a patch; its failure fails the call.
@ -43,7 +49,7 @@ fn selected_handler(
)
}
pub fn snapshot(py: Python<'_>, asynchronous: bool) -> PyResult<Vec<Subscriber>> {
pub fn snapshot(py: Python<'_>, asynchronous: bool) -> PyResult<Vec<(Subscription, Handlers)>> {
let mode = if asynchronous {
ExecutionMode::Async
} else {
@ -64,23 +70,28 @@ pub fn snapshot(py: Python<'_>, asynchronous: bool) -> PyResult<Vec<Subscriber>>
})
})
.collect::<PyResult<BTreeSet<_>>>()?;
Ok(Subscriber {
name: subscriber.getattr("name")?.extract()?,
events,
observe: selected_handler(&subscriber, mode, "on_event", "async_on_event")?,
intercept: selected_handler(&subscriber, mode, "before_send", "async_before_send")?,
})
let name: String = subscriber.getattr("name")?.extract()?;
let observe = selected_handler(&subscriber, mode, "on_event", "async_on_event")?;
let intercept =
selected_handler(&subscriber, mode, "before_send", "async_before_send")?;
Ok((
Subscription {
name: name.clone(),
events,
observes: observe.is_some(),
intercepts: intercept.is_some(),
},
Handlers {
name,
observe,
intercept,
},
))
})
.collect()
}
impl Subscriber {
pub fn observer(&self, kind: EventKind) -> Option<&Handler> {
self.observe
.as_ref()
.filter(|_| self.events.contains(&kind))
}
impl Handlers {
pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
for handler in self.observe.iter().chain(&self.intercept) {
visit.call(handler.object())?;

View file

@ -1,18 +1,18 @@
use std::collections::BTreeSet;
use litellm_auth::SecretValue;
use litellm_callbacks_v1::{EventKind, Subscription};
use litellm_host::event::{MachineEvent, RawResponse, RequestContext, Timing, WireRequest};
use litellm_host_python::{LifecycleEvent, LifecycleStep, PythonLifecycle, from_py};
use pyo3::exceptions::asyncio::CancelledError;
use pyo3::prelude::*;
use pyo3::types::PyDict;
use pyo3::{exceptions::asyncio::CancelledError, prelude::*, types::PyDict};
use serde_json::{Value, json};
use crate::adapter::V1PythonLifecycle;
use crate::call::V1PythonSurface;
use crate::subscribers::{Handler, Subscriber};
use crate::test_support::{local, namespace};
use litellm_callbacks_v1::EventKind;
use crate::{
adapter::V1PythonLifecycle,
call::V1PythonSurface,
subscribers::{Handler, Handlers},
test_support::{local, namespace},
};
const TIMING: Timing = Timing {
start_time: 10.0,
@ -25,7 +25,7 @@ fn subscriber(
events: &[EventKind],
observe: Option<(&str, bool)>,
intercept: Option<(&str, bool)>,
) -> Subscriber {
) -> (Subscription, Handlers) {
let handler = |spec: Option<(&str, bool)>| {
spec.map(|(local_name, asynchronous)| {
let callable = local(locals, local_name).unbind();
@ -36,18 +36,28 @@ fn subscriber(
}
})
};
Subscriber {
let handlers = Handlers {
name: name.to_string(),
events: events.iter().copied().collect::<BTreeSet<_>>(),
observe: handler(observe),
intercept: handler(intercept),
}
};
(
Subscription {
name: name.to_string(),
events: events.iter().copied().collect::<BTreeSet<_>>(),
observes: handlers.observe.is_some(),
intercepts: handlers.intercept.is_some(),
},
handlers,
)
}
fn lifecycle(subscribers: Vec<Subscriber>, asynchronous: bool) -> V1PythonLifecycle {
fn lifecycle(subscribers: Vec<(Subscription, Handlers)>, asynchronous: bool) -> V1PythonLifecycle {
let (subscriptions, handlers): (Vec<_>, Vec<_>) = subscribers.into_iter().unzip();
V1PythonLifecycle::new(
V1PythonSurface { call_type: "ocr" },
subscribers,
subscriptions,
handlers,
asynchronous,
)
}
@ -245,7 +255,7 @@ def record(event):
["call.started"]
);
let reports = py
.import("litellm.rust_bridge.callbacks_v1_python")
.import("litellm.rust_bridge.callbacks_v1")
.unwrap()
.getattr("reports")
.unwrap();

View file

@ -1,7 +1,6 @@
use std::ffi::CStr;
use pyo3::prelude::*;
use pyo3::types::PyDict;
use pyo3::{prelude::*, types::PyDict};
pub(crate) const PYTHON_CONTRACT: &str = include_str!("../python_contract.json");
@ -11,10 +10,10 @@ import json
import sys
import types
for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_v1_python'):
for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_v1'):
sys.modules.setdefault(name, types.ModuleType(name))
module = sys.modules['litellm.rust_bridge.callbacks_v1_python']
module = sys.modules['litellm.rust_bridge.callbacks_v1']
contract = json.loads(python_contract)
def contracted(name, fake):

View file

@ -1,5 +1,10 @@
- Target invariants, not completion claims
- Keep this crate the runtime-neutral v1 callback contract and nothing else: envelopes, sequencing, redaction, handler selection and validated wire patches
- Keep this crate the runtime-neutral v1 callback contract and nothing else: envelopes, redaction, handler selection, validated wire patches, and the per-call session that sequences and routes them
- `CallSession` is the whole protocol of one call and does no I/O: which envelope a moment produces, which subscriptions receive it, how interceptor patches fold into the wire. A host adapter projects its runtime's values into facts, asks the session, and runs handlers its own way; it never decides protocol. A second host (out of process, for the gateway) reuses the session unchanged
- The two planes are in the types. An `Emission` has no return path, so nothing an observer does re-enters the session. An `Interception` takes exactly one patch per interceptor, in registration order, and `request_sending` yields the wire the host must send
- A terminal method consumes the session, which is how a call has exactly one terminal envelope, last
- `Subscription` is a subscriber as data (name, events, observes, intercepts) and serializes; handlers stay in the host adapter at the same index. It is what a host that loads handlers elsewhere reports back
- Protocol tests live here and need no interpreter; they are the conformance suite every host adapter inherits. An adapter's own tests cover projection and execution only
- This crate may depend on `litellm-host`, but never on `litellm-host-python`, PyO3 or a runtime registry
- `golden/v1` pins every serialized envelope shape; every `EventKind` has a golden file by construction, and Python's `EVENTS` and `SUPPORTED_SCHEMAS` are tested against those files
- Terms (shared ones are in `host-python/AGENTS.md`)
@ -7,10 +12,14 @@
- Handler: one callable of a subscriber; an observer receives envelopes and cannot affect the call; an interceptor receives `RequestFacts` and returns a patch
- Handler selection (`select_handler`): an async call prefers the async handler and falls back to the sync one; a sync call never selects an async handler
- Event: one thing that happened in a call; `EventKind` is its dotted name (`call.started`, `request.sending`, `response.received`, `call.succeeded`, `call.failed`), the serde tag and the subscription name at once
- Envelope: schema + `call_id` + `call_type` + `seq` + event; the only value an observer receives; `Sequencer` numbers a call's envelopes from zero
- Envelope: schema + `call_id` + `call_type` + `seq` + event; the only value an observer receives; the session numbers a call's envelopes from zero
- Facts (`CallFacts`, `RequestFacts`, `TimingFacts`, `ErrorFacts`): a redacted, serializable projection of runtime state; never a live object
- Patch (`WirePatch`): an interceptor's edit of the wire: header `remove`, then header `set`, optional `body` replacement; `apply` validates it
- Redaction: credential headers by name (`CREDENTIAL_HEADERS`) and optional params by the route's secret fields become `[REDACTED]`; shape never changes
- Observers are asynchronous to the call. This is contract semantics, not a property of one host
- An envelope may reach an observer after the call has moved on or returned. Envelopes of one call arrive in `seq` order; nothing orders two calls. Delivery is at most once
- Running every observer inline before the step returns, as the in-process CPython host does, is one legal schedule and promises nothing a callback may rely on. A host that queues, batches or crosses a process boundary is equally conforming
- Nothing an observer does or returns has a path back into the call. Interceptors are the opposite: on the call path by nature, answered before the request leaves
- No runtime object of a call reaches a callback contract value
- `RequestFacts::new` is the only request projection and always applies the contract's credential redaction; its fields are private so no other crate can build one
- A patch cannot change the URL or set or remove a credential header

View file

@ -3,7 +3,7 @@
//! redacted, serializable projection that never holds a runtime object of the call.
use litellm_host::event::{FailureOrigin, RequestContext, Timing, WireRequest};
use serde::Serialize;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use strum::{EnumString, IntoStaticStr, VariantArray};
@ -57,17 +57,24 @@ pub enum Event {
PartialOrd,
Ord,
Debug,
Serialize,
Deserialize,
)]
pub enum EventKind {
#[strum(serialize = "call.started")]
#[serde(rename = "call.started")]
CallStarted,
#[strum(serialize = "request.sending")]
#[serde(rename = "request.sending")]
RequestSending,
#[strum(serialize = "response.received")]
#[serde(rename = "response.received")]
ResponseReceived,
#[strum(serialize = "call.succeeded")]
#[serde(rename = "call.succeeded")]
CallSucceeded,
#[strum(serialize = "call.failed")]
#[serde(rename = "call.failed")]
CallFailed,
}
@ -156,14 +163,14 @@ impl Event {
}
}
pub struct Sequencer {
pub(crate) struct Sequencer {
call_id: String,
call_type: &'static str,
next: u32,
}
impl Sequencer {
pub fn new(call_id: String, call_type: &'static str) -> Self {
pub(crate) fn new(call_id: String, call_type: &'static str) -> Self {
Self {
call_id,
call_type,
@ -171,7 +178,7 @@ impl Sequencer {
}
}
pub fn envelope(&mut self, event: Event) -> Envelope {
pub(crate) fn envelope(&mut self, event: Event) -> Envelope {
let envelope = Envelope {
schema: SCHEMA_V1,
call_id: self.call_id.clone(),

View file

@ -1,15 +1,17 @@
//! The runtime-neutral v1 callback contract: envelopes and their sequencing, credential
//! redaction, handler selection and validated wire patches. No PyO3 and no registry; a
//! language host marshals these values to its own subscribers.
//! The runtime-neutral v1 callback contract: envelopes, credential redaction, handler
//! selection, validated wire patches, and the per-call [`CallSession`] that says which
//! envelope goes to whom and folds interceptor patches. No PyO3, no registry and no I/O; a
//! host projects its own values into facts and runs its own subscribers.
mod envelope;
mod patch;
mod policy;
mod redact;
mod session;
pub use envelope::{
CallFacts, Envelope, ErrorFacts, Event, EventKind, Origin, RequestFacts, SCHEMA_V1, Sequencer,
TimingFacts,
CallFacts, Envelope, ErrorFacts, Event, EventKind, Origin, RequestFacts, SCHEMA_V1, TimingFacts,
};
pub use patch::{HeaderPatch, PatchError, WirePatch, apply};
pub use policy::{ExecutionMode, HandlerSelection, select_handler};
pub use session::{CallSession, Emission, Interception, Subscription, Turn};

View file

@ -0,0 +1,374 @@
//! One call's v1 protocol, with no I/O and no runtime: which envelope each moment of the
//! call produces, who receives it, and how interceptor patches fold into the wire
//! request. A host projects its own values into facts, asks the session what to deliver,
//! and runs the handlers however it can: inline in CPython, or across a process boundary.
//!
//! The two planes are visible in the types. An [`Emission`] has no way back: nothing an
//! observer does or returns re-enters the session. An [`Interception`] is answered, one
//! patch per interceptor, before the request leaves.
use std::collections::BTreeSet;
use litellm_host::event::{FailureOrigin, RequestContext, Timing, WireRequest};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::{
envelope::{CallFacts, Envelope, ErrorFacts, Event, EventKind, RequestFacts, Sequencer},
patch::{PatchError, WirePatch, apply},
};
/// What one subscriber asked for, as data: no handler, no runtime object. It is what a
/// host that runs handlers elsewhere reports back once it has loaded them.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Subscription {
pub name: String,
pub events: BTreeSet<EventKind>,
pub observes: bool,
pub intercepts: bool,
}
/// One envelope and the subscribers it goes to, as indexes into the session's
/// subscriptions, in registration order.
#[derive(Clone, Debug, PartialEq)]
pub struct Emission {
pub envelope: Envelope,
pub observers: Vec<usize>,
}
/// An interceptor's turn: who is asked, and the request as the earlier ones left it.
#[derive(Clone, Debug, PartialEq)]
pub struct Turn {
pub subscriber: usize,
pub request: RequestFacts,
}
/// The wire request on its way through the interceptors, in registration order.
#[derive(Debug)]
pub struct Interception {
wire: WireRequest,
context: RequestContext,
interceptors: Vec<usize>,
position: usize,
}
pub struct CallSession {
sequencer: Sequencer,
subscriptions: Vec<Subscription>,
streamed: bool,
}
impl Interception {
/// The interceptor to ask next, or `None` once every one has answered.
pub fn turn(&self) -> Option<Turn> {
self.interceptors
.get(self.position)
.map(|&subscriber| Turn {
subscriber,
request: RequestFacts::new(&self.wire, &self.context),
})
}
/// The current interceptor's answer; an invalid patch ends the interception.
pub fn patched(self, patch: WirePatch) -> Result<Self, PatchError> {
Ok(Self {
wire: apply(self.wire, patch)?,
position: self.position + 1,
..self
})
}
}
impl CallSession {
/// Opens the call. `call.started` is always envelope zero.
pub fn start(
call_id: String,
call_type: &'static str,
subscriptions: Vec<Subscription>,
facts: CallFacts,
) -> (Self, Emission) {
let mut session = Self {
sequencer: Sequencer::new(call_id, call_type),
subscriptions,
streamed: false,
};
let emission = session.emission(Event::CallStarted(facts));
(session, emission)
}
fn emission(&mut self, event: Event) -> Emission {
let kind = event.kind();
Emission {
envelope: self.sequencer.envelope(event),
observers: self
.subscriptions
.iter()
.enumerate()
.filter(|(_, subscription)| {
subscription.observes && subscription.events.contains(&kind)
})
.map(|(index, _)| index)
.collect(),
}
}
pub fn interception(&self, wire: WireRequest, context: RequestContext) -> Interception {
Interception {
wire,
context,
interceptors: self
.subscriptions
.iter()
.enumerate()
.filter(|(_, subscription)| subscription.intercepts)
.map(|(index, _)| index)
.collect(),
position: 0,
}
}
/// Closes an interception: `request.sending` describes the wire as the last
/// interceptor left it, which is the wire the host must send.
pub fn request_sending(&mut self, interception: Interception) -> (Emission, WireRequest) {
let facts = RequestFacts::new(&interception.wire, &interception.context);
(
self.emission(Event::RequestSending(facts)),
interception.wire,
)
}
pub fn response_received(&mut self, body: String) -> Emission {
self.emission(Event::ResponseReceived { body })
}
/// The call's stream was handed to the caller; the terminal envelope says so.
pub fn opened(&mut self) {
self.streamed = true;
}
/// A terminal envelope consumes the session, so a call has exactly one, last.
pub fn succeeded(
mut self,
timing: Timing,
response: Value,
response_error: Option<String>,
) -> Emission {
let streamed = self.streamed;
self.emission(Event::CallSucceeded {
timing: timing.into(),
streamed,
response,
response_error,
})
}
pub fn failed(mut self, timing: Timing, origin: FailureOrigin, error: ErrorFacts) -> Emission {
let streamed = self.streamed;
self.emission(Event::CallFailed {
timing: timing.into(),
streamed,
origin: origin.into(),
error,
})
}
}
#[cfg(test)]
mod tests {
use serde_json::{Map, json};
use strum::VariantArray;
use super::*;
const TIMING: Timing = Timing {
start_time: 1.0,
end_time: 2.0,
};
fn subscription(name: &str, events: &[EventKind], intercepts: bool) -> Subscription {
Subscription {
name: name.to_string(),
events: events.iter().copied().collect(),
observes: !events.is_empty(),
intercepts,
}
}
fn facts() -> CallFacts {
CallFacts {
start_time: 1.0,
asynchronous: true,
metadata: Map::new(),
metadata_dropped: Vec::new(),
}
}
fn wire() -> WireRequest {
WireRequest {
url: "https://provider.example".to_string(),
headers: Vec::new(),
body: json!({"input": "original"}),
}
}
fn context() -> RequestContext {
RequestContext {
model: "model".to_string(),
custom_llm_provider: "provider".to_string(),
optional_params: json!({}),
secret_fields: Vec::new(),
api_key: None,
}
}
fn body(value: Value) -> WirePatch {
serde_json::from_value(json!({"body": value})).unwrap()
}
fn seen_body(turn: &Turn) -> Value {
serde_json::to_value(&turn.request).unwrap()["body"].clone()
}
fn error() -> ErrorFacts {
ErrorFacts {
class: "builtins.ValueError".to_string(),
message: "bad".to_string(),
status_code: Some(400),
}
}
fn start(subscriptions: Vec<Subscription>) -> (CallSession, Emission) {
CallSession::start("call".to_string(), "ocr", subscriptions, facts())
}
#[test]
fn a_call_numbers_its_envelopes_from_the_start_to_one_terminal() {
let (mut session, started) = start(vec![subscription("all", EventKind::VARIANTS, false)]);
let (sending, _) = {
let interception = session.interception(wire(), context());
session.request_sending(interception)
};
let received = session.response_received("{}".to_string());
let terminal = session.succeeded(TIMING, json!({"id": "response"}), None);
let seen: Vec<(u32, EventKind)> = [started, sending, received, terminal]
.iter()
.map(|emission| (emission.envelope.seq, emission.envelope.event.kind()))
.collect();
assert_eq!(
seen,
vec![
(0, EventKind::CallStarted),
(1, EventKind::RequestSending),
(2, EventKind::ResponseReceived),
(3, EventKind::CallSucceeded),
]
);
}
#[test]
fn an_envelope_goes_only_to_observers_subscribed_to_its_kind_in_registration_order() {
let (session, started) = start(vec![
subscription("terminal", &[EventKind::CallFailed], false),
subscription("interceptor", &[], true),
subscription(
"both",
&[EventKind::CallStarted, EventKind::CallFailed],
false,
),
subscription("started", &[EventKind::CallStarted], true),
]);
assert_eq!(started.observers, vec![2, 3]);
assert_eq!(
session
.failed(TIMING, FailureOrigin::Call, error())
.observers,
vec![0, 2]
);
}
#[test]
fn each_interceptor_sees_the_request_as_the_earlier_ones_left_it() {
let (mut session, _) = start(vec![
subscription("first", &[], true),
subscription("observer", &[EventKind::RequestSending], false),
subscription("second", &[], true),
]);
let interception = session.interception(wire(), context());
let first = interception.turn().unwrap();
assert_eq!(
(first.subscriber, seen_body(&first)),
(0, json!({"input": "original"}))
);
let interception = interception
.patched(body(json!({"input": "first"})))
.unwrap();
let second = interception.turn().unwrap();
assert_eq!(
(second.subscriber, seen_body(&second)),
(2, json!({"input": "first"}))
);
let interception = interception.patched(WirePatch::default()).unwrap();
assert_eq!(interception.turn(), None);
let (sending, sent) = session.request_sending(interception);
assert_eq!(sent.body, json!({"input": "first"}));
assert_eq!(sending.observers, vec![1]);
assert_eq!(
serde_json::to_value(&sending.envelope).unwrap()["event"]["body"],
sent.body
);
}
#[test]
fn an_invalid_patch_ends_the_interception() {
let (session, _) = start(vec![subscription("interceptor", &[], true)]);
let patch: WirePatch =
serde_json::from_value(json!({"headers": {"set": [["Authorization", "stolen"]]}}))
.unwrap();
assert_eq!(
session
.interception(wire(), context())
.patched(patch)
.unwrap_err(),
PatchError::ProtectedHeader("Authorization".to_string())
);
}
#[test]
fn a_terminal_envelope_says_whether_the_call_streamed() {
let (mut streamed, _) = start(Vec::new());
streamed.opened();
let (plain, _) = start(Vec::new());
for (session, expected) in [(streamed, true), (plain, false)] {
let Event::CallFailed { streamed, .. } = session
.failed(TIMING, FailureOrigin::Host, error())
.envelope
.event
else {
panic!("call.failed expected");
};
assert_eq!(streamed, expected);
}
}
#[test]
fn a_subscription_is_plain_data_a_remote_host_can_report() {
let reported = subscription("remote", &[EventKind::CallSucceeded], true);
let wire = serde_json::to_value(&reported).unwrap();
assert_eq!(
wire,
json!({"name": "remote", "events": ["call.succeeded"], "observes": true, "intercepts": true})
);
assert_eq!(
serde_json::from_value::<Subscription>(wire).unwrap(),
reported
);
}
}

View file

@ -0,0 +1,171 @@
# Migrating built-in callbacks to the v1 contract
How a built-in logger moves from `CustomLogger` (`litellm/integrations/`) to the v1
contract (`litellm/callbacks_v1/__init__.py`), and what the scaffolding in `builtin/` is
for while that happens. The rules for writing one port are in `builtin/AGENTS.md`; this
file is the flow around them.
## The shape of the migration
- Ports live in a parallel tree, `builtin/<name>.py`, one frozen class each declaring
`SinkPort` or `InterceptorPort` from `builtin/port.py` as its base. The legacy logger
is never edited, renamed, or taught about its port.
- Everything the migration needs to say is in `builtin/manifest.py`: `Gap`, each port's
registration name, its twin, its status and its gaps. A port carries none of it, so
cutting over deletes that file and leaves the ports untouched.
- A port is inert. Importing it registers nothing and the product never registers it;
only tests do. Writing a port therefore cannot change behaviour for anyone.
- A port goes live by removing its legacy twin. The two are never active on the same
call, so there is no dual delivery, no ownership table, and no per-call skip logic.
- v1 envelopes only exist on native (Rust) routes. A port can only replace its twin once
every route the twin serves emits them.
- Built-in and custom callbacks share one interface, the v1 contract. In the SDK a custom
callback is free-form Python, in process, never sandboxed. On the gateway it runs in an
out-of-process extension host driven by the same `CallSession` (`litellm-rust/crates/
callbacks-v1`). A built-in port is pure methods over values with every effect in
`builtin/runtime.py`, which also owns the two subscribers (`Sink`, `Patcher`); that is
hygiene for built-ins (secrets, a later Rust port), not the isolation mechanism.
Porting comes first, and on purpose, because its first product is not the port. It is the
list of things the contract cannot do yet. The contract grows from what real built-ins
needed, not from guesses.
## The loop
```
port a built-in ──► record gaps ──► read the backlog ──► grow the contract
▲ │
└──────── delete gaps and their allowances ◄─────────────┘
```
### 1. Port inertly
Write `payload()` against the golden envelopes. Every time the twin reads something the
envelopes do not carry, do one of two things and write a `Gap` either way:
- leave the field out of the vendor body, or
- let `builtin/runtime.py` work around it for every port at once (`CallJoin` recovers the
model for the terminal envelope; `Outbox` keeps vendor I/O off the call path). A
workaround never goes inside a port: a port that needs a thread, a socket or state has
found a `capability` gap.
A `Gap` is handwritten, in `manifest.py` beside the port's entry. It has a `kind`
(`fact`, `event`, `capability`), a `key`, the `legacy` input the twin read, and a `note`. The `key` names what the contract would have
to grow (`cost`, `usage`, `identity`), not what the twin happened to call it, so that two
ports missing the same thing say so in the same word.
Then write the parity test: one real native call, the twin and the port each posting to
their own recording vendor, `assert_parity` on the two bodies. Every difference needs an
`Allowed(path, reason)`. If the difference exists because of a gap, it cites the gap:
`Allowed("data.cost", "no cost fact on call.succeeded", gap="cost")`. An `Allowed` with no
`gap` is a difference the port means to keep (each side stamps its own clock, the port
omits a key the twin sent as null).
Add the port to `manifest.py` as `exploring`, then `parity` once the parity test passes.
### 2. Read the backlog
```bash
python -m litellm.callbacks_v1.builtin.manifest
```
prints the ledger (every gap, per port) and the backlog (every gap key, with the ports
waiting on it, most wanted first). A key several ports wait on is contract work. A key
one port waits on is either contract work or a candidate for "v1 deliberately does not
do this"; decide which before building it.
### 3. Grow the contract
The contract is defined in Rust (`litellm-rust/crates/callbacks-v1`, golden envelopes in
`golden/v1/`) and mirrored by the types in `litellm/callbacks_v1/__init__.py`. Add the
fact, event, or capability there, for every native route, and update the golden files.
### 4. Close the gap
For each port that listed the key:
1. Use the new fact in `payload()`, or drop the workaround.
2. Delete the `Gap`.
3. Run the parity tests. They now fail in exactly the places that need attention:
- an `Allowed` still citing the deleted key fails with "cites gap ..., which the port
does not list";
- an `Allowed` whose difference is gone fails as stale;
- a difference the new fact did not actually fix fails as unexplained once its
`Allowed` is removed.
4. Delete those `Allowed` entries until the test is green again. Do not re-allow a
difference to get there.
The order of 1 and 2 does not matter; the tests fail until both are done.
### 5. Cut over
A port is `ready` when its parity test passes, its manifest gaps are empty, and every route its twin
serves emits v1 envelopes. `test_conventions.py` refuses `ready` with an open gap. A
`ready` port replaces its twin: register the port where the twin was registered, remove
the twin, in one change.
## What is checked and what is trusted
| Claim | Checked by |
|---|---|
| Every difference from the twin is explained | `assert_parity`: an uncovered difference fails |
| The allow-list does not outlive the differences | `assert_parity`: a stale `Allowed` fails |
| The allow-list does not outlive the gaps | `assert_parity`: an `Allowed` citing an unlisted key fails |
| A port is not `ready` with open gaps | `test_conventions.py` |
| The gap list is complete | nobody |
The last row is the honest one. The gap list is only as complete as its author. For a
sink, the parity diff is the real source of truth and the gaps are its readable summary: anything the
author forgot still shows up as an unexplained difference. Three kinds of gap never show
up in a diff, and the handwritten entry is their only record:
- `capability` and `event` gaps (observers run inline; an interceptor cannot see
`call_type`), because they are about how the port runs, not what it sends;
- gaps the port works around (`terminal_model`), because the workaround makes the bodies
equal;
- gaps the parity call does not exercise (`request_user`, when the test sends no `user`).
Closing one of these is a manual edit with no failing test to prompt it. When the contract
change lands, grep the key.
## Open decisions
Porting does not settle these, and no port may settle one by accident. Each is kept out of
the ports so that it can be decided once, in the host.
- **Where `Config` comes from.** A port takes a finished `Config`. Whether the host
resolves it from the twin's environment variables, from config.yaml, or through the
Rust settings precedence is undecided; `from_env` on a port only records the names.
- **The gateway's extension host.** Custom Python callbacks run out of process there:
the worker's confinement profile, the interceptor timeout and failure policy, and
whether built-ins run in a trusted worker or move to Rust. The interface is the v1
Protocol itself over a socket, not the ports' pure-method shape.
- **Who holds secrets.** A `Config` carries the vendor's API key and a `Delivery` carries
it as a header, so a port sees the credential. A sandboxed callback should name a secret
and let the host put it on the wire. `off_path_delivery` is the gap that would carry it.
## Differences that are never closed
Some legacy inputs will not come to v1 (`hidden_params` is the likely example). Those are
not gaps; a gap is something the contract intends to grow. When that decision is made for
a key, delete the `Gap`, drop `gap=` from its `Allowed` entries and reword their reasons
as the decision, and move the feature to the port's `Not ported yet:` docstring line or to
the release notes. A port must not sit at `parity` forever behind a gap nobody will close.
`parity` means every difference is explained. It does not mean the port is equivalent:
`generic_api` is at `parity` while shipping about a fifth of `StandardLoggingPayload`.
Only `ready` means equivalent.
## What survives the migration
| Piece | After every twin is gone |
|---|---|
| The ports, `SinkPort`, `InterceptorPort`, `Delivery`, `CallRecord`, `Batching` | stay; this is the product and its interface |
| `runtime.py`: `Sink`, `Outbox`, `Transport` / `HttpxTransport` | stays until the contract delivers off the call path itself (`off_path_delivery`); then the host owns it |
| `CallJoin` | stays only if the terminal envelope never carries model and metadata; it is a workaround for the `terminal_model` gap |
| `manifest.py` entire: `Gap`, every gap list, the names, `ledger()`, `backlog()`, the statuses, `Entry.legacy` | deleted with the last twin |
| The twin checks in `test_conventions.py` | deleted with the last twin |
| `assert_parity`, `Allowed`, `test_parity.py` | deleted; there is no twin left to compare with. The payload tests on golden envelopes remain as the ports' tests |
At that point `port.py` holds the two protocols and what they are written in, which is the
whole of what a callback author, built-in or custom, needs besides the contract types.

View file

@ -0,0 +1,419 @@
"""The v1 callback contract as a callback author sees it.
A registered callback observes a call through envelopes it cannot influence, and may
influence one only by returning a patch that Rust validates. The schema itself is owned
by `litellm-rust/crates/callbacks-v1`; this module is its typed Python face and the
process-wide registry. `litellm.rust_bridge.callbacks_v1` is the marshalling the
native call reaches for, and is not part of this surface.
Two ways to register. A single function takes a decorator, which derives the name and the
handler slot for you and returns the function unchanged:
@on_event("call.succeeded", "call.failed")
def record(event: EnvelopeV1) -> None: ...
@before_send
async def add_header(request: RequestFactsV1) -> WirePatchV1 | None: ...
An object with several handlers, or with state, declares them by the protocol names and
goes through `register`.
Every rejectable mistake is rejected at registration, never during a call, and each static
rule here mirrors one `RegistrationError`: the handler protocols mirror `MissingHandler`
and `HandlerKindMismatch`, and `EventName` mirrors `UnknownEvent`. Annotate `events` with
`frozenset[EventName]` on the object form to get that check before import time; the
decorator infers it from its arguments.
"""
import inspect
import threading
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass
from typing import Final, Literal, Protocol, TypeAlias, TypeVar
from typing_extensions import ReadOnly, TypeAliasType, TypedDict
__all__ = (
"EVENTS",
"SCHEMA",
"SUPPORTED_SCHEMAS",
"AsyncInterceptor",
"AsyncObserver",
"CallFactsV1",
"CallFailedV1",
"CallStartedV1",
"CallSucceededV1",
"Callback",
"CallbackBase",
"DuplicateName",
"EmptyObserverEvents",
"EnvelopeV1",
"ErrorFactsV1",
"EventName",
"EventV1",
"HandlerKindMismatch",
"HeaderPatchV1",
"Interceptor",
"InterceptorFn",
"JSONValue",
"MissingHandler",
"Observer",
"ObserverFn",
"RegistrationError",
"RequestFactsV1",
"RequestSendingV1",
"ResponseReceivedV1",
"SchemaVersion",
"Subscriber",
"TimingFactsV1",
"UnknownEvent",
"UnsupportedSchema",
"WirePatchV1",
"before_send",
"on_event",
"register",
"snapshot",
"unregister",
)
JSONScalar: TypeAlias = None | bool | int | float | str
JSONValue = TypeAliasType( # rebind-ok: named recursive alias required for runtime validation
"JSONValue", JSONScalar | Sequence["JSONValue"] | Mapping[str, "JSONValue"]
)
EventName: TypeAlias = Literal[
"call.started",
"request.sending",
"response.received",
"call.succeeded",
"call.failed",
]
SchemaVersion: TypeAlias = Literal[1]
SCHEMA: Final[SchemaVersion] = 1
SUPPORTED_SCHEMAS: Final[frozenset[SchemaVersion]] = frozenset({SCHEMA})
EVENTS: Final[frozenset[EventName]] = frozenset(
{
"call.started",
"request.sending",
"response.received",
"call.succeeded",
"call.failed",
}
)
class TimingFactsV1(TypedDict):
start_time: ReadOnly[float]
end_time: ReadOnly[float]
duration: ReadOnly[float]
class CallFactsV1(TypedDict):
start_time: ReadOnly[float]
asynchronous: ReadOnly[bool]
metadata: ReadOnly[Mapping[str, JSONValue]]
metadata_dropped: ReadOnly[Sequence[str]]
class RequestFactsV1(TypedDict):
model: ReadOnly[str]
custom_llm_provider: ReadOnly[str]
optional_params: ReadOnly[JSONValue]
url: ReadOnly[str]
headers: ReadOnly[Sequence[tuple[str, str]]]
body: ReadOnly[JSONValue]
ErrorFactsV1 = TypedDict( # rebind-ok: runtime TypedDict type for the reserved class field
"ErrorFactsV1",
{ # mutable-ok: functional TypedDict syntax requires a dict for the reserved class field
"class": ReadOnly[str],
"message": ReadOnly[str],
"status_code": ReadOnly[int | None],
},
)
class CallStartedV1(CallFactsV1):
type: ReadOnly[Literal["call.started"]]
class RequestSendingV1(RequestFactsV1):
type: ReadOnly[Literal["request.sending"]]
class ResponseReceivedV1(TypedDict):
type: ReadOnly[Literal["response.received"]]
body: ReadOnly[str]
class CallSucceededV1(TypedDict):
type: ReadOnly[Literal["call.succeeded"]]
timing: ReadOnly[TimingFactsV1]
streamed: ReadOnly[bool]
response: ReadOnly[JSONValue]
response_error: ReadOnly[str | None]
class CallFailedV1(TypedDict):
type: ReadOnly[Literal["call.failed"]]
timing: ReadOnly[TimingFactsV1]
streamed: ReadOnly[bool]
origin: ReadOnly[Literal["call", "host"]]
error: ReadOnly[ErrorFactsV1]
EventV1: TypeAlias = CallStartedV1 | RequestSendingV1 | ResponseReceivedV1 | CallSucceededV1 | CallFailedV1
class EnvelopeV1(TypedDict):
schema: ReadOnly[Literal[1]]
call_id: ReadOnly[str]
call_type: ReadOnly[str]
seq: ReadOnly[int]
event: ReadOnly[EventV1]
class HeaderPatchV1(TypedDict, total=False):
remove: ReadOnly[Sequence[str]]
set: ReadOnly[Sequence[tuple[str, str]]]
class WirePatchV1(TypedDict, total=False):
headers: ReadOnly[HeaderPatchV1]
body: ReadOnly[JSONValue]
class CallbackBase(Protocol):
"""What every callback declares, whatever it handles.
`schema` is not here: the module you imported from already fixes it. Declaring one
anyway is still honoured, so a callback written against another version is rejected
by `register` rather than silently reinterpreted.
"""
name: str
events: frozenset[EventName]
class Observer(CallbackBase, Protocol):
"""Receives envelopes on a sync call; its errors are reported and swallowed.
Observers are asynchronous to the call: an envelope may arrive after the call has
moved on or returned, in `seq` order within one call and in no order across calls, at
most once. This host happens to run observers inline; do not rely on it.
"""
def on_event(self, event: EnvelopeV1) -> None: ...
class AsyncObserver(CallbackBase, Protocol):
"""Receives envelopes on an async call; its errors are reported and swallowed.
Asynchronous to the call in the same sense as `Observer`."""
async def async_on_event(self, event: EnvelopeV1) -> None: ...
class Interceptor(CallbackBase, Protocol):
"""Patches the sync request before it is sent; its failure fails the call."""
def before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ...
class AsyncInterceptor(CallbackBase, Protocol):
"""Patches the async request before it is sent; its failure fails the call."""
async def async_before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ...
Callback: TypeAlias = Observer | AsyncObserver | Interceptor | AsyncInterceptor
# The decorators return the function unchanged, so each is generic over the exact function
# it decorated; without that, the decorated name would widen to the union below and stop
# being directly callable at its own signature.
_SyncObserverFn: TypeAlias = Callable[[EnvelopeV1], None] # mutable-ok: Callable's bracketed parameter list is call syntax, not a list literal
_AsyncObserverFn: TypeAlias = Callable[[EnvelopeV1], Awaitable[None]] # mutable-ok: Callable's bracketed parameter list is call syntax, not a list literal
_SyncInterceptorFn: TypeAlias = Callable[[RequestFactsV1], WirePatchV1 | None] # mutable-ok: Callable's bracketed parameter list is call syntax, not a list literal
_AsyncInterceptorFn: TypeAlias = Callable[[RequestFactsV1], Awaitable[WirePatchV1 | None]] # mutable-ok: Callable's bracketed parameter list is call syntax, not a list literal
ObserverFn: TypeAlias = _SyncObserverFn | _AsyncObserverFn
InterceptorFn: TypeAlias = _SyncInterceptorFn | _AsyncInterceptorFn
ObserverFnT = TypeVar("ObserverFnT", bound=ObserverFn) # rebind-ok: a TypeVar must bind to a bare name to be recognised as one
InterceptorFnT = TypeVar("InterceptorFnT", bound=InterceptorFn) # rebind-ok: a TypeVar must bind to a bare name to be recognised as one
class RegistrationError(ValueError):
pass
class UnsupportedSchema(RegistrationError):
pass
class UnknownEvent(RegistrationError):
pass
class MissingHandler(RegistrationError):
pass
class EmptyObserverEvents(RegistrationError):
pass
class HandlerKindMismatch(RegistrationError):
pass
class DuplicateName(RegistrationError):
pass
@dataclass(frozen=True, slots=True)
class Subscriber:
name: str
schema: SchemaVersion
events: frozenset[EventName]
on_event: object | None
async_on_event: object | None
before_send: object | None
async_before_send: object | None
class _Registry:
def __init__(self) -> None:
self.subscribers: tuple[Subscriber, ...] = ()
self.lock: Final = threading.Lock()
_REGISTRY: Final = _Registry()
def _handler(callback: Callback, name: str) -> object | None:
return getattr(callback, name, None)
def _validate_kind(name: str, handler: object | None, asynchronous: bool) -> None:
if handler is None:
return
if not callable(handler):
raise HandlerKindMismatch(f"{name} must be callable")
is_async: Final = inspect.iscoroutinefunction(handler)
if is_async != asynchronous:
expected: Final = "async" if asynchronous else "sync"
raise HandlerKindMismatch(f"{name} must be {expected}")
def _validate_events(events: frozenset[EventName], observing: bool) -> None:
unknown: Final = events - EVENTS
if unknown:
raise UnknownEvent(f"unknown callback events: {sorted(unknown)}")
if observing and not events:
raise EmptyObserverEvents("observer callback events cannot be empty")
def _insert(subscriber: Subscriber) -> None:
with _REGISTRY.lock:
if any(existing.name == subscriber.name for existing in _REGISTRY.subscribers):
raise DuplicateName(f"duplicate callback name: {subscriber.name}")
_REGISTRY.subscribers = (*_REGISTRY.subscribers, subscriber)
def register(callback: Callback) -> None:
"""Registers an object whose handler methods are named as the protocols declare them."""
schema: Final = getattr(callback, "schema", SCHEMA)
if schema not in SUPPORTED_SCHEMAS:
raise UnsupportedSchema(f"unsupported callback schema: {schema}")
events: Final = frozenset(callback.events)
on_event: Final = _handler(callback, "on_event")
async_on_event: Final = _handler(callback, "async_on_event")
before_send: Final = _handler(callback, "before_send")
async_before_send: Final = _handler(callback, "async_before_send")
handlers: Final = (on_event, async_on_event, before_send, async_before_send)
if all(handler is None for handler in handlers):
raise MissingHandler("callback must define at least one handler")
_validate_events(events, on_event is not None or async_on_event is not None)
_validate_kind("on_event", on_event, False)
_validate_kind("async_on_event", async_on_event, True)
_validate_kind("before_send", before_send, False)
_validate_kind("async_before_send", async_before_send, True)
_insert(
Subscriber(
name=callback.name,
schema=schema,
events=events,
on_event=on_event,
async_on_event=async_on_event,
before_send=before_send,
async_before_send=async_before_send,
)
)
def _derived_name(function: object) -> str:
module: Final = getattr(function, "__module__", "") or ""
qualified: Final = getattr(function, "__qualname__", None) or repr(function)
return f"{module}.{qualified}" if module else str(qualified)
def on_event(*events: EventName, name: str | None = None) -> Callable[[ObserverFnT], ObserverFnT]:
"""Registers one function as an observer of `events`; `async def` selects the async slot.
The function is returned unchanged, so it stays directly callable and testable. Its
registry name defaults to `module.qualname`, which is unique by construction.
"""
def decorate(function: ObserverFnT) -> ObserverFnT:
subscribed: Final = frozenset(events)
_validate_events(subscribed, True)
asynchronous: Final = inspect.iscoroutinefunction(function)
_insert(
Subscriber(
name=name if name is not None else _derived_name(function),
schema=SCHEMA,
events=subscribed,
on_event=None if asynchronous else function,
async_on_event=function if asynchronous else None,
before_send=None,
async_before_send=None,
)
)
return function
return decorate
def before_send(function: InterceptorFnT) -> InterceptorFnT:
"""Registers one function as an interceptor; `async def` selects the async slot.
An interceptor subscribes to no events, and its failure fails the call.
"""
asynchronous: Final = inspect.iscoroutinefunction(function)
_insert(
Subscriber(
name=_derived_name(function),
schema=SCHEMA,
events=frozenset(),
on_event=None,
async_on_event=None,
before_send=None if asynchronous else function,
async_before_send=function if asynchronous else None,
)
)
return function
def unregister(callback: Callback | Subscriber | str) -> None:
"""Drops the callback of this name, given the object, a `Subscriber`, or the name."""
name: Final = callback if isinstance(callback, str) else callback.name
with _REGISTRY.lock:
_REGISTRY.subscribers = tuple(subscriber for subscriber in _REGISTRY.subscribers if subscriber.name != name)
def snapshot() -> tuple[Subscriber, ...]:
return _REGISTRY.subscribers

View file

@ -0,0 +1,44 @@
- Target invariants, not completion claims
- This package holds built-in callbacks ported from `CustomLogger` to the v1 contract (`litellm/callbacks_v1/__init__.py`), and nothing else
- Porting is exploration: its first product is the gap ledger (`python -m litellm.callbacks_v1.builtin.manifest`), which says what the contract has to grow
- `../MIGRATION.md` is the whole flow from first port to removing the legacy twin; this file is the rules for writing one port
- A port is never registered by the product while its legacy twin exists. Importing anything here registers nothing; only tests call `register`
- A port goes live by removing or unregistering its legacy twin. The two are never active together, so there is no per-call skip logic anywhere
- Never edit the legacy twin to make a port easier
- A port is a frozen class of pure methods; the host does everything else
- Built-in and custom callbacks share one interface, the v1 contract. This rule is hygiene for built-ins, not the sandbox: a built-in holds vendor secrets and is the likeliest to be ported to Rust, and a pure transform is what survives both. Custom Python callbacks stay free-form; on the gateway they are isolated by an out-of-process extension host, not by this rule
- A port opens no socket, starts no thread, keeps no state between calls and reads neither the environment nor the clock. Whatever it needs arrives as a value: contract facts, its `Config`, `now`
- `port.py` is the whole interface: the two protocols a port declares as its base (`SinkPort`, `InterceptorPort`) and what they are written in (`CallRecord`, `Delivery`, `Batching`). Nothing in it is migration scaffolding, and a port imports nothing else from this package
- `runtime.py` is the in-process host: `Sink` and `Patcher`, the two subscribers, with `CallJoin`, `Transport`, the clock and `outbox.py` behind them. They are the only things here that implement the v1 contract, so a port never writes `name`, `events` or a handler. Each takes the registration name, because the name is the registration's and not the port's: one port configured twice is two subscribers. It stands in for what a host should own; when the contract grows off-path delivery and a joined call record, `runtime.py` is deleted and the ports do not change
- `manifest.py` is the migration, all of it: `Gap`, the name a port registers under, its twin, its status and its gaps. At cutover it is deleted outright and no port changes
- `test_conventions.py` holds a port's imports to a pure allow-list and checks that it declares the protocol of its kind. A port importing anything from `runtime.py` is a failure: wiring is the host's, and the tests do it
- Terms
- Legacy twin: the `CustomLogger` subclass (or function of one) a port replaces, named in the port's docstring and in `manifest.py`
- Sink: a port that observes calls and tells a vendor (`SinkPort`, run by `runtime.Sink`). Interceptor: a port that patches the wire request (`InterceptorPort`, run by `runtime.Patcher`)
- Gap: one legacy input the contract does not supply: a missing `fact`, a missing `event`, or a missing `capability`
- Its `key` names what the contract would have to grow (`cost`, `usage`, `identity`), not what the twin read. Reuse an existing key when another port already waits on the same thing; the backlog counts ports per key
- Parity: the port and its twin handled the same call and every difference in what they sent is `Allowed` with a reason
- What can be ported: a twin that only overrides the success/failure/pre/post log hooks is a sink; one that only rewrites the outgoing request is an interceptor. A twin overriding any other hook (guardrail, routing, prompt management, agentic loop, stream rewriting, audit log) has no v1 shape and stays out; `test_conventions.py` enforces this from the twin's own class
- Every port module has the same parts, in this order
- Docstring: one line of what it is, `Legacy twin: <path>`, and `Not ported yet:` for legacy features left out on purpose
- `Config`: a frozen dataclass of plain data. Where its values come from (environment, config.yaml, the UI) is the host's business and an open decision in `../MIGRATION.md`; a port never reads the environment. A `from_env(env)` over a mapping is allowed as the record of the twin's variable names, not as the way a port gets configured
- The vendor's types, as `TypedDict`s, and any module-private pure helpers
- The port class: a frozen dataclass whose only field is `config`, declaring `SinkPort[<its record>]` or `InterceptorPort` as its base. Both kinds have exactly the same parts
- `payload(...)`: the port. For a sink, one `CallRecord` and the host's `now` to the vendor's body; for an interceptor, one `RequestFactsV1` to the `WirePatchV1` it needs. `None` means nothing to send or nothing to patch, which is also how a port filters
- A sink adds `deliveries(batch)`, a pure function from a batch of bodies to the `Delivery`s that carry them, headers and framing included, and `batching`, how large a batch the host should collect
- Nothing else. Not its name, not its gaps (`manifest.py` holds both), and nothing of the contract: no `events`, no `on_event`, no `before_send`, no subscriber class, no `build`. A port is registered by nobody; the tests name it, wrap it in `runtime.Sink` or `runtime.Patcher` and register that
- A sink's handler never does I/O: v1 observers run inline in the call, where legacy callbacks ran on an executor or the logging worker
- `runtime.Sink` joins envelopes with `CallJoin`, calls `payload` with the clock's `now`, and `put`s the result on its `Outbox`; the outbox thread calls `deliveries` and sends, `batching` at a time
- So a sink has only the sync `on_event`; an async call falls back to it, and it never blocks
- The terminal envelope carries neither model nor metadata, which is why `CallJoin` exists. A cancelled call has no terminal envelope, which is why it is bounded
- Observer errors are reported and swallowed by the contract; a port does not wrap its handler in `try`
- Tests, three layers; a port is not done without the first two
- Payload (`tests/test_litellm/callbacks_v1/builtin/test_<name>.py`): `payload` on the golden envelopes Rust pins, asserting the whole vendor body. No Rust needed
- Parity (`tests/test_litellm_rust/builtin/test_parity.py`): one native call with the twin in `callbacks=[...]` and the port registered by the test, each posting to its own recording vendor; `assert_parity` on the two raw bodies. Cover success and failure when the twin logs both
- An interceptor whose twin never runs on a native route is compared function to function on the same bodies instead, plus one native call proving the patch reaches the provider
- Conventions (`test_conventions.py`): runs for every manifest entry with no per-port code, the pure-import rule and the declared-protocol check included; add the port to `EXAMPLES` and `SUBSCRIBERS` in `support.py`
- `Allowed` differences
- Every entry has a reason. One that exists because of a gap cites it as `gap="<key>"`, and `assert_parity` fails if the port's `GAPS` has no such key. One with no `gap` is a difference the port keeps on purpose
- Patterns are as narrow as the difference. A stale entry fails the test, so closing a gap in the contract forces the list to shrink, and deleting a `Gap` fails every entry still citing it
- Never allow a difference to get a test green; either port the behaviour or record the gap
- Gaps and status live in `manifest.py`, next to the entry: `exploring` (payload tests pass), `parity` (parity test passes), `ready` (parity, no open gap, and every route the twin serves emits v1 envelopes). Only `ready` ports may replace their twin

View file

@ -0,0 +1,6 @@
"""Built-in callbacks ported to the v1 contract.
Nothing here is registered by importing it, and nothing in the product imports it yet:
a port is exercised by its tests only, until its legacy twin is removed. `AGENTS.md`
beside this file is the porting convention, `manifest.py` is the list of ports.
"""

View file

@ -0,0 +1,149 @@
"""Anthropic prompt-cache breakpoints on the v1 contract: a `before_send` body patch.
Legacy twin: `AnthropicCacheControlHook.apply_to_anthropic_messages_request` in
`litellm.integrations.anthropic_cache_control_hook`, the Messages-route half of that hook.
Not ported yet: the OpenAI `prompt_cache_breakpoint` dialect, `tool_config` points, the
default-injection seeding, and the chat-completions path (that route is not native).
"""
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
from litellm.callbacks_v1 import JSONValue, RequestFactsV1, WirePatchV1
from litellm.callbacks_v1.builtin.port import InterceptorPort
MAX_CACHE_CONTROL_BLOCKS: Final = 4
Role: TypeAlias = Literal["user", "system", "assistant"]
Block: TypeAlias = Mapping[str, JSONValue]
@dataclass(frozen=True, slots=True)
class InjectionPoint:
"""Where to put a breakpoint: the system prompt, every message of `role`, or the message at `index`."""
role: Role | None = None
index: int | None = None
ttl: Literal["5m", "1h"] | None = None
@property
def control(self) -> Block:
if self.ttl is None:
return MappingProxyType({"type": "ephemeral"})
return MappingProxyType({"type": "ephemeral", "ttl": self.ttl})
@dataclass(frozen=True, slots=True)
class Config:
points: tuple[InjectionPoint, ...]
providers: frozenset[str] = frozenset({"anthropic"})
def _marked(block: JSONValue) -> bool:
return isinstance(block, Mapping) and block.get("cache_control") is not None
def _blocks(content: JSONValue) -> Sequence[JSONValue]:
return content if isinstance(content, Sequence) and not isinstance(content, str) else ()
def _breakpoints(message: JSONValue) -> int:
if not isinstance(message, Mapping):
return 0
return int(_marked(message)) + sum(1 for block in _blocks(message.get("content")) if _marked(block))
def _with_last_block_marked(blocks: Sequence[JSONValue], control: Block) -> Sequence[JSONValue] | None:
if not blocks or not isinstance(blocks[-1], Mapping):
return None
return (*blocks[:-1], MappingProxyType({**blocks[-1], "cache_control": control}))
def _as_blocks(message: Block) -> Block:
content: Final = message.get("content")
if not isinstance(content, str):
return message
return MappingProxyType({**message, "content": (MappingProxyType({"type": "text", "text": content}),)})
def _targets(point: InjectionPoint, messages: Sequence[Block]) -> tuple[int, ...]:
if point.index is not None:
index: Final = point.index + len(messages) if point.index < 0 else point.index
return (index,) if 0 <= index < len(messages) else ()
return tuple(i for i, message in enumerate(messages) if message.get("role") == point.role)
def _mark_system(system: JSONValue, control: Block) -> JSONValue:
if isinstance(system, str):
return (MappingProxyType({"type": "text", "text": system, "cache_control": control}),)
blocks: Final = _blocks(system)
if any(_marked(block) for block in blocks):
return system
return _with_last_block_marked(blocks, control) or system
def _mark_messages(messages: Sequence[Block], targets: Sequence[tuple[int, Block]], budget: int) -> Sequence[Block]:
if budget <= 0 or not targets:
return messages
(index, control), rest = targets[0], targets[1:]
marked: Final = (
None
if _breakpoints(messages[index])
else _with_last_block_marked(_blocks(messages[index].get("content")), control)
)
if marked is None:
return _mark_messages(messages, rest, budget)
message: Final = MappingProxyType({**messages[index], "content": marked})
updated: Final = (*messages[:index], message, *messages[index + 1 :])
return _mark_messages(updated, rest, budget - 1)
@dataclass(frozen=True, slots=True)
class AnthropicCacheControl(InterceptorPort):
"""A body patch that adds the configured breakpoints to a Messages request."""
config: Config
def payload(self, request: RequestFactsV1) -> WirePatchV1 | None:
"""The patch this request needs, or `None` for a provider or body this port leaves alone."""
if request["custom_llm_provider"] not in self.config.providers:
return None
body: Final = self.marked(request["body"])
if body is None:
return None
patch: Final[WirePatchV1] = {"body": body}
return patch
def marked(self, body: JSONValue) -> JSONValue | None:
"""The Messages body with the configured breakpoints added, or `None` to leave it alone."""
if not isinstance(body, Mapping) or not self.config.points:
return None
raw_messages: Final = body.get("messages")
if not isinstance(raw_messages, Sequence) or isinstance(raw_messages, str):
return None
messages: Final = tuple(_as_blocks(message) for message in raw_messages if isinstance(message, Mapping))
if len(messages) != len(raw_messages):
return None
system: Final = body.get("system")
system_point: Final = next((point for point in self.config.points if point.role == "system"), None)
used: Final = sum(_breakpoints(message) for message in messages) + sum(1 for b in _blocks(system) if _marked(b))
new_system: Final = (
_mark_system(system, system_point.control)
if system_point is not None and system is not None and used < MAX_CACHE_CONTROL_BLOCKS
else system
)
used_after_system: Final = used + int(new_system != system)
targets: Final = tuple(
(index, point.control)
for point in self.config.points
if point.role != "system"
for index in _targets(point, messages)
)
new_messages: Final = _mark_messages(messages, targets, MAX_CACHE_CONTROL_BLOCKS - used_after_system)
replaced: Final = MappingProxyType({"messages": new_messages, "system": new_system})
return MappingProxyType({key: replaced.get(key, value) for key, value in body.items()})

View file

@ -0,0 +1,129 @@
"""Generic API logger on the v1 contract: batches of call records to one HTTP endpoint.
Legacy twin: `litellm.integrations.generic_api.generic_api_callback.GenericAPILogger`.
The legacy logger ships the whole `StandardLoggingPayload`; this port ships the fields of
it that v1 facts can fill, under the same names; `manifest.py` lists the rest as gaps.
Not ported yet: `callback_name` presets from `generic_api_compatible_callbacks.json`,
retries, and the pre-StandardLoggingPayload body behind `litellm.generic_api_use_v1`.
"""
import json
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import datetime
from typing import Final, Literal, TypeAlias
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.callbacks_v1 import JSONValue
from litellm.callbacks_v1.builtin.port import Batching, CallRecord, Delivery, SinkPort
LogFormat: TypeAlias = Literal["json_array", "ndjson", "single"]
ApiEventType: TypeAlias = Literal["llm_api_success", "llm_api_failure"]
@dataclass(frozen=True, slots=True)
class Config:
endpoint: str
headers: tuple[tuple[str, str], ...] = (("Content-Type", "application/json"),)
event_types: frozenset[ApiEventType] = frozenset({"llm_api_success", "llm_api_failure"})
log_format: LogFormat = "json_array"
batch_size: int = 512
flush_interval: float = 5.0
@staticmethod
def from_env(env: Mapping[str, str]) -> "Config":
endpoint: Final = env.get("GENERIC_LOGGER_ENDPOINT")
if endpoint is None:
raise ValueError("endpoint not set for GenericAPILogger, GENERIC_LOGGER_ENDPOINT not found")
pairs: Final = (item.split("=", 1) for item in env.get("GENERIC_LOGGER_HEADERS", "").split(",") if "=" in item)
return Config(
endpoint=endpoint,
headers=(("Content-Type", "application/json"), *((key.strip(), value.strip()) for key, value in pairs)),
)
class ErrorInformation(TypedDict):
error_class: ReadOnly[str]
error_message: ReadOnly[str]
error_code: ReadOnly[str]
class Record(TypedDict):
"""The `StandardLoggingPayload` fields v1 can fill, under their legacy names."""
id: ReadOnly[str]
litellm_call_id: ReadOnly[str]
call_type: ReadOnly[str]
stream: ReadOnly[bool]
status: ReadOnly[Literal["success", "failure"]]
startTime: ReadOnly[float]
endTime: ReadOnly[float]
response_time: ReadOnly[float]
model: ReadOnly[str | None]
custom_llm_provider: ReadOnly[str | None]
model_parameters: ReadOnly[JSONValue]
response: ReadOnly[NotRequired[JSONValue]]
error_str: ReadOnly[NotRequired[str]]
error_information: ReadOnly[NotRequired[ErrorInformation]]
@dataclass(frozen=True, slots=True)
class GenericApi(SinkPort[Record]):
"""One record per finished call, batched into one endpoint."""
config: Config
@property
def batching(self) -> Batching:
return Batching(self.config.batch_size, self.config.flush_interval)
def payload(self, call: CallRecord, _now: datetime) -> Record | None:
"""The record for a finished call, or `None` when `config.event_types` leaves its outcome out.
A failed call has `error_*` where a successful one has `response`. The time is the call's own,
so the clock the host passes is unused."""
terminal: Final = call.terminal
wanted: Final = "llm_api_success" if terminal["type"] == "call.succeeded" else "llm_api_failure"
if wanted not in self.config.event_types:
return None
request: Final = call.request
response: Final = terminal["response"] if terminal["type"] == "call.succeeded" else None
response_id: Final = response.get("id") if isinstance(response, Mapping) else None
common: Final[Record] = {
"id": response_id if isinstance(response_id, str) else call.call_id,
"litellm_call_id": call.call_id,
"call_type": call.call_type,
"stream": terminal["streamed"],
"status": "success" if terminal["type"] == "call.succeeded" else "failure",
"startTime": terminal["timing"]["start_time"],
"endTime": terminal["timing"]["end_time"],
"response_time": terminal["timing"]["duration"],
"model": request["model"] if request is not None else None,
"custom_llm_provider": request["custom_llm_provider"] if request is not None else None,
"model_parameters": request["optional_params"] if request is not None else None,
}
if terminal["type"] == "call.succeeded":
succeeded: Final[Record] = {**common, "response": terminal["response"]}
return succeeded
error: Final = terminal["error"]
failed: Final[Record] = {
**common,
"error_str": error["message"],
"error_information": {
"error_class": error["class"].rsplit(".", 1)[-1],
"error_message": error["message"],
"error_code": str(error["status_code"]) if error["status_code"] is not None else "",
},
}
return failed
def deliveries(self, batch: tuple[Record, ...]) -> tuple[Delivery, ...]:
if self.config.log_format == "single":
return tuple(Delivery(self.config.endpoint, self.config.headers, json.dumps(r).encode()) for r in batch)
body: Final = (
json.dumps(batch)
if self.config.log_format == "json_array"
else "\n".join(json.dumps(record) for record in batch)
)
return (Delivery(self.config.endpoint, self.config.headers, body.encode()),)

View file

@ -0,0 +1,218 @@
"""Every built-in that has a v1 port, its legacy twin, how far the port has come, and the
gaps it is waiting on.
This whole module is migration scaffolding, and it is all of it: `port.py` and the ports
themselves carry none. Read by tests only. Nothing here registers a callback or decides
what runs in a call -- a port goes live by removing its legacy twin, not by a switch in
this table -- and when the last twin is gone this file is deleted outright.
The registration name lives here rather than on the port because it is the host's: a port
is a value, and the same one configured twice is two subscribers under two names.
`python -m litellm.callbacks_v1.builtin.manifest` prints the gap ledger and the backlog.
"""
from dataclasses import dataclass
from types import ModuleType
from typing import Final, Literal, TypeAlias
from litellm.callbacks_v1.builtin import anthropic_cache_control, generic_api, openmeter
Kind: TypeAlias = Literal["sink", "interceptor"]
GapKind: TypeAlias = Literal["fact", "event", "capability"]
# exploring: the port exists and its payload tests pass.
# parity: its differential test against the legacy twin passes, every difference explained.
# ready: parity, no open gap, and every route the legacy twin serves emits v1 envelopes.
Status: TypeAlias = Literal["exploring", "parity", "ready"]
@dataclass(frozen=True, slots=True)
class Gap:
"""One legacy input the v1 contract does not supply.
`fact` is a value missing from an envelope, `event` is a moment no envelope marks,
and `capability` is something the contract does not let a callback do at all.
`key` names what the contract would have to grow, not what the legacy logger read, so
two ports missing the same thing share a key and the ledger can count them. It is also
what an `Allowed` parity difference cites, which is how closing a gap finds its tests.
"""
kind: GapKind
key: str
legacy: str
note: str
@dataclass(frozen=True, slots=True)
class Entry:
"""One port: where it lives, what it registers as, what it replaces, and where it has got to."""
module: ModuleType
name: str
kind: Kind
legacy: str
status: Status
gaps: tuple[Gap, ...]
OPENMETER_GAPS: Final = (
Gap("fact", "cost", "kwargs['response_cost']", "no cost fact on call.succeeded; `data.cost` is always null"),
Gap(
"fact",
"usage",
"response_obj['usage'] of a ModelResponse",
"token counts are only reachable by parsing `response` per provider shape; needs a usage fact",
),
Gap(
"fact",
"request_user",
"kwargs['user']",
"the request's `user` is not projected; subject comes from metadata only",
),
Gap(
"fact",
"litellm_metadata",
"litellm_params['metadata'] fed by `litellm_metadata`",
"call.started projects only the `metadata` keyword; the proxy's identity on Messages-style routes is invisible",
),
Gap(
"fact",
"terminal_model",
"kwargs['model']",
"the terminal envelope has no model, so the port joins request.sending by call_id",
),
Gap(
"capability",
"off_path_delivery",
"Logging runs success callbacks on an executor / the logging worker",
"v1 observers run inline in the call; the port needs its own outbox to stay off the call path",
),
)
ANTHROPIC_CACHE_CONTROL_GAPS: Final = (
Gap(
"fact",
"request_params",
"kwargs['cache_control_injection_points']",
"per-request injection points are not in RequestFacts; the port only knows the points it was built with",
),
Gap(
"fact",
"request_call_type",
"call_type",
"RequestFacts has no call_type, so the port recognises a Messages request by the body's shape",
),
Gap(
"capability",
"route_neutral_rewrite",
"get_chat_completion_prompt rewrites the route-neutral messages before transformation",
"before_send sees only the provider's wire body, so every dialect needs its own body walker",
),
)
# What the legacy StandardLoggingPayload has and v1 facts cannot fill, by what the
# contract would have to grow to fill it.
_MISSING_FACTS: Final = (
("cost", "response_cost, cost_breakdown, saved_cache_cost, autorouter_savings", "no cost facts"),
("usage", "total_tokens, prompt_tokens, completion_tokens", "no usage fact; only the raw provider response"),
("cache", "cache_hit, cache_key", "no cache fact"),
(
"routing",
"model_id, model_group, api_base, model_map_information",
"no routing fact; `url` is the wire URL, not api_base",
),
("identity", "metadata.user_api_key_*, end_user, requester_ip_address, user_agent", "no identity fact"),
("tags", "request_tags, request_model_access_groups", "no tags fact"),
("correlation", "trace_id, session_id", "no correlation beyond call_id"),
("first_token_time", "completionStartTime", "no first-token time in `timing`"),
("no_equivalent", "hidden_params, guardrail_information, standard_built_in_tools_params", "no equivalent"),
("prompt", "messages", "only the provider-shaped wire body; no route-neutral prompt projection"),
(
"normalised_response",
"response",
"legacy logs the OpenAI-normalised ModelResponse; v1 carries the route's own public response",
),
(
"error_detail",
"error_information.traceback, llm_provider, error_provider_request_id",
"ErrorFacts is class/message/status only",
),
)
GENERIC_API_GAPS: Final = (
*(Gap("fact", key, f"standard_logging_object: {fields}", note) for key, fields, note in _MISSING_FACTS),
Gap(
"fact",
"terminal_model",
"standard_logging_object: model, custom_llm_provider, model_parameters",
"the terminal envelope has none of them, so the port joins request.sending by call_id",
),
Gap(
"capability",
"off_path_delivery",
"CustomBatchLogger.periodic_flush on the proxy's event loop",
"v1 observers run inline in the call; batching and I/O live on the port's outbox thread",
),
)
PORTS: Final = (
Entry(
openmeter,
"openmeter",
"sink",
"litellm.integrations.openmeter:OpenMeterLogger",
"parity",
OPENMETER_GAPS,
),
Entry(
generic_api,
"generic_api",
"sink",
"litellm.integrations.generic_api.generic_api_callback:GenericAPILogger",
"parity",
GENERIC_API_GAPS,
),
Entry(
anthropic_cache_control,
"anthropic_cache_control",
"interceptor",
"litellm.integrations.anthropic_cache_control_hook:AnthropicCacheControlHook",
"parity",
ANTHROPIC_CACHE_CONTROL_GAPS,
),
)
def entry(name: str) -> Entry:
"""The manifest entry a port registers under; how a test reaches its gaps."""
(found,) = (candidate for candidate in PORTS if candidate.name == name)
return found
def ledger() -> str:
"""Every open gap, grouped by port: what the contract has to grow before ports can be `ready`."""
return "\n".join(
line
for port in PORTS
for line in (
f"{port.name} [{port.kind}, {port.status}] <- {port.legacy}",
*(f" {gap.kind:<10} {gap.key}: {gap.legacy}\n {gap.note}" for gap in port.gaps),
)
)
def backlog() -> str:
"""Every gap key with the ports that are waiting on it, most wanted first: the order the contract should grow in."""
keys: Final = frozenset(gap.key for port in PORTS for gap in port.gaps)
waiting: Final = tuple(
(key, tuple(port.name for port in PORTS if any(gap.key == key for gap in port.gaps))) for key in sorted(keys)
)
return "\n".join(
f" {key:<22} {', '.join(names)}" for key, names in sorted(waiting, key=lambda item: -len(item[1]))
)
if __name__ == "__main__":
print(f"{ledger()}\n\nbacklog, by how many ports wait on it\n{backlog()}") # noqa: T201 # the ledger is this module's command-line output

View file

@ -0,0 +1,125 @@
"""OpenMeter on the v1 contract: one CloudEvent per successful call.
Legacy twin: `litellm.integrations.openmeter.OpenMeterLogger`.
"""
import json
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import datetime
from typing import Final
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.callbacks_v1 import JSONValue
from litellm.callbacks_v1.builtin.port import Batching, CallRecord, Delivery, SinkPort
_USAGE_KEYS: Final = ("prompt_tokens", "completion_tokens", "total_tokens")
@dataclass(frozen=True, slots=True)
class Config:
api_key: str
endpoint: str = "https://openmeter.cloud"
event_type: str = "litellm_tokens"
@staticmethod
def from_env(env: Mapping[str, str]) -> "Config":
api_key: Final = env.get("OPENMETER_API_KEY")
if api_key is None:
raise ValueError("Missing keys=['OPENMETER_API_KEY'] in environment.")
return Config(
api_key=api_key,
endpoint=env.get("OPENMETER_API_ENDPOINT", "https://openmeter.cloud"),
event_type=env.get("OPENMETER_EVENT_TYPE", "litellm_tokens"),
)
@property
def url(self) -> str:
return f"{self.endpoint.rstrip('/')}/api/v1/events"
class Usage(TypedDict):
prompt_tokens: ReadOnly[JSONValue]
completion_tokens: ReadOnly[JSONValue]
total_tokens: ReadOnly[JSONValue]
class MeterData(TypedDict):
model: ReadOnly[str | None]
cost: ReadOnly[float | None]
prompt_tokens: ReadOnly[NotRequired[JSONValue]]
completion_tokens: ReadOnly[NotRequired[JSONValue]]
total_tokens: ReadOnly[NotRequired[JSONValue]]
class CloudEvent(TypedDict):
specversion: ReadOnly[str]
type: ReadOnly[str]
id: ReadOnly[str]
time: ReadOnly[str]
subject: ReadOnly[str]
source: ReadOnly[str]
data: ReadOnly[MeterData]
def _usage(response: JSONValue) -> Usage | None:
"""Token counts when the response carries them in the OpenAI shape, as the legacy logger reads them."""
usage: Final = response.get("usage") if isinstance(response, Mapping) else None
if not isinstance(usage, Mapping) or not any(key in usage for key in _USAGE_KEYS):
return None
counted: Final[Usage] = {
"prompt_tokens": usage.get("prompt_tokens", 0),
"completion_tokens": usage.get("completion_tokens", 0),
"total_tokens": usage.get("total_tokens"),
}
return counted
def _metered(model: str | None, usage: Usage) -> MeterData:
data: Final[MeterData] = {"model": model, "cost": None, **usage}
return data
@dataclass(frozen=True, slots=True)
class OpenMeter(SinkPort[CloudEvent]):
"""One CloudEvent per successful call, one request per event."""
config: Config
@property
def batching(self) -> Batching:
"""One event per request, as the legacy logger posts them."""
return Batching()
def payload(self, call: CallRecord, now: datetime) -> CloudEvent | None:
"""The CloudEvent for a finished call, or `None` when OpenMeter has nothing to meter."""
if call.terminal["type"] != "call.succeeded":
return None
subject: Final = call.started["metadata"].get("user_api_key_user_id") if call.started is not None else None
if subject is None:
raise ValueError("OpenMeter: user is required")
response: Final = call.terminal["response"]
response_id: Final = response.get("id") if isinstance(response, Mapping) else None
model: Final = call.request["model"] if call.request is not None else None
usage: Final = _usage(response)
unmetered: Final[MeterData] = {"model": model, "cost": None}
data: Final = unmetered if usage is None else _metered(model, usage)
event: Final[CloudEvent] = {
"specversion": "1.0",
"type": self.config.event_type,
"id": response_id if isinstance(response_id, str) else call.call_id,
"time": now.isoformat(),
"subject": str(subject),
"source": "litellm-proxy",
"data": data,
}
return event
def deliveries(self, batch: tuple[CloudEvent, ...]) -> tuple[Delivery, ...]:
"""One request per CloudEvent, as the legacy logger posts them."""
headers: Final = (
("Content-Type", "application/cloudevents+json"),
("Authorization", f"Bearer {self.config.api_key}"),
)
return tuple(Delivery(self.config.url, headers, json.dumps(event).encode()) for event in batch)

View file

@ -0,0 +1,95 @@
"""Where a sink's handler leaves its work, so that the handler never does I/O.
The v1 contract runs an observer inline in the call it observes. A handler that posted to
a vendor would put that vendor's latency into every call, so a sink's handler only builds
a value and `put`s it here; one daemon thread per outbox delivers it, alone or in batches.
"""
import atexit
import os
import queue
import threading
import time
from collections.abc import Callable
from typing import Final, Generic, TypeVar
from litellm._logging import verbose_logger
T = TypeVar("T") # rebind-ok: a TypeVar must bind to a bare name to be recognised as one
class Outbox(Generic[T]):
"""Delivers items off the call path, `batch_size` at a time or every `flush_interval`.
`deliver` gets a non-empty batch and may raise: the failure is logged and the batch is
dropped, as the legacy batch loggers do. A full outbox drops the new item rather than
block the call that produced it.
"""
def __init__(
self,
name: str,
deliver: Callable[[tuple[T, ...]], None],
batch_size: int = 1,
flush_interval: float = 5.0,
capacity: int = 50_000,
) -> None:
self._name: Final = name
self._deliver: Final = deliver
self._batch_size: Final = max(1, batch_size)
self._flush_interval: Final = flush_interval
self._items: Final[queue.Queue[T]] = queue.Queue(maxsize=capacity)
self._lock: Final = threading.Lock()
self._flush_now: Final = threading.Event()
self._worker_pid: int | None = None
atexit.register(self.flush)
def put(self, item: T) -> None:
self._ensure_worker()
try:
self._items.put_nowait(item)
except queue.Full:
verbose_logger.warning("callback %s outbox is full; dropping one item", self._name)
def flush(self, timeout: float = 5.0) -> bool:
"""Delivers everything already put; returns whether it drained within `timeout`."""
deadline: Final = time.monotonic() + timeout
self._flush_now.set()
while self._items.unfinished_tasks and time.monotonic() < deadline:
time.sleep(0.005)
return not self._items.unfinished_tasks
def _ensure_worker(self) -> None:
# A forked child inherits the queue but not the thread, so the worker is per process.
with self._lock:
if self._worker_pid == os.getpid():
return
self._worker_pid = os.getpid()
threading.Thread(target=self._run, name=f"callback-outbox-{self._name}", daemon=True).start()
def _next_batch(self) -> tuple[T, ...]:
first: Final = self._items.get()
deadline: Final = time.monotonic() + self._flush_interval
batch: Final[list[T]] = [first] # mutable-ok: a batch is gathered from a blocking queue one item at a time
while len(batch) < self._batch_size:
remaining = deadline - time.monotonic() # rebind-ok: the wait shrinks on every pass of the gather loop
if remaining <= 0 or (self._flush_now.is_set() and self._items.empty()):
break
try:
batch.append(self._items.get(timeout=min(remaining, 0.01)))
except queue.Empty:
continue
return tuple(batch)
def _run(self) -> None:
while True:
batch = self._next_batch() # rebind-ok: the worker loop takes a new batch on every pass
try:
self._deliver(batch)
except Exception: # noqa: BLE001 # a vendor failure must never end the delivery thread
verbose_logger.exception("callback %s failed to deliver %d item(s)", self._name, len(batch))
finally:
for _ in batch:
self._items.task_done()
if self._items.empty():
self._flush_now.clear()

View file

@ -0,0 +1,90 @@
"""The shape every port has, and nothing that can act.
A port is a frozen class that holds its `Config` and names one of the protocols below as
its base: pure methods from contract values to what the port produces. Declaring the base
is what holds it to the shape, at the class rather than at some call site far away; a
method that drifts fails there, and a missing one fails wherever the port is built. It opens no socket,
starts no thread and reads neither the environment nor the clock; the host hands it all of
those as values. That is the shape a sandboxed custom callback is confined to, and
built-ins take it first so both share one interface.
A port has no identity of its own. It is not named, does not know which events reach it
and implements nothing of the v1 contract: `runtime.py` names it and wraps it into a
subscriber, so the same port configured twice is two registrations rather than a clash.
Nothing in this file is migration scaffolding; `manifest.py` holds all of that.
"""
from dataclasses import dataclass
from datetime import datetime
from typing import Protocol, TypeVar
from litellm.callbacks_v1 import (
CallFailedV1,
CallStartedV1,
CallSucceededV1,
RequestFactsV1,
RequestSendingV1,
WirePatchV1,
)
R = TypeVar("R") # rebind-ok: a TypeVar must bind to a bare name to be recognised as one
@dataclass(frozen=True, slots=True)
class Delivery:
"""One request to a vendor, fully decided; sending it is the only thing left."""
url: str
headers: tuple[tuple[str, str], ...]
body: bytes
@dataclass(frozen=True, slots=True)
class CallRecord:
"""Every envelope of one finished call a sink needs, joined by `call_id`.
`request` is absent when the call failed before anything was sent.
"""
call_id: str
call_type: str
started: CallStartedV1 | None
request: RequestSendingV1 | None
terminal: CallSucceededV1 | CallFailedV1
@dataclass(frozen=True, slots=True)
class Batching:
"""How many records the host carries to the vendor at once, and how long it waits."""
size: int = 1
interval: float = 5.0
class SinkPort(Protocol[R]):
"""A port that observes finished calls and tells a vendor.
`payload` is the port: one finished call and the current time to the vendor's record,
or `None` for nothing to send, which is also how a port filters. `deliveries` frames a
batch of those records as requests, headers and framing included, and `batching` says
how large a batch the host should collect. The two are separate because the handler
runs inline in the call and the delivery does not.
"""
def payload(self, call: CallRecord, now: datetime, /) -> R | None: ...
def deliveries(self, batch: tuple[R, ...], /) -> tuple[Delivery, ...]: ...
@property
def batching(self) -> Batching: ...
class InterceptorPort(Protocol):
"""A port that patches the wire request before it is sent; its failure fails the call.
`payload` is the patch the request needs, or `None` to leave it alone. It decides
whether the request is one this port acts on at all: the host applies what it returns
and filters nothing.
"""
def payload(self, request: RequestFactsV1, /) -> WirePatchV1 | None: ...

View file

@ -0,0 +1,156 @@
"""The in-process host of a port: everything a port is not allowed to do itself.
Joining a call's envelopes, reading the clock, batching off the call path and talking to
the vendor all live here, once, so a port stays pure methods over values (`port.py`).
`Sink` and `Patcher` are the two subscribers: each names a port of its kind and wraps it,
and they are the only things in this package that implement the v1 contract, so a port
never writes `name`, `events` or a handler. The name is the caller's because it is the
registration's, not the port's: one port configured twice is two subscribers. This is not a sandbox and isolates nothing: it is plain
in-process Python. It is a stand-in for what a host should own (join, queue, batch,
deliver). When the contract grows off-path delivery and a joined call record, this module
is deleted and the ports do not change. Custom callbacks are isolated by the gateway's
extension host, in Rust.
"""
import threading
from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Final, Generic, Protocol, TypeVar
from litellm.callbacks_v1 import (
CallStartedV1,
EnvelopeV1,
EventName,
Interceptor,
Observer,
RequestFactsV1,
RequestSendingV1,
WirePatchV1,
)
from litellm.callbacks_v1.builtin.outbox import Outbox
from litellm.callbacks_v1.builtin.port import CallRecord, Delivery, InterceptorPort, SinkPort
R = TypeVar("R") # rebind-ok: a TypeVar must bind to a bare name to be recognised as one
class Transport(Protocol):
def send(self, delivery: Delivery) -> None: ...
class HttpxTransport:
"""Sends through the same client the legacy loggers use, so proxy TLS settings apply."""
def __init__(self) -> None:
from litellm.llms.custom_httpx.http_handler import HTTPHandler
self._client: Final = HTTPHandler()
def send(self, delivery: Delivery) -> None:
headers: Final = dict(delivery.headers) # mutable-ok: the httpx client takes its headers as a dict
self._client.post(url=delivery.url, headers=headers, content=delivery.body) # pyright: ignore[reportUnknownMemberType] # the legacy HTTP handler is only partly typed
def utc_now() -> datetime:
return datetime.now(timezone.utc)
@dataclass(frozen=True, slots=True)
class _OpenCall:
started: CallStartedV1 | None = None
request: RequestSendingV1 | None = None
JOINED_EVENTS: Final[frozenset[EventName]] = frozenset(
{"call.started", "request.sending", "call.succeeded", "call.failed"}
)
class CallJoin:
"""Joins a call's envelopes into one `CallRecord` at its terminal envelope.
The terminal envelope carries neither the model nor the metadata, so a sink has to
remember the earlier ones. A cancelled call never gets a terminal envelope, which is
why the open calls are bounded: the oldest is forgotten rather than leaked.
"""
def __init__(self, capacity: int = 10_000) -> None:
self._capacity: Final = capacity
self._lock: Final = threading.Lock()
self._open: Final[dict[str, _OpenCall]] = {} # mutable-ok: per-call join state is an evolving keyed store
def accept(self, envelope: EnvelopeV1) -> CallRecord | None:
call_id: Final = envelope["call_id"]
event: Final = envelope["event"]
with self._lock:
if event["type"] == "call.succeeded" or event["type"] == "call.failed":
seen: Final = self._open.pop(call_id, _OpenCall())
return CallRecord(call_id, envelope["call_type"], seen.started, seen.request, event)
current: Final = self._open.get(call_id, _OpenCall())
if event["type"] == "call.started":
self._open[call_id] = _OpenCall(event, current.request)
elif event["type"] == "request.sending":
self._open[call_id] = _OpenCall(current.started, event)
while len(self._open) > self._capacity:
del self._open[next(iter(self._open))]
return None
def open_calls(self) -> int:
with self._lock:
return len(self._open)
class Sink(Observer, Generic[R]):
"""The subscriber of every `SinkPort`: the port supplies pure methods, this runs them.
The handler only joins and enqueues, so the sync one serves async calls too and never
blocks; the outbox thread calls `deliveries` and sends.
"""
def __init__(
self,
name: str,
port: SinkPort[R],
transport: Transport,
clock: Callable[[], datetime] = utc_now,
) -> None:
self.name = name
self.events = JOINED_EVENTS
self._port: Final = port
self._transport: Final = transport
self._clock: Final = clock
self._calls: Final = CallJoin()
self.outbox: Final[Outbox[R]] = Outbox(
name, self._deliver, batch_size=port.batching.size, flush_interval=port.batching.interval
)
def _deliver(self, batch: tuple[R, ...]) -> None:
for delivery in self._port.deliveries(batch):
self._transport.send(delivery)
def on_event(self, event: EnvelopeV1) -> None:
call: Final = self._calls.accept(event)
if call is None:
return
record: Final = self._port.payload(call, self._clock())
if record is not None:
self.outbox.put(record)
def open_calls(self) -> int:
return self._calls.open_calls()
class Patcher(Interceptor):
"""The subscriber of every `InterceptorPort`: the contract's `before_send` over `payload`.
An interceptor runs on the call path by definition, so there is nothing to join, batch
or defer here; this exists so that an interceptor port declares as little as a sink does.
"""
def __init__(self, name: str, port: InterceptorPort) -> None:
self.name = name
self.events: frozenset[EventName] = frozenset()
self._port: Final = port
def before_send(self, request: RequestFactsV1) -> WirePatchV1 | None:
return self._port.payload(request)

View file

@ -0,0 +1,486 @@
"""The extension host: a process that runs v1 callbacks for a gateway that does not.
python -m litellm.callbacks_v1.host --fd 3 --load my_pkg.audit --load my_pkg.cb:callback
The gateway never imports a callback. It starts (or attaches to) this worker, which imports
the callbacks exactly as an SDK process would, reports what they subscribed to, and then
only answers: the gateway sends envelopes and interception requests over one socket, the
worker runs the handlers and sends back patches. The worker cannot call into the gateway;
there is no frame for it. A callback file is the same file in the SDK and here.
This module isolates nothing by itself. What the worker can reach (environment, files,
network) is whatever the process it runs in was given; confinement is the spawner's job.
Frames are a 4-byte big-endian length and that many bytes of JSON, one object with a
`type`. `litellm-rust/crates/callbacks-v1-exthost/golden` pins every frame for both sides.
worker -> gateway hello {schema, subscriptions: [{name, events, observes, intercepts}]}
gateway -> worker event {subscriber, envelope} no reply
gateway -> worker intercept {id, subscriber, request}
worker -> gateway patch {id, patch} or error {id, error_class, message}
worker -> gateway report {subscriber, event, message} an observer raised
gateway -> worker flush {id}
worker -> gateway flushed {id} every event sent before the flush was handled
`subscriber` is an index into the hello's `subscriptions`. Envelopes reach one subscriber
in the order they were sent; an interception never waits behind events.
A handler is scheduled by the kind its author declared, which is the author's decision and
not this module's guess. An `async` handler is a task on the worker's loop, so one that only
waits costs no thread and concurrency is the loop's rather than a pool size; a sync handler
gets a thread, because running it blocks one, and the loop keeps turning while it does. Every
frame leaves through one writer, so no handler and no loop callback blocks on the socket.
"""
import argparse
import asyncio
import importlib
import json
import queue
import socket
import sys
import threading
from collections.abc import Coroutine, Mapping, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass
from typing import Final, Literal, Protocol, TypeAlias
from typing_extensions import ReadOnly, TypedDict
from litellm.callbacks_v1 import SCHEMA, JSONValue, SchemaVersion, Subscriber, register, snapshot
MAX_FRAME_BYTES: Final = 64 * 1024 * 1024
INTERCEPT_THREADS: Final = 8
OBSERVER_QUEUE: Final = 1024
class Handler(Protocol):
"""A handler the worker calls on a thread, because running it blocks one."""
def __call__(self, argument: JSONValue, /) -> object: ...
class AsyncHandler(Protocol):
"""A handler the worker awaits on its loop, because waiting is all it does."""
def __call__(self, argument: JSONValue, /) -> Coroutine[object, object, object]: ...
class SubscriptionData(TypedDict):
name: ReadOnly[str]
events: ReadOnly[Sequence[str]]
observes: ReadOnly[bool]
intercepts: ReadOnly[bool]
class HelloFrame(TypedDict):
type: ReadOnly[Literal["hello"]]
schema: ReadOnly[SchemaVersion]
subscriptions: ReadOnly[Sequence[SubscriptionData]]
class PatchFrame(TypedDict):
type: ReadOnly[Literal["patch"]]
id: ReadOnly[JSONValue]
patch: ReadOnly[JSONValue]
class ErrorFrame(TypedDict):
type: ReadOnly[Literal["error"]]
id: ReadOnly[JSONValue]
error_class: ReadOnly[str]
message: ReadOnly[str]
class ReportFrame(TypedDict):
type: ReadOnly[Literal["report"]]
subscriber: ReadOnly[int]
event: ReadOnly[JSONValue]
message: ReadOnly[str]
class FlushedFrame(TypedDict):
type: ReadOnly[Literal["flushed"]]
id: ReadOnly[JSONValue]
Sent: TypeAlias = HelloFrame | PatchFrame | ErrorFrame | ReportFrame | FlushedFrame
Received: TypeAlias = Mapping[str, JSONValue]
class ProtocolError(Exception):
"""The peer sent something that is not a frame of this protocol; the worker stops."""
def _read_exactly(sock: socket.socket, size: int) -> bytes | None:
"""`size` bytes, or `None` when the peer closed before the first of them."""
chunks: Final[list[bytes]] = [] # mutable-ok: a socket yields a frame in pieces
remaining = size # rebind-ok: counts down as the pieces arrive
while remaining:
chunk = sock.recv(min(remaining, 1 << 20)) # rebind-ok: one piece per pass
if not chunk:
if chunks:
raise ProtocolError("connection closed inside a frame")
return None
chunks.append(chunk)
remaining -= len(chunk)
return b"".join(chunks)
def read_frame(sock: socket.socket) -> Received | None:
"""The next frame, or `None` once the peer has closed the connection."""
header: Final = _read_exactly(sock, 4)
if header is None:
return None
size: Final = int.from_bytes(header, "big")
if size > MAX_FRAME_BYTES:
raise ProtocolError(f"frame of {size} bytes exceeds {MAX_FRAME_BYTES}")
body: Final = _read_exactly(sock, size)
if body is None:
raise ProtocolError("connection closed inside a frame")
frame: Final[JSONValue] = json.loads(body) # pyright: ignore[reportAny] # untyped until the checks below
if not isinstance(frame, Mapping) or not isinstance(frame.get("type"), str):
raise ProtocolError("a frame is a JSON object with a string `type`")
return frame
def encode_frame(frame: Sent | Received) -> bytes:
body: Final = json.dumps(frame, separators=(",", ":")).encode()
return len(body).to_bytes(4, "big") + body
def load(reference: str) -> None:
"""Imports `module`, whose decorators register as they run, or registers `module:attr`."""
module_name, _, attribute = reference.partition(":")
module: Final = importlib.import_module(module_name)
if attribute:
register(getattr(module, attribute)) # pyright: ignore[reportAny] # the attribute is whatever the operator named; `register` validates it
def _subscription(subscriber: Subscriber) -> SubscriptionData:
data: Final[SubscriptionData] = {
"name": subscriber.name,
"events": tuple(sorted(subscriber.events)),
"observes": subscriber.on_event is not None or subscriber.async_on_event is not None,
"intercepts": subscriber.before_send is not None or subscriber.async_before_send is not None,
}
return data
def hello(subscribers: Sequence[Subscriber]) -> HelloFrame:
frame: Final[HelloFrame] = {
"type": "hello",
"schema": SCHEMA,
"subscriptions": tuple(_subscription(subscriber) for subscriber in subscribers),
}
return frame
def _plain(value: object) -> JSONValue:
"""A handler's return value as JSON can carry it; callbacks may build patches from read-only views."""
if isinstance(value, Mapping):
# A handler's return value is untyped until it is encoded, and json.dumps wants a dict.
return {str(k): _plain(v) for k, v in value.items()} # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # mutable-ok: json.dumps encodes dict, not Mapping
if isinstance(value, Sequence) and not isinstance(value, (str, bytes)):
return tuple(_plain(item) for item in value) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # a handler's return value is untyped until it is encoded
if value is None or isinstance(value, (bool, int, float, str)):
return value
raise TypeError(f"a callback returned {type(value).__name__}, which JSON cannot carry")
def _sync(handler: object | None) -> Handler | None:
"""A handler the worker calls on a thread; `register` has already checked its kind."""
return handler if callable(handler) else None
def _asynchronous(handler: object | None) -> AsyncHandler | None:
"""A handler the worker awaits on its loop; `register` has already checked its kind."""
return handler if callable(handler) else None # pyright: ignore[reportReturnType] # `register` validated the kind and `callable` narrows no further
@dataclass(frozen=True, slots=True)
class _Marker:
"""A flush's place in one lane's queue, resolved once everything queued before it is handled."""
reached: Future[None]
@dataclass(frozen=True, slots=True)
class _Stop:
"""The end of one lane's queue; the consumer returns once everything before it is handled."""
_Queued: TypeAlias = JSONValue | _Marker | _Stop
@dataclass(frozen=True, slots=True)
class _Lane:
"""One subscriber's handlers, each kept in the kind its author declared.
The kind is the author's scheduling decision and the worker routes on it: a handler that
waits on the network is `async` and costs no thread, one that occupies the CPU is sync
and gets a thread so that the loop keeps turning while it runs. Collapsing the two would
discard that decision, and it cannot be recovered by calling a handler to see what comes
back: calling a sync one has already run it, on whatever thread asked. `events` is the
thread a sync observer's envelopes are handled on, in order, and exists only for one.
"""
observe: Handler | None
async_observe: AsyncHandler | None
intercept: Handler | None
async_intercept: AsyncHandler | None
events: ThreadPoolExecutor | None
class Worker:
"""Serves one gateway connection over the subscribers registered when it was built.
Nothing that waits holds a thread it does not need. An async handler is a task on the
worker's loop, so concurrency is the loop's and not a pool size; a sync handler has a
thread because it blocks one; and every frame leaves through one writer, so no handler
and no loop callback ever blocks on the socket.
"""
def __init__(self, sock: socket.socket, subscribers: Sequence[Subscriber]) -> None:
self._sock: Final = sock
self._subscribers: Final = tuple(subscribers)
self._loop: Final = asyncio.new_event_loop()
self._outgoing: Final[queue.Queue[bytes | None]] = queue.Queue()
self._writer: Final = threading.Thread(target=self._write, name="callback-writer", daemon=True)
self._intercepts: Final = ThreadPoolExecutor(INTERCEPT_THREADS, thread_name_prefix="callback-intercept")
self._queues: Final[
dict[int, asyncio.Queue[_Queued]]
] = {} # mutable-ok: filled once on the loop before serving
self._consumers: Final[list[asyncio.Task[None]]] = [] # mutable-ok: filled once on the loop before serving
self._pending: Final[set[asyncio.Task[None]]] = (
set()
) # mutable-ok: the loop's strong references to its live tasks
self._lanes: Final = tuple(
_Lane(
_sync(subscriber.on_event),
_asynchronous(subscriber.async_on_event),
_sync(subscriber.before_send),
_asynchronous(subscriber.async_before_send),
ThreadPoolExecutor(1, thread_name_prefix=f"callback-events-{index}")
if subscriber.on_event is not None and subscriber.async_on_event is None
else None,
)
for index, subscriber in enumerate(self._subscribers)
)
def _write(self) -> None:
"""The only thread that writes to the socket, so that no caller ever blocks on it."""
for data in iter(self._outgoing.get, None):
try:
self._sock.sendall(data)
except OSError:
return # the gateway is gone; the read loop ends on its own
def _send(self, frame: Sent) -> None:
self._outgoing.put(encode_frame(frame))
def _report(self, index: int, envelope: JSONValue, error: BaseException) -> ReportFrame:
event: Final = envelope.get("event") if isinstance(envelope, Mapping) else None
kind: Final = event.get("type") if isinstance(event, Mapping) else None
frame: Final[ReportFrame] = {"type": "report", "subscriber": index, "event": kind, "message": str(error)}
return frame
def _patched(self, frame_id: JSONValue, result: object) -> PatchFrame:
frame: Final[PatchFrame] = {"type": "patch", "id": frame_id, "patch": _plain(result)}
return frame
def _failed(self, frame_id: JSONValue, error: BaseException) -> ErrorFrame:
name: Final = f"{type(error).__module__}.{type(error).__qualname__}"
frame: Final[ErrorFrame] = {"type": "error", "id": frame_id, "error_class": name, "message": str(error)}
return frame
def _observe(self, index: int, handler: Handler, envelope: JSONValue) -> None:
"""One sync observer on its lane's thread, in the order its envelopes arrived."""
try:
handler(envelope)
except Exception as error: # noqa: BLE001 # an observer's failure is reported and swallowed, by contract
self._send(self._report(index, envelope, error))
async def _consume(self, index: int, handler: AsyncHandler, envelopes: "asyncio.Queue[_Queued]") -> None:
"""One async observer's whole lane: one item at a time, so order holds without a thread."""
while True:
item: _Queued = await envelopes.get()
if isinstance(item, _Stop):
return
if isinstance(item, _Marker):
item.reached.set_result(None)
continue
try:
await handler(item)
except Exception as error: # noqa: BLE001 # an observer's failure is reported and swallowed, by contract
self._send(self._report(index, item, error))
def _intercept(self, frame_id: JSONValue, handler: Handler, request: JSONValue) -> None:
"""One sync interceptor on a pool thread, which it holds for as long as it runs."""
try:
self._send(self._patched(frame_id, handler(request)))
except Exception as error: # noqa: BLE001 # an interceptor's failure is the gateway's to act on
self._send(self._failed(frame_id, error))
async def _await_intercept(self, frame_id: JSONValue, handler: AsyncHandler, request: JSONValue) -> None:
"""One async interceptor as a task, so that a thousand of them wait on one thread."""
try:
self._send(self._patched(frame_id, await handler(request)))
except Exception as error: # noqa: BLE001 # an interceptor's failure is the gateway's to act on
self._send(self._failed(frame_id, error))
def _keep(self, task: "asyncio.Task[None]") -> None:
"""asyncio keeps only a weak reference to a task; this is the strong one."""
self._pending.add(task)
task.add_done_callback(self._pending.discard)
def _spawn(self, frame_id: JSONValue, handler: AsyncHandler, request: JSONValue) -> None:
"""On the loop: starts the interception and returns, so the read loop stays free."""
self._keep(self._loop.create_task(self._await_intercept(frame_id, handler, request)))
def _enqueue(self, index: int, envelope: JSONValue) -> None:
"""On the loop: a bounded lane drops rather than grow without limit, and says so."""
try:
self._queues[index].put_nowait(envelope)
except asyncio.QueueFull:
full: Final = RuntimeError(f"observer queue is full at {OBSERVER_QUEUE}; this envelope was dropped")
self._send(self._report(index, envelope, full))
def _mark(self, index: int, reached: "Future[None]") -> None:
"""On the loop: a marker waits for room rather than drop, so it keeps its place."""
self._keep(self._loop.create_task(self._queues[index].put(_Marker(reached))))
async def _start(self) -> None:
"""On the loop: one queue and one consumer per async observer, before the first frame."""
for index, lane in enumerate(self._lanes):
if lane.async_observe is not None:
envelopes: asyncio.Queue[_Queued] = asyncio.Queue(OBSERVER_QUEUE)
self._queues[index] = envelopes
self._consumers.append(self._loop.create_task(self._consume(index, lane.async_observe, envelopes)))
async def _drain(self) -> None:
"""On the loop: every lane sees a stop behind what was queued, then every task ends."""
for envelopes in self._queues.values():
await envelopes.put(_Stop())
outstanding: Final = (*self._consumers, *self._pending)
if outstanding:
_ = await asyncio.gather(*outstanding, return_exceptions=True)
def _lane(self, frame: Received) -> tuple[int, _Lane]:
index: Final = frame.get("subscriber")
if not isinstance(index, int) or isinstance(index, bool) or not 0 <= index < len(self._lanes):
raise ProtocolError(f"unknown subscriber: {index!r}")
return index, self._lanes[index]
def _flush(self, frame_id: JSONValue) -> None:
"""A marker at the back of each lane passes only after everything queued before it."""
markers: Final[list[Future[None]]] = [] # mutable-ok: one marker per lane, collected and then waited on
for index, lane in enumerate(self._lanes):
if index in self._queues:
reached: Future[None] = Future()
self._loop.call_soon_threadsafe(self._mark, index, reached)
markers.append(reached)
elif lane.events is not None:
markers.append(lane.events.submit(lambda: None))
for marker in markers:
marker.result()
flushed: Final[FlushedFrame] = {"type": "flushed", "id": frame_id}
self._send(flushed)
def _dispatch(self, frame: Received) -> None:
"""Routes one frame and returns; nothing here runs a handler or touches the socket."""
kind: Final = frame["type"]
if kind == "event":
index, observed = self._lane(frame)
envelope: Final = frame.get("envelope")
if observed.async_observe is not None:
self._loop.call_soon_threadsafe(self._enqueue, index, envelope)
elif observed.observe is not None and observed.events is not None:
_ = observed.events.submit(self._observe, index, observed.observe, envelope)
elif kind == "intercept":
_, asked = self._lane(frame)
if asked.async_intercept is not None:
self._loop.call_soon_threadsafe(
self._spawn, frame.get("id"), asked.async_intercept, frame.get("request")
)
elif asked.intercept is not None:
_ = self._intercepts.submit(self._intercept, frame.get("id"), asked.intercept, frame.get("request"))
else:
unpatched: Final[PatchFrame] = {"type": "patch", "id": frame.get("id"), "patch": None}
self._send(unpatched)
elif kind == "flush":
_ = self._intercepts.submit(self._flush, frame.get("id"))
else:
raise ProtocolError(f"unknown frame type: {kind!r}")
def serve(self) -> None:
"""Says hello, then answers until the gateway closes the connection."""
threading.Thread(target=self._loop.run_forever, name="callback-loop", daemon=True).start()
self._writer.start()
try:
_ = asyncio.run_coroutine_threadsafe(self._start(), self._loop).result()
self._send(hello(self._subscribers))
while (frame := read_frame(self._sock)) is not None:
self._dispatch(frame)
finally:
# In this order: no new flush can start, so every marker still resolves behind a
# live consumer; then the lanes end; then the writer, once nothing can send again.
self._intercepts.shutdown(wait=True)
_ = asyncio.run_coroutine_threadsafe(self._drain(), self._loop).result()
for lane in self._lanes:
if lane.events is not None:
lane.events.shutdown(wait=True)
self._loop.call_soon_threadsafe(self._loop.stop)
self._outgoing.put(None)
self._writer.join()
@dataclass(frozen=True, slots=True)
class Arguments:
fd: int | None
connect: str | None
load: tuple[str, ...]
def _connect(arguments: Arguments) -> socket.socket:
if arguments.fd is not None:
return socket.socket(fileno=arguments.fd)
if arguments.connect is None:
raise ValueError("one of --fd and --connect is required")
connected: Final = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
connected.connect(arguments.connect)
return connected
def _option(parsed: Mapping[str, object], name: str) -> object:
return parsed.get(name)
def _arguments(argv: Sequence[str] | None) -> Arguments:
parser: Final = argparse.ArgumentParser(prog="python -m litellm.callbacks_v1.host")
where: Final = parser.add_mutually_exclusive_group(required=True)
where.add_argument("--fd", type=int, help="an inherited, connected socket (the gateway spawned this worker)")
where.add_argument("--connect", metavar="PATH", help="a unix socket the gateway listens on (a sidecar)")
parser.add_argument("--load", action="append", metavar="MODULE[:ATTR]", help="a callback to import")
parsed: Final = vars(parser.parse_args(argv))
fd: Final = _option(parsed, "fd")
connect: Final = _option(parsed, "connect")
load_: Final = _option(parsed, "load")
return Arguments(
fd if isinstance(fd, int) else None,
connect if isinstance(connect, str) else None,
tuple(str(item) for item in load_) if isinstance(load_, (list, tuple)) else (), # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # argparse collects `--load` untyped
)
def main(argv: Sequence[str] | None = None) -> int:
arguments: Final = _arguments(argv)
for reference in arguments.load:
load(reference)
with _connect(arguments) as sock:
Worker(sock, snapshot()).serve()
return 0
if __name__ == "__main__":
sys.exit(main())

View file

@ -0,0 +1,53 @@
"""The v1 contract functions the native call reaches for, by name.
`python_contract.json` pins these four on both sides. The typed authoring surface — the
callback protocols, the envelope and patch types, and the registry — is
`litellm.callbacks_v1`; `snapshot` is re-exported from there because the registry has one
home, and the rest here is marshalling no callback author calls.
"""
import json
import uuid
from typing import Final, TypeGuard # noqa: TID251 # recursive JSON validation
from litellm.callbacks_v1 import JSONValue, snapshot
__all__ = ("new_call_id", "project_response", "report", "snapshot")
def _is_json_value(
value: object,
) -> TypeGuard[JSONValue]: # guard-ok: recursively validates the complete JSON value
if value is None or isinstance(value, (bool, int, float, str)):
return True
if isinstance(value, list):
return all(
_is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # list items are validated recursively
for item in value # pyright: ignore[reportUnknownVariableType] # list items are validated recursively
)
if isinstance(value, dict):
return all(
isinstance(key, str) and _is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # mapping entries are validated recursively
for key, item in value.items() # pyright: ignore[reportUnknownVariableType] # mapping entries are validated recursively
)
return False
def project_response(response: object) -> JSONValue:
model_dump: Final = getattr(response, "model_dump", None)
projected: Final = model_dump(mode="json") if callable(model_dump) else response
json.dumps(projected, allow_nan=False)
if _is_json_value(projected):
return projected
raise TypeError(f"unsupported response type: {type(response).__qualname__}")
def new_call_id() -> str:
return str(uuid.uuid4())
def report(name: str, event: str, error: BaseException) -> None:
from litellm._logging import verbose_logger
exception: Final = (type(error), error, error.__traceback__)
verbose_logger.exception("callback %s failed while handling %s", name, event, exc_info=exception)

View file

@ -1,272 +0,0 @@
import inspect
import json
import threading
import uuid
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Final, Literal, Protocol, TypeAlias, TypeGuard # noqa: TID251 # recursive JSON validation
from typing_extensions import ReadOnly, TypeAliasType, TypedDict
JSONScalar: TypeAlias = None | bool | int | float | str
JSONValue = TypeAliasType( # rebind-ok: named recursive alias required for runtime validation
"JSONValue", JSONScalar | Sequence["JSONValue"] | Mapping[str, "JSONValue"]
)
SUPPORTED_SCHEMAS: Final = frozenset({1})
EVENTS: Final = frozenset(
{
"call.started",
"request.sending",
"response.received",
"call.succeeded",
"call.failed",
}
)
class TimingFactsV1(TypedDict):
start_time: ReadOnly[float]
end_time: ReadOnly[float]
duration: ReadOnly[float]
class CallFactsV1(TypedDict):
start_time: ReadOnly[float]
asynchronous: ReadOnly[bool]
metadata: ReadOnly[Mapping[str, JSONValue]]
metadata_dropped: ReadOnly[Sequence[str]]
class RequestFactsV1(TypedDict):
model: ReadOnly[str]
custom_llm_provider: ReadOnly[str]
optional_params: ReadOnly[JSONValue]
url: ReadOnly[str]
headers: ReadOnly[Sequence[tuple[str, str]]]
body: ReadOnly[JSONValue]
ErrorFactsV1 = TypedDict( # rebind-ok: runtime TypedDict type for the reserved class field
"ErrorFactsV1",
{ # mutable-ok: functional TypedDict syntax requires a dict for the reserved class field
"class": ReadOnly[str],
"message": ReadOnly[str],
"status_code": ReadOnly[int | None],
},
)
class CallStartedV1(CallFactsV1):
type: ReadOnly[Literal["call.started"]]
class RequestSendingV1(RequestFactsV1):
type: ReadOnly[Literal["request.sending"]]
class ResponseReceivedV1(TypedDict):
type: ReadOnly[Literal["response.received"]]
body: ReadOnly[str]
class CallSucceededV1(TypedDict):
type: ReadOnly[Literal["call.succeeded"]]
timing: ReadOnly[TimingFactsV1]
streamed: ReadOnly[bool]
response: ReadOnly[JSONValue]
response_error: ReadOnly[str | None]
class CallFailedV1(TypedDict):
type: ReadOnly[Literal["call.failed"]]
timing: ReadOnly[TimingFactsV1]
streamed: ReadOnly[bool]
origin: ReadOnly[Literal["call", "host"]]
error: ReadOnly[ErrorFactsV1]
EventV1: TypeAlias = CallStartedV1 | RequestSendingV1 | ResponseReceivedV1 | CallSucceededV1 | CallFailedV1
class EnvelopeV1(TypedDict):
schema: ReadOnly[Literal[1]]
call_id: ReadOnly[str]
call_type: ReadOnly[str]
seq: ReadOnly[int]
event: ReadOnly[EventV1]
class HeaderPatchV1(TypedDict, total=False):
remove: ReadOnly[Sequence[str]]
set: ReadOnly[Sequence[tuple[str, str]]]
class WirePatchV1(TypedDict, total=False):
headers: ReadOnly[HeaderPatchV1]
body: ReadOnly[JSONValue]
class Callback(Protocol):
name: str
schema: int
events: frozenset[str]
def on_event(self, event: EnvelopeV1) -> None: ...
async def async_on_event(self, event: EnvelopeV1) -> None: ...
def before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ...
async def async_before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ...
class RegistrationError(ValueError):
pass
class UnsupportedSchema(RegistrationError):
pass
class UnknownEvent(RegistrationError):
pass
class MissingHandler(RegistrationError):
pass
class EmptyObserverEvents(RegistrationError):
pass
class HandlerKindMismatch(RegistrationError):
pass
class DuplicateName(RegistrationError):
pass
@dataclass(frozen=True, slots=True)
class Subscriber:
name: str
schema: int
events: frozenset[str]
on_event: object | None
async_on_event: object | None
before_send: object | None
async_before_send: object | None
class _Registry:
def __init__(self) -> None:
self.subscribers: tuple[Subscriber, ...] = ()
self.lock: Final = threading.Lock()
_REGISTRY: Final = _Registry()
def _handler(callback: Callback, name: str) -> object | None:
return getattr(callback, name, None)
def _validate_kind(name: str, handler: object | None, asynchronous: bool) -> None:
if handler is None:
return
if not callable(handler):
raise HandlerKindMismatch(f"{name} must be callable")
is_async: Final = inspect.iscoroutinefunction(handler)
if is_async != asynchronous:
expected: Final = "async" if asynchronous else "sync"
raise HandlerKindMismatch(f"{name} must be {expected}")
def register(callback: Callback) -> None:
schema: Final = callback.schema
if schema not in SUPPORTED_SCHEMAS:
raise UnsupportedSchema(f"unsupported callback schema: {schema}")
events: Final = frozenset(callback.events)
unknown: Final = events - EVENTS
if unknown:
raise UnknownEvent(f"unknown callback events: {sorted(unknown)}")
on_event: Final = _handler(callback, "on_event")
async_on_event: Final = _handler(callback, "async_on_event")
before_send: Final = _handler(callback, "before_send")
async_before_send: Final = _handler(callback, "async_before_send")
handlers: Final = (on_event, async_on_event, before_send, async_before_send)
if all(handler is None for handler in handlers):
raise MissingHandler("callback must define at least one handler")
if (on_event is not None or async_on_event is not None) and not events:
raise EmptyObserverEvents("observer callback events cannot be empty")
_validate_kind("on_event", on_event, False)
_validate_kind("async_on_event", async_on_event, True)
_validate_kind("before_send", before_send, False)
_validate_kind("async_before_send", async_before_send, True)
subscriber: Final = Subscriber(
name=callback.name,
schema=schema,
events=events,
on_event=on_event,
async_on_event=async_on_event,
before_send=before_send,
async_before_send=async_before_send,
)
with _REGISTRY.lock:
if any(existing.name == subscriber.name for existing in _REGISTRY.subscribers):
raise DuplicateName(f"duplicate callback name: {subscriber.name}")
_REGISTRY.subscribers = (*_REGISTRY.subscribers, subscriber)
def unregister(callback: Callback) -> None:
with _REGISTRY.lock:
_REGISTRY.subscribers = tuple(
subscriber for subscriber in _REGISTRY.subscribers if subscriber.name != callback.name
)
def snapshot() -> tuple[Subscriber, ...]:
return _REGISTRY.subscribers
def _is_json_value(
value: object,
) -> TypeGuard[JSONValue]: # guard-ok: recursively validates the complete JSON value
if value is None or isinstance(value, (bool, int, float, str)):
return True
if isinstance(value, list):
return all(
_is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # list items are validated recursively
for item in value # pyright: ignore[reportUnknownVariableType] # list items are validated recursively
)
if isinstance(value, dict):
return all(
isinstance(key, str) and _is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # mapping entries are validated recursively
for key, item in value.items() # pyright: ignore[reportUnknownVariableType] # mapping entries are validated recursively
)
return False
def project_response(response: object) -> JSONValue:
model_dump: Final = getattr(response, "model_dump", None)
projected: Final = model_dump(mode="json") if callable(model_dump) else response
json.dumps(projected, allow_nan=False)
if _is_json_value(projected):
return projected
raise TypeError(f"unsupported response type: {type(response).__qualname__}")
def new_call_id() -> str:
return str(uuid.uuid4())
def report(name: str, event: str, error: BaseException) -> None:
from litellm._logging import verbose_logger
exception: Final = (type(error), error, error.__traceback__)
verbose_logger.exception("callback %s failed while handling %s", name, event, exc_info=exception)

View file

@ -0,0 +1,152 @@
"""What every port's tests share: golden envelopes, a fake transport, and the parity diff.
The parity diff is the migration test. A legacy logger and its port handle the same call,
and every difference between what they send the vendor must be `Allowed` with a reason;
an unexplained one fails. An `Allowed` entry that no longer matches anything fails too,
and so does one citing a `Gap` its port no longer lists, so the allow-list and the gap
ledger shrink together as the contract grows instead of rotting apart.
"""
import json
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass, field
from datetime import datetime, timezone
from fnmatch import fnmatchcase
from pathlib import Path
from typing import Final
from litellm.callbacks_v1 import Callback, EnvelopeV1, EventName
from litellm.callbacks_v1.builtin import anthropic_cache_control, generic_api, openmeter
from litellm.callbacks_v1.builtin.manifest import Gap
from litellm.callbacks_v1.builtin.port import Delivery
from litellm.callbacks_v1.builtin.runtime import Patcher, Sink, Transport
ROOT: Final = Path(__file__).parents[4]
GOLDEN_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1/golden/v1"
FIXED_NOW: Final = datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc)
def golden(kind: EventName) -> EnvelopeV1:
"""The envelope Rust pins for `kind`; a port tested on it cannot drift from the contract.
`tests/test_litellm/test_callbacks_v1.py` is what validates these files against the types."""
return json.loads((GOLDEN_PATH / f"{kind}.json").read_text())
def golden_call(terminal: EventName, **metadata: str) -> tuple[EnvelopeV1, ...]:
"""One call's envelopes in order, ending in `terminal`, with `metadata` on call.started."""
started: Final = golden("call.started")
with_metadata: Final[EnvelopeV1] = {
**started,
"event": {**started["event"], "metadata": {**started["event"]["metadata"], **metadata}}, # pyright: ignore[reportTypedDictNotRequiredAccess, reportGeneralTypeIssues] # the golden call.started event always has metadata
}
return (with_metadata, golden("request.sending"), golden("response.received"), golden(terminal))
@dataclass
class FakeTransport:
deliveries: list[Delivery] = field(default_factory=list)
def send(self, delivery: Delivery) -> None:
self.deliveries.append(delivery)
def bodies(self) -> list[object]:
return [json.loads(delivery.body) for delivery in self.deliveries]
def plain(value: object) -> object:
"""A contract value as plain dicts and lists, so it compares and serialises like JSON."""
if isinstance(value, Mapping):
return {key: plain(item) for key, item in value.items()}
if isinstance(value, Sequence) and not isinstance(value, (str, bytes)):
return [plain(item) for item in value]
return value
# One configured port per manifest entry; `test_conventions` requires one for every entry.
# A port is pure and unnamed, so an example is a value: the host, not the port, is what
# needs a transport and what decides the registration name.
OPENMETER: Final = openmeter.OpenMeter(openmeter.Config(api_key="test-key", endpoint="http://openmeter.test"))
GENERIC_API: Final = generic_api.GenericApi(generic_api.Config(endpoint="http://generic.test/logs", batch_size=1))
ANTHROPIC_CACHE_CONTROL: Final = anthropic_cache_control.AnthropicCacheControl(
anthropic_cache_control.Config(points=(anthropic_cache_control.InjectionPoint(role="system"),))
)
EXAMPLES: Final[Mapping[str, object]] = {
"openmeter": OPENMETER,
"generic_api": GENERIC_API,
"anthropic_cache_control": ANTHROPIC_CACHE_CONTROL,
}
# The subscriber a host builds for each example port, under the manifest's name. Naming and
# wrapping are the host's job, which is why these live here and not in the ports; each call
# is also where the type checker holds a port to its protocol. The clock is fixed so a port
# that stamps a time is comparable run to run, and one that does not is handed it anyway:
# that is the point of the two kinds having one shape.
SUBSCRIBERS: Final[Mapping[str, Callable[[Transport], Callback]]] = {
"openmeter": lambda transport: Sink("openmeter", OPENMETER, transport, lambda: FIXED_NOW),
"generic_api": lambda transport: Sink("generic_api", GENERIC_API, transport, lambda: FIXED_NOW),
"anthropic_cache_control": lambda _transport: Patcher("anthropic_cache_control", ANTHROPIC_CACHE_CONTROL),
}
@dataclass(frozen=True, slots=True)
class Allowed:
"""A difference from the legacy twin that is intended. `path` is an fnmatch pattern over
dotted paths (`data.cost`, `0.metadata.*`; a list index is a segment); `reason` says why.
`gap` is the `Gap.key` this difference exists because of. A difference with no `gap` is
one the port intends to keep; a difference with one goes away when the contract grows."""
path: str
reason: str
gap: str | None = None
def _leaves(value: object, prefix: str = "") -> Mapping[str, object]:
if isinstance(value, Mapping) and value:
return {
path: leaf
for key, item in value.items()
for path, leaf in _leaves(item, f"{prefix}.{key}" if prefix else str(key)).items()
}
if isinstance(value, list) and value:
return {
path: leaf
for index, item in enumerate(value)
for path, leaf in _leaves(item, f"{prefix}.{index}" if prefix else str(index)).items()
}
return {prefix: value}
def unexplained(legacy: object, port: object, allowed: Sequence[Allowed], gaps: Sequence[Gap]) -> tuple[str, ...]:
"""Every difference between two vendor bodies that `allowed` does not cover, plus every
`allowed` entry that covered nothing or cites a gap not in `gaps`. Empty means parity."""
assert all(entry.reason.strip() for entry in allowed), "every Allowed difference needs a reason"
open_keys: Final = {gap.key for gap in gaps}
closed: Final = [entry for entry in allowed if entry.gap is not None and entry.gap not in open_keys]
before: Final = _leaves(plain(legacy))
after: Final = _leaves(plain(port))
differences: Final = {
**{path: f"missing in port: {path} (legacy sent {before[path]!r})" for path in before.keys() - after.keys()},
**{path: f"only in port: {path} = {after[path]!r}" for path in after.keys() - before.keys()},
**{
path: f"differs: {path}: legacy {before[path]!r} != port {after[path]!r}"
for path in before.keys() & after.keys()
if before[path] != after[path]
},
}
covered: Final = {path: [entry for entry in allowed if fnmatchcase(path, entry.path)] for path in differences}
stale: Final = [entry for entry in allowed if not any(entry in entries for entries in covered.values())]
return (
*(message for path, message in sorted(differences.items()) if not covered[path]),
*(f"stale Allowed({entry.path!r}): it no longer matches any difference" for entry in stale),
*(f"Allowed({entry.path!r}) cites gap {entry.gap!r}, which the port does not list" for entry in closed),
)
def assert_parity(legacy: bytes, port: bytes, allowed: Sequence[Allowed], gaps: Sequence[Gap]) -> None:
"""Compares two raw vendor request bodies as JSON and fails with one line per unexplained
difference, so the fix is to port it or to allow it. Two empty bodies are not parity.
`gaps` is the port's own `GAPS`, which every `Allowed.gap` has to be found in."""
assert legacy and port, "nothing to compare: a side sent no body"
problems: Final = unexplained(json.loads(legacy), json.loads(port), allowed, gaps)
assert not problems, f"{len(problems)} unexplained difference(s) from the legacy twin:\n " + "\n ".join(problems)

View file

@ -0,0 +1,81 @@
"""Function-level parity: the port's `marked` against the legacy hook's Messages-route function
on the same bodies. The legacy hook never runs on the native route, so there is no call to share."""
import copy
import pytest
from litellm.callbacks_v1.builtin import anthropic_cache_control as port
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from tests.test_litellm.callbacks_v1.builtin.support import golden, plain
EPHEMERAL = {"type": "ephemeral"}
TURNS = [
{"role": "user", "content": "first"},
{"role": "assistant", "content": [{"type": "text", "text": "second"}]},
{"role": "user", "content": [{"type": "text", "text": "third"}, {"type": "text", "text": "fourth"}]},
]
ALREADY_MARKED = [
{"role": "user", "content": [{"type": "text", "text": "kept", "cache_control": {"type": "ephemeral", "ttl": "1h"}}]}
]
FIVE_USERS = [{"role": "user", "content": f"turn {index}"} for index in range(5)]
CASES = {
"system string": ({"system": "be brief", "messages": TURNS}, [("system", None, None)]),
"system blocks": (
{"system": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}], "messages": TURNS},
[("system", None, None)],
),
"no system to mark": ({"messages": TURNS}, [("system", None, None)]),
"every user turn": ({"messages": TURNS}, [("user", None, None)]),
"last message by index": ({"messages": TURNS}, [(None, -1, None)]),
"index out of range": ({"messages": TURNS}, [(None, 9, None)]),
"ttl": ({"system": "be brief", "messages": TURNS}, [("system", None, "1h"), (None, -1, "5m")]),
"client breakpoint is kept": ({"messages": ALREADY_MARKED}, [("user", None, None)]),
"stops at four breakpoints": (
{"system": "be brief", "messages": FIVE_USERS},
[("system", None, None), ("user", None, None)],
),
}
def legacy(body: dict, points: list[tuple]) -> dict:
injection_points = [
{"location": "message", "role": role, "index": index, "control": {**EPHEMERAL, **({"ttl": ttl} if ttl else {})}}
for role, index, ttl in points
]
messages, system, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
copy.deepcopy(body["messages"]),
copy.deepcopy(body.get("system")),
injection_points, # pyright: ignore[reportArgumentType] # plain dicts stand in for the legacy TypedDicts
)
return {**body, "messages": messages, **({"system": system} if "system" in body else {})}
@pytest.mark.parametrize("name", CASES)
def test_the_port_marks_a_messages_body_exactly_as_the_legacy_hook_does(name: str) -> None:
body, points = CASES[name]
config = port.Config(points=tuple(port.InjectionPoint(role, index, ttl) for role, index, ttl in points))
original = copy.deepcopy(body)
assert plain(port.AnthropicCacheControl(config).marked(body)) == legacy(body, points)
assert body == original
def test_the_patch_covers_only_the_configured_providers() -> None:
request = {**golden("request.sending")["event"], "body": {"system": "be brief", "messages": TURNS}}
config = port.Config(points=(port.InjectionPoint(role="system"),))
interceptor = port.AnthropicCacheControl(config)
assert interceptor.payload(request) is None # pyright: ignore[reportArgumentType] # the golden event plus a Messages body
patch = interceptor.payload({**request, "custom_llm_provider": "anthropic"}) # pyright: ignore[reportArgumentType] # the golden event plus a Messages body
assert patch is not None and set(patch) == {"body"}
assert plain(patch["body"])["system"] == [{"type": "text", "text": "be brief", "cache_control": EPHEMERAL}] # pyright: ignore[reportIndexIssue, reportTypedDictNotRequiredAccess, reportCallIssue, reportArgumentType] # the patch body is the Messages mapping
def test_a_body_that_is_not_a_messages_request_is_left_alone() -> None:
config = port.Config(points=(port.InjectionPoint(role="user"),))
assert port.AnthropicCacheControl(config).marked({"input": "hello"}) is None
assert port.AnthropicCacheControl(port.Config(points=())).marked({"messages": TURNS}) is None

View file

@ -0,0 +1,180 @@
"""The rules every port obeys, checked for every manifest entry. A new port gets these for free."""
import ast
import importlib
import subprocess
import sys
from collections.abc import Generator
from dataclasses import is_dataclass
from pathlib import Path
from typing import Final
import pytest
from litellm import callbacks_v1
from litellm.callbacks_v1.builtin import port as port_vocabulary
from litellm.callbacks_v1.builtin.manifest import PORTS, Entry, Gap, ledger
from litellm.callbacks_v1.builtin.port import InterceptorPort, SinkPort
from litellm.callbacks_v1.builtin.runtime import Sink
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.integrations.custom_logger import CustomLogger
from tests.test_litellm.callbacks_v1.builtin.support import EXAMPLES, SUBSCRIBERS, FakeTransport, golden_call
# The legacy hooks a v1 sink can stand in for. A legacy twin overriding any other
# CustomLogger hook does something the v1 contract has no place for.
SINK_HOOKS: Final = frozenset(
{
"log_pre_api_call",
"log_post_api_call",
"log_success_event",
"log_failure_event",
"async_log_success_event",
"async_log_failure_event",
"async_log_pre_api_call",
}
)
PUBLIC_HOOKS: Final = frozenset(
name
for name, value in vars(CustomLogger).items()
if callable(value) and (name.startswith(("log_", "async_")) or name.endswith("_hook"))
)
# What a port may import. A port is values and pure methods, the shape a sandboxed custom
# callback is confined to; anything that can reach a socket, a thread, the environment or the
# rest of litellm belongs to `runtime.py`, which a port may not import at all. A new pure
# stdlib module is a reviewed line here.
PURE_MODULES: Final = frozenset(
{
"base64",
"collections.abc",
"dataclasses",
"datetime",
"hashlib",
"json",
"math",
"re",
"types",
"typing",
"typing_extensions",
"urllib.parse",
"litellm.callbacks_v1",
"litellm.callbacks_v1.builtin.port",
}
)
# What a port never carries. `runtime.py` names it and wraps it into a subscriber, and
# `manifest.py` holds everything the migration needs to say about it.
NOT_A_PORTS: Final = frozenset(
{"NAME", "GAPS", "events", "on_event", "async_on_event", "before_send", "async_before_send"}
)
pytestmark = pytest.mark.parametrize("entry", PORTS, ids=[entry.name for entry in PORTS])
@pytest.fixture(autouse=True)
def clean_registry() -> Generator[None]:
yield
for subscriber in callbacks_v1.snapshot():
callbacks_v1.unregister(subscriber)
def _legacy(entry: Entry) -> type:
module, _, attribute = entry.legacy.partition(":")
return getattr(importlib.import_module(module), attribute)
def test_importing_a_port_registers_nothing(entry: Entry) -> None:
script: Final = (
f"import {entry.module.__name__}; from litellm import callbacks_v1; "
"import sys; sys.exit(len(callbacks_v1.snapshot()))"
)
assert subprocess.run([sys.executable, "-c", script], check=False).returncode == 0
def _impure_imports(source: str) -> tuple[str, ...]:
nodes: Final = tuple(ast.walk(ast.parse(source)))
plain: Final = tuple(alias.name for node in nodes if isinstance(node, ast.Import) for alias in node.names)
froms: Final = tuple(
str(node.module)
for node in nodes
if isinstance(node, ast.ImportFrom) and (node.level or node.module not in PURE_MODULES)
)
return (*(name for name in plain if name not in PURE_MODULES), *froms)
def test_a_port_imports_nothing_that_can_act(entry: Entry) -> None:
for module in (entry.module, port_vocabulary):
source = Path(str(module.__file__)).read_text() # rebind-ok: one source per checked module
assert _impure_imports(source) == (), f"{module.__name__} imports outside the pure allow-list"
def test_a_port_is_a_frozen_class_over_a_frozen_config(entry: Entry) -> None:
example: Final = EXAMPLES[entry.name]
assert all(isinstance(gap, Gap) for gap in entry.gaps)
assert is_dataclass(example) and type(example).__dataclass_params__.frozen
assert is_dataclass(entry.module.Config) and entry.module.Config.__dataclass_params__.frozen
assert "Legacy twin:" in (entry.module.__doc__ or "")
def test_a_port_declares_the_protocol_of_its_kind_and_nothing_of_the_contract(entry: Entry) -> None:
"""A port names its protocol as a base, so the type checker holds it to the shape at the
class. This is the same claim at runtime, for a port whose module a type checker never saw."""
declared: Final = type(EXAMPLES[entry.name]).__mro__
expected: Final = SinkPort if entry.kind == "sink" else InterceptorPort
assert expected in declared, f"{entry.name} is a {entry.kind}; it should declare {expected.__name__} as its base"
assert (SinkPort in declared) == (entry.kind == "sink")
example: Final = EXAMPLES[entry.name]
carried: Final = {member for member in NOT_A_PORTS if hasattr(example, member)}
assert not carried, f"{entry.name} carries {sorted(carried)}; that is runtime.py's or manifest.py's"
def test_a_port_registers_under_its_manifest_name_with_the_handlers_of_its_kind(entry: Entry) -> None:
callbacks_v1.register(SUBSCRIBERS[entry.name](FakeTransport()))
(subscriber,) = callbacks_v1.snapshot()
assert subscriber.name == entry.name
observes: Final = subscriber.on_event is not None or subscriber.async_on_event is not None
intercepts: Final = subscriber.before_send is not None or subscriber.async_before_send is not None
assert (observes, intercepts) == ((True, False) if entry.kind == "sink" else (False, True))
# A sink's handler only enqueues, so the sync one serves async calls too.
assert subscriber.async_on_event is None
def test_a_sink_handler_does_no_io_and_keeps_no_finished_call(entry: Entry) -> None:
if entry.kind != "sink":
pytest.skip("interceptors have no outbox")
transport: Final = FakeTransport()
callback: Final = SUBSCRIBERS[entry.name](transport)
assert isinstance(callback, Sink), "a sink port supplies pure methods; `runtime.Sink` is its only subscriber"
for envelope in golden_call("call.succeeded", user_api_key_user_id="user-1"):
callback.on_event(envelope)
assert callback.outbox.flush()
assert len(transport.deliveries) == 1
assert callback.open_calls() == 0
def test_the_legacy_twin_resolves_and_a_sink_twin_is_only_a_sink(entry: Entry) -> None:
legacy: Final = _legacy(entry)
if entry.kind != "sink":
return
overridden: Final = {
name
for cls in legacy.__mro__
if cls not in (CustomLogger, CustomBatchLogger, object)
for name in vars(cls)
if name in PUBLIC_HOOKS
}
assert overridden <= SINK_HOOKS, f"{entry.legacy} also overrides {sorted(overridden - SINK_HOOKS)}"
def test_a_port_names_each_gap_once_by_what_the_contract_lacks(entry: Entry) -> None:
keys: Final = tuple(gap.key for gap in entry.gaps)
assert len(keys) == len(frozenset(keys)), f"{entry.name} lists a gap key twice: {sorted(keys)}"
assert all(key and key == key.lower() and key.replace("_", "").isalnum() for key in keys)
def test_ready_means_no_open_gap(entry: Entry) -> None:
assert entry.status != "ready" or entry.gaps == ()
assert entry.name in ledger()

View file

@ -0,0 +1,101 @@
import json
from datetime import datetime, timezone
import pytest
from litellm.callbacks_v1.builtin import generic_api
from litellm.callbacks_v1.builtin.port import CallRecord
from litellm.callbacks_v1.builtin.runtime import CallJoin, Sink
from tests.test_litellm.callbacks_v1.builtin.support import golden_call
PORT = generic_api.GenericApi(generic_api.Config(endpoint="http://generic.test"))
FAILURES_ONLY = generic_api.GenericApi(
generic_api.Config(endpoint="http://generic.test", event_types=frozenset({"llm_api_failure"}))
)
# A sink is handed the clock whether or not it stamps a time; this one does not.
UNUSED_NOW = datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc)
COMMON = {
"litellm_call_id": "call-123",
"call_type": "ocr",
"startTime": 10.0,
"model": "model",
"custom_llm_provider": "provider",
"model_parameters": {"temperature": 0.2},
}
def joined(terminal: str) -> CallRecord:
join = CallJoin()
records = [join.accept(envelope) for envelope in golden_call(terminal)] # pyright: ignore[reportArgumentType] # terminal is an EventName literal at every call site
(record,) = [record for record in records if record is not None]
return record
def test_a_succeeded_call_uses_the_standard_logging_payload_names() -> None:
assert PORT.payload(joined("call.succeeded"), UNUSED_NOW) == {
**COMMON,
"id": "response",
"stream": False,
"status": "success",
"endTime": 12.5,
"response_time": 2.5,
"response": {"id": "response"},
}
def test_a_failed_call_carries_error_information_instead_of_a_response() -> None:
assert PORT.payload(joined("call.failed"), UNUSED_NOW) == {
**COMMON,
"id": "call-123",
"stream": True,
"status": "failure",
"endTime": 11.0,
"response_time": 1.0,
"error_str": "bad request",
"error_information": {"error_class": "ValueError", "error_message": "bad request", "error_code": "400"},
}
@pytest.mark.parametrize(
("log_format", "bodies"),
[
("json_array", ['[{"id": "a"}, {"id": "b"}]']),
("ndjson", ['{"id": "a"}\n{"id": "b"}']),
("single", ['{"id": "a"}', '{"id": "b"}']),
],
)
def test_a_batch_is_framed_by_log_format(log_format: generic_api.LogFormat, bodies: list[str]) -> None:
port = generic_api.GenericApi(generic_api.Config(endpoint="http://generic.test", log_format=log_format))
batch = ({"id": "a"}, {"id": "b"})
sent = port.deliveries(batch) # pyright: ignore[reportArgumentType] # framing does not depend on the record's fields
assert [delivery.body.decode() for delivery in sent] == bodies
assert {delivery.url for delivery in sent} == {"http://generic.test"}
def test_an_outcome_left_out_of_event_types_has_no_record() -> None:
assert FAILURES_ONLY.payload(joined("call.succeeded"), UNUSED_NOW) is None
assert FAILURES_ONLY.payload(joined("call.failed"), UNUSED_NOW) is not None
def test_event_types_filter_what_is_queued() -> None:
from tests.test_litellm.callbacks_v1.builtin.support import FakeTransport
transport = FakeTransport()
sink = Sink("generic_api", FAILURES_ONLY, transport)
for terminal in ("call.succeeded", "call.failed"):
for envelope in golden_call(terminal):
sink.on_event(envelope)
assert sink.outbox.flush()
assert [record["status"] for body in transport.bodies() for record in body] == ["failure"] # pyright: ignore[reportGeneralTypeIssues, reportIndexIssue] # json bodies are lists of records here
def test_config_parses_the_legacy_header_variable() -> None:
config = generic_api.Config.from_env(
{"GENERIC_LOGGER_ENDPOINT": "http://e", "GENERIC_LOGGER_HEADERS": "a=1, b = 2"}
)
assert config.headers == (("Content-Type", "application/json"), ("a", "1"), ("b", "2"))
assert json.loads(json.dumps(config.headers)) == [["Content-Type", "application/json"], ["a", "1"], ["b", "2"]]

View file

@ -0,0 +1,54 @@
import pytest
from litellm.callbacks_v1.builtin import openmeter
from litellm.callbacks_v1.builtin.port import CallRecord
from litellm.callbacks_v1.builtin.runtime import CallJoin
from tests.test_litellm.callbacks_v1.builtin.support import FIXED_NOW, golden_call
PORT = openmeter.OpenMeter(openmeter.Config(api_key="test-key", endpoint="http://openmeter.test/"))
def joined(terminal: str, **metadata: str) -> CallRecord:
join = CallJoin()
records = [join.accept(envelope) for envelope in golden_call(terminal, **metadata)] # pyright: ignore[reportArgumentType] # terminal is an EventName literal at every call site
(record,) = [record for record in records if record is not None]
return record
def test_a_succeeded_call_becomes_one_cloud_event() -> None:
assert PORT.payload(joined("call.succeeded", user_api_key_user_id="user-1"), FIXED_NOW) == {
"specversion": "1.0",
"type": "litellm_tokens",
"id": "response",
"time": "2026-01-02T03:04:05+00:00",
"subject": "user-1",
"source": "litellm-proxy",
"data": {"model": "model", "cost": None},
}
def test_a_failed_call_is_not_metered() -> None:
assert PORT.payload(joined("call.failed", user_api_key_user_id="user-1"), FIXED_NOW) is None
def test_a_call_without_a_subject_is_an_error_like_the_legacy_logger() -> None:
with pytest.raises(ValueError, match="user is required"):
PORT.payload(joined("call.succeeded"), FIXED_NOW)
def test_each_event_is_its_own_authorised_request() -> None:
(delivery,) = PORT.deliveries(({"id": "a"},)) # pyright: ignore[reportArgumentType] # framing does not depend on the event's fields
assert delivery.url == "http://openmeter.test/api/v1/events"
assert dict(delivery.headers) == {
"Content-Type": "application/cloudevents+json",
"Authorization": "Bearer test-key",
}
assert delivery.body == b'{"id": "a"}'
def test_config_reads_the_legacy_environment_variables() -> None:
config = openmeter.Config.from_env({"OPENMETER_API_KEY": "k", "OPENMETER_API_ENDPOINT": "http://host/"})
assert (config.api_key, config.url, config.event_type) == ("k", "http://host/api/v1/events", "litellm_tokens")
with pytest.raises(ValueError, match="OPENMETER_API_KEY"):
openmeter.Config.from_env({})

View file

@ -0,0 +1,94 @@
import threading
import time
from litellm.callbacks_v1.builtin.outbox import Outbox
from litellm.callbacks_v1.builtin.runtime import CallJoin
from tests.test_litellm.callbacks_v1.builtin.support import golden, golden_call
def test_put_returns_before_a_slow_delivery_finishes() -> None:
release = threading.Event()
delivered: list[tuple[int, ...]] = []
def deliver(batch: tuple[int, ...]) -> None:
release.wait(2)
delivered.append(batch)
outbox = Outbox("slow", deliver)
started = time.monotonic()
outbox.put(1)
assert time.monotonic() - started < 0.5
assert delivered == []
release.set()
assert outbox.flush()
assert delivered == [(1,)]
def test_items_are_delivered_batch_size_at_a_time_and_flush_sends_the_rest() -> None:
delivered: list[tuple[int, ...]] = []
outbox = Outbox("batched", delivered.append, batch_size=2, flush_interval=60)
for item in (1, 2, 3):
outbox.put(item)
assert outbox.flush()
assert delivered == [(1, 2), (3,)]
def test_a_failed_delivery_is_dropped_and_the_next_one_still_goes_out() -> None:
delivered: list[tuple[int, ...]] = []
def deliver(batch: tuple[int, ...]) -> None:
if batch == (1,):
raise RuntimeError("vendor is down")
delivered.append(batch)
outbox = Outbox("flaky", deliver)
outbox.put(1)
outbox.put(2)
assert outbox.flush()
assert delivered == [(2,)]
def test_a_full_outbox_drops_the_new_item_instead_of_blocking_the_call() -> None:
release = threading.Event()
outbox = Outbox("full", lambda batch: release.wait(2) and None, capacity=1)
outbox.put(1)
time.sleep(0.05)
outbox.put(2)
started = time.monotonic()
outbox.put(3)
assert time.monotonic() - started < 0.5
release.set()
assert outbox.flush()
def test_the_join_returns_one_record_at_the_terminal_envelope_and_forgets_the_call() -> None:
join = CallJoin()
started, request, received, terminal = golden_call("call.succeeded")
assert [join.accept(envelope) for envelope in (started, request, received)] == [None, None, None]
record = join.accept(terminal)
assert record is not None
assert (record.call_id, record.call_type) == ("call-123", "ocr")
assert (record.started, record.request, record.terminal) == (started["event"], request["event"], terminal["event"])
assert join.open_calls() == 0
def test_a_call_that_failed_before_sending_has_no_request() -> None:
record = CallJoin().accept(golden("call.failed"))
assert record is not None and record.request is None and record.started is None
def test_calls_that_never_finish_are_bounded() -> None:
join = CallJoin(capacity=2)
started = golden("call.started")
for call_id in ("a", "b", "c"):
join.accept({**started, "call_id": call_id})
assert join.open_calls() == 2

View file

@ -0,0 +1,34 @@
"""The parity diff itself: what it reports is what keeps the allow-lists and the gap ledger honest."""
from types import MappingProxyType
from typing import Final
from litellm.callbacks_v1.builtin.manifest import Gap
from tests.test_litellm.callbacks_v1.builtin.support import Allowed, unexplained
COST: Final = Gap("fact", "cost", "kwargs['response_cost']", "no cost fact")
LEGACY: Final = MappingProxyType({"id": "call-1", "data": MappingProxyType({"cost": 0.25, "model": "claude"})})
PORT: Final = MappingProxyType({"id": "call-1", "data": MappingProxyType({"cost": None, "model": "claude"})})
def test_identical_bodies_need_no_allowance() -> None:
assert unexplained(LEGACY, LEGACY, (), ()) == ()
def test_a_difference_nobody_allowed_is_reported_by_path() -> None:
assert unexplained(LEGACY, PORT, (), ()) == ("differs: data.cost: legacy 0.25 != port None",)
def test_a_difference_allowed_by_an_open_gap_is_parity() -> None:
assert unexplained(LEGACY, PORT, (Allowed("data.cost", "no cost fact", gap="cost"),), (COST,)) == ()
def test_an_allowance_outlives_neither_its_difference_nor_its_gap() -> None:
allowed: Final = (Allowed("data.cost", "no cost fact", gap="cost"),)
assert unexplained(LEGACY, LEGACY, allowed, (COST,)) == (
"stale Allowed('data.cost'): it no longer matches any difference",
)
assert unexplained(LEGACY, PORT, allowed, ()) == (
"Allowed('data.cost') cites gap 'cost', which the port does not list",
)

View file

@ -0,0 +1,299 @@
"""The extension host worker, driven over a socketpair the way a gateway would drive it."""
import asyncio
import os
import socket
import subprocess
import sys
import threading
from collections.abc import Generator
from pathlib import Path
from types import MappingProxyType
import pytest
from litellm import callbacks_v1
from litellm.callbacks_v1 import EnvelopeV1, RequestFactsV1, WirePatchV1
from litellm.callbacks_v1.host import INTERCEPT_THREADS, ProtocolError, Worker, encode_frame, read_frame
from tests.test_litellm.callbacks_v1.builtin.support import golden
@pytest.fixture(autouse=True)
def clean_registry() -> Generator[None]:
yield
for subscriber in callbacks_v1.snapshot():
callbacks_v1.unregister(subscriber)
class Gateway:
"""The gateway's end of the socket: starts a worker on a thread and speaks frames to it."""
def __init__(self) -> None:
self.sock, worker_sock = socket.socketpair()
self.sock.settimeout(5)
self.thread = threading.Thread(target=Worker(worker_sock, callbacks_v1.snapshot()).serve, daemon=True)
self.thread.start()
self.hello = self.read()
def send(self, **frame: object) -> None:
self.sock.sendall(encode_frame(frame))
def read(self) -> dict:
frame = read_frame(self.sock)
assert frame is not None
return dict(frame)
def flush(self) -> list[dict]:
"""Everything the worker said before it confirmed the flush."""
self.send(type="flush", id="f")
frames = []
while (frame := self.read()) != {"type": "flushed", "id": "f"}:
frames.append(frame)
return frames
def close(self) -> None:
self.sock.close()
self.thread.join(5)
assert not self.thread.is_alive()
def test_hello_reports_every_subscription_as_data() -> None:
@callbacks_v1.on_event("call.succeeded", "call.failed", name="recorder")
def record(event: EnvelopeV1) -> None: ...
@callbacks_v1.before_send
async def patch(request: RequestFactsV1) -> WirePatchV1 | None: ...
gateway = Gateway()
assert gateway.hello == {
"type": "hello",
"schema": 1,
"subscriptions": [
{"name": "recorder", "events": ["call.failed", "call.succeeded"], "observes": True, "intercepts": False},
{"name": f"{__name__}.{patch.__qualname__}", "events": [], "observes": False, "intercepts": True},
],
}
gateway.close()
def test_one_subscriber_gets_its_envelopes_in_the_order_they_were_sent() -> None:
seen: list[int] = []
@callbacks_v1.on_event("response.received", name="ordered")
async def record(event: EnvelopeV1) -> None:
seen.append(event["seq"])
gateway = Gateway()
for seq in range(50):
gateway.send(type="event", subscriber=0, envelope={**golden("response.received"), "seq": seq})
assert gateway.flush() == []
assert seen == list(range(50))
gateway.close()
def test_an_observer_that_raises_is_reported_and_the_next_envelope_still_arrives() -> None:
seen: list[str] = []
@callbacks_v1.on_event("call.started", "call.failed", name="flaky")
def record(event: EnvelopeV1) -> None:
seen.append(event["event"]["type"])
if event["event"]["type"] == "call.started":
raise ValueError("boom")
gateway = Gateway()
gateway.send(type="event", subscriber=0, envelope=golden("call.started"))
gateway.send(type="event", subscriber=0, envelope=golden("call.failed"))
assert gateway.flush() == [{"type": "report", "subscriber": 0, "event": "call.started", "message": "boom"}]
assert seen == ["call.started", "call.failed"]
gateway.close()
def test_an_interceptor_answers_with_its_patch_even_from_a_read_only_view() -> None:
@callbacks_v1.before_send
def patch(request: RequestFactsV1) -> WirePatchV1 | None:
return {"body": MappingProxyType({"model": request["model"], "blocks": ({"a": 1},)})}
gateway = Gateway()
gateway.send(type="intercept", id=7, subscriber=0, request=golden("request.sending")["event"])
assert gateway.read() == {"type": "patch", "id": 7, "patch": {"body": {"model": "model", "blocks": [{"a": 1}]}}}
gateway.close()
def test_an_interceptor_that_raises_is_an_error_for_the_gateway_to_act_on() -> None:
@callbacks_v1.before_send
async def patch(request: RequestFactsV1) -> WirePatchV1 | None:
raise PermissionError("not this model")
gateway = Gateway()
gateway.send(type="intercept", id="a", subscriber=0, request=golden("request.sending")["event"])
assert gateway.read() == {
"type": "error",
"id": "a",
"error_class": "builtins.PermissionError",
"message": "not this model",
}
gateway.close()
def test_an_interception_does_not_wait_behind_a_slow_observer() -> None:
release = threading.Event()
@callbacks_v1.on_event("call.started", name="slow")
def slow(event: EnvelopeV1) -> None:
release.wait(5)
@callbacks_v1.before_send
def patch(request: RequestFactsV1) -> WirePatchV1 | None:
return None
gateway = Gateway()
gateway.send(type="event", subscriber=0, envelope=golden("call.started"))
gateway.send(type="intercept", id=1, subscriber=1, request=golden("request.sending")["event"])
assert gateway.read() == {"type": "patch", "id": 1, "patch": None}
release.set()
gateway.close()
def test_every_async_interception_is_in_flight_at_once() -> None:
"""Concurrency is the loop's, not a pool size: an async handler that only waits holds no
thread, so far more interceptions run together than there are intercept threads. Awaiting
each other is how they prove it — none of them can finish until all of them have started."""
burst = INTERCEPT_THREADS * 4
started = 0
everyone: asyncio.Event | None = None
@callbacks_v1.before_send
async def patch(request: RequestFactsV1) -> WirePatchV1 | None:
nonlocal started, everyone
if everyone is None: # every handler runs on the one loop, so this races with nothing
everyone = asyncio.Event()
started += 1
if started == burst:
everyone.set()
await asyncio.wait_for(everyone.wait(), 3)
return None
gateway = Gateway()
for frame_id in range(burst):
gateway.send(type="intercept", id=frame_id, subscriber=0, request=golden("request.sending")["event"])
answers = sorted((gateway.read() for _ in range(burst)), key=lambda frame: frame["id"])
assert answers == [{"type": "patch", "id": frame_id, "patch": None} for frame_id in range(burst)]
gateway.close()
def test_a_frame_the_protocol_does_not_know_stops_the_worker() -> None:
sock, worker_sock = socket.socketpair()
worker = Worker(worker_sock, ())
sock.sendall(encode_frame({"type": "shell", "command": "id"}))
with pytest.raises(ProtocolError, match="unknown frame type"):
worker.serve()
sock.close()
CALLBACK_MODULE = """
import os
from litellm.callbacks_v1 import before_send
@before_send
def leak(request):
return {"body": {"env": sorted(os.environ), "cwd_files": sorted(os.listdir("."))}}
"""
def test_a_spawned_worker_sees_only_the_environment_it_was_given(tmp_path: Path) -> None:
modules = tmp_path / "modules"
modules.mkdir()
(modules / "leaky.py").write_text(CALLBACK_MODULE)
empty = tmp_path / "empty"
empty.mkdir()
gateway_sock, worker_sock = socket.socketpair()
gateway_sock.settimeout(60)
repo = Path(__file__).parents[3]
given = {"PYTHONPATH": os.pathsep.join((str(repo), str(modules))), "LITELLM_MODE": "PRODUCTION"}
process = subprocess.Popen(
[sys.executable, "-m", "litellm.callbacks_v1.host", "--fd", str(worker_sock.fileno()), "--load", "leaky"],
pass_fds=(worker_sock.fileno(),),
env=given,
cwd=empty,
)
worker_sock.close()
try:
hello = read_frame(gateway_sock)
assert hello is not None and [sub["name"] for sub in hello["subscriptions"]] == ["leaky.leak"] # pyright: ignore[reportIndexIssue, reportGeneralTypeIssues] # the hello frame's subscriptions are a list of objects
gateway_sock.sendall(encode_frame({"type": "intercept", "id": 1, "subscriber": 0, "request": {}}))
answer = read_frame(gateway_sock)
finally:
gateway_sock.close()
assert process.wait(30) == 0
assert answer is not None
leaked = answer["patch"]["body"] # pyright: ignore[reportIndexIssue, reportOptionalSubscript] # the patch frame carries the callback's body
assert set(leaked["env"]) - {"LC_CTYPE", "__CF_USER_TEXT_ENCODING"} <= set(given)
assert leaked["cwd_files"] == []
GOLDEN_FRAMES = Path(__file__).parents[3] / "litellm-rust/crates/callbacks-v1-exthost/golden"
def golden_frame(name: str) -> dict:
import json
return json.loads((GOLDEN_FRAMES / name).read_text())
def test_the_worker_answers_the_golden_frames_rust_sends_with_the_golden_frames_rust_parses() -> None:
"""Both sides are pinned to one directory: Rust serialises `to_worker` and parses
`from_worker`; here the worker is fed the first and must produce the shape of the second."""
seen: list[object] = []
@callbacks_v1.on_event("call.started", name="recorder")
def record(event: EnvelopeV1) -> None:
seen.append(event)
raise ValueError("boom")
@callbacks_v1.before_send
def patch(request: RequestFactsV1) -> WirePatchV1 | None:
seen.append(request)
return {"headers": {"remove": ["X-Trace"], "set": [("X-Team", "core")]}, "body": {"input": "patched"}}
event, intercept, flush = (golden_frame(f"to_worker/{name}.json") for name in ("event", "intercept", "flush"))
gateway = Gateway()
assert set(gateway.hello) == set(golden_frame("from_worker/hello.json"))
assert {key for sub in gateway.hello["subscriptions"] for key in sub} == {
key for sub in golden_frame("from_worker/hello.json")["subscriptions"] for key in sub
}
gateway.send(**intercept)
assert gateway.read() == golden_frame("from_worker/patch.json")
gateway.send(**event)
gateway.send(**flush)
assert gateway.read() == golden_frame("from_worker/report.json")
assert gateway.read() == golden_frame("from_worker/flushed.json")
assert seen == [intercept["request"], event["envelope"]]
gateway.close()
def test_the_authoring_surface_and_the_worker_import_nothing_but_the_standard_library() -> None:
"""A worker that could run without the rest of litellm stays possible only while these two
files need nothing from it. `typing_extensions` is the one third-party import."""
import ast
allowed = set(sys.stdlib_module_names) | {"typing_extensions", "litellm.callbacks_v1"}
for path in ("litellm/callbacks_v1/__init__.py", "litellm/callbacks_v1/host.py"):
tree = ast.parse((Path(__file__).parents[3] / path).read_text())
imported = {
alias.name if isinstance(node, ast.Import) else str(node.module)
for node in ast.walk(tree)
if isinstance(node, (ast.Import, ast.ImportFrom))
for alias in node.names
}
assert {name for name in imported if name not in allowed and name.split(".")[0] not in allowed} == set(), path

View file

@ -10,8 +10,8 @@ from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.rust_bridge import callbacks_legacy_python as callbacks
from litellm.rust_bridge.callbacks_legacy_python import check_limits, setup
from litellm.rust_bridge import callbacks_legacy as callbacks
from litellm.rust_bridge.callbacks_legacy import check_limits, setup
_OCR_KWARGS: Final = MappingProxyType(
{

View file

@ -0,0 +1,31 @@
import inspect
from pathlib import Path
from typing import Final
import pytest
from pydantic import BaseModel, TypeAdapter
from litellm.rust_bridge import callbacks_v1 as callbacks
ROOT: Final = Path(__file__).parents[3]
CONTRACT_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1-python/python_contract.json"
def test_the_rust_contract_matches_the_shim_signatures() -> None:
contract: Final = TypeAdapter(dict[str, list[str]]).validate_json(CONTRACT_PATH.read_text())
assert contract == {name: list(inspect.signature(getattr(callbacks, name)).parameters) for name in contract}
class ResponseModel(BaseModel):
value: int
def test_project_response_supports_pydantic_and_json_values() -> None:
assert callbacks.project_response(ResponseModel(value=3)) == {"value": 3}
original: Final = {"nested": [1, True, None]}
assert callbacks.project_response(original) is original
def test_project_response_rejects_unsupported_objects() -> None:
with pytest.raises(TypeError):
callbacks.project_response(object())

View file

@ -1,178 +0,0 @@
import inspect
import warnings
from collections.abc import Generator
from pathlib import Path
from typing import Final, cast
import pytest
from pydantic import BaseModel, TypeAdapter
from litellm.rust_bridge import callbacks_v1_python as callbacks
ROOT: Final = Path(__file__).parents[3]
CONTRACT_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1-python/python_contract.json"
GOLDEN_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1/golden/v1"
def _clear_registry() -> None:
for subscriber in callbacks.snapshot():
callbacks.unregister(cast(callbacks.Callback, subscriber))
@pytest.fixture(autouse=True)
def clear_registry() -> Generator[None, None, None]:
_clear_registry()
yield
_clear_registry()
def test_the_rust_contract_matches_the_shim_signatures() -> None:
contract: Final = TypeAdapter(dict[str, list[str]]).validate_json(CONTRACT_PATH.read_text())
assert contract == {name: list(inspect.signature(getattr(callbacks, name)).parameters) for name in contract}
def test_v1_golden_envelopes_match_the_typed_contract() -> None:
with warnings.catch_warnings():
warnings.filterwarnings("ignore", message="Items .*ReadOnly.*", category=UserWarning)
adapter: Final = TypeAdapter(callbacks.EnvelopeV1)
envelopes: Final = [adapter.validate_json(path.read_text(), strict=True) for path in GOLDEN_PATH.glob("*.json")]
assert {envelope["event"]["type"] for envelope in envelopes} == callbacks.EVENTS
assert {envelope["schema"] for envelope in envelopes} == callbacks.SUPPORTED_SCHEMAS
class ValidCallback:
name: str = "valid"
schema: int = 1
events: frozenset[str] = frozenset({"call.started"})
def on_event(self, event: callbacks.EnvelopeV1) -> None:
del event
@pytest.mark.parametrize(
("callback", "error"),
[
(
type(
"Unsupported",
(),
{"name": "unsupported", "schema": 2, "events": frozenset(), "before_send": lambda *_: None},
)(),
callbacks.UnsupportedSchema,
),
(
type(
"Unknown",
(),
{"name": "unknown", "schema": 1, "events": frozenset({"new.event"}), "on_event": lambda *_: None},
)(),
callbacks.UnknownEvent,
),
(
type("Missing", (), {"name": "missing", "schema": 1, "events": frozenset()})(),
callbacks.MissingHandler,
),
(
type("Empty", (), {"name": "empty", "schema": 1, "events": frozenset(), "on_event": lambda *_: None})(),
callbacks.EmptyObserverEvents,
),
(
type(
"AsyncFunction",
(),
{
"name": "async-function",
"schema": 1,
"events": frozenset({"call.started"}),
"async_on_event": lambda *_: None,
},
)(),
callbacks.HandlerKindMismatch,
),
],
ids=["schema", "event", "missing", "empty-events", "async-kind"],
)
def test_register_rejects_invalid_callbacks(callback: object, error: type[ValueError]) -> None:
with pytest.raises(error):
callbacks.register(callback) # pyright: ignore[reportArgumentType] # invalid callback shapes are the subject of the test
class WrongSyncKind:
name: str = "wrong-sync-kind"
schema: int = 1
events: frozenset[str] = frozenset({"call.started"})
async def on_event(self, event: callbacks.EnvelopeV1) -> None:
del event
def test_register_rejects_coroutine_in_sync_slot() -> None:
with pytest.raises(callbacks.HandlerKindMismatch):
callbacks.register(cast(callbacks.Callback, WrongSyncKind()))
class AsyncObserver:
name: str = "async-observer"
schema: int = 1
events: frozenset[str] = frozenset({"call.started"})
async def async_on_event(self, event: callbacks.EnvelopeV1) -> None:
del event
class SyncInterceptor:
name: str = "sync-interceptor"
schema: int = 1
events: frozenset[str] = frozenset()
def before_send(self, request: callbacks.RequestFactsV1) -> callbacks.WirePatchV1 | None:
del request
return None
class AsyncInterceptor:
name: str = "async-interceptor"
schema: int = 1
events: frozenset[str] = frozenset()
async def async_before_send(self, request: callbacks.RequestFactsV1) -> callbacks.WirePatchV1 | None:
del request
return None
@pytest.mark.parametrize("callback", [ValidCallback(), AsyncObserver(), SyncInterceptor(), AsyncInterceptor()])
def test_register_accepts_each_handler_family(callback: callbacks.Callback) -> None:
callbacks.register(callback)
assert len(callbacks.snapshot()) == 1
def test_register_rejects_duplicate_names() -> None:
callbacks.register(cast(callbacks.Callback, ValidCallback()))
with pytest.raises(callbacks.DuplicateName):
callbacks.register(cast(callbacks.Callback, ValidCallback()))
def test_snapshot_is_stable_across_later_registry_writes() -> None:
first: Final = ValidCallback()
second: Final = AsyncObserver()
callbacks.register(cast(callbacks.Callback, first))
captured: Final = callbacks.snapshot()
callbacks.register(cast(callbacks.Callback, second))
callbacks.unregister(cast(callbacks.Callback, first))
assert callbacks.snapshot() is not captured
assert tuple(subscriber.name for subscriber in captured) == ("valid",)
class ResponseModel(BaseModel):
value: int
def test_project_response_supports_pydantic_and_json_values() -> None:
assert callbacks.project_response(ResponseModel(value=3)) == {"value": 3}
original: Final = {"nested": [1, True, None]}
assert callbacks.project_response(original) is original
def test_project_response_rejects_unsupported_objects() -> None:
with pytest.raises(TypeError):
callbacks.project_response(object())

View file

@ -0,0 +1,236 @@
import warnings
from collections.abc import Generator
from pathlib import Path
from typing import Final, get_args
import pytest
from pydantic import TypeAdapter
from litellm import callbacks_v1
from litellm.callbacks_v1 import EnvelopeV1, EventName, RequestFactsV1, SchemaVersion, WirePatchV1
ROOT: Final = Path(__file__).parents[2]
GOLDEN_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1/golden/v1"
def _clear_registry() -> None:
for subscriber in callbacks_v1.snapshot():
callbacks_v1.unregister(subscriber)
@pytest.fixture(autouse=True)
def clear_registry() -> Generator[None, None, None]:
_clear_registry()
yield
_clear_registry()
def test_the_event_names_and_schemas_are_the_literal_aliases() -> None:
assert callbacks_v1.EVENTS == frozenset(get_args(EventName))
assert callbacks_v1.SUPPORTED_SCHEMAS == frozenset(get_args(SchemaVersion))
def test_v1_golden_envelopes_match_the_typed_contract() -> None:
with warnings.catch_warnings():
warnings.filterwarnings("ignore", message="Items .*ReadOnly.*", category=UserWarning)
adapter: Final = TypeAdapter(callbacks_v1.EnvelopeV1)
envelopes: Final = [adapter.validate_json(path.read_text(), strict=True) for path in GOLDEN_PATH.glob("*.json")]
assert {envelope["event"]["type"] for envelope in envelopes} == callbacks_v1.EVENTS
assert {envelope["schema"] for envelope in envelopes} == callbacks_v1.SUPPORTED_SCHEMAS
class SyncObserver:
name: str = "valid"
events: frozenset[EventName] = frozenset({"call.started"})
def on_event(self, event: callbacks_v1.EnvelopeV1) -> None:
del event
class AsyncObserver:
name: str = "async-observer"
events: frozenset[EventName] = frozenset({"call.started"})
async def async_on_event(self, event: callbacks_v1.EnvelopeV1) -> None:
del event
class SyncInterceptor:
name: str = "sync-interceptor"
events: frozenset[EventName] = frozenset()
def before_send(self, request: callbacks_v1.RequestFactsV1) -> callbacks_v1.WirePatchV1 | None:
del request
return None
class AsyncInterceptor:
name: str = "async-interceptor"
events: frozenset[EventName] = frozenset()
async def async_before_send(self, request: callbacks_v1.RequestFactsV1) -> callbacks_v1.WirePatchV1 | None:
del request
return None
class WrongSyncKind:
name: str = "wrong-sync-kind"
events: frozenset[EventName] = frozenset({"call.started"})
async def on_event(self, event: callbacks_v1.EnvelopeV1) -> None:
del event
@pytest.mark.parametrize(
("callback", "error"),
[
(
type(
"Unsupported",
(),
{"name": "unsupported", "schema": 2, "events": frozenset(), "before_send": lambda *_: None},
)(),
callbacks_v1.UnsupportedSchema,
),
(
type(
"Unknown",
(),
{"name": "unknown", "schema": 1, "events": frozenset({"new.event"}), "on_event": lambda *_: None},
)(),
callbacks_v1.UnknownEvent,
),
(
type("Missing", (), {"name": "missing", "schema": 1, "events": frozenset()})(),
callbacks_v1.MissingHandler,
),
(
type("Empty", (), {"name": "empty", "schema": 1, "events": frozenset(), "on_event": lambda *_: None})(),
callbacks_v1.EmptyObserverEvents,
),
(
type(
"AsyncFunction",
(),
{
"name": "async-function",
"schema": 1,
"events": frozenset({"call.started"}),
"async_on_event": lambda *_: None,
},
)(),
callbacks_v1.HandlerKindMismatch,
),
(WrongSyncKind(), callbacks_v1.HandlerKindMismatch),
],
ids=["schema", "event", "missing", "empty-events", "async-kind", "coroutine-in-sync-slot"],
)
def test_register_rejects_invalid_callbacks(callback: object, error: type[ValueError]) -> None:
with pytest.raises(error):
callbacks_v1.register(callback) # pyright: ignore[reportArgumentType] # invalid callback shapes are the subject of the test
@pytest.mark.parametrize("callback", [SyncObserver(), AsyncObserver(), SyncInterceptor(), AsyncInterceptor()])
def test_register_accepts_each_handler_family(callback: callbacks_v1.Callback) -> None:
callbacks_v1.register(callback)
assert len(callbacks_v1.snapshot()) == 1
def test_register_rejects_duplicate_names() -> None:
callbacks_v1.register(SyncObserver())
with pytest.raises(callbacks_v1.DuplicateName):
callbacks_v1.register(SyncObserver())
def test_snapshot_is_stable_across_later_registry_writes() -> None:
first: Final = SyncObserver()
second: Final = AsyncObserver()
callbacks_v1.register(first)
captured: Final = callbacks_v1.snapshot()
callbacks_v1.register(second)
callbacks_v1.unregister(first)
assert callbacks_v1.snapshot() is not captured
assert tuple(subscriber.name for subscriber in captured) == ("valid",)
def test_register_defaults_the_schema_when_the_callback_declares_none() -> None:
callbacks_v1.register(SyncObserver())
assert callbacks_v1.snapshot()[0].schema == callbacks_v1.SCHEMA
def test_register_still_rejects_a_declared_foreign_schema() -> None:
class Foreign(SyncObserver):
name: str = "foreign"
schema: int = 2
with pytest.raises(callbacks_v1.UnsupportedSchema):
callbacks_v1.register(Foreign())
def test_on_event_registers_a_sync_function_in_the_sync_slot() -> None:
@callbacks_v1.on_event("call.started", "call.succeeded")
def record(event: EnvelopeV1) -> None:
del event
subscriber: Final = callbacks_v1.snapshot()[0]
assert subscriber.on_event is record
assert subscriber.async_on_event is None
assert subscriber.events == frozenset({"call.started", "call.succeeded"})
assert subscriber.name.endswith("record")
assert record( # the function is returned unchanged, so it stays directly callable
{"schema": 1, "call_id": "c", "call_type": "ocr", "seq": 0, "event": {"type": "response.received", "body": ""}}
) is None
def test_on_event_routes_a_coroutine_function_to_the_async_slot() -> None:
@callbacks_v1.on_event("call.failed", name="named-observer")
async def record(event: EnvelopeV1) -> None:
del event
subscriber: Final = callbacks_v1.snapshot()[0]
assert subscriber.async_on_event is record
assert subscriber.on_event is None
assert subscriber.name == "named-observer"
def test_before_send_routes_by_kind_and_subscribes_to_no_events() -> None:
@callbacks_v1.before_send
def sync_patch(request: RequestFactsV1) -> WirePatchV1 | None:
del request
return None
@callbacks_v1.before_send
async def async_patch(request: RequestFactsV1) -> WirePatchV1 | None:
del request
return None
first, second = callbacks_v1.snapshot()
assert (first.before_send, first.async_before_send) == (sync_patch, None)
assert (second.before_send, second.async_before_send) == (None, async_patch)
assert first.events == second.events == frozenset()
def test_on_event_rejects_an_unknown_event_at_decoration() -> None:
with pytest.raises(callbacks_v1.UnknownEvent):
@callbacks_v1.on_event("call.startd") # pyright: ignore[reportArgumentType] # the typo is the subject of the test
def record(event: EnvelopeV1) -> None:
del event
def test_on_event_rejects_an_empty_event_set_at_decoration() -> None:
with pytest.raises(callbacks_v1.EmptyObserverEvents):
@callbacks_v1.on_event()
def record(event: EnvelopeV1) -> None:
del event
def test_unregister_accepts_a_bare_name() -> None:
@callbacks_v1.on_event("call.started", name="by-name")
def record(event: EnvelopeV1) -> None:
del event
assert len(callbacks_v1.snapshot()) == 1
callbacks_v1.unregister("by-name")
assert callbacks_v1.snapshot() == ()

View file

@ -0,0 +1,215 @@
"""Differential parity: one native call reaches a legacy logger and its v1 port, each posting
to its own recording vendor, and every difference between the two vendor bodies is `Allowed`.
The native chain runs the legacy contract and then v1 on the same call, so nothing is
replayed or mocked: both sides see the one real call. Registration happens in fixtures only.
"""
from collections.abc import Generator
from datetime import datetime, timezone
from typing import Final
import pytest
import litellm
from litellm import callbacks_v1
from litellm.callbacks_v1.builtin import anthropic_cache_control, generic_api, openmeter
from litellm.callbacks_v1.builtin.manifest import entry
from litellm.callbacks_v1.builtin.runtime import HttpxTransport, Patcher, Sink
from litellm.integrations.generic_api.generic_api_callback import GenericAPILogger
from litellm.integrations.openmeter import OpenMeterLogger
from tests.test_litellm.callbacks_v1.builtin.support import Allowed, assert_parity
from tests.test_litellm_rust.support.callback_recorder import drain_logging
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec, recording_service
from tests.test_litellm_rust.support.requests import MESSAGES, MESSAGES_MODEL, MESSAGES_RESPONSE
pytestmark = pytest.mark.requires_rust_extension
METADATA: Final = {"user_api_key_user_id": "user-1"}
@pytest.fixture
def provider(recording_server: RecordingServer) -> RecordingServer:
recording_server.default_response = ResponseSpec(body=MESSAGES_RESPONSE)
return recording_server
@pytest.fixture
def legacy_vendor() -> Generator[RecordingServer]:
with recording_service() as server:
yield server
@pytest.fixture
def port_vendor() -> Generator[RecordingServer]:
with recording_service() as server:
yield server
@pytest.fixture(autouse=True)
def clean_registry() -> Generator[None]:
yield
for subscriber in callbacks_v1.snapshot():
callbacks_v1.unregister(subscriber)
def arguments(server: RecordingServer, **kwargs: object) -> dict[str, object]:
# Identity rides in `metadata` here because call.started projects only that keyword; the proxy
# sends it as `litellm_metadata` on this route, which v1 does not see yet (see entry("openmeter").gaps).
return {
"model": MESSAGES_MODEL,
"messages": [dict(message) for message in MESSAGES],
"max_tokens": 64,
"api_key": "test-key",
"api_base": server.base_url,
"metadata": METADATA,
**kwargs,
}
OPENMETER_ALLOWED: Final = (
Allowed("time", "each side stamps its own clock"),
Allowed("data.cost", "no cost fact on call.succeeded", gap="cost"),
Allowed("data.*_tokens", "no usage fact; legacy counts tokens off the OpenAI-normalised response", gap="usage"),
)
@pytest.mark.asyncio
async def test_openmeter_meters_a_native_call_like_its_legacy_twin(
provider: RecordingServer,
legacy_vendor: RecordingServer,
port_vendor: RecordingServer,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("OPENMETER_API_KEY", "test-key")
monkeypatch.setenv("OPENMETER_API_ENDPOINT", legacy_vendor.base_url)
port: Final = openmeter.OpenMeter(openmeter.Config(api_key="test-key", endpoint=port_vendor.base_url))
sink: Final = Sink("openmeter", port, HttpxTransport(), lambda: datetime.now(timezone.utc))
callbacks_v1.register(sink)
await litellm.anthropic.messages.acreate(**arguments(provider, callbacks=[OpenMeterLogger()]))
await drain_logging()
assert sink.outbox.flush()
(legacy_request,) = legacy_vendor.requests
(port_request,) = port_vendor.requests
assert port_request.path == legacy_request.path == "/api/v1/events"
assert port_request.headers["authorization"] == legacy_request.headers["authorization"]
assert port_request.headers["content-type"] == legacy_request.headers["content-type"]
assert_parity(legacy_request.raw_body, port_request.raw_body, OPENMETER_ALLOWED, entry("openmeter").gaps)
# What the legacy StandardLoggingPayload has and v1 facts cannot fill yet, on any outcome.
GENERIC_API_GAP_ALLOWED: Final = (
Allowed("0.stream", "legacy leaves stream null on a non-streaming call, v1 always says false"),
Allowed("0.completionStartTime", "no first-token time in timing", gap="first_token_time"),
Allowed("0.response_cost*", "no cost facts", gap="cost"),
Allowed("0.cost_breakdown*", "no cost facts", gap="cost"),
Allowed("0.saved_cache_cost", "no cost facts", gap="cost"),
Allowed("0.autorouter_savings", "no cost facts", gap="cost"),
Allowed("0.*_tokens", "no usage fact", gap="usage"),
Allowed("0.cache_*", "no cache fact", gap="cache"),
Allowed("0.model_id", "no routing fact", gap="routing"),
Allowed("0.model_group", "no routing fact", gap="routing"),
Allowed("0.model_map_information.*", "no routing fact", gap="routing"),
Allowed("0.api_base", "no routing fact", gap="routing"),
Allowed("0.metadata.*", "no identity fact", gap="identity"),
Allowed("0.end_user", "no identity fact", gap="identity"),
Allowed("0.requester_ip_address", "no identity fact", gap="identity"),
Allowed("0.user_agent", "no identity fact", gap="identity"),
Allowed("0.request_tags", "no tags fact", gap="tags"),
Allowed("0.request_model_access_groups", "no tags fact", gap="tags"),
Allowed("0.trace_id", "no correlation beyond call_id", gap="correlation"),
Allowed("0.session_id", "no correlation beyond call_id", gap="correlation"),
Allowed("0.hidden_params.*", "no equivalent", gap="no_equivalent"),
Allowed("0.guardrail_information", "no equivalent", gap="no_equivalent"),
Allowed("0.standard_built_in_tools_params.*", "no equivalent", gap="no_equivalent"),
Allowed("0.status_fields.*", "derived from guardrail_information, which v1 has no fact for", gap="no_equivalent"),
Allowed("0.messages.*", "no route-neutral prompt projection", gap="prompt"),
Allowed(
"0.model_parameters.metadata.*",
"this test sends identity as `metadata`, which Anthropic also accepts; legacy keeps it out of model_parameters",
),
)
GENERIC_API_ALLOWED: Final = (
*GENERIC_API_GAP_ALLOWED,
Allowed(
"0.response.*",
"legacy logs the OpenAI-normalised response, v1 carries the route's own",
gap="normalised_response",
),
Allowed("0.error_str", "legacy sends null on success, the port omits the key"),
Allowed("0.error_information.*", "legacy sends an empty shell on success, the port omits the key"),
)
@pytest.mark.asyncio
async def test_generic_api_logs_a_native_call_like_its_legacy_twin(
provider: RecordingServer, legacy_vendor: RecordingServer, port_vendor: RecordingServer
) -> None:
legacy: Final = GenericAPILogger(endpoint=legacy_vendor.base_url, batch_size=1)
port: Final = generic_api.GenericApi(generic_api.Config(endpoint=port_vendor.base_url, batch_size=1))
sink: Final = Sink("generic_api", port, HttpxTransport())
callbacks_v1.register(sink)
await litellm.anthropic.messages.acreate(**arguments(provider, callbacks=[legacy]))
await drain_logging()
assert sink.outbox.flush()
(legacy_request,) = legacy_vendor.requests
(port_request,) = port_vendor.requests
assert port_request.headers["content-type"] == legacy_request.headers["content-type"]
assert_parity(legacy_request.raw_body, port_request.raw_body, GENERIC_API_ALLOWED, entry("generic_api").gaps)
GENERIC_API_FAILURE_ALLOWED: Final = (
*GENERIC_API_GAP_ALLOWED,
Allowed("0.response", "legacy sends an empty object on failure, the port omits the key"),
Allowed("0.error_information.error_budget_*", "ErrorFacts is class/message/status only", gap="error_detail"),
Allowed("0.error_information.error_rate_limit_*", "ErrorFacts is class/message/status only", gap="error_detail"),
Allowed(
"0.error_information.error_provider_request_id", "ErrorFacts is class/message/status only", gap="error_detail"
),
Allowed("0.error_information.llm_provider", "ErrorFacts is class/message/status only", gap="error_detail"),
Allowed("0.error_information.traceback", "ErrorFacts is class/message/status only", gap="error_detail"),
)
@pytest.mark.asyncio
async def test_generic_api_logs_a_failed_native_call_like_its_legacy_twin(
provider: RecordingServer, legacy_vendor: RecordingServer, port_vendor: RecordingServer
) -> None:
provider.enqueue(
ResponseSpec(body={"type": "error", "error": {"type": "invalid_request_error", "message": "bad"}}, status=400)
)
legacy: Final = GenericAPILogger(endpoint=legacy_vendor.base_url, batch_size=1)
port: Final = generic_api.GenericApi(generic_api.Config(endpoint=port_vendor.base_url, batch_size=1))
sink: Final = Sink("generic_api", port, HttpxTransport())
callbacks_v1.register(sink)
with pytest.raises(litellm.BadRequestError):
await litellm.anthropic.messages.acreate(**arguments(provider, callbacks=[legacy]))
await drain_logging()
assert sink.outbox.flush()
(legacy_request,) = legacy_vendor.requests
(port_request,) = port_vendor.requests
assert_parity(
legacy_request.raw_body, port_request.raw_body, GENERIC_API_FAILURE_ALLOWED, entry("generic_api").gaps
)
def test_the_cache_control_patch_survives_the_contract_and_reaches_the_provider(provider: RecordingServer) -> None:
config: Final = anthropic_cache_control.Config(
points=(anthropic_cache_control.InjectionPoint(role="system"), anthropic_cache_control.InjectionPoint(index=-1))
)
callbacks_v1.register(Patcher("anthropic_cache_control", anthropic_cache_control.AnthropicCacheControl(config)))
litellm.anthropic.messages.create(**arguments(provider, system="be brief"))
sent: Final = provider.requests[-1].body
assert isinstance(sent, dict)
assert sent["system"] == [{"type": "text", "text": "be brief", "cache_control": {"type": "ephemeral"}}]
assert sent["messages"][-1]["content"][-1]["cache_control"] == {"type": "ephemeral"}
assert sent["model"] == MESSAGES_MODEL.split("/", 1)[1]

View file

@ -0,0 +1,146 @@
from collections.abc import AsyncIterator, Generator
from typing import Final
import pytest
import litellm
from litellm import callbacks_v1
from litellm.callbacks_v1 import EVENTS, EnvelopeV1, EventName, RequestFactsV1, WirePatchV1
from litellm.integrations.custom_logger import CustomLogger
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
from tests.test_litellm_rust.support.requests import (
MESSAGES,
MESSAGES_EVENTS,
MESSAGES_MODEL,
MESSAGES_RESPONSE,
)
pytestmark = pytest.mark.requires_rust_extension
STREAM: Final = ResponseSpec(body=None, events=MESSAGES_EVENTS)
@pytest.fixture
def messages_server(recording_server: RecordingServer) -> RecordingServer:
recording_server.default_response = ResponseSpec(body=MESSAGES_RESPONSE)
return recording_server
def arguments(server: RecordingServer, **kwargs: object) -> dict[str, object]:
return {
"model": MESSAGES_MODEL,
"messages": [dict(message) for message in MESSAGES],
"max_tokens": 64,
"api_key": "test-key",
"api_base": server.base_url,
**kwargs,
}
class BothObserver:
"""Declares both kinds; an async call must select the async one and never the sync one."""
name: Final = "both-observer"
schema: Final = 1
events: Final = EVENTS
def __init__(self) -> None:
self.envelopes: Final[list[EnvelopeV1]] = []
self.sync_calls: Final[list[EnvelopeV1]] = []
def on_event(self, event: EnvelopeV1) -> None:
self.sync_calls.append(event)
async def async_on_event(self, event: EnvelopeV1) -> None:
self.envelopes.append(event)
class AsyncInterceptor:
name: Final = "async-interceptor"
schema: Final = 1
events: Final[frozenset[EventName]] = frozenset()
def __init__(self) -> None:
self.requests: Final[list[RequestFactsV1]] = []
async def async_before_send(self, request: RequestFactsV1) -> WirePatchV1 | None:
self.requests.append(request)
return {"headers": {"set": [("x-v1-async-interceptor", "seen")]}}
@pytest.fixture
def observer() -> Generator[BothObserver]:
subscriber: Final = BothObserver()
callbacks_v1.register(subscriber)
try:
yield subscriber
finally:
callbacks_v1.unregister(subscriber)
@pytest.fixture
def interceptor() -> Generator[AsyncInterceptor]:
subscriber: Final = AsyncInterceptor()
callbacks_v1.register(subscriber)
try:
yield subscriber
finally:
callbacks_v1.unregister(subscriber)
class LegacyCallId(CustomLogger):
def __init__(self) -> None:
self.call_ids: Final[list[str]] = []
def log_pre_api_call(self, model, messages, kwargs):
self.call_ids.append(kwargs["litellm_call_id"])
@pytest.mark.asyncio
async def test_native_messages_async_call_selects_the_async_observer_and_shares_the_legacy_call_id(
messages_server: RecordingServer, observer: BothObserver
) -> None:
legacy: Final = LegacyCallId()
await litellm.anthropic.messages.acreate(**arguments(messages_server, callbacks=[legacy]))
assert observer.sync_calls == []
assert [envelope["event"]["type"] for envelope in observer.envelopes] == [
"call.started",
"request.sending",
"response.received",
"call.succeeded",
]
assert [envelope["seq"] for envelope in observer.envelopes] == [0, 1, 2, 3]
assert {envelope["call_type"] for envelope in observer.envelopes} == {"anthropic_messages"}
assert len(legacy.call_ids) == 1
assert {envelope["call_id"] for envelope in observer.envelopes} == set(legacy.call_ids)
@pytest.mark.asyncio
async def test_native_messages_async_interceptor_patch_reaches_the_provider(
messages_server: RecordingServer, interceptor: AsyncInterceptor
) -> None:
await litellm.anthropic.messages.acreate(**arguments(messages_server))
assert len(interceptor.requests) == 1
request: Final = interceptor.requests[0]
assert request["custom_llm_provider"] == "anthropic"
assert request["headers"] != []
assert messages_server.requests[-1].headers["x-v1-async-interceptor"] == "seen"
@pytest.mark.asyncio
async def test_native_messages_stream_marks_the_terminal_envelope_streamed(
messages_server: RecordingServer, observer: BothObserver
) -> None:
messages_server.enqueue(STREAM)
stream: Final = await litellm.anthropic.messages.acreate(**arguments(messages_server, stream=True))
assert isinstance(stream, AsyncIterator)
delivered: Final = [chunk async for chunk in stream]
assert delivered
terminal: Final = observer.envelopes[-1]
assert terminal["event"]["type"] == "call.succeeded"
assert terminal["event"]["streamed"] is True

View file

@ -3,9 +3,9 @@ from typing import Final
import pytest
from litellm import callbacks_v1
from litellm.callbacks_v1 import EVENTS, EnvelopeV1, EventName, RequestFactsV1, WirePatchV1
from litellm.integrations.custom_logger import CustomLogger
from litellm.rust_bridge import callbacks_v1_python
from litellm.rust_bridge.callbacks_v1_python import EVENTS, EnvelopeV1, RequestFactsV1, WirePatchV1
from tests.test_litellm_rust.support.callback_recorder import drain_logging
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
from tests.test_litellm_rust.support.requests import (
@ -39,7 +39,7 @@ class Observer:
class Interceptor:
name: Final = "interceptor"
schema: Final = 1
events: Final[frozenset[str]] = frozenset()
events: Final[frozenset[EventName]] = frozenset()
def __init__(self) -> None:
self.bodies: Final[list[object]] = []
@ -52,21 +52,21 @@ class Interceptor:
@pytest.fixture
def observer() -> Generator[Observer]:
subscriber: Final = Observer()
callbacks_v1_python.register(subscriber)
callbacks_v1.register(subscriber)
try:
yield subscriber
finally:
callbacks_v1_python.unregister(subscriber)
callbacks_v1.unregister(subscriber)
@pytest.fixture
def interceptor() -> Generator[Interceptor]:
subscriber: Final = Interceptor()
callbacks_v1_python.register(subscriber)
callbacks_v1.register(subscriber)
try:
yield subscriber
finally:
callbacks_v1_python.unregister(subscriber)
callbacks_v1.unregister(subscriber)
class LegacyCallId(CustomLogger):
@ -139,3 +139,28 @@ def test_native_ocr_failure_reaches_both_contracts_once(
]
assert [envelope["event"]["type"] for envelope in terminal] == ["call.failed"]
assert observer.envelopes[-1] is terminal[0]
def test_native_ocr_decorated_function_callbacks_run_through_the_native_call(
ocr_server: RecordingServer,
) -> None:
seen: Final[list[str]] = []
@callbacks_v1.on_event("call.started", "call.succeeded", name="decorated-observer")
def record(event: EnvelopeV1) -> None:
seen.append(event["event"]["type"])
@callbacks_v1.before_send
def patch(request: RequestFactsV1) -> WirePatchV1 | None:
del request
return {"headers": {"set": [("x-v1-decorated", "seen")]}}
try:
call_native_ocr(ocr_server)
finally:
callbacks_v1.unregister("decorated-observer")
callbacks_v1.unregister(f"{patch.__module__}.{patch.__qualname__}")
assert seen == ["call.started", "call.succeeded"]
assert ocr_server.requests[-1].headers["x-v1-decorated"] == "seen"
assert callbacks_v1.snapshot() == ()