mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
wip
This commit is contained in:
parent
57aa3f2895
commit
e0ead49d96
63 changed files with 5269 additions and 632 deletions
11
litellm-rust/Cargo.lock
generated
11
litellm-rust/Cargo.lock
generated
|
|
@ -2057,6 +2057,17 @@ dependencies = [
|
|||
"thiserror 2.0.19",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-callbacks-v1-exthost"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"litellm-callbacks-v1",
|
||||
"litellm-host",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-callbacks-v1-python"
|
||||
version = "0.1.0"
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ litellm-host = { path = "crates/host" }
|
|||
litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
|
||||
litellm-callbacks-v1 = { path = "crates/callbacks-v1" }
|
||||
litellm-callbacks-v1-python = { path = "crates/callbacks-v1-python" }
|
||||
litellm-callbacks-v1-exthost = { path = "crates/callbacks-v1-exthost" }
|
||||
litellm-framing = { path = "crates/framer" }
|
||||
litellm-auth = { path = "crates/auth" }
|
||||
litellm-auth-aws = { path = "crates/auth-aws" }
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
//! The Python contract of this crate: every function of
|
||||
//! `litellm/rust_bridge/callbacks_legacy_python.py` that Rust calls, by name.
|
||||
//! `litellm/rust_bridge/callbacks_legacy.py` that Rust calls, by name.
|
||||
|
||||
use pyo3::prelude::*;
|
||||
use strum::{IntoStaticStr, VariantArray};
|
||||
|
||||
const MODULE: &str = "litellm.rust_bridge.callbacks_legacy_python";
|
||||
const MODULE: &str = "litellm.rust_bridge.callbacks_legacy";
|
||||
|
||||
/// Every litellm Python internal the native call still borrows, grouped by the subsystem it
|
||||
/// belongs to. Rust drives the call; these exist only so behaviour that Python owns today
|
||||
|
|
@ -12,7 +12,7 @@ const MODULE: &str = "litellm.rust_bridge.callbacks_legacy_python";
|
|||
/// A group is deleted once Rust owns that subsystem, so this enum only shrinks. Calling a
|
||||
/// user's own callback is not borrowing and does not belong here.
|
||||
///
|
||||
/// `litellm/rust_bridge/callbacks_legacy_python.py` is the only Python module behind it, and
|
||||
/// `litellm/rust_bridge/callbacks_legacy.py` is the only Python module behind it, and
|
||||
/// `python_contract.json` pins each function's parameters on both sides.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum LegacyPython {
|
||||
|
|
|
|||
|
|
@ -5,12 +5,12 @@ use pyo3::types::{PyDict, PyTuple};
|
|||
|
||||
use crate::{LegacyPythonLifecycle, LegacyPythonSurface, PublicCall};
|
||||
|
||||
/// The parameters of every `callbacks_legacy_python` function, as the real module declares them.
|
||||
/// `tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py` pins this file to the Python
|
||||
/// The parameters of every `callbacks.legacy` function, as the real module declares them.
|
||||
/// `tests/test_litellm/rust_bridge/test_callbacks_legacy.py` pins this file to the Python
|
||||
/// signatures, and [`namespace`] binds every fake call against it.
|
||||
pub(crate) const PYTHON_CONTRACT: &str = include_str!("../python_contract.json");
|
||||
|
||||
/// Stand-ins for `callbacks_legacy_python`, the only Python module the crate calls. Tests
|
||||
/// Stand-ins for `callbacks.legacy`, the only Python module the crate calls. Tests
|
||||
/// share one interpreter and run concurrently, so each fake is installed idempotently and
|
||||
/// forwards to the per-test `StubLogger` it is handed (directly, or as `kwargs['logger']`).
|
||||
/// Every fake is bound against the contract first, so a call the real module would reject
|
||||
|
|
@ -23,10 +23,10 @@ import sys
|
|||
import traceback
|
||||
import types
|
||||
|
||||
for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_legacy_python'):
|
||||
for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_legacy'):
|
||||
sys.modules.setdefault(name, types.ModuleType(name))
|
||||
|
||||
legacy = sys.modules['litellm.rust_bridge.callbacks_legacy_python']
|
||||
legacy = sys.modules['litellm.rust_bridge.callbacks_legacy']
|
||||
CONTRACT = json.loads(python_contract)
|
||||
|
||||
|
||||
|
|
|
|||
18
litellm-rust/crates/callbacks-v1-exthost/AGENTS.md
Normal file
18
litellm-rust/crates/callbacks-v1-exthost/AGENTS.md
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
- Target invariants, not completion claims
|
||||
- Keep this crate the out-of-process host of `litellm-callbacks-v1` and nothing else: the wire protocol to an extension host worker and the client that speaks it
|
||||
- The protocol of a call (which envelope, to whom, how patches fold, patch validation) belongs to `litellm-callbacks-v1`'s `CallSession`; this crate sends what the session says and brings answers back to it. It never decides protocol and never validates a patch itself
|
||||
- No PyO3 and no interpreter in this crate or in the process that uses it. The worker is `python -m litellm.callbacks_v1.host` (`litellm/callbacks_v1/host.py`), which loads callbacks with the SDK's own `register` and decorators; a callback file is the same file in the SDK and behind this host
|
||||
- `callbacks-v1-python` is the sibling adapter for the SDK (in process, inline). The two share the session and nothing else
|
||||
- Terms (contract terms are in `callbacks-v1/AGENTS.md`)
|
||||
- Worker: the process that runs callbacks. Gateway: the process that owns calls and this crate
|
||||
- Frame: a 4-byte big-endian length and one JSON object tagged by `type`. `ToWorker` is `event`, `intercept`, `flush`; `FromWorker` is `hello`, `patch`, `error`, `report`, `flushed`
|
||||
- Hello: the worker's first frame, its `Subscription`s as data; `subscriber` in every later frame is an index into it
|
||||
- Report: an observer's swallowed failure, told to the gateway for logging only
|
||||
- The worker only answers. There is no frame by which it calls into the gateway; a host service (sending vendor HTTP, a secret by name) would be a new frame and a contract capability, never an escape hatch
|
||||
- Everything from a worker is untrusted: frames are size-capped before they are read, unknown frame types and unknown fields are errors, and a patch is data until `Interception::patched` validates it
|
||||
- This crate isolates nothing. What a worker can reach (environment, files, user, network) is what the process it runs in was given; the spawner clears the environment and picks the working directory, and the deployment (a sidecar, a sandbox runtime, another host) supplies the rest
|
||||
- Observers are asynchronous to the call: `observe` writes and returns. Only `intercept` waits, and only it may carry a timeout or a failure policy
|
||||
- An interceptor that raises is `ExtHostError::Interceptor`; whether that fails the call or skips the patch is the gateway's policy per subscriber, not the worker's
|
||||
- `golden/` pins every frame for both sides: Rust serialises `to_worker` and parses `from_worker`; `tests/test_litellm/callbacks_v1/test_host.py` feeds the worker the first and expects the second. The frames embed the contract's own golden envelope and request, so the two golden directories cannot drift
|
||||
- `Worker` is the blocking reference client: what a conforming gateway does, in the simplest form that can be tested end to end. The production host (async, supervised, a bounded queue per subscriber, restart, timeouts) replaces the client and keeps the protocol
|
||||
- `tests/worker.rs` is end to end against a real Python worker and is `#[ignore]`d because it needs an interpreter that can import `litellm`; run it with `PYO3_PYTHON=... cargo test -p litellm-callbacks-v1-exthost -- --ignored`
|
||||
15
litellm-rust/crates/callbacks-v1-exthost/Cargo.toml
Normal file
15
litellm-rust/crates/callbacks-v1-exthost/Cargo.toml
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
[package]
|
||||
name = "litellm-callbacks-v1-exthost"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
litellm-callbacks-v1.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
litellm-host.workspace = true
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
{
|
||||
"type": "error",
|
||||
"id": 1,
|
||||
"error_class": "builtins.PermissionError",
|
||||
"message": "not this model"
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"type": "flushed",
|
||||
"id": 2
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"type": "hello",
|
||||
"schema": 1,
|
||||
"subscriptions": [
|
||||
{
|
||||
"name": "recorder",
|
||||
"events": [
|
||||
"call.failed",
|
||||
"call.succeeded"
|
||||
],
|
||||
"observes": true,
|
||||
"intercepts": false
|
||||
},
|
||||
{
|
||||
"name": "my_pkg.cb.patch",
|
||||
"events": [],
|
||||
"observes": false,
|
||||
"intercepts": true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
{
|
||||
"type": "patch",
|
||||
"id": 1,
|
||||
"patch": {
|
||||
"headers": {
|
||||
"remove": [
|
||||
"X-Trace"
|
||||
],
|
||||
"set": [
|
||||
[
|
||||
"X-Team",
|
||||
"core"
|
||||
]
|
||||
]
|
||||
},
|
||||
"body": {
|
||||
"input": "patched"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"type": "patch",
|
||||
"id": 1,
|
||||
"patch": null
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
{
|
||||
"type": "report",
|
||||
"subscriber": 0,
|
||||
"event": "call.started",
|
||||
"message": "boom"
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"type": "event",
|
||||
"subscriber": 0,
|
||||
"envelope": {
|
||||
"schema": 1,
|
||||
"call_id": "call-123",
|
||||
"call_type": "ocr",
|
||||
"seq": 0,
|
||||
"event": {
|
||||
"type": "call.started",
|
||||
"start_time": 10.0,
|
||||
"asynchronous": true,
|
||||
"metadata": {
|
||||
"team": "core"
|
||||
},
|
||||
"metadata_dropped": [
|
||||
"broken"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"type": "flush",
|
||||
"id": 2
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
{
|
||||
"type": "intercept",
|
||||
"id": 1,
|
||||
"subscriber": 1,
|
||||
"request": {
|
||||
"model": "model",
|
||||
"custom_llm_provider": "provider",
|
||||
"optional_params": {
|
||||
"temperature": 0.2
|
||||
},
|
||||
"url": "https://provider.example/v1",
|
||||
"headers": [
|
||||
[
|
||||
"Authorization",
|
||||
"[REDACTED]"
|
||||
]
|
||||
],
|
||||
"body": {
|
||||
"input": "hello"
|
||||
}
|
||||
}
|
||||
}
|
||||
15
litellm-rust/crates/callbacks-v1-exthost/src/lib.rs
Normal file
15
litellm-rust/crates/callbacks-v1-exthost/src/lib.rs
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
//! The out-of-process host of the v1 callback contract: the gateway's side of the socket to
|
||||
//! an extension host worker (`python -m litellm.callbacks_v1.host`). The gateway never loads
|
||||
//! a callback; the worker reports its subscriptions, and from then on this crate sends it
|
||||
//! what a [`CallSession`](litellm_callbacks_v1::CallSession) says to deliver and brings
|
||||
//! back interceptor patches for the session to validate.
|
||||
//!
|
||||
//! [`protocol`] is the wire, pinned by `golden/` on both sides. [`Worker`] is the blocking
|
||||
//! reference client: what a conforming gateway does, in the simplest form that can be
|
||||
//! tested end to end. The production host (async, supervised, queued) reuses the protocol
|
||||
//! and replaces the client.
|
||||
|
||||
pub mod protocol;
|
||||
mod worker;
|
||||
|
||||
pub use worker::{ExtHostError, Report, Worker};
|
||||
254
litellm-rust/crates/callbacks-v1-exthost/src/protocol.rs
Normal file
254
litellm-rust/crates/callbacks-v1-exthost/src/protocol.rs
Normal file
|
|
@ -0,0 +1,254 @@
|
|||
//! Frames: a 4-byte big-endian length, then that many bytes of JSON, one object tagged by
|
||||
//! `type`. The worker only answers; there is no frame by which it calls into the gateway.
|
||||
//! Everything a worker sends is untrusted: sizes are capped, unknown frames and fields are
|
||||
//! errors, and a patch is only data until the session validates it.
|
||||
|
||||
use std::io::{self, Read, Write};
|
||||
|
||||
use litellm_callbacks_v1::{Envelope, RequestFacts, Subscription, WirePatch};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub const MAX_FRAME_BYTES: usize = 64 * 1024 * 1024;
|
||||
|
||||
/// `subscriber` is an index into the hello's subscriptions.
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ToWorker<'a> {
|
||||
/// Fire and forget: nothing an observer does comes back, except a [`FromWorker::Report`].
|
||||
Event {
|
||||
subscriber: usize,
|
||||
envelope: &'a Envelope,
|
||||
},
|
||||
Intercept {
|
||||
id: u64,
|
||||
subscriber: usize,
|
||||
request: &'a RequestFacts,
|
||||
},
|
||||
/// Answered by `Flushed` once every event sent before it has been handled.
|
||||
Flush { id: u64 },
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum FromWorker {
|
||||
Hello {
|
||||
schema: u32,
|
||||
subscriptions: Vec<Subscription>,
|
||||
},
|
||||
/// `None` is an interceptor that returned nothing.
|
||||
Patch {
|
||||
id: u64,
|
||||
patch: Option<WirePatch>,
|
||||
},
|
||||
Error {
|
||||
id: u64,
|
||||
error_class: String,
|
||||
message: String,
|
||||
},
|
||||
/// An observer raised; the worker swallowed it, as the contract says.
|
||||
Report {
|
||||
subscriber: usize,
|
||||
event: Option<String>,
|
||||
message: String,
|
||||
},
|
||||
Flushed {
|
||||
id: u64,
|
||||
},
|
||||
}
|
||||
|
||||
pub fn write_frame(writer: &mut impl Write, frame: &ToWorker<'_>) -> io::Result<()> {
|
||||
let body = serde_json::to_vec(frame)?;
|
||||
let length = u32::try_from(body.len())
|
||||
.ok()
|
||||
.filter(|_| body.len() <= MAX_FRAME_BYTES)
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "frame too large"))?;
|
||||
writer.write_all(&length.to_be_bytes())?;
|
||||
writer.write_all(&body)?;
|
||||
writer.flush()
|
||||
}
|
||||
|
||||
/// The next frame, or `None` when the worker closed the connection between frames.
|
||||
pub fn read_frame(reader: &mut impl Read) -> io::Result<Option<FromWorker>> {
|
||||
let mut header = [0u8; 4];
|
||||
match reader.read_exact(&mut header) {
|
||||
Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),
|
||||
other => other?,
|
||||
}
|
||||
let length = u32::from_be_bytes(header) as usize;
|
||||
if length > MAX_FRAME_BYTES {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("frame of {length} bytes exceeds {MAX_FRAME_BYTES}"),
|
||||
));
|
||||
}
|
||||
let mut body = vec![0u8; length];
|
||||
reader.read_exact(&mut body)?;
|
||||
serde_json::from_slice(&body)
|
||||
.map(Some)
|
||||
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use litellm_callbacks_v1::{CallFacts, CallSession, EventKind, HeaderPatch};
|
||||
use litellm_host::event::{RequestContext, WireRequest};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn subscriptions() -> Vec<Subscription> {
|
||||
vec![
|
||||
Subscription {
|
||||
name: "recorder".to_string(),
|
||||
events: BTreeSet::from([EventKind::CallFailed, EventKind::CallSucceeded]),
|
||||
observes: true,
|
||||
intercepts: false,
|
||||
},
|
||||
Subscription {
|
||||
name: "my_pkg.cb.patch".to_string(),
|
||||
events: BTreeSet::new(),
|
||||
observes: false,
|
||||
intercepts: true,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
fn golden(text: &str) -> Value {
|
||||
serde_json::from_str(text).unwrap()
|
||||
}
|
||||
|
||||
/// The frames carry the contract's own golden envelope and request, so the two golden
|
||||
/// directories cannot drift apart.
|
||||
#[test]
|
||||
fn frames_to_the_worker_match_golden() {
|
||||
let facts = CallFacts {
|
||||
start_time: 10.0,
|
||||
asynchronous: true,
|
||||
metadata: json!({"team": "core"}).as_object().unwrap().clone(),
|
||||
metadata_dropped: vec!["broken".to_string()],
|
||||
};
|
||||
let (session, started) =
|
||||
CallSession::start("call-123".to_string(), "ocr", subscriptions(), facts);
|
||||
let interception = session.interception(
|
||||
WireRequest {
|
||||
url: "https://provider.example/v1".to_string(),
|
||||
headers: vec![("Authorization".to_string(), "secret".to_string())],
|
||||
body: json!({"input": "hello"}),
|
||||
},
|
||||
RequestContext {
|
||||
model: "model".to_string(),
|
||||
custom_llm_provider: "provider".to_string(),
|
||||
optional_params: json!({"temperature": 0.2}),
|
||||
secret_fields: Vec::new(),
|
||||
api_key: None,
|
||||
},
|
||||
);
|
||||
let turn = interception.turn().unwrap();
|
||||
|
||||
let frames = [
|
||||
(
|
||||
ToWorker::Event {
|
||||
subscriber: 0,
|
||||
envelope: &started.envelope,
|
||||
},
|
||||
include_str!("../golden/to_worker/event.json"),
|
||||
),
|
||||
(
|
||||
ToWorker::Intercept {
|
||||
id: 1,
|
||||
subscriber: turn.subscriber,
|
||||
request: &turn.request,
|
||||
},
|
||||
include_str!("../golden/to_worker/intercept.json"),
|
||||
),
|
||||
(
|
||||
ToWorker::Flush { id: 2 },
|
||||
include_str!("../golden/to_worker/flush.json"),
|
||||
),
|
||||
];
|
||||
for (frame, expected) in frames {
|
||||
assert_eq!(serde_json::to_value(&frame).unwrap(), golden(expected));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frames_from_the_worker_parse_from_golden() {
|
||||
let parsed = |text: &str| serde_json::from_str::<FromWorker>(text).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
parsed(include_str!("../golden/from_worker/hello.json")),
|
||||
FromWorker::Hello {
|
||||
schema: 1,
|
||||
subscriptions: subscriptions()
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
parsed(include_str!("../golden/from_worker/patch.json")),
|
||||
FromWorker::Patch {
|
||||
id: 1,
|
||||
patch: Some(WirePatch {
|
||||
headers: HeaderPatch {
|
||||
remove: vec!["X-Trace".to_string()],
|
||||
set: vec![("X-Team".to_string(), "core".to_string())],
|
||||
},
|
||||
body: Some(json!({"input": "patched"})),
|
||||
}),
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
parsed(include_str!("../golden/from_worker/patch_none.json")),
|
||||
FromWorker::Patch { id: 1, patch: None }
|
||||
);
|
||||
assert_eq!(
|
||||
parsed(include_str!("../golden/from_worker/error.json")),
|
||||
FromWorker::Error {
|
||||
id: 1,
|
||||
error_class: "builtins.PermissionError".to_string(),
|
||||
message: "not this model".to_string(),
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
parsed(include_str!("../golden/from_worker/report.json")),
|
||||
FromWorker::Report {
|
||||
subscriber: 0,
|
||||
event: Some("call.started".to_string()),
|
||||
message: "boom".to_string(),
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
parsed(include_str!("../golden/from_worker/flushed.json")),
|
||||
FromWorker::Flushed { id: 2 }
|
||||
);
|
||||
}
|
||||
|
||||
fn framed(body: &[u8]) -> Vec<u8> {
|
||||
[&(body.len() as u32).to_be_bytes()[..], body].concat()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_worker_cannot_say_anything_the_protocol_does_not_name() {
|
||||
for body in [
|
||||
&br#"{"type":"call_gateway","method":"get_secret"}"#[..],
|
||||
br#"{"type":"flushed","id":1,"extra":true}"#,
|
||||
br#"{"type":"patch","id":1,"patch":{"url":"https://evil.example"}}"#,
|
||||
br#"[1,2,3]"#,
|
||||
] {
|
||||
let error = read_frame(&mut framed(body).as_slice()).unwrap_err();
|
||||
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_oversized_frame_is_refused_before_it_is_read() {
|
||||
let header = ((MAX_FRAME_BYTES + 1) as u32).to_be_bytes();
|
||||
let error = read_frame(&mut header.as_slice()).unwrap_err();
|
||||
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_closed_connection_between_frames_is_the_end_not_an_error() {
|
||||
assert_eq!(read_frame(&mut [].as_slice()).unwrap(), None);
|
||||
}
|
||||
}
|
||||
214
litellm-rust/crates/callbacks-v1-exthost/src/worker.rs
Normal file
214
litellm-rust/crates/callbacks-v1-exthost/src/worker.rs
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
//! The blocking reference client of one worker. It starts the worker on a unix socket it
|
||||
//! listens on (so nothing but a path is handed over, and an attached sidecar connects the
|
||||
//! same way), reads the hello, and then does exactly what a [`CallSession`] says: sends an
|
||||
//! emission's envelope to its observers and asks each interceptor for its patch in turn.
|
||||
//! The session, not this client, decides the protocol and validates every patch.
|
||||
|
||||
use std::{
|
||||
io,
|
||||
os::unix::net::{UnixListener, UnixStream},
|
||||
path::Path,
|
||||
process::{Child, Command},
|
||||
};
|
||||
|
||||
use litellm_callbacks_v1::{Emission, Interception, PatchError, SCHEMA_V1, Subscription};
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::protocol::{FromWorker, ToWorker, read_frame, write_frame};
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum ExtHostError {
|
||||
#[error("extension host i/o: {0}")]
|
||||
Io(#[from] io::Error),
|
||||
#[error("extension host protocol: {0}")]
|
||||
Protocol(String),
|
||||
#[error("worker speaks schema {0}, this gateway speaks {SCHEMA_V1}")]
|
||||
Schema(u32),
|
||||
/// The interceptor raised. Whether that fails the call or skips the patch is the
|
||||
/// gateway's policy for this subscriber, not the worker's.
|
||||
#[error("callback {subscriber}: {error_class}: {message}")]
|
||||
Interceptor {
|
||||
subscriber: String,
|
||||
error_class: String,
|
||||
message: String,
|
||||
},
|
||||
#[error("callback {subscriber}: {error}")]
|
||||
Patch {
|
||||
subscriber: String,
|
||||
error: PatchError,
|
||||
},
|
||||
}
|
||||
|
||||
/// An observer's swallowed failure, as the worker reported it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Report {
|
||||
pub subscriber: String,
|
||||
pub event: Option<String>,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
pub struct Worker {
|
||||
stream: UnixStream,
|
||||
child: Option<Child>,
|
||||
subscriptions: Vec<Subscription>,
|
||||
reports: Vec<Report>,
|
||||
next_id: u64,
|
||||
}
|
||||
|
||||
impl Worker {
|
||||
/// Starts `command` with `--connect <socket>` appended and waits for its hello. What
|
||||
/// the worker can reach is whatever `command` was given: its environment, its working
|
||||
/// directory, its user. This crate grants nothing.
|
||||
pub fn spawn(mut command: Command, socket: &Path) -> Result<Self, ExtHostError> {
|
||||
let listener = UnixListener::bind(socket)?;
|
||||
let child = command.arg("--connect").arg(socket).spawn()?;
|
||||
let (stream, _) = listener.accept()?;
|
||||
Self::greeted(stream, Some(child))
|
||||
}
|
||||
|
||||
/// Takes a worker that connected by itself, a sidecar for example.
|
||||
pub fn attach(stream: UnixStream) -> Result<Self, ExtHostError> {
|
||||
Self::greeted(stream, None)
|
||||
}
|
||||
|
||||
fn greeted(mut stream: UnixStream, child: Option<Child>) -> Result<Self, ExtHostError> {
|
||||
match read_frame(&mut stream)? {
|
||||
Some(FromWorker::Hello {
|
||||
schema: SCHEMA_V1,
|
||||
subscriptions,
|
||||
}) => Ok(Self {
|
||||
stream,
|
||||
child,
|
||||
subscriptions,
|
||||
reports: Vec::new(),
|
||||
next_id: 0,
|
||||
}),
|
||||
Some(FromWorker::Hello { schema, .. }) => Err(ExtHostError::Schema(schema)),
|
||||
other => Err(ExtHostError::Protocol(format!(
|
||||
"expected hello, got {other:?}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
/// What the worker's callbacks subscribed to: the session's subscriptions for every
|
||||
/// call this worker serves.
|
||||
pub fn subscriptions(&self) -> &[Subscription] {
|
||||
&self.subscriptions
|
||||
}
|
||||
|
||||
fn name(&self, subscriber: usize) -> String {
|
||||
self.subscriptions
|
||||
.get(subscriber)
|
||||
.map_or_else(|| format!("#{subscriber}"), |found| found.name.clone())
|
||||
}
|
||||
|
||||
fn id(&mut self) -> u64 {
|
||||
self.next_id += 1;
|
||||
self.next_id
|
||||
}
|
||||
|
||||
/// Sends the envelope to each of its observers and returns at once: observers are
|
||||
/// asynchronous to the call.
|
||||
pub fn observe(&mut self, emission: &Emission) -> Result<(), ExtHostError> {
|
||||
for &subscriber in &emission.observers {
|
||||
write_frame(
|
||||
&mut self.stream,
|
||||
&ToWorker::Event {
|
||||
subscriber,
|
||||
envelope: &emission.envelope,
|
||||
},
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The next frame that is not a report; reports are kept for [`Worker::flush`].
|
||||
fn answer(&mut self) -> Result<FromWorker, ExtHostError> {
|
||||
loop {
|
||||
match read_frame(&mut self.stream)? {
|
||||
Some(FromWorker::Report {
|
||||
subscriber,
|
||||
event,
|
||||
message,
|
||||
}) => {
|
||||
let subscriber = self.name(subscriber);
|
||||
self.reports.push(Report {
|
||||
subscriber,
|
||||
event,
|
||||
message,
|
||||
});
|
||||
}
|
||||
Some(frame) => return Ok(frame),
|
||||
None => return Err(ExtHostError::Protocol("worker closed".to_string())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Asks every interceptor for its patch, in the session's order, and folds each into
|
||||
/// the interception, which validates it.
|
||||
pub fn intercept(&mut self, interception: Interception) -> Result<Interception, ExtHostError> {
|
||||
let mut current = interception;
|
||||
while let Some(turn) = current.turn() {
|
||||
let id = self.id();
|
||||
write_frame(
|
||||
&mut self.stream,
|
||||
&ToWorker::Intercept {
|
||||
id,
|
||||
subscriber: turn.subscriber,
|
||||
request: &turn.request,
|
||||
},
|
||||
)?;
|
||||
let subscriber = self.name(turn.subscriber);
|
||||
current = match self.answer()? {
|
||||
FromWorker::Patch {
|
||||
id: answered,
|
||||
patch,
|
||||
} if answered == id => current
|
||||
.patched(patch.unwrap_or_default())
|
||||
.map_err(|error| ExtHostError::Patch { subscriber, error })?,
|
||||
FromWorker::Error {
|
||||
id: answered,
|
||||
error_class,
|
||||
message,
|
||||
} if answered == id => {
|
||||
return Err(ExtHostError::Interceptor {
|
||||
subscriber,
|
||||
error_class,
|
||||
message,
|
||||
});
|
||||
}
|
||||
other => {
|
||||
return Err(ExtHostError::Protocol(format!(
|
||||
"expected the answer to intercept {id}, got {other:?}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
}
|
||||
Ok(current)
|
||||
}
|
||||
|
||||
/// Waits until the worker has handled every event sent so far, and returns what its
|
||||
/// observers reported since the last flush.
|
||||
pub fn flush(&mut self) -> Result<Vec<Report>, ExtHostError> {
|
||||
let id = self.id();
|
||||
write_frame(&mut self.stream, &ToWorker::Flush { id })?;
|
||||
match self.answer()? {
|
||||
FromWorker::Flushed { id: answered } if answered == id => {
|
||||
Ok(std::mem::take(&mut self.reports))
|
||||
}
|
||||
other => Err(ExtHostError::Protocol(format!(
|
||||
"expected flushed {id}, got {other:?}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Worker {
|
||||
/// Closing the socket is how a worker is told to stop; it exits once its queues drain.
|
||||
fn drop(&mut self) {
|
||||
let _ = self.stream.shutdown(std::net::Shutdown::Both);
|
||||
if let Some(child) = self.child.as_mut() {
|
||||
let _ = child.wait();
|
||||
}
|
||||
}
|
||||
}
|
||||
239
litellm-rust/crates/callbacks-v1-exthost/tests/worker.rs
Normal file
239
litellm-rust/crates/callbacks-v1-exthost/tests/worker.rs
Normal file
|
|
@ -0,0 +1,239 @@
|
|||
//! End to end: a real `CallSession` driven through a real Python worker. The callback file
|
||||
//! below is an ordinary SDK callback; nothing in it knows it runs out of process.
|
||||
//!
|
||||
//! Ignored by default because it needs an interpreter that can import `litellm`:
|
||||
//!
|
||||
//! PYO3_PYTHON=/path/to/.venv/bin/python cargo test -p litellm-callbacks-v1-exthost -- --ignored
|
||||
|
||||
use std::{
|
||||
env, fs,
|
||||
path::{Path, PathBuf},
|
||||
process::Command,
|
||||
};
|
||||
|
||||
use litellm_callbacks_v1::{CallFacts, CallSession, ErrorFacts, PatchError};
|
||||
use litellm_callbacks_v1_exthost::{ExtHostError, Report, Worker};
|
||||
use litellm_host::event::{FailureOrigin, RequestContext, Timing, WireRequest};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
const CALLBACKS: &str = r#"
|
||||
import json
|
||||
import os
|
||||
|
||||
from litellm.callbacks_v1 import before_send, on_event
|
||||
|
||||
LOG = os.environ["CALLBACK_LOG"]
|
||||
|
||||
|
||||
@on_event("call.started", "request.sending", "response.received", "call.succeeded", name="recorder")
|
||||
async def record(event):
|
||||
with open(LOG, "a") as log:
|
||||
log.write(json.dumps([event["seq"], event["event"]["type"], event["event"].get("body")]) + "\n")
|
||||
|
||||
|
||||
@on_event("call.started", name="flaky")
|
||||
def flaky(event):
|
||||
raise ValueError("boom")
|
||||
|
||||
|
||||
@before_send
|
||||
def tag(request):
|
||||
return {"headers": {"set": [["X-Team", "core"]]}, "body": {**request["body"], "tagged": True}}
|
||||
|
||||
|
||||
@before_send
|
||||
async def guard(request):
|
||||
if request["body"].get("input") == "steal":
|
||||
return {"headers": {"set": [["Authorization", "stolen"]]}}
|
||||
if request["body"].get("input") == "refuse":
|
||||
raise PermissionError("not this input")
|
||||
return None
|
||||
"#;
|
||||
|
||||
const TIMING: Timing = Timing {
|
||||
start_time: 10.0,
|
||||
end_time: 12.5,
|
||||
};
|
||||
|
||||
struct Scratch(PathBuf);
|
||||
|
||||
impl Scratch {
|
||||
fn new(name: &str) -> Self {
|
||||
let path = env::temp_dir().join(format!("exthost-{}-{name}", std::process::id()));
|
||||
let _ = fs::remove_dir_all(&path);
|
||||
fs::create_dir_all(path.join("modules")).unwrap();
|
||||
fs::create_dir_all(path.join("cwd")).unwrap();
|
||||
fs::write(path.join("modules/gateway_callbacks.py"), CALLBACKS).unwrap();
|
||||
Self(path)
|
||||
}
|
||||
|
||||
fn log(&self) -> Vec<Value> {
|
||||
fs::read_to_string(self.0.join("log"))
|
||||
.unwrap_or_default()
|
||||
.lines()
|
||||
.map(|line| serde_json::from_str(line).unwrap())
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Scratch {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::remove_dir_all(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
/// The worker gets a cleared environment and an empty working directory: it sees what it
|
||||
/// is given and nothing of the gateway's.
|
||||
fn worker(scratch: &Scratch) -> Worker {
|
||||
let python = env::var("PYO3_PYTHON").expect("PYO3_PYTHON names an interpreter with litellm");
|
||||
let repository = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..");
|
||||
let python_path = env::join_paths([repository, scratch.0.join("modules")]).unwrap();
|
||||
let mut command = Command::new(python);
|
||||
command
|
||||
.args([
|
||||
"-m",
|
||||
"litellm.callbacks_v1.host",
|
||||
"--load",
|
||||
"gateway_callbacks",
|
||||
])
|
||||
.env_clear()
|
||||
.env("PYTHONPATH", python_path)
|
||||
.env("LITELLM_MODE", "PRODUCTION")
|
||||
.env("CALLBACK_LOG", scratch.0.join("log"))
|
||||
.current_dir(scratch.0.join("cwd"));
|
||||
Worker::spawn(command, &scratch.0.join("socket")).unwrap()
|
||||
}
|
||||
|
||||
fn wire(input: &str) -> WireRequest {
|
||||
WireRequest {
|
||||
url: "https://provider.example/v1".to_string(),
|
||||
headers: vec![("Authorization".to_string(), "secret".to_string())],
|
||||
body: json!({"input": input}),
|
||||
}
|
||||
}
|
||||
|
||||
fn context() -> RequestContext {
|
||||
RequestContext {
|
||||
model: "model".to_string(),
|
||||
custom_llm_provider: "provider".to_string(),
|
||||
optional_params: json!({}),
|
||||
secret_fields: Vec::new(),
|
||||
api_key: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn start(worker: &mut Worker) -> CallSession {
|
||||
let facts = CallFacts {
|
||||
start_time: TIMING.start_time,
|
||||
asynchronous: true,
|
||||
metadata: Map::new(),
|
||||
metadata_dropped: Vec::new(),
|
||||
};
|
||||
let (session, started) = CallSession::start(
|
||||
"call-123".to_string(),
|
||||
"ocr",
|
||||
worker.subscriptions().to_vec(),
|
||||
facts,
|
||||
);
|
||||
worker.observe(&started).unwrap();
|
||||
session
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "needs PYO3_PYTHON to name an interpreter that can import litellm"]
|
||||
fn a_call_reaches_python_callbacks_and_their_patches_reach_the_wire() {
|
||||
let scratch = Scratch::new("call");
|
||||
let mut worker = worker(&scratch);
|
||||
let names: Vec<&str> = worker
|
||||
.subscriptions()
|
||||
.iter()
|
||||
.map(|subscription| subscription.name.as_str())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
names,
|
||||
[
|
||||
"recorder",
|
||||
"flaky",
|
||||
"gateway_callbacks.tag",
|
||||
"gateway_callbacks.guard"
|
||||
]
|
||||
);
|
||||
|
||||
let mut session = start(&mut worker);
|
||||
let interception = worker
|
||||
.intercept(session.interception(wire("hello"), context()))
|
||||
.unwrap();
|
||||
let (sending, sent) = session.request_sending(interception);
|
||||
worker.observe(&sending).unwrap();
|
||||
worker
|
||||
.observe(&session.response_received("{}".to_string()))
|
||||
.unwrap();
|
||||
worker
|
||||
.observe(&session.succeeded(TIMING, json!({"id": "response"}), None))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(sent.body, json!({"input": "hello", "tagged": true}));
|
||||
assert!(
|
||||
sent.headers
|
||||
.contains(&("X-Team".to_string(), "core".to_string()))
|
||||
);
|
||||
assert!(
|
||||
sent.headers
|
||||
.contains(&("Authorization".to_string(), "secret".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
worker.flush().unwrap(),
|
||||
vec![Report {
|
||||
subscriber: "flaky".to_string(),
|
||||
event: Some("call.started".to_string()),
|
||||
message: "boom".to_string(),
|
||||
}]
|
||||
);
|
||||
assert_eq!(
|
||||
scratch.log(),
|
||||
vec![
|
||||
json!([0, "call.started", null]),
|
||||
json!([1, "request.sending", {"input": "hello", "tagged": true}]),
|
||||
json!([2, "response.received", "{}"]),
|
||||
json!([3, "call.succeeded", null]),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "needs PYO3_PYTHON to name an interpreter that can import litellm"]
|
||||
fn a_worker_cannot_patch_a_credential_and_its_refusal_is_the_gateways_to_act_on() {
|
||||
let scratch = Scratch::new("refusals");
|
||||
let mut worker = worker(&scratch);
|
||||
|
||||
let session = start(&mut worker);
|
||||
let stolen = worker
|
||||
.intercept(session.interception(wire("steal"), context()))
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
stolen,
|
||||
ExtHostError::Patch { ref subscriber, error: PatchError::ProtectedHeader(_) }
|
||||
if subscriber == "gateway_callbacks.guard"
|
||||
));
|
||||
|
||||
let refused = worker
|
||||
.intercept(session.interception(wire("refuse"), context()))
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
refused,
|
||||
ExtHostError::Interceptor { ref error_class, ref message, .. }
|
||||
if error_class == "builtins.PermissionError" && message == "not this input"
|
||||
));
|
||||
|
||||
let failed = session.failed(
|
||||
TIMING,
|
||||
FailureOrigin::Call,
|
||||
ErrorFacts {
|
||||
class: "builtins.PermissionError".to_string(),
|
||||
message: "not this input".to_string(),
|
||||
status_code: None,
|
||||
},
|
||||
);
|
||||
assert!(failed.observers.is_empty());
|
||||
worker.flush().unwrap();
|
||||
}
|
||||
|
|
@ -1,7 +1,9 @@
|
|||
- Target invariants, not completion claims
|
||||
- Keep this crate the Python runtime adapter for `litellm-callbacks-v1` and nothing else: the subscriber snapshot and one `PythonLifecycle` that delivers the core contract
|
||||
- Envelopes, sequencing, redaction, handler selection and wire patch validation belong to `litellm-callbacks-v1`
|
||||
- `litellm/rust_bridge/callbacks_v1_python.py` is the only Python module behind it; `python_contract.json` pins its functions on both sides
|
||||
- Envelopes, sequencing, routing, the interceptor fold, redaction, handler selection and wire patch validation belong to `litellm-callbacks-v1`; this adapter drives its `CallSession` and adds only projection (Python arguments, response and exception into facts) and execution (calling handlers, `Await`/`resume`, cancellation, GC traversal)
|
||||
- The snapshot splits each subscriber into the contract's `Subscription` and this crate's `Handlers`, kept at the same index
|
||||
- `litellm/rust_bridge/callbacks_v1.py` is the only Python module Rust imports; `python_contract.json` pins its functions on both sides
|
||||
- The typed authoring surface (callback protocols, envelope and patch types, the registry) is `litellm/callbacks_v1/__init__.py`; the bridge module re-exports `snapshot` from it and otherwise holds only marshalling
|
||||
- Terms (contract terms are in `callbacks-v1/AGENTS.md`, lifecycle terms in `host-python/AGENTS.md`)
|
||||
- Registry: the process-wide subscriber list; it lives in Python (`register`/`unregister`), never in Rust
|
||||
- Snapshot: that list read once per call, with one observer and one interceptor handler already selected per subscriber for the call's mode (`subscribers.rs`)
|
||||
|
|
|
|||
|
|
@ -1,23 +1,25 @@
|
|||
//! The v1 lifecycle: each step builds one envelope, hands every subscribed observer a
|
||||
//! fresh conversion of it, and, before the request is sent, folds every interceptor's
|
||||
//! patch into the wire request.
|
||||
//! The v1 lifecycle in CPython. The contract's [`CallSession`] decides which envelope a
|
||||
//! step produces, who receives it and how patches fold into the wire; this adapter only
|
||||
//! projects Python values into facts and runs the Python handlers: every observer of an
|
||||
//! emission gets a fresh conversion of its envelope, inline, and every interceptor's
|
||||
//! answer goes back to the session.
|
||||
|
||||
use litellm_callbacks_v1::{
|
||||
CallFacts, Envelope, ErrorFacts, Event, RequestFacts, Sequencer, WirePatch, apply,
|
||||
CallFacts, CallSession, Emission, Envelope, ErrorFacts, Interception, Subscription, WirePatch,
|
||||
};
|
||||
use litellm_host::event::{MachineEvent, RequestContext, WireRequest};
|
||||
use litellm_host_python::{
|
||||
LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, is_cancellation, missing_state, to_py,
|
||||
};
|
||||
use pyo3::exceptions::PyValueError;
|
||||
use pyo3::gc::{PyTraverseError, PyVisit};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyDict, PyString};
|
||||
use pyo3::{
|
||||
exceptions::PyValueError,
|
||||
gc::{PyTraverseError, PyVisit},
|
||||
prelude::*,
|
||||
types::{PyDict, PyString},
|
||||
};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::call::V1PythonSurface;
|
||||
use crate::python::V1Python;
|
||||
use crate::subscribers::Subscriber;
|
||||
use crate::{call::V1PythonSurface, python::V1Python, subscribers::Handlers};
|
||||
|
||||
/// What a step hands back to the driver once every observer of its envelope has run.
|
||||
enum Continuation {
|
||||
|
|
@ -26,26 +28,29 @@ enum Continuation {
|
|||
Wire(Box<WireRequest>),
|
||||
}
|
||||
|
||||
/// The suspended step; `index` is the subscriber whose awaitable the driver is awaiting.
|
||||
/// The suspended step, while the driver awaits one handler's awaitable.
|
||||
enum Pending {
|
||||
/// `position` is the place in `emission.observers` of the observer being awaited.
|
||||
Observe {
|
||||
envelope: Envelope,
|
||||
index: usize,
|
||||
emission: Emission,
|
||||
position: usize,
|
||||
then: Continuation,
|
||||
},
|
||||
/// `subscriber` is the interceptor whose turn it is.
|
||||
Intercept {
|
||||
wire: Box<WireRequest>,
|
||||
context: RequestContext,
|
||||
index: usize,
|
||||
interception: Interception,
|
||||
subscriber: usize,
|
||||
},
|
||||
}
|
||||
|
||||
pub struct V1PythonLifecycle {
|
||||
surface: V1PythonSurface,
|
||||
subscribers: Vec<Subscriber>,
|
||||
/// Held until `begin` learns the call's identity and hands them to the session.
|
||||
subscriptions: Vec<Subscription>,
|
||||
/// The Python side of each subscription, at the same index.
|
||||
handlers: Vec<Handlers>,
|
||||
asynchronous: bool,
|
||||
sequencer: Option<Sequencer>,
|
||||
streamed: bool,
|
||||
session: Option<CallSession>,
|
||||
pending: Option<Pending>,
|
||||
closed: bool,
|
||||
}
|
||||
|
|
@ -70,27 +75,21 @@ impl Continuation {
|
|||
impl V1PythonLifecycle {
|
||||
pub(crate) fn new(
|
||||
surface: V1PythonSurface,
|
||||
subscribers: Vec<Subscriber>,
|
||||
subscriptions: Vec<Subscription>,
|
||||
handlers: Vec<Handlers>,
|
||||
asynchronous: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
surface,
|
||||
subscribers,
|
||||
subscriptions,
|
||||
handlers,
|
||||
asynchronous,
|
||||
sequencer: None,
|
||||
streamed: false,
|
||||
session: None,
|
||||
pending: None,
|
||||
closed: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn envelope(&mut self, event: Event) -> PyResult<Envelope> {
|
||||
self.sequencer
|
||||
.as_mut()
|
||||
.map(|sequencer| sequencer.envelope(event))
|
||||
.ok_or_else(missing_state)
|
||||
}
|
||||
|
||||
fn report(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
|
|
@ -98,7 +97,7 @@ impl V1PythonLifecycle {
|
|||
envelope: &Envelope,
|
||||
error: PyErr,
|
||||
) -> PyResult<()> {
|
||||
let name = &self.subscribers[index].name;
|
||||
let name = &self.handlers[index].name;
|
||||
let event: &'static str = envelope.event.kind().into();
|
||||
match V1Python::Report.call(py, (name, event, error.value(py))) {
|
||||
Err(report_error) if is_cancellation(py, &report_error) => Err(report_error),
|
||||
|
|
@ -106,86 +105,87 @@ impl V1PythonLifecycle {
|
|||
}
|
||||
}
|
||||
|
||||
/// Runs the emission's observers inline from `from`, which is one legal schedule of
|
||||
/// the contract and nothing a callback may rely on.
|
||||
fn observe(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
envelope: Envelope,
|
||||
emission: Emission,
|
||||
from: usize,
|
||||
then: Continuation,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
for index in from..self.subscribers.len() {
|
||||
let Some(handler) = self.subscribers[index]
|
||||
.observer(envelope.event.kind())
|
||||
for position in from..emission.observers.len() {
|
||||
let index = emission.observers[position];
|
||||
let Some(handler) = self.handlers[index]
|
||||
.observe
|
||||
.as_ref()
|
||||
.map(|handler| handler.clone_ref(py))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let outcome = to_py(py, &envelope).and_then(|argument| handler.call(py, argument));
|
||||
let outcome =
|
||||
to_py(py, &emission.envelope).and_then(|argument| handler.call(py, argument));
|
||||
match outcome {
|
||||
Ok(awaitable) if handler.is_async() => {
|
||||
self.pending = Some(Pending::Observe {
|
||||
envelope,
|
||||
index,
|
||||
emission,
|
||||
position,
|
||||
then,
|
||||
});
|
||||
return Ok(LifecycleStep::Await(awaitable.unbind()));
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(error) if is_cancellation(py, &error) => return Err(error),
|
||||
Err(error) => self.report(py, index, &envelope, error)?,
|
||||
Err(error) => self.report(py, index, &emission.envelope, error)?,
|
||||
}
|
||||
}
|
||||
Ok(then.into_step())
|
||||
}
|
||||
|
||||
fn apply_patch(
|
||||
fn patched(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
index: usize,
|
||||
wire: WireRequest,
|
||||
interception: Interception,
|
||||
subscriber: usize,
|
||||
result: Py<PyAny>,
|
||||
) -> PyResult<WireRequest> {
|
||||
) -> PyResult<Interception> {
|
||||
let patch = if result.bind(py).is_none() {
|
||||
WirePatch::default()
|
||||
} else {
|
||||
from_py::<WirePatch>(result.bind(py))?
|
||||
};
|
||||
let name = &self.subscribers[index].name;
|
||||
apply(wire, patch)
|
||||
let name = &self.handlers[subscriber].name;
|
||||
interception
|
||||
.patched(patch)
|
||||
.map_err(|error| PyValueError::new_err(format!("callback {name}: {error}")))
|
||||
}
|
||||
|
||||
fn intercept(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
wire: Box<WireRequest>,
|
||||
context: RequestContext,
|
||||
from: usize,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
let mut current = wire;
|
||||
for index in from..self.subscribers.len() {
|
||||
let Some(handler) = self.subscribers[index]
|
||||
fn intercept(&mut self, py: Python<'_>, interception: Interception) -> PyResult<LifecycleStep> {
|
||||
let mut current = interception;
|
||||
while let Some(turn) = current.turn() {
|
||||
let Some(handler) = self.handlers[turn.subscriber]
|
||||
.intercept
|
||||
.as_ref()
|
||||
.map(|handler| handler.clone_ref(py))
|
||||
else {
|
||||
continue;
|
||||
return Err(missing_state());
|
||||
};
|
||||
let request = to_py(py, &RequestFacts::new(¤t, &context))?;
|
||||
let result = handler.call(py, request)?;
|
||||
let result = handler.call(py, to_py(py, &turn.request)?)?;
|
||||
if handler.is_async() {
|
||||
self.pending = Some(Pending::Intercept {
|
||||
wire: current,
|
||||
context,
|
||||
index,
|
||||
interception: current,
|
||||
subscriber: turn.subscriber,
|
||||
});
|
||||
return Ok(LifecycleStep::Await(result.unbind()));
|
||||
}
|
||||
current = Box::new(self.apply_patch(py, index, *current, result.unbind())?);
|
||||
current = self.patched(py, current, turn.subscriber, result.unbind())?;
|
||||
}
|
||||
let envelope =
|
||||
self.envelope(Event::RequestSending(RequestFacts::new(¤t, &context)))?;
|
||||
self.observe(py, envelope, 0, Continuation::Wire(current))
|
||||
let (emission, wire) = self
|
||||
.session
|
||||
.as_mut()
|
||||
.ok_or_else(missing_state)?
|
||||
.request_sending(current);
|
||||
self.observe(py, emission, 0, Continuation::Wire(Box::new(wire)))
|
||||
}
|
||||
|
||||
fn metadata(arguments: &Bound<'_, PyDict>) -> PyResult<(Map<String, Value>, Vec<String>)> {
|
||||
|
|
@ -246,14 +246,19 @@ impl PythonLifecycle for V1PythonLifecycle {
|
|||
) -> PyResult<LifecycleStep> {
|
||||
let call_id = Self::call_id(py, arguments.bind(py))?;
|
||||
let (metadata, metadata_dropped) = Self::metadata(arguments.bind(py))?;
|
||||
self.sequencer = Some(Sequencer::new(call_id, self.surface.call_type));
|
||||
let envelope = self.envelope(Event::CallStarted(CallFacts {
|
||||
start_time,
|
||||
asynchronous: self.asynchronous,
|
||||
metadata,
|
||||
metadata_dropped,
|
||||
}))?;
|
||||
self.observe(py, envelope, 0, Continuation::Arguments(arguments))
|
||||
let (session, emission) = CallSession::start(
|
||||
call_id,
|
||||
self.surface.call_type,
|
||||
std::mem::take(&mut self.subscriptions),
|
||||
CallFacts {
|
||||
start_time,
|
||||
asynchronous: self.asynchronous,
|
||||
metadata,
|
||||
metadata_dropped,
|
||||
},
|
||||
);
|
||||
self.session = Some(session);
|
||||
self.observe(py, emission, 0, Continuation::Arguments(arguments))
|
||||
}
|
||||
|
||||
fn before_send(
|
||||
|
|
@ -262,16 +267,23 @@ impl PythonLifecycle for V1PythonLifecycle {
|
|||
wire: Box<WireRequest>,
|
||||
context: &RequestContext,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
self.intercept(py, wire, context.clone(), 0)
|
||||
let interception = self
|
||||
.session
|
||||
.as_ref()
|
||||
.ok_or_else(missing_state)?
|
||||
.interception(*wire, context.clone());
|
||||
self.intercept(py, interception)
|
||||
}
|
||||
|
||||
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
|
||||
match event {
|
||||
LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => {
|
||||
let envelope = self.envelope(Event::ResponseReceived {
|
||||
body: raw.body.clone(),
|
||||
})?;
|
||||
self.observe(py, envelope, 0, Continuation::Done)
|
||||
let emission = self
|
||||
.session
|
||||
.as_mut()
|
||||
.ok_or_else(missing_state)?
|
||||
.response_received(raw.body.clone());
|
||||
self.observe(py, emission, 0, Continuation::Done)
|
||||
}
|
||||
LifecycleEvent::Succeeded { timing, response } => {
|
||||
let projected = V1Python::ProjectResponse
|
||||
|
|
@ -282,13 +294,9 @@ impl PythonLifecycle for V1PythonLifecycle {
|
|||
Err(error) if is_cancellation(py, &error) => return Err(error),
|
||||
Err(error) => (Value::Null, Some(error.to_string())),
|
||||
};
|
||||
let envelope = self.envelope(Event::CallSucceeded {
|
||||
timing: timing.into(),
|
||||
streamed: self.streamed,
|
||||
response,
|
||||
response_error,
|
||||
})?;
|
||||
self.observe(py, envelope, 0, Continuation::Done)
|
||||
let session = self.session.take().ok_or_else(missing_state)?;
|
||||
let emission = session.succeeded(timing, response, response_error);
|
||||
self.observe(py, emission, 0, Continuation::Done)
|
||||
}
|
||||
LifecycleEvent::Failed {
|
||||
timing,
|
||||
|
|
@ -296,44 +304,41 @@ impl PythonLifecycle for V1PythonLifecycle {
|
|||
error,
|
||||
} => {
|
||||
let facts = Self::error_facts(py, error)?;
|
||||
let envelope = self.envelope(Event::CallFailed {
|
||||
timing: timing.into(),
|
||||
streamed: self.streamed,
|
||||
origin: origin.into(),
|
||||
error: facts,
|
||||
})?;
|
||||
self.observe(py, envelope, 0, Continuation::Done)
|
||||
let session = self.session.take().ok_or_else(missing_state)?;
|
||||
let emission = session.failed(timing, origin, facts);
|
||||
self.observe(py, emission, 0, Continuation::Done)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn opened(&mut self, _py: Python<'_>) -> PyResult<()> {
|
||||
self.streamed = true;
|
||||
if let Some(session) = self.session.as_mut() {
|
||||
session.opened();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn resume(&mut self, py: Python<'_>, result: PyResult<Py<PyAny>>) -> PyResult<LifecycleStep> {
|
||||
match self.pending.take().ok_or_else(missing_state)? {
|
||||
Pending::Observe {
|
||||
envelope,
|
||||
index,
|
||||
emission,
|
||||
position,
|
||||
then,
|
||||
} => {
|
||||
if let Err(error) = result {
|
||||
if is_cancellation(py, &error) {
|
||||
return Err(error);
|
||||
}
|
||||
self.report(py, index, &envelope, error)?;
|
||||
self.report(py, emission.observers[position], &emission.envelope, error)?;
|
||||
}
|
||||
self.observe(py, envelope, index + 1, then)
|
||||
self.observe(py, emission, position + 1, then)
|
||||
}
|
||||
Pending::Intercept {
|
||||
wire,
|
||||
context,
|
||||
index,
|
||||
interception,
|
||||
subscriber,
|
||||
} => {
|
||||
let patched = self.apply_patch(py, index, *wire, result?)?;
|
||||
self.intercept(py, Box::new(patched), context, index + 1)
|
||||
let patched = self.patched(py, interception, subscriber, result?)?;
|
||||
self.intercept(py, patched)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -344,12 +349,14 @@ impl PythonLifecycle for V1PythonLifecycle {
|
|||
}
|
||||
self.closed = true;
|
||||
self.pending = None;
|
||||
self.subscribers.clear();
|
||||
self.session = None;
|
||||
self.subscriptions.clear();
|
||||
self.handlers.clear();
|
||||
}
|
||||
|
||||
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
for subscriber in &self.subscribers {
|
||||
subscriber.traverse(visit)?;
|
||||
for handlers in &self.handlers {
|
||||
handlers.traverse(visit)?;
|
||||
}
|
||||
match &self.pending {
|
||||
Some(Pending::Observe { then, .. }) => then.traverse(visit),
|
||||
|
|
|
|||
|
|
@ -2,8 +2,7 @@
|
|||
|
||||
use pyo3::prelude::*;
|
||||
|
||||
use crate::adapter::V1PythonLifecycle;
|
||||
use crate::subscribers::snapshot;
|
||||
use crate::{adapter::V1PythonLifecycle, subscribers::snapshot};
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct V1PythonSurface {
|
||||
|
|
@ -19,7 +18,9 @@ impl V1PythonLifecycle {
|
|||
surface: V1PythonSurface,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Option<Self>> {
|
||||
let subscribers = snapshot(py, asynchronous)?;
|
||||
Ok((!subscribers.is_empty()).then(|| Self::new(surface, subscribers, asynchronous)))
|
||||
let (subscriptions, handlers): (Vec<_>, Vec<_>) =
|
||||
snapshot(py, asynchronous)?.into_iter().unzip();
|
||||
Ok((!subscriptions.is_empty())
|
||||
.then(|| Self::new(surface, subscriptions, handlers, asynchronous)))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
//! The Python contract of this crate: every function of
|
||||
//! `litellm/rust_bridge/callbacks_v1_python.py` that Rust calls, by name.
|
||||
//! `litellm/rust_bridge/callbacks_v1.py` that Rust calls, by name.
|
||||
|
||||
use pyo3::prelude::*;
|
||||
use strum::{IntoStaticStr, VariantArray};
|
||||
|
||||
const MODULE: &str = "litellm.rust_bridge.callbacks_v1_python";
|
||||
const MODULE: &str = "litellm.rust_bridge.callbacks_v1";
|
||||
|
||||
#[derive(Clone, Copy, Debug, IntoStaticStr, PartialEq, Eq, VariantArray)]
|
||||
pub(crate) enum V1Python {
|
||||
|
|
|
|||
|
|
@ -1,13 +1,19 @@
|
|||
//! The per-call subscriber snapshot: what Python's `register` recorded, read once, with
|
||||
//! one handler chosen per role for the call's execution mode. `register` has already
|
||||
//! rejected everything rejectable, so this only parses.
|
||||
//! rejected everything rejectable, so this only parses. Each subscriber splits in two: the
|
||||
//! [`Subscription`] the contract's session routes by, and the [`Handlers`] only this host
|
||||
//! can call.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use litellm_callbacks_v1::{EventKind, ExecutionMode, HandlerSelection, select_handler};
|
||||
use pyo3::exceptions::PyValueError;
|
||||
use pyo3::gc::{PyTraverseError, PyVisit};
|
||||
use pyo3::prelude::*;
|
||||
use litellm_callbacks_v1::{
|
||||
EventKind, ExecutionMode, HandlerSelection, Subscription, select_handler,
|
||||
};
|
||||
use pyo3::{
|
||||
exceptions::PyValueError,
|
||||
gc::{PyTraverseError, PyVisit},
|
||||
prelude::*,
|
||||
};
|
||||
|
||||
use crate::python::V1Python;
|
||||
|
||||
|
|
@ -17,9 +23,9 @@ pub enum Handler {
|
|||
Async(Py<PyAny>),
|
||||
}
|
||||
|
||||
pub struct Subscriber {
|
||||
/// The Python callables of one subscriber, at the index of its [`Subscription`].
|
||||
pub struct Handlers {
|
||||
pub name: String,
|
||||
pub events: BTreeSet<EventKind>,
|
||||
/// `on_event` or `async_on_event`: cannot affect the call.
|
||||
pub observe: Option<Handler>,
|
||||
/// `before_send` or `async_before_send`: returns a patch; its failure fails the call.
|
||||
|
|
@ -43,7 +49,7 @@ fn selected_handler(
|
|||
)
|
||||
}
|
||||
|
||||
pub fn snapshot(py: Python<'_>, asynchronous: bool) -> PyResult<Vec<Subscriber>> {
|
||||
pub fn snapshot(py: Python<'_>, asynchronous: bool) -> PyResult<Vec<(Subscription, Handlers)>> {
|
||||
let mode = if asynchronous {
|
||||
ExecutionMode::Async
|
||||
} else {
|
||||
|
|
@ -64,23 +70,28 @@ pub fn snapshot(py: Python<'_>, asynchronous: bool) -> PyResult<Vec<Subscriber>>
|
|||
})
|
||||
})
|
||||
.collect::<PyResult<BTreeSet<_>>>()?;
|
||||
Ok(Subscriber {
|
||||
name: subscriber.getattr("name")?.extract()?,
|
||||
events,
|
||||
observe: selected_handler(&subscriber, mode, "on_event", "async_on_event")?,
|
||||
intercept: selected_handler(&subscriber, mode, "before_send", "async_before_send")?,
|
||||
})
|
||||
let name: String = subscriber.getattr("name")?.extract()?;
|
||||
let observe = selected_handler(&subscriber, mode, "on_event", "async_on_event")?;
|
||||
let intercept =
|
||||
selected_handler(&subscriber, mode, "before_send", "async_before_send")?;
|
||||
Ok((
|
||||
Subscription {
|
||||
name: name.clone(),
|
||||
events,
|
||||
observes: observe.is_some(),
|
||||
intercepts: intercept.is_some(),
|
||||
},
|
||||
Handlers {
|
||||
name,
|
||||
observe,
|
||||
intercept,
|
||||
},
|
||||
))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl Subscriber {
|
||||
pub fn observer(&self, kind: EventKind) -> Option<&Handler> {
|
||||
self.observe
|
||||
.as_ref()
|
||||
.filter(|_| self.events.contains(&kind))
|
||||
}
|
||||
|
||||
impl Handlers {
|
||||
pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
for handler in self.observe.iter().chain(&self.intercept) {
|
||||
visit.call(handler.object())?;
|
||||
|
|
|
|||
|
|
@ -1,18 +1,18 @@
|
|||
use std::collections::BTreeSet;
|
||||
|
||||
use litellm_auth::SecretValue;
|
||||
use litellm_callbacks_v1::{EventKind, Subscription};
|
||||
use litellm_host::event::{MachineEvent, RawResponse, RequestContext, Timing, WireRequest};
|
||||
use litellm_host_python::{LifecycleEvent, LifecycleStep, PythonLifecycle, from_py};
|
||||
use pyo3::exceptions::asyncio::CancelledError;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::PyDict;
|
||||
use pyo3::{exceptions::asyncio::CancelledError, prelude::*, types::PyDict};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::adapter::V1PythonLifecycle;
|
||||
use crate::call::V1PythonSurface;
|
||||
use crate::subscribers::{Handler, Subscriber};
|
||||
use crate::test_support::{local, namespace};
|
||||
use litellm_callbacks_v1::EventKind;
|
||||
use crate::{
|
||||
adapter::V1PythonLifecycle,
|
||||
call::V1PythonSurface,
|
||||
subscribers::{Handler, Handlers},
|
||||
test_support::{local, namespace},
|
||||
};
|
||||
|
||||
const TIMING: Timing = Timing {
|
||||
start_time: 10.0,
|
||||
|
|
@ -25,7 +25,7 @@ fn subscriber(
|
|||
events: &[EventKind],
|
||||
observe: Option<(&str, bool)>,
|
||||
intercept: Option<(&str, bool)>,
|
||||
) -> Subscriber {
|
||||
) -> (Subscription, Handlers) {
|
||||
let handler = |spec: Option<(&str, bool)>| {
|
||||
spec.map(|(local_name, asynchronous)| {
|
||||
let callable = local(locals, local_name).unbind();
|
||||
|
|
@ -36,18 +36,28 @@ fn subscriber(
|
|||
}
|
||||
})
|
||||
};
|
||||
Subscriber {
|
||||
let handlers = Handlers {
|
||||
name: name.to_string(),
|
||||
events: events.iter().copied().collect::<BTreeSet<_>>(),
|
||||
observe: handler(observe),
|
||||
intercept: handler(intercept),
|
||||
}
|
||||
};
|
||||
(
|
||||
Subscription {
|
||||
name: name.to_string(),
|
||||
events: events.iter().copied().collect::<BTreeSet<_>>(),
|
||||
observes: handlers.observe.is_some(),
|
||||
intercepts: handlers.intercept.is_some(),
|
||||
},
|
||||
handlers,
|
||||
)
|
||||
}
|
||||
|
||||
fn lifecycle(subscribers: Vec<Subscriber>, asynchronous: bool) -> V1PythonLifecycle {
|
||||
fn lifecycle(subscribers: Vec<(Subscription, Handlers)>, asynchronous: bool) -> V1PythonLifecycle {
|
||||
let (subscriptions, handlers): (Vec<_>, Vec<_>) = subscribers.into_iter().unzip();
|
||||
V1PythonLifecycle::new(
|
||||
V1PythonSurface { call_type: "ocr" },
|
||||
subscribers,
|
||||
subscriptions,
|
||||
handlers,
|
||||
asynchronous,
|
||||
)
|
||||
}
|
||||
|
|
@ -245,7 +255,7 @@ def record(event):
|
|||
["call.started"]
|
||||
);
|
||||
let reports = py
|
||||
.import("litellm.rust_bridge.callbacks_v1_python")
|
||||
.import("litellm.rust_bridge.callbacks_v1")
|
||||
.unwrap()
|
||||
.getattr("reports")
|
||||
.unwrap();
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
use std::ffi::CStr;
|
||||
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::PyDict;
|
||||
use pyo3::{prelude::*, types::PyDict};
|
||||
|
||||
pub(crate) const PYTHON_CONTRACT: &str = include_str!("../python_contract.json");
|
||||
|
||||
|
|
@ -11,10 +10,10 @@ import json
|
|||
import sys
|
||||
import types
|
||||
|
||||
for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_v1_python'):
|
||||
for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_v1'):
|
||||
sys.modules.setdefault(name, types.ModuleType(name))
|
||||
|
||||
module = sys.modules['litellm.rust_bridge.callbacks_v1_python']
|
||||
module = sys.modules['litellm.rust_bridge.callbacks_v1']
|
||||
contract = json.loads(python_contract)
|
||||
|
||||
def contracted(name, fake):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,10 @@
|
|||
- Target invariants, not completion claims
|
||||
- Keep this crate the runtime-neutral v1 callback contract and nothing else: envelopes, sequencing, redaction, handler selection and validated wire patches
|
||||
- Keep this crate the runtime-neutral v1 callback contract and nothing else: envelopes, redaction, handler selection, validated wire patches, and the per-call session that sequences and routes them
|
||||
- `CallSession` is the whole protocol of one call and does no I/O: which envelope a moment produces, which subscriptions receive it, how interceptor patches fold into the wire. A host adapter projects its runtime's values into facts, asks the session, and runs handlers its own way; it never decides protocol. A second host (out of process, for the gateway) reuses the session unchanged
|
||||
- The two planes are in the types. An `Emission` has no return path, so nothing an observer does re-enters the session. An `Interception` takes exactly one patch per interceptor, in registration order, and `request_sending` yields the wire the host must send
|
||||
- A terminal method consumes the session, which is how a call has exactly one terminal envelope, last
|
||||
- `Subscription` is a subscriber as data (name, events, observes, intercepts) and serializes; handlers stay in the host adapter at the same index. It is what a host that loads handlers elsewhere reports back
|
||||
- Protocol tests live here and need no interpreter; they are the conformance suite every host adapter inherits. An adapter's own tests cover projection and execution only
|
||||
- This crate may depend on `litellm-host`, but never on `litellm-host-python`, PyO3 or a runtime registry
|
||||
- `golden/v1` pins every serialized envelope shape; every `EventKind` has a golden file by construction, and Python's `EVENTS` and `SUPPORTED_SCHEMAS` are tested against those files
|
||||
- Terms (shared ones are in `host-python/AGENTS.md`)
|
||||
|
|
@ -7,10 +12,14 @@
|
|||
- Handler: one callable of a subscriber; an observer receives envelopes and cannot affect the call; an interceptor receives `RequestFacts` and returns a patch
|
||||
- Handler selection (`select_handler`): an async call prefers the async handler and falls back to the sync one; a sync call never selects an async handler
|
||||
- Event: one thing that happened in a call; `EventKind` is its dotted name (`call.started`, `request.sending`, `response.received`, `call.succeeded`, `call.failed`), the serde tag and the subscription name at once
|
||||
- Envelope: schema + `call_id` + `call_type` + `seq` + event; the only value an observer receives; `Sequencer` numbers a call's envelopes from zero
|
||||
- Envelope: schema + `call_id` + `call_type` + `seq` + event; the only value an observer receives; the session numbers a call's envelopes from zero
|
||||
- Facts (`CallFacts`, `RequestFacts`, `TimingFacts`, `ErrorFacts`): a redacted, serializable projection of runtime state; never a live object
|
||||
- Patch (`WirePatch`): an interceptor's edit of the wire: header `remove`, then header `set`, optional `body` replacement; `apply` validates it
|
||||
- Redaction: credential headers by name (`CREDENTIAL_HEADERS`) and optional params by the route's secret fields become `[REDACTED]`; shape never changes
|
||||
- Observers are asynchronous to the call. This is contract semantics, not a property of one host
|
||||
- An envelope may reach an observer after the call has moved on or returned. Envelopes of one call arrive in `seq` order; nothing orders two calls. Delivery is at most once
|
||||
- Running every observer inline before the step returns, as the in-process CPython host does, is one legal schedule and promises nothing a callback may rely on. A host that queues, batches or crosses a process boundary is equally conforming
|
||||
- Nothing an observer does or returns has a path back into the call. Interceptors are the opposite: on the call path by nature, answered before the request leaves
|
||||
- No runtime object of a call reaches a callback contract value
|
||||
- `RequestFacts::new` is the only request projection and always applies the contract's credential redaction; its fields are private so no other crate can build one
|
||||
- A patch cannot change the URL or set or remove a credential header
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
//! redacted, serializable projection that never holds a runtime object of the call.
|
||||
|
||||
use litellm_host::event::{FailureOrigin, RequestContext, Timing, WireRequest};
|
||||
use serde::Serialize;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
use strum::{EnumString, IntoStaticStr, VariantArray};
|
||||
|
||||
|
|
@ -57,17 +57,24 @@ pub enum Event {
|
|||
PartialOrd,
|
||||
Ord,
|
||||
Debug,
|
||||
Serialize,
|
||||
Deserialize,
|
||||
)]
|
||||
pub enum EventKind {
|
||||
#[strum(serialize = "call.started")]
|
||||
#[serde(rename = "call.started")]
|
||||
CallStarted,
|
||||
#[strum(serialize = "request.sending")]
|
||||
#[serde(rename = "request.sending")]
|
||||
RequestSending,
|
||||
#[strum(serialize = "response.received")]
|
||||
#[serde(rename = "response.received")]
|
||||
ResponseReceived,
|
||||
#[strum(serialize = "call.succeeded")]
|
||||
#[serde(rename = "call.succeeded")]
|
||||
CallSucceeded,
|
||||
#[strum(serialize = "call.failed")]
|
||||
#[serde(rename = "call.failed")]
|
||||
CallFailed,
|
||||
}
|
||||
|
||||
|
|
@ -156,14 +163,14 @@ impl Event {
|
|||
}
|
||||
}
|
||||
|
||||
pub struct Sequencer {
|
||||
pub(crate) struct Sequencer {
|
||||
call_id: String,
|
||||
call_type: &'static str,
|
||||
next: u32,
|
||||
}
|
||||
|
||||
impl Sequencer {
|
||||
pub fn new(call_id: String, call_type: &'static str) -> Self {
|
||||
pub(crate) fn new(call_id: String, call_type: &'static str) -> Self {
|
||||
Self {
|
||||
call_id,
|
||||
call_type,
|
||||
|
|
@ -171,7 +178,7 @@ impl Sequencer {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn envelope(&mut self, event: Event) -> Envelope {
|
||||
pub(crate) fn envelope(&mut self, event: Event) -> Envelope {
|
||||
let envelope = Envelope {
|
||||
schema: SCHEMA_V1,
|
||||
call_id: self.call_id.clone(),
|
||||
|
|
|
|||
|
|
@ -1,15 +1,17 @@
|
|||
//! The runtime-neutral v1 callback contract: envelopes and their sequencing, credential
|
||||
//! redaction, handler selection and validated wire patches. No PyO3 and no registry; a
|
||||
//! language host marshals these values to its own subscribers.
|
||||
//! The runtime-neutral v1 callback contract: envelopes, credential redaction, handler
|
||||
//! selection, validated wire patches, and the per-call [`CallSession`] that says which
|
||||
//! envelope goes to whom and folds interceptor patches. No PyO3, no registry and no I/O; a
|
||||
//! host projects its own values into facts and runs its own subscribers.
|
||||
|
||||
mod envelope;
|
||||
mod patch;
|
||||
mod policy;
|
||||
mod redact;
|
||||
mod session;
|
||||
|
||||
pub use envelope::{
|
||||
CallFacts, Envelope, ErrorFacts, Event, EventKind, Origin, RequestFacts, SCHEMA_V1, Sequencer,
|
||||
TimingFacts,
|
||||
CallFacts, Envelope, ErrorFacts, Event, EventKind, Origin, RequestFacts, SCHEMA_V1, TimingFacts,
|
||||
};
|
||||
pub use patch::{HeaderPatch, PatchError, WirePatch, apply};
|
||||
pub use policy::{ExecutionMode, HandlerSelection, select_handler};
|
||||
pub use session::{CallSession, Emission, Interception, Subscription, Turn};
|
||||
|
|
|
|||
374
litellm-rust/crates/callbacks-v1/src/session.rs
Normal file
374
litellm-rust/crates/callbacks-v1/src/session.rs
Normal file
|
|
@ -0,0 +1,374 @@
|
|||
//! One call's v1 protocol, with no I/O and no runtime: which envelope each moment of the
|
||||
//! call produces, who receives it, and how interceptor patches fold into the wire
|
||||
//! request. A host projects its own values into facts, asks the session what to deliver,
|
||||
//! and runs the handlers however it can: inline in CPython, or across a process boundary.
|
||||
//!
|
||||
//! The two planes are visible in the types. An [`Emission`] has no way back: nothing an
|
||||
//! observer does or returns re-enters the session. An [`Interception`] is answered, one
|
||||
//! patch per interceptor, before the request leaves.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use litellm_host::event::{FailureOrigin, RequestContext, Timing, WireRequest};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
envelope::{CallFacts, Envelope, ErrorFacts, Event, EventKind, RequestFacts, Sequencer},
|
||||
patch::{PatchError, WirePatch, apply},
|
||||
};
|
||||
|
||||
/// What one subscriber asked for, as data: no handler, no runtime object. It is what a
|
||||
/// host that runs handlers elsewhere reports back once it has loaded them.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Subscription {
|
||||
pub name: String,
|
||||
pub events: BTreeSet<EventKind>,
|
||||
pub observes: bool,
|
||||
pub intercepts: bool,
|
||||
}
|
||||
|
||||
/// One envelope and the subscribers it goes to, as indexes into the session's
|
||||
/// subscriptions, in registration order.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Emission {
|
||||
pub envelope: Envelope,
|
||||
pub observers: Vec<usize>,
|
||||
}
|
||||
|
||||
/// An interceptor's turn: who is asked, and the request as the earlier ones left it.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Turn {
|
||||
pub subscriber: usize,
|
||||
pub request: RequestFacts,
|
||||
}
|
||||
|
||||
/// The wire request on its way through the interceptors, in registration order.
|
||||
#[derive(Debug)]
|
||||
pub struct Interception {
|
||||
wire: WireRequest,
|
||||
context: RequestContext,
|
||||
interceptors: Vec<usize>,
|
||||
position: usize,
|
||||
}
|
||||
|
||||
pub struct CallSession {
|
||||
sequencer: Sequencer,
|
||||
subscriptions: Vec<Subscription>,
|
||||
streamed: bool,
|
||||
}
|
||||
|
||||
impl Interception {
|
||||
/// The interceptor to ask next, or `None` once every one has answered.
|
||||
pub fn turn(&self) -> Option<Turn> {
|
||||
self.interceptors
|
||||
.get(self.position)
|
||||
.map(|&subscriber| Turn {
|
||||
subscriber,
|
||||
request: RequestFacts::new(&self.wire, &self.context),
|
||||
})
|
||||
}
|
||||
|
||||
/// The current interceptor's answer; an invalid patch ends the interception.
|
||||
pub fn patched(self, patch: WirePatch) -> Result<Self, PatchError> {
|
||||
Ok(Self {
|
||||
wire: apply(self.wire, patch)?,
|
||||
position: self.position + 1,
|
||||
..self
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl CallSession {
|
||||
/// Opens the call. `call.started` is always envelope zero.
|
||||
pub fn start(
|
||||
call_id: String,
|
||||
call_type: &'static str,
|
||||
subscriptions: Vec<Subscription>,
|
||||
facts: CallFacts,
|
||||
) -> (Self, Emission) {
|
||||
let mut session = Self {
|
||||
sequencer: Sequencer::new(call_id, call_type),
|
||||
subscriptions,
|
||||
streamed: false,
|
||||
};
|
||||
let emission = session.emission(Event::CallStarted(facts));
|
||||
(session, emission)
|
||||
}
|
||||
|
||||
fn emission(&mut self, event: Event) -> Emission {
|
||||
let kind = event.kind();
|
||||
Emission {
|
||||
envelope: self.sequencer.envelope(event),
|
||||
observers: self
|
||||
.subscriptions
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, subscription)| {
|
||||
subscription.observes && subscription.events.contains(&kind)
|
||||
})
|
||||
.map(|(index, _)| index)
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn interception(&self, wire: WireRequest, context: RequestContext) -> Interception {
|
||||
Interception {
|
||||
wire,
|
||||
context,
|
||||
interceptors: self
|
||||
.subscriptions
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, subscription)| subscription.intercepts)
|
||||
.map(|(index, _)| index)
|
||||
.collect(),
|
||||
position: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Closes an interception: `request.sending` describes the wire as the last
|
||||
/// interceptor left it, which is the wire the host must send.
|
||||
pub fn request_sending(&mut self, interception: Interception) -> (Emission, WireRequest) {
|
||||
let facts = RequestFacts::new(&interception.wire, &interception.context);
|
||||
(
|
||||
self.emission(Event::RequestSending(facts)),
|
||||
interception.wire,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn response_received(&mut self, body: String) -> Emission {
|
||||
self.emission(Event::ResponseReceived { body })
|
||||
}
|
||||
|
||||
/// The call's stream was handed to the caller; the terminal envelope says so.
|
||||
pub fn opened(&mut self) {
|
||||
self.streamed = true;
|
||||
}
|
||||
|
||||
/// A terminal envelope consumes the session, so a call has exactly one, last.
|
||||
pub fn succeeded(
|
||||
mut self,
|
||||
timing: Timing,
|
||||
response: Value,
|
||||
response_error: Option<String>,
|
||||
) -> Emission {
|
||||
let streamed = self.streamed;
|
||||
self.emission(Event::CallSucceeded {
|
||||
timing: timing.into(),
|
||||
streamed,
|
||||
response,
|
||||
response_error,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn failed(mut self, timing: Timing, origin: FailureOrigin, error: ErrorFacts) -> Emission {
|
||||
let streamed = self.streamed;
|
||||
self.emission(Event::CallFailed {
|
||||
timing: timing.into(),
|
||||
streamed,
|
||||
origin: origin.into(),
|
||||
error,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::{Map, json};
|
||||
use strum::VariantArray;
|
||||
|
||||
use super::*;
|
||||
|
||||
const TIMING: Timing = Timing {
|
||||
start_time: 1.0,
|
||||
end_time: 2.0,
|
||||
};
|
||||
|
||||
fn subscription(name: &str, events: &[EventKind], intercepts: bool) -> Subscription {
|
||||
Subscription {
|
||||
name: name.to_string(),
|
||||
events: events.iter().copied().collect(),
|
||||
observes: !events.is_empty(),
|
||||
intercepts,
|
||||
}
|
||||
}
|
||||
|
||||
fn facts() -> CallFacts {
|
||||
CallFacts {
|
||||
start_time: 1.0,
|
||||
asynchronous: true,
|
||||
metadata: Map::new(),
|
||||
metadata_dropped: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn wire() -> WireRequest {
|
||||
WireRequest {
|
||||
url: "https://provider.example".to_string(),
|
||||
headers: Vec::new(),
|
||||
body: json!({"input": "original"}),
|
||||
}
|
||||
}
|
||||
|
||||
fn context() -> RequestContext {
|
||||
RequestContext {
|
||||
model: "model".to_string(),
|
||||
custom_llm_provider: "provider".to_string(),
|
||||
optional_params: json!({}),
|
||||
secret_fields: Vec::new(),
|
||||
api_key: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn body(value: Value) -> WirePatch {
|
||||
serde_json::from_value(json!({"body": value})).unwrap()
|
||||
}
|
||||
|
||||
fn seen_body(turn: &Turn) -> Value {
|
||||
serde_json::to_value(&turn.request).unwrap()["body"].clone()
|
||||
}
|
||||
|
||||
fn error() -> ErrorFacts {
|
||||
ErrorFacts {
|
||||
class: "builtins.ValueError".to_string(),
|
||||
message: "bad".to_string(),
|
||||
status_code: Some(400),
|
||||
}
|
||||
}
|
||||
|
||||
fn start(subscriptions: Vec<Subscription>) -> (CallSession, Emission) {
|
||||
CallSession::start("call".to_string(), "ocr", subscriptions, facts())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_call_numbers_its_envelopes_from_the_start_to_one_terminal() {
|
||||
let (mut session, started) = start(vec![subscription("all", EventKind::VARIANTS, false)]);
|
||||
let (sending, _) = {
|
||||
let interception = session.interception(wire(), context());
|
||||
session.request_sending(interception)
|
||||
};
|
||||
let received = session.response_received("{}".to_string());
|
||||
let terminal = session.succeeded(TIMING, json!({"id": "response"}), None);
|
||||
|
||||
let seen: Vec<(u32, EventKind)> = [started, sending, received, terminal]
|
||||
.iter()
|
||||
.map(|emission| (emission.envelope.seq, emission.envelope.event.kind()))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
seen,
|
||||
vec![
|
||||
(0, EventKind::CallStarted),
|
||||
(1, EventKind::RequestSending),
|
||||
(2, EventKind::ResponseReceived),
|
||||
(3, EventKind::CallSucceeded),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_envelope_goes_only_to_observers_subscribed_to_its_kind_in_registration_order() {
|
||||
let (session, started) = start(vec![
|
||||
subscription("terminal", &[EventKind::CallFailed], false),
|
||||
subscription("interceptor", &[], true),
|
||||
subscription(
|
||||
"both",
|
||||
&[EventKind::CallStarted, EventKind::CallFailed],
|
||||
false,
|
||||
),
|
||||
subscription("started", &[EventKind::CallStarted], true),
|
||||
]);
|
||||
|
||||
assert_eq!(started.observers, vec![2, 3]);
|
||||
assert_eq!(
|
||||
session
|
||||
.failed(TIMING, FailureOrigin::Call, error())
|
||||
.observers,
|
||||
vec![0, 2]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_interceptor_sees_the_request_as_the_earlier_ones_left_it() {
|
||||
let (mut session, _) = start(vec![
|
||||
subscription("first", &[], true),
|
||||
subscription("observer", &[EventKind::RequestSending], false),
|
||||
subscription("second", &[], true),
|
||||
]);
|
||||
let interception = session.interception(wire(), context());
|
||||
|
||||
let first = interception.turn().unwrap();
|
||||
assert_eq!(
|
||||
(first.subscriber, seen_body(&first)),
|
||||
(0, json!({"input": "original"}))
|
||||
);
|
||||
let interception = interception
|
||||
.patched(body(json!({"input": "first"})))
|
||||
.unwrap();
|
||||
|
||||
let second = interception.turn().unwrap();
|
||||
assert_eq!(
|
||||
(second.subscriber, seen_body(&second)),
|
||||
(2, json!({"input": "first"}))
|
||||
);
|
||||
let interception = interception.patched(WirePatch::default()).unwrap();
|
||||
assert_eq!(interception.turn(), None);
|
||||
|
||||
let (sending, sent) = session.request_sending(interception);
|
||||
assert_eq!(sent.body, json!({"input": "first"}));
|
||||
assert_eq!(sending.observers, vec![1]);
|
||||
assert_eq!(
|
||||
serde_json::to_value(&sending.envelope).unwrap()["event"]["body"],
|
||||
sent.body
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_invalid_patch_ends_the_interception() {
|
||||
let (session, _) = start(vec![subscription("interceptor", &[], true)]);
|
||||
let patch: WirePatch =
|
||||
serde_json::from_value(json!({"headers": {"set": [["Authorization", "stolen"]]}}))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
session
|
||||
.interception(wire(), context())
|
||||
.patched(patch)
|
||||
.unwrap_err(),
|
||||
PatchError::ProtectedHeader("Authorization".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_terminal_envelope_says_whether_the_call_streamed() {
|
||||
let (mut streamed, _) = start(Vec::new());
|
||||
streamed.opened();
|
||||
let (plain, _) = start(Vec::new());
|
||||
|
||||
for (session, expected) in [(streamed, true), (plain, false)] {
|
||||
let Event::CallFailed { streamed, .. } = session
|
||||
.failed(TIMING, FailureOrigin::Host, error())
|
||||
.envelope
|
||||
.event
|
||||
else {
|
||||
panic!("call.failed expected");
|
||||
};
|
||||
assert_eq!(streamed, expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_subscription_is_plain_data_a_remote_host_can_report() {
|
||||
let reported = subscription("remote", &[EventKind::CallSucceeded], true);
|
||||
let wire = serde_json::to_value(&reported).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
wire,
|
||||
json!({"name": "remote", "events": ["call.succeeded"], "observes": true, "intercepts": true})
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<Subscription>(wire).unwrap(),
|
||||
reported
|
||||
);
|
||||
}
|
||||
}
|
||||
171
litellm/callbacks_v1/MIGRATION.md
Normal file
171
litellm/callbacks_v1/MIGRATION.md
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
# Migrating built-in callbacks to the v1 contract
|
||||
|
||||
How a built-in logger moves from `CustomLogger` (`litellm/integrations/`) to the v1
|
||||
contract (`litellm/callbacks_v1/__init__.py`), and what the scaffolding in `builtin/` is
|
||||
for while that happens. The rules for writing one port are in `builtin/AGENTS.md`; this
|
||||
file is the flow around them.
|
||||
|
||||
## The shape of the migration
|
||||
|
||||
- Ports live in a parallel tree, `builtin/<name>.py`, one frozen class each declaring
|
||||
`SinkPort` or `InterceptorPort` from `builtin/port.py` as its base. The legacy logger
|
||||
is never edited, renamed, or taught about its port.
|
||||
- Everything the migration needs to say is in `builtin/manifest.py`: `Gap`, each port's
|
||||
registration name, its twin, its status and its gaps. A port carries none of it, so
|
||||
cutting over deletes that file and leaves the ports untouched.
|
||||
- A port is inert. Importing it registers nothing and the product never registers it;
|
||||
only tests do. Writing a port therefore cannot change behaviour for anyone.
|
||||
- A port goes live by removing its legacy twin. The two are never active on the same
|
||||
call, so there is no dual delivery, no ownership table, and no per-call skip logic.
|
||||
- v1 envelopes only exist on native (Rust) routes. A port can only replace its twin once
|
||||
every route the twin serves emits them.
|
||||
- Built-in and custom callbacks share one interface, the v1 contract. In the SDK a custom
|
||||
callback is free-form Python, in process, never sandboxed. On the gateway it runs in an
|
||||
out-of-process extension host driven by the same `CallSession` (`litellm-rust/crates/
|
||||
callbacks-v1`). A built-in port is pure methods over values with every effect in
|
||||
`builtin/runtime.py`, which also owns the two subscribers (`Sink`, `Patcher`); that is
|
||||
hygiene for built-ins (secrets, a later Rust port), not the isolation mechanism.
|
||||
|
||||
Porting comes first, and on purpose, because its first product is not the port. It is the
|
||||
list of things the contract cannot do yet. The contract grows from what real built-ins
|
||||
needed, not from guesses.
|
||||
|
||||
## The loop
|
||||
|
||||
```
|
||||
port a built-in ──► record gaps ──► read the backlog ──► grow the contract
|
||||
▲ │
|
||||
└──────── delete gaps and their allowances ◄─────────────┘
|
||||
```
|
||||
|
||||
### 1. Port inertly
|
||||
|
||||
Write `payload()` against the golden envelopes. Every time the twin reads something the
|
||||
envelopes do not carry, do one of two things and write a `Gap` either way:
|
||||
|
||||
- leave the field out of the vendor body, or
|
||||
- let `builtin/runtime.py` work around it for every port at once (`CallJoin` recovers the
|
||||
model for the terminal envelope; `Outbox` keeps vendor I/O off the call path). A
|
||||
workaround never goes inside a port: a port that needs a thread, a socket or state has
|
||||
found a `capability` gap.
|
||||
|
||||
A `Gap` is handwritten, in `manifest.py` beside the port's entry. It has a `kind`
|
||||
(`fact`, `event`, `capability`), a `key`, the `legacy` input the twin read, and a `note`. The `key` names what the contract would have
|
||||
to grow (`cost`, `usage`, `identity`), not what the twin happened to call it, so that two
|
||||
ports missing the same thing say so in the same word.
|
||||
|
||||
Then write the parity test: one real native call, the twin and the port each posting to
|
||||
their own recording vendor, `assert_parity` on the two bodies. Every difference needs an
|
||||
`Allowed(path, reason)`. If the difference exists because of a gap, it cites the gap:
|
||||
`Allowed("data.cost", "no cost fact on call.succeeded", gap="cost")`. An `Allowed` with no
|
||||
`gap` is a difference the port means to keep (each side stamps its own clock, the port
|
||||
omits a key the twin sent as null).
|
||||
|
||||
Add the port to `manifest.py` as `exploring`, then `parity` once the parity test passes.
|
||||
|
||||
### 2. Read the backlog
|
||||
|
||||
```bash
|
||||
python -m litellm.callbacks_v1.builtin.manifest
|
||||
```
|
||||
|
||||
prints the ledger (every gap, per port) and the backlog (every gap key, with the ports
|
||||
waiting on it, most wanted first). A key several ports wait on is contract work. A key
|
||||
one port waits on is either contract work or a candidate for "v1 deliberately does not
|
||||
do this"; decide which before building it.
|
||||
|
||||
### 3. Grow the contract
|
||||
|
||||
The contract is defined in Rust (`litellm-rust/crates/callbacks-v1`, golden envelopes in
|
||||
`golden/v1/`) and mirrored by the types in `litellm/callbacks_v1/__init__.py`. Add the
|
||||
fact, event, or capability there, for every native route, and update the golden files.
|
||||
|
||||
### 4. Close the gap
|
||||
|
||||
For each port that listed the key:
|
||||
|
||||
1. Use the new fact in `payload()`, or drop the workaround.
|
||||
2. Delete the `Gap`.
|
||||
3. Run the parity tests. They now fail in exactly the places that need attention:
|
||||
- an `Allowed` still citing the deleted key fails with "cites gap ..., which the port
|
||||
does not list";
|
||||
- an `Allowed` whose difference is gone fails as stale;
|
||||
- a difference the new fact did not actually fix fails as unexplained once its
|
||||
`Allowed` is removed.
|
||||
4. Delete those `Allowed` entries until the test is green again. Do not re-allow a
|
||||
difference to get there.
|
||||
|
||||
The order of 1 and 2 does not matter; the tests fail until both are done.
|
||||
|
||||
### 5. Cut over
|
||||
|
||||
A port is `ready` when its parity test passes, its manifest gaps are empty, and every route its twin
|
||||
serves emits v1 envelopes. `test_conventions.py` refuses `ready` with an open gap. A
|
||||
`ready` port replaces its twin: register the port where the twin was registered, remove
|
||||
the twin, in one change.
|
||||
|
||||
## What is checked and what is trusted
|
||||
|
||||
| Claim | Checked by |
|
||||
|---|---|
|
||||
| Every difference from the twin is explained | `assert_parity`: an uncovered difference fails |
|
||||
| The allow-list does not outlive the differences | `assert_parity`: a stale `Allowed` fails |
|
||||
| The allow-list does not outlive the gaps | `assert_parity`: an `Allowed` citing an unlisted key fails |
|
||||
| A port is not `ready` with open gaps | `test_conventions.py` |
|
||||
| The gap list is complete | nobody |
|
||||
|
||||
The last row is the honest one. The gap list is only as complete as its author. For a
|
||||
sink, the parity diff is the real source of truth and the gaps are its readable summary: anything the
|
||||
author forgot still shows up as an unexplained difference. Three kinds of gap never show
|
||||
up in a diff, and the handwritten entry is their only record:
|
||||
|
||||
- `capability` and `event` gaps (observers run inline; an interceptor cannot see
|
||||
`call_type`), because they are about how the port runs, not what it sends;
|
||||
- gaps the port works around (`terminal_model`), because the workaround makes the bodies
|
||||
equal;
|
||||
- gaps the parity call does not exercise (`request_user`, when the test sends no `user`).
|
||||
|
||||
Closing one of these is a manual edit with no failing test to prompt it. When the contract
|
||||
change lands, grep the key.
|
||||
|
||||
## Open decisions
|
||||
|
||||
Porting does not settle these, and no port may settle one by accident. Each is kept out of
|
||||
the ports so that it can be decided once, in the host.
|
||||
|
||||
- **Where `Config` comes from.** A port takes a finished `Config`. Whether the host
|
||||
resolves it from the twin's environment variables, from config.yaml, or through the
|
||||
Rust settings precedence is undecided; `from_env` on a port only records the names.
|
||||
- **The gateway's extension host.** Custom Python callbacks run out of process there:
|
||||
the worker's confinement profile, the interceptor timeout and failure policy, and
|
||||
whether built-ins run in a trusted worker or move to Rust. The interface is the v1
|
||||
Protocol itself over a socket, not the ports' pure-method shape.
|
||||
- **Who holds secrets.** A `Config` carries the vendor's API key and a `Delivery` carries
|
||||
it as a header, so a port sees the credential. A sandboxed callback should name a secret
|
||||
and let the host put it on the wire. `off_path_delivery` is the gap that would carry it.
|
||||
|
||||
## Differences that are never closed
|
||||
|
||||
Some legacy inputs will not come to v1 (`hidden_params` is the likely example). Those are
|
||||
not gaps; a gap is something the contract intends to grow. When that decision is made for
|
||||
a key, delete the `Gap`, drop `gap=` from its `Allowed` entries and reword their reasons
|
||||
as the decision, and move the feature to the port's `Not ported yet:` docstring line or to
|
||||
the release notes. A port must not sit at `parity` forever behind a gap nobody will close.
|
||||
|
||||
`parity` means every difference is explained. It does not mean the port is equivalent:
|
||||
`generic_api` is at `parity` while shipping about a fifth of `StandardLoggingPayload`.
|
||||
Only `ready` means equivalent.
|
||||
|
||||
## What survives the migration
|
||||
|
||||
| Piece | After every twin is gone |
|
||||
|---|---|
|
||||
| The ports, `SinkPort`, `InterceptorPort`, `Delivery`, `CallRecord`, `Batching` | stay; this is the product and its interface |
|
||||
| `runtime.py`: `Sink`, `Outbox`, `Transport` / `HttpxTransport` | stays until the contract delivers off the call path itself (`off_path_delivery`); then the host owns it |
|
||||
| `CallJoin` | stays only if the terminal envelope never carries model and metadata; it is a workaround for the `terminal_model` gap |
|
||||
| `manifest.py` entire: `Gap`, every gap list, the names, `ledger()`, `backlog()`, the statuses, `Entry.legacy` | deleted with the last twin |
|
||||
| The twin checks in `test_conventions.py` | deleted with the last twin |
|
||||
| `assert_parity`, `Allowed`, `test_parity.py` | deleted; there is no twin left to compare with. The payload tests on golden envelopes remain as the ports' tests |
|
||||
|
||||
At that point `port.py` holds the two protocols and what they are written in, which is the
|
||||
whole of what a callback author, built-in or custom, needs besides the contract types.
|
||||
419
litellm/callbacks_v1/__init__.py
Normal file
419
litellm/callbacks_v1/__init__.py
Normal file
|
|
@ -0,0 +1,419 @@
|
|||
"""The v1 callback contract as a callback author sees it.
|
||||
|
||||
A registered callback observes a call through envelopes it cannot influence, and may
|
||||
influence one only by returning a patch that Rust validates. The schema itself is owned
|
||||
by `litellm-rust/crates/callbacks-v1`; this module is its typed Python face and the
|
||||
process-wide registry. `litellm.rust_bridge.callbacks_v1` is the marshalling the
|
||||
native call reaches for, and is not part of this surface.
|
||||
|
||||
Two ways to register. A single function takes a decorator, which derives the name and the
|
||||
handler slot for you and returns the function unchanged:
|
||||
|
||||
@on_event("call.succeeded", "call.failed")
|
||||
def record(event: EnvelopeV1) -> None: ...
|
||||
|
||||
@before_send
|
||||
async def add_header(request: RequestFactsV1) -> WirePatchV1 | None: ...
|
||||
|
||||
An object with several handlers, or with state, declares them by the protocol names and
|
||||
goes through `register`.
|
||||
|
||||
Every rejectable mistake is rejected at registration, never during a call, and each static
|
||||
rule here mirrors one `RegistrationError`: the handler protocols mirror `MissingHandler`
|
||||
and `HandlerKindMismatch`, and `EventName` mirrors `UnknownEvent`. Annotate `events` with
|
||||
`frozenset[EventName]` on the object form to get that check before import time; the
|
||||
decorator infers it from its arguments.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
import threading
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Literal, Protocol, TypeAlias, TypeVar
|
||||
|
||||
from typing_extensions import ReadOnly, TypeAliasType, TypedDict
|
||||
|
||||
__all__ = (
|
||||
"EVENTS",
|
||||
"SCHEMA",
|
||||
"SUPPORTED_SCHEMAS",
|
||||
"AsyncInterceptor",
|
||||
"AsyncObserver",
|
||||
"CallFactsV1",
|
||||
"CallFailedV1",
|
||||
"CallStartedV1",
|
||||
"CallSucceededV1",
|
||||
"Callback",
|
||||
"CallbackBase",
|
||||
"DuplicateName",
|
||||
"EmptyObserverEvents",
|
||||
"EnvelopeV1",
|
||||
"ErrorFactsV1",
|
||||
"EventName",
|
||||
"EventV1",
|
||||
"HandlerKindMismatch",
|
||||
"HeaderPatchV1",
|
||||
"Interceptor",
|
||||
"InterceptorFn",
|
||||
"JSONValue",
|
||||
"MissingHandler",
|
||||
"Observer",
|
||||
"ObserverFn",
|
||||
"RegistrationError",
|
||||
"RequestFactsV1",
|
||||
"RequestSendingV1",
|
||||
"ResponseReceivedV1",
|
||||
"SchemaVersion",
|
||||
"Subscriber",
|
||||
"TimingFactsV1",
|
||||
"UnknownEvent",
|
||||
"UnsupportedSchema",
|
||||
"WirePatchV1",
|
||||
"before_send",
|
||||
"on_event",
|
||||
"register",
|
||||
"snapshot",
|
||||
"unregister",
|
||||
)
|
||||
|
||||
JSONScalar: TypeAlias = None | bool | int | float | str
|
||||
JSONValue = TypeAliasType( # rebind-ok: named recursive alias required for runtime validation
|
||||
"JSONValue", JSONScalar | Sequence["JSONValue"] | Mapping[str, "JSONValue"]
|
||||
)
|
||||
|
||||
EventName: TypeAlias = Literal[
|
||||
"call.started",
|
||||
"request.sending",
|
||||
"response.received",
|
||||
"call.succeeded",
|
||||
"call.failed",
|
||||
]
|
||||
SchemaVersion: TypeAlias = Literal[1]
|
||||
|
||||
SCHEMA: Final[SchemaVersion] = 1
|
||||
SUPPORTED_SCHEMAS: Final[frozenset[SchemaVersion]] = frozenset({SCHEMA})
|
||||
EVENTS: Final[frozenset[EventName]] = frozenset(
|
||||
{
|
||||
"call.started",
|
||||
"request.sending",
|
||||
"response.received",
|
||||
"call.succeeded",
|
||||
"call.failed",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class TimingFactsV1(TypedDict):
|
||||
start_time: ReadOnly[float]
|
||||
end_time: ReadOnly[float]
|
||||
duration: ReadOnly[float]
|
||||
|
||||
|
||||
class CallFactsV1(TypedDict):
|
||||
start_time: ReadOnly[float]
|
||||
asynchronous: ReadOnly[bool]
|
||||
metadata: ReadOnly[Mapping[str, JSONValue]]
|
||||
metadata_dropped: ReadOnly[Sequence[str]]
|
||||
|
||||
|
||||
class RequestFactsV1(TypedDict):
|
||||
model: ReadOnly[str]
|
||||
custom_llm_provider: ReadOnly[str]
|
||||
optional_params: ReadOnly[JSONValue]
|
||||
url: ReadOnly[str]
|
||||
headers: ReadOnly[Sequence[tuple[str, str]]]
|
||||
body: ReadOnly[JSONValue]
|
||||
|
||||
|
||||
ErrorFactsV1 = TypedDict( # rebind-ok: runtime TypedDict type for the reserved class field
|
||||
"ErrorFactsV1",
|
||||
{ # mutable-ok: functional TypedDict syntax requires a dict for the reserved class field
|
||||
"class": ReadOnly[str],
|
||||
"message": ReadOnly[str],
|
||||
"status_code": ReadOnly[int | None],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class CallStartedV1(CallFactsV1):
|
||||
type: ReadOnly[Literal["call.started"]]
|
||||
|
||||
|
||||
class RequestSendingV1(RequestFactsV1):
|
||||
type: ReadOnly[Literal["request.sending"]]
|
||||
|
||||
|
||||
class ResponseReceivedV1(TypedDict):
|
||||
type: ReadOnly[Literal["response.received"]]
|
||||
body: ReadOnly[str]
|
||||
|
||||
|
||||
class CallSucceededV1(TypedDict):
|
||||
type: ReadOnly[Literal["call.succeeded"]]
|
||||
timing: ReadOnly[TimingFactsV1]
|
||||
streamed: ReadOnly[bool]
|
||||
response: ReadOnly[JSONValue]
|
||||
response_error: ReadOnly[str | None]
|
||||
|
||||
|
||||
class CallFailedV1(TypedDict):
|
||||
type: ReadOnly[Literal["call.failed"]]
|
||||
timing: ReadOnly[TimingFactsV1]
|
||||
streamed: ReadOnly[bool]
|
||||
origin: ReadOnly[Literal["call", "host"]]
|
||||
error: ReadOnly[ErrorFactsV1]
|
||||
|
||||
|
||||
EventV1: TypeAlias = CallStartedV1 | RequestSendingV1 | ResponseReceivedV1 | CallSucceededV1 | CallFailedV1
|
||||
|
||||
|
||||
class EnvelopeV1(TypedDict):
|
||||
schema: ReadOnly[Literal[1]]
|
||||
call_id: ReadOnly[str]
|
||||
call_type: ReadOnly[str]
|
||||
seq: ReadOnly[int]
|
||||
event: ReadOnly[EventV1]
|
||||
|
||||
|
||||
class HeaderPatchV1(TypedDict, total=False):
|
||||
remove: ReadOnly[Sequence[str]]
|
||||
set: ReadOnly[Sequence[tuple[str, str]]]
|
||||
|
||||
|
||||
class WirePatchV1(TypedDict, total=False):
|
||||
headers: ReadOnly[HeaderPatchV1]
|
||||
body: ReadOnly[JSONValue]
|
||||
|
||||
|
||||
class CallbackBase(Protocol):
|
||||
"""What every callback declares, whatever it handles.
|
||||
|
||||
`schema` is not here: the module you imported from already fixes it. Declaring one
|
||||
anyway is still honoured, so a callback written against another version is rejected
|
||||
by `register` rather than silently reinterpreted.
|
||||
"""
|
||||
|
||||
name: str
|
||||
events: frozenset[EventName]
|
||||
|
||||
|
||||
class Observer(CallbackBase, Protocol):
|
||||
"""Receives envelopes on a sync call; its errors are reported and swallowed.
|
||||
|
||||
Observers are asynchronous to the call: an envelope may arrive after the call has
|
||||
moved on or returned, in `seq` order within one call and in no order across calls, at
|
||||
most once. This host happens to run observers inline; do not rely on it.
|
||||
"""
|
||||
|
||||
def on_event(self, event: EnvelopeV1) -> None: ...
|
||||
|
||||
|
||||
class AsyncObserver(CallbackBase, Protocol):
|
||||
"""Receives envelopes on an async call; its errors are reported and swallowed.
|
||||
Asynchronous to the call in the same sense as `Observer`."""
|
||||
|
||||
async def async_on_event(self, event: EnvelopeV1) -> None: ...
|
||||
|
||||
|
||||
class Interceptor(CallbackBase, Protocol):
|
||||
"""Patches the sync request before it is sent; its failure fails the call."""
|
||||
|
||||
def before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ...
|
||||
|
||||
|
||||
class AsyncInterceptor(CallbackBase, Protocol):
|
||||
"""Patches the async request before it is sent; its failure fails the call."""
|
||||
|
||||
async def async_before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ...
|
||||
|
||||
|
||||
Callback: TypeAlias = Observer | AsyncObserver | Interceptor | AsyncInterceptor
|
||||
|
||||
# The decorators return the function unchanged, so each is generic over the exact function
|
||||
# it decorated; without that, the decorated name would widen to the union below and stop
|
||||
# being directly callable at its own signature.
|
||||
_SyncObserverFn: TypeAlias = Callable[[EnvelopeV1], None] # mutable-ok: Callable's bracketed parameter list is call syntax, not a list literal
|
||||
_AsyncObserverFn: TypeAlias = Callable[[EnvelopeV1], Awaitable[None]] # mutable-ok: Callable's bracketed parameter list is call syntax, not a list literal
|
||||
_SyncInterceptorFn: TypeAlias = Callable[[RequestFactsV1], WirePatchV1 | None] # mutable-ok: Callable's bracketed parameter list is call syntax, not a list literal
|
||||
_AsyncInterceptorFn: TypeAlias = Callable[[RequestFactsV1], Awaitable[WirePatchV1 | None]] # mutable-ok: Callable's bracketed parameter list is call syntax, not a list literal
|
||||
|
||||
ObserverFn: TypeAlias = _SyncObserverFn | _AsyncObserverFn
|
||||
InterceptorFn: TypeAlias = _SyncInterceptorFn | _AsyncInterceptorFn
|
||||
ObserverFnT = TypeVar("ObserverFnT", bound=ObserverFn) # rebind-ok: a TypeVar must bind to a bare name to be recognised as one
|
||||
InterceptorFnT = TypeVar("InterceptorFnT", bound=InterceptorFn) # rebind-ok: a TypeVar must bind to a bare name to be recognised as one
|
||||
|
||||
|
||||
class RegistrationError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class UnsupportedSchema(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
class UnknownEvent(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
class MissingHandler(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
class EmptyObserverEvents(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
class HandlerKindMismatch(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
class DuplicateName(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Subscriber:
|
||||
name: str
|
||||
schema: SchemaVersion
|
||||
events: frozenset[EventName]
|
||||
on_event: object | None
|
||||
async_on_event: object | None
|
||||
before_send: object | None
|
||||
async_before_send: object | None
|
||||
|
||||
|
||||
class _Registry:
|
||||
def __init__(self) -> None:
|
||||
self.subscribers: tuple[Subscriber, ...] = ()
|
||||
self.lock: Final = threading.Lock()
|
||||
|
||||
|
||||
_REGISTRY: Final = _Registry()
|
||||
|
||||
|
||||
def _handler(callback: Callback, name: str) -> object | None:
|
||||
return getattr(callback, name, None)
|
||||
|
||||
|
||||
def _validate_kind(name: str, handler: object | None, asynchronous: bool) -> None:
|
||||
if handler is None:
|
||||
return
|
||||
if not callable(handler):
|
||||
raise HandlerKindMismatch(f"{name} must be callable")
|
||||
is_async: Final = inspect.iscoroutinefunction(handler)
|
||||
if is_async != asynchronous:
|
||||
expected: Final = "async" if asynchronous else "sync"
|
||||
raise HandlerKindMismatch(f"{name} must be {expected}")
|
||||
|
||||
|
||||
def _validate_events(events: frozenset[EventName], observing: bool) -> None:
|
||||
unknown: Final = events - EVENTS
|
||||
if unknown:
|
||||
raise UnknownEvent(f"unknown callback events: {sorted(unknown)}")
|
||||
if observing and not events:
|
||||
raise EmptyObserverEvents("observer callback events cannot be empty")
|
||||
|
||||
|
||||
def _insert(subscriber: Subscriber) -> None:
|
||||
with _REGISTRY.lock:
|
||||
if any(existing.name == subscriber.name for existing in _REGISTRY.subscribers):
|
||||
raise DuplicateName(f"duplicate callback name: {subscriber.name}")
|
||||
_REGISTRY.subscribers = (*_REGISTRY.subscribers, subscriber)
|
||||
|
||||
|
||||
def register(callback: Callback) -> None:
|
||||
"""Registers an object whose handler methods are named as the protocols declare them."""
|
||||
schema: Final = getattr(callback, "schema", SCHEMA)
|
||||
if schema not in SUPPORTED_SCHEMAS:
|
||||
raise UnsupportedSchema(f"unsupported callback schema: {schema}")
|
||||
events: Final = frozenset(callback.events)
|
||||
|
||||
on_event: Final = _handler(callback, "on_event")
|
||||
async_on_event: Final = _handler(callback, "async_on_event")
|
||||
before_send: Final = _handler(callback, "before_send")
|
||||
async_before_send: Final = _handler(callback, "async_before_send")
|
||||
handlers: Final = (on_event, async_on_event, before_send, async_before_send)
|
||||
if all(handler is None for handler in handlers):
|
||||
raise MissingHandler("callback must define at least one handler")
|
||||
_validate_events(events, on_event is not None or async_on_event is not None)
|
||||
|
||||
_validate_kind("on_event", on_event, False)
|
||||
_validate_kind("async_on_event", async_on_event, True)
|
||||
_validate_kind("before_send", before_send, False)
|
||||
_validate_kind("async_before_send", async_before_send, True)
|
||||
|
||||
_insert(
|
||||
Subscriber(
|
||||
name=callback.name,
|
||||
schema=schema,
|
||||
events=events,
|
||||
on_event=on_event,
|
||||
async_on_event=async_on_event,
|
||||
before_send=before_send,
|
||||
async_before_send=async_before_send,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _derived_name(function: object) -> str:
|
||||
module: Final = getattr(function, "__module__", "") or ""
|
||||
qualified: Final = getattr(function, "__qualname__", None) or repr(function)
|
||||
return f"{module}.{qualified}" if module else str(qualified)
|
||||
|
||||
|
||||
def on_event(*events: EventName, name: str | None = None) -> Callable[[ObserverFnT], ObserverFnT]:
|
||||
"""Registers one function as an observer of `events`; `async def` selects the async slot.
|
||||
|
||||
The function is returned unchanged, so it stays directly callable and testable. Its
|
||||
registry name defaults to `module.qualname`, which is unique by construction.
|
||||
"""
|
||||
|
||||
def decorate(function: ObserverFnT) -> ObserverFnT:
|
||||
subscribed: Final = frozenset(events)
|
||||
_validate_events(subscribed, True)
|
||||
asynchronous: Final = inspect.iscoroutinefunction(function)
|
||||
_insert(
|
||||
Subscriber(
|
||||
name=name if name is not None else _derived_name(function),
|
||||
schema=SCHEMA,
|
||||
events=subscribed,
|
||||
on_event=None if asynchronous else function,
|
||||
async_on_event=function if asynchronous else None,
|
||||
before_send=None,
|
||||
async_before_send=None,
|
||||
)
|
||||
)
|
||||
return function
|
||||
|
||||
return decorate
|
||||
|
||||
|
||||
def before_send(function: InterceptorFnT) -> InterceptorFnT:
|
||||
"""Registers one function as an interceptor; `async def` selects the async slot.
|
||||
|
||||
An interceptor subscribes to no events, and its failure fails the call.
|
||||
"""
|
||||
asynchronous: Final = inspect.iscoroutinefunction(function)
|
||||
_insert(
|
||||
Subscriber(
|
||||
name=_derived_name(function),
|
||||
schema=SCHEMA,
|
||||
events=frozenset(),
|
||||
on_event=None,
|
||||
async_on_event=None,
|
||||
before_send=None if asynchronous else function,
|
||||
async_before_send=function if asynchronous else None,
|
||||
)
|
||||
)
|
||||
return function
|
||||
|
||||
|
||||
def unregister(callback: Callback | Subscriber | str) -> None:
|
||||
"""Drops the callback of this name, given the object, a `Subscriber`, or the name."""
|
||||
name: Final = callback if isinstance(callback, str) else callback.name
|
||||
with _REGISTRY.lock:
|
||||
_REGISTRY.subscribers = tuple(subscriber for subscriber in _REGISTRY.subscribers if subscriber.name != name)
|
||||
|
||||
|
||||
def snapshot() -> tuple[Subscriber, ...]:
|
||||
return _REGISTRY.subscribers
|
||||
44
litellm/callbacks_v1/builtin/AGENTS.md
Normal file
44
litellm/callbacks_v1/builtin/AGENTS.md
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
- Target invariants, not completion claims
|
||||
- This package holds built-in callbacks ported from `CustomLogger` to the v1 contract (`litellm/callbacks_v1/__init__.py`), and nothing else
|
||||
- Porting is exploration: its first product is the gap ledger (`python -m litellm.callbacks_v1.builtin.manifest`), which says what the contract has to grow
|
||||
- `../MIGRATION.md` is the whole flow from first port to removing the legacy twin; this file is the rules for writing one port
|
||||
- A port is never registered by the product while its legacy twin exists. Importing anything here registers nothing; only tests call `register`
|
||||
- A port goes live by removing or unregistering its legacy twin. The two are never active together, so there is no per-call skip logic anywhere
|
||||
- Never edit the legacy twin to make a port easier
|
||||
- A port is a frozen class of pure methods; the host does everything else
|
||||
- Built-in and custom callbacks share one interface, the v1 contract. This rule is hygiene for built-ins, not the sandbox: a built-in holds vendor secrets and is the likeliest to be ported to Rust, and a pure transform is what survives both. Custom Python callbacks stay free-form; on the gateway they are isolated by an out-of-process extension host, not by this rule
|
||||
- A port opens no socket, starts no thread, keeps no state between calls and reads neither the environment nor the clock. Whatever it needs arrives as a value: contract facts, its `Config`, `now`
|
||||
- `port.py` is the whole interface: the two protocols a port declares as its base (`SinkPort`, `InterceptorPort`) and what they are written in (`CallRecord`, `Delivery`, `Batching`). Nothing in it is migration scaffolding, and a port imports nothing else from this package
|
||||
- `runtime.py` is the in-process host: `Sink` and `Patcher`, the two subscribers, with `CallJoin`, `Transport`, the clock and `outbox.py` behind them. They are the only things here that implement the v1 contract, so a port never writes `name`, `events` or a handler. Each takes the registration name, because the name is the registration's and not the port's: one port configured twice is two subscribers. It stands in for what a host should own; when the contract grows off-path delivery and a joined call record, `runtime.py` is deleted and the ports do not change
|
||||
- `manifest.py` is the migration, all of it: `Gap`, the name a port registers under, its twin, its status and its gaps. At cutover it is deleted outright and no port changes
|
||||
- `test_conventions.py` holds a port's imports to a pure allow-list and checks that it declares the protocol of its kind. A port importing anything from `runtime.py` is a failure: wiring is the host's, and the tests do it
|
||||
- Terms
|
||||
- Legacy twin: the `CustomLogger` subclass (or function of one) a port replaces, named in the port's docstring and in `manifest.py`
|
||||
- Sink: a port that observes calls and tells a vendor (`SinkPort`, run by `runtime.Sink`). Interceptor: a port that patches the wire request (`InterceptorPort`, run by `runtime.Patcher`)
|
||||
- Gap: one legacy input the contract does not supply: a missing `fact`, a missing `event`, or a missing `capability`
|
||||
- Its `key` names what the contract would have to grow (`cost`, `usage`, `identity`), not what the twin read. Reuse an existing key when another port already waits on the same thing; the backlog counts ports per key
|
||||
- Parity: the port and its twin handled the same call and every difference in what they sent is `Allowed` with a reason
|
||||
- What can be ported: a twin that only overrides the success/failure/pre/post log hooks is a sink; one that only rewrites the outgoing request is an interceptor. A twin overriding any other hook (guardrail, routing, prompt management, agentic loop, stream rewriting, audit log) has no v1 shape and stays out; `test_conventions.py` enforces this from the twin's own class
|
||||
- Every port module has the same parts, in this order
|
||||
- Docstring: one line of what it is, `Legacy twin: <path>`, and `Not ported yet:` for legacy features left out on purpose
|
||||
- `Config`: a frozen dataclass of plain data. Where its values come from (environment, config.yaml, the UI) is the host's business and an open decision in `../MIGRATION.md`; a port never reads the environment. A `from_env(env)` over a mapping is allowed as the record of the twin's variable names, not as the way a port gets configured
|
||||
- The vendor's types, as `TypedDict`s, and any module-private pure helpers
|
||||
- The port class: a frozen dataclass whose only field is `config`, declaring `SinkPort[<its record>]` or `InterceptorPort` as its base. Both kinds have exactly the same parts
|
||||
- `payload(...)`: the port. For a sink, one `CallRecord` and the host's `now` to the vendor's body; for an interceptor, one `RequestFactsV1` to the `WirePatchV1` it needs. `None` means nothing to send or nothing to patch, which is also how a port filters
|
||||
- A sink adds `deliveries(batch)`, a pure function from a batch of bodies to the `Delivery`s that carry them, headers and framing included, and `batching`, how large a batch the host should collect
|
||||
- Nothing else. Not its name, not its gaps (`manifest.py` holds both), and nothing of the contract: no `events`, no `on_event`, no `before_send`, no subscriber class, no `build`. A port is registered by nobody; the tests name it, wrap it in `runtime.Sink` or `runtime.Patcher` and register that
|
||||
- A sink's handler never does I/O: v1 observers run inline in the call, where legacy callbacks ran on an executor or the logging worker
|
||||
- `runtime.Sink` joins envelopes with `CallJoin`, calls `payload` with the clock's `now`, and `put`s the result on its `Outbox`; the outbox thread calls `deliveries` and sends, `batching` at a time
|
||||
- So a sink has only the sync `on_event`; an async call falls back to it, and it never blocks
|
||||
- The terminal envelope carries neither model nor metadata, which is why `CallJoin` exists. A cancelled call has no terminal envelope, which is why it is bounded
|
||||
- Observer errors are reported and swallowed by the contract; a port does not wrap its handler in `try`
|
||||
- Tests, three layers; a port is not done without the first two
|
||||
- Payload (`tests/test_litellm/callbacks_v1/builtin/test_<name>.py`): `payload` on the golden envelopes Rust pins, asserting the whole vendor body. No Rust needed
|
||||
- Parity (`tests/test_litellm_rust/builtin/test_parity.py`): one native call with the twin in `callbacks=[...]` and the port registered by the test, each posting to its own recording vendor; `assert_parity` on the two raw bodies. Cover success and failure when the twin logs both
|
||||
- An interceptor whose twin never runs on a native route is compared function to function on the same bodies instead, plus one native call proving the patch reaches the provider
|
||||
- Conventions (`test_conventions.py`): runs for every manifest entry with no per-port code, the pure-import rule and the declared-protocol check included; add the port to `EXAMPLES` and `SUBSCRIBERS` in `support.py`
|
||||
- `Allowed` differences
|
||||
- Every entry has a reason. One that exists because of a gap cites it as `gap="<key>"`, and `assert_parity` fails if the port's `GAPS` has no such key. One with no `gap` is a difference the port keeps on purpose
|
||||
- Patterns are as narrow as the difference. A stale entry fails the test, so closing a gap in the contract forces the list to shrink, and deleting a `Gap` fails every entry still citing it
|
||||
- Never allow a difference to get a test green; either port the behaviour or record the gap
|
||||
- Gaps and status live in `manifest.py`, next to the entry: `exploring` (payload tests pass), `parity` (parity test passes), `ready` (parity, no open gap, and every route the twin serves emits v1 envelopes). Only `ready` ports may replace their twin
|
||||
6
litellm/callbacks_v1/builtin/__init__.py
Normal file
6
litellm/callbacks_v1/builtin/__init__.py
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
"""Built-in callbacks ported to the v1 contract.
|
||||
|
||||
Nothing here is registered by importing it, and nothing in the product imports it yet:
|
||||
a port is exercised by its tests only, until its legacy twin is removed. `AGENTS.md`
|
||||
beside this file is the porting convention, `manifest.py` is the list of ports.
|
||||
"""
|
||||
149
litellm/callbacks_v1/builtin/anthropic_cache_control.py
Normal file
149
litellm/callbacks_v1/builtin/anthropic_cache_control.py
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
"""Anthropic prompt-cache breakpoints on the v1 contract: a `before_send` body patch.
|
||||
|
||||
Legacy twin: `AnthropicCacheControlHook.apply_to_anthropic_messages_request` in
|
||||
`litellm.integrations.anthropic_cache_control_hook`, the Messages-route half of that hook.
|
||||
|
||||
Not ported yet: the OpenAI `prompt_cache_breakpoint` dialect, `tool_config` points, the
|
||||
default-injection seeding, and the chat-completions path (that route is not native).
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
from litellm.callbacks_v1 import JSONValue, RequestFactsV1, WirePatchV1
|
||||
from litellm.callbacks_v1.builtin.port import InterceptorPort
|
||||
|
||||
MAX_CACHE_CONTROL_BLOCKS: Final = 4
|
||||
|
||||
Role: TypeAlias = Literal["user", "system", "assistant"]
|
||||
Block: TypeAlias = Mapping[str, JSONValue]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InjectionPoint:
|
||||
"""Where to put a breakpoint: the system prompt, every message of `role`, or the message at `index`."""
|
||||
|
||||
role: Role | None = None
|
||||
index: int | None = None
|
||||
ttl: Literal["5m", "1h"] | None = None
|
||||
|
||||
@property
|
||||
def control(self) -> Block:
|
||||
if self.ttl is None:
|
||||
return MappingProxyType({"type": "ephemeral"})
|
||||
return MappingProxyType({"type": "ephemeral", "ttl": self.ttl})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Config:
|
||||
points: tuple[InjectionPoint, ...]
|
||||
providers: frozenset[str] = frozenset({"anthropic"})
|
||||
|
||||
|
||||
def _marked(block: JSONValue) -> bool:
|
||||
return isinstance(block, Mapping) and block.get("cache_control") is not None
|
||||
|
||||
|
||||
def _blocks(content: JSONValue) -> Sequence[JSONValue]:
|
||||
return content if isinstance(content, Sequence) and not isinstance(content, str) else ()
|
||||
|
||||
|
||||
def _breakpoints(message: JSONValue) -> int:
|
||||
if not isinstance(message, Mapping):
|
||||
return 0
|
||||
return int(_marked(message)) + sum(1 for block in _blocks(message.get("content")) if _marked(block))
|
||||
|
||||
|
||||
def _with_last_block_marked(blocks: Sequence[JSONValue], control: Block) -> Sequence[JSONValue] | None:
|
||||
if not blocks or not isinstance(blocks[-1], Mapping):
|
||||
return None
|
||||
return (*blocks[:-1], MappingProxyType({**blocks[-1], "cache_control": control}))
|
||||
|
||||
|
||||
def _as_blocks(message: Block) -> Block:
|
||||
content: Final = message.get("content")
|
||||
if not isinstance(content, str):
|
||||
return message
|
||||
return MappingProxyType({**message, "content": (MappingProxyType({"type": "text", "text": content}),)})
|
||||
|
||||
|
||||
def _targets(point: InjectionPoint, messages: Sequence[Block]) -> tuple[int, ...]:
|
||||
if point.index is not None:
|
||||
index: Final = point.index + len(messages) if point.index < 0 else point.index
|
||||
return (index,) if 0 <= index < len(messages) else ()
|
||||
return tuple(i for i, message in enumerate(messages) if message.get("role") == point.role)
|
||||
|
||||
|
||||
def _mark_system(system: JSONValue, control: Block) -> JSONValue:
|
||||
if isinstance(system, str):
|
||||
return (MappingProxyType({"type": "text", "text": system, "cache_control": control}),)
|
||||
blocks: Final = _blocks(system)
|
||||
if any(_marked(block) for block in blocks):
|
||||
return system
|
||||
return _with_last_block_marked(blocks, control) or system
|
||||
|
||||
|
||||
def _mark_messages(messages: Sequence[Block], targets: Sequence[tuple[int, Block]], budget: int) -> Sequence[Block]:
|
||||
if budget <= 0 or not targets:
|
||||
return messages
|
||||
(index, control), rest = targets[0], targets[1:]
|
||||
marked: Final = (
|
||||
None
|
||||
if _breakpoints(messages[index])
|
||||
else _with_last_block_marked(_blocks(messages[index].get("content")), control)
|
||||
)
|
||||
if marked is None:
|
||||
return _mark_messages(messages, rest, budget)
|
||||
message: Final = MappingProxyType({**messages[index], "content": marked})
|
||||
updated: Final = (*messages[:index], message, *messages[index + 1 :])
|
||||
return _mark_messages(updated, rest, budget - 1)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AnthropicCacheControl(InterceptorPort):
|
||||
"""A body patch that adds the configured breakpoints to a Messages request."""
|
||||
|
||||
config: Config
|
||||
|
||||
def payload(self, request: RequestFactsV1) -> WirePatchV1 | None:
|
||||
"""The patch this request needs, or `None` for a provider or body this port leaves alone."""
|
||||
if request["custom_llm_provider"] not in self.config.providers:
|
||||
return None
|
||||
body: Final = self.marked(request["body"])
|
||||
if body is None:
|
||||
return None
|
||||
patch: Final[WirePatchV1] = {"body": body}
|
||||
return patch
|
||||
|
||||
def marked(self, body: JSONValue) -> JSONValue | None:
|
||||
"""The Messages body with the configured breakpoints added, or `None` to leave it alone."""
|
||||
if not isinstance(body, Mapping) or not self.config.points:
|
||||
return None
|
||||
raw_messages: Final = body.get("messages")
|
||||
if not isinstance(raw_messages, Sequence) or isinstance(raw_messages, str):
|
||||
return None
|
||||
messages: Final = tuple(_as_blocks(message) for message in raw_messages if isinstance(message, Mapping))
|
||||
if len(messages) != len(raw_messages):
|
||||
return None
|
||||
|
||||
system: Final = body.get("system")
|
||||
system_point: Final = next((point for point in self.config.points if point.role == "system"), None)
|
||||
used: Final = sum(_breakpoints(message) for message in messages) + sum(1 for b in _blocks(system) if _marked(b))
|
||||
new_system: Final = (
|
||||
_mark_system(system, system_point.control)
|
||||
if system_point is not None and system is not None and used < MAX_CACHE_CONTROL_BLOCKS
|
||||
else system
|
||||
)
|
||||
used_after_system: Final = used + int(new_system != system)
|
||||
|
||||
targets: Final = tuple(
|
||||
(index, point.control)
|
||||
for point in self.config.points
|
||||
if point.role != "system"
|
||||
for index in _targets(point, messages)
|
||||
)
|
||||
new_messages: Final = _mark_messages(messages, targets, MAX_CACHE_CONTROL_BLOCKS - used_after_system)
|
||||
replaced: Final = MappingProxyType({"messages": new_messages, "system": new_system})
|
||||
return MappingProxyType({key: replaced.get(key, value) for key, value in body.items()})
|
||||
129
litellm/callbacks_v1/builtin/generic_api.py
Normal file
129
litellm/callbacks_v1/builtin/generic_api.py
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
"""Generic API logger on the v1 contract: batches of call records to one HTTP endpoint.
|
||||
|
||||
Legacy twin: `litellm.integrations.generic_api.generic_api_callback.GenericAPILogger`.
|
||||
The legacy logger ships the whole `StandardLoggingPayload`; this port ships the fields of
|
||||
it that v1 facts can fill, under the same names; `manifest.py` lists the rest as gaps.
|
||||
|
||||
Not ported yet: `callback_name` presets from `generic_api_compatible_callbacks.json`,
|
||||
retries, and the pre-StandardLoggingPayload body behind `litellm.generic_api_use_v1`.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm.callbacks_v1 import JSONValue
|
||||
from litellm.callbacks_v1.builtin.port import Batching, CallRecord, Delivery, SinkPort
|
||||
|
||||
LogFormat: TypeAlias = Literal["json_array", "ndjson", "single"]
|
||||
ApiEventType: TypeAlias = Literal["llm_api_success", "llm_api_failure"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Config:
|
||||
endpoint: str
|
||||
headers: tuple[tuple[str, str], ...] = (("Content-Type", "application/json"),)
|
||||
event_types: frozenset[ApiEventType] = frozenset({"llm_api_success", "llm_api_failure"})
|
||||
log_format: LogFormat = "json_array"
|
||||
batch_size: int = 512
|
||||
flush_interval: float = 5.0
|
||||
|
||||
@staticmethod
|
||||
def from_env(env: Mapping[str, str]) -> "Config":
|
||||
endpoint: Final = env.get("GENERIC_LOGGER_ENDPOINT")
|
||||
if endpoint is None:
|
||||
raise ValueError("endpoint not set for GenericAPILogger, GENERIC_LOGGER_ENDPOINT not found")
|
||||
pairs: Final = (item.split("=", 1) for item in env.get("GENERIC_LOGGER_HEADERS", "").split(",") if "=" in item)
|
||||
return Config(
|
||||
endpoint=endpoint,
|
||||
headers=(("Content-Type", "application/json"), *((key.strip(), value.strip()) for key, value in pairs)),
|
||||
)
|
||||
|
||||
|
||||
class ErrorInformation(TypedDict):
|
||||
error_class: ReadOnly[str]
|
||||
error_message: ReadOnly[str]
|
||||
error_code: ReadOnly[str]
|
||||
|
||||
|
||||
class Record(TypedDict):
|
||||
"""The `StandardLoggingPayload` fields v1 can fill, under their legacy names."""
|
||||
|
||||
id: ReadOnly[str]
|
||||
litellm_call_id: ReadOnly[str]
|
||||
call_type: ReadOnly[str]
|
||||
stream: ReadOnly[bool]
|
||||
status: ReadOnly[Literal["success", "failure"]]
|
||||
startTime: ReadOnly[float]
|
||||
endTime: ReadOnly[float]
|
||||
response_time: ReadOnly[float]
|
||||
model: ReadOnly[str | None]
|
||||
custom_llm_provider: ReadOnly[str | None]
|
||||
model_parameters: ReadOnly[JSONValue]
|
||||
response: ReadOnly[NotRequired[JSONValue]]
|
||||
error_str: ReadOnly[NotRequired[str]]
|
||||
error_information: ReadOnly[NotRequired[ErrorInformation]]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GenericApi(SinkPort[Record]):
|
||||
"""One record per finished call, batched into one endpoint."""
|
||||
|
||||
config: Config
|
||||
|
||||
@property
|
||||
def batching(self) -> Batching:
|
||||
return Batching(self.config.batch_size, self.config.flush_interval)
|
||||
|
||||
def payload(self, call: CallRecord, _now: datetime) -> Record | None:
|
||||
"""The record for a finished call, or `None` when `config.event_types` leaves its outcome out.
|
||||
A failed call has `error_*` where a successful one has `response`. The time is the call's own,
|
||||
so the clock the host passes is unused."""
|
||||
terminal: Final = call.terminal
|
||||
wanted: Final = "llm_api_success" if terminal["type"] == "call.succeeded" else "llm_api_failure"
|
||||
if wanted not in self.config.event_types:
|
||||
return None
|
||||
request: Final = call.request
|
||||
response: Final = terminal["response"] if terminal["type"] == "call.succeeded" else None
|
||||
response_id: Final = response.get("id") if isinstance(response, Mapping) else None
|
||||
common: Final[Record] = {
|
||||
"id": response_id if isinstance(response_id, str) else call.call_id,
|
||||
"litellm_call_id": call.call_id,
|
||||
"call_type": call.call_type,
|
||||
"stream": terminal["streamed"],
|
||||
"status": "success" if terminal["type"] == "call.succeeded" else "failure",
|
||||
"startTime": terminal["timing"]["start_time"],
|
||||
"endTime": terminal["timing"]["end_time"],
|
||||
"response_time": terminal["timing"]["duration"],
|
||||
"model": request["model"] if request is not None else None,
|
||||
"custom_llm_provider": request["custom_llm_provider"] if request is not None else None,
|
||||
"model_parameters": request["optional_params"] if request is not None else None,
|
||||
}
|
||||
if terminal["type"] == "call.succeeded":
|
||||
succeeded: Final[Record] = {**common, "response": terminal["response"]}
|
||||
return succeeded
|
||||
error: Final = terminal["error"]
|
||||
failed: Final[Record] = {
|
||||
**common,
|
||||
"error_str": error["message"],
|
||||
"error_information": {
|
||||
"error_class": error["class"].rsplit(".", 1)[-1],
|
||||
"error_message": error["message"],
|
||||
"error_code": str(error["status_code"]) if error["status_code"] is not None else "",
|
||||
},
|
||||
}
|
||||
return failed
|
||||
|
||||
def deliveries(self, batch: tuple[Record, ...]) -> tuple[Delivery, ...]:
|
||||
if self.config.log_format == "single":
|
||||
return tuple(Delivery(self.config.endpoint, self.config.headers, json.dumps(r).encode()) for r in batch)
|
||||
body: Final = (
|
||||
json.dumps(batch)
|
||||
if self.config.log_format == "json_array"
|
||||
else "\n".join(json.dumps(record) for record in batch)
|
||||
)
|
||||
return (Delivery(self.config.endpoint, self.config.headers, body.encode()),)
|
||||
218
litellm/callbacks_v1/builtin/manifest.py
Normal file
218
litellm/callbacks_v1/builtin/manifest.py
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
"""Every built-in that has a v1 port, its legacy twin, how far the port has come, and the
|
||||
gaps it is waiting on.
|
||||
|
||||
This whole module is migration scaffolding, and it is all of it: `port.py` and the ports
|
||||
themselves carry none. Read by tests only. Nothing here registers a callback or decides
|
||||
what runs in a call -- a port goes live by removing its legacy twin, not by a switch in
|
||||
this table -- and when the last twin is gone this file is deleted outright.
|
||||
|
||||
The registration name lives here rather than on the port because it is the host's: a port
|
||||
is a value, and the same one configured twice is two subscribers under two names.
|
||||
|
||||
`python -m litellm.callbacks_v1.builtin.manifest` prints the gap ledger and the backlog.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from types import ModuleType
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
from litellm.callbacks_v1.builtin import anthropic_cache_control, generic_api, openmeter
|
||||
|
||||
Kind: TypeAlias = Literal["sink", "interceptor"]
|
||||
GapKind: TypeAlias = Literal["fact", "event", "capability"]
|
||||
|
||||
# exploring: the port exists and its payload tests pass.
|
||||
# parity: its differential test against the legacy twin passes, every difference explained.
|
||||
# ready: parity, no open gap, and every route the legacy twin serves emits v1 envelopes.
|
||||
Status: TypeAlias = Literal["exploring", "parity", "ready"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Gap:
|
||||
"""One legacy input the v1 contract does not supply.
|
||||
|
||||
`fact` is a value missing from an envelope, `event` is a moment no envelope marks,
|
||||
and `capability` is something the contract does not let a callback do at all.
|
||||
|
||||
`key` names what the contract would have to grow, not what the legacy logger read, so
|
||||
two ports missing the same thing share a key and the ledger can count them. It is also
|
||||
what an `Allowed` parity difference cites, which is how closing a gap finds its tests.
|
||||
"""
|
||||
|
||||
kind: GapKind
|
||||
key: str
|
||||
legacy: str
|
||||
note: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Entry:
|
||||
"""One port: where it lives, what it registers as, what it replaces, and where it has got to."""
|
||||
|
||||
module: ModuleType
|
||||
name: str
|
||||
kind: Kind
|
||||
legacy: str
|
||||
status: Status
|
||||
gaps: tuple[Gap, ...]
|
||||
|
||||
|
||||
OPENMETER_GAPS: Final = (
|
||||
Gap("fact", "cost", "kwargs['response_cost']", "no cost fact on call.succeeded; `data.cost` is always null"),
|
||||
Gap(
|
||||
"fact",
|
||||
"usage",
|
||||
"response_obj['usage'] of a ModelResponse",
|
||||
"token counts are only reachable by parsing `response` per provider shape; needs a usage fact",
|
||||
),
|
||||
Gap(
|
||||
"fact",
|
||||
"request_user",
|
||||
"kwargs['user']",
|
||||
"the request's `user` is not projected; subject comes from metadata only",
|
||||
),
|
||||
Gap(
|
||||
"fact",
|
||||
"litellm_metadata",
|
||||
"litellm_params['metadata'] fed by `litellm_metadata`",
|
||||
"call.started projects only the `metadata` keyword; the proxy's identity on Messages-style routes is invisible",
|
||||
),
|
||||
Gap(
|
||||
"fact",
|
||||
"terminal_model",
|
||||
"kwargs['model']",
|
||||
"the terminal envelope has no model, so the port joins request.sending by call_id",
|
||||
),
|
||||
Gap(
|
||||
"capability",
|
||||
"off_path_delivery",
|
||||
"Logging runs success callbacks on an executor / the logging worker",
|
||||
"v1 observers run inline in the call; the port needs its own outbox to stay off the call path",
|
||||
),
|
||||
)
|
||||
|
||||
ANTHROPIC_CACHE_CONTROL_GAPS: Final = (
|
||||
Gap(
|
||||
"fact",
|
||||
"request_params",
|
||||
"kwargs['cache_control_injection_points']",
|
||||
"per-request injection points are not in RequestFacts; the port only knows the points it was built with",
|
||||
),
|
||||
Gap(
|
||||
"fact",
|
||||
"request_call_type",
|
||||
"call_type",
|
||||
"RequestFacts has no call_type, so the port recognises a Messages request by the body's shape",
|
||||
),
|
||||
Gap(
|
||||
"capability",
|
||||
"route_neutral_rewrite",
|
||||
"get_chat_completion_prompt rewrites the route-neutral messages before transformation",
|
||||
"before_send sees only the provider's wire body, so every dialect needs its own body walker",
|
||||
),
|
||||
)
|
||||
|
||||
# What the legacy StandardLoggingPayload has and v1 facts cannot fill, by what the
|
||||
# contract would have to grow to fill it.
|
||||
_MISSING_FACTS: Final = (
|
||||
("cost", "response_cost, cost_breakdown, saved_cache_cost, autorouter_savings", "no cost facts"),
|
||||
("usage", "total_tokens, prompt_tokens, completion_tokens", "no usage fact; only the raw provider response"),
|
||||
("cache", "cache_hit, cache_key", "no cache fact"),
|
||||
(
|
||||
"routing",
|
||||
"model_id, model_group, api_base, model_map_information",
|
||||
"no routing fact; `url` is the wire URL, not api_base",
|
||||
),
|
||||
("identity", "metadata.user_api_key_*, end_user, requester_ip_address, user_agent", "no identity fact"),
|
||||
("tags", "request_tags, request_model_access_groups", "no tags fact"),
|
||||
("correlation", "trace_id, session_id", "no correlation beyond call_id"),
|
||||
("first_token_time", "completionStartTime", "no first-token time in `timing`"),
|
||||
("no_equivalent", "hidden_params, guardrail_information, standard_built_in_tools_params", "no equivalent"),
|
||||
("prompt", "messages", "only the provider-shaped wire body; no route-neutral prompt projection"),
|
||||
(
|
||||
"normalised_response",
|
||||
"response",
|
||||
"legacy logs the OpenAI-normalised ModelResponse; v1 carries the route's own public response",
|
||||
),
|
||||
(
|
||||
"error_detail",
|
||||
"error_information.traceback, llm_provider, error_provider_request_id",
|
||||
"ErrorFacts is class/message/status only",
|
||||
),
|
||||
)
|
||||
|
||||
GENERIC_API_GAPS: Final = (
|
||||
*(Gap("fact", key, f"standard_logging_object: {fields}", note) for key, fields, note in _MISSING_FACTS),
|
||||
Gap(
|
||||
"fact",
|
||||
"terminal_model",
|
||||
"standard_logging_object: model, custom_llm_provider, model_parameters",
|
||||
"the terminal envelope has none of them, so the port joins request.sending by call_id",
|
||||
),
|
||||
Gap(
|
||||
"capability",
|
||||
"off_path_delivery",
|
||||
"CustomBatchLogger.periodic_flush on the proxy's event loop",
|
||||
"v1 observers run inline in the call; batching and I/O live on the port's outbox thread",
|
||||
),
|
||||
)
|
||||
|
||||
PORTS: Final = (
|
||||
Entry(
|
||||
openmeter,
|
||||
"openmeter",
|
||||
"sink",
|
||||
"litellm.integrations.openmeter:OpenMeterLogger",
|
||||
"parity",
|
||||
OPENMETER_GAPS,
|
||||
),
|
||||
Entry(
|
||||
generic_api,
|
||||
"generic_api",
|
||||
"sink",
|
||||
"litellm.integrations.generic_api.generic_api_callback:GenericAPILogger",
|
||||
"parity",
|
||||
GENERIC_API_GAPS,
|
||||
),
|
||||
Entry(
|
||||
anthropic_cache_control,
|
||||
"anthropic_cache_control",
|
||||
"interceptor",
|
||||
"litellm.integrations.anthropic_cache_control_hook:AnthropicCacheControlHook",
|
||||
"parity",
|
||||
ANTHROPIC_CACHE_CONTROL_GAPS,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def entry(name: str) -> Entry:
|
||||
"""The manifest entry a port registers under; how a test reaches its gaps."""
|
||||
(found,) = (candidate for candidate in PORTS if candidate.name == name)
|
||||
return found
|
||||
|
||||
|
||||
def ledger() -> str:
|
||||
"""Every open gap, grouped by port: what the contract has to grow before ports can be `ready`."""
|
||||
return "\n".join(
|
||||
line
|
||||
for port in PORTS
|
||||
for line in (
|
||||
f"{port.name} [{port.kind}, {port.status}] <- {port.legacy}",
|
||||
*(f" {gap.kind:<10} {gap.key}: {gap.legacy}\n {gap.note}" for gap in port.gaps),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def backlog() -> str:
|
||||
"""Every gap key with the ports that are waiting on it, most wanted first: the order the contract should grow in."""
|
||||
keys: Final = frozenset(gap.key for port in PORTS for gap in port.gaps)
|
||||
waiting: Final = tuple(
|
||||
(key, tuple(port.name for port in PORTS if any(gap.key == key for gap in port.gaps))) for key in sorted(keys)
|
||||
)
|
||||
return "\n".join(
|
||||
f" {key:<22} {', '.join(names)}" for key, names in sorted(waiting, key=lambda item: -len(item[1]))
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(f"{ledger()}\n\nbacklog, by how many ports wait on it\n{backlog()}") # noqa: T201 # the ledger is this module's command-line output
|
||||
125
litellm/callbacks_v1/builtin/openmeter.py
Normal file
125
litellm/callbacks_v1/builtin/openmeter.py
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
"""OpenMeter on the v1 contract: one CloudEvent per successful call.
|
||||
|
||||
Legacy twin: `litellm.integrations.openmeter.OpenMeterLogger`.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm.callbacks_v1 import JSONValue
|
||||
from litellm.callbacks_v1.builtin.port import Batching, CallRecord, Delivery, SinkPort
|
||||
|
||||
_USAGE_KEYS: Final = ("prompt_tokens", "completion_tokens", "total_tokens")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Config:
|
||||
api_key: str
|
||||
endpoint: str = "https://openmeter.cloud"
|
||||
event_type: str = "litellm_tokens"
|
||||
|
||||
@staticmethod
|
||||
def from_env(env: Mapping[str, str]) -> "Config":
|
||||
api_key: Final = env.get("OPENMETER_API_KEY")
|
||||
if api_key is None:
|
||||
raise ValueError("Missing keys=['OPENMETER_API_KEY'] in environment.")
|
||||
return Config(
|
||||
api_key=api_key,
|
||||
endpoint=env.get("OPENMETER_API_ENDPOINT", "https://openmeter.cloud"),
|
||||
event_type=env.get("OPENMETER_EVENT_TYPE", "litellm_tokens"),
|
||||
)
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return f"{self.endpoint.rstrip('/')}/api/v1/events"
|
||||
|
||||
|
||||
class Usage(TypedDict):
|
||||
prompt_tokens: ReadOnly[JSONValue]
|
||||
completion_tokens: ReadOnly[JSONValue]
|
||||
total_tokens: ReadOnly[JSONValue]
|
||||
|
||||
|
||||
class MeterData(TypedDict):
|
||||
model: ReadOnly[str | None]
|
||||
cost: ReadOnly[float | None]
|
||||
prompt_tokens: ReadOnly[NotRequired[JSONValue]]
|
||||
completion_tokens: ReadOnly[NotRequired[JSONValue]]
|
||||
total_tokens: ReadOnly[NotRequired[JSONValue]]
|
||||
|
||||
|
||||
class CloudEvent(TypedDict):
|
||||
specversion: ReadOnly[str]
|
||||
type: ReadOnly[str]
|
||||
id: ReadOnly[str]
|
||||
time: ReadOnly[str]
|
||||
subject: ReadOnly[str]
|
||||
source: ReadOnly[str]
|
||||
data: ReadOnly[MeterData]
|
||||
|
||||
|
||||
def _usage(response: JSONValue) -> Usage | None:
|
||||
"""Token counts when the response carries them in the OpenAI shape, as the legacy logger reads them."""
|
||||
usage: Final = response.get("usage") if isinstance(response, Mapping) else None
|
||||
if not isinstance(usage, Mapping) or not any(key in usage for key in _USAGE_KEYS):
|
||||
return None
|
||||
counted: Final[Usage] = {
|
||||
"prompt_tokens": usage.get("prompt_tokens", 0),
|
||||
"completion_tokens": usage.get("completion_tokens", 0),
|
||||
"total_tokens": usage.get("total_tokens"),
|
||||
}
|
||||
return counted
|
||||
|
||||
|
||||
def _metered(model: str | None, usage: Usage) -> MeterData:
|
||||
data: Final[MeterData] = {"model": model, "cost": None, **usage}
|
||||
return data
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class OpenMeter(SinkPort[CloudEvent]):
|
||||
"""One CloudEvent per successful call, one request per event."""
|
||||
|
||||
config: Config
|
||||
|
||||
@property
|
||||
def batching(self) -> Batching:
|
||||
"""One event per request, as the legacy logger posts them."""
|
||||
return Batching()
|
||||
|
||||
def payload(self, call: CallRecord, now: datetime) -> CloudEvent | None:
|
||||
"""The CloudEvent for a finished call, or `None` when OpenMeter has nothing to meter."""
|
||||
if call.terminal["type"] != "call.succeeded":
|
||||
return None
|
||||
subject: Final = call.started["metadata"].get("user_api_key_user_id") if call.started is not None else None
|
||||
if subject is None:
|
||||
raise ValueError("OpenMeter: user is required")
|
||||
response: Final = call.terminal["response"]
|
||||
response_id: Final = response.get("id") if isinstance(response, Mapping) else None
|
||||
model: Final = call.request["model"] if call.request is not None else None
|
||||
usage: Final = _usage(response)
|
||||
unmetered: Final[MeterData] = {"model": model, "cost": None}
|
||||
data: Final = unmetered if usage is None else _metered(model, usage)
|
||||
event: Final[CloudEvent] = {
|
||||
"specversion": "1.0",
|
||||
"type": self.config.event_type,
|
||||
"id": response_id if isinstance(response_id, str) else call.call_id,
|
||||
"time": now.isoformat(),
|
||||
"subject": str(subject),
|
||||
"source": "litellm-proxy",
|
||||
"data": data,
|
||||
}
|
||||
return event
|
||||
|
||||
def deliveries(self, batch: tuple[CloudEvent, ...]) -> tuple[Delivery, ...]:
|
||||
"""One request per CloudEvent, as the legacy logger posts them."""
|
||||
headers: Final = (
|
||||
("Content-Type", "application/cloudevents+json"),
|
||||
("Authorization", f"Bearer {self.config.api_key}"),
|
||||
)
|
||||
return tuple(Delivery(self.config.url, headers, json.dumps(event).encode()) for event in batch)
|
||||
95
litellm/callbacks_v1/builtin/outbox.py
Normal file
95
litellm/callbacks_v1/builtin/outbox.py
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
"""Where a sink's handler leaves its work, so that the handler never does I/O.
|
||||
|
||||
The v1 contract runs an observer inline in the call it observes. A handler that posted to
|
||||
a vendor would put that vendor's latency into every call, so a sink's handler only builds
|
||||
a value and `put`s it here; one daemon thread per outbox delivers it, alone or in batches.
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Final, Generic, TypeVar
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
T = TypeVar("T") # rebind-ok: a TypeVar must bind to a bare name to be recognised as one
|
||||
|
||||
|
||||
class Outbox(Generic[T]):
|
||||
"""Delivers items off the call path, `batch_size` at a time or every `flush_interval`.
|
||||
|
||||
`deliver` gets a non-empty batch and may raise: the failure is logged and the batch is
|
||||
dropped, as the legacy batch loggers do. A full outbox drops the new item rather than
|
||||
block the call that produced it.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
deliver: Callable[[tuple[T, ...]], None],
|
||||
batch_size: int = 1,
|
||||
flush_interval: float = 5.0,
|
||||
capacity: int = 50_000,
|
||||
) -> None:
|
||||
self._name: Final = name
|
||||
self._deliver: Final = deliver
|
||||
self._batch_size: Final = max(1, batch_size)
|
||||
self._flush_interval: Final = flush_interval
|
||||
self._items: Final[queue.Queue[T]] = queue.Queue(maxsize=capacity)
|
||||
self._lock: Final = threading.Lock()
|
||||
self._flush_now: Final = threading.Event()
|
||||
self._worker_pid: int | None = None
|
||||
atexit.register(self.flush)
|
||||
|
||||
def put(self, item: T) -> None:
|
||||
self._ensure_worker()
|
||||
try:
|
||||
self._items.put_nowait(item)
|
||||
except queue.Full:
|
||||
verbose_logger.warning("callback %s outbox is full; dropping one item", self._name)
|
||||
|
||||
def flush(self, timeout: float = 5.0) -> bool:
|
||||
"""Delivers everything already put; returns whether it drained within `timeout`."""
|
||||
deadline: Final = time.monotonic() + timeout
|
||||
self._flush_now.set()
|
||||
while self._items.unfinished_tasks and time.monotonic() < deadline:
|
||||
time.sleep(0.005)
|
||||
return not self._items.unfinished_tasks
|
||||
|
||||
def _ensure_worker(self) -> None:
|
||||
# A forked child inherits the queue but not the thread, so the worker is per process.
|
||||
with self._lock:
|
||||
if self._worker_pid == os.getpid():
|
||||
return
|
||||
self._worker_pid = os.getpid()
|
||||
threading.Thread(target=self._run, name=f"callback-outbox-{self._name}", daemon=True).start()
|
||||
|
||||
def _next_batch(self) -> tuple[T, ...]:
|
||||
first: Final = self._items.get()
|
||||
deadline: Final = time.monotonic() + self._flush_interval
|
||||
batch: Final[list[T]] = [first] # mutable-ok: a batch is gathered from a blocking queue one item at a time
|
||||
while len(batch) < self._batch_size:
|
||||
remaining = deadline - time.monotonic() # rebind-ok: the wait shrinks on every pass of the gather loop
|
||||
if remaining <= 0 or (self._flush_now.is_set() and self._items.empty()):
|
||||
break
|
||||
try:
|
||||
batch.append(self._items.get(timeout=min(remaining, 0.01)))
|
||||
except queue.Empty:
|
||||
continue
|
||||
return tuple(batch)
|
||||
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
batch = self._next_batch() # rebind-ok: the worker loop takes a new batch on every pass
|
||||
try:
|
||||
self._deliver(batch)
|
||||
except Exception: # noqa: BLE001 # a vendor failure must never end the delivery thread
|
||||
verbose_logger.exception("callback %s failed to deliver %d item(s)", self._name, len(batch))
|
||||
finally:
|
||||
for _ in batch:
|
||||
self._items.task_done()
|
||||
if self._items.empty():
|
||||
self._flush_now.clear()
|
||||
90
litellm/callbacks_v1/builtin/port.py
Normal file
90
litellm/callbacks_v1/builtin/port.py
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
"""The shape every port has, and nothing that can act.
|
||||
|
||||
A port is a frozen class that holds its `Config` and names one of the protocols below as
|
||||
its base: pure methods from contract values to what the port produces. Declaring the base
|
||||
is what holds it to the shape, at the class rather than at some call site far away; a
|
||||
method that drifts fails there, and a missing one fails wherever the port is built. It opens no socket,
|
||||
starts no thread and reads neither the environment nor the clock; the host hands it all of
|
||||
those as values. That is the shape a sandboxed custom callback is confined to, and
|
||||
built-ins take it first so both share one interface.
|
||||
|
||||
A port has no identity of its own. It is not named, does not know which events reach it
|
||||
and implements nothing of the v1 contract: `runtime.py` names it and wraps it into a
|
||||
subscriber, so the same port configured twice is two registrations rather than a clash.
|
||||
Nothing in this file is migration scaffolding; `manifest.py` holds all of that.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Protocol, TypeVar
|
||||
|
||||
from litellm.callbacks_v1 import (
|
||||
CallFailedV1,
|
||||
CallStartedV1,
|
||||
CallSucceededV1,
|
||||
RequestFactsV1,
|
||||
RequestSendingV1,
|
||||
WirePatchV1,
|
||||
)
|
||||
|
||||
R = TypeVar("R") # rebind-ok: a TypeVar must bind to a bare name to be recognised as one
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Delivery:
|
||||
"""One request to a vendor, fully decided; sending it is the only thing left."""
|
||||
|
||||
url: str
|
||||
headers: tuple[tuple[str, str], ...]
|
||||
body: bytes
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CallRecord:
|
||||
"""Every envelope of one finished call a sink needs, joined by `call_id`.
|
||||
|
||||
`request` is absent when the call failed before anything was sent.
|
||||
"""
|
||||
|
||||
call_id: str
|
||||
call_type: str
|
||||
started: CallStartedV1 | None
|
||||
request: RequestSendingV1 | None
|
||||
terminal: CallSucceededV1 | CallFailedV1
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Batching:
|
||||
"""How many records the host carries to the vendor at once, and how long it waits."""
|
||||
|
||||
size: int = 1
|
||||
interval: float = 5.0
|
||||
|
||||
|
||||
class SinkPort(Protocol[R]):
|
||||
"""A port that observes finished calls and tells a vendor.
|
||||
|
||||
`payload` is the port: one finished call and the current time to the vendor's record,
|
||||
or `None` for nothing to send, which is also how a port filters. `deliveries` frames a
|
||||
batch of those records as requests, headers and framing included, and `batching` says
|
||||
how large a batch the host should collect. The two are separate because the handler
|
||||
runs inline in the call and the delivery does not.
|
||||
"""
|
||||
|
||||
def payload(self, call: CallRecord, now: datetime, /) -> R | None: ...
|
||||
|
||||
def deliveries(self, batch: tuple[R, ...], /) -> tuple[Delivery, ...]: ...
|
||||
|
||||
@property
|
||||
def batching(self) -> Batching: ...
|
||||
|
||||
|
||||
class InterceptorPort(Protocol):
|
||||
"""A port that patches the wire request before it is sent; its failure fails the call.
|
||||
|
||||
`payload` is the patch the request needs, or `None` to leave it alone. It decides
|
||||
whether the request is one this port acts on at all: the host applies what it returns
|
||||
and filters nothing.
|
||||
"""
|
||||
|
||||
def payload(self, request: RequestFactsV1, /) -> WirePatchV1 | None: ...
|
||||
156
litellm/callbacks_v1/builtin/runtime.py
Normal file
156
litellm/callbacks_v1/builtin/runtime.py
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
"""The in-process host of a port: everything a port is not allowed to do itself.
|
||||
|
||||
Joining a call's envelopes, reading the clock, batching off the call path and talking to
|
||||
the vendor all live here, once, so a port stays pure methods over values (`port.py`).
|
||||
`Sink` and `Patcher` are the two subscribers: each names a port of its kind and wraps it,
|
||||
and they are the only things in this package that implement the v1 contract, so a port
|
||||
never writes `name`, `events` or a handler. The name is the caller's because it is the
|
||||
registration's, not the port's: one port configured twice is two subscribers. This is not a sandbox and isolates nothing: it is plain
|
||||
in-process Python. It is a stand-in for what a host should own (join, queue, batch,
|
||||
deliver). When the contract grows off-path delivery and a joined call record, this module
|
||||
is deleted and the ports do not change. Custom callbacks are isolated by the gateway's
|
||||
extension host, in Rust.
|
||||
"""
|
||||
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final, Generic, Protocol, TypeVar
|
||||
|
||||
from litellm.callbacks_v1 import (
|
||||
CallStartedV1,
|
||||
EnvelopeV1,
|
||||
EventName,
|
||||
Interceptor,
|
||||
Observer,
|
||||
RequestFactsV1,
|
||||
RequestSendingV1,
|
||||
WirePatchV1,
|
||||
)
|
||||
from litellm.callbacks_v1.builtin.outbox import Outbox
|
||||
from litellm.callbacks_v1.builtin.port import CallRecord, Delivery, InterceptorPort, SinkPort
|
||||
|
||||
R = TypeVar("R") # rebind-ok: a TypeVar must bind to a bare name to be recognised as one
|
||||
|
||||
|
||||
class Transport(Protocol):
|
||||
def send(self, delivery: Delivery) -> None: ...
|
||||
|
||||
|
||||
class HttpxTransport:
|
||||
"""Sends through the same client the legacy loggers use, so proxy TLS settings apply."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
self._client: Final = HTTPHandler()
|
||||
|
||||
def send(self, delivery: Delivery) -> None:
|
||||
headers: Final = dict(delivery.headers) # mutable-ok: the httpx client takes its headers as a dict
|
||||
self._client.post(url=delivery.url, headers=headers, content=delivery.body) # pyright: ignore[reportUnknownMemberType] # the legacy HTTP handler is only partly typed
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _OpenCall:
|
||||
started: CallStartedV1 | None = None
|
||||
request: RequestSendingV1 | None = None
|
||||
|
||||
|
||||
JOINED_EVENTS: Final[frozenset[EventName]] = frozenset(
|
||||
{"call.started", "request.sending", "call.succeeded", "call.failed"}
|
||||
)
|
||||
|
||||
|
||||
class CallJoin:
|
||||
"""Joins a call's envelopes into one `CallRecord` at its terminal envelope.
|
||||
|
||||
The terminal envelope carries neither the model nor the metadata, so a sink has to
|
||||
remember the earlier ones. A cancelled call never gets a terminal envelope, which is
|
||||
why the open calls are bounded: the oldest is forgotten rather than leaked.
|
||||
"""
|
||||
|
||||
def __init__(self, capacity: int = 10_000) -> None:
|
||||
self._capacity: Final = capacity
|
||||
self._lock: Final = threading.Lock()
|
||||
self._open: Final[dict[str, _OpenCall]] = {} # mutable-ok: per-call join state is an evolving keyed store
|
||||
|
||||
def accept(self, envelope: EnvelopeV1) -> CallRecord | None:
|
||||
call_id: Final = envelope["call_id"]
|
||||
event: Final = envelope["event"]
|
||||
with self._lock:
|
||||
if event["type"] == "call.succeeded" or event["type"] == "call.failed":
|
||||
seen: Final = self._open.pop(call_id, _OpenCall())
|
||||
return CallRecord(call_id, envelope["call_type"], seen.started, seen.request, event)
|
||||
current: Final = self._open.get(call_id, _OpenCall())
|
||||
if event["type"] == "call.started":
|
||||
self._open[call_id] = _OpenCall(event, current.request)
|
||||
elif event["type"] == "request.sending":
|
||||
self._open[call_id] = _OpenCall(current.started, event)
|
||||
while len(self._open) > self._capacity:
|
||||
del self._open[next(iter(self._open))]
|
||||
return None
|
||||
|
||||
def open_calls(self) -> int:
|
||||
with self._lock:
|
||||
return len(self._open)
|
||||
|
||||
|
||||
class Sink(Observer, Generic[R]):
|
||||
"""The subscriber of every `SinkPort`: the port supplies pure methods, this runs them.
|
||||
|
||||
The handler only joins and enqueues, so the sync one serves async calls too and never
|
||||
blocks; the outbox thread calls `deliveries` and sends.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
port: SinkPort[R],
|
||||
transport: Transport,
|
||||
clock: Callable[[], datetime] = utc_now,
|
||||
) -> None:
|
||||
self.name = name
|
||||
self.events = JOINED_EVENTS
|
||||
self._port: Final = port
|
||||
self._transport: Final = transport
|
||||
self._clock: Final = clock
|
||||
self._calls: Final = CallJoin()
|
||||
self.outbox: Final[Outbox[R]] = Outbox(
|
||||
name, self._deliver, batch_size=port.batching.size, flush_interval=port.batching.interval
|
||||
)
|
||||
|
||||
def _deliver(self, batch: tuple[R, ...]) -> None:
|
||||
for delivery in self._port.deliveries(batch):
|
||||
self._transport.send(delivery)
|
||||
|
||||
def on_event(self, event: EnvelopeV1) -> None:
|
||||
call: Final = self._calls.accept(event)
|
||||
if call is None:
|
||||
return
|
||||
record: Final = self._port.payload(call, self._clock())
|
||||
if record is not None:
|
||||
self.outbox.put(record)
|
||||
|
||||
def open_calls(self) -> int:
|
||||
return self._calls.open_calls()
|
||||
|
||||
|
||||
class Patcher(Interceptor):
|
||||
"""The subscriber of every `InterceptorPort`: the contract's `before_send` over `payload`.
|
||||
|
||||
An interceptor runs on the call path by definition, so there is nothing to join, batch
|
||||
or defer here; this exists so that an interceptor port declares as little as a sink does.
|
||||
"""
|
||||
|
||||
def __init__(self, name: str, port: InterceptorPort) -> None:
|
||||
self.name = name
|
||||
self.events: frozenset[EventName] = frozenset()
|
||||
self._port: Final = port
|
||||
|
||||
def before_send(self, request: RequestFactsV1) -> WirePatchV1 | None:
|
||||
return self._port.payload(request)
|
||||
486
litellm/callbacks_v1/host.py
Normal file
486
litellm/callbacks_v1/host.py
Normal file
|
|
@ -0,0 +1,486 @@
|
|||
"""The extension host: a process that runs v1 callbacks for a gateway that does not.
|
||||
|
||||
python -m litellm.callbacks_v1.host --fd 3 --load my_pkg.audit --load my_pkg.cb:callback
|
||||
|
||||
The gateway never imports a callback. It starts (or attaches to) this worker, which imports
|
||||
the callbacks exactly as an SDK process would, reports what they subscribed to, and then
|
||||
only answers: the gateway sends envelopes and interception requests over one socket, the
|
||||
worker runs the handlers and sends back patches. The worker cannot call into the gateway;
|
||||
there is no frame for it. A callback file is the same file in the SDK and here.
|
||||
|
||||
This module isolates nothing by itself. What the worker can reach (environment, files,
|
||||
network) is whatever the process it runs in was given; confinement is the spawner's job.
|
||||
|
||||
Frames are a 4-byte big-endian length and that many bytes of JSON, one object with a
|
||||
`type`. `litellm-rust/crates/callbacks-v1-exthost/golden` pins every frame for both sides.
|
||||
|
||||
worker -> gateway hello {schema, subscriptions: [{name, events, observes, intercepts}]}
|
||||
gateway -> worker event {subscriber, envelope} no reply
|
||||
gateway -> worker intercept {id, subscriber, request}
|
||||
worker -> gateway patch {id, patch} or error {id, error_class, message}
|
||||
worker -> gateway report {subscriber, event, message} an observer raised
|
||||
gateway -> worker flush {id}
|
||||
worker -> gateway flushed {id} every event sent before the flush was handled
|
||||
|
||||
`subscriber` is an index into the hello's `subscriptions`. Envelopes reach one subscriber
|
||||
in the order they were sent; an interception never waits behind events.
|
||||
|
||||
A handler is scheduled by the kind its author declared, which is the author's decision and
|
||||
not this module's guess. An `async` handler is a task on the worker's loop, so one that only
|
||||
waits costs no thread and concurrency is the loop's rather than a pool size; a sync handler
|
||||
gets a thread, because running it blocks one, and the loop keeps turning while it does. Every
|
||||
frame leaves through one writer, so no handler and no loop callback blocks on the socket.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import importlib
|
||||
import json
|
||||
import queue
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
from collections.abc import Coroutine, Mapping, Sequence
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Literal, Protocol, TypeAlias
|
||||
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm.callbacks_v1 import SCHEMA, JSONValue, SchemaVersion, Subscriber, register, snapshot
|
||||
|
||||
MAX_FRAME_BYTES: Final = 64 * 1024 * 1024
|
||||
INTERCEPT_THREADS: Final = 8
|
||||
OBSERVER_QUEUE: Final = 1024
|
||||
|
||||
|
||||
class Handler(Protocol):
|
||||
"""A handler the worker calls on a thread, because running it blocks one."""
|
||||
|
||||
def __call__(self, argument: JSONValue, /) -> object: ...
|
||||
|
||||
|
||||
class AsyncHandler(Protocol):
|
||||
"""A handler the worker awaits on its loop, because waiting is all it does."""
|
||||
|
||||
def __call__(self, argument: JSONValue, /) -> Coroutine[object, object, object]: ...
|
||||
|
||||
|
||||
class SubscriptionData(TypedDict):
|
||||
name: ReadOnly[str]
|
||||
events: ReadOnly[Sequence[str]]
|
||||
observes: ReadOnly[bool]
|
||||
intercepts: ReadOnly[bool]
|
||||
|
||||
|
||||
class HelloFrame(TypedDict):
|
||||
type: ReadOnly[Literal["hello"]]
|
||||
schema: ReadOnly[SchemaVersion]
|
||||
subscriptions: ReadOnly[Sequence[SubscriptionData]]
|
||||
|
||||
|
||||
class PatchFrame(TypedDict):
|
||||
type: ReadOnly[Literal["patch"]]
|
||||
id: ReadOnly[JSONValue]
|
||||
patch: ReadOnly[JSONValue]
|
||||
|
||||
|
||||
class ErrorFrame(TypedDict):
|
||||
type: ReadOnly[Literal["error"]]
|
||||
id: ReadOnly[JSONValue]
|
||||
error_class: ReadOnly[str]
|
||||
message: ReadOnly[str]
|
||||
|
||||
|
||||
class ReportFrame(TypedDict):
|
||||
type: ReadOnly[Literal["report"]]
|
||||
subscriber: ReadOnly[int]
|
||||
event: ReadOnly[JSONValue]
|
||||
message: ReadOnly[str]
|
||||
|
||||
|
||||
class FlushedFrame(TypedDict):
|
||||
type: ReadOnly[Literal["flushed"]]
|
||||
id: ReadOnly[JSONValue]
|
||||
|
||||
|
||||
Sent: TypeAlias = HelloFrame | PatchFrame | ErrorFrame | ReportFrame | FlushedFrame
|
||||
Received: TypeAlias = Mapping[str, JSONValue]
|
||||
|
||||
|
||||
class ProtocolError(Exception):
|
||||
"""The peer sent something that is not a frame of this protocol; the worker stops."""
|
||||
|
||||
|
||||
def _read_exactly(sock: socket.socket, size: int) -> bytes | None:
|
||||
"""`size` bytes, or `None` when the peer closed before the first of them."""
|
||||
chunks: Final[list[bytes]] = [] # mutable-ok: a socket yields a frame in pieces
|
||||
remaining = size # rebind-ok: counts down as the pieces arrive
|
||||
while remaining:
|
||||
chunk = sock.recv(min(remaining, 1 << 20)) # rebind-ok: one piece per pass
|
||||
if not chunk:
|
||||
if chunks:
|
||||
raise ProtocolError("connection closed inside a frame")
|
||||
return None
|
||||
chunks.append(chunk)
|
||||
remaining -= len(chunk)
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
def read_frame(sock: socket.socket) -> Received | None:
|
||||
"""The next frame, or `None` once the peer has closed the connection."""
|
||||
header: Final = _read_exactly(sock, 4)
|
||||
if header is None:
|
||||
return None
|
||||
size: Final = int.from_bytes(header, "big")
|
||||
if size > MAX_FRAME_BYTES:
|
||||
raise ProtocolError(f"frame of {size} bytes exceeds {MAX_FRAME_BYTES}")
|
||||
body: Final = _read_exactly(sock, size)
|
||||
if body is None:
|
||||
raise ProtocolError("connection closed inside a frame")
|
||||
frame: Final[JSONValue] = json.loads(body) # pyright: ignore[reportAny] # untyped until the checks below
|
||||
if not isinstance(frame, Mapping) or not isinstance(frame.get("type"), str):
|
||||
raise ProtocolError("a frame is a JSON object with a string `type`")
|
||||
return frame
|
||||
|
||||
|
||||
def encode_frame(frame: Sent | Received) -> bytes:
|
||||
body: Final = json.dumps(frame, separators=(",", ":")).encode()
|
||||
return len(body).to_bytes(4, "big") + body
|
||||
|
||||
|
||||
def load(reference: str) -> None:
|
||||
"""Imports `module`, whose decorators register as they run, or registers `module:attr`."""
|
||||
module_name, _, attribute = reference.partition(":")
|
||||
module: Final = importlib.import_module(module_name)
|
||||
if attribute:
|
||||
register(getattr(module, attribute)) # pyright: ignore[reportAny] # the attribute is whatever the operator named; `register` validates it
|
||||
|
||||
|
||||
def _subscription(subscriber: Subscriber) -> SubscriptionData:
|
||||
data: Final[SubscriptionData] = {
|
||||
"name": subscriber.name,
|
||||
"events": tuple(sorted(subscriber.events)),
|
||||
"observes": subscriber.on_event is not None or subscriber.async_on_event is not None,
|
||||
"intercepts": subscriber.before_send is not None or subscriber.async_before_send is not None,
|
||||
}
|
||||
return data
|
||||
|
||||
|
||||
def hello(subscribers: Sequence[Subscriber]) -> HelloFrame:
|
||||
frame: Final[HelloFrame] = {
|
||||
"type": "hello",
|
||||
"schema": SCHEMA,
|
||||
"subscriptions": tuple(_subscription(subscriber) for subscriber in subscribers),
|
||||
}
|
||||
return frame
|
||||
|
||||
|
||||
def _plain(value: object) -> JSONValue:
|
||||
"""A handler's return value as JSON can carry it; callbacks may build patches from read-only views."""
|
||||
if isinstance(value, Mapping):
|
||||
# A handler's return value is untyped until it is encoded, and json.dumps wants a dict.
|
||||
return {str(k): _plain(v) for k, v in value.items()} # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # mutable-ok: json.dumps encodes dict, not Mapping
|
||||
if isinstance(value, Sequence) and not isinstance(value, (str, bytes)):
|
||||
return tuple(_plain(item) for item in value) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # a handler's return value is untyped until it is encoded
|
||||
if value is None or isinstance(value, (bool, int, float, str)):
|
||||
return value
|
||||
raise TypeError(f"a callback returned {type(value).__name__}, which JSON cannot carry")
|
||||
|
||||
|
||||
def _sync(handler: object | None) -> Handler | None:
|
||||
"""A handler the worker calls on a thread; `register` has already checked its kind."""
|
||||
return handler if callable(handler) else None
|
||||
|
||||
|
||||
def _asynchronous(handler: object | None) -> AsyncHandler | None:
|
||||
"""A handler the worker awaits on its loop; `register` has already checked its kind."""
|
||||
return handler if callable(handler) else None # pyright: ignore[reportReturnType] # `register` validated the kind and `callable` narrows no further
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Marker:
|
||||
"""A flush's place in one lane's queue, resolved once everything queued before it is handled."""
|
||||
|
||||
reached: Future[None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Stop:
|
||||
"""The end of one lane's queue; the consumer returns once everything before it is handled."""
|
||||
|
||||
|
||||
_Queued: TypeAlias = JSONValue | _Marker | _Stop
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Lane:
|
||||
"""One subscriber's handlers, each kept in the kind its author declared.
|
||||
|
||||
The kind is the author's scheduling decision and the worker routes on it: a handler that
|
||||
waits on the network is `async` and costs no thread, one that occupies the CPU is sync
|
||||
and gets a thread so that the loop keeps turning while it runs. Collapsing the two would
|
||||
discard that decision, and it cannot be recovered by calling a handler to see what comes
|
||||
back: calling a sync one has already run it, on whatever thread asked. `events` is the
|
||||
thread a sync observer's envelopes are handled on, in order, and exists only for one.
|
||||
"""
|
||||
|
||||
observe: Handler | None
|
||||
async_observe: AsyncHandler | None
|
||||
intercept: Handler | None
|
||||
async_intercept: AsyncHandler | None
|
||||
events: ThreadPoolExecutor | None
|
||||
|
||||
|
||||
class Worker:
|
||||
"""Serves one gateway connection over the subscribers registered when it was built.
|
||||
|
||||
Nothing that waits holds a thread it does not need. An async handler is a task on the
|
||||
worker's loop, so concurrency is the loop's and not a pool size; a sync handler has a
|
||||
thread because it blocks one; and every frame leaves through one writer, so no handler
|
||||
and no loop callback ever blocks on the socket.
|
||||
"""
|
||||
|
||||
def __init__(self, sock: socket.socket, subscribers: Sequence[Subscriber]) -> None:
|
||||
self._sock: Final = sock
|
||||
self._subscribers: Final = tuple(subscribers)
|
||||
self._loop: Final = asyncio.new_event_loop()
|
||||
self._outgoing: Final[queue.Queue[bytes | None]] = queue.Queue()
|
||||
self._writer: Final = threading.Thread(target=self._write, name="callback-writer", daemon=True)
|
||||
self._intercepts: Final = ThreadPoolExecutor(INTERCEPT_THREADS, thread_name_prefix="callback-intercept")
|
||||
self._queues: Final[
|
||||
dict[int, asyncio.Queue[_Queued]]
|
||||
] = {} # mutable-ok: filled once on the loop before serving
|
||||
self._consumers: Final[list[asyncio.Task[None]]] = [] # mutable-ok: filled once on the loop before serving
|
||||
self._pending: Final[set[asyncio.Task[None]]] = (
|
||||
set()
|
||||
) # mutable-ok: the loop's strong references to its live tasks
|
||||
self._lanes: Final = tuple(
|
||||
_Lane(
|
||||
_sync(subscriber.on_event),
|
||||
_asynchronous(subscriber.async_on_event),
|
||||
_sync(subscriber.before_send),
|
||||
_asynchronous(subscriber.async_before_send),
|
||||
ThreadPoolExecutor(1, thread_name_prefix=f"callback-events-{index}")
|
||||
if subscriber.on_event is not None and subscriber.async_on_event is None
|
||||
else None,
|
||||
)
|
||||
for index, subscriber in enumerate(self._subscribers)
|
||||
)
|
||||
|
||||
def _write(self) -> None:
|
||||
"""The only thread that writes to the socket, so that no caller ever blocks on it."""
|
||||
for data in iter(self._outgoing.get, None):
|
||||
try:
|
||||
self._sock.sendall(data)
|
||||
except OSError:
|
||||
return # the gateway is gone; the read loop ends on its own
|
||||
|
||||
def _send(self, frame: Sent) -> None:
|
||||
self._outgoing.put(encode_frame(frame))
|
||||
|
||||
def _report(self, index: int, envelope: JSONValue, error: BaseException) -> ReportFrame:
|
||||
event: Final = envelope.get("event") if isinstance(envelope, Mapping) else None
|
||||
kind: Final = event.get("type") if isinstance(event, Mapping) else None
|
||||
frame: Final[ReportFrame] = {"type": "report", "subscriber": index, "event": kind, "message": str(error)}
|
||||
return frame
|
||||
|
||||
def _patched(self, frame_id: JSONValue, result: object) -> PatchFrame:
|
||||
frame: Final[PatchFrame] = {"type": "patch", "id": frame_id, "patch": _plain(result)}
|
||||
return frame
|
||||
|
||||
def _failed(self, frame_id: JSONValue, error: BaseException) -> ErrorFrame:
|
||||
name: Final = f"{type(error).__module__}.{type(error).__qualname__}"
|
||||
frame: Final[ErrorFrame] = {"type": "error", "id": frame_id, "error_class": name, "message": str(error)}
|
||||
return frame
|
||||
|
||||
def _observe(self, index: int, handler: Handler, envelope: JSONValue) -> None:
|
||||
"""One sync observer on its lane's thread, in the order its envelopes arrived."""
|
||||
try:
|
||||
handler(envelope)
|
||||
except Exception as error: # noqa: BLE001 # an observer's failure is reported and swallowed, by contract
|
||||
self._send(self._report(index, envelope, error))
|
||||
|
||||
async def _consume(self, index: int, handler: AsyncHandler, envelopes: "asyncio.Queue[_Queued]") -> None:
|
||||
"""One async observer's whole lane: one item at a time, so order holds without a thread."""
|
||||
while True:
|
||||
item: _Queued = await envelopes.get()
|
||||
if isinstance(item, _Stop):
|
||||
return
|
||||
if isinstance(item, _Marker):
|
||||
item.reached.set_result(None)
|
||||
continue
|
||||
try:
|
||||
await handler(item)
|
||||
except Exception as error: # noqa: BLE001 # an observer's failure is reported and swallowed, by contract
|
||||
self._send(self._report(index, item, error))
|
||||
|
||||
def _intercept(self, frame_id: JSONValue, handler: Handler, request: JSONValue) -> None:
|
||||
"""One sync interceptor on a pool thread, which it holds for as long as it runs."""
|
||||
try:
|
||||
self._send(self._patched(frame_id, handler(request)))
|
||||
except Exception as error: # noqa: BLE001 # an interceptor's failure is the gateway's to act on
|
||||
self._send(self._failed(frame_id, error))
|
||||
|
||||
async def _await_intercept(self, frame_id: JSONValue, handler: AsyncHandler, request: JSONValue) -> None:
|
||||
"""One async interceptor as a task, so that a thousand of them wait on one thread."""
|
||||
try:
|
||||
self._send(self._patched(frame_id, await handler(request)))
|
||||
except Exception as error: # noqa: BLE001 # an interceptor's failure is the gateway's to act on
|
||||
self._send(self._failed(frame_id, error))
|
||||
|
||||
def _keep(self, task: "asyncio.Task[None]") -> None:
|
||||
"""asyncio keeps only a weak reference to a task; this is the strong one."""
|
||||
self._pending.add(task)
|
||||
task.add_done_callback(self._pending.discard)
|
||||
|
||||
def _spawn(self, frame_id: JSONValue, handler: AsyncHandler, request: JSONValue) -> None:
|
||||
"""On the loop: starts the interception and returns, so the read loop stays free."""
|
||||
self._keep(self._loop.create_task(self._await_intercept(frame_id, handler, request)))
|
||||
|
||||
def _enqueue(self, index: int, envelope: JSONValue) -> None:
|
||||
"""On the loop: a bounded lane drops rather than grow without limit, and says so."""
|
||||
try:
|
||||
self._queues[index].put_nowait(envelope)
|
||||
except asyncio.QueueFull:
|
||||
full: Final = RuntimeError(f"observer queue is full at {OBSERVER_QUEUE}; this envelope was dropped")
|
||||
self._send(self._report(index, envelope, full))
|
||||
|
||||
def _mark(self, index: int, reached: "Future[None]") -> None:
|
||||
"""On the loop: a marker waits for room rather than drop, so it keeps its place."""
|
||||
self._keep(self._loop.create_task(self._queues[index].put(_Marker(reached))))
|
||||
|
||||
async def _start(self) -> None:
|
||||
"""On the loop: one queue and one consumer per async observer, before the first frame."""
|
||||
for index, lane in enumerate(self._lanes):
|
||||
if lane.async_observe is not None:
|
||||
envelopes: asyncio.Queue[_Queued] = asyncio.Queue(OBSERVER_QUEUE)
|
||||
self._queues[index] = envelopes
|
||||
self._consumers.append(self._loop.create_task(self._consume(index, lane.async_observe, envelopes)))
|
||||
|
||||
async def _drain(self) -> None:
|
||||
"""On the loop: every lane sees a stop behind what was queued, then every task ends."""
|
||||
for envelopes in self._queues.values():
|
||||
await envelopes.put(_Stop())
|
||||
outstanding: Final = (*self._consumers, *self._pending)
|
||||
if outstanding:
|
||||
_ = await asyncio.gather(*outstanding, return_exceptions=True)
|
||||
|
||||
def _lane(self, frame: Received) -> tuple[int, _Lane]:
|
||||
index: Final = frame.get("subscriber")
|
||||
if not isinstance(index, int) or isinstance(index, bool) or not 0 <= index < len(self._lanes):
|
||||
raise ProtocolError(f"unknown subscriber: {index!r}")
|
||||
return index, self._lanes[index]
|
||||
|
||||
def _flush(self, frame_id: JSONValue) -> None:
|
||||
"""A marker at the back of each lane passes only after everything queued before it."""
|
||||
markers: Final[list[Future[None]]] = [] # mutable-ok: one marker per lane, collected and then waited on
|
||||
for index, lane in enumerate(self._lanes):
|
||||
if index in self._queues:
|
||||
reached: Future[None] = Future()
|
||||
self._loop.call_soon_threadsafe(self._mark, index, reached)
|
||||
markers.append(reached)
|
||||
elif lane.events is not None:
|
||||
markers.append(lane.events.submit(lambda: None))
|
||||
for marker in markers:
|
||||
marker.result()
|
||||
flushed: Final[FlushedFrame] = {"type": "flushed", "id": frame_id}
|
||||
self._send(flushed)
|
||||
|
||||
def _dispatch(self, frame: Received) -> None:
|
||||
"""Routes one frame and returns; nothing here runs a handler or touches the socket."""
|
||||
kind: Final = frame["type"]
|
||||
if kind == "event":
|
||||
index, observed = self._lane(frame)
|
||||
envelope: Final = frame.get("envelope")
|
||||
if observed.async_observe is not None:
|
||||
self._loop.call_soon_threadsafe(self._enqueue, index, envelope)
|
||||
elif observed.observe is not None and observed.events is not None:
|
||||
_ = observed.events.submit(self._observe, index, observed.observe, envelope)
|
||||
elif kind == "intercept":
|
||||
_, asked = self._lane(frame)
|
||||
if asked.async_intercept is not None:
|
||||
self._loop.call_soon_threadsafe(
|
||||
self._spawn, frame.get("id"), asked.async_intercept, frame.get("request")
|
||||
)
|
||||
elif asked.intercept is not None:
|
||||
_ = self._intercepts.submit(self._intercept, frame.get("id"), asked.intercept, frame.get("request"))
|
||||
else:
|
||||
unpatched: Final[PatchFrame] = {"type": "patch", "id": frame.get("id"), "patch": None}
|
||||
self._send(unpatched)
|
||||
elif kind == "flush":
|
||||
_ = self._intercepts.submit(self._flush, frame.get("id"))
|
||||
else:
|
||||
raise ProtocolError(f"unknown frame type: {kind!r}")
|
||||
|
||||
def serve(self) -> None:
|
||||
"""Says hello, then answers until the gateway closes the connection."""
|
||||
threading.Thread(target=self._loop.run_forever, name="callback-loop", daemon=True).start()
|
||||
self._writer.start()
|
||||
try:
|
||||
_ = asyncio.run_coroutine_threadsafe(self._start(), self._loop).result()
|
||||
self._send(hello(self._subscribers))
|
||||
while (frame := read_frame(self._sock)) is not None:
|
||||
self._dispatch(frame)
|
||||
finally:
|
||||
# In this order: no new flush can start, so every marker still resolves behind a
|
||||
# live consumer; then the lanes end; then the writer, once nothing can send again.
|
||||
self._intercepts.shutdown(wait=True)
|
||||
_ = asyncio.run_coroutine_threadsafe(self._drain(), self._loop).result()
|
||||
for lane in self._lanes:
|
||||
if lane.events is not None:
|
||||
lane.events.shutdown(wait=True)
|
||||
self._loop.call_soon_threadsafe(self._loop.stop)
|
||||
self._outgoing.put(None)
|
||||
self._writer.join()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Arguments:
|
||||
fd: int | None
|
||||
connect: str | None
|
||||
load: tuple[str, ...]
|
||||
|
||||
|
||||
def _connect(arguments: Arguments) -> socket.socket:
|
||||
if arguments.fd is not None:
|
||||
return socket.socket(fileno=arguments.fd)
|
||||
if arguments.connect is None:
|
||||
raise ValueError("one of --fd and --connect is required")
|
||||
connected: Final = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
connected.connect(arguments.connect)
|
||||
return connected
|
||||
|
||||
|
||||
def _option(parsed: Mapping[str, object], name: str) -> object:
|
||||
return parsed.get(name)
|
||||
|
||||
|
||||
def _arguments(argv: Sequence[str] | None) -> Arguments:
|
||||
parser: Final = argparse.ArgumentParser(prog="python -m litellm.callbacks_v1.host")
|
||||
where: Final = parser.add_mutually_exclusive_group(required=True)
|
||||
where.add_argument("--fd", type=int, help="an inherited, connected socket (the gateway spawned this worker)")
|
||||
where.add_argument("--connect", metavar="PATH", help="a unix socket the gateway listens on (a sidecar)")
|
||||
parser.add_argument("--load", action="append", metavar="MODULE[:ATTR]", help="a callback to import")
|
||||
parsed: Final = vars(parser.parse_args(argv))
|
||||
fd: Final = _option(parsed, "fd")
|
||||
connect: Final = _option(parsed, "connect")
|
||||
load_: Final = _option(parsed, "load")
|
||||
return Arguments(
|
||||
fd if isinstance(fd, int) else None,
|
||||
connect if isinstance(connect, str) else None,
|
||||
tuple(str(item) for item in load_) if isinstance(load_, (list, tuple)) else (), # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # argparse collects `--load` untyped
|
||||
)
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
arguments: Final = _arguments(argv)
|
||||
for reference in arguments.load:
|
||||
load(reference)
|
||||
with _connect(arguments) as sock:
|
||||
Worker(sock, snapshot()).serve()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
53
litellm/rust_bridge/callbacks_v1.py
Normal file
53
litellm/rust_bridge/callbacks_v1.py
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
"""The v1 contract functions the native call reaches for, by name.
|
||||
|
||||
`python_contract.json` pins these four on both sides. The typed authoring surface — the
|
||||
callback protocols, the envelope and patch types, and the registry — is
|
||||
`litellm.callbacks_v1`; `snapshot` is re-exported from there because the registry has one
|
||||
home, and the rest here is marshalling no callback author calls.
|
||||
"""
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from typing import Final, TypeGuard # noqa: TID251 # recursive JSON validation
|
||||
|
||||
from litellm.callbacks_v1 import JSONValue, snapshot
|
||||
|
||||
__all__ = ("new_call_id", "project_response", "report", "snapshot")
|
||||
|
||||
|
||||
def _is_json_value(
|
||||
value: object,
|
||||
) -> TypeGuard[JSONValue]: # guard-ok: recursively validates the complete JSON value
|
||||
if value is None or isinstance(value, (bool, int, float, str)):
|
||||
return True
|
||||
if isinstance(value, list):
|
||||
return all(
|
||||
_is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # list items are validated recursively
|
||||
for item in value # pyright: ignore[reportUnknownVariableType] # list items are validated recursively
|
||||
)
|
||||
if isinstance(value, dict):
|
||||
return all(
|
||||
isinstance(key, str) and _is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # mapping entries are validated recursively
|
||||
for key, item in value.items() # pyright: ignore[reportUnknownVariableType] # mapping entries are validated recursively
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def project_response(response: object) -> JSONValue:
|
||||
model_dump: Final = getattr(response, "model_dump", None)
|
||||
projected: Final = model_dump(mode="json") if callable(model_dump) else response
|
||||
json.dumps(projected, allow_nan=False)
|
||||
if _is_json_value(projected):
|
||||
return projected
|
||||
raise TypeError(f"unsupported response type: {type(response).__qualname__}")
|
||||
|
||||
|
||||
def new_call_id() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def report(name: str, event: str, error: BaseException) -> None:
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
exception: Final = (type(error), error, error.__traceback__)
|
||||
verbose_logger.exception("callback %s failed while handling %s", name, event, exc_info=exception)
|
||||
|
|
@ -1,272 +0,0 @@
|
|||
import inspect
|
||||
import json
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Literal, Protocol, TypeAlias, TypeGuard # noqa: TID251 # recursive JSON validation
|
||||
|
||||
from typing_extensions import ReadOnly, TypeAliasType, TypedDict
|
||||
|
||||
JSONScalar: TypeAlias = None | bool | int | float | str
|
||||
JSONValue = TypeAliasType( # rebind-ok: named recursive alias required for runtime validation
|
||||
"JSONValue", JSONScalar | Sequence["JSONValue"] | Mapping[str, "JSONValue"]
|
||||
)
|
||||
|
||||
SUPPORTED_SCHEMAS: Final = frozenset({1})
|
||||
EVENTS: Final = frozenset(
|
||||
{
|
||||
"call.started",
|
||||
"request.sending",
|
||||
"response.received",
|
||||
"call.succeeded",
|
||||
"call.failed",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class TimingFactsV1(TypedDict):
|
||||
start_time: ReadOnly[float]
|
||||
end_time: ReadOnly[float]
|
||||
duration: ReadOnly[float]
|
||||
|
||||
|
||||
class CallFactsV1(TypedDict):
|
||||
start_time: ReadOnly[float]
|
||||
asynchronous: ReadOnly[bool]
|
||||
metadata: ReadOnly[Mapping[str, JSONValue]]
|
||||
metadata_dropped: ReadOnly[Sequence[str]]
|
||||
|
||||
|
||||
class RequestFactsV1(TypedDict):
|
||||
model: ReadOnly[str]
|
||||
custom_llm_provider: ReadOnly[str]
|
||||
optional_params: ReadOnly[JSONValue]
|
||||
url: ReadOnly[str]
|
||||
headers: ReadOnly[Sequence[tuple[str, str]]]
|
||||
body: ReadOnly[JSONValue]
|
||||
|
||||
|
||||
ErrorFactsV1 = TypedDict( # rebind-ok: runtime TypedDict type for the reserved class field
|
||||
"ErrorFactsV1",
|
||||
{ # mutable-ok: functional TypedDict syntax requires a dict for the reserved class field
|
||||
"class": ReadOnly[str],
|
||||
"message": ReadOnly[str],
|
||||
"status_code": ReadOnly[int | None],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class CallStartedV1(CallFactsV1):
|
||||
type: ReadOnly[Literal["call.started"]]
|
||||
|
||||
|
||||
class RequestSendingV1(RequestFactsV1):
|
||||
type: ReadOnly[Literal["request.sending"]]
|
||||
|
||||
|
||||
class ResponseReceivedV1(TypedDict):
|
||||
type: ReadOnly[Literal["response.received"]]
|
||||
body: ReadOnly[str]
|
||||
|
||||
|
||||
class CallSucceededV1(TypedDict):
|
||||
type: ReadOnly[Literal["call.succeeded"]]
|
||||
timing: ReadOnly[TimingFactsV1]
|
||||
streamed: ReadOnly[bool]
|
||||
response: ReadOnly[JSONValue]
|
||||
response_error: ReadOnly[str | None]
|
||||
|
||||
|
||||
class CallFailedV1(TypedDict):
|
||||
type: ReadOnly[Literal["call.failed"]]
|
||||
timing: ReadOnly[TimingFactsV1]
|
||||
streamed: ReadOnly[bool]
|
||||
origin: ReadOnly[Literal["call", "host"]]
|
||||
error: ReadOnly[ErrorFactsV1]
|
||||
|
||||
|
||||
EventV1: TypeAlias = CallStartedV1 | RequestSendingV1 | ResponseReceivedV1 | CallSucceededV1 | CallFailedV1
|
||||
|
||||
|
||||
class EnvelopeV1(TypedDict):
|
||||
schema: ReadOnly[Literal[1]]
|
||||
call_id: ReadOnly[str]
|
||||
call_type: ReadOnly[str]
|
||||
seq: ReadOnly[int]
|
||||
event: ReadOnly[EventV1]
|
||||
|
||||
|
||||
class HeaderPatchV1(TypedDict, total=False):
|
||||
remove: ReadOnly[Sequence[str]]
|
||||
set: ReadOnly[Sequence[tuple[str, str]]]
|
||||
|
||||
|
||||
class WirePatchV1(TypedDict, total=False):
|
||||
headers: ReadOnly[HeaderPatchV1]
|
||||
body: ReadOnly[JSONValue]
|
||||
|
||||
|
||||
class Callback(Protocol):
|
||||
name: str
|
||||
schema: int
|
||||
events: frozenset[str]
|
||||
|
||||
def on_event(self, event: EnvelopeV1) -> None: ...
|
||||
|
||||
async def async_on_event(self, event: EnvelopeV1) -> None: ...
|
||||
|
||||
def before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ...
|
||||
|
||||
async def async_before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ...
|
||||
|
||||
|
||||
class RegistrationError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class UnsupportedSchema(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
class UnknownEvent(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
class MissingHandler(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
class EmptyObserverEvents(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
class HandlerKindMismatch(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
class DuplicateName(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Subscriber:
|
||||
name: str
|
||||
schema: int
|
||||
events: frozenset[str]
|
||||
on_event: object | None
|
||||
async_on_event: object | None
|
||||
before_send: object | None
|
||||
async_before_send: object | None
|
||||
|
||||
|
||||
class _Registry:
|
||||
def __init__(self) -> None:
|
||||
self.subscribers: tuple[Subscriber, ...] = ()
|
||||
self.lock: Final = threading.Lock()
|
||||
|
||||
|
||||
_REGISTRY: Final = _Registry()
|
||||
|
||||
|
||||
def _handler(callback: Callback, name: str) -> object | None:
|
||||
return getattr(callback, name, None)
|
||||
|
||||
|
||||
def _validate_kind(name: str, handler: object | None, asynchronous: bool) -> None:
|
||||
if handler is None:
|
||||
return
|
||||
if not callable(handler):
|
||||
raise HandlerKindMismatch(f"{name} must be callable")
|
||||
is_async: Final = inspect.iscoroutinefunction(handler)
|
||||
if is_async != asynchronous:
|
||||
expected: Final = "async" if asynchronous else "sync"
|
||||
raise HandlerKindMismatch(f"{name} must be {expected}")
|
||||
|
||||
|
||||
def register(callback: Callback) -> None:
|
||||
schema: Final = callback.schema
|
||||
if schema not in SUPPORTED_SCHEMAS:
|
||||
raise UnsupportedSchema(f"unsupported callback schema: {schema}")
|
||||
events: Final = frozenset(callback.events)
|
||||
unknown: Final = events - EVENTS
|
||||
if unknown:
|
||||
raise UnknownEvent(f"unknown callback events: {sorted(unknown)}")
|
||||
|
||||
on_event: Final = _handler(callback, "on_event")
|
||||
async_on_event: Final = _handler(callback, "async_on_event")
|
||||
before_send: Final = _handler(callback, "before_send")
|
||||
async_before_send: Final = _handler(callback, "async_before_send")
|
||||
handlers: Final = (on_event, async_on_event, before_send, async_before_send)
|
||||
if all(handler is None for handler in handlers):
|
||||
raise MissingHandler("callback must define at least one handler")
|
||||
if (on_event is not None or async_on_event is not None) and not events:
|
||||
raise EmptyObserverEvents("observer callback events cannot be empty")
|
||||
|
||||
_validate_kind("on_event", on_event, False)
|
||||
_validate_kind("async_on_event", async_on_event, True)
|
||||
_validate_kind("before_send", before_send, False)
|
||||
_validate_kind("async_before_send", async_before_send, True)
|
||||
|
||||
subscriber: Final = Subscriber(
|
||||
name=callback.name,
|
||||
schema=schema,
|
||||
events=events,
|
||||
on_event=on_event,
|
||||
async_on_event=async_on_event,
|
||||
before_send=before_send,
|
||||
async_before_send=async_before_send,
|
||||
)
|
||||
with _REGISTRY.lock:
|
||||
if any(existing.name == subscriber.name for existing in _REGISTRY.subscribers):
|
||||
raise DuplicateName(f"duplicate callback name: {subscriber.name}")
|
||||
_REGISTRY.subscribers = (*_REGISTRY.subscribers, subscriber)
|
||||
|
||||
|
||||
def unregister(callback: Callback) -> None:
|
||||
with _REGISTRY.lock:
|
||||
_REGISTRY.subscribers = tuple(
|
||||
subscriber for subscriber in _REGISTRY.subscribers if subscriber.name != callback.name
|
||||
)
|
||||
|
||||
|
||||
def snapshot() -> tuple[Subscriber, ...]:
|
||||
return _REGISTRY.subscribers
|
||||
|
||||
|
||||
def _is_json_value(
|
||||
value: object,
|
||||
) -> TypeGuard[JSONValue]: # guard-ok: recursively validates the complete JSON value
|
||||
if value is None or isinstance(value, (bool, int, float, str)):
|
||||
return True
|
||||
if isinstance(value, list):
|
||||
return all(
|
||||
_is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # list items are validated recursively
|
||||
for item in value # pyright: ignore[reportUnknownVariableType] # list items are validated recursively
|
||||
)
|
||||
if isinstance(value, dict):
|
||||
return all(
|
||||
isinstance(key, str) and _is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # mapping entries are validated recursively
|
||||
for key, item in value.items() # pyright: ignore[reportUnknownVariableType] # mapping entries are validated recursively
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def project_response(response: object) -> JSONValue:
|
||||
model_dump: Final = getattr(response, "model_dump", None)
|
||||
projected: Final = model_dump(mode="json") if callable(model_dump) else response
|
||||
json.dumps(projected, allow_nan=False)
|
||||
if _is_json_value(projected):
|
||||
return projected
|
||||
raise TypeError(f"unsupported response type: {type(response).__qualname__}")
|
||||
|
||||
|
||||
def new_call_id() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def report(name: str, event: str, error: BaseException) -> None:
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
exception: Final = (type(error), error, error.__traceback__)
|
||||
verbose_logger.exception("callback %s failed while handling %s", name, event, exc_info=exception)
|
||||
0
tests/test_litellm/callbacks_v1/__init__.py
Normal file
0
tests/test_litellm/callbacks_v1/__init__.py
Normal file
0
tests/test_litellm/callbacks_v1/builtin/__init__.py
Normal file
0
tests/test_litellm/callbacks_v1/builtin/__init__.py
Normal file
152
tests/test_litellm/callbacks_v1/builtin/support.py
Normal file
152
tests/test_litellm/callbacks_v1/builtin/support.py
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
"""What every port's tests share: golden envelopes, a fake transport, and the parity diff.
|
||||
|
||||
The parity diff is the migration test. A legacy logger and its port handle the same call,
|
||||
and every difference between what they send the vendor must be `Allowed` with a reason;
|
||||
an unexplained one fails. An `Allowed` entry that no longer matches anything fails too,
|
||||
and so does one citing a `Gap` its port no longer lists, so the allow-list and the gap
|
||||
ledger shrink together as the contract grows instead of rotting apart.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from fnmatch import fnmatchcase
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from litellm.callbacks_v1 import Callback, EnvelopeV1, EventName
|
||||
from litellm.callbacks_v1.builtin import anthropic_cache_control, generic_api, openmeter
|
||||
from litellm.callbacks_v1.builtin.manifest import Gap
|
||||
from litellm.callbacks_v1.builtin.port import Delivery
|
||||
from litellm.callbacks_v1.builtin.runtime import Patcher, Sink, Transport
|
||||
|
||||
ROOT: Final = Path(__file__).parents[4]
|
||||
GOLDEN_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1/golden/v1"
|
||||
FIXED_NOW: Final = datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def golden(kind: EventName) -> EnvelopeV1:
|
||||
"""The envelope Rust pins for `kind`; a port tested on it cannot drift from the contract.
|
||||
`tests/test_litellm/test_callbacks_v1.py` is what validates these files against the types."""
|
||||
return json.loads((GOLDEN_PATH / f"{kind}.json").read_text())
|
||||
|
||||
|
||||
def golden_call(terminal: EventName, **metadata: str) -> tuple[EnvelopeV1, ...]:
|
||||
"""One call's envelopes in order, ending in `terminal`, with `metadata` on call.started."""
|
||||
started: Final = golden("call.started")
|
||||
with_metadata: Final[EnvelopeV1] = {
|
||||
**started,
|
||||
"event": {**started["event"], "metadata": {**started["event"]["metadata"], **metadata}}, # pyright: ignore[reportTypedDictNotRequiredAccess, reportGeneralTypeIssues] # the golden call.started event always has metadata
|
||||
}
|
||||
return (with_metadata, golden("request.sending"), golden("response.received"), golden(terminal))
|
||||
|
||||
|
||||
@dataclass
|
||||
class FakeTransport:
|
||||
deliveries: list[Delivery] = field(default_factory=list)
|
||||
|
||||
def send(self, delivery: Delivery) -> None:
|
||||
self.deliveries.append(delivery)
|
||||
|
||||
def bodies(self) -> list[object]:
|
||||
return [json.loads(delivery.body) for delivery in self.deliveries]
|
||||
|
||||
|
||||
def plain(value: object) -> object:
|
||||
"""A contract value as plain dicts and lists, so it compares and serialises like JSON."""
|
||||
if isinstance(value, Mapping):
|
||||
return {key: plain(item) for key, item in value.items()}
|
||||
if isinstance(value, Sequence) and not isinstance(value, (str, bytes)):
|
||||
return [plain(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
# One configured port per manifest entry; `test_conventions` requires one for every entry.
|
||||
# A port is pure and unnamed, so an example is a value: the host, not the port, is what
|
||||
# needs a transport and what decides the registration name.
|
||||
OPENMETER: Final = openmeter.OpenMeter(openmeter.Config(api_key="test-key", endpoint="http://openmeter.test"))
|
||||
GENERIC_API: Final = generic_api.GenericApi(generic_api.Config(endpoint="http://generic.test/logs", batch_size=1))
|
||||
ANTHROPIC_CACHE_CONTROL: Final = anthropic_cache_control.AnthropicCacheControl(
|
||||
anthropic_cache_control.Config(points=(anthropic_cache_control.InjectionPoint(role="system"),))
|
||||
)
|
||||
EXAMPLES: Final[Mapping[str, object]] = {
|
||||
"openmeter": OPENMETER,
|
||||
"generic_api": GENERIC_API,
|
||||
"anthropic_cache_control": ANTHROPIC_CACHE_CONTROL,
|
||||
}
|
||||
|
||||
# The subscriber a host builds for each example port, under the manifest's name. Naming and
|
||||
# wrapping are the host's job, which is why these live here and not in the ports; each call
|
||||
# is also where the type checker holds a port to its protocol. The clock is fixed so a port
|
||||
# that stamps a time is comparable run to run, and one that does not is handed it anyway:
|
||||
# that is the point of the two kinds having one shape.
|
||||
SUBSCRIBERS: Final[Mapping[str, Callable[[Transport], Callback]]] = {
|
||||
"openmeter": lambda transport: Sink("openmeter", OPENMETER, transport, lambda: FIXED_NOW),
|
||||
"generic_api": lambda transport: Sink("generic_api", GENERIC_API, transport, lambda: FIXED_NOW),
|
||||
"anthropic_cache_control": lambda _transport: Patcher("anthropic_cache_control", ANTHROPIC_CACHE_CONTROL),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Allowed:
|
||||
"""A difference from the legacy twin that is intended. `path` is an fnmatch pattern over
|
||||
dotted paths (`data.cost`, `0.metadata.*`; a list index is a segment); `reason` says why.
|
||||
|
||||
`gap` is the `Gap.key` this difference exists because of. A difference with no `gap` is
|
||||
one the port intends to keep; a difference with one goes away when the contract grows."""
|
||||
|
||||
path: str
|
||||
reason: str
|
||||
gap: str | None = None
|
||||
|
||||
|
||||
def _leaves(value: object, prefix: str = "") -> Mapping[str, object]:
|
||||
if isinstance(value, Mapping) and value:
|
||||
return {
|
||||
path: leaf
|
||||
for key, item in value.items()
|
||||
for path, leaf in _leaves(item, f"{prefix}.{key}" if prefix else str(key)).items()
|
||||
}
|
||||
if isinstance(value, list) and value:
|
||||
return {
|
||||
path: leaf
|
||||
for index, item in enumerate(value)
|
||||
for path, leaf in _leaves(item, f"{prefix}.{index}" if prefix else str(index)).items()
|
||||
}
|
||||
return {prefix: value}
|
||||
|
||||
|
||||
def unexplained(legacy: object, port: object, allowed: Sequence[Allowed], gaps: Sequence[Gap]) -> tuple[str, ...]:
|
||||
"""Every difference between two vendor bodies that `allowed` does not cover, plus every
|
||||
`allowed` entry that covered nothing or cites a gap not in `gaps`. Empty means parity."""
|
||||
assert all(entry.reason.strip() for entry in allowed), "every Allowed difference needs a reason"
|
||||
open_keys: Final = {gap.key for gap in gaps}
|
||||
closed: Final = [entry for entry in allowed if entry.gap is not None and entry.gap not in open_keys]
|
||||
before: Final = _leaves(plain(legacy))
|
||||
after: Final = _leaves(plain(port))
|
||||
differences: Final = {
|
||||
**{path: f"missing in port: {path} (legacy sent {before[path]!r})" for path in before.keys() - after.keys()},
|
||||
**{path: f"only in port: {path} = {after[path]!r}" for path in after.keys() - before.keys()},
|
||||
**{
|
||||
path: f"differs: {path}: legacy {before[path]!r} != port {after[path]!r}"
|
||||
for path in before.keys() & after.keys()
|
||||
if before[path] != after[path]
|
||||
},
|
||||
}
|
||||
covered: Final = {path: [entry for entry in allowed if fnmatchcase(path, entry.path)] for path in differences}
|
||||
stale: Final = [entry for entry in allowed if not any(entry in entries for entries in covered.values())]
|
||||
return (
|
||||
*(message for path, message in sorted(differences.items()) if not covered[path]),
|
||||
*(f"stale Allowed({entry.path!r}): it no longer matches any difference" for entry in stale),
|
||||
*(f"Allowed({entry.path!r}) cites gap {entry.gap!r}, which the port does not list" for entry in closed),
|
||||
)
|
||||
|
||||
|
||||
def assert_parity(legacy: bytes, port: bytes, allowed: Sequence[Allowed], gaps: Sequence[Gap]) -> None:
|
||||
"""Compares two raw vendor request bodies as JSON and fails with one line per unexplained
|
||||
difference, so the fix is to port it or to allow it. Two empty bodies are not parity.
|
||||
`gaps` is the port's own `GAPS`, which every `Allowed.gap` has to be found in."""
|
||||
assert legacy and port, "nothing to compare: a side sent no body"
|
||||
problems: Final = unexplained(json.loads(legacy), json.loads(port), allowed, gaps)
|
||||
assert not problems, f"{len(problems)} unexplained difference(s) from the legacy twin:\n " + "\n ".join(problems)
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
"""Function-level parity: the port's `marked` against the legacy hook's Messages-route function
|
||||
on the same bodies. The legacy hook never runs on the native route, so there is no call to share."""
|
||||
|
||||
import copy
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.callbacks_v1.builtin import anthropic_cache_control as port
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
||||
from tests.test_litellm.callbacks_v1.builtin.support import golden, plain
|
||||
|
||||
EPHEMERAL = {"type": "ephemeral"}
|
||||
TURNS = [
|
||||
{"role": "user", "content": "first"},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "second"}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "third"}, {"type": "text", "text": "fourth"}]},
|
||||
]
|
||||
ALREADY_MARKED = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "kept", "cache_control": {"type": "ephemeral", "ttl": "1h"}}]}
|
||||
]
|
||||
FIVE_USERS = [{"role": "user", "content": f"turn {index}"} for index in range(5)]
|
||||
|
||||
CASES = {
|
||||
"system string": ({"system": "be brief", "messages": TURNS}, [("system", None, None)]),
|
||||
"system blocks": (
|
||||
{"system": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}], "messages": TURNS},
|
||||
[("system", None, None)],
|
||||
),
|
||||
"no system to mark": ({"messages": TURNS}, [("system", None, None)]),
|
||||
"every user turn": ({"messages": TURNS}, [("user", None, None)]),
|
||||
"last message by index": ({"messages": TURNS}, [(None, -1, None)]),
|
||||
"index out of range": ({"messages": TURNS}, [(None, 9, None)]),
|
||||
"ttl": ({"system": "be brief", "messages": TURNS}, [("system", None, "1h"), (None, -1, "5m")]),
|
||||
"client breakpoint is kept": ({"messages": ALREADY_MARKED}, [("user", None, None)]),
|
||||
"stops at four breakpoints": (
|
||||
{"system": "be brief", "messages": FIVE_USERS},
|
||||
[("system", None, None), ("user", None, None)],
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def legacy(body: dict, points: list[tuple]) -> dict:
|
||||
injection_points = [
|
||||
{"location": "message", "role": role, "index": index, "control": {**EPHEMERAL, **({"ttl": ttl} if ttl else {})}}
|
||||
for role, index, ttl in points
|
||||
]
|
||||
messages, system, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
|
||||
copy.deepcopy(body["messages"]),
|
||||
copy.deepcopy(body.get("system")),
|
||||
injection_points, # pyright: ignore[reportArgumentType] # plain dicts stand in for the legacy TypedDicts
|
||||
)
|
||||
return {**body, "messages": messages, **({"system": system} if "system" in body else {})}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", CASES)
|
||||
def test_the_port_marks_a_messages_body_exactly_as_the_legacy_hook_does(name: str) -> None:
|
||||
body, points = CASES[name]
|
||||
config = port.Config(points=tuple(port.InjectionPoint(role, index, ttl) for role, index, ttl in points))
|
||||
original = copy.deepcopy(body)
|
||||
|
||||
assert plain(port.AnthropicCacheControl(config).marked(body)) == legacy(body, points)
|
||||
assert body == original
|
||||
|
||||
|
||||
def test_the_patch_covers_only_the_configured_providers() -> None:
|
||||
request = {**golden("request.sending")["event"], "body": {"system": "be brief", "messages": TURNS}}
|
||||
config = port.Config(points=(port.InjectionPoint(role="system"),))
|
||||
|
||||
interceptor = port.AnthropicCacheControl(config)
|
||||
|
||||
assert interceptor.payload(request) is None # pyright: ignore[reportArgumentType] # the golden event plus a Messages body
|
||||
patch = interceptor.payload({**request, "custom_llm_provider": "anthropic"}) # pyright: ignore[reportArgumentType] # the golden event plus a Messages body
|
||||
|
||||
assert patch is not None and set(patch) == {"body"}
|
||||
assert plain(patch["body"])["system"] == [{"type": "text", "text": "be brief", "cache_control": EPHEMERAL}] # pyright: ignore[reportIndexIssue, reportTypedDictNotRequiredAccess, reportCallIssue, reportArgumentType] # the patch body is the Messages mapping
|
||||
|
||||
|
||||
def test_a_body_that_is_not_a_messages_request_is_left_alone() -> None:
|
||||
config = port.Config(points=(port.InjectionPoint(role="user"),))
|
||||
assert port.AnthropicCacheControl(config).marked({"input": "hello"}) is None
|
||||
assert port.AnthropicCacheControl(port.Config(points=())).marked({"messages": TURNS}) is None
|
||||
180
tests/test_litellm/callbacks_v1/builtin/test_conventions.py
Normal file
180
tests/test_litellm/callbacks_v1/builtin/test_conventions.py
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
"""The rules every port obeys, checked for every manifest entry. A new port gets these for free."""
|
||||
|
||||
import ast
|
||||
import importlib
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Generator
|
||||
from dataclasses import is_dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm import callbacks_v1
|
||||
from litellm.callbacks_v1.builtin import port as port_vocabulary
|
||||
from litellm.callbacks_v1.builtin.manifest import PORTS, Entry, Gap, ledger
|
||||
from litellm.callbacks_v1.builtin.port import InterceptorPort, SinkPort
|
||||
from litellm.callbacks_v1.builtin.runtime import Sink
|
||||
from litellm.integrations.custom_batch_logger import CustomBatchLogger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from tests.test_litellm.callbacks_v1.builtin.support import EXAMPLES, SUBSCRIBERS, FakeTransport, golden_call
|
||||
|
||||
# The legacy hooks a v1 sink can stand in for. A legacy twin overriding any other
|
||||
# CustomLogger hook does something the v1 contract has no place for.
|
||||
SINK_HOOKS: Final = frozenset(
|
||||
{
|
||||
"log_pre_api_call",
|
||||
"log_post_api_call",
|
||||
"log_success_event",
|
||||
"log_failure_event",
|
||||
"async_log_success_event",
|
||||
"async_log_failure_event",
|
||||
"async_log_pre_api_call",
|
||||
}
|
||||
)
|
||||
PUBLIC_HOOKS: Final = frozenset(
|
||||
name
|
||||
for name, value in vars(CustomLogger).items()
|
||||
if callable(value) and (name.startswith(("log_", "async_")) or name.endswith("_hook"))
|
||||
)
|
||||
|
||||
# What a port may import. A port is values and pure methods, the shape a sandboxed custom
|
||||
# callback is confined to; anything that can reach a socket, a thread, the environment or the
|
||||
# rest of litellm belongs to `runtime.py`, which a port may not import at all. A new pure
|
||||
# stdlib module is a reviewed line here.
|
||||
PURE_MODULES: Final = frozenset(
|
||||
{
|
||||
"base64",
|
||||
"collections.abc",
|
||||
"dataclasses",
|
||||
"datetime",
|
||||
"hashlib",
|
||||
"json",
|
||||
"math",
|
||||
"re",
|
||||
"types",
|
||||
"typing",
|
||||
"typing_extensions",
|
||||
"urllib.parse",
|
||||
"litellm.callbacks_v1",
|
||||
"litellm.callbacks_v1.builtin.port",
|
||||
}
|
||||
)
|
||||
# What a port never carries. `runtime.py` names it and wraps it into a subscriber, and
|
||||
# `manifest.py` holds everything the migration needs to say about it.
|
||||
NOT_A_PORTS: Final = frozenset(
|
||||
{"NAME", "GAPS", "events", "on_event", "async_on_event", "before_send", "async_before_send"}
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.parametrize("entry", PORTS, ids=[entry.name for entry in PORTS])
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clean_registry() -> Generator[None]:
|
||||
yield
|
||||
for subscriber in callbacks_v1.snapshot():
|
||||
callbacks_v1.unregister(subscriber)
|
||||
|
||||
|
||||
def _legacy(entry: Entry) -> type:
|
||||
module, _, attribute = entry.legacy.partition(":")
|
||||
return getattr(importlib.import_module(module), attribute)
|
||||
|
||||
|
||||
def test_importing_a_port_registers_nothing(entry: Entry) -> None:
|
||||
script: Final = (
|
||||
f"import {entry.module.__name__}; from litellm import callbacks_v1; "
|
||||
"import sys; sys.exit(len(callbacks_v1.snapshot()))"
|
||||
)
|
||||
assert subprocess.run([sys.executable, "-c", script], check=False).returncode == 0
|
||||
|
||||
|
||||
def _impure_imports(source: str) -> tuple[str, ...]:
|
||||
nodes: Final = tuple(ast.walk(ast.parse(source)))
|
||||
plain: Final = tuple(alias.name for node in nodes if isinstance(node, ast.Import) for alias in node.names)
|
||||
froms: Final = tuple(
|
||||
str(node.module)
|
||||
for node in nodes
|
||||
if isinstance(node, ast.ImportFrom) and (node.level or node.module not in PURE_MODULES)
|
||||
)
|
||||
return (*(name for name in plain if name not in PURE_MODULES), *froms)
|
||||
|
||||
|
||||
def test_a_port_imports_nothing_that_can_act(entry: Entry) -> None:
|
||||
for module in (entry.module, port_vocabulary):
|
||||
source = Path(str(module.__file__)).read_text() # rebind-ok: one source per checked module
|
||||
assert _impure_imports(source) == (), f"{module.__name__} imports outside the pure allow-list"
|
||||
|
||||
|
||||
def test_a_port_is_a_frozen_class_over_a_frozen_config(entry: Entry) -> None:
|
||||
example: Final = EXAMPLES[entry.name]
|
||||
assert all(isinstance(gap, Gap) for gap in entry.gaps)
|
||||
assert is_dataclass(example) and type(example).__dataclass_params__.frozen
|
||||
assert is_dataclass(entry.module.Config) and entry.module.Config.__dataclass_params__.frozen
|
||||
assert "Legacy twin:" in (entry.module.__doc__ or "")
|
||||
|
||||
|
||||
def test_a_port_declares_the_protocol_of_its_kind_and_nothing_of_the_contract(entry: Entry) -> None:
|
||||
"""A port names its protocol as a base, so the type checker holds it to the shape at the
|
||||
class. This is the same claim at runtime, for a port whose module a type checker never saw."""
|
||||
declared: Final = type(EXAMPLES[entry.name]).__mro__
|
||||
expected: Final = SinkPort if entry.kind == "sink" else InterceptorPort
|
||||
assert expected in declared, f"{entry.name} is a {entry.kind}; it should declare {expected.__name__} as its base"
|
||||
assert (SinkPort in declared) == (entry.kind == "sink")
|
||||
|
||||
example: Final = EXAMPLES[entry.name]
|
||||
carried: Final = {member for member in NOT_A_PORTS if hasattr(example, member)}
|
||||
assert not carried, f"{entry.name} carries {sorted(carried)}; that is runtime.py's or manifest.py's"
|
||||
|
||||
|
||||
def test_a_port_registers_under_its_manifest_name_with_the_handlers_of_its_kind(entry: Entry) -> None:
|
||||
callbacks_v1.register(SUBSCRIBERS[entry.name](FakeTransport()))
|
||||
|
||||
(subscriber,) = callbacks_v1.snapshot()
|
||||
assert subscriber.name == entry.name
|
||||
observes: Final = subscriber.on_event is not None or subscriber.async_on_event is not None
|
||||
intercepts: Final = subscriber.before_send is not None or subscriber.async_before_send is not None
|
||||
assert (observes, intercepts) == ((True, False) if entry.kind == "sink" else (False, True))
|
||||
# A sink's handler only enqueues, so the sync one serves async calls too.
|
||||
assert subscriber.async_on_event is None
|
||||
|
||||
|
||||
def test_a_sink_handler_does_no_io_and_keeps_no_finished_call(entry: Entry) -> None:
|
||||
if entry.kind != "sink":
|
||||
pytest.skip("interceptors have no outbox")
|
||||
transport: Final = FakeTransport()
|
||||
callback: Final = SUBSCRIBERS[entry.name](transport)
|
||||
assert isinstance(callback, Sink), "a sink port supplies pure methods; `runtime.Sink` is its only subscriber"
|
||||
|
||||
for envelope in golden_call("call.succeeded", user_api_key_user_id="user-1"):
|
||||
callback.on_event(envelope)
|
||||
|
||||
assert callback.outbox.flush()
|
||||
assert len(transport.deliveries) == 1
|
||||
assert callback.open_calls() == 0
|
||||
|
||||
|
||||
def test_the_legacy_twin_resolves_and_a_sink_twin_is_only_a_sink(entry: Entry) -> None:
|
||||
legacy: Final = _legacy(entry)
|
||||
if entry.kind != "sink":
|
||||
return
|
||||
overridden: Final = {
|
||||
name
|
||||
for cls in legacy.__mro__
|
||||
if cls not in (CustomLogger, CustomBatchLogger, object)
|
||||
for name in vars(cls)
|
||||
if name in PUBLIC_HOOKS
|
||||
}
|
||||
assert overridden <= SINK_HOOKS, f"{entry.legacy} also overrides {sorted(overridden - SINK_HOOKS)}"
|
||||
|
||||
|
||||
def test_a_port_names_each_gap_once_by_what_the_contract_lacks(entry: Entry) -> None:
|
||||
keys: Final = tuple(gap.key for gap in entry.gaps)
|
||||
assert len(keys) == len(frozenset(keys)), f"{entry.name} lists a gap key twice: {sorted(keys)}"
|
||||
assert all(key and key == key.lower() and key.replace("_", "").isalnum() for key in keys)
|
||||
|
||||
|
||||
def test_ready_means_no_open_gap(entry: Entry) -> None:
|
||||
assert entry.status != "ready" or entry.gaps == ()
|
||||
assert entry.name in ledger()
|
||||
101
tests/test_litellm/callbacks_v1/builtin/test_generic_api.py
Normal file
101
tests/test_litellm/callbacks_v1/builtin/test_generic_api.py
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
import json
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.callbacks_v1.builtin import generic_api
|
||||
from litellm.callbacks_v1.builtin.port import CallRecord
|
||||
from litellm.callbacks_v1.builtin.runtime import CallJoin, Sink
|
||||
from tests.test_litellm.callbacks_v1.builtin.support import golden_call
|
||||
|
||||
PORT = generic_api.GenericApi(generic_api.Config(endpoint="http://generic.test"))
|
||||
FAILURES_ONLY = generic_api.GenericApi(
|
||||
generic_api.Config(endpoint="http://generic.test", event_types=frozenset({"llm_api_failure"}))
|
||||
)
|
||||
# A sink is handed the clock whether or not it stamps a time; this one does not.
|
||||
UNUSED_NOW = datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc)
|
||||
|
||||
COMMON = {
|
||||
"litellm_call_id": "call-123",
|
||||
"call_type": "ocr",
|
||||
"startTime": 10.0,
|
||||
"model": "model",
|
||||
"custom_llm_provider": "provider",
|
||||
"model_parameters": {"temperature": 0.2},
|
||||
}
|
||||
|
||||
|
||||
def joined(terminal: str) -> CallRecord:
|
||||
join = CallJoin()
|
||||
records = [join.accept(envelope) for envelope in golden_call(terminal)] # pyright: ignore[reportArgumentType] # terminal is an EventName literal at every call site
|
||||
(record,) = [record for record in records if record is not None]
|
||||
return record
|
||||
|
||||
|
||||
def test_a_succeeded_call_uses_the_standard_logging_payload_names() -> None:
|
||||
assert PORT.payload(joined("call.succeeded"), UNUSED_NOW) == {
|
||||
**COMMON,
|
||||
"id": "response",
|
||||
"stream": False,
|
||||
"status": "success",
|
||||
"endTime": 12.5,
|
||||
"response_time": 2.5,
|
||||
"response": {"id": "response"},
|
||||
}
|
||||
|
||||
|
||||
def test_a_failed_call_carries_error_information_instead_of_a_response() -> None:
|
||||
assert PORT.payload(joined("call.failed"), UNUSED_NOW) == {
|
||||
**COMMON,
|
||||
"id": "call-123",
|
||||
"stream": True,
|
||||
"status": "failure",
|
||||
"endTime": 11.0,
|
||||
"response_time": 1.0,
|
||||
"error_str": "bad request",
|
||||
"error_information": {"error_class": "ValueError", "error_message": "bad request", "error_code": "400"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("log_format", "bodies"),
|
||||
[
|
||||
("json_array", ['[{"id": "a"}, {"id": "b"}]']),
|
||||
("ndjson", ['{"id": "a"}\n{"id": "b"}']),
|
||||
("single", ['{"id": "a"}', '{"id": "b"}']),
|
||||
],
|
||||
)
|
||||
def test_a_batch_is_framed_by_log_format(log_format: generic_api.LogFormat, bodies: list[str]) -> None:
|
||||
port = generic_api.GenericApi(generic_api.Config(endpoint="http://generic.test", log_format=log_format))
|
||||
batch = ({"id": "a"}, {"id": "b"})
|
||||
|
||||
sent = port.deliveries(batch) # pyright: ignore[reportArgumentType] # framing does not depend on the record's fields
|
||||
|
||||
assert [delivery.body.decode() for delivery in sent] == bodies
|
||||
assert {delivery.url for delivery in sent} == {"http://generic.test"}
|
||||
|
||||
|
||||
def test_an_outcome_left_out_of_event_types_has_no_record() -> None:
|
||||
assert FAILURES_ONLY.payload(joined("call.succeeded"), UNUSED_NOW) is None
|
||||
assert FAILURES_ONLY.payload(joined("call.failed"), UNUSED_NOW) is not None
|
||||
|
||||
|
||||
def test_event_types_filter_what_is_queued() -> None:
|
||||
from tests.test_litellm.callbacks_v1.builtin.support import FakeTransport
|
||||
|
||||
transport = FakeTransport()
|
||||
sink = Sink("generic_api", FAILURES_ONLY, transport)
|
||||
for terminal in ("call.succeeded", "call.failed"):
|
||||
for envelope in golden_call(terminal):
|
||||
sink.on_event(envelope)
|
||||
|
||||
assert sink.outbox.flush()
|
||||
assert [record["status"] for body in transport.bodies() for record in body] == ["failure"] # pyright: ignore[reportGeneralTypeIssues, reportIndexIssue] # json bodies are lists of records here
|
||||
|
||||
|
||||
def test_config_parses_the_legacy_header_variable() -> None:
|
||||
config = generic_api.Config.from_env(
|
||||
{"GENERIC_LOGGER_ENDPOINT": "http://e", "GENERIC_LOGGER_HEADERS": "a=1, b = 2"}
|
||||
)
|
||||
assert config.headers == (("Content-Type", "application/json"), ("a", "1"), ("b", "2"))
|
||||
assert json.loads(json.dumps(config.headers)) == [["Content-Type", "application/json"], ["a", "1"], ["b", "2"]]
|
||||
54
tests/test_litellm/callbacks_v1/builtin/test_openmeter.py
Normal file
54
tests/test_litellm/callbacks_v1/builtin/test_openmeter.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
import pytest
|
||||
|
||||
from litellm.callbacks_v1.builtin import openmeter
|
||||
from litellm.callbacks_v1.builtin.port import CallRecord
|
||||
from litellm.callbacks_v1.builtin.runtime import CallJoin
|
||||
from tests.test_litellm.callbacks_v1.builtin.support import FIXED_NOW, golden_call
|
||||
|
||||
PORT = openmeter.OpenMeter(openmeter.Config(api_key="test-key", endpoint="http://openmeter.test/"))
|
||||
|
||||
|
||||
def joined(terminal: str, **metadata: str) -> CallRecord:
|
||||
join = CallJoin()
|
||||
records = [join.accept(envelope) for envelope in golden_call(terminal, **metadata)] # pyright: ignore[reportArgumentType] # terminal is an EventName literal at every call site
|
||||
(record,) = [record for record in records if record is not None]
|
||||
return record
|
||||
|
||||
|
||||
def test_a_succeeded_call_becomes_one_cloud_event() -> None:
|
||||
assert PORT.payload(joined("call.succeeded", user_api_key_user_id="user-1"), FIXED_NOW) == {
|
||||
"specversion": "1.0",
|
||||
"type": "litellm_tokens",
|
||||
"id": "response",
|
||||
"time": "2026-01-02T03:04:05+00:00",
|
||||
"subject": "user-1",
|
||||
"source": "litellm-proxy",
|
||||
"data": {"model": "model", "cost": None},
|
||||
}
|
||||
|
||||
|
||||
def test_a_failed_call_is_not_metered() -> None:
|
||||
assert PORT.payload(joined("call.failed", user_api_key_user_id="user-1"), FIXED_NOW) is None
|
||||
|
||||
|
||||
def test_a_call_without_a_subject_is_an_error_like_the_legacy_logger() -> None:
|
||||
with pytest.raises(ValueError, match="user is required"):
|
||||
PORT.payload(joined("call.succeeded"), FIXED_NOW)
|
||||
|
||||
|
||||
def test_each_event_is_its_own_authorised_request() -> None:
|
||||
(delivery,) = PORT.deliveries(({"id": "a"},)) # pyright: ignore[reportArgumentType] # framing does not depend on the event's fields
|
||||
|
||||
assert delivery.url == "http://openmeter.test/api/v1/events"
|
||||
assert dict(delivery.headers) == {
|
||||
"Content-Type": "application/cloudevents+json",
|
||||
"Authorization": "Bearer test-key",
|
||||
}
|
||||
assert delivery.body == b'{"id": "a"}'
|
||||
|
||||
|
||||
def test_config_reads_the_legacy_environment_variables() -> None:
|
||||
config = openmeter.Config.from_env({"OPENMETER_API_KEY": "k", "OPENMETER_API_ENDPOINT": "http://host/"})
|
||||
assert (config.api_key, config.url, config.event_type) == ("k", "http://host/api/v1/events", "litellm_tokens")
|
||||
with pytest.raises(ValueError, match="OPENMETER_API_KEY"):
|
||||
openmeter.Config.from_env({})
|
||||
94
tests/test_litellm/callbacks_v1/builtin/test_outbox.py
Normal file
94
tests/test_litellm/callbacks_v1/builtin/test_outbox.py
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
import threading
|
||||
import time
|
||||
|
||||
from litellm.callbacks_v1.builtin.outbox import Outbox
|
||||
from litellm.callbacks_v1.builtin.runtime import CallJoin
|
||||
from tests.test_litellm.callbacks_v1.builtin.support import golden, golden_call
|
||||
|
||||
|
||||
def test_put_returns_before_a_slow_delivery_finishes() -> None:
|
||||
release = threading.Event()
|
||||
delivered: list[tuple[int, ...]] = []
|
||||
|
||||
def deliver(batch: tuple[int, ...]) -> None:
|
||||
release.wait(2)
|
||||
delivered.append(batch)
|
||||
|
||||
outbox = Outbox("slow", deliver)
|
||||
started = time.monotonic()
|
||||
outbox.put(1)
|
||||
|
||||
assert time.monotonic() - started < 0.5
|
||||
assert delivered == []
|
||||
release.set()
|
||||
assert outbox.flush()
|
||||
assert delivered == [(1,)]
|
||||
|
||||
|
||||
def test_items_are_delivered_batch_size_at_a_time_and_flush_sends_the_rest() -> None:
|
||||
delivered: list[tuple[int, ...]] = []
|
||||
outbox = Outbox("batched", delivered.append, batch_size=2, flush_interval=60)
|
||||
|
||||
for item in (1, 2, 3):
|
||||
outbox.put(item)
|
||||
|
||||
assert outbox.flush()
|
||||
assert delivered == [(1, 2), (3,)]
|
||||
|
||||
|
||||
def test_a_failed_delivery_is_dropped_and_the_next_one_still_goes_out() -> None:
|
||||
delivered: list[tuple[int, ...]] = []
|
||||
|
||||
def deliver(batch: tuple[int, ...]) -> None:
|
||||
if batch == (1,):
|
||||
raise RuntimeError("vendor is down")
|
||||
delivered.append(batch)
|
||||
|
||||
outbox = Outbox("flaky", deliver)
|
||||
outbox.put(1)
|
||||
outbox.put(2)
|
||||
|
||||
assert outbox.flush()
|
||||
assert delivered == [(2,)]
|
||||
|
||||
|
||||
def test_a_full_outbox_drops_the_new_item_instead_of_blocking_the_call() -> None:
|
||||
release = threading.Event()
|
||||
outbox = Outbox("full", lambda batch: release.wait(2) and None, capacity=1)
|
||||
outbox.put(1)
|
||||
time.sleep(0.05)
|
||||
outbox.put(2)
|
||||
|
||||
started = time.monotonic()
|
||||
outbox.put(3)
|
||||
|
||||
assert time.monotonic() - started < 0.5
|
||||
release.set()
|
||||
assert outbox.flush()
|
||||
|
||||
|
||||
def test_the_join_returns_one_record_at_the_terminal_envelope_and_forgets_the_call() -> None:
|
||||
join = CallJoin()
|
||||
started, request, received, terminal = golden_call("call.succeeded")
|
||||
|
||||
assert [join.accept(envelope) for envelope in (started, request, received)] == [None, None, None]
|
||||
record = join.accept(terminal)
|
||||
|
||||
assert record is not None
|
||||
assert (record.call_id, record.call_type) == ("call-123", "ocr")
|
||||
assert (record.started, record.request, record.terminal) == (started["event"], request["event"], terminal["event"])
|
||||
assert join.open_calls() == 0
|
||||
|
||||
|
||||
def test_a_call_that_failed_before_sending_has_no_request() -> None:
|
||||
record = CallJoin().accept(golden("call.failed"))
|
||||
assert record is not None and record.request is None and record.started is None
|
||||
|
||||
|
||||
def test_calls_that_never_finish_are_bounded() -> None:
|
||||
join = CallJoin(capacity=2)
|
||||
started = golden("call.started")
|
||||
for call_id in ("a", "b", "c"):
|
||||
join.accept({**started, "call_id": call_id})
|
||||
|
||||
assert join.open_calls() == 2
|
||||
34
tests/test_litellm/callbacks_v1/builtin/test_parity_diff.py
Normal file
34
tests/test_litellm/callbacks_v1/builtin/test_parity_diff.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
"""The parity diff itself: what it reports is what keeps the allow-lists and the gap ledger honest."""
|
||||
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from litellm.callbacks_v1.builtin.manifest import Gap
|
||||
from tests.test_litellm.callbacks_v1.builtin.support import Allowed, unexplained
|
||||
|
||||
COST: Final = Gap("fact", "cost", "kwargs['response_cost']", "no cost fact")
|
||||
LEGACY: Final = MappingProxyType({"id": "call-1", "data": MappingProxyType({"cost": 0.25, "model": "claude"})})
|
||||
PORT: Final = MappingProxyType({"id": "call-1", "data": MappingProxyType({"cost": None, "model": "claude"})})
|
||||
|
||||
|
||||
def test_identical_bodies_need_no_allowance() -> None:
|
||||
assert unexplained(LEGACY, LEGACY, (), ()) == ()
|
||||
|
||||
|
||||
def test_a_difference_nobody_allowed_is_reported_by_path() -> None:
|
||||
assert unexplained(LEGACY, PORT, (), ()) == ("differs: data.cost: legacy 0.25 != port None",)
|
||||
|
||||
|
||||
def test_a_difference_allowed_by_an_open_gap_is_parity() -> None:
|
||||
assert unexplained(LEGACY, PORT, (Allowed("data.cost", "no cost fact", gap="cost"),), (COST,)) == ()
|
||||
|
||||
|
||||
def test_an_allowance_outlives_neither_its_difference_nor_its_gap() -> None:
|
||||
allowed: Final = (Allowed("data.cost", "no cost fact", gap="cost"),)
|
||||
|
||||
assert unexplained(LEGACY, LEGACY, allowed, (COST,)) == (
|
||||
"stale Allowed('data.cost'): it no longer matches any difference",
|
||||
)
|
||||
assert unexplained(LEGACY, PORT, allowed, ()) == (
|
||||
"Allowed('data.cost') cites gap 'cost', which the port does not list",
|
||||
)
|
||||
299
tests/test_litellm/callbacks_v1/test_host.py
Normal file
299
tests/test_litellm/callbacks_v1/test_host.py
Normal file
|
|
@ -0,0 +1,299 @@
|
|||
"""The extension host worker, driven over a socketpair the way a gateway would drive it."""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm import callbacks_v1
|
||||
from litellm.callbacks_v1 import EnvelopeV1, RequestFactsV1, WirePatchV1
|
||||
from litellm.callbacks_v1.host import INTERCEPT_THREADS, ProtocolError, Worker, encode_frame, read_frame
|
||||
from tests.test_litellm.callbacks_v1.builtin.support import golden
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clean_registry() -> Generator[None]:
|
||||
yield
|
||||
for subscriber in callbacks_v1.snapshot():
|
||||
callbacks_v1.unregister(subscriber)
|
||||
|
||||
|
||||
class Gateway:
|
||||
"""The gateway's end of the socket: starts a worker on a thread and speaks frames to it."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.sock, worker_sock = socket.socketpair()
|
||||
self.sock.settimeout(5)
|
||||
self.thread = threading.Thread(target=Worker(worker_sock, callbacks_v1.snapshot()).serve, daemon=True)
|
||||
self.thread.start()
|
||||
self.hello = self.read()
|
||||
|
||||
def send(self, **frame: object) -> None:
|
||||
self.sock.sendall(encode_frame(frame))
|
||||
|
||||
def read(self) -> dict:
|
||||
frame = read_frame(self.sock)
|
||||
assert frame is not None
|
||||
return dict(frame)
|
||||
|
||||
def flush(self) -> list[dict]:
|
||||
"""Everything the worker said before it confirmed the flush."""
|
||||
self.send(type="flush", id="f")
|
||||
frames = []
|
||||
while (frame := self.read()) != {"type": "flushed", "id": "f"}:
|
||||
frames.append(frame)
|
||||
return frames
|
||||
|
||||
def close(self) -> None:
|
||||
self.sock.close()
|
||||
self.thread.join(5)
|
||||
assert not self.thread.is_alive()
|
||||
|
||||
|
||||
def test_hello_reports_every_subscription_as_data() -> None:
|
||||
@callbacks_v1.on_event("call.succeeded", "call.failed", name="recorder")
|
||||
def record(event: EnvelopeV1) -> None: ...
|
||||
|
||||
@callbacks_v1.before_send
|
||||
async def patch(request: RequestFactsV1) -> WirePatchV1 | None: ...
|
||||
|
||||
gateway = Gateway()
|
||||
|
||||
assert gateway.hello == {
|
||||
"type": "hello",
|
||||
"schema": 1,
|
||||
"subscriptions": [
|
||||
{"name": "recorder", "events": ["call.failed", "call.succeeded"], "observes": True, "intercepts": False},
|
||||
{"name": f"{__name__}.{patch.__qualname__}", "events": [], "observes": False, "intercepts": True},
|
||||
],
|
||||
}
|
||||
gateway.close()
|
||||
|
||||
|
||||
def test_one_subscriber_gets_its_envelopes_in_the_order_they_were_sent() -> None:
|
||||
seen: list[int] = []
|
||||
|
||||
@callbacks_v1.on_event("response.received", name="ordered")
|
||||
async def record(event: EnvelopeV1) -> None:
|
||||
seen.append(event["seq"])
|
||||
|
||||
gateway = Gateway()
|
||||
for seq in range(50):
|
||||
gateway.send(type="event", subscriber=0, envelope={**golden("response.received"), "seq": seq})
|
||||
|
||||
assert gateway.flush() == []
|
||||
assert seen == list(range(50))
|
||||
gateway.close()
|
||||
|
||||
|
||||
def test_an_observer_that_raises_is_reported_and_the_next_envelope_still_arrives() -> None:
|
||||
seen: list[str] = []
|
||||
|
||||
@callbacks_v1.on_event("call.started", "call.failed", name="flaky")
|
||||
def record(event: EnvelopeV1) -> None:
|
||||
seen.append(event["event"]["type"])
|
||||
if event["event"]["type"] == "call.started":
|
||||
raise ValueError("boom")
|
||||
|
||||
gateway = Gateway()
|
||||
gateway.send(type="event", subscriber=0, envelope=golden("call.started"))
|
||||
gateway.send(type="event", subscriber=0, envelope=golden("call.failed"))
|
||||
|
||||
assert gateway.flush() == [{"type": "report", "subscriber": 0, "event": "call.started", "message": "boom"}]
|
||||
assert seen == ["call.started", "call.failed"]
|
||||
gateway.close()
|
||||
|
||||
|
||||
def test_an_interceptor_answers_with_its_patch_even_from_a_read_only_view() -> None:
|
||||
@callbacks_v1.before_send
|
||||
def patch(request: RequestFactsV1) -> WirePatchV1 | None:
|
||||
return {"body": MappingProxyType({"model": request["model"], "blocks": ({"a": 1},)})}
|
||||
|
||||
gateway = Gateway()
|
||||
gateway.send(type="intercept", id=7, subscriber=0, request=golden("request.sending")["event"])
|
||||
|
||||
assert gateway.read() == {"type": "patch", "id": 7, "patch": {"body": {"model": "model", "blocks": [{"a": 1}]}}}
|
||||
gateway.close()
|
||||
|
||||
|
||||
def test_an_interceptor_that_raises_is_an_error_for_the_gateway_to_act_on() -> None:
|
||||
@callbacks_v1.before_send
|
||||
async def patch(request: RequestFactsV1) -> WirePatchV1 | None:
|
||||
raise PermissionError("not this model")
|
||||
|
||||
gateway = Gateway()
|
||||
gateway.send(type="intercept", id="a", subscriber=0, request=golden("request.sending")["event"])
|
||||
|
||||
assert gateway.read() == {
|
||||
"type": "error",
|
||||
"id": "a",
|
||||
"error_class": "builtins.PermissionError",
|
||||
"message": "not this model",
|
||||
}
|
||||
gateway.close()
|
||||
|
||||
|
||||
def test_an_interception_does_not_wait_behind_a_slow_observer() -> None:
|
||||
release = threading.Event()
|
||||
|
||||
@callbacks_v1.on_event("call.started", name="slow")
|
||||
def slow(event: EnvelopeV1) -> None:
|
||||
release.wait(5)
|
||||
|
||||
@callbacks_v1.before_send
|
||||
def patch(request: RequestFactsV1) -> WirePatchV1 | None:
|
||||
return None
|
||||
|
||||
gateway = Gateway()
|
||||
gateway.send(type="event", subscriber=0, envelope=golden("call.started"))
|
||||
gateway.send(type="intercept", id=1, subscriber=1, request=golden("request.sending")["event"])
|
||||
|
||||
assert gateway.read() == {"type": "patch", "id": 1, "patch": None}
|
||||
release.set()
|
||||
gateway.close()
|
||||
|
||||
|
||||
def test_every_async_interception_is_in_flight_at_once() -> None:
|
||||
"""Concurrency is the loop's, not a pool size: an async handler that only waits holds no
|
||||
thread, so far more interceptions run together than there are intercept threads. Awaiting
|
||||
each other is how they prove it — none of them can finish until all of them have started."""
|
||||
burst = INTERCEPT_THREADS * 4
|
||||
started = 0
|
||||
everyone: asyncio.Event | None = None
|
||||
|
||||
@callbacks_v1.before_send
|
||||
async def patch(request: RequestFactsV1) -> WirePatchV1 | None:
|
||||
nonlocal started, everyone
|
||||
if everyone is None: # every handler runs on the one loop, so this races with nothing
|
||||
everyone = asyncio.Event()
|
||||
started += 1
|
||||
if started == burst:
|
||||
everyone.set()
|
||||
await asyncio.wait_for(everyone.wait(), 3)
|
||||
return None
|
||||
|
||||
gateway = Gateway()
|
||||
for frame_id in range(burst):
|
||||
gateway.send(type="intercept", id=frame_id, subscriber=0, request=golden("request.sending")["event"])
|
||||
|
||||
answers = sorted((gateway.read() for _ in range(burst)), key=lambda frame: frame["id"])
|
||||
assert answers == [{"type": "patch", "id": frame_id, "patch": None} for frame_id in range(burst)]
|
||||
gateway.close()
|
||||
|
||||
|
||||
def test_a_frame_the_protocol_does_not_know_stops_the_worker() -> None:
|
||||
sock, worker_sock = socket.socketpair()
|
||||
worker = Worker(worker_sock, ())
|
||||
sock.sendall(encode_frame({"type": "shell", "command": "id"}))
|
||||
|
||||
with pytest.raises(ProtocolError, match="unknown frame type"):
|
||||
worker.serve()
|
||||
sock.close()
|
||||
|
||||
|
||||
CALLBACK_MODULE = """
|
||||
import os
|
||||
from litellm.callbacks_v1 import before_send
|
||||
|
||||
@before_send
|
||||
def leak(request):
|
||||
return {"body": {"env": sorted(os.environ), "cwd_files": sorted(os.listdir("."))}}
|
||||
"""
|
||||
|
||||
|
||||
def test_a_spawned_worker_sees_only_the_environment_it_was_given(tmp_path: Path) -> None:
|
||||
modules = tmp_path / "modules"
|
||||
modules.mkdir()
|
||||
(modules / "leaky.py").write_text(CALLBACK_MODULE)
|
||||
empty = tmp_path / "empty"
|
||||
empty.mkdir()
|
||||
gateway_sock, worker_sock = socket.socketpair()
|
||||
gateway_sock.settimeout(60)
|
||||
repo = Path(__file__).parents[3]
|
||||
given = {"PYTHONPATH": os.pathsep.join((str(repo), str(modules))), "LITELLM_MODE": "PRODUCTION"}
|
||||
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, "-m", "litellm.callbacks_v1.host", "--fd", str(worker_sock.fileno()), "--load", "leaky"],
|
||||
pass_fds=(worker_sock.fileno(),),
|
||||
env=given,
|
||||
cwd=empty,
|
||||
)
|
||||
worker_sock.close()
|
||||
try:
|
||||
hello = read_frame(gateway_sock)
|
||||
assert hello is not None and [sub["name"] for sub in hello["subscriptions"]] == ["leaky.leak"] # pyright: ignore[reportIndexIssue, reportGeneralTypeIssues] # the hello frame's subscriptions are a list of objects
|
||||
gateway_sock.sendall(encode_frame({"type": "intercept", "id": 1, "subscriber": 0, "request": {}}))
|
||||
answer = read_frame(gateway_sock)
|
||||
finally:
|
||||
gateway_sock.close()
|
||||
assert process.wait(30) == 0
|
||||
|
||||
assert answer is not None
|
||||
leaked = answer["patch"]["body"] # pyright: ignore[reportIndexIssue, reportOptionalSubscript] # the patch frame carries the callback's body
|
||||
assert set(leaked["env"]) - {"LC_CTYPE", "__CF_USER_TEXT_ENCODING"} <= set(given)
|
||||
assert leaked["cwd_files"] == []
|
||||
|
||||
|
||||
GOLDEN_FRAMES = Path(__file__).parents[3] / "litellm-rust/crates/callbacks-v1-exthost/golden"
|
||||
|
||||
|
||||
def golden_frame(name: str) -> dict:
|
||||
import json
|
||||
|
||||
return json.loads((GOLDEN_FRAMES / name).read_text())
|
||||
|
||||
|
||||
def test_the_worker_answers_the_golden_frames_rust_sends_with_the_golden_frames_rust_parses() -> None:
|
||||
"""Both sides are pinned to one directory: Rust serialises `to_worker` and parses
|
||||
`from_worker`; here the worker is fed the first and must produce the shape of the second."""
|
||||
seen: list[object] = []
|
||||
|
||||
@callbacks_v1.on_event("call.started", name="recorder")
|
||||
def record(event: EnvelopeV1) -> None:
|
||||
seen.append(event)
|
||||
raise ValueError("boom")
|
||||
|
||||
@callbacks_v1.before_send
|
||||
def patch(request: RequestFactsV1) -> WirePatchV1 | None:
|
||||
seen.append(request)
|
||||
return {"headers": {"remove": ["X-Trace"], "set": [("X-Team", "core")]}, "body": {"input": "patched"}}
|
||||
|
||||
event, intercept, flush = (golden_frame(f"to_worker/{name}.json") for name in ("event", "intercept", "flush"))
|
||||
gateway = Gateway()
|
||||
assert set(gateway.hello) == set(golden_frame("from_worker/hello.json"))
|
||||
assert {key for sub in gateway.hello["subscriptions"] for key in sub} == {
|
||||
key for sub in golden_frame("from_worker/hello.json")["subscriptions"] for key in sub
|
||||
}
|
||||
|
||||
gateway.send(**intercept)
|
||||
assert gateway.read() == golden_frame("from_worker/patch.json")
|
||||
gateway.send(**event)
|
||||
gateway.send(**flush)
|
||||
assert gateway.read() == golden_frame("from_worker/report.json")
|
||||
assert gateway.read() == golden_frame("from_worker/flushed.json")
|
||||
assert seen == [intercept["request"], event["envelope"]]
|
||||
gateway.close()
|
||||
|
||||
|
||||
def test_the_authoring_surface_and_the_worker_import_nothing_but_the_standard_library() -> None:
|
||||
"""A worker that could run without the rest of litellm stays possible only while these two
|
||||
files need nothing from it. `typing_extensions` is the one third-party import."""
|
||||
import ast
|
||||
|
||||
allowed = set(sys.stdlib_module_names) | {"typing_extensions", "litellm.callbacks_v1"}
|
||||
for path in ("litellm/callbacks_v1/__init__.py", "litellm/callbacks_v1/host.py"):
|
||||
tree = ast.parse((Path(__file__).parents[3] / path).read_text())
|
||||
imported = {
|
||||
alias.name if isinstance(node, ast.Import) else str(node.module)
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, (ast.Import, ast.ImportFrom))
|
||||
for alias in node.names
|
||||
}
|
||||
assert {name for name in imported if name not in allowed and name.split(".")[0] not in allowed} == set(), path
|
||||
|
|
@ -10,8 +10,8 @@ from pydantic import TypeAdapter
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.rust_bridge import callbacks_legacy_python as callbacks
|
||||
from litellm.rust_bridge.callbacks_legacy_python import check_limits, setup
|
||||
from litellm.rust_bridge import callbacks_legacy as callbacks
|
||||
from litellm.rust_bridge.callbacks_legacy import check_limits, setup
|
||||
|
||||
_OCR_KWARGS: Final = MappingProxyType(
|
||||
{
|
||||
31
tests/test_litellm/rust_bridge/test_callbacks_v1.py
Normal file
31
tests/test_litellm/rust_bridge/test_callbacks_v1.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
import inspect
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
from litellm.rust_bridge import callbacks_v1 as callbacks
|
||||
|
||||
ROOT: Final = Path(__file__).parents[3]
|
||||
CONTRACT_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1-python/python_contract.json"
|
||||
|
||||
|
||||
def test_the_rust_contract_matches_the_shim_signatures() -> None:
|
||||
contract: Final = TypeAdapter(dict[str, list[str]]).validate_json(CONTRACT_PATH.read_text())
|
||||
assert contract == {name: list(inspect.signature(getattr(callbacks, name)).parameters) for name in contract}
|
||||
|
||||
|
||||
class ResponseModel(BaseModel):
|
||||
value: int
|
||||
|
||||
|
||||
def test_project_response_supports_pydantic_and_json_values() -> None:
|
||||
assert callbacks.project_response(ResponseModel(value=3)) == {"value": 3}
|
||||
original: Final = {"nested": [1, True, None]}
|
||||
assert callbacks.project_response(original) is original
|
||||
|
||||
|
||||
def test_project_response_rejects_unsupported_objects() -> None:
|
||||
with pytest.raises(TypeError):
|
||||
callbacks.project_response(object())
|
||||
|
|
@ -1,178 +0,0 @@
|
|||
import inspect
|
||||
import warnings
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
from litellm.rust_bridge import callbacks_v1_python as callbacks
|
||||
|
||||
ROOT: Final = Path(__file__).parents[3]
|
||||
CONTRACT_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1-python/python_contract.json"
|
||||
GOLDEN_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1/golden/v1"
|
||||
|
||||
|
||||
def _clear_registry() -> None:
|
||||
for subscriber in callbacks.snapshot():
|
||||
callbacks.unregister(cast(callbacks.Callback, subscriber))
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_registry() -> Generator[None, None, None]:
|
||||
_clear_registry()
|
||||
yield
|
||||
_clear_registry()
|
||||
|
||||
|
||||
def test_the_rust_contract_matches_the_shim_signatures() -> None:
|
||||
contract: Final = TypeAdapter(dict[str, list[str]]).validate_json(CONTRACT_PATH.read_text())
|
||||
assert contract == {name: list(inspect.signature(getattr(callbacks, name)).parameters) for name in contract}
|
||||
|
||||
|
||||
def test_v1_golden_envelopes_match_the_typed_contract() -> None:
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings("ignore", message="Items .*ReadOnly.*", category=UserWarning)
|
||||
adapter: Final = TypeAdapter(callbacks.EnvelopeV1)
|
||||
envelopes: Final = [adapter.validate_json(path.read_text(), strict=True) for path in GOLDEN_PATH.glob("*.json")]
|
||||
assert {envelope["event"]["type"] for envelope in envelopes} == callbacks.EVENTS
|
||||
assert {envelope["schema"] for envelope in envelopes} == callbacks.SUPPORTED_SCHEMAS
|
||||
|
||||
|
||||
class ValidCallback:
|
||||
name: str = "valid"
|
||||
schema: int = 1
|
||||
events: frozenset[str] = frozenset({"call.started"})
|
||||
|
||||
def on_event(self, event: callbacks.EnvelopeV1) -> None:
|
||||
del event
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("callback", "error"),
|
||||
[
|
||||
(
|
||||
type(
|
||||
"Unsupported",
|
||||
(),
|
||||
{"name": "unsupported", "schema": 2, "events": frozenset(), "before_send": lambda *_: None},
|
||||
)(),
|
||||
callbacks.UnsupportedSchema,
|
||||
),
|
||||
(
|
||||
type(
|
||||
"Unknown",
|
||||
(),
|
||||
{"name": "unknown", "schema": 1, "events": frozenset({"new.event"}), "on_event": lambda *_: None},
|
||||
)(),
|
||||
callbacks.UnknownEvent,
|
||||
),
|
||||
(
|
||||
type("Missing", (), {"name": "missing", "schema": 1, "events": frozenset()})(),
|
||||
callbacks.MissingHandler,
|
||||
),
|
||||
(
|
||||
type("Empty", (), {"name": "empty", "schema": 1, "events": frozenset(), "on_event": lambda *_: None})(),
|
||||
callbacks.EmptyObserverEvents,
|
||||
),
|
||||
(
|
||||
type(
|
||||
"AsyncFunction",
|
||||
(),
|
||||
{
|
||||
"name": "async-function",
|
||||
"schema": 1,
|
||||
"events": frozenset({"call.started"}),
|
||||
"async_on_event": lambda *_: None,
|
||||
},
|
||||
)(),
|
||||
callbacks.HandlerKindMismatch,
|
||||
),
|
||||
],
|
||||
ids=["schema", "event", "missing", "empty-events", "async-kind"],
|
||||
)
|
||||
def test_register_rejects_invalid_callbacks(callback: object, error: type[ValueError]) -> None:
|
||||
with pytest.raises(error):
|
||||
callbacks.register(callback) # pyright: ignore[reportArgumentType] # invalid callback shapes are the subject of the test
|
||||
|
||||
|
||||
class WrongSyncKind:
|
||||
name: str = "wrong-sync-kind"
|
||||
schema: int = 1
|
||||
events: frozenset[str] = frozenset({"call.started"})
|
||||
|
||||
async def on_event(self, event: callbacks.EnvelopeV1) -> None:
|
||||
del event
|
||||
|
||||
|
||||
def test_register_rejects_coroutine_in_sync_slot() -> None:
|
||||
with pytest.raises(callbacks.HandlerKindMismatch):
|
||||
callbacks.register(cast(callbacks.Callback, WrongSyncKind()))
|
||||
|
||||
|
||||
class AsyncObserver:
|
||||
name: str = "async-observer"
|
||||
schema: int = 1
|
||||
events: frozenset[str] = frozenset({"call.started"})
|
||||
|
||||
async def async_on_event(self, event: callbacks.EnvelopeV1) -> None:
|
||||
del event
|
||||
|
||||
|
||||
class SyncInterceptor:
|
||||
name: str = "sync-interceptor"
|
||||
schema: int = 1
|
||||
events: frozenset[str] = frozenset()
|
||||
|
||||
def before_send(self, request: callbacks.RequestFactsV1) -> callbacks.WirePatchV1 | None:
|
||||
del request
|
||||
return None
|
||||
|
||||
|
||||
class AsyncInterceptor:
|
||||
name: str = "async-interceptor"
|
||||
schema: int = 1
|
||||
events: frozenset[str] = frozenset()
|
||||
|
||||
async def async_before_send(self, request: callbacks.RequestFactsV1) -> callbacks.WirePatchV1 | None:
|
||||
del request
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("callback", [ValidCallback(), AsyncObserver(), SyncInterceptor(), AsyncInterceptor()])
|
||||
def test_register_accepts_each_handler_family(callback: callbacks.Callback) -> None:
|
||||
callbacks.register(callback)
|
||||
assert len(callbacks.snapshot()) == 1
|
||||
|
||||
|
||||
def test_register_rejects_duplicate_names() -> None:
|
||||
callbacks.register(cast(callbacks.Callback, ValidCallback()))
|
||||
with pytest.raises(callbacks.DuplicateName):
|
||||
callbacks.register(cast(callbacks.Callback, ValidCallback()))
|
||||
|
||||
|
||||
def test_snapshot_is_stable_across_later_registry_writes() -> None:
|
||||
first: Final = ValidCallback()
|
||||
second: Final = AsyncObserver()
|
||||
callbacks.register(cast(callbacks.Callback, first))
|
||||
captured: Final = callbacks.snapshot()
|
||||
callbacks.register(cast(callbacks.Callback, second))
|
||||
callbacks.unregister(cast(callbacks.Callback, first))
|
||||
assert callbacks.snapshot() is not captured
|
||||
assert tuple(subscriber.name for subscriber in captured) == ("valid",)
|
||||
|
||||
|
||||
class ResponseModel(BaseModel):
|
||||
value: int
|
||||
|
||||
|
||||
def test_project_response_supports_pydantic_and_json_values() -> None:
|
||||
assert callbacks.project_response(ResponseModel(value=3)) == {"value": 3}
|
||||
original: Final = {"nested": [1, True, None]}
|
||||
assert callbacks.project_response(original) is original
|
||||
|
||||
|
||||
def test_project_response_rejects_unsupported_objects() -> None:
|
||||
with pytest.raises(TypeError):
|
||||
callbacks.project_response(object())
|
||||
236
tests/test_litellm/test_callbacks_v1.py
Normal file
236
tests/test_litellm/test_callbacks_v1.py
Normal file
|
|
@ -0,0 +1,236 @@
|
|||
import warnings
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
from typing import Final, get_args
|
||||
|
||||
import pytest
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm import callbacks_v1
|
||||
from litellm.callbacks_v1 import EnvelopeV1, EventName, RequestFactsV1, SchemaVersion, WirePatchV1
|
||||
|
||||
ROOT: Final = Path(__file__).parents[2]
|
||||
GOLDEN_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1/golden/v1"
|
||||
|
||||
|
||||
def _clear_registry() -> None:
|
||||
for subscriber in callbacks_v1.snapshot():
|
||||
callbacks_v1.unregister(subscriber)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_registry() -> Generator[None, None, None]:
|
||||
_clear_registry()
|
||||
yield
|
||||
_clear_registry()
|
||||
|
||||
|
||||
def test_the_event_names_and_schemas_are_the_literal_aliases() -> None:
|
||||
assert callbacks_v1.EVENTS == frozenset(get_args(EventName))
|
||||
assert callbacks_v1.SUPPORTED_SCHEMAS == frozenset(get_args(SchemaVersion))
|
||||
|
||||
|
||||
def test_v1_golden_envelopes_match_the_typed_contract() -> None:
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings("ignore", message="Items .*ReadOnly.*", category=UserWarning)
|
||||
adapter: Final = TypeAdapter(callbacks_v1.EnvelopeV1)
|
||||
envelopes: Final = [adapter.validate_json(path.read_text(), strict=True) for path in GOLDEN_PATH.glob("*.json")]
|
||||
assert {envelope["event"]["type"] for envelope in envelopes} == callbacks_v1.EVENTS
|
||||
assert {envelope["schema"] for envelope in envelopes} == callbacks_v1.SUPPORTED_SCHEMAS
|
||||
|
||||
|
||||
class SyncObserver:
|
||||
name: str = "valid"
|
||||
events: frozenset[EventName] = frozenset({"call.started"})
|
||||
|
||||
def on_event(self, event: callbacks_v1.EnvelopeV1) -> None:
|
||||
del event
|
||||
|
||||
|
||||
class AsyncObserver:
|
||||
name: str = "async-observer"
|
||||
events: frozenset[EventName] = frozenset({"call.started"})
|
||||
|
||||
async def async_on_event(self, event: callbacks_v1.EnvelopeV1) -> None:
|
||||
del event
|
||||
|
||||
|
||||
class SyncInterceptor:
|
||||
name: str = "sync-interceptor"
|
||||
events: frozenset[EventName] = frozenset()
|
||||
|
||||
def before_send(self, request: callbacks_v1.RequestFactsV1) -> callbacks_v1.WirePatchV1 | None:
|
||||
del request
|
||||
return None
|
||||
|
||||
|
||||
class AsyncInterceptor:
|
||||
name: str = "async-interceptor"
|
||||
events: frozenset[EventName] = frozenset()
|
||||
|
||||
async def async_before_send(self, request: callbacks_v1.RequestFactsV1) -> callbacks_v1.WirePatchV1 | None:
|
||||
del request
|
||||
return None
|
||||
|
||||
|
||||
class WrongSyncKind:
|
||||
name: str = "wrong-sync-kind"
|
||||
events: frozenset[EventName] = frozenset({"call.started"})
|
||||
|
||||
async def on_event(self, event: callbacks_v1.EnvelopeV1) -> None:
|
||||
del event
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("callback", "error"),
|
||||
[
|
||||
(
|
||||
type(
|
||||
"Unsupported",
|
||||
(),
|
||||
{"name": "unsupported", "schema": 2, "events": frozenset(), "before_send": lambda *_: None},
|
||||
)(),
|
||||
callbacks_v1.UnsupportedSchema,
|
||||
),
|
||||
(
|
||||
type(
|
||||
"Unknown",
|
||||
(),
|
||||
{"name": "unknown", "schema": 1, "events": frozenset({"new.event"}), "on_event": lambda *_: None},
|
||||
)(),
|
||||
callbacks_v1.UnknownEvent,
|
||||
),
|
||||
(
|
||||
type("Missing", (), {"name": "missing", "schema": 1, "events": frozenset()})(),
|
||||
callbacks_v1.MissingHandler,
|
||||
),
|
||||
(
|
||||
type("Empty", (), {"name": "empty", "schema": 1, "events": frozenset(), "on_event": lambda *_: None})(),
|
||||
callbacks_v1.EmptyObserverEvents,
|
||||
),
|
||||
(
|
||||
type(
|
||||
"AsyncFunction",
|
||||
(),
|
||||
{
|
||||
"name": "async-function",
|
||||
"schema": 1,
|
||||
"events": frozenset({"call.started"}),
|
||||
"async_on_event": lambda *_: None,
|
||||
},
|
||||
)(),
|
||||
callbacks_v1.HandlerKindMismatch,
|
||||
),
|
||||
(WrongSyncKind(), callbacks_v1.HandlerKindMismatch),
|
||||
],
|
||||
ids=["schema", "event", "missing", "empty-events", "async-kind", "coroutine-in-sync-slot"],
|
||||
)
|
||||
def test_register_rejects_invalid_callbacks(callback: object, error: type[ValueError]) -> None:
|
||||
with pytest.raises(error):
|
||||
callbacks_v1.register(callback) # pyright: ignore[reportArgumentType] # invalid callback shapes are the subject of the test
|
||||
|
||||
|
||||
@pytest.mark.parametrize("callback", [SyncObserver(), AsyncObserver(), SyncInterceptor(), AsyncInterceptor()])
|
||||
def test_register_accepts_each_handler_family(callback: callbacks_v1.Callback) -> None:
|
||||
callbacks_v1.register(callback)
|
||||
assert len(callbacks_v1.snapshot()) == 1
|
||||
|
||||
|
||||
def test_register_rejects_duplicate_names() -> None:
|
||||
callbacks_v1.register(SyncObserver())
|
||||
with pytest.raises(callbacks_v1.DuplicateName):
|
||||
callbacks_v1.register(SyncObserver())
|
||||
|
||||
|
||||
def test_snapshot_is_stable_across_later_registry_writes() -> None:
|
||||
first: Final = SyncObserver()
|
||||
second: Final = AsyncObserver()
|
||||
callbacks_v1.register(first)
|
||||
captured: Final = callbacks_v1.snapshot()
|
||||
callbacks_v1.register(second)
|
||||
callbacks_v1.unregister(first)
|
||||
assert callbacks_v1.snapshot() is not captured
|
||||
assert tuple(subscriber.name for subscriber in captured) == ("valid",)
|
||||
|
||||
|
||||
def test_register_defaults_the_schema_when_the_callback_declares_none() -> None:
|
||||
callbacks_v1.register(SyncObserver())
|
||||
assert callbacks_v1.snapshot()[0].schema == callbacks_v1.SCHEMA
|
||||
|
||||
|
||||
def test_register_still_rejects_a_declared_foreign_schema() -> None:
|
||||
class Foreign(SyncObserver):
|
||||
name: str = "foreign"
|
||||
schema: int = 2
|
||||
|
||||
with pytest.raises(callbacks_v1.UnsupportedSchema):
|
||||
callbacks_v1.register(Foreign())
|
||||
|
||||
|
||||
def test_on_event_registers_a_sync_function_in_the_sync_slot() -> None:
|
||||
@callbacks_v1.on_event("call.started", "call.succeeded")
|
||||
def record(event: EnvelopeV1) -> None:
|
||||
del event
|
||||
|
||||
subscriber: Final = callbacks_v1.snapshot()[0]
|
||||
assert subscriber.on_event is record
|
||||
assert subscriber.async_on_event is None
|
||||
assert subscriber.events == frozenset({"call.started", "call.succeeded"})
|
||||
assert subscriber.name.endswith("record")
|
||||
assert record( # the function is returned unchanged, so it stays directly callable
|
||||
{"schema": 1, "call_id": "c", "call_type": "ocr", "seq": 0, "event": {"type": "response.received", "body": ""}}
|
||||
) is None
|
||||
|
||||
|
||||
def test_on_event_routes_a_coroutine_function_to_the_async_slot() -> None:
|
||||
@callbacks_v1.on_event("call.failed", name="named-observer")
|
||||
async def record(event: EnvelopeV1) -> None:
|
||||
del event
|
||||
|
||||
subscriber: Final = callbacks_v1.snapshot()[0]
|
||||
assert subscriber.async_on_event is record
|
||||
assert subscriber.on_event is None
|
||||
assert subscriber.name == "named-observer"
|
||||
|
||||
|
||||
def test_before_send_routes_by_kind_and_subscribes_to_no_events() -> None:
|
||||
@callbacks_v1.before_send
|
||||
def sync_patch(request: RequestFactsV1) -> WirePatchV1 | None:
|
||||
del request
|
||||
return None
|
||||
|
||||
@callbacks_v1.before_send
|
||||
async def async_patch(request: RequestFactsV1) -> WirePatchV1 | None:
|
||||
del request
|
||||
return None
|
||||
|
||||
first, second = callbacks_v1.snapshot()
|
||||
assert (first.before_send, first.async_before_send) == (sync_patch, None)
|
||||
assert (second.before_send, second.async_before_send) == (None, async_patch)
|
||||
assert first.events == second.events == frozenset()
|
||||
|
||||
|
||||
def test_on_event_rejects_an_unknown_event_at_decoration() -> None:
|
||||
with pytest.raises(callbacks_v1.UnknownEvent):
|
||||
|
||||
@callbacks_v1.on_event("call.startd") # pyright: ignore[reportArgumentType] # the typo is the subject of the test
|
||||
def record(event: EnvelopeV1) -> None:
|
||||
del event
|
||||
|
||||
|
||||
def test_on_event_rejects_an_empty_event_set_at_decoration() -> None:
|
||||
with pytest.raises(callbacks_v1.EmptyObserverEvents):
|
||||
|
||||
@callbacks_v1.on_event()
|
||||
def record(event: EnvelopeV1) -> None:
|
||||
del event
|
||||
|
||||
|
||||
def test_unregister_accepts_a_bare_name() -> None:
|
||||
@callbacks_v1.on_event("call.started", name="by-name")
|
||||
def record(event: EnvelopeV1) -> None:
|
||||
del event
|
||||
|
||||
assert len(callbacks_v1.snapshot()) == 1
|
||||
callbacks_v1.unregister("by-name")
|
||||
assert callbacks_v1.snapshot() == ()
|
||||
0
tests/test_litellm_rust/builtin/__init__.py
Normal file
0
tests/test_litellm_rust/builtin/__init__.py
Normal file
215
tests/test_litellm_rust/builtin/test_parity.py
Normal file
215
tests/test_litellm_rust/builtin/test_parity.py
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
"""Differential parity: one native call reaches a legacy logger and its v1 port, each posting
|
||||
to its own recording vendor, and every difference between the two vendor bodies is `Allowed`.
|
||||
|
||||
The native chain runs the legacy contract and then v1 on the same call, so nothing is
|
||||
replayed or mocked: both sides see the one real call. Registration happens in fixtures only.
|
||||
"""
|
||||
|
||||
from collections.abc import Generator
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import callbacks_v1
|
||||
from litellm.callbacks_v1.builtin import anthropic_cache_control, generic_api, openmeter
|
||||
from litellm.callbacks_v1.builtin.manifest import entry
|
||||
from litellm.callbacks_v1.builtin.runtime import HttpxTransport, Patcher, Sink
|
||||
from litellm.integrations.generic_api.generic_api_callback import GenericAPILogger
|
||||
from litellm.integrations.openmeter import OpenMeterLogger
|
||||
from tests.test_litellm.callbacks_v1.builtin.support import Allowed, assert_parity
|
||||
from tests.test_litellm_rust.support.callback_recorder import drain_logging
|
||||
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec, recording_service
|
||||
from tests.test_litellm_rust.support.requests import MESSAGES, MESSAGES_MODEL, MESSAGES_RESPONSE
|
||||
|
||||
pytestmark = pytest.mark.requires_rust_extension
|
||||
|
||||
METADATA: Final = {"user_api_key_user_id": "user-1"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def provider(recording_server: RecordingServer) -> RecordingServer:
|
||||
recording_server.default_response = ResponseSpec(body=MESSAGES_RESPONSE)
|
||||
return recording_server
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def legacy_vendor() -> Generator[RecordingServer]:
|
||||
with recording_service() as server:
|
||||
yield server
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def port_vendor() -> Generator[RecordingServer]:
|
||||
with recording_service() as server:
|
||||
yield server
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clean_registry() -> Generator[None]:
|
||||
yield
|
||||
for subscriber in callbacks_v1.snapshot():
|
||||
callbacks_v1.unregister(subscriber)
|
||||
|
||||
|
||||
def arguments(server: RecordingServer, **kwargs: object) -> dict[str, object]:
|
||||
# Identity rides in `metadata` here because call.started projects only that keyword; the proxy
|
||||
# sends it as `litellm_metadata` on this route, which v1 does not see yet (see entry("openmeter").gaps).
|
||||
return {
|
||||
"model": MESSAGES_MODEL,
|
||||
"messages": [dict(message) for message in MESSAGES],
|
||||
"max_tokens": 64,
|
||||
"api_key": "test-key",
|
||||
"api_base": server.base_url,
|
||||
"metadata": METADATA,
|
||||
**kwargs,
|
||||
}
|
||||
|
||||
|
||||
OPENMETER_ALLOWED: Final = (
|
||||
Allowed("time", "each side stamps its own clock"),
|
||||
Allowed("data.cost", "no cost fact on call.succeeded", gap="cost"),
|
||||
Allowed("data.*_tokens", "no usage fact; legacy counts tokens off the OpenAI-normalised response", gap="usage"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openmeter_meters_a_native_call_like_its_legacy_twin(
|
||||
provider: RecordingServer,
|
||||
legacy_vendor: RecordingServer,
|
||||
port_vendor: RecordingServer,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("OPENMETER_API_KEY", "test-key")
|
||||
monkeypatch.setenv("OPENMETER_API_ENDPOINT", legacy_vendor.base_url)
|
||||
port: Final = openmeter.OpenMeter(openmeter.Config(api_key="test-key", endpoint=port_vendor.base_url))
|
||||
sink: Final = Sink("openmeter", port, HttpxTransport(), lambda: datetime.now(timezone.utc))
|
||||
callbacks_v1.register(sink)
|
||||
|
||||
await litellm.anthropic.messages.acreate(**arguments(provider, callbacks=[OpenMeterLogger()]))
|
||||
await drain_logging()
|
||||
assert sink.outbox.flush()
|
||||
|
||||
(legacy_request,) = legacy_vendor.requests
|
||||
(port_request,) = port_vendor.requests
|
||||
assert port_request.path == legacy_request.path == "/api/v1/events"
|
||||
assert port_request.headers["authorization"] == legacy_request.headers["authorization"]
|
||||
assert port_request.headers["content-type"] == legacy_request.headers["content-type"]
|
||||
assert_parity(legacy_request.raw_body, port_request.raw_body, OPENMETER_ALLOWED, entry("openmeter").gaps)
|
||||
|
||||
|
||||
# What the legacy StandardLoggingPayload has and v1 facts cannot fill yet, on any outcome.
|
||||
GENERIC_API_GAP_ALLOWED: Final = (
|
||||
Allowed("0.stream", "legacy leaves stream null on a non-streaming call, v1 always says false"),
|
||||
Allowed("0.completionStartTime", "no first-token time in timing", gap="first_token_time"),
|
||||
Allowed("0.response_cost*", "no cost facts", gap="cost"),
|
||||
Allowed("0.cost_breakdown*", "no cost facts", gap="cost"),
|
||||
Allowed("0.saved_cache_cost", "no cost facts", gap="cost"),
|
||||
Allowed("0.autorouter_savings", "no cost facts", gap="cost"),
|
||||
Allowed("0.*_tokens", "no usage fact", gap="usage"),
|
||||
Allowed("0.cache_*", "no cache fact", gap="cache"),
|
||||
Allowed("0.model_id", "no routing fact", gap="routing"),
|
||||
Allowed("0.model_group", "no routing fact", gap="routing"),
|
||||
Allowed("0.model_map_information.*", "no routing fact", gap="routing"),
|
||||
Allowed("0.api_base", "no routing fact", gap="routing"),
|
||||
Allowed("0.metadata.*", "no identity fact", gap="identity"),
|
||||
Allowed("0.end_user", "no identity fact", gap="identity"),
|
||||
Allowed("0.requester_ip_address", "no identity fact", gap="identity"),
|
||||
Allowed("0.user_agent", "no identity fact", gap="identity"),
|
||||
Allowed("0.request_tags", "no tags fact", gap="tags"),
|
||||
Allowed("0.request_model_access_groups", "no tags fact", gap="tags"),
|
||||
Allowed("0.trace_id", "no correlation beyond call_id", gap="correlation"),
|
||||
Allowed("0.session_id", "no correlation beyond call_id", gap="correlation"),
|
||||
Allowed("0.hidden_params.*", "no equivalent", gap="no_equivalent"),
|
||||
Allowed("0.guardrail_information", "no equivalent", gap="no_equivalent"),
|
||||
Allowed("0.standard_built_in_tools_params.*", "no equivalent", gap="no_equivalent"),
|
||||
Allowed("0.status_fields.*", "derived from guardrail_information, which v1 has no fact for", gap="no_equivalent"),
|
||||
Allowed("0.messages.*", "no route-neutral prompt projection", gap="prompt"),
|
||||
Allowed(
|
||||
"0.model_parameters.metadata.*",
|
||||
"this test sends identity as `metadata`, which Anthropic also accepts; legacy keeps it out of model_parameters",
|
||||
),
|
||||
)
|
||||
|
||||
GENERIC_API_ALLOWED: Final = (
|
||||
*GENERIC_API_GAP_ALLOWED,
|
||||
Allowed(
|
||||
"0.response.*",
|
||||
"legacy logs the OpenAI-normalised response, v1 carries the route's own",
|
||||
gap="normalised_response",
|
||||
),
|
||||
Allowed("0.error_str", "legacy sends null on success, the port omits the key"),
|
||||
Allowed("0.error_information.*", "legacy sends an empty shell on success, the port omits the key"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generic_api_logs_a_native_call_like_its_legacy_twin(
|
||||
provider: RecordingServer, legacy_vendor: RecordingServer, port_vendor: RecordingServer
|
||||
) -> None:
|
||||
legacy: Final = GenericAPILogger(endpoint=legacy_vendor.base_url, batch_size=1)
|
||||
port: Final = generic_api.GenericApi(generic_api.Config(endpoint=port_vendor.base_url, batch_size=1))
|
||||
sink: Final = Sink("generic_api", port, HttpxTransport())
|
||||
callbacks_v1.register(sink)
|
||||
|
||||
await litellm.anthropic.messages.acreate(**arguments(provider, callbacks=[legacy]))
|
||||
await drain_logging()
|
||||
assert sink.outbox.flush()
|
||||
|
||||
(legacy_request,) = legacy_vendor.requests
|
||||
(port_request,) = port_vendor.requests
|
||||
assert port_request.headers["content-type"] == legacy_request.headers["content-type"]
|
||||
assert_parity(legacy_request.raw_body, port_request.raw_body, GENERIC_API_ALLOWED, entry("generic_api").gaps)
|
||||
|
||||
|
||||
GENERIC_API_FAILURE_ALLOWED: Final = (
|
||||
*GENERIC_API_GAP_ALLOWED,
|
||||
Allowed("0.response", "legacy sends an empty object on failure, the port omits the key"),
|
||||
Allowed("0.error_information.error_budget_*", "ErrorFacts is class/message/status only", gap="error_detail"),
|
||||
Allowed("0.error_information.error_rate_limit_*", "ErrorFacts is class/message/status only", gap="error_detail"),
|
||||
Allowed(
|
||||
"0.error_information.error_provider_request_id", "ErrorFacts is class/message/status only", gap="error_detail"
|
||||
),
|
||||
Allowed("0.error_information.llm_provider", "ErrorFacts is class/message/status only", gap="error_detail"),
|
||||
Allowed("0.error_information.traceback", "ErrorFacts is class/message/status only", gap="error_detail"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generic_api_logs_a_failed_native_call_like_its_legacy_twin(
|
||||
provider: RecordingServer, legacy_vendor: RecordingServer, port_vendor: RecordingServer
|
||||
) -> None:
|
||||
provider.enqueue(
|
||||
ResponseSpec(body={"type": "error", "error": {"type": "invalid_request_error", "message": "bad"}}, status=400)
|
||||
)
|
||||
legacy: Final = GenericAPILogger(endpoint=legacy_vendor.base_url, batch_size=1)
|
||||
port: Final = generic_api.GenericApi(generic_api.Config(endpoint=port_vendor.base_url, batch_size=1))
|
||||
sink: Final = Sink("generic_api", port, HttpxTransport())
|
||||
callbacks_v1.register(sink)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
await litellm.anthropic.messages.acreate(**arguments(provider, callbacks=[legacy]))
|
||||
await drain_logging()
|
||||
assert sink.outbox.flush()
|
||||
|
||||
(legacy_request,) = legacy_vendor.requests
|
||||
(port_request,) = port_vendor.requests
|
||||
assert_parity(
|
||||
legacy_request.raw_body, port_request.raw_body, GENERIC_API_FAILURE_ALLOWED, entry("generic_api").gaps
|
||||
)
|
||||
|
||||
|
||||
def test_the_cache_control_patch_survives_the_contract_and_reaches_the_provider(provider: RecordingServer) -> None:
|
||||
config: Final = anthropic_cache_control.Config(
|
||||
points=(anthropic_cache_control.InjectionPoint(role="system"), anthropic_cache_control.InjectionPoint(index=-1))
|
||||
)
|
||||
callbacks_v1.register(Patcher("anthropic_cache_control", anthropic_cache_control.AnthropicCacheControl(config)))
|
||||
|
||||
litellm.anthropic.messages.create(**arguments(provider, system="be brief"))
|
||||
|
||||
sent: Final = provider.requests[-1].body
|
||||
assert isinstance(sent, dict)
|
||||
assert sent["system"] == [{"type": "text", "text": "be brief", "cache_control": {"type": "ephemeral"}}]
|
||||
assert sent["messages"][-1]["content"][-1]["cache_control"] == {"type": "ephemeral"}
|
||||
assert sent["model"] == MESSAGES_MODEL.split("/", 1)[1]
|
||||
146
tests/test_litellm_rust/messages/test_callbacks_v1.py
Normal file
146
tests/test_litellm_rust/messages/test_callbacks_v1.py
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
from collections.abc import AsyncIterator, Generator
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import callbacks_v1
|
||||
from litellm.callbacks_v1 import EVENTS, EnvelopeV1, EventName, RequestFactsV1, WirePatchV1
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
|
||||
from tests.test_litellm_rust.support.requests import (
|
||||
MESSAGES,
|
||||
MESSAGES_EVENTS,
|
||||
MESSAGES_MODEL,
|
||||
MESSAGES_RESPONSE,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.requires_rust_extension
|
||||
|
||||
STREAM: Final = ResponseSpec(body=None, events=MESSAGES_EVENTS)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def messages_server(recording_server: RecordingServer) -> RecordingServer:
|
||||
recording_server.default_response = ResponseSpec(body=MESSAGES_RESPONSE)
|
||||
return recording_server
|
||||
|
||||
|
||||
def arguments(server: RecordingServer, **kwargs: object) -> dict[str, object]:
|
||||
return {
|
||||
"model": MESSAGES_MODEL,
|
||||
"messages": [dict(message) for message in MESSAGES],
|
||||
"max_tokens": 64,
|
||||
"api_key": "test-key",
|
||||
"api_base": server.base_url,
|
||||
**kwargs,
|
||||
}
|
||||
|
||||
|
||||
class BothObserver:
|
||||
"""Declares both kinds; an async call must select the async one and never the sync one."""
|
||||
|
||||
name: Final = "both-observer"
|
||||
schema: Final = 1
|
||||
events: Final = EVENTS
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.envelopes: Final[list[EnvelopeV1]] = []
|
||||
self.sync_calls: Final[list[EnvelopeV1]] = []
|
||||
|
||||
def on_event(self, event: EnvelopeV1) -> None:
|
||||
self.sync_calls.append(event)
|
||||
|
||||
async def async_on_event(self, event: EnvelopeV1) -> None:
|
||||
self.envelopes.append(event)
|
||||
|
||||
|
||||
class AsyncInterceptor:
|
||||
name: Final = "async-interceptor"
|
||||
schema: Final = 1
|
||||
events: Final[frozenset[EventName]] = frozenset()
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.requests: Final[list[RequestFactsV1]] = []
|
||||
|
||||
async def async_before_send(self, request: RequestFactsV1) -> WirePatchV1 | None:
|
||||
self.requests.append(request)
|
||||
return {"headers": {"set": [("x-v1-async-interceptor", "seen")]}}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def observer() -> Generator[BothObserver]:
|
||||
subscriber: Final = BothObserver()
|
||||
callbacks_v1.register(subscriber)
|
||||
try:
|
||||
yield subscriber
|
||||
finally:
|
||||
callbacks_v1.unregister(subscriber)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def interceptor() -> Generator[AsyncInterceptor]:
|
||||
subscriber: Final = AsyncInterceptor()
|
||||
callbacks_v1.register(subscriber)
|
||||
try:
|
||||
yield subscriber
|
||||
finally:
|
||||
callbacks_v1.unregister(subscriber)
|
||||
|
||||
|
||||
class LegacyCallId(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
self.call_ids: Final[list[str]] = []
|
||||
|
||||
def log_pre_api_call(self, model, messages, kwargs):
|
||||
self.call_ids.append(kwargs["litellm_call_id"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_messages_async_call_selects_the_async_observer_and_shares_the_legacy_call_id(
|
||||
messages_server: RecordingServer, observer: BothObserver
|
||||
) -> None:
|
||||
legacy: Final = LegacyCallId()
|
||||
|
||||
await litellm.anthropic.messages.acreate(**arguments(messages_server, callbacks=[legacy]))
|
||||
|
||||
assert observer.sync_calls == []
|
||||
assert [envelope["event"]["type"] for envelope in observer.envelopes] == [
|
||||
"call.started",
|
||||
"request.sending",
|
||||
"response.received",
|
||||
"call.succeeded",
|
||||
]
|
||||
assert [envelope["seq"] for envelope in observer.envelopes] == [0, 1, 2, 3]
|
||||
assert {envelope["call_type"] for envelope in observer.envelopes} == {"anthropic_messages"}
|
||||
assert len(legacy.call_ids) == 1
|
||||
assert {envelope["call_id"] for envelope in observer.envelopes} == set(legacy.call_ids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_messages_async_interceptor_patch_reaches_the_provider(
|
||||
messages_server: RecordingServer, interceptor: AsyncInterceptor
|
||||
) -> None:
|
||||
await litellm.anthropic.messages.acreate(**arguments(messages_server))
|
||||
|
||||
assert len(interceptor.requests) == 1
|
||||
request: Final = interceptor.requests[0]
|
||||
assert request["custom_llm_provider"] == "anthropic"
|
||||
assert request["headers"] != []
|
||||
assert messages_server.requests[-1].headers["x-v1-async-interceptor"] == "seen"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_messages_stream_marks_the_terminal_envelope_streamed(
|
||||
messages_server: RecordingServer, observer: BothObserver
|
||||
) -> None:
|
||||
messages_server.enqueue(STREAM)
|
||||
|
||||
stream: Final = await litellm.anthropic.messages.acreate(**arguments(messages_server, stream=True))
|
||||
assert isinstance(stream, AsyncIterator)
|
||||
delivered: Final = [chunk async for chunk in stream]
|
||||
|
||||
assert delivered
|
||||
terminal: Final = observer.envelopes[-1]
|
||||
assert terminal["event"]["type"] == "call.succeeded"
|
||||
assert terminal["event"]["streamed"] is True
|
||||
|
|
@ -3,9 +3,9 @@ from typing import Final
|
|||
|
||||
import pytest
|
||||
|
||||
from litellm import callbacks_v1
|
||||
from litellm.callbacks_v1 import EVENTS, EnvelopeV1, EventName, RequestFactsV1, WirePatchV1
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.rust_bridge import callbacks_v1_python
|
||||
from litellm.rust_bridge.callbacks_v1_python import EVENTS, EnvelopeV1, RequestFactsV1, WirePatchV1
|
||||
from tests.test_litellm_rust.support.callback_recorder import drain_logging
|
||||
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
|
||||
from tests.test_litellm_rust.support.requests import (
|
||||
|
|
@ -39,7 +39,7 @@ class Observer:
|
|||
class Interceptor:
|
||||
name: Final = "interceptor"
|
||||
schema: Final = 1
|
||||
events: Final[frozenset[str]] = frozenset()
|
||||
events: Final[frozenset[EventName]] = frozenset()
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.bodies: Final[list[object]] = []
|
||||
|
|
@ -52,21 +52,21 @@ class Interceptor:
|
|||
@pytest.fixture
|
||||
def observer() -> Generator[Observer]:
|
||||
subscriber: Final = Observer()
|
||||
callbacks_v1_python.register(subscriber)
|
||||
callbacks_v1.register(subscriber)
|
||||
try:
|
||||
yield subscriber
|
||||
finally:
|
||||
callbacks_v1_python.unregister(subscriber)
|
||||
callbacks_v1.unregister(subscriber)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def interceptor() -> Generator[Interceptor]:
|
||||
subscriber: Final = Interceptor()
|
||||
callbacks_v1_python.register(subscriber)
|
||||
callbacks_v1.register(subscriber)
|
||||
try:
|
||||
yield subscriber
|
||||
finally:
|
||||
callbacks_v1_python.unregister(subscriber)
|
||||
callbacks_v1.unregister(subscriber)
|
||||
|
||||
|
||||
class LegacyCallId(CustomLogger):
|
||||
|
|
@ -139,3 +139,28 @@ def test_native_ocr_failure_reaches_both_contracts_once(
|
|||
]
|
||||
assert [envelope["event"]["type"] for envelope in terminal] == ["call.failed"]
|
||||
assert observer.envelopes[-1] is terminal[0]
|
||||
|
||||
|
||||
def test_native_ocr_decorated_function_callbacks_run_through_the_native_call(
|
||||
ocr_server: RecordingServer,
|
||||
) -> None:
|
||||
seen: Final[list[str]] = []
|
||||
|
||||
@callbacks_v1.on_event("call.started", "call.succeeded", name="decorated-observer")
|
||||
def record(event: EnvelopeV1) -> None:
|
||||
seen.append(event["event"]["type"])
|
||||
|
||||
@callbacks_v1.before_send
|
||||
def patch(request: RequestFactsV1) -> WirePatchV1 | None:
|
||||
del request
|
||||
return {"headers": {"set": [("x-v1-decorated", "seen")]}}
|
||||
|
||||
try:
|
||||
call_native_ocr(ocr_server)
|
||||
finally:
|
||||
callbacks_v1.unregister("decorated-observer")
|
||||
callbacks_v1.unregister(f"{patch.__module__}.{patch.__qualname__}")
|
||||
|
||||
assert seen == ["call.started", "call.succeeded"]
|
||||
assert ocr_server.requests[-1].headers["x-v1-decorated"] == "seen"
|
||||
assert callbacks_v1.snapshot() == ()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue