From e0ead49d968c4e42f84aee4349fdb5d9bbac36fc Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Sat, 19 Sep 2026 21:00:51 -0700 Subject: [PATCH] wip --- litellm-rust/Cargo.lock | 11 + litellm-rust/Cargo.toml | 1 + .../callbacks-legacy-python/src/python.rs | 6 +- .../callbacks-legacy-python/tests/support.rs | 10 +- .../crates/callbacks-v1-exthost/AGENTS.md | 18 + .../crates/callbacks-v1-exthost/Cargo.toml | 15 + .../golden/from_worker/error.json | 6 + .../golden/from_worker/flushed.json | 4 + .../golden/from_worker/hello.json | 21 + .../golden/from_worker/patch.json | 20 + .../golden/from_worker/patch_none.json | 5 + .../golden/from_worker/report.json | 6 + .../golden/to_worker/event.json | 21 + .../golden/to_worker/flush.json | 4 + .../golden/to_worker/intercept.json | 22 + .../crates/callbacks-v1-exthost/src/lib.rs | 15 + .../callbacks-v1-exthost/src/protocol.rs | 254 +++++++++ .../crates/callbacks-v1-exthost/src/worker.rs | 214 ++++++++ .../callbacks-v1-exthost/tests/worker.rs | 239 +++++++++ .../crates/callbacks-v1-python/AGENTS.md | 6 +- .../crates/callbacks-v1-python/src/adapter.rs | 219 ++++---- .../crates/callbacks-v1-python/src/call.rs | 9 +- .../crates/callbacks-v1-python/src/python.rs | 4 +- .../callbacks-v1-python/src/subscribers.rs | 53 +- .../callbacks-v1-python/tests/lifecycle.rs | 40 +- .../callbacks-v1-python/tests/support.rs | 7 +- litellm-rust/crates/callbacks-v1/AGENTS.md | 13 +- .../crates/callbacks-v1/src/envelope.rs | 15 +- litellm-rust/crates/callbacks-v1/src/lib.rs | 12 +- .../crates/callbacks-v1/src/session.rs | 374 ++++++++++++++ litellm/callbacks_v1/MIGRATION.md | 171 ++++++ litellm/callbacks_v1/__init__.py | 419 +++++++++++++++ litellm/callbacks_v1/builtin/AGENTS.md | 44 ++ litellm/callbacks_v1/builtin/__init__.py | 6 + .../builtin/anthropic_cache_control.py | 149 ++++++ litellm/callbacks_v1/builtin/generic_api.py | 129 +++++ litellm/callbacks_v1/builtin/manifest.py | 218 ++++++++ litellm/callbacks_v1/builtin/openmeter.py | 125 +++++ litellm/callbacks_v1/builtin/outbox.py | 95 ++++ litellm/callbacks_v1/builtin/port.py | 90 ++++ litellm/callbacks_v1/builtin/runtime.py | 156 ++++++ litellm/callbacks_v1/host.py | 486 ++++++++++++++++++ ...s_legacy_python.py => callbacks_legacy.py} | 0 litellm/rust_bridge/callbacks_v1.py | 53 ++ litellm/rust_bridge/callbacks_v1_python.py | 272 ---------- tests/test_litellm/callbacks_v1/__init__.py | 0 .../callbacks_v1/builtin/__init__.py | 0 .../callbacks_v1/builtin/support.py | 152 ++++++ .../builtin/test_anthropic_cache_control.py | 81 +++ .../callbacks_v1/builtin/test_conventions.py | 180 +++++++ .../callbacks_v1/builtin/test_generic_api.py | 101 ++++ .../callbacks_v1/builtin/test_openmeter.py | 54 ++ .../callbacks_v1/builtin/test_outbox.py | 94 ++++ .../callbacks_v1/builtin/test_parity_diff.py | 34 ++ tests/test_litellm/callbacks_v1/test_host.py | 299 +++++++++++ ...acy_python.py => test_callbacks_legacy.py} | 4 +- .../rust_bridge/test_callbacks_v1.py | 31 ++ .../rust_bridge/test_callbacks_v1_python.py | 178 ------- tests/test_litellm/test_callbacks_v1.py | 236 +++++++++ tests/test_litellm_rust/builtin/__init__.py | 0 .../test_litellm_rust/builtin/test_parity.py | 215 ++++++++ .../messages/test_callbacks_v1.py | 146 ++++++ .../ocr/test_callbacks_v1.py | 39 +- 63 files changed, 5269 insertions(+), 632 deletions(-) create mode 100644 litellm-rust/crates/callbacks-v1-exthost/AGENTS.md create mode 100644 litellm-rust/crates/callbacks-v1-exthost/Cargo.toml create mode 100644 litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/error.json create mode 100644 litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/flushed.json create mode 100644 litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/hello.json create mode 100644 litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/patch.json create mode 100644 litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/patch_none.json create mode 100644 litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/report.json create mode 100644 litellm-rust/crates/callbacks-v1-exthost/golden/to_worker/event.json create mode 100644 litellm-rust/crates/callbacks-v1-exthost/golden/to_worker/flush.json create mode 100644 litellm-rust/crates/callbacks-v1-exthost/golden/to_worker/intercept.json create mode 100644 litellm-rust/crates/callbacks-v1-exthost/src/lib.rs create mode 100644 litellm-rust/crates/callbacks-v1-exthost/src/protocol.rs create mode 100644 litellm-rust/crates/callbacks-v1-exthost/src/worker.rs create mode 100644 litellm-rust/crates/callbacks-v1-exthost/tests/worker.rs create mode 100644 litellm-rust/crates/callbacks-v1/src/session.rs create mode 100644 litellm/callbacks_v1/MIGRATION.md create mode 100644 litellm/callbacks_v1/__init__.py create mode 100644 litellm/callbacks_v1/builtin/AGENTS.md create mode 100644 litellm/callbacks_v1/builtin/__init__.py create mode 100644 litellm/callbacks_v1/builtin/anthropic_cache_control.py create mode 100644 litellm/callbacks_v1/builtin/generic_api.py create mode 100644 litellm/callbacks_v1/builtin/manifest.py create mode 100644 litellm/callbacks_v1/builtin/openmeter.py create mode 100644 litellm/callbacks_v1/builtin/outbox.py create mode 100644 litellm/callbacks_v1/builtin/port.py create mode 100644 litellm/callbacks_v1/builtin/runtime.py create mode 100644 litellm/callbacks_v1/host.py rename litellm/rust_bridge/{callbacks_legacy_python.py => callbacks_legacy.py} (100%) create mode 100644 litellm/rust_bridge/callbacks_v1.py delete mode 100644 litellm/rust_bridge/callbacks_v1_python.py create mode 100644 tests/test_litellm/callbacks_v1/__init__.py create mode 100644 tests/test_litellm/callbacks_v1/builtin/__init__.py create mode 100644 tests/test_litellm/callbacks_v1/builtin/support.py create mode 100644 tests/test_litellm/callbacks_v1/builtin/test_anthropic_cache_control.py create mode 100644 tests/test_litellm/callbacks_v1/builtin/test_conventions.py create mode 100644 tests/test_litellm/callbacks_v1/builtin/test_generic_api.py create mode 100644 tests/test_litellm/callbacks_v1/builtin/test_openmeter.py create mode 100644 tests/test_litellm/callbacks_v1/builtin/test_outbox.py create mode 100644 tests/test_litellm/callbacks_v1/builtin/test_parity_diff.py create mode 100644 tests/test_litellm/callbacks_v1/test_host.py rename tests/test_litellm/rust_bridge/{test_callbacks_legacy_python.py => test_callbacks_legacy.py} (95%) create mode 100644 tests/test_litellm/rust_bridge/test_callbacks_v1.py delete mode 100644 tests/test_litellm/rust_bridge/test_callbacks_v1_python.py create mode 100644 tests/test_litellm/test_callbacks_v1.py create mode 100644 tests/test_litellm_rust/builtin/__init__.py create mode 100644 tests/test_litellm_rust/builtin/test_parity.py create mode 100644 tests/test_litellm_rust/messages/test_callbacks_v1.py diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index a02d9803528..7c50106e614 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2057,6 +2057,17 @@ dependencies = [ "thiserror 2.0.19", ] +[[package]] +name = "litellm-callbacks-v1-exthost" +version = "0.1.0" +dependencies = [ + "litellm-callbacks-v1", + "litellm-host", + "serde", + "serde_json", + "thiserror 2.0.19", +] + [[package]] name = "litellm-callbacks-v1-python" version = "0.1.0" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index bfc84b3dcc9..0c01897a42e 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -14,6 +14,7 @@ litellm-host = { path = "crates/host" } litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" } litellm-callbacks-v1 = { path = "crates/callbacks-v1" } litellm-callbacks-v1-python = { path = "crates/callbacks-v1-python" } +litellm-callbacks-v1-exthost = { path = "crates/callbacks-v1-exthost" } litellm-framing = { path = "crates/framer" } litellm-auth = { path = "crates/auth" } litellm-auth-aws = { path = "crates/auth-aws" } diff --git a/litellm-rust/crates/callbacks-legacy-python/src/python.rs b/litellm-rust/crates/callbacks-legacy-python/src/python.rs index 1b225e03bae..35016ca962e 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/python.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/python.rs @@ -1,10 +1,10 @@ //! The Python contract of this crate: every function of -//! `litellm/rust_bridge/callbacks_legacy_python.py` that Rust calls, by name. +//! `litellm/rust_bridge/callbacks_legacy.py` that Rust calls, by name. use pyo3::prelude::*; use strum::{IntoStaticStr, VariantArray}; -const MODULE: &str = "litellm.rust_bridge.callbacks_legacy_python"; +const MODULE: &str = "litellm.rust_bridge.callbacks_legacy"; /// Every litellm Python internal the native call still borrows, grouped by the subsystem it /// belongs to. Rust drives the call; these exist only so behaviour that Python owns today @@ -12,7 +12,7 @@ const MODULE: &str = "litellm.rust_bridge.callbacks_legacy_python"; /// A group is deleted once Rust owns that subsystem, so this enum only shrinks. Calling a /// user's own callback is not borrowing and does not belong here. /// -/// `litellm/rust_bridge/callbacks_legacy_python.py` is the only Python module behind it, and +/// `litellm/rust_bridge/callbacks_legacy.py` is the only Python module behind it, and /// `python_contract.json` pins each function's parameters on both sides. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum LegacyPython { diff --git a/litellm-rust/crates/callbacks-legacy-python/tests/support.rs b/litellm-rust/crates/callbacks-legacy-python/tests/support.rs index 245cdfc89bf..c7a95944c8e 100644 --- a/litellm-rust/crates/callbacks-legacy-python/tests/support.rs +++ b/litellm-rust/crates/callbacks-legacy-python/tests/support.rs @@ -5,12 +5,12 @@ use pyo3::types::{PyDict, PyTuple}; use crate::{LegacyPythonLifecycle, LegacyPythonSurface, PublicCall}; -/// The parameters of every `callbacks_legacy_python` function, as the real module declares them. -/// `tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py` pins this file to the Python +/// The parameters of every `callbacks.legacy` function, as the real module declares them. +/// `tests/test_litellm/rust_bridge/test_callbacks_legacy.py` pins this file to the Python /// signatures, and [`namespace`] binds every fake call against it. pub(crate) const PYTHON_CONTRACT: &str = include_str!("../python_contract.json"); -/// Stand-ins for `callbacks_legacy_python`, the only Python module the crate calls. Tests +/// Stand-ins for `callbacks.legacy`, the only Python module the crate calls. Tests /// share one interpreter and run concurrently, so each fake is installed idempotently and /// forwards to the per-test `StubLogger` it is handed (directly, or as `kwargs['logger']`). /// Every fake is bound against the contract first, so a call the real module would reject @@ -23,10 +23,10 @@ import sys import traceback import types -for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_legacy_python'): +for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_legacy'): sys.modules.setdefault(name, types.ModuleType(name)) -legacy = sys.modules['litellm.rust_bridge.callbacks_legacy_python'] +legacy = sys.modules['litellm.rust_bridge.callbacks_legacy'] CONTRACT = json.loads(python_contract) diff --git a/litellm-rust/crates/callbacks-v1-exthost/AGENTS.md b/litellm-rust/crates/callbacks-v1-exthost/AGENTS.md new file mode 100644 index 00000000000..b90180091aa --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-exthost/AGENTS.md @@ -0,0 +1,18 @@ +- Target invariants, not completion claims +- Keep this crate the out-of-process host of `litellm-callbacks-v1` and nothing else: the wire protocol to an extension host worker and the client that speaks it + - The protocol of a call (which envelope, to whom, how patches fold, patch validation) belongs to `litellm-callbacks-v1`'s `CallSession`; this crate sends what the session says and brings answers back to it. It never decides protocol and never validates a patch itself + - No PyO3 and no interpreter in this crate or in the process that uses it. The worker is `python -m litellm.callbacks_v1.host` (`litellm/callbacks_v1/host.py`), which loads callbacks with the SDK's own `register` and decorators; a callback file is the same file in the SDK and behind this host + - `callbacks-v1-python` is the sibling adapter for the SDK (in process, inline). The two share the session and nothing else +- Terms (contract terms are in `callbacks-v1/AGENTS.md`) + - Worker: the process that runs callbacks. Gateway: the process that owns calls and this crate + - Frame: a 4-byte big-endian length and one JSON object tagged by `type`. `ToWorker` is `event`, `intercept`, `flush`; `FromWorker` is `hello`, `patch`, `error`, `report`, `flushed` + - Hello: the worker's first frame, its `Subscription`s as data; `subscriber` in every later frame is an index into it + - Report: an observer's swallowed failure, told to the gateway for logging only +- The worker only answers. There is no frame by which it calls into the gateway; a host service (sending vendor HTTP, a secret by name) would be a new frame and a contract capability, never an escape hatch +- Everything from a worker is untrusted: frames are size-capped before they are read, unknown frame types and unknown fields are errors, and a patch is data until `Interception::patched` validates it +- This crate isolates nothing. What a worker can reach (environment, files, user, network) is what the process it runs in was given; the spawner clears the environment and picks the working directory, and the deployment (a sidecar, a sandbox runtime, another host) supplies the rest +- Observers are asynchronous to the call: `observe` writes and returns. Only `intercept` waits, and only it may carry a timeout or a failure policy + - An interceptor that raises is `ExtHostError::Interceptor`; whether that fails the call or skips the patch is the gateway's policy per subscriber, not the worker's +- `golden/` pins every frame for both sides: Rust serialises `to_worker` and parses `from_worker`; `tests/test_litellm/callbacks_v1/test_host.py` feeds the worker the first and expects the second. The frames embed the contract's own golden envelope and request, so the two golden directories cannot drift +- `Worker` is the blocking reference client: what a conforming gateway does, in the simplest form that can be tested end to end. The production host (async, supervised, a bounded queue per subscriber, restart, timeouts) replaces the client and keeps the protocol +- `tests/worker.rs` is end to end against a real Python worker and is `#[ignore]`d because it needs an interpreter that can import `litellm`; run it with `PYO3_PYTHON=... cargo test -p litellm-callbacks-v1-exthost -- --ignored` diff --git a/litellm-rust/crates/callbacks-v1-exthost/Cargo.toml b/litellm-rust/crates/callbacks-v1-exthost/Cargo.toml new file mode 100644 index 00000000000..e1a5736d7c2 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-exthost/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "litellm-callbacks-v1-exthost" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-callbacks-v1.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true + +[dev-dependencies] +litellm-host.workspace = true diff --git a/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/error.json b/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/error.json new file mode 100644 index 00000000000..2584fa55b7a --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/error.json @@ -0,0 +1,6 @@ +{ + "type": "error", + "id": 1, + "error_class": "builtins.PermissionError", + "message": "not this model" +} diff --git a/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/flushed.json b/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/flushed.json new file mode 100644 index 00000000000..e9c8814e51d --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/flushed.json @@ -0,0 +1,4 @@ +{ + "type": "flushed", + "id": 2 +} diff --git a/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/hello.json b/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/hello.json new file mode 100644 index 00000000000..9ff1a5f2e6d --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/hello.json @@ -0,0 +1,21 @@ +{ + "type": "hello", + "schema": 1, + "subscriptions": [ + { + "name": "recorder", + "events": [ + "call.failed", + "call.succeeded" + ], + "observes": true, + "intercepts": false + }, + { + "name": "my_pkg.cb.patch", + "events": [], + "observes": false, + "intercepts": true + } + ] +} diff --git a/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/patch.json b/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/patch.json new file mode 100644 index 00000000000..5f899ced7c4 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/patch.json @@ -0,0 +1,20 @@ +{ + "type": "patch", + "id": 1, + "patch": { + "headers": { + "remove": [ + "X-Trace" + ], + "set": [ + [ + "X-Team", + "core" + ] + ] + }, + "body": { + "input": "patched" + } + } +} diff --git a/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/patch_none.json b/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/patch_none.json new file mode 100644 index 00000000000..928276edc0f --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/patch_none.json @@ -0,0 +1,5 @@ +{ + "type": "patch", + "id": 1, + "patch": null +} diff --git a/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/report.json b/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/report.json new file mode 100644 index 00000000000..00ca6644302 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-exthost/golden/from_worker/report.json @@ -0,0 +1,6 @@ +{ + "type": "report", + "subscriber": 0, + "event": "call.started", + "message": "boom" +} diff --git a/litellm-rust/crates/callbacks-v1-exthost/golden/to_worker/event.json b/litellm-rust/crates/callbacks-v1-exthost/golden/to_worker/event.json new file mode 100644 index 00000000000..5573f5bd88e --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-exthost/golden/to_worker/event.json @@ -0,0 +1,21 @@ +{ + "type": "event", + "subscriber": 0, + "envelope": { + "schema": 1, + "call_id": "call-123", + "call_type": "ocr", + "seq": 0, + "event": { + "type": "call.started", + "start_time": 10.0, + "asynchronous": true, + "metadata": { + "team": "core" + }, + "metadata_dropped": [ + "broken" + ] + } + } +} diff --git a/litellm-rust/crates/callbacks-v1-exthost/golden/to_worker/flush.json b/litellm-rust/crates/callbacks-v1-exthost/golden/to_worker/flush.json new file mode 100644 index 00000000000..853062d6df1 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-exthost/golden/to_worker/flush.json @@ -0,0 +1,4 @@ +{ + "type": "flush", + "id": 2 +} diff --git a/litellm-rust/crates/callbacks-v1-exthost/golden/to_worker/intercept.json b/litellm-rust/crates/callbacks-v1-exthost/golden/to_worker/intercept.json new file mode 100644 index 00000000000..a7b14863756 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-exthost/golden/to_worker/intercept.json @@ -0,0 +1,22 @@ +{ + "type": "intercept", + "id": 1, + "subscriber": 1, + "request": { + "model": "model", + "custom_llm_provider": "provider", + "optional_params": { + "temperature": 0.2 + }, + "url": "https://provider.example/v1", + "headers": [ + [ + "Authorization", + "[REDACTED]" + ] + ], + "body": { + "input": "hello" + } + } +} diff --git a/litellm-rust/crates/callbacks-v1-exthost/src/lib.rs b/litellm-rust/crates/callbacks-v1-exthost/src/lib.rs new file mode 100644 index 00000000000..a75588e23c4 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-exthost/src/lib.rs @@ -0,0 +1,15 @@ +//! The out-of-process host of the v1 callback contract: the gateway's side of the socket to +//! an extension host worker (`python -m litellm.callbacks_v1.host`). The gateway never loads +//! a callback; the worker reports its subscriptions, and from then on this crate sends it +//! what a [`CallSession`](litellm_callbacks_v1::CallSession) says to deliver and brings +//! back interceptor patches for the session to validate. +//! +//! [`protocol`] is the wire, pinned by `golden/` on both sides. [`Worker`] is the blocking +//! reference client: what a conforming gateway does, in the simplest form that can be +//! tested end to end. The production host (async, supervised, queued) reuses the protocol +//! and replaces the client. + +pub mod protocol; +mod worker; + +pub use worker::{ExtHostError, Report, Worker}; diff --git a/litellm-rust/crates/callbacks-v1-exthost/src/protocol.rs b/litellm-rust/crates/callbacks-v1-exthost/src/protocol.rs new file mode 100644 index 00000000000..fa453bc2c01 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-exthost/src/protocol.rs @@ -0,0 +1,254 @@ +//! Frames: a 4-byte big-endian length, then that many bytes of JSON, one object tagged by +//! `type`. The worker only answers; there is no frame by which it calls into the gateway. +//! Everything a worker sends is untrusted: sizes are capped, unknown frames and fields are +//! errors, and a patch is only data until the session validates it. + +use std::io::{self, Read, Write}; + +use litellm_callbacks_v1::{Envelope, RequestFacts, Subscription, WirePatch}; +use serde::{Deserialize, Serialize}; + +pub const MAX_FRAME_BYTES: usize = 64 * 1024 * 1024; + +/// `subscriber` is an index into the hello's subscriptions. +#[derive(Debug, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ToWorker<'a> { + /// Fire and forget: nothing an observer does comes back, except a [`FromWorker::Report`]. + Event { + subscriber: usize, + envelope: &'a Envelope, + }, + Intercept { + id: u64, + subscriber: usize, + request: &'a RequestFacts, + }, + /// Answered by `Flushed` once every event sent before it has been handled. + Flush { id: u64 }, +} + +#[derive(Debug, PartialEq, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum FromWorker { + Hello { + schema: u32, + subscriptions: Vec, + }, + /// `None` is an interceptor that returned nothing. + Patch { + id: u64, + patch: Option, + }, + Error { + id: u64, + error_class: String, + message: String, + }, + /// An observer raised; the worker swallowed it, as the contract says. + Report { + subscriber: usize, + event: Option, + message: String, + }, + Flushed { + id: u64, + }, +} + +pub fn write_frame(writer: &mut impl Write, frame: &ToWorker<'_>) -> io::Result<()> { + let body = serde_json::to_vec(frame)?; + let length = u32::try_from(body.len()) + .ok() + .filter(|_| body.len() <= MAX_FRAME_BYTES) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "frame too large"))?; + writer.write_all(&length.to_be_bytes())?; + writer.write_all(&body)?; + writer.flush() +} + +/// The next frame, or `None` when the worker closed the connection between frames. +pub fn read_frame(reader: &mut impl Read) -> io::Result> { + let mut header = [0u8; 4]; + match reader.read_exact(&mut header) { + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(None), + other => other?, + } + let length = u32::from_be_bytes(header) as usize; + if length > MAX_FRAME_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("frame of {length} bytes exceeds {MAX_FRAME_BYTES}"), + )); + } + let mut body = vec![0u8; length]; + reader.read_exact(&mut body)?; + serde_json::from_slice(&body) + .map(Some) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error)) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use litellm_callbacks_v1::{CallFacts, CallSession, EventKind, HeaderPatch}; + use litellm_host::event::{RequestContext, WireRequest}; + use serde_json::{Value, json}; + + use super::*; + + fn subscriptions() -> Vec { + vec![ + Subscription { + name: "recorder".to_string(), + events: BTreeSet::from([EventKind::CallFailed, EventKind::CallSucceeded]), + observes: true, + intercepts: false, + }, + Subscription { + name: "my_pkg.cb.patch".to_string(), + events: BTreeSet::new(), + observes: false, + intercepts: true, + }, + ] + } + + fn golden(text: &str) -> Value { + serde_json::from_str(text).unwrap() + } + + /// The frames carry the contract's own golden envelope and request, so the two golden + /// directories cannot drift apart. + #[test] + fn frames_to_the_worker_match_golden() { + let facts = CallFacts { + start_time: 10.0, + asynchronous: true, + metadata: json!({"team": "core"}).as_object().unwrap().clone(), + metadata_dropped: vec!["broken".to_string()], + }; + let (session, started) = + CallSession::start("call-123".to_string(), "ocr", subscriptions(), facts); + let interception = session.interception( + WireRequest { + url: "https://provider.example/v1".to_string(), + headers: vec![("Authorization".to_string(), "secret".to_string())], + body: json!({"input": "hello"}), + }, + RequestContext { + model: "model".to_string(), + custom_llm_provider: "provider".to_string(), + optional_params: json!({"temperature": 0.2}), + secret_fields: Vec::new(), + api_key: None, + }, + ); + let turn = interception.turn().unwrap(); + + let frames = [ + ( + ToWorker::Event { + subscriber: 0, + envelope: &started.envelope, + }, + include_str!("../golden/to_worker/event.json"), + ), + ( + ToWorker::Intercept { + id: 1, + subscriber: turn.subscriber, + request: &turn.request, + }, + include_str!("../golden/to_worker/intercept.json"), + ), + ( + ToWorker::Flush { id: 2 }, + include_str!("../golden/to_worker/flush.json"), + ), + ]; + for (frame, expected) in frames { + assert_eq!(serde_json::to_value(&frame).unwrap(), golden(expected)); + } + } + + #[test] + fn frames_from_the_worker_parse_from_golden() { + let parsed = |text: &str| serde_json::from_str::(text).unwrap(); + + assert_eq!( + parsed(include_str!("../golden/from_worker/hello.json")), + FromWorker::Hello { + schema: 1, + subscriptions: subscriptions() + } + ); + assert_eq!( + parsed(include_str!("../golden/from_worker/patch.json")), + FromWorker::Patch { + id: 1, + patch: Some(WirePatch { + headers: HeaderPatch { + remove: vec!["X-Trace".to_string()], + set: vec![("X-Team".to_string(), "core".to_string())], + }, + body: Some(json!({"input": "patched"})), + }), + } + ); + assert_eq!( + parsed(include_str!("../golden/from_worker/patch_none.json")), + FromWorker::Patch { id: 1, patch: None } + ); + assert_eq!( + parsed(include_str!("../golden/from_worker/error.json")), + FromWorker::Error { + id: 1, + error_class: "builtins.PermissionError".to_string(), + message: "not this model".to_string(), + } + ); + assert_eq!( + parsed(include_str!("../golden/from_worker/report.json")), + FromWorker::Report { + subscriber: 0, + event: Some("call.started".to_string()), + message: "boom".to_string(), + } + ); + assert_eq!( + parsed(include_str!("../golden/from_worker/flushed.json")), + FromWorker::Flushed { id: 2 } + ); + } + + fn framed(body: &[u8]) -> Vec { + [&(body.len() as u32).to_be_bytes()[..], body].concat() + } + + #[test] + fn a_worker_cannot_say_anything_the_protocol_does_not_name() { + for body in [ + &br#"{"type":"call_gateway","method":"get_secret"}"#[..], + br#"{"type":"flushed","id":1,"extra":true}"#, + br#"{"type":"patch","id":1,"patch":{"url":"https://evil.example"}}"#, + br#"[1,2,3]"#, + ] { + let error = read_frame(&mut framed(body).as_slice()).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + } + } + + #[test] + fn an_oversized_frame_is_refused_before_it_is_read() { + let header = ((MAX_FRAME_BYTES + 1) as u32).to_be_bytes(); + let error = read_frame(&mut header.as_slice()).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + } + + #[test] + fn a_closed_connection_between_frames_is_the_end_not_an_error() { + assert_eq!(read_frame(&mut [].as_slice()).unwrap(), None); + } +} diff --git a/litellm-rust/crates/callbacks-v1-exthost/src/worker.rs b/litellm-rust/crates/callbacks-v1-exthost/src/worker.rs new file mode 100644 index 00000000000..40ece1a9111 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-exthost/src/worker.rs @@ -0,0 +1,214 @@ +//! The blocking reference client of one worker. It starts the worker on a unix socket it +//! listens on (so nothing but a path is handed over, and an attached sidecar connects the +//! same way), reads the hello, and then does exactly what a [`CallSession`] says: sends an +//! emission's envelope to its observers and asks each interceptor for its patch in turn. +//! The session, not this client, decides the protocol and validates every patch. + +use std::{ + io, + os::unix::net::{UnixListener, UnixStream}, + path::Path, + process::{Child, Command}, +}; + +use litellm_callbacks_v1::{Emission, Interception, PatchError, SCHEMA_V1, Subscription}; +use thiserror::Error; + +use crate::protocol::{FromWorker, ToWorker, read_frame, write_frame}; + +#[derive(Debug, Error)] +pub enum ExtHostError { + #[error("extension host i/o: {0}")] + Io(#[from] io::Error), + #[error("extension host protocol: {0}")] + Protocol(String), + #[error("worker speaks schema {0}, this gateway speaks {SCHEMA_V1}")] + Schema(u32), + /// The interceptor raised. Whether that fails the call or skips the patch is the + /// gateway's policy for this subscriber, not the worker's. + #[error("callback {subscriber}: {error_class}: {message}")] + Interceptor { + subscriber: String, + error_class: String, + message: String, + }, + #[error("callback {subscriber}: {error}")] + Patch { + subscriber: String, + error: PatchError, + }, +} + +/// An observer's swallowed failure, as the worker reported it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Report { + pub subscriber: String, + pub event: Option, + pub message: String, +} + +pub struct Worker { + stream: UnixStream, + child: Option, + subscriptions: Vec, + reports: Vec, + next_id: u64, +} + +impl Worker { + /// Starts `command` with `--connect ` appended and waits for its hello. What + /// the worker can reach is whatever `command` was given: its environment, its working + /// directory, its user. This crate grants nothing. + pub fn spawn(mut command: Command, socket: &Path) -> Result { + let listener = UnixListener::bind(socket)?; + let child = command.arg("--connect").arg(socket).spawn()?; + let (stream, _) = listener.accept()?; + Self::greeted(stream, Some(child)) + } + + /// Takes a worker that connected by itself, a sidecar for example. + pub fn attach(stream: UnixStream) -> Result { + Self::greeted(stream, None) + } + + fn greeted(mut stream: UnixStream, child: Option) -> Result { + match read_frame(&mut stream)? { + Some(FromWorker::Hello { + schema: SCHEMA_V1, + subscriptions, + }) => Ok(Self { + stream, + child, + subscriptions, + reports: Vec::new(), + next_id: 0, + }), + Some(FromWorker::Hello { schema, .. }) => Err(ExtHostError::Schema(schema)), + other => Err(ExtHostError::Protocol(format!( + "expected hello, got {other:?}" + ))), + } + } + + /// What the worker's callbacks subscribed to: the session's subscriptions for every + /// call this worker serves. + pub fn subscriptions(&self) -> &[Subscription] { + &self.subscriptions + } + + fn name(&self, subscriber: usize) -> String { + self.subscriptions + .get(subscriber) + .map_or_else(|| format!("#{subscriber}"), |found| found.name.clone()) + } + + fn id(&mut self) -> u64 { + self.next_id += 1; + self.next_id + } + + /// Sends the envelope to each of its observers and returns at once: observers are + /// asynchronous to the call. + pub fn observe(&mut self, emission: &Emission) -> Result<(), ExtHostError> { + for &subscriber in &emission.observers { + write_frame( + &mut self.stream, + &ToWorker::Event { + subscriber, + envelope: &emission.envelope, + }, + )?; + } + Ok(()) + } + + /// The next frame that is not a report; reports are kept for [`Worker::flush`]. + fn answer(&mut self) -> Result { + loop { + match read_frame(&mut self.stream)? { + Some(FromWorker::Report { + subscriber, + event, + message, + }) => { + let subscriber = self.name(subscriber); + self.reports.push(Report { + subscriber, + event, + message, + }); + } + Some(frame) => return Ok(frame), + None => return Err(ExtHostError::Protocol("worker closed".to_string())), + } + } + } + + /// Asks every interceptor for its patch, in the session's order, and folds each into + /// the interception, which validates it. + pub fn intercept(&mut self, interception: Interception) -> Result { + let mut current = interception; + while let Some(turn) = current.turn() { + let id = self.id(); + write_frame( + &mut self.stream, + &ToWorker::Intercept { + id, + subscriber: turn.subscriber, + request: &turn.request, + }, + )?; + let subscriber = self.name(turn.subscriber); + current = match self.answer()? { + FromWorker::Patch { + id: answered, + patch, + } if answered == id => current + .patched(patch.unwrap_or_default()) + .map_err(|error| ExtHostError::Patch { subscriber, error })?, + FromWorker::Error { + id: answered, + error_class, + message, + } if answered == id => { + return Err(ExtHostError::Interceptor { + subscriber, + error_class, + message, + }); + } + other => { + return Err(ExtHostError::Protocol(format!( + "expected the answer to intercept {id}, got {other:?}" + ))); + } + }; + } + Ok(current) + } + + /// Waits until the worker has handled every event sent so far, and returns what its + /// observers reported since the last flush. + pub fn flush(&mut self) -> Result, ExtHostError> { + let id = self.id(); + write_frame(&mut self.stream, &ToWorker::Flush { id })?; + match self.answer()? { + FromWorker::Flushed { id: answered } if answered == id => { + Ok(std::mem::take(&mut self.reports)) + } + other => Err(ExtHostError::Protocol(format!( + "expected flushed {id}, got {other:?}" + ))), + } + } +} + +impl Drop for Worker { + /// Closing the socket is how a worker is told to stop; it exits once its queues drain. + fn drop(&mut self) { + let _ = self.stream.shutdown(std::net::Shutdown::Both); + if let Some(child) = self.child.as_mut() { + let _ = child.wait(); + } + } +} diff --git a/litellm-rust/crates/callbacks-v1-exthost/tests/worker.rs b/litellm-rust/crates/callbacks-v1-exthost/tests/worker.rs new file mode 100644 index 00000000000..f0e37c51e41 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-exthost/tests/worker.rs @@ -0,0 +1,239 @@ +//! End to end: a real `CallSession` driven through a real Python worker. The callback file +//! below is an ordinary SDK callback; nothing in it knows it runs out of process. +//! +//! Ignored by default because it needs an interpreter that can import `litellm`: +//! +//! PYO3_PYTHON=/path/to/.venv/bin/python cargo test -p litellm-callbacks-v1-exthost -- --ignored + +use std::{ + env, fs, + path::{Path, PathBuf}, + process::Command, +}; + +use litellm_callbacks_v1::{CallFacts, CallSession, ErrorFacts, PatchError}; +use litellm_callbacks_v1_exthost::{ExtHostError, Report, Worker}; +use litellm_host::event::{FailureOrigin, RequestContext, Timing, WireRequest}; +use serde_json::{Map, Value, json}; + +const CALLBACKS: &str = r#" +import json +import os + +from litellm.callbacks_v1 import before_send, on_event + +LOG = os.environ["CALLBACK_LOG"] + + +@on_event("call.started", "request.sending", "response.received", "call.succeeded", name="recorder") +async def record(event): + with open(LOG, "a") as log: + log.write(json.dumps([event["seq"], event["event"]["type"], event["event"].get("body")]) + "\n") + + +@on_event("call.started", name="flaky") +def flaky(event): + raise ValueError("boom") + + +@before_send +def tag(request): + return {"headers": {"set": [["X-Team", "core"]]}, "body": {**request["body"], "tagged": True}} + + +@before_send +async def guard(request): + if request["body"].get("input") == "steal": + return {"headers": {"set": [["Authorization", "stolen"]]}} + if request["body"].get("input") == "refuse": + raise PermissionError("not this input") + return None +"#; + +const TIMING: Timing = Timing { + start_time: 10.0, + end_time: 12.5, +}; + +struct Scratch(PathBuf); + +impl Scratch { + fn new(name: &str) -> Self { + let path = env::temp_dir().join(format!("exthost-{}-{name}", std::process::id())); + let _ = fs::remove_dir_all(&path); + fs::create_dir_all(path.join("modules")).unwrap(); + fs::create_dir_all(path.join("cwd")).unwrap(); + fs::write(path.join("modules/gateway_callbacks.py"), CALLBACKS).unwrap(); + Self(path) + } + + fn log(&self) -> Vec { + fs::read_to_string(self.0.join("log")) + .unwrap_or_default() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +/// The worker gets a cleared environment and an empty working directory: it sees what it +/// is given and nothing of the gateway's. +fn worker(scratch: &Scratch) -> Worker { + let python = env::var("PYO3_PYTHON").expect("PYO3_PYTHON names an interpreter with litellm"); + let repository = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); + let python_path = env::join_paths([repository, scratch.0.join("modules")]).unwrap(); + let mut command = Command::new(python); + command + .args([ + "-m", + "litellm.callbacks_v1.host", + "--load", + "gateway_callbacks", + ]) + .env_clear() + .env("PYTHONPATH", python_path) + .env("LITELLM_MODE", "PRODUCTION") + .env("CALLBACK_LOG", scratch.0.join("log")) + .current_dir(scratch.0.join("cwd")); + Worker::spawn(command, &scratch.0.join("socket")).unwrap() +} + +fn wire(input: &str) -> WireRequest { + WireRequest { + url: "https://provider.example/v1".to_string(), + headers: vec![("Authorization".to_string(), "secret".to_string())], + body: json!({"input": input}), + } +} + +fn context() -> RequestContext { + RequestContext { + model: "model".to_string(), + custom_llm_provider: "provider".to_string(), + optional_params: json!({}), + secret_fields: Vec::new(), + api_key: None, + } +} + +fn start(worker: &mut Worker) -> CallSession { + let facts = CallFacts { + start_time: TIMING.start_time, + asynchronous: true, + metadata: Map::new(), + metadata_dropped: Vec::new(), + }; + let (session, started) = CallSession::start( + "call-123".to_string(), + "ocr", + worker.subscriptions().to_vec(), + facts, + ); + worker.observe(&started).unwrap(); + session +} + +#[test] +#[ignore = "needs PYO3_PYTHON to name an interpreter that can import litellm"] +fn a_call_reaches_python_callbacks_and_their_patches_reach_the_wire() { + let scratch = Scratch::new("call"); + let mut worker = worker(&scratch); + let names: Vec<&str> = worker + .subscriptions() + .iter() + .map(|subscription| subscription.name.as_str()) + .collect(); + assert_eq!( + names, + [ + "recorder", + "flaky", + "gateway_callbacks.tag", + "gateway_callbacks.guard" + ] + ); + + let mut session = start(&mut worker); + let interception = worker + .intercept(session.interception(wire("hello"), context())) + .unwrap(); + let (sending, sent) = session.request_sending(interception); + worker.observe(&sending).unwrap(); + worker + .observe(&session.response_received("{}".to_string())) + .unwrap(); + worker + .observe(&session.succeeded(TIMING, json!({"id": "response"}), None)) + .unwrap(); + + assert_eq!(sent.body, json!({"input": "hello", "tagged": true})); + assert!( + sent.headers + .contains(&("X-Team".to_string(), "core".to_string())) + ); + assert!( + sent.headers + .contains(&("Authorization".to_string(), "secret".to_string())) + ); + assert_eq!( + worker.flush().unwrap(), + vec![Report { + subscriber: "flaky".to_string(), + event: Some("call.started".to_string()), + message: "boom".to_string(), + }] + ); + assert_eq!( + scratch.log(), + vec![ + json!([0, "call.started", null]), + json!([1, "request.sending", {"input": "hello", "tagged": true}]), + json!([2, "response.received", "{}"]), + json!([3, "call.succeeded", null]), + ] + ); +} + +#[test] +#[ignore = "needs PYO3_PYTHON to name an interpreter that can import litellm"] +fn a_worker_cannot_patch_a_credential_and_its_refusal_is_the_gateways_to_act_on() { + let scratch = Scratch::new("refusals"); + let mut worker = worker(&scratch); + + let session = start(&mut worker); + let stolen = worker + .intercept(session.interception(wire("steal"), context())) + .unwrap_err(); + assert!(matches!( + stolen, + ExtHostError::Patch { ref subscriber, error: PatchError::ProtectedHeader(_) } + if subscriber == "gateway_callbacks.guard" + )); + + let refused = worker + .intercept(session.interception(wire("refuse"), context())) + .unwrap_err(); + assert!(matches!( + refused, + ExtHostError::Interceptor { ref error_class, ref message, .. } + if error_class == "builtins.PermissionError" && message == "not this input" + )); + + let failed = session.failed( + TIMING, + FailureOrigin::Call, + ErrorFacts { + class: "builtins.PermissionError".to_string(), + message: "not this input".to_string(), + status_code: None, + }, + ); + assert!(failed.observers.is_empty()); + worker.flush().unwrap(); +} diff --git a/litellm-rust/crates/callbacks-v1-python/AGENTS.md b/litellm-rust/crates/callbacks-v1-python/AGENTS.md index c28c89134d2..48c41c42bae 100644 --- a/litellm-rust/crates/callbacks-v1-python/AGENTS.md +++ b/litellm-rust/crates/callbacks-v1-python/AGENTS.md @@ -1,7 +1,9 @@ - Target invariants, not completion claims - Keep this crate the Python runtime adapter for `litellm-callbacks-v1` and nothing else: the subscriber snapshot and one `PythonLifecycle` that delivers the core contract - - Envelopes, sequencing, redaction, handler selection and wire patch validation belong to `litellm-callbacks-v1` - - `litellm/rust_bridge/callbacks_v1_python.py` is the only Python module behind it; `python_contract.json` pins its functions on both sides + - Envelopes, sequencing, routing, the interceptor fold, redaction, handler selection and wire patch validation belong to `litellm-callbacks-v1`; this adapter drives its `CallSession` and adds only projection (Python arguments, response and exception into facts) and execution (calling handlers, `Await`/`resume`, cancellation, GC traversal) + - The snapshot splits each subscriber into the contract's `Subscription` and this crate's `Handlers`, kept at the same index + - `litellm/rust_bridge/callbacks_v1.py` is the only Python module Rust imports; `python_contract.json` pins its functions on both sides + - The typed authoring surface (callback protocols, envelope and patch types, the registry) is `litellm/callbacks_v1/__init__.py`; the bridge module re-exports `snapshot` from it and otherwise holds only marshalling - Terms (contract terms are in `callbacks-v1/AGENTS.md`, lifecycle terms in `host-python/AGENTS.md`) - Registry: the process-wide subscriber list; it lives in Python (`register`/`unregister`), never in Rust - Snapshot: that list read once per call, with one observer and one interceptor handler already selected per subscriber for the call's mode (`subscribers.rs`) diff --git a/litellm-rust/crates/callbacks-v1-python/src/adapter.rs b/litellm-rust/crates/callbacks-v1-python/src/adapter.rs index d834d2c69b9..fbff0291f0f 100644 --- a/litellm-rust/crates/callbacks-v1-python/src/adapter.rs +++ b/litellm-rust/crates/callbacks-v1-python/src/adapter.rs @@ -1,23 +1,25 @@ -//! The v1 lifecycle: each step builds one envelope, hands every subscribed observer a -//! fresh conversion of it, and, before the request is sent, folds every interceptor's -//! patch into the wire request. +//! The v1 lifecycle in CPython. The contract's [`CallSession`] decides which envelope a +//! step produces, who receives it and how patches fold into the wire; this adapter only +//! projects Python values into facts and runs the Python handlers: every observer of an +//! emission gets a fresh conversion of its envelope, inline, and every interceptor's +//! answer goes back to the session. use litellm_callbacks_v1::{ - CallFacts, Envelope, ErrorFacts, Event, RequestFacts, Sequencer, WirePatch, apply, + CallFacts, CallSession, Emission, Envelope, ErrorFacts, Interception, Subscription, WirePatch, }; use litellm_host::event::{MachineEvent, RequestContext, WireRequest}; use litellm_host_python::{ LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, is_cancellation, missing_state, to_py, }; -use pyo3::exceptions::PyValueError; -use pyo3::gc::{PyTraverseError, PyVisit}; -use pyo3::prelude::*; -use pyo3::types::{PyDict, PyString}; +use pyo3::{ + exceptions::PyValueError, + gc::{PyTraverseError, PyVisit}, + prelude::*, + types::{PyDict, PyString}, +}; use serde_json::{Map, Value}; -use crate::call::V1PythonSurface; -use crate::python::V1Python; -use crate::subscribers::Subscriber; +use crate::{call::V1PythonSurface, python::V1Python, subscribers::Handlers}; /// What a step hands back to the driver once every observer of its envelope has run. enum Continuation { @@ -26,26 +28,29 @@ enum Continuation { Wire(Box), } -/// The suspended step; `index` is the subscriber whose awaitable the driver is awaiting. +/// The suspended step, while the driver awaits one handler's awaitable. enum Pending { + /// `position` is the place in `emission.observers` of the observer being awaited. Observe { - envelope: Envelope, - index: usize, + emission: Emission, + position: usize, then: Continuation, }, + /// `subscriber` is the interceptor whose turn it is. Intercept { - wire: Box, - context: RequestContext, - index: usize, + interception: Interception, + subscriber: usize, }, } pub struct V1PythonLifecycle { surface: V1PythonSurface, - subscribers: Vec, + /// Held until `begin` learns the call's identity and hands them to the session. + subscriptions: Vec, + /// The Python side of each subscription, at the same index. + handlers: Vec, asynchronous: bool, - sequencer: Option, - streamed: bool, + session: Option, pending: Option, closed: bool, } @@ -70,27 +75,21 @@ impl Continuation { impl V1PythonLifecycle { pub(crate) fn new( surface: V1PythonSurface, - subscribers: Vec, + subscriptions: Vec, + handlers: Vec, asynchronous: bool, ) -> Self { Self { surface, - subscribers, + subscriptions, + handlers, asynchronous, - sequencer: None, - streamed: false, + session: None, pending: None, closed: false, } } - fn envelope(&mut self, event: Event) -> PyResult { - self.sequencer - .as_mut() - .map(|sequencer| sequencer.envelope(event)) - .ok_or_else(missing_state) - } - fn report( &self, py: Python<'_>, @@ -98,7 +97,7 @@ impl V1PythonLifecycle { envelope: &Envelope, error: PyErr, ) -> PyResult<()> { - let name = &self.subscribers[index].name; + let name = &self.handlers[index].name; let event: &'static str = envelope.event.kind().into(); match V1Python::Report.call(py, (name, event, error.value(py))) { Err(report_error) if is_cancellation(py, &report_error) => Err(report_error), @@ -106,86 +105,87 @@ impl V1PythonLifecycle { } } + /// Runs the emission's observers inline from `from`, which is one legal schedule of + /// the contract and nothing a callback may rely on. fn observe( &mut self, py: Python<'_>, - envelope: Envelope, + emission: Emission, from: usize, then: Continuation, ) -> PyResult { - for index in from..self.subscribers.len() { - let Some(handler) = self.subscribers[index] - .observer(envelope.event.kind()) + for position in from..emission.observers.len() { + let index = emission.observers[position]; + let Some(handler) = self.handlers[index] + .observe + .as_ref() .map(|handler| handler.clone_ref(py)) else { continue; }; - let outcome = to_py(py, &envelope).and_then(|argument| handler.call(py, argument)); + let outcome = + to_py(py, &emission.envelope).and_then(|argument| handler.call(py, argument)); match outcome { Ok(awaitable) if handler.is_async() => { self.pending = Some(Pending::Observe { - envelope, - index, + emission, + position, then, }); return Ok(LifecycleStep::Await(awaitable.unbind())); } Ok(_) => {} Err(error) if is_cancellation(py, &error) => return Err(error), - Err(error) => self.report(py, index, &envelope, error)?, + Err(error) => self.report(py, index, &emission.envelope, error)?, } } Ok(then.into_step()) } - fn apply_patch( + fn patched( &self, py: Python<'_>, - index: usize, - wire: WireRequest, + interception: Interception, + subscriber: usize, result: Py, - ) -> PyResult { + ) -> PyResult { let patch = if result.bind(py).is_none() { WirePatch::default() } else { from_py::(result.bind(py))? }; - let name = &self.subscribers[index].name; - apply(wire, patch) + let name = &self.handlers[subscriber].name; + interception + .patched(patch) .map_err(|error| PyValueError::new_err(format!("callback {name}: {error}"))) } - fn intercept( - &mut self, - py: Python<'_>, - wire: Box, - context: RequestContext, - from: usize, - ) -> PyResult { - let mut current = wire; - for index in from..self.subscribers.len() { - let Some(handler) = self.subscribers[index] + fn intercept(&mut self, py: Python<'_>, interception: Interception) -> PyResult { + let mut current = interception; + while let Some(turn) = current.turn() { + let Some(handler) = self.handlers[turn.subscriber] .intercept .as_ref() .map(|handler| handler.clone_ref(py)) else { - continue; + return Err(missing_state()); }; - let request = to_py(py, &RequestFacts::new(¤t, &context))?; - let result = handler.call(py, request)?; + let result = handler.call(py, to_py(py, &turn.request)?)?; if handler.is_async() { self.pending = Some(Pending::Intercept { - wire: current, - context, - index, + interception: current, + subscriber: turn.subscriber, }); return Ok(LifecycleStep::Await(result.unbind())); } - current = Box::new(self.apply_patch(py, index, *current, result.unbind())?); + current = self.patched(py, current, turn.subscriber, result.unbind())?; } - let envelope = - self.envelope(Event::RequestSending(RequestFacts::new(¤t, &context)))?; - self.observe(py, envelope, 0, Continuation::Wire(current)) + let (emission, wire) = self + .session + .as_mut() + .ok_or_else(missing_state)? + .request_sending(current); + self.observe(py, emission, 0, Continuation::Wire(Box::new(wire))) } fn metadata(arguments: &Bound<'_, PyDict>) -> PyResult<(Map, Vec)> { @@ -246,14 +246,19 @@ impl PythonLifecycle for V1PythonLifecycle { ) -> PyResult { let call_id = Self::call_id(py, arguments.bind(py))?; let (metadata, metadata_dropped) = Self::metadata(arguments.bind(py))?; - self.sequencer = Some(Sequencer::new(call_id, self.surface.call_type)); - let envelope = self.envelope(Event::CallStarted(CallFacts { - start_time, - asynchronous: self.asynchronous, - metadata, - metadata_dropped, - }))?; - self.observe(py, envelope, 0, Continuation::Arguments(arguments)) + let (session, emission) = CallSession::start( + call_id, + self.surface.call_type, + std::mem::take(&mut self.subscriptions), + CallFacts { + start_time, + asynchronous: self.asynchronous, + metadata, + metadata_dropped, + }, + ); + self.session = Some(session); + self.observe(py, emission, 0, Continuation::Arguments(arguments)) } fn before_send( @@ -262,16 +267,23 @@ impl PythonLifecycle for V1PythonLifecycle { wire: Box, context: &RequestContext, ) -> PyResult { - self.intercept(py, wire, context.clone(), 0) + let interception = self + .session + .as_ref() + .ok_or_else(missing_state)? + .interception(*wire, context.clone()); + self.intercept(py, interception) } fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult { match event { LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => { - let envelope = self.envelope(Event::ResponseReceived { - body: raw.body.clone(), - })?; - self.observe(py, envelope, 0, Continuation::Done) + let emission = self + .session + .as_mut() + .ok_or_else(missing_state)? + .response_received(raw.body.clone()); + self.observe(py, emission, 0, Continuation::Done) } LifecycleEvent::Succeeded { timing, response } => { let projected = V1Python::ProjectResponse @@ -282,13 +294,9 @@ impl PythonLifecycle for V1PythonLifecycle { Err(error) if is_cancellation(py, &error) => return Err(error), Err(error) => (Value::Null, Some(error.to_string())), }; - let envelope = self.envelope(Event::CallSucceeded { - timing: timing.into(), - streamed: self.streamed, - response, - response_error, - })?; - self.observe(py, envelope, 0, Continuation::Done) + let session = self.session.take().ok_or_else(missing_state)?; + let emission = session.succeeded(timing, response, response_error); + self.observe(py, emission, 0, Continuation::Done) } LifecycleEvent::Failed { timing, @@ -296,44 +304,41 @@ impl PythonLifecycle for V1PythonLifecycle { error, } => { let facts = Self::error_facts(py, error)?; - let envelope = self.envelope(Event::CallFailed { - timing: timing.into(), - streamed: self.streamed, - origin: origin.into(), - error: facts, - })?; - self.observe(py, envelope, 0, Continuation::Done) + let session = self.session.take().ok_or_else(missing_state)?; + let emission = session.failed(timing, origin, facts); + self.observe(py, emission, 0, Continuation::Done) } } } fn opened(&mut self, _py: Python<'_>) -> PyResult<()> { - self.streamed = true; + if let Some(session) = self.session.as_mut() { + session.opened(); + } Ok(()) } fn resume(&mut self, py: Python<'_>, result: PyResult>) -> PyResult { match self.pending.take().ok_or_else(missing_state)? { Pending::Observe { - envelope, - index, + emission, + position, then, } => { if let Err(error) = result { if is_cancellation(py, &error) { return Err(error); } - self.report(py, index, &envelope, error)?; + self.report(py, emission.observers[position], &emission.envelope, error)?; } - self.observe(py, envelope, index + 1, then) + self.observe(py, emission, position + 1, then) } Pending::Intercept { - wire, - context, - index, + interception, + subscriber, } => { - let patched = self.apply_patch(py, index, *wire, result?)?; - self.intercept(py, Box::new(patched), context, index + 1) + let patched = self.patched(py, interception, subscriber, result?)?; + self.intercept(py, patched) } } } @@ -344,12 +349,14 @@ impl PythonLifecycle for V1PythonLifecycle { } self.closed = true; self.pending = None; - self.subscribers.clear(); + self.session = None; + self.subscriptions.clear(); + self.handlers.clear(); } fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { - for subscriber in &self.subscribers { - subscriber.traverse(visit)?; + for handlers in &self.handlers { + handlers.traverse(visit)?; } match &self.pending { Some(Pending::Observe { then, .. }) => then.traverse(visit), diff --git a/litellm-rust/crates/callbacks-v1-python/src/call.rs b/litellm-rust/crates/callbacks-v1-python/src/call.rs index 5826b993446..f4dd860397c 100644 --- a/litellm-rust/crates/callbacks-v1-python/src/call.rs +++ b/litellm-rust/crates/callbacks-v1-python/src/call.rs @@ -2,8 +2,7 @@ use pyo3::prelude::*; -use crate::adapter::V1PythonLifecycle; -use crate::subscribers::snapshot; +use crate::{adapter::V1PythonLifecycle, subscribers::snapshot}; #[derive(Clone, Copy, Debug)] pub struct V1PythonSurface { @@ -19,7 +18,9 @@ impl V1PythonLifecycle { surface: V1PythonSurface, asynchronous: bool, ) -> PyResult> { - let subscribers = snapshot(py, asynchronous)?; - Ok((!subscribers.is_empty()).then(|| Self::new(surface, subscribers, asynchronous))) + let (subscriptions, handlers): (Vec<_>, Vec<_>) = + snapshot(py, asynchronous)?.into_iter().unzip(); + Ok((!subscriptions.is_empty()) + .then(|| Self::new(surface, subscriptions, handlers, asynchronous))) } } diff --git a/litellm-rust/crates/callbacks-v1-python/src/python.rs b/litellm-rust/crates/callbacks-v1-python/src/python.rs index 0fd75cab0d2..6fb19e4d7e5 100644 --- a/litellm-rust/crates/callbacks-v1-python/src/python.rs +++ b/litellm-rust/crates/callbacks-v1-python/src/python.rs @@ -1,10 +1,10 @@ //! The Python contract of this crate: every function of -//! `litellm/rust_bridge/callbacks_v1_python.py` that Rust calls, by name. +//! `litellm/rust_bridge/callbacks_v1.py` that Rust calls, by name. use pyo3::prelude::*; use strum::{IntoStaticStr, VariantArray}; -const MODULE: &str = "litellm.rust_bridge.callbacks_v1_python"; +const MODULE: &str = "litellm.rust_bridge.callbacks_v1"; #[derive(Clone, Copy, Debug, IntoStaticStr, PartialEq, Eq, VariantArray)] pub(crate) enum V1Python { diff --git a/litellm-rust/crates/callbacks-v1-python/src/subscribers.rs b/litellm-rust/crates/callbacks-v1-python/src/subscribers.rs index a63e3aa842f..70f0a7a84af 100644 --- a/litellm-rust/crates/callbacks-v1-python/src/subscribers.rs +++ b/litellm-rust/crates/callbacks-v1-python/src/subscribers.rs @@ -1,13 +1,19 @@ //! The per-call subscriber snapshot: what Python's `register` recorded, read once, with //! one handler chosen per role for the call's execution mode. `register` has already -//! rejected everything rejectable, so this only parses. +//! rejected everything rejectable, so this only parses. Each subscriber splits in two: the +//! [`Subscription`] the contract's session routes by, and the [`Handlers`] only this host +//! can call. use std::collections::BTreeSet; -use litellm_callbacks_v1::{EventKind, ExecutionMode, HandlerSelection, select_handler}; -use pyo3::exceptions::PyValueError; -use pyo3::gc::{PyTraverseError, PyVisit}; -use pyo3::prelude::*; +use litellm_callbacks_v1::{ + EventKind, ExecutionMode, HandlerSelection, Subscription, select_handler, +}; +use pyo3::{ + exceptions::PyValueError, + gc::{PyTraverseError, PyVisit}, + prelude::*, +}; use crate::python::V1Python; @@ -17,9 +23,9 @@ pub enum Handler { Async(Py), } -pub struct Subscriber { +/// The Python callables of one subscriber, at the index of its [`Subscription`]. +pub struct Handlers { pub name: String, - pub events: BTreeSet, /// `on_event` or `async_on_event`: cannot affect the call. pub observe: Option, /// `before_send` or `async_before_send`: returns a patch; its failure fails the call. @@ -43,7 +49,7 @@ fn selected_handler( ) } -pub fn snapshot(py: Python<'_>, asynchronous: bool) -> PyResult> { +pub fn snapshot(py: Python<'_>, asynchronous: bool) -> PyResult> { let mode = if asynchronous { ExecutionMode::Async } else { @@ -64,23 +70,28 @@ pub fn snapshot(py: Python<'_>, asynchronous: bool) -> PyResult> }) }) .collect::>>()?; - Ok(Subscriber { - name: subscriber.getattr("name")?.extract()?, - events, - observe: selected_handler(&subscriber, mode, "on_event", "async_on_event")?, - intercept: selected_handler(&subscriber, mode, "before_send", "async_before_send")?, - }) + let name: String = subscriber.getattr("name")?.extract()?; + let observe = selected_handler(&subscriber, mode, "on_event", "async_on_event")?; + let intercept = + selected_handler(&subscriber, mode, "before_send", "async_before_send")?; + Ok(( + Subscription { + name: name.clone(), + events, + observes: observe.is_some(), + intercepts: intercept.is_some(), + }, + Handlers { + name, + observe, + intercept, + }, + )) }) .collect() } -impl Subscriber { - pub fn observer(&self, kind: EventKind) -> Option<&Handler> { - self.observe - .as_ref() - .filter(|_| self.events.contains(&kind)) - } - +impl Handlers { pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { for handler in self.observe.iter().chain(&self.intercept) { visit.call(handler.object())?; diff --git a/litellm-rust/crates/callbacks-v1-python/tests/lifecycle.rs b/litellm-rust/crates/callbacks-v1-python/tests/lifecycle.rs index 6d961e4ca9c..490ca667bdb 100644 --- a/litellm-rust/crates/callbacks-v1-python/tests/lifecycle.rs +++ b/litellm-rust/crates/callbacks-v1-python/tests/lifecycle.rs @@ -1,18 +1,18 @@ use std::collections::BTreeSet; use litellm_auth::SecretValue; +use litellm_callbacks_v1::{EventKind, Subscription}; use litellm_host::event::{MachineEvent, RawResponse, RequestContext, Timing, WireRequest}; use litellm_host_python::{LifecycleEvent, LifecycleStep, PythonLifecycle, from_py}; -use pyo3::exceptions::asyncio::CancelledError; -use pyo3::prelude::*; -use pyo3::types::PyDict; +use pyo3::{exceptions::asyncio::CancelledError, prelude::*, types::PyDict}; use serde_json::{Value, json}; -use crate::adapter::V1PythonLifecycle; -use crate::call::V1PythonSurface; -use crate::subscribers::{Handler, Subscriber}; -use crate::test_support::{local, namespace}; -use litellm_callbacks_v1::EventKind; +use crate::{ + adapter::V1PythonLifecycle, + call::V1PythonSurface, + subscribers::{Handler, Handlers}, + test_support::{local, namespace}, +}; const TIMING: Timing = Timing { start_time: 10.0, @@ -25,7 +25,7 @@ fn subscriber( events: &[EventKind], observe: Option<(&str, bool)>, intercept: Option<(&str, bool)>, -) -> Subscriber { +) -> (Subscription, Handlers) { let handler = |spec: Option<(&str, bool)>| { spec.map(|(local_name, asynchronous)| { let callable = local(locals, local_name).unbind(); @@ -36,18 +36,28 @@ fn subscriber( } }) }; - Subscriber { + let handlers = Handlers { name: name.to_string(), - events: events.iter().copied().collect::>(), observe: handler(observe), intercept: handler(intercept), - } + }; + ( + Subscription { + name: name.to_string(), + events: events.iter().copied().collect::>(), + observes: handlers.observe.is_some(), + intercepts: handlers.intercept.is_some(), + }, + handlers, + ) } -fn lifecycle(subscribers: Vec, asynchronous: bool) -> V1PythonLifecycle { +fn lifecycle(subscribers: Vec<(Subscription, Handlers)>, asynchronous: bool) -> V1PythonLifecycle { + let (subscriptions, handlers): (Vec<_>, Vec<_>) = subscribers.into_iter().unzip(); V1PythonLifecycle::new( V1PythonSurface { call_type: "ocr" }, - subscribers, + subscriptions, + handlers, asynchronous, ) } @@ -245,7 +255,7 @@ def record(event): ["call.started"] ); let reports = py - .import("litellm.rust_bridge.callbacks_v1_python") + .import("litellm.rust_bridge.callbacks_v1") .unwrap() .getattr("reports") .unwrap(); diff --git a/litellm-rust/crates/callbacks-v1-python/tests/support.rs b/litellm-rust/crates/callbacks-v1-python/tests/support.rs index 62898c57e04..3f7b3eb0f87 100644 --- a/litellm-rust/crates/callbacks-v1-python/tests/support.rs +++ b/litellm-rust/crates/callbacks-v1-python/tests/support.rs @@ -1,7 +1,6 @@ use std::ffi::CStr; -use pyo3::prelude::*; -use pyo3::types::PyDict; +use pyo3::{prelude::*, types::PyDict}; pub(crate) const PYTHON_CONTRACT: &str = include_str!("../python_contract.json"); @@ -11,10 +10,10 @@ import json import sys import types -for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_v1_python'): +for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_v1'): sys.modules.setdefault(name, types.ModuleType(name)) -module = sys.modules['litellm.rust_bridge.callbacks_v1_python'] +module = sys.modules['litellm.rust_bridge.callbacks_v1'] contract = json.loads(python_contract) def contracted(name, fake): diff --git a/litellm-rust/crates/callbacks-v1/AGENTS.md b/litellm-rust/crates/callbacks-v1/AGENTS.md index 2b989ffa1e6..b7621ba3331 100644 --- a/litellm-rust/crates/callbacks-v1/AGENTS.md +++ b/litellm-rust/crates/callbacks-v1/AGENTS.md @@ -1,5 +1,10 @@ - Target invariants, not completion claims -- Keep this crate the runtime-neutral v1 callback contract and nothing else: envelopes, sequencing, redaction, handler selection and validated wire patches +- Keep this crate the runtime-neutral v1 callback contract and nothing else: envelopes, redaction, handler selection, validated wire patches, and the per-call session that sequences and routes them + - `CallSession` is the whole protocol of one call and does no I/O: which envelope a moment produces, which subscriptions receive it, how interceptor patches fold into the wire. A host adapter projects its runtime's values into facts, asks the session, and runs handlers its own way; it never decides protocol. A second host (out of process, for the gateway) reuses the session unchanged + - The two planes are in the types. An `Emission` has no return path, so nothing an observer does re-enters the session. An `Interception` takes exactly one patch per interceptor, in registration order, and `request_sending` yields the wire the host must send + - A terminal method consumes the session, which is how a call has exactly one terminal envelope, last + - `Subscription` is a subscriber as data (name, events, observes, intercepts) and serializes; handlers stay in the host adapter at the same index. It is what a host that loads handlers elsewhere reports back + - Protocol tests live here and need no interpreter; they are the conformance suite every host adapter inherits. An adapter's own tests cover projection and execution only - This crate may depend on `litellm-host`, but never on `litellm-host-python`, PyO3 or a runtime registry - `golden/v1` pins every serialized envelope shape; every `EventKind` has a golden file by construction, and Python's `EVENTS` and `SUPPORTED_SCHEMAS` are tested against those files - Terms (shared ones are in `host-python/AGENTS.md`) @@ -7,10 +12,14 @@ - Handler: one callable of a subscriber; an observer receives envelopes and cannot affect the call; an interceptor receives `RequestFacts` and returns a patch - Handler selection (`select_handler`): an async call prefers the async handler and falls back to the sync one; a sync call never selects an async handler - Event: one thing that happened in a call; `EventKind` is its dotted name (`call.started`, `request.sending`, `response.received`, `call.succeeded`, `call.failed`), the serde tag and the subscription name at once - - Envelope: schema + `call_id` + `call_type` + `seq` + event; the only value an observer receives; `Sequencer` numbers a call's envelopes from zero + - Envelope: schema + `call_id` + `call_type` + `seq` + event; the only value an observer receives; the session numbers a call's envelopes from zero - Facts (`CallFacts`, `RequestFacts`, `TimingFacts`, `ErrorFacts`): a redacted, serializable projection of runtime state; never a live object - Patch (`WirePatch`): an interceptor's edit of the wire: header `remove`, then header `set`, optional `body` replacement; `apply` validates it - Redaction: credential headers by name (`CREDENTIAL_HEADERS`) and optional params by the route's secret fields become `[REDACTED]`; shape never changes +- Observers are asynchronous to the call. This is contract semantics, not a property of one host + - An envelope may reach an observer after the call has moved on or returned. Envelopes of one call arrive in `seq` order; nothing orders two calls. Delivery is at most once + - Running every observer inline before the step returns, as the in-process CPython host does, is one legal schedule and promises nothing a callback may rely on. A host that queues, batches or crosses a process boundary is equally conforming + - Nothing an observer does or returns has a path back into the call. Interceptors are the opposite: on the call path by nature, answered before the request leaves - No runtime object of a call reaches a callback contract value - `RequestFacts::new` is the only request projection and always applies the contract's credential redaction; its fields are private so no other crate can build one - A patch cannot change the URL or set or remove a credential header diff --git a/litellm-rust/crates/callbacks-v1/src/envelope.rs b/litellm-rust/crates/callbacks-v1/src/envelope.rs index 7f78b999a32..3e69afb649b 100644 --- a/litellm-rust/crates/callbacks-v1/src/envelope.rs +++ b/litellm-rust/crates/callbacks-v1/src/envelope.rs @@ -3,7 +3,7 @@ //! redacted, serializable projection that never holds a runtime object of the call. use litellm_host::event::{FailureOrigin, RequestContext, Timing, WireRequest}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use strum::{EnumString, IntoStaticStr, VariantArray}; @@ -57,17 +57,24 @@ pub enum Event { PartialOrd, Ord, Debug, + Serialize, + Deserialize, )] pub enum EventKind { #[strum(serialize = "call.started")] + #[serde(rename = "call.started")] CallStarted, #[strum(serialize = "request.sending")] + #[serde(rename = "request.sending")] RequestSending, #[strum(serialize = "response.received")] + #[serde(rename = "response.received")] ResponseReceived, #[strum(serialize = "call.succeeded")] + #[serde(rename = "call.succeeded")] CallSucceeded, #[strum(serialize = "call.failed")] + #[serde(rename = "call.failed")] CallFailed, } @@ -156,14 +163,14 @@ impl Event { } } -pub struct Sequencer { +pub(crate) struct Sequencer { call_id: String, call_type: &'static str, next: u32, } impl Sequencer { - pub fn new(call_id: String, call_type: &'static str) -> Self { + pub(crate) fn new(call_id: String, call_type: &'static str) -> Self { Self { call_id, call_type, @@ -171,7 +178,7 @@ impl Sequencer { } } - pub fn envelope(&mut self, event: Event) -> Envelope { + pub(crate) fn envelope(&mut self, event: Event) -> Envelope { let envelope = Envelope { schema: SCHEMA_V1, call_id: self.call_id.clone(), diff --git a/litellm-rust/crates/callbacks-v1/src/lib.rs b/litellm-rust/crates/callbacks-v1/src/lib.rs index 2d5b27df591..f381a83093b 100644 --- a/litellm-rust/crates/callbacks-v1/src/lib.rs +++ b/litellm-rust/crates/callbacks-v1/src/lib.rs @@ -1,15 +1,17 @@ -//! The runtime-neutral v1 callback contract: envelopes and their sequencing, credential -//! redaction, handler selection and validated wire patches. No PyO3 and no registry; a -//! language host marshals these values to its own subscribers. +//! The runtime-neutral v1 callback contract: envelopes, credential redaction, handler +//! selection, validated wire patches, and the per-call [`CallSession`] that says which +//! envelope goes to whom and folds interceptor patches. No PyO3, no registry and no I/O; a +//! host projects its own values into facts and runs its own subscribers. mod envelope; mod patch; mod policy; mod redact; +mod session; pub use envelope::{ - CallFacts, Envelope, ErrorFacts, Event, EventKind, Origin, RequestFacts, SCHEMA_V1, Sequencer, - TimingFacts, + CallFacts, Envelope, ErrorFacts, Event, EventKind, Origin, RequestFacts, SCHEMA_V1, TimingFacts, }; pub use patch::{HeaderPatch, PatchError, WirePatch, apply}; pub use policy::{ExecutionMode, HandlerSelection, select_handler}; +pub use session::{CallSession, Emission, Interception, Subscription, Turn}; diff --git a/litellm-rust/crates/callbacks-v1/src/session.rs b/litellm-rust/crates/callbacks-v1/src/session.rs new file mode 100644 index 00000000000..530d134b900 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1/src/session.rs @@ -0,0 +1,374 @@ +//! One call's v1 protocol, with no I/O and no runtime: which envelope each moment of the +//! call produces, who receives it, and how interceptor patches fold into the wire +//! request. A host projects its own values into facts, asks the session what to deliver, +//! and runs the handlers however it can: inline in CPython, or across a process boundary. +//! +//! The two planes are visible in the types. An [`Emission`] has no way back: nothing an +//! observer does or returns re-enters the session. An [`Interception`] is answered, one +//! patch per interceptor, before the request leaves. + +use std::collections::BTreeSet; + +use litellm_host::event::{FailureOrigin, RequestContext, Timing, WireRequest}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::{ + envelope::{CallFacts, Envelope, ErrorFacts, Event, EventKind, RequestFacts, Sequencer}, + patch::{PatchError, WirePatch, apply}, +}; + +/// What one subscriber asked for, as data: no handler, no runtime object. It is what a +/// host that runs handlers elsewhere reports back once it has loaded them. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Subscription { + pub name: String, + pub events: BTreeSet, + pub observes: bool, + pub intercepts: bool, +} + +/// One envelope and the subscribers it goes to, as indexes into the session's +/// subscriptions, in registration order. +#[derive(Clone, Debug, PartialEq)] +pub struct Emission { + pub envelope: Envelope, + pub observers: Vec, +} + +/// An interceptor's turn: who is asked, and the request as the earlier ones left it. +#[derive(Clone, Debug, PartialEq)] +pub struct Turn { + pub subscriber: usize, + pub request: RequestFacts, +} + +/// The wire request on its way through the interceptors, in registration order. +#[derive(Debug)] +pub struct Interception { + wire: WireRequest, + context: RequestContext, + interceptors: Vec, + position: usize, +} + +pub struct CallSession { + sequencer: Sequencer, + subscriptions: Vec, + streamed: bool, +} + +impl Interception { + /// The interceptor to ask next, or `None` once every one has answered. + pub fn turn(&self) -> Option { + self.interceptors + .get(self.position) + .map(|&subscriber| Turn { + subscriber, + request: RequestFacts::new(&self.wire, &self.context), + }) + } + + /// The current interceptor's answer; an invalid patch ends the interception. + pub fn patched(self, patch: WirePatch) -> Result { + Ok(Self { + wire: apply(self.wire, patch)?, + position: self.position + 1, + ..self + }) + } +} + +impl CallSession { + /// Opens the call. `call.started` is always envelope zero. + pub fn start( + call_id: String, + call_type: &'static str, + subscriptions: Vec, + facts: CallFacts, + ) -> (Self, Emission) { + let mut session = Self { + sequencer: Sequencer::new(call_id, call_type), + subscriptions, + streamed: false, + }; + let emission = session.emission(Event::CallStarted(facts)); + (session, emission) + } + + fn emission(&mut self, event: Event) -> Emission { + let kind = event.kind(); + Emission { + envelope: self.sequencer.envelope(event), + observers: self + .subscriptions + .iter() + .enumerate() + .filter(|(_, subscription)| { + subscription.observes && subscription.events.contains(&kind) + }) + .map(|(index, _)| index) + .collect(), + } + } + + pub fn interception(&self, wire: WireRequest, context: RequestContext) -> Interception { + Interception { + wire, + context, + interceptors: self + .subscriptions + .iter() + .enumerate() + .filter(|(_, subscription)| subscription.intercepts) + .map(|(index, _)| index) + .collect(), + position: 0, + } + } + + /// Closes an interception: `request.sending` describes the wire as the last + /// interceptor left it, which is the wire the host must send. + pub fn request_sending(&mut self, interception: Interception) -> (Emission, WireRequest) { + let facts = RequestFacts::new(&interception.wire, &interception.context); + ( + self.emission(Event::RequestSending(facts)), + interception.wire, + ) + } + + pub fn response_received(&mut self, body: String) -> Emission { + self.emission(Event::ResponseReceived { body }) + } + + /// The call's stream was handed to the caller; the terminal envelope says so. + pub fn opened(&mut self) { + self.streamed = true; + } + + /// A terminal envelope consumes the session, so a call has exactly one, last. + pub fn succeeded( + mut self, + timing: Timing, + response: Value, + response_error: Option, + ) -> Emission { + let streamed = self.streamed; + self.emission(Event::CallSucceeded { + timing: timing.into(), + streamed, + response, + response_error, + }) + } + + pub fn failed(mut self, timing: Timing, origin: FailureOrigin, error: ErrorFacts) -> Emission { + let streamed = self.streamed; + self.emission(Event::CallFailed { + timing: timing.into(), + streamed, + origin: origin.into(), + error, + }) + } +} + +#[cfg(test)] +mod tests { + use serde_json::{Map, json}; + use strum::VariantArray; + + use super::*; + + const TIMING: Timing = Timing { + start_time: 1.0, + end_time: 2.0, + }; + + fn subscription(name: &str, events: &[EventKind], intercepts: bool) -> Subscription { + Subscription { + name: name.to_string(), + events: events.iter().copied().collect(), + observes: !events.is_empty(), + intercepts, + } + } + + fn facts() -> CallFacts { + CallFacts { + start_time: 1.0, + asynchronous: true, + metadata: Map::new(), + metadata_dropped: Vec::new(), + } + } + + fn wire() -> WireRequest { + WireRequest { + url: "https://provider.example".to_string(), + headers: Vec::new(), + body: json!({"input": "original"}), + } + } + + fn context() -> RequestContext { + RequestContext { + model: "model".to_string(), + custom_llm_provider: "provider".to_string(), + optional_params: json!({}), + secret_fields: Vec::new(), + api_key: None, + } + } + + fn body(value: Value) -> WirePatch { + serde_json::from_value(json!({"body": value})).unwrap() + } + + fn seen_body(turn: &Turn) -> Value { + serde_json::to_value(&turn.request).unwrap()["body"].clone() + } + + fn error() -> ErrorFacts { + ErrorFacts { + class: "builtins.ValueError".to_string(), + message: "bad".to_string(), + status_code: Some(400), + } + } + + fn start(subscriptions: Vec) -> (CallSession, Emission) { + CallSession::start("call".to_string(), "ocr", subscriptions, facts()) + } + + #[test] + fn a_call_numbers_its_envelopes_from_the_start_to_one_terminal() { + let (mut session, started) = start(vec![subscription("all", EventKind::VARIANTS, false)]); + let (sending, _) = { + let interception = session.interception(wire(), context()); + session.request_sending(interception) + }; + let received = session.response_received("{}".to_string()); + let terminal = session.succeeded(TIMING, json!({"id": "response"}), None); + + let seen: Vec<(u32, EventKind)> = [started, sending, received, terminal] + .iter() + .map(|emission| (emission.envelope.seq, emission.envelope.event.kind())) + .collect(); + assert_eq!( + seen, + vec![ + (0, EventKind::CallStarted), + (1, EventKind::RequestSending), + (2, EventKind::ResponseReceived), + (3, EventKind::CallSucceeded), + ] + ); + } + + #[test] + fn an_envelope_goes_only_to_observers_subscribed_to_its_kind_in_registration_order() { + let (session, started) = start(vec![ + subscription("terminal", &[EventKind::CallFailed], false), + subscription("interceptor", &[], true), + subscription( + "both", + &[EventKind::CallStarted, EventKind::CallFailed], + false, + ), + subscription("started", &[EventKind::CallStarted], true), + ]); + + assert_eq!(started.observers, vec![2, 3]); + assert_eq!( + session + .failed(TIMING, FailureOrigin::Call, error()) + .observers, + vec![0, 2] + ); + } + + #[test] + fn each_interceptor_sees_the_request_as_the_earlier_ones_left_it() { + let (mut session, _) = start(vec![ + subscription("first", &[], true), + subscription("observer", &[EventKind::RequestSending], false), + subscription("second", &[], true), + ]); + let interception = session.interception(wire(), context()); + + let first = interception.turn().unwrap(); + assert_eq!( + (first.subscriber, seen_body(&first)), + (0, json!({"input": "original"})) + ); + let interception = interception + .patched(body(json!({"input": "first"}))) + .unwrap(); + + let second = interception.turn().unwrap(); + assert_eq!( + (second.subscriber, seen_body(&second)), + (2, json!({"input": "first"})) + ); + let interception = interception.patched(WirePatch::default()).unwrap(); + assert_eq!(interception.turn(), None); + + let (sending, sent) = session.request_sending(interception); + assert_eq!(sent.body, json!({"input": "first"})); + assert_eq!(sending.observers, vec![1]); + assert_eq!( + serde_json::to_value(&sending.envelope).unwrap()["event"]["body"], + sent.body + ); + } + + #[test] + fn an_invalid_patch_ends_the_interception() { + let (session, _) = start(vec![subscription("interceptor", &[], true)]); + let patch: WirePatch = + serde_json::from_value(json!({"headers": {"set": [["Authorization", "stolen"]]}})) + .unwrap(); + + assert_eq!( + session + .interception(wire(), context()) + .patched(patch) + .unwrap_err(), + PatchError::ProtectedHeader("Authorization".to_string()) + ); + } + + #[test] + fn a_terminal_envelope_says_whether_the_call_streamed() { + let (mut streamed, _) = start(Vec::new()); + streamed.opened(); + let (plain, _) = start(Vec::new()); + + for (session, expected) in [(streamed, true), (plain, false)] { + let Event::CallFailed { streamed, .. } = session + .failed(TIMING, FailureOrigin::Host, error()) + .envelope + .event + else { + panic!("call.failed expected"); + }; + assert_eq!(streamed, expected); + } + } + + #[test] + fn a_subscription_is_plain_data_a_remote_host_can_report() { + let reported = subscription("remote", &[EventKind::CallSucceeded], true); + let wire = serde_json::to_value(&reported).unwrap(); + + assert_eq!( + wire, + json!({"name": "remote", "events": ["call.succeeded"], "observes": true, "intercepts": true}) + ); + assert_eq!( + serde_json::from_value::(wire).unwrap(), + reported + ); + } +} diff --git a/litellm/callbacks_v1/MIGRATION.md b/litellm/callbacks_v1/MIGRATION.md new file mode 100644 index 00000000000..8825b78019b --- /dev/null +++ b/litellm/callbacks_v1/MIGRATION.md @@ -0,0 +1,171 @@ +# Migrating built-in callbacks to the v1 contract + +How a built-in logger moves from `CustomLogger` (`litellm/integrations/`) to the v1 +contract (`litellm/callbacks_v1/__init__.py`), and what the scaffolding in `builtin/` is +for while that happens. The rules for writing one port are in `builtin/AGENTS.md`; this +file is the flow around them. + +## The shape of the migration + +- Ports live in a parallel tree, `builtin/.py`, one frozen class each declaring + `SinkPort` or `InterceptorPort` from `builtin/port.py` as its base. The legacy logger + is never edited, renamed, or taught about its port. +- Everything the migration needs to say is in `builtin/manifest.py`: `Gap`, each port's + registration name, its twin, its status and its gaps. A port carries none of it, so + cutting over deletes that file and leaves the ports untouched. +- A port is inert. Importing it registers nothing and the product never registers it; + only tests do. Writing a port therefore cannot change behaviour for anyone. +- A port goes live by removing its legacy twin. The two are never active on the same + call, so there is no dual delivery, no ownership table, and no per-call skip logic. +- v1 envelopes only exist on native (Rust) routes. A port can only replace its twin once + every route the twin serves emits them. +- Built-in and custom callbacks share one interface, the v1 contract. In the SDK a custom + callback is free-form Python, in process, never sandboxed. On the gateway it runs in an + out-of-process extension host driven by the same `CallSession` (`litellm-rust/crates/ + callbacks-v1`). A built-in port is pure methods over values with every effect in + `builtin/runtime.py`, which also owns the two subscribers (`Sink`, `Patcher`); that is + hygiene for built-ins (secrets, a later Rust port), not the isolation mechanism. + +Porting comes first, and on purpose, because its first product is not the port. It is the +list of things the contract cannot do yet. The contract grows from what real built-ins +needed, not from guesses. + +## The loop + +``` + port a built-in ──► record gaps ──► read the backlog ──► grow the contract + ▲ │ + └──────── delete gaps and their allowances ◄─────────────┘ +``` + +### 1. Port inertly + +Write `payload()` against the golden envelopes. Every time the twin reads something the +envelopes do not carry, do one of two things and write a `Gap` either way: + +- leave the field out of the vendor body, or +- let `builtin/runtime.py` work around it for every port at once (`CallJoin` recovers the + model for the terminal envelope; `Outbox` keeps vendor I/O off the call path). A + workaround never goes inside a port: a port that needs a thread, a socket or state has + found a `capability` gap. + +A `Gap` is handwritten, in `manifest.py` beside the port's entry. It has a `kind` +(`fact`, `event`, `capability`), a `key`, the `legacy` input the twin read, and a `note`. The `key` names what the contract would have +to grow (`cost`, `usage`, `identity`), not what the twin happened to call it, so that two +ports missing the same thing say so in the same word. + +Then write the parity test: one real native call, the twin and the port each posting to +their own recording vendor, `assert_parity` on the two bodies. Every difference needs an +`Allowed(path, reason)`. If the difference exists because of a gap, it cites the gap: +`Allowed("data.cost", "no cost fact on call.succeeded", gap="cost")`. An `Allowed` with no +`gap` is a difference the port means to keep (each side stamps its own clock, the port +omits a key the twin sent as null). + +Add the port to `manifest.py` as `exploring`, then `parity` once the parity test passes. + +### 2. Read the backlog + +```bash +python -m litellm.callbacks_v1.builtin.manifest +``` + +prints the ledger (every gap, per port) and the backlog (every gap key, with the ports +waiting on it, most wanted first). A key several ports wait on is contract work. A key +one port waits on is either contract work or a candidate for "v1 deliberately does not +do this"; decide which before building it. + +### 3. Grow the contract + +The contract is defined in Rust (`litellm-rust/crates/callbacks-v1`, golden envelopes in +`golden/v1/`) and mirrored by the types in `litellm/callbacks_v1/__init__.py`. Add the +fact, event, or capability there, for every native route, and update the golden files. + +### 4. Close the gap + +For each port that listed the key: + +1. Use the new fact in `payload()`, or drop the workaround. +2. Delete the `Gap`. +3. Run the parity tests. They now fail in exactly the places that need attention: + - an `Allowed` still citing the deleted key fails with "cites gap ..., which the port + does not list"; + - an `Allowed` whose difference is gone fails as stale; + - a difference the new fact did not actually fix fails as unexplained once its + `Allowed` is removed. +4. Delete those `Allowed` entries until the test is green again. Do not re-allow a + difference to get there. + +The order of 1 and 2 does not matter; the tests fail until both are done. + +### 5. Cut over + +A port is `ready` when its parity test passes, its manifest gaps are empty, and every route its twin +serves emits v1 envelopes. `test_conventions.py` refuses `ready` with an open gap. A +`ready` port replaces its twin: register the port where the twin was registered, remove +the twin, in one change. + +## What is checked and what is trusted + +| Claim | Checked by | +|---|---| +| Every difference from the twin is explained | `assert_parity`: an uncovered difference fails | +| The allow-list does not outlive the differences | `assert_parity`: a stale `Allowed` fails | +| The allow-list does not outlive the gaps | `assert_parity`: an `Allowed` citing an unlisted key fails | +| A port is not `ready` with open gaps | `test_conventions.py` | +| The gap list is complete | nobody | + +The last row is the honest one. The gap list is only as complete as its author. For a +sink, the parity diff is the real source of truth and the gaps are its readable summary: anything the +author forgot still shows up as an unexplained difference. Three kinds of gap never show +up in a diff, and the handwritten entry is their only record: + +- `capability` and `event` gaps (observers run inline; an interceptor cannot see + `call_type`), because they are about how the port runs, not what it sends; +- gaps the port works around (`terminal_model`), because the workaround makes the bodies + equal; +- gaps the parity call does not exercise (`request_user`, when the test sends no `user`). + +Closing one of these is a manual edit with no failing test to prompt it. When the contract +change lands, grep the key. + +## Open decisions + +Porting does not settle these, and no port may settle one by accident. Each is kept out of +the ports so that it can be decided once, in the host. + +- **Where `Config` comes from.** A port takes a finished `Config`. Whether the host + resolves it from the twin's environment variables, from config.yaml, or through the + Rust settings precedence is undecided; `from_env` on a port only records the names. +- **The gateway's extension host.** Custom Python callbacks run out of process there: + the worker's confinement profile, the interceptor timeout and failure policy, and + whether built-ins run in a trusted worker or move to Rust. The interface is the v1 + Protocol itself over a socket, not the ports' pure-method shape. +- **Who holds secrets.** A `Config` carries the vendor's API key and a `Delivery` carries + it as a header, so a port sees the credential. A sandboxed callback should name a secret + and let the host put it on the wire. `off_path_delivery` is the gap that would carry it. + +## Differences that are never closed + +Some legacy inputs will not come to v1 (`hidden_params` is the likely example). Those are +not gaps; a gap is something the contract intends to grow. When that decision is made for +a key, delete the `Gap`, drop `gap=` from its `Allowed` entries and reword their reasons +as the decision, and move the feature to the port's `Not ported yet:` docstring line or to +the release notes. A port must not sit at `parity` forever behind a gap nobody will close. + +`parity` means every difference is explained. It does not mean the port is equivalent: +`generic_api` is at `parity` while shipping about a fifth of `StandardLoggingPayload`. +Only `ready` means equivalent. + +## What survives the migration + +| Piece | After every twin is gone | +|---|---| +| The ports, `SinkPort`, `InterceptorPort`, `Delivery`, `CallRecord`, `Batching` | stay; this is the product and its interface | +| `runtime.py`: `Sink`, `Outbox`, `Transport` / `HttpxTransport` | stays until the contract delivers off the call path itself (`off_path_delivery`); then the host owns it | +| `CallJoin` | stays only if the terminal envelope never carries model and metadata; it is a workaround for the `terminal_model` gap | +| `manifest.py` entire: `Gap`, every gap list, the names, `ledger()`, `backlog()`, the statuses, `Entry.legacy` | deleted with the last twin | +| The twin checks in `test_conventions.py` | deleted with the last twin | +| `assert_parity`, `Allowed`, `test_parity.py` | deleted; there is no twin left to compare with. The payload tests on golden envelopes remain as the ports' tests | + +At that point `port.py` holds the two protocols and what they are written in, which is the +whole of what a callback author, built-in or custom, needs besides the contract types. diff --git a/litellm/callbacks_v1/__init__.py b/litellm/callbacks_v1/__init__.py new file mode 100644 index 00000000000..99d075e6120 --- /dev/null +++ b/litellm/callbacks_v1/__init__.py @@ -0,0 +1,419 @@ +"""The v1 callback contract as a callback author sees it. + +A registered callback observes a call through envelopes it cannot influence, and may +influence one only by returning a patch that Rust validates. The schema itself is owned +by `litellm-rust/crates/callbacks-v1`; this module is its typed Python face and the +process-wide registry. `litellm.rust_bridge.callbacks_v1` is the marshalling the +native call reaches for, and is not part of this surface. + +Two ways to register. A single function takes a decorator, which derives the name and the +handler slot for you and returns the function unchanged: + + @on_event("call.succeeded", "call.failed") + def record(event: EnvelopeV1) -> None: ... + + @before_send + async def add_header(request: RequestFactsV1) -> WirePatchV1 | None: ... + +An object with several handlers, or with state, declares them by the protocol names and +goes through `register`. + +Every rejectable mistake is rejected at registration, never during a call, and each static +rule here mirrors one `RegistrationError`: the handler protocols mirror `MissingHandler` +and `HandlerKindMismatch`, and `EventName` mirrors `UnknownEvent`. Annotate `events` with +`frozenset[EventName]` on the object form to get that check before import time; the +decorator infers it from its arguments. +""" + +import inspect +import threading +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import Final, Literal, Protocol, TypeAlias, TypeVar + +from typing_extensions import ReadOnly, TypeAliasType, TypedDict + +__all__ = ( + "EVENTS", + "SCHEMA", + "SUPPORTED_SCHEMAS", + "AsyncInterceptor", + "AsyncObserver", + "CallFactsV1", + "CallFailedV1", + "CallStartedV1", + "CallSucceededV1", + "Callback", + "CallbackBase", + "DuplicateName", + "EmptyObserverEvents", + "EnvelopeV1", + "ErrorFactsV1", + "EventName", + "EventV1", + "HandlerKindMismatch", + "HeaderPatchV1", + "Interceptor", + "InterceptorFn", + "JSONValue", + "MissingHandler", + "Observer", + "ObserverFn", + "RegistrationError", + "RequestFactsV1", + "RequestSendingV1", + "ResponseReceivedV1", + "SchemaVersion", + "Subscriber", + "TimingFactsV1", + "UnknownEvent", + "UnsupportedSchema", + "WirePatchV1", + "before_send", + "on_event", + "register", + "snapshot", + "unregister", +) + +JSONScalar: TypeAlias = None | bool | int | float | str +JSONValue = TypeAliasType( # rebind-ok: named recursive alias required for runtime validation + "JSONValue", JSONScalar | Sequence["JSONValue"] | Mapping[str, "JSONValue"] +) + +EventName: TypeAlias = Literal[ + "call.started", + "request.sending", + "response.received", + "call.succeeded", + "call.failed", +] +SchemaVersion: TypeAlias = Literal[1] + +SCHEMA: Final[SchemaVersion] = 1 +SUPPORTED_SCHEMAS: Final[frozenset[SchemaVersion]] = frozenset({SCHEMA}) +EVENTS: Final[frozenset[EventName]] = frozenset( + { + "call.started", + "request.sending", + "response.received", + "call.succeeded", + "call.failed", + } +) + + +class TimingFactsV1(TypedDict): + start_time: ReadOnly[float] + end_time: ReadOnly[float] + duration: ReadOnly[float] + + +class CallFactsV1(TypedDict): + start_time: ReadOnly[float] + asynchronous: ReadOnly[bool] + metadata: ReadOnly[Mapping[str, JSONValue]] + metadata_dropped: ReadOnly[Sequence[str]] + + +class RequestFactsV1(TypedDict): + model: ReadOnly[str] + custom_llm_provider: ReadOnly[str] + optional_params: ReadOnly[JSONValue] + url: ReadOnly[str] + headers: ReadOnly[Sequence[tuple[str, str]]] + body: ReadOnly[JSONValue] + + +ErrorFactsV1 = TypedDict( # rebind-ok: runtime TypedDict type for the reserved class field + "ErrorFactsV1", + { # mutable-ok: functional TypedDict syntax requires a dict for the reserved class field + "class": ReadOnly[str], + "message": ReadOnly[str], + "status_code": ReadOnly[int | None], + }, +) + + +class CallStartedV1(CallFactsV1): + type: ReadOnly[Literal["call.started"]] + + +class RequestSendingV1(RequestFactsV1): + type: ReadOnly[Literal["request.sending"]] + + +class ResponseReceivedV1(TypedDict): + type: ReadOnly[Literal["response.received"]] + body: ReadOnly[str] + + +class CallSucceededV1(TypedDict): + type: ReadOnly[Literal["call.succeeded"]] + timing: ReadOnly[TimingFactsV1] + streamed: ReadOnly[bool] + response: ReadOnly[JSONValue] + response_error: ReadOnly[str | None] + + +class CallFailedV1(TypedDict): + type: ReadOnly[Literal["call.failed"]] + timing: ReadOnly[TimingFactsV1] + streamed: ReadOnly[bool] + origin: ReadOnly[Literal["call", "host"]] + error: ReadOnly[ErrorFactsV1] + + +EventV1: TypeAlias = CallStartedV1 | RequestSendingV1 | ResponseReceivedV1 | CallSucceededV1 | CallFailedV1 + + +class EnvelopeV1(TypedDict): + schema: ReadOnly[Literal[1]] + call_id: ReadOnly[str] + call_type: ReadOnly[str] + seq: ReadOnly[int] + event: ReadOnly[EventV1] + + +class HeaderPatchV1(TypedDict, total=False): + remove: ReadOnly[Sequence[str]] + set: ReadOnly[Sequence[tuple[str, str]]] + + +class WirePatchV1(TypedDict, total=False): + headers: ReadOnly[HeaderPatchV1] + body: ReadOnly[JSONValue] + + +class CallbackBase(Protocol): + """What every callback declares, whatever it handles. + + `schema` is not here: the module you imported from already fixes it. Declaring one + anyway is still honoured, so a callback written against another version is rejected + by `register` rather than silently reinterpreted. + """ + + name: str + events: frozenset[EventName] + + +class Observer(CallbackBase, Protocol): + """Receives envelopes on a sync call; its errors are reported and swallowed. + + Observers are asynchronous to the call: an envelope may arrive after the call has + moved on or returned, in `seq` order within one call and in no order across calls, at + most once. This host happens to run observers inline; do not rely on it. + """ + + def on_event(self, event: EnvelopeV1) -> None: ... + + +class AsyncObserver(CallbackBase, Protocol): + """Receives envelopes on an async call; its errors are reported and swallowed. + Asynchronous to the call in the same sense as `Observer`.""" + + async def async_on_event(self, event: EnvelopeV1) -> None: ... + + +class Interceptor(CallbackBase, Protocol): + """Patches the sync request before it is sent; its failure fails the call.""" + + def before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ... + + +class AsyncInterceptor(CallbackBase, Protocol): + """Patches the async request before it is sent; its failure fails the call.""" + + async def async_before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ... + + +Callback: TypeAlias = Observer | AsyncObserver | Interceptor | AsyncInterceptor + +# The decorators return the function unchanged, so each is generic over the exact function +# it decorated; without that, the decorated name would widen to the union below and stop +# being directly callable at its own signature. +_SyncObserverFn: TypeAlias = Callable[[EnvelopeV1], None] # mutable-ok: Callable's bracketed parameter list is call syntax, not a list literal +_AsyncObserverFn: TypeAlias = Callable[[EnvelopeV1], Awaitable[None]] # mutable-ok: Callable's bracketed parameter list is call syntax, not a list literal +_SyncInterceptorFn: TypeAlias = Callable[[RequestFactsV1], WirePatchV1 | None] # mutable-ok: Callable's bracketed parameter list is call syntax, not a list literal +_AsyncInterceptorFn: TypeAlias = Callable[[RequestFactsV1], Awaitable[WirePatchV1 | None]] # mutable-ok: Callable's bracketed parameter list is call syntax, not a list literal + +ObserverFn: TypeAlias = _SyncObserverFn | _AsyncObserverFn +InterceptorFn: TypeAlias = _SyncInterceptorFn | _AsyncInterceptorFn +ObserverFnT = TypeVar("ObserverFnT", bound=ObserverFn) # rebind-ok: a TypeVar must bind to a bare name to be recognised as one +InterceptorFnT = TypeVar("InterceptorFnT", bound=InterceptorFn) # rebind-ok: a TypeVar must bind to a bare name to be recognised as one + + +class RegistrationError(ValueError): + pass + + +class UnsupportedSchema(RegistrationError): + pass + + +class UnknownEvent(RegistrationError): + pass + + +class MissingHandler(RegistrationError): + pass + + +class EmptyObserverEvents(RegistrationError): + pass + + +class HandlerKindMismatch(RegistrationError): + pass + + +class DuplicateName(RegistrationError): + pass + + +@dataclass(frozen=True, slots=True) +class Subscriber: + name: str + schema: SchemaVersion + events: frozenset[EventName] + on_event: object | None + async_on_event: object | None + before_send: object | None + async_before_send: object | None + + +class _Registry: + def __init__(self) -> None: + self.subscribers: tuple[Subscriber, ...] = () + self.lock: Final = threading.Lock() + + +_REGISTRY: Final = _Registry() + + +def _handler(callback: Callback, name: str) -> object | None: + return getattr(callback, name, None) + + +def _validate_kind(name: str, handler: object | None, asynchronous: bool) -> None: + if handler is None: + return + if not callable(handler): + raise HandlerKindMismatch(f"{name} must be callable") + is_async: Final = inspect.iscoroutinefunction(handler) + if is_async != asynchronous: + expected: Final = "async" if asynchronous else "sync" + raise HandlerKindMismatch(f"{name} must be {expected}") + + +def _validate_events(events: frozenset[EventName], observing: bool) -> None: + unknown: Final = events - EVENTS + if unknown: + raise UnknownEvent(f"unknown callback events: {sorted(unknown)}") + if observing and not events: + raise EmptyObserverEvents("observer callback events cannot be empty") + + +def _insert(subscriber: Subscriber) -> None: + with _REGISTRY.lock: + if any(existing.name == subscriber.name for existing in _REGISTRY.subscribers): + raise DuplicateName(f"duplicate callback name: {subscriber.name}") + _REGISTRY.subscribers = (*_REGISTRY.subscribers, subscriber) + + +def register(callback: Callback) -> None: + """Registers an object whose handler methods are named as the protocols declare them.""" + schema: Final = getattr(callback, "schema", SCHEMA) + if schema not in SUPPORTED_SCHEMAS: + raise UnsupportedSchema(f"unsupported callback schema: {schema}") + events: Final = frozenset(callback.events) + + on_event: Final = _handler(callback, "on_event") + async_on_event: Final = _handler(callback, "async_on_event") + before_send: Final = _handler(callback, "before_send") + async_before_send: Final = _handler(callback, "async_before_send") + handlers: Final = (on_event, async_on_event, before_send, async_before_send) + if all(handler is None for handler in handlers): + raise MissingHandler("callback must define at least one handler") + _validate_events(events, on_event is not None or async_on_event is not None) + + _validate_kind("on_event", on_event, False) + _validate_kind("async_on_event", async_on_event, True) + _validate_kind("before_send", before_send, False) + _validate_kind("async_before_send", async_before_send, True) + + _insert( + Subscriber( + name=callback.name, + schema=schema, + events=events, + on_event=on_event, + async_on_event=async_on_event, + before_send=before_send, + async_before_send=async_before_send, + ) + ) + + +def _derived_name(function: object) -> str: + module: Final = getattr(function, "__module__", "") or "" + qualified: Final = getattr(function, "__qualname__", None) or repr(function) + return f"{module}.{qualified}" if module else str(qualified) + + +def on_event(*events: EventName, name: str | None = None) -> Callable[[ObserverFnT], ObserverFnT]: + """Registers one function as an observer of `events`; `async def` selects the async slot. + + The function is returned unchanged, so it stays directly callable and testable. Its + registry name defaults to `module.qualname`, which is unique by construction. + """ + + def decorate(function: ObserverFnT) -> ObserverFnT: + subscribed: Final = frozenset(events) + _validate_events(subscribed, True) + asynchronous: Final = inspect.iscoroutinefunction(function) + _insert( + Subscriber( + name=name if name is not None else _derived_name(function), + schema=SCHEMA, + events=subscribed, + on_event=None if asynchronous else function, + async_on_event=function if asynchronous else None, + before_send=None, + async_before_send=None, + ) + ) + return function + + return decorate + + +def before_send(function: InterceptorFnT) -> InterceptorFnT: + """Registers one function as an interceptor; `async def` selects the async slot. + + An interceptor subscribes to no events, and its failure fails the call. + """ + asynchronous: Final = inspect.iscoroutinefunction(function) + _insert( + Subscriber( + name=_derived_name(function), + schema=SCHEMA, + events=frozenset(), + on_event=None, + async_on_event=None, + before_send=None if asynchronous else function, + async_before_send=function if asynchronous else None, + ) + ) + return function + + +def unregister(callback: Callback | Subscriber | str) -> None: + """Drops the callback of this name, given the object, a `Subscriber`, or the name.""" + name: Final = callback if isinstance(callback, str) else callback.name + with _REGISTRY.lock: + _REGISTRY.subscribers = tuple(subscriber for subscriber in _REGISTRY.subscribers if subscriber.name != name) + + +def snapshot() -> tuple[Subscriber, ...]: + return _REGISTRY.subscribers diff --git a/litellm/callbacks_v1/builtin/AGENTS.md b/litellm/callbacks_v1/builtin/AGENTS.md new file mode 100644 index 00000000000..824853bf9fc --- /dev/null +++ b/litellm/callbacks_v1/builtin/AGENTS.md @@ -0,0 +1,44 @@ +- Target invariants, not completion claims +- This package holds built-in callbacks ported from `CustomLogger` to the v1 contract (`litellm/callbacks_v1/__init__.py`), and nothing else + - Porting is exploration: its first product is the gap ledger (`python -m litellm.callbacks_v1.builtin.manifest`), which says what the contract has to grow + - `../MIGRATION.md` is the whole flow from first port to removing the legacy twin; this file is the rules for writing one port + - A port is never registered by the product while its legacy twin exists. Importing anything here registers nothing; only tests call `register` + - A port goes live by removing or unregistering its legacy twin. The two are never active together, so there is no per-call skip logic anywhere + - Never edit the legacy twin to make a port easier +- A port is a frozen class of pure methods; the host does everything else + - Built-in and custom callbacks share one interface, the v1 contract. This rule is hygiene for built-ins, not the sandbox: a built-in holds vendor secrets and is the likeliest to be ported to Rust, and a pure transform is what survives both. Custom Python callbacks stay free-form; on the gateway they are isolated by an out-of-process extension host, not by this rule + - A port opens no socket, starts no thread, keeps no state between calls and reads neither the environment nor the clock. Whatever it needs arrives as a value: contract facts, its `Config`, `now` + - `port.py` is the whole interface: the two protocols a port declares as its base (`SinkPort`, `InterceptorPort`) and what they are written in (`CallRecord`, `Delivery`, `Batching`). Nothing in it is migration scaffolding, and a port imports nothing else from this package + - `runtime.py` is the in-process host: `Sink` and `Patcher`, the two subscribers, with `CallJoin`, `Transport`, the clock and `outbox.py` behind them. They are the only things here that implement the v1 contract, so a port never writes `name`, `events` or a handler. Each takes the registration name, because the name is the registration's and not the port's: one port configured twice is two subscribers. It stands in for what a host should own; when the contract grows off-path delivery and a joined call record, `runtime.py` is deleted and the ports do not change + - `manifest.py` is the migration, all of it: `Gap`, the name a port registers under, its twin, its status and its gaps. At cutover it is deleted outright and no port changes + - `test_conventions.py` holds a port's imports to a pure allow-list and checks that it declares the protocol of its kind. A port importing anything from `runtime.py` is a failure: wiring is the host's, and the tests do it +- Terms + - Legacy twin: the `CustomLogger` subclass (or function of one) a port replaces, named in the port's docstring and in `manifest.py` + - Sink: a port that observes calls and tells a vendor (`SinkPort`, run by `runtime.Sink`). Interceptor: a port that patches the wire request (`InterceptorPort`, run by `runtime.Patcher`) + - Gap: one legacy input the contract does not supply: a missing `fact`, a missing `event`, or a missing `capability` + - Its `key` names what the contract would have to grow (`cost`, `usage`, `identity`), not what the twin read. Reuse an existing key when another port already waits on the same thing; the backlog counts ports per key + - Parity: the port and its twin handled the same call and every difference in what they sent is `Allowed` with a reason +- What can be ported: a twin that only overrides the success/failure/pre/post log hooks is a sink; one that only rewrites the outgoing request is an interceptor. A twin overriding any other hook (guardrail, routing, prompt management, agentic loop, stream rewriting, audit log) has no v1 shape and stays out; `test_conventions.py` enforces this from the twin's own class +- Every port module has the same parts, in this order + - Docstring: one line of what it is, `Legacy twin: `, and `Not ported yet:` for legacy features left out on purpose + - `Config`: a frozen dataclass of plain data. Where its values come from (environment, config.yaml, the UI) is the host's business and an open decision in `../MIGRATION.md`; a port never reads the environment. A `from_env(env)` over a mapping is allowed as the record of the twin's variable names, not as the way a port gets configured + - The vendor's types, as `TypedDict`s, and any module-private pure helpers + - The port class: a frozen dataclass whose only field is `config`, declaring `SinkPort[]` or `InterceptorPort` as its base. Both kinds have exactly the same parts + - `payload(...)`: the port. For a sink, one `CallRecord` and the host's `now` to the vendor's body; for an interceptor, one `RequestFactsV1` to the `WirePatchV1` it needs. `None` means nothing to send or nothing to patch, which is also how a port filters + - A sink adds `deliveries(batch)`, a pure function from a batch of bodies to the `Delivery`s that carry them, headers and framing included, and `batching`, how large a batch the host should collect + - Nothing else. Not its name, not its gaps (`manifest.py` holds both), and nothing of the contract: no `events`, no `on_event`, no `before_send`, no subscriber class, no `build`. A port is registered by nobody; the tests name it, wrap it in `runtime.Sink` or `runtime.Patcher` and register that +- A sink's handler never does I/O: v1 observers run inline in the call, where legacy callbacks ran on an executor or the logging worker + - `runtime.Sink` joins envelopes with `CallJoin`, calls `payload` with the clock's `now`, and `put`s the result on its `Outbox`; the outbox thread calls `deliveries` and sends, `batching` at a time + - So a sink has only the sync `on_event`; an async call falls back to it, and it never blocks + - The terminal envelope carries neither model nor metadata, which is why `CallJoin` exists. A cancelled call has no terminal envelope, which is why it is bounded + - Observer errors are reported and swallowed by the contract; a port does not wrap its handler in `try` +- Tests, three layers; a port is not done without the first two + - Payload (`tests/test_litellm/callbacks_v1/builtin/test_.py`): `payload` on the golden envelopes Rust pins, asserting the whole vendor body. No Rust needed + - Parity (`tests/test_litellm_rust/builtin/test_parity.py`): one native call with the twin in `callbacks=[...]` and the port registered by the test, each posting to its own recording vendor; `assert_parity` on the two raw bodies. Cover success and failure when the twin logs both + - An interceptor whose twin never runs on a native route is compared function to function on the same bodies instead, plus one native call proving the patch reaches the provider + - Conventions (`test_conventions.py`): runs for every manifest entry with no per-port code, the pure-import rule and the declared-protocol check included; add the port to `EXAMPLES` and `SUBSCRIBERS` in `support.py` +- `Allowed` differences + - Every entry has a reason. One that exists because of a gap cites it as `gap=""`, and `assert_parity` fails if the port's `GAPS` has no such key. One with no `gap` is a difference the port keeps on purpose + - Patterns are as narrow as the difference. A stale entry fails the test, so closing a gap in the contract forces the list to shrink, and deleting a `Gap` fails every entry still citing it + - Never allow a difference to get a test green; either port the behaviour or record the gap +- Gaps and status live in `manifest.py`, next to the entry: `exploring` (payload tests pass), `parity` (parity test passes), `ready` (parity, no open gap, and every route the twin serves emits v1 envelopes). Only `ready` ports may replace their twin diff --git a/litellm/callbacks_v1/builtin/__init__.py b/litellm/callbacks_v1/builtin/__init__.py new file mode 100644 index 00000000000..948958605db --- /dev/null +++ b/litellm/callbacks_v1/builtin/__init__.py @@ -0,0 +1,6 @@ +"""Built-in callbacks ported to the v1 contract. + +Nothing here is registered by importing it, and nothing in the product imports it yet: +a port is exercised by its tests only, until its legacy twin is removed. `AGENTS.md` +beside this file is the porting convention, `manifest.py` is the list of ports. +""" diff --git a/litellm/callbacks_v1/builtin/anthropic_cache_control.py b/litellm/callbacks_v1/builtin/anthropic_cache_control.py new file mode 100644 index 00000000000..602908b275c --- /dev/null +++ b/litellm/callbacks_v1/builtin/anthropic_cache_control.py @@ -0,0 +1,149 @@ +"""Anthropic prompt-cache breakpoints on the v1 contract: a `before_send` body patch. + +Legacy twin: `AnthropicCacheControlHook.apply_to_anthropic_messages_request` in +`litellm.integrations.anthropic_cache_control_hook`, the Messages-route half of that hook. + +Not ported yet: the OpenAI `prompt_cache_breakpoint` dialect, `tool_config` points, the +default-injection seeding, and the chat-completions path (that route is not native). +""" + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from litellm.callbacks_v1 import JSONValue, RequestFactsV1, WirePatchV1 +from litellm.callbacks_v1.builtin.port import InterceptorPort + +MAX_CACHE_CONTROL_BLOCKS: Final = 4 + +Role: TypeAlias = Literal["user", "system", "assistant"] +Block: TypeAlias = Mapping[str, JSONValue] + + +@dataclass(frozen=True, slots=True) +class InjectionPoint: + """Where to put a breakpoint: the system prompt, every message of `role`, or the message at `index`.""" + + role: Role | None = None + index: int | None = None + ttl: Literal["5m", "1h"] | None = None + + @property + def control(self) -> Block: + if self.ttl is None: + return MappingProxyType({"type": "ephemeral"}) + return MappingProxyType({"type": "ephemeral", "ttl": self.ttl}) + + +@dataclass(frozen=True, slots=True) +class Config: + points: tuple[InjectionPoint, ...] + providers: frozenset[str] = frozenset({"anthropic"}) + + +def _marked(block: JSONValue) -> bool: + return isinstance(block, Mapping) and block.get("cache_control") is not None + + +def _blocks(content: JSONValue) -> Sequence[JSONValue]: + return content if isinstance(content, Sequence) and not isinstance(content, str) else () + + +def _breakpoints(message: JSONValue) -> int: + if not isinstance(message, Mapping): + return 0 + return int(_marked(message)) + sum(1 for block in _blocks(message.get("content")) if _marked(block)) + + +def _with_last_block_marked(blocks: Sequence[JSONValue], control: Block) -> Sequence[JSONValue] | None: + if not blocks or not isinstance(blocks[-1], Mapping): + return None + return (*blocks[:-1], MappingProxyType({**blocks[-1], "cache_control": control})) + + +def _as_blocks(message: Block) -> Block: + content: Final = message.get("content") + if not isinstance(content, str): + return message + return MappingProxyType({**message, "content": (MappingProxyType({"type": "text", "text": content}),)}) + + +def _targets(point: InjectionPoint, messages: Sequence[Block]) -> tuple[int, ...]: + if point.index is not None: + index: Final = point.index + len(messages) if point.index < 0 else point.index + return (index,) if 0 <= index < len(messages) else () + return tuple(i for i, message in enumerate(messages) if message.get("role") == point.role) + + +def _mark_system(system: JSONValue, control: Block) -> JSONValue: + if isinstance(system, str): + return (MappingProxyType({"type": "text", "text": system, "cache_control": control}),) + blocks: Final = _blocks(system) + if any(_marked(block) for block in blocks): + return system + return _with_last_block_marked(blocks, control) or system + + +def _mark_messages(messages: Sequence[Block], targets: Sequence[tuple[int, Block]], budget: int) -> Sequence[Block]: + if budget <= 0 or not targets: + return messages + (index, control), rest = targets[0], targets[1:] + marked: Final = ( + None + if _breakpoints(messages[index]) + else _with_last_block_marked(_blocks(messages[index].get("content")), control) + ) + if marked is None: + return _mark_messages(messages, rest, budget) + message: Final = MappingProxyType({**messages[index], "content": marked}) + updated: Final = (*messages[:index], message, *messages[index + 1 :]) + return _mark_messages(updated, rest, budget - 1) + + +@dataclass(frozen=True, slots=True) +class AnthropicCacheControl(InterceptorPort): + """A body patch that adds the configured breakpoints to a Messages request.""" + + config: Config + + def payload(self, request: RequestFactsV1) -> WirePatchV1 | None: + """The patch this request needs, or `None` for a provider or body this port leaves alone.""" + if request["custom_llm_provider"] not in self.config.providers: + return None + body: Final = self.marked(request["body"]) + if body is None: + return None + patch: Final[WirePatchV1] = {"body": body} + return patch + + def marked(self, body: JSONValue) -> JSONValue | None: + """The Messages body with the configured breakpoints added, or `None` to leave it alone.""" + if not isinstance(body, Mapping) or not self.config.points: + return None + raw_messages: Final = body.get("messages") + if not isinstance(raw_messages, Sequence) or isinstance(raw_messages, str): + return None + messages: Final = tuple(_as_blocks(message) for message in raw_messages if isinstance(message, Mapping)) + if len(messages) != len(raw_messages): + return None + + system: Final = body.get("system") + system_point: Final = next((point for point in self.config.points if point.role == "system"), None) + used: Final = sum(_breakpoints(message) for message in messages) + sum(1 for b in _blocks(system) if _marked(b)) + new_system: Final = ( + _mark_system(system, system_point.control) + if system_point is not None and system is not None and used < MAX_CACHE_CONTROL_BLOCKS + else system + ) + used_after_system: Final = used + int(new_system != system) + + targets: Final = tuple( + (index, point.control) + for point in self.config.points + if point.role != "system" + for index in _targets(point, messages) + ) + new_messages: Final = _mark_messages(messages, targets, MAX_CACHE_CONTROL_BLOCKS - used_after_system) + replaced: Final = MappingProxyType({"messages": new_messages, "system": new_system}) + return MappingProxyType({key: replaced.get(key, value) for key, value in body.items()}) diff --git a/litellm/callbacks_v1/builtin/generic_api.py b/litellm/callbacks_v1/builtin/generic_api.py new file mode 100644 index 00000000000..bfa071459d8 --- /dev/null +++ b/litellm/callbacks_v1/builtin/generic_api.py @@ -0,0 +1,129 @@ +"""Generic API logger on the v1 contract: batches of call records to one HTTP endpoint. + +Legacy twin: `litellm.integrations.generic_api.generic_api_callback.GenericAPILogger`. +The legacy logger ships the whole `StandardLoggingPayload`; this port ships the fields of +it that v1 facts can fill, under the same names; `manifest.py` lists the rest as gaps. + +Not ported yet: `callback_name` presets from `generic_api_compatible_callbacks.json`, +retries, and the pre-StandardLoggingPayload body behind `litellm.generic_api_use_v1`. +""" + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime +from typing import Final, Literal, TypeAlias + +from typing_extensions import NotRequired, ReadOnly, TypedDict + +from litellm.callbacks_v1 import JSONValue +from litellm.callbacks_v1.builtin.port import Batching, CallRecord, Delivery, SinkPort + +LogFormat: TypeAlias = Literal["json_array", "ndjson", "single"] +ApiEventType: TypeAlias = Literal["llm_api_success", "llm_api_failure"] + + +@dataclass(frozen=True, slots=True) +class Config: + endpoint: str + headers: tuple[tuple[str, str], ...] = (("Content-Type", "application/json"),) + event_types: frozenset[ApiEventType] = frozenset({"llm_api_success", "llm_api_failure"}) + log_format: LogFormat = "json_array" + batch_size: int = 512 + flush_interval: float = 5.0 + + @staticmethod + def from_env(env: Mapping[str, str]) -> "Config": + endpoint: Final = env.get("GENERIC_LOGGER_ENDPOINT") + if endpoint is None: + raise ValueError("endpoint not set for GenericAPILogger, GENERIC_LOGGER_ENDPOINT not found") + pairs: Final = (item.split("=", 1) for item in env.get("GENERIC_LOGGER_HEADERS", "").split(",") if "=" in item) + return Config( + endpoint=endpoint, + headers=(("Content-Type", "application/json"), *((key.strip(), value.strip()) for key, value in pairs)), + ) + + +class ErrorInformation(TypedDict): + error_class: ReadOnly[str] + error_message: ReadOnly[str] + error_code: ReadOnly[str] + + +class Record(TypedDict): + """The `StandardLoggingPayload` fields v1 can fill, under their legacy names.""" + + id: ReadOnly[str] + litellm_call_id: ReadOnly[str] + call_type: ReadOnly[str] + stream: ReadOnly[bool] + status: ReadOnly[Literal["success", "failure"]] + startTime: ReadOnly[float] + endTime: ReadOnly[float] + response_time: ReadOnly[float] + model: ReadOnly[str | None] + custom_llm_provider: ReadOnly[str | None] + model_parameters: ReadOnly[JSONValue] + response: ReadOnly[NotRequired[JSONValue]] + error_str: ReadOnly[NotRequired[str]] + error_information: ReadOnly[NotRequired[ErrorInformation]] + + +@dataclass(frozen=True, slots=True) +class GenericApi(SinkPort[Record]): + """One record per finished call, batched into one endpoint.""" + + config: Config + + @property + def batching(self) -> Batching: + return Batching(self.config.batch_size, self.config.flush_interval) + + def payload(self, call: CallRecord, _now: datetime) -> Record | None: + """The record for a finished call, or `None` when `config.event_types` leaves its outcome out. + A failed call has `error_*` where a successful one has `response`. The time is the call's own, + so the clock the host passes is unused.""" + terminal: Final = call.terminal + wanted: Final = "llm_api_success" if terminal["type"] == "call.succeeded" else "llm_api_failure" + if wanted not in self.config.event_types: + return None + request: Final = call.request + response: Final = terminal["response"] if terminal["type"] == "call.succeeded" else None + response_id: Final = response.get("id") if isinstance(response, Mapping) else None + common: Final[Record] = { + "id": response_id if isinstance(response_id, str) else call.call_id, + "litellm_call_id": call.call_id, + "call_type": call.call_type, + "stream": terminal["streamed"], + "status": "success" if terminal["type"] == "call.succeeded" else "failure", + "startTime": terminal["timing"]["start_time"], + "endTime": terminal["timing"]["end_time"], + "response_time": terminal["timing"]["duration"], + "model": request["model"] if request is not None else None, + "custom_llm_provider": request["custom_llm_provider"] if request is not None else None, + "model_parameters": request["optional_params"] if request is not None else None, + } + if terminal["type"] == "call.succeeded": + succeeded: Final[Record] = {**common, "response": terminal["response"]} + return succeeded + error: Final = terminal["error"] + failed: Final[Record] = { + **common, + "error_str": error["message"], + "error_information": { + "error_class": error["class"].rsplit(".", 1)[-1], + "error_message": error["message"], + "error_code": str(error["status_code"]) if error["status_code"] is not None else "", + }, + } + return failed + + def deliveries(self, batch: tuple[Record, ...]) -> tuple[Delivery, ...]: + if self.config.log_format == "single": + return tuple(Delivery(self.config.endpoint, self.config.headers, json.dumps(r).encode()) for r in batch) + body: Final = ( + json.dumps(batch) + if self.config.log_format == "json_array" + else "\n".join(json.dumps(record) for record in batch) + ) + return (Delivery(self.config.endpoint, self.config.headers, body.encode()),) diff --git a/litellm/callbacks_v1/builtin/manifest.py b/litellm/callbacks_v1/builtin/manifest.py new file mode 100644 index 00000000000..c951ee33f8a --- /dev/null +++ b/litellm/callbacks_v1/builtin/manifest.py @@ -0,0 +1,218 @@ +"""Every built-in that has a v1 port, its legacy twin, how far the port has come, and the +gaps it is waiting on. + +This whole module is migration scaffolding, and it is all of it: `port.py` and the ports +themselves carry none. Read by tests only. Nothing here registers a callback or decides +what runs in a call -- a port goes live by removing its legacy twin, not by a switch in +this table -- and when the last twin is gone this file is deleted outright. + +The registration name lives here rather than on the port because it is the host's: a port +is a value, and the same one configured twice is two subscribers under two names. + +`python -m litellm.callbacks_v1.builtin.manifest` prints the gap ledger and the backlog. +""" + +from dataclasses import dataclass +from types import ModuleType +from typing import Final, Literal, TypeAlias + +from litellm.callbacks_v1.builtin import anthropic_cache_control, generic_api, openmeter + +Kind: TypeAlias = Literal["sink", "interceptor"] +GapKind: TypeAlias = Literal["fact", "event", "capability"] + +# exploring: the port exists and its payload tests pass. +# parity: its differential test against the legacy twin passes, every difference explained. +# ready: parity, no open gap, and every route the legacy twin serves emits v1 envelopes. +Status: TypeAlias = Literal["exploring", "parity", "ready"] + + +@dataclass(frozen=True, slots=True) +class Gap: + """One legacy input the v1 contract does not supply. + + `fact` is a value missing from an envelope, `event` is a moment no envelope marks, + and `capability` is something the contract does not let a callback do at all. + + `key` names what the contract would have to grow, not what the legacy logger read, so + two ports missing the same thing share a key and the ledger can count them. It is also + what an `Allowed` parity difference cites, which is how closing a gap finds its tests. + """ + + kind: GapKind + key: str + legacy: str + note: str + + +@dataclass(frozen=True, slots=True) +class Entry: + """One port: where it lives, what it registers as, what it replaces, and where it has got to.""" + + module: ModuleType + name: str + kind: Kind + legacy: str + status: Status + gaps: tuple[Gap, ...] + + +OPENMETER_GAPS: Final = ( + Gap("fact", "cost", "kwargs['response_cost']", "no cost fact on call.succeeded; `data.cost` is always null"), + Gap( + "fact", + "usage", + "response_obj['usage'] of a ModelResponse", + "token counts are only reachable by parsing `response` per provider shape; needs a usage fact", + ), + Gap( + "fact", + "request_user", + "kwargs['user']", + "the request's `user` is not projected; subject comes from metadata only", + ), + Gap( + "fact", + "litellm_metadata", + "litellm_params['metadata'] fed by `litellm_metadata`", + "call.started projects only the `metadata` keyword; the proxy's identity on Messages-style routes is invisible", + ), + Gap( + "fact", + "terminal_model", + "kwargs['model']", + "the terminal envelope has no model, so the port joins request.sending by call_id", + ), + Gap( + "capability", + "off_path_delivery", + "Logging runs success callbacks on an executor / the logging worker", + "v1 observers run inline in the call; the port needs its own outbox to stay off the call path", + ), +) + +ANTHROPIC_CACHE_CONTROL_GAPS: Final = ( + Gap( + "fact", + "request_params", + "kwargs['cache_control_injection_points']", + "per-request injection points are not in RequestFacts; the port only knows the points it was built with", + ), + Gap( + "fact", + "request_call_type", + "call_type", + "RequestFacts has no call_type, so the port recognises a Messages request by the body's shape", + ), + Gap( + "capability", + "route_neutral_rewrite", + "get_chat_completion_prompt rewrites the route-neutral messages before transformation", + "before_send sees only the provider's wire body, so every dialect needs its own body walker", + ), +) + +# What the legacy StandardLoggingPayload has and v1 facts cannot fill, by what the +# contract would have to grow to fill it. +_MISSING_FACTS: Final = ( + ("cost", "response_cost, cost_breakdown, saved_cache_cost, autorouter_savings", "no cost facts"), + ("usage", "total_tokens, prompt_tokens, completion_tokens", "no usage fact; only the raw provider response"), + ("cache", "cache_hit, cache_key", "no cache fact"), + ( + "routing", + "model_id, model_group, api_base, model_map_information", + "no routing fact; `url` is the wire URL, not api_base", + ), + ("identity", "metadata.user_api_key_*, end_user, requester_ip_address, user_agent", "no identity fact"), + ("tags", "request_tags, request_model_access_groups", "no tags fact"), + ("correlation", "trace_id, session_id", "no correlation beyond call_id"), + ("first_token_time", "completionStartTime", "no first-token time in `timing`"), + ("no_equivalent", "hidden_params, guardrail_information, standard_built_in_tools_params", "no equivalent"), + ("prompt", "messages", "only the provider-shaped wire body; no route-neutral prompt projection"), + ( + "normalised_response", + "response", + "legacy logs the OpenAI-normalised ModelResponse; v1 carries the route's own public response", + ), + ( + "error_detail", + "error_information.traceback, llm_provider, error_provider_request_id", + "ErrorFacts is class/message/status only", + ), +) + +GENERIC_API_GAPS: Final = ( + *(Gap("fact", key, f"standard_logging_object: {fields}", note) for key, fields, note in _MISSING_FACTS), + Gap( + "fact", + "terminal_model", + "standard_logging_object: model, custom_llm_provider, model_parameters", + "the terminal envelope has none of them, so the port joins request.sending by call_id", + ), + Gap( + "capability", + "off_path_delivery", + "CustomBatchLogger.periodic_flush on the proxy's event loop", + "v1 observers run inline in the call; batching and I/O live on the port's outbox thread", + ), +) + +PORTS: Final = ( + Entry( + openmeter, + "openmeter", + "sink", + "litellm.integrations.openmeter:OpenMeterLogger", + "parity", + OPENMETER_GAPS, + ), + Entry( + generic_api, + "generic_api", + "sink", + "litellm.integrations.generic_api.generic_api_callback:GenericAPILogger", + "parity", + GENERIC_API_GAPS, + ), + Entry( + anthropic_cache_control, + "anthropic_cache_control", + "interceptor", + "litellm.integrations.anthropic_cache_control_hook:AnthropicCacheControlHook", + "parity", + ANTHROPIC_CACHE_CONTROL_GAPS, + ), +) + + +def entry(name: str) -> Entry: + """The manifest entry a port registers under; how a test reaches its gaps.""" + (found,) = (candidate for candidate in PORTS if candidate.name == name) + return found + + +def ledger() -> str: + """Every open gap, grouped by port: what the contract has to grow before ports can be `ready`.""" + return "\n".join( + line + for port in PORTS + for line in ( + f"{port.name} [{port.kind}, {port.status}] <- {port.legacy}", + *(f" {gap.kind:<10} {gap.key}: {gap.legacy}\n {gap.note}" for gap in port.gaps), + ) + ) + + +def backlog() -> str: + """Every gap key with the ports that are waiting on it, most wanted first: the order the contract should grow in.""" + keys: Final = frozenset(gap.key for port in PORTS for gap in port.gaps) + waiting: Final = tuple( + (key, tuple(port.name for port in PORTS if any(gap.key == key for gap in port.gaps))) for key in sorted(keys) + ) + return "\n".join( + f" {key:<22} {', '.join(names)}" for key, names in sorted(waiting, key=lambda item: -len(item[1])) + ) + + +if __name__ == "__main__": + print(f"{ledger()}\n\nbacklog, by how many ports wait on it\n{backlog()}") # noqa: T201 # the ledger is this module's command-line output diff --git a/litellm/callbacks_v1/builtin/openmeter.py b/litellm/callbacks_v1/builtin/openmeter.py new file mode 100644 index 00000000000..bde73deacaf --- /dev/null +++ b/litellm/callbacks_v1/builtin/openmeter.py @@ -0,0 +1,125 @@ +"""OpenMeter on the v1 contract: one CloudEvent per successful call. + +Legacy twin: `litellm.integrations.openmeter.OpenMeterLogger`. +""" + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime +from typing import Final + +from typing_extensions import NotRequired, ReadOnly, TypedDict + +from litellm.callbacks_v1 import JSONValue +from litellm.callbacks_v1.builtin.port import Batching, CallRecord, Delivery, SinkPort + +_USAGE_KEYS: Final = ("prompt_tokens", "completion_tokens", "total_tokens") + + +@dataclass(frozen=True, slots=True) +class Config: + api_key: str + endpoint: str = "https://openmeter.cloud" + event_type: str = "litellm_tokens" + + @staticmethod + def from_env(env: Mapping[str, str]) -> "Config": + api_key: Final = env.get("OPENMETER_API_KEY") + if api_key is None: + raise ValueError("Missing keys=['OPENMETER_API_KEY'] in environment.") + return Config( + api_key=api_key, + endpoint=env.get("OPENMETER_API_ENDPOINT", "https://openmeter.cloud"), + event_type=env.get("OPENMETER_EVENT_TYPE", "litellm_tokens"), + ) + + @property + def url(self) -> str: + return f"{self.endpoint.rstrip('/')}/api/v1/events" + + +class Usage(TypedDict): + prompt_tokens: ReadOnly[JSONValue] + completion_tokens: ReadOnly[JSONValue] + total_tokens: ReadOnly[JSONValue] + + +class MeterData(TypedDict): + model: ReadOnly[str | None] + cost: ReadOnly[float | None] + prompt_tokens: ReadOnly[NotRequired[JSONValue]] + completion_tokens: ReadOnly[NotRequired[JSONValue]] + total_tokens: ReadOnly[NotRequired[JSONValue]] + + +class CloudEvent(TypedDict): + specversion: ReadOnly[str] + type: ReadOnly[str] + id: ReadOnly[str] + time: ReadOnly[str] + subject: ReadOnly[str] + source: ReadOnly[str] + data: ReadOnly[MeterData] + + +def _usage(response: JSONValue) -> Usage | None: + """Token counts when the response carries them in the OpenAI shape, as the legacy logger reads them.""" + usage: Final = response.get("usage") if isinstance(response, Mapping) else None + if not isinstance(usage, Mapping) or not any(key in usage for key in _USAGE_KEYS): + return None + counted: Final[Usage] = { + "prompt_tokens": usage.get("prompt_tokens", 0), + "completion_tokens": usage.get("completion_tokens", 0), + "total_tokens": usage.get("total_tokens"), + } + return counted + + +def _metered(model: str | None, usage: Usage) -> MeterData: + data: Final[MeterData] = {"model": model, "cost": None, **usage} + return data + + +@dataclass(frozen=True, slots=True) +class OpenMeter(SinkPort[CloudEvent]): + """One CloudEvent per successful call, one request per event.""" + + config: Config + + @property + def batching(self) -> Batching: + """One event per request, as the legacy logger posts them.""" + return Batching() + + def payload(self, call: CallRecord, now: datetime) -> CloudEvent | None: + """The CloudEvent for a finished call, or `None` when OpenMeter has nothing to meter.""" + if call.terminal["type"] != "call.succeeded": + return None + subject: Final = call.started["metadata"].get("user_api_key_user_id") if call.started is not None else None + if subject is None: + raise ValueError("OpenMeter: user is required") + response: Final = call.terminal["response"] + response_id: Final = response.get("id") if isinstance(response, Mapping) else None + model: Final = call.request["model"] if call.request is not None else None + usage: Final = _usage(response) + unmetered: Final[MeterData] = {"model": model, "cost": None} + data: Final = unmetered if usage is None else _metered(model, usage) + event: Final[CloudEvent] = { + "specversion": "1.0", + "type": self.config.event_type, + "id": response_id if isinstance(response_id, str) else call.call_id, + "time": now.isoformat(), + "subject": str(subject), + "source": "litellm-proxy", + "data": data, + } + return event + + def deliveries(self, batch: tuple[CloudEvent, ...]) -> tuple[Delivery, ...]: + """One request per CloudEvent, as the legacy logger posts them.""" + headers: Final = ( + ("Content-Type", "application/cloudevents+json"), + ("Authorization", f"Bearer {self.config.api_key}"), + ) + return tuple(Delivery(self.config.url, headers, json.dumps(event).encode()) for event in batch) diff --git a/litellm/callbacks_v1/builtin/outbox.py b/litellm/callbacks_v1/builtin/outbox.py new file mode 100644 index 00000000000..253823357f9 --- /dev/null +++ b/litellm/callbacks_v1/builtin/outbox.py @@ -0,0 +1,95 @@ +"""Where a sink's handler leaves its work, so that the handler never does I/O. + +The v1 contract runs an observer inline in the call it observes. A handler that posted to +a vendor would put that vendor's latency into every call, so a sink's handler only builds +a value and `put`s it here; one daemon thread per outbox delivers it, alone or in batches. +""" + +import atexit +import os +import queue +import threading +import time +from collections.abc import Callable +from typing import Final, Generic, TypeVar + +from litellm._logging import verbose_logger + +T = TypeVar("T") # rebind-ok: a TypeVar must bind to a bare name to be recognised as one + + +class Outbox(Generic[T]): + """Delivers items off the call path, `batch_size` at a time or every `flush_interval`. + + `deliver` gets a non-empty batch and may raise: the failure is logged and the batch is + dropped, as the legacy batch loggers do. A full outbox drops the new item rather than + block the call that produced it. + """ + + def __init__( + self, + name: str, + deliver: Callable[[tuple[T, ...]], None], + batch_size: int = 1, + flush_interval: float = 5.0, + capacity: int = 50_000, + ) -> None: + self._name: Final = name + self._deliver: Final = deliver + self._batch_size: Final = max(1, batch_size) + self._flush_interval: Final = flush_interval + self._items: Final[queue.Queue[T]] = queue.Queue(maxsize=capacity) + self._lock: Final = threading.Lock() + self._flush_now: Final = threading.Event() + self._worker_pid: int | None = None + atexit.register(self.flush) + + def put(self, item: T) -> None: + self._ensure_worker() + try: + self._items.put_nowait(item) + except queue.Full: + verbose_logger.warning("callback %s outbox is full; dropping one item", self._name) + + def flush(self, timeout: float = 5.0) -> bool: + """Delivers everything already put; returns whether it drained within `timeout`.""" + deadline: Final = time.monotonic() + timeout + self._flush_now.set() + while self._items.unfinished_tasks and time.monotonic() < deadline: + time.sleep(0.005) + return not self._items.unfinished_tasks + + def _ensure_worker(self) -> None: + # A forked child inherits the queue but not the thread, so the worker is per process. + with self._lock: + if self._worker_pid == os.getpid(): + return + self._worker_pid = os.getpid() + threading.Thread(target=self._run, name=f"callback-outbox-{self._name}", daemon=True).start() + + def _next_batch(self) -> tuple[T, ...]: + first: Final = self._items.get() + deadline: Final = time.monotonic() + self._flush_interval + batch: Final[list[T]] = [first] # mutable-ok: a batch is gathered from a blocking queue one item at a time + while len(batch) < self._batch_size: + remaining = deadline - time.monotonic() # rebind-ok: the wait shrinks on every pass of the gather loop + if remaining <= 0 or (self._flush_now.is_set() and self._items.empty()): + break + try: + batch.append(self._items.get(timeout=min(remaining, 0.01))) + except queue.Empty: + continue + return tuple(batch) + + def _run(self) -> None: + while True: + batch = self._next_batch() # rebind-ok: the worker loop takes a new batch on every pass + try: + self._deliver(batch) + except Exception: # noqa: BLE001 # a vendor failure must never end the delivery thread + verbose_logger.exception("callback %s failed to deliver %d item(s)", self._name, len(batch)) + finally: + for _ in batch: + self._items.task_done() + if self._items.empty(): + self._flush_now.clear() diff --git a/litellm/callbacks_v1/builtin/port.py b/litellm/callbacks_v1/builtin/port.py new file mode 100644 index 00000000000..72b4883629e --- /dev/null +++ b/litellm/callbacks_v1/builtin/port.py @@ -0,0 +1,90 @@ +"""The shape every port has, and nothing that can act. + +A port is a frozen class that holds its `Config` and names one of the protocols below as +its base: pure methods from contract values to what the port produces. Declaring the base +is what holds it to the shape, at the class rather than at some call site far away; a +method that drifts fails there, and a missing one fails wherever the port is built. It opens no socket, +starts no thread and reads neither the environment nor the clock; the host hands it all of +those as values. That is the shape a sandboxed custom callback is confined to, and +built-ins take it first so both share one interface. + +A port has no identity of its own. It is not named, does not know which events reach it +and implements nothing of the v1 contract: `runtime.py` names it and wraps it into a +subscriber, so the same port configured twice is two registrations rather than a clash. +Nothing in this file is migration scaffolding; `manifest.py` holds all of that. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Protocol, TypeVar + +from litellm.callbacks_v1 import ( + CallFailedV1, + CallStartedV1, + CallSucceededV1, + RequestFactsV1, + RequestSendingV1, + WirePatchV1, +) + +R = TypeVar("R") # rebind-ok: a TypeVar must bind to a bare name to be recognised as one + + +@dataclass(frozen=True, slots=True) +class Delivery: + """One request to a vendor, fully decided; sending it is the only thing left.""" + + url: str + headers: tuple[tuple[str, str], ...] + body: bytes + + +@dataclass(frozen=True, slots=True) +class CallRecord: + """Every envelope of one finished call a sink needs, joined by `call_id`. + + `request` is absent when the call failed before anything was sent. + """ + + call_id: str + call_type: str + started: CallStartedV1 | None + request: RequestSendingV1 | None + terminal: CallSucceededV1 | CallFailedV1 + + +@dataclass(frozen=True, slots=True) +class Batching: + """How many records the host carries to the vendor at once, and how long it waits.""" + + size: int = 1 + interval: float = 5.0 + + +class SinkPort(Protocol[R]): + """A port that observes finished calls and tells a vendor. + + `payload` is the port: one finished call and the current time to the vendor's record, + or `None` for nothing to send, which is also how a port filters. `deliveries` frames a + batch of those records as requests, headers and framing included, and `batching` says + how large a batch the host should collect. The two are separate because the handler + runs inline in the call and the delivery does not. + """ + + def payload(self, call: CallRecord, now: datetime, /) -> R | None: ... + + def deliveries(self, batch: tuple[R, ...], /) -> tuple[Delivery, ...]: ... + + @property + def batching(self) -> Batching: ... + + +class InterceptorPort(Protocol): + """A port that patches the wire request before it is sent; its failure fails the call. + + `payload` is the patch the request needs, or `None` to leave it alone. It decides + whether the request is one this port acts on at all: the host applies what it returns + and filters nothing. + """ + + def payload(self, request: RequestFactsV1, /) -> WirePatchV1 | None: ... diff --git a/litellm/callbacks_v1/builtin/runtime.py b/litellm/callbacks_v1/builtin/runtime.py new file mode 100644 index 00000000000..cd3e2856139 --- /dev/null +++ b/litellm/callbacks_v1/builtin/runtime.py @@ -0,0 +1,156 @@ +"""The in-process host of a port: everything a port is not allowed to do itself. + +Joining a call's envelopes, reading the clock, batching off the call path and talking to +the vendor all live here, once, so a port stays pure methods over values (`port.py`). +`Sink` and `Patcher` are the two subscribers: each names a port of its kind and wraps it, +and they are the only things in this package that implement the v1 contract, so a port +never writes `name`, `events` or a handler. The name is the caller's because it is the +registration's, not the port's: one port configured twice is two subscribers. This is not a sandbox and isolates nothing: it is plain +in-process Python. It is a stand-in for what a host should own (join, queue, batch, +deliver). When the contract grows off-path delivery and a joined call record, this module +is deleted and the ports do not change. Custom callbacks are isolated by the gateway's +extension host, in Rust. +""" + +import threading +from collections.abc import Callable +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Final, Generic, Protocol, TypeVar + +from litellm.callbacks_v1 import ( + CallStartedV1, + EnvelopeV1, + EventName, + Interceptor, + Observer, + RequestFactsV1, + RequestSendingV1, + WirePatchV1, +) +from litellm.callbacks_v1.builtin.outbox import Outbox +from litellm.callbacks_v1.builtin.port import CallRecord, Delivery, InterceptorPort, SinkPort + +R = TypeVar("R") # rebind-ok: a TypeVar must bind to a bare name to be recognised as one + + +class Transport(Protocol): + def send(self, delivery: Delivery) -> None: ... + + +class HttpxTransport: + """Sends through the same client the legacy loggers use, so proxy TLS settings apply.""" + + def __init__(self) -> None: + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + self._client: Final = HTTPHandler() + + def send(self, delivery: Delivery) -> None: + headers: Final = dict(delivery.headers) # mutable-ok: the httpx client takes its headers as a dict + self._client.post(url=delivery.url, headers=headers, content=delivery.body) # pyright: ignore[reportUnknownMemberType] # the legacy HTTP handler is only partly typed + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +@dataclass(frozen=True, slots=True) +class _OpenCall: + started: CallStartedV1 | None = None + request: RequestSendingV1 | None = None + + +JOINED_EVENTS: Final[frozenset[EventName]] = frozenset( + {"call.started", "request.sending", "call.succeeded", "call.failed"} +) + + +class CallJoin: + """Joins a call's envelopes into one `CallRecord` at its terminal envelope. + + The terminal envelope carries neither the model nor the metadata, so a sink has to + remember the earlier ones. A cancelled call never gets a terminal envelope, which is + why the open calls are bounded: the oldest is forgotten rather than leaked. + """ + + def __init__(self, capacity: int = 10_000) -> None: + self._capacity: Final = capacity + self._lock: Final = threading.Lock() + self._open: Final[dict[str, _OpenCall]] = {} # mutable-ok: per-call join state is an evolving keyed store + + def accept(self, envelope: EnvelopeV1) -> CallRecord | None: + call_id: Final = envelope["call_id"] + event: Final = envelope["event"] + with self._lock: + if event["type"] == "call.succeeded" or event["type"] == "call.failed": + seen: Final = self._open.pop(call_id, _OpenCall()) + return CallRecord(call_id, envelope["call_type"], seen.started, seen.request, event) + current: Final = self._open.get(call_id, _OpenCall()) + if event["type"] == "call.started": + self._open[call_id] = _OpenCall(event, current.request) + elif event["type"] == "request.sending": + self._open[call_id] = _OpenCall(current.started, event) + while len(self._open) > self._capacity: + del self._open[next(iter(self._open))] + return None + + def open_calls(self) -> int: + with self._lock: + return len(self._open) + + +class Sink(Observer, Generic[R]): + """The subscriber of every `SinkPort`: the port supplies pure methods, this runs them. + + The handler only joins and enqueues, so the sync one serves async calls too and never + blocks; the outbox thread calls `deliveries` and sends. + """ + + def __init__( + self, + name: str, + port: SinkPort[R], + transport: Transport, + clock: Callable[[], datetime] = utc_now, + ) -> None: + self.name = name + self.events = JOINED_EVENTS + self._port: Final = port + self._transport: Final = transport + self._clock: Final = clock + self._calls: Final = CallJoin() + self.outbox: Final[Outbox[R]] = Outbox( + name, self._deliver, batch_size=port.batching.size, flush_interval=port.batching.interval + ) + + def _deliver(self, batch: tuple[R, ...]) -> None: + for delivery in self._port.deliveries(batch): + self._transport.send(delivery) + + def on_event(self, event: EnvelopeV1) -> None: + call: Final = self._calls.accept(event) + if call is None: + return + record: Final = self._port.payload(call, self._clock()) + if record is not None: + self.outbox.put(record) + + def open_calls(self) -> int: + return self._calls.open_calls() + + +class Patcher(Interceptor): + """The subscriber of every `InterceptorPort`: the contract's `before_send` over `payload`. + + An interceptor runs on the call path by definition, so there is nothing to join, batch + or defer here; this exists so that an interceptor port declares as little as a sink does. + """ + + def __init__(self, name: str, port: InterceptorPort) -> None: + self.name = name + self.events: frozenset[EventName] = frozenset() + self._port: Final = port + + def before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: + return self._port.payload(request) diff --git a/litellm/callbacks_v1/host.py b/litellm/callbacks_v1/host.py new file mode 100644 index 00000000000..8eddf4b60d6 --- /dev/null +++ b/litellm/callbacks_v1/host.py @@ -0,0 +1,486 @@ +"""The extension host: a process that runs v1 callbacks for a gateway that does not. + + python -m litellm.callbacks_v1.host --fd 3 --load my_pkg.audit --load my_pkg.cb:callback + +The gateway never imports a callback. It starts (or attaches to) this worker, which imports +the callbacks exactly as an SDK process would, reports what they subscribed to, and then +only answers: the gateway sends envelopes and interception requests over one socket, the +worker runs the handlers and sends back patches. The worker cannot call into the gateway; +there is no frame for it. A callback file is the same file in the SDK and here. + +This module isolates nothing by itself. What the worker can reach (environment, files, +network) is whatever the process it runs in was given; confinement is the spawner's job. + +Frames are a 4-byte big-endian length and that many bytes of JSON, one object with a +`type`. `litellm-rust/crates/callbacks-v1-exthost/golden` pins every frame for both sides. + + worker -> gateway hello {schema, subscriptions: [{name, events, observes, intercepts}]} + gateway -> worker event {subscriber, envelope} no reply + gateway -> worker intercept {id, subscriber, request} + worker -> gateway patch {id, patch} or error {id, error_class, message} + worker -> gateway report {subscriber, event, message} an observer raised + gateway -> worker flush {id} + worker -> gateway flushed {id} every event sent before the flush was handled + +`subscriber` is an index into the hello's `subscriptions`. Envelopes reach one subscriber +in the order they were sent; an interception never waits behind events. + +A handler is scheduled by the kind its author declared, which is the author's decision and +not this module's guess. An `async` handler is a task on the worker's loop, so one that only +waits costs no thread and concurrency is the loop's rather than a pool size; a sync handler +gets a thread, because running it blocks one, and the loop keeps turning while it does. Every +frame leaves through one writer, so no handler and no loop callback blocks on the socket. +""" + +import argparse +import asyncio +import importlib +import json +import queue +import socket +import sys +import threading +from collections.abc import Coroutine, Mapping, Sequence +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass +from typing import Final, Literal, Protocol, TypeAlias + +from typing_extensions import ReadOnly, TypedDict + +from litellm.callbacks_v1 import SCHEMA, JSONValue, SchemaVersion, Subscriber, register, snapshot + +MAX_FRAME_BYTES: Final = 64 * 1024 * 1024 +INTERCEPT_THREADS: Final = 8 +OBSERVER_QUEUE: Final = 1024 + + +class Handler(Protocol): + """A handler the worker calls on a thread, because running it blocks one.""" + + def __call__(self, argument: JSONValue, /) -> object: ... + + +class AsyncHandler(Protocol): + """A handler the worker awaits on its loop, because waiting is all it does.""" + + def __call__(self, argument: JSONValue, /) -> Coroutine[object, object, object]: ... + + +class SubscriptionData(TypedDict): + name: ReadOnly[str] + events: ReadOnly[Sequence[str]] + observes: ReadOnly[bool] + intercepts: ReadOnly[bool] + + +class HelloFrame(TypedDict): + type: ReadOnly[Literal["hello"]] + schema: ReadOnly[SchemaVersion] + subscriptions: ReadOnly[Sequence[SubscriptionData]] + + +class PatchFrame(TypedDict): + type: ReadOnly[Literal["patch"]] + id: ReadOnly[JSONValue] + patch: ReadOnly[JSONValue] + + +class ErrorFrame(TypedDict): + type: ReadOnly[Literal["error"]] + id: ReadOnly[JSONValue] + error_class: ReadOnly[str] + message: ReadOnly[str] + + +class ReportFrame(TypedDict): + type: ReadOnly[Literal["report"]] + subscriber: ReadOnly[int] + event: ReadOnly[JSONValue] + message: ReadOnly[str] + + +class FlushedFrame(TypedDict): + type: ReadOnly[Literal["flushed"]] + id: ReadOnly[JSONValue] + + +Sent: TypeAlias = HelloFrame | PatchFrame | ErrorFrame | ReportFrame | FlushedFrame +Received: TypeAlias = Mapping[str, JSONValue] + + +class ProtocolError(Exception): + """The peer sent something that is not a frame of this protocol; the worker stops.""" + + +def _read_exactly(sock: socket.socket, size: int) -> bytes | None: + """`size` bytes, or `None` when the peer closed before the first of them.""" + chunks: Final[list[bytes]] = [] # mutable-ok: a socket yields a frame in pieces + remaining = size # rebind-ok: counts down as the pieces arrive + while remaining: + chunk = sock.recv(min(remaining, 1 << 20)) # rebind-ok: one piece per pass + if not chunk: + if chunks: + raise ProtocolError("connection closed inside a frame") + return None + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + + +def read_frame(sock: socket.socket) -> Received | None: + """The next frame, or `None` once the peer has closed the connection.""" + header: Final = _read_exactly(sock, 4) + if header is None: + return None + size: Final = int.from_bytes(header, "big") + if size > MAX_FRAME_BYTES: + raise ProtocolError(f"frame of {size} bytes exceeds {MAX_FRAME_BYTES}") + body: Final = _read_exactly(sock, size) + if body is None: + raise ProtocolError("connection closed inside a frame") + frame: Final[JSONValue] = json.loads(body) # pyright: ignore[reportAny] # untyped until the checks below + if not isinstance(frame, Mapping) or not isinstance(frame.get("type"), str): + raise ProtocolError("a frame is a JSON object with a string `type`") + return frame + + +def encode_frame(frame: Sent | Received) -> bytes: + body: Final = json.dumps(frame, separators=(",", ":")).encode() + return len(body).to_bytes(4, "big") + body + + +def load(reference: str) -> None: + """Imports `module`, whose decorators register as they run, or registers `module:attr`.""" + module_name, _, attribute = reference.partition(":") + module: Final = importlib.import_module(module_name) + if attribute: + register(getattr(module, attribute)) # pyright: ignore[reportAny] # the attribute is whatever the operator named; `register` validates it + + +def _subscription(subscriber: Subscriber) -> SubscriptionData: + data: Final[SubscriptionData] = { + "name": subscriber.name, + "events": tuple(sorted(subscriber.events)), + "observes": subscriber.on_event is not None or subscriber.async_on_event is not None, + "intercepts": subscriber.before_send is not None or subscriber.async_before_send is not None, + } + return data + + +def hello(subscribers: Sequence[Subscriber]) -> HelloFrame: + frame: Final[HelloFrame] = { + "type": "hello", + "schema": SCHEMA, + "subscriptions": tuple(_subscription(subscriber) for subscriber in subscribers), + } + return frame + + +def _plain(value: object) -> JSONValue: + """A handler's return value as JSON can carry it; callbacks may build patches from read-only views.""" + if isinstance(value, Mapping): + # A handler's return value is untyped until it is encoded, and json.dumps wants a dict. + return {str(k): _plain(v) for k, v in value.items()} # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # mutable-ok: json.dumps encodes dict, not Mapping + if isinstance(value, Sequence) and not isinstance(value, (str, bytes)): + return tuple(_plain(item) for item in value) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # a handler's return value is untyped until it is encoded + if value is None or isinstance(value, (bool, int, float, str)): + return value + raise TypeError(f"a callback returned {type(value).__name__}, which JSON cannot carry") + + +def _sync(handler: object | None) -> Handler | None: + """A handler the worker calls on a thread; `register` has already checked its kind.""" + return handler if callable(handler) else None + + +def _asynchronous(handler: object | None) -> AsyncHandler | None: + """A handler the worker awaits on its loop; `register` has already checked its kind.""" + return handler if callable(handler) else None # pyright: ignore[reportReturnType] # `register` validated the kind and `callable` narrows no further + + +@dataclass(frozen=True, slots=True) +class _Marker: + """A flush's place in one lane's queue, resolved once everything queued before it is handled.""" + + reached: Future[None] + + +@dataclass(frozen=True, slots=True) +class _Stop: + """The end of one lane's queue; the consumer returns once everything before it is handled.""" + + +_Queued: TypeAlias = JSONValue | _Marker | _Stop + + +@dataclass(frozen=True, slots=True) +class _Lane: + """One subscriber's handlers, each kept in the kind its author declared. + + The kind is the author's scheduling decision and the worker routes on it: a handler that + waits on the network is `async` and costs no thread, one that occupies the CPU is sync + and gets a thread so that the loop keeps turning while it runs. Collapsing the two would + discard that decision, and it cannot be recovered by calling a handler to see what comes + back: calling a sync one has already run it, on whatever thread asked. `events` is the + thread a sync observer's envelopes are handled on, in order, and exists only for one. + """ + + observe: Handler | None + async_observe: AsyncHandler | None + intercept: Handler | None + async_intercept: AsyncHandler | None + events: ThreadPoolExecutor | None + + +class Worker: + """Serves one gateway connection over the subscribers registered when it was built. + + Nothing that waits holds a thread it does not need. An async handler is a task on the + worker's loop, so concurrency is the loop's and not a pool size; a sync handler has a + thread because it blocks one; and every frame leaves through one writer, so no handler + and no loop callback ever blocks on the socket. + """ + + def __init__(self, sock: socket.socket, subscribers: Sequence[Subscriber]) -> None: + self._sock: Final = sock + self._subscribers: Final = tuple(subscribers) + self._loop: Final = asyncio.new_event_loop() + self._outgoing: Final[queue.Queue[bytes | None]] = queue.Queue() + self._writer: Final = threading.Thread(target=self._write, name="callback-writer", daemon=True) + self._intercepts: Final = ThreadPoolExecutor(INTERCEPT_THREADS, thread_name_prefix="callback-intercept") + self._queues: Final[ + dict[int, asyncio.Queue[_Queued]] + ] = {} # mutable-ok: filled once on the loop before serving + self._consumers: Final[list[asyncio.Task[None]]] = [] # mutable-ok: filled once on the loop before serving + self._pending: Final[set[asyncio.Task[None]]] = ( + set() + ) # mutable-ok: the loop's strong references to its live tasks + self._lanes: Final = tuple( + _Lane( + _sync(subscriber.on_event), + _asynchronous(subscriber.async_on_event), + _sync(subscriber.before_send), + _asynchronous(subscriber.async_before_send), + ThreadPoolExecutor(1, thread_name_prefix=f"callback-events-{index}") + if subscriber.on_event is not None and subscriber.async_on_event is None + else None, + ) + for index, subscriber in enumerate(self._subscribers) + ) + + def _write(self) -> None: + """The only thread that writes to the socket, so that no caller ever blocks on it.""" + for data in iter(self._outgoing.get, None): + try: + self._sock.sendall(data) + except OSError: + return # the gateway is gone; the read loop ends on its own + + def _send(self, frame: Sent) -> None: + self._outgoing.put(encode_frame(frame)) + + def _report(self, index: int, envelope: JSONValue, error: BaseException) -> ReportFrame: + event: Final = envelope.get("event") if isinstance(envelope, Mapping) else None + kind: Final = event.get("type") if isinstance(event, Mapping) else None + frame: Final[ReportFrame] = {"type": "report", "subscriber": index, "event": kind, "message": str(error)} + return frame + + def _patched(self, frame_id: JSONValue, result: object) -> PatchFrame: + frame: Final[PatchFrame] = {"type": "patch", "id": frame_id, "patch": _plain(result)} + return frame + + def _failed(self, frame_id: JSONValue, error: BaseException) -> ErrorFrame: + name: Final = f"{type(error).__module__}.{type(error).__qualname__}" + frame: Final[ErrorFrame] = {"type": "error", "id": frame_id, "error_class": name, "message": str(error)} + return frame + + def _observe(self, index: int, handler: Handler, envelope: JSONValue) -> None: + """One sync observer on its lane's thread, in the order its envelopes arrived.""" + try: + handler(envelope) + except Exception as error: # noqa: BLE001 # an observer's failure is reported and swallowed, by contract + self._send(self._report(index, envelope, error)) + + async def _consume(self, index: int, handler: AsyncHandler, envelopes: "asyncio.Queue[_Queued]") -> None: + """One async observer's whole lane: one item at a time, so order holds without a thread.""" + while True: + item: _Queued = await envelopes.get() + if isinstance(item, _Stop): + return + if isinstance(item, _Marker): + item.reached.set_result(None) + continue + try: + await handler(item) + except Exception as error: # noqa: BLE001 # an observer's failure is reported and swallowed, by contract + self._send(self._report(index, item, error)) + + def _intercept(self, frame_id: JSONValue, handler: Handler, request: JSONValue) -> None: + """One sync interceptor on a pool thread, which it holds for as long as it runs.""" + try: + self._send(self._patched(frame_id, handler(request))) + except Exception as error: # noqa: BLE001 # an interceptor's failure is the gateway's to act on + self._send(self._failed(frame_id, error)) + + async def _await_intercept(self, frame_id: JSONValue, handler: AsyncHandler, request: JSONValue) -> None: + """One async interceptor as a task, so that a thousand of them wait on one thread.""" + try: + self._send(self._patched(frame_id, await handler(request))) + except Exception as error: # noqa: BLE001 # an interceptor's failure is the gateway's to act on + self._send(self._failed(frame_id, error)) + + def _keep(self, task: "asyncio.Task[None]") -> None: + """asyncio keeps only a weak reference to a task; this is the strong one.""" + self._pending.add(task) + task.add_done_callback(self._pending.discard) + + def _spawn(self, frame_id: JSONValue, handler: AsyncHandler, request: JSONValue) -> None: + """On the loop: starts the interception and returns, so the read loop stays free.""" + self._keep(self._loop.create_task(self._await_intercept(frame_id, handler, request))) + + def _enqueue(self, index: int, envelope: JSONValue) -> None: + """On the loop: a bounded lane drops rather than grow without limit, and says so.""" + try: + self._queues[index].put_nowait(envelope) + except asyncio.QueueFull: + full: Final = RuntimeError(f"observer queue is full at {OBSERVER_QUEUE}; this envelope was dropped") + self._send(self._report(index, envelope, full)) + + def _mark(self, index: int, reached: "Future[None]") -> None: + """On the loop: a marker waits for room rather than drop, so it keeps its place.""" + self._keep(self._loop.create_task(self._queues[index].put(_Marker(reached)))) + + async def _start(self) -> None: + """On the loop: one queue and one consumer per async observer, before the first frame.""" + for index, lane in enumerate(self._lanes): + if lane.async_observe is not None: + envelopes: asyncio.Queue[_Queued] = asyncio.Queue(OBSERVER_QUEUE) + self._queues[index] = envelopes + self._consumers.append(self._loop.create_task(self._consume(index, lane.async_observe, envelopes))) + + async def _drain(self) -> None: + """On the loop: every lane sees a stop behind what was queued, then every task ends.""" + for envelopes in self._queues.values(): + await envelopes.put(_Stop()) + outstanding: Final = (*self._consumers, *self._pending) + if outstanding: + _ = await asyncio.gather(*outstanding, return_exceptions=True) + + def _lane(self, frame: Received) -> tuple[int, _Lane]: + index: Final = frame.get("subscriber") + if not isinstance(index, int) or isinstance(index, bool) or not 0 <= index < len(self._lanes): + raise ProtocolError(f"unknown subscriber: {index!r}") + return index, self._lanes[index] + + def _flush(self, frame_id: JSONValue) -> None: + """A marker at the back of each lane passes only after everything queued before it.""" + markers: Final[list[Future[None]]] = [] # mutable-ok: one marker per lane, collected and then waited on + for index, lane in enumerate(self._lanes): + if index in self._queues: + reached: Future[None] = Future() + self._loop.call_soon_threadsafe(self._mark, index, reached) + markers.append(reached) + elif lane.events is not None: + markers.append(lane.events.submit(lambda: None)) + for marker in markers: + marker.result() + flushed: Final[FlushedFrame] = {"type": "flushed", "id": frame_id} + self._send(flushed) + + def _dispatch(self, frame: Received) -> None: + """Routes one frame and returns; nothing here runs a handler or touches the socket.""" + kind: Final = frame["type"] + if kind == "event": + index, observed = self._lane(frame) + envelope: Final = frame.get("envelope") + if observed.async_observe is not None: + self._loop.call_soon_threadsafe(self._enqueue, index, envelope) + elif observed.observe is not None and observed.events is not None: + _ = observed.events.submit(self._observe, index, observed.observe, envelope) + elif kind == "intercept": + _, asked = self._lane(frame) + if asked.async_intercept is not None: + self._loop.call_soon_threadsafe( + self._spawn, frame.get("id"), asked.async_intercept, frame.get("request") + ) + elif asked.intercept is not None: + _ = self._intercepts.submit(self._intercept, frame.get("id"), asked.intercept, frame.get("request")) + else: + unpatched: Final[PatchFrame] = {"type": "patch", "id": frame.get("id"), "patch": None} + self._send(unpatched) + elif kind == "flush": + _ = self._intercepts.submit(self._flush, frame.get("id")) + else: + raise ProtocolError(f"unknown frame type: {kind!r}") + + def serve(self) -> None: + """Says hello, then answers until the gateway closes the connection.""" + threading.Thread(target=self._loop.run_forever, name="callback-loop", daemon=True).start() + self._writer.start() + try: + _ = asyncio.run_coroutine_threadsafe(self._start(), self._loop).result() + self._send(hello(self._subscribers)) + while (frame := read_frame(self._sock)) is not None: + self._dispatch(frame) + finally: + # In this order: no new flush can start, so every marker still resolves behind a + # live consumer; then the lanes end; then the writer, once nothing can send again. + self._intercepts.shutdown(wait=True) + _ = asyncio.run_coroutine_threadsafe(self._drain(), self._loop).result() + for lane in self._lanes: + if lane.events is not None: + lane.events.shutdown(wait=True) + self._loop.call_soon_threadsafe(self._loop.stop) + self._outgoing.put(None) + self._writer.join() + + +@dataclass(frozen=True, slots=True) +class Arguments: + fd: int | None + connect: str | None + load: tuple[str, ...] + + +def _connect(arguments: Arguments) -> socket.socket: + if arguments.fd is not None: + return socket.socket(fileno=arguments.fd) + if arguments.connect is None: + raise ValueError("one of --fd and --connect is required") + connected: Final = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + connected.connect(arguments.connect) + return connected + + +def _option(parsed: Mapping[str, object], name: str) -> object: + return parsed.get(name) + + +def _arguments(argv: Sequence[str] | None) -> Arguments: + parser: Final = argparse.ArgumentParser(prog="python -m litellm.callbacks_v1.host") + where: Final = parser.add_mutually_exclusive_group(required=True) + where.add_argument("--fd", type=int, help="an inherited, connected socket (the gateway spawned this worker)") + where.add_argument("--connect", metavar="PATH", help="a unix socket the gateway listens on (a sidecar)") + parser.add_argument("--load", action="append", metavar="MODULE[:ATTR]", help="a callback to import") + parsed: Final = vars(parser.parse_args(argv)) + fd: Final = _option(parsed, "fd") + connect: Final = _option(parsed, "connect") + load_: Final = _option(parsed, "load") + return Arguments( + fd if isinstance(fd, int) else None, + connect if isinstance(connect, str) else None, + tuple(str(item) for item in load_) if isinstance(load_, (list, tuple)) else (), # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # argparse collects `--load` untyped + ) + + +def main(argv: Sequence[str] | None = None) -> int: + arguments: Final = _arguments(argv) + for reference in arguments.load: + load(reference) + with _connect(arguments) as sock: + Worker(sock, snapshot()).serve() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/litellm/rust_bridge/callbacks_legacy_python.py b/litellm/rust_bridge/callbacks_legacy.py similarity index 100% rename from litellm/rust_bridge/callbacks_legacy_python.py rename to litellm/rust_bridge/callbacks_legacy.py diff --git a/litellm/rust_bridge/callbacks_v1.py b/litellm/rust_bridge/callbacks_v1.py new file mode 100644 index 00000000000..927061d4c16 --- /dev/null +++ b/litellm/rust_bridge/callbacks_v1.py @@ -0,0 +1,53 @@ +"""The v1 contract functions the native call reaches for, by name. + +`python_contract.json` pins these four on both sides. The typed authoring surface — the +callback protocols, the envelope and patch types, and the registry — is +`litellm.callbacks_v1`; `snapshot` is re-exported from there because the registry has one +home, and the rest here is marshalling no callback author calls. +""" + +import json +import uuid +from typing import Final, TypeGuard # noqa: TID251 # recursive JSON validation + +from litellm.callbacks_v1 import JSONValue, snapshot + +__all__ = ("new_call_id", "project_response", "report", "snapshot") + + +def _is_json_value( + value: object, +) -> TypeGuard[JSONValue]: # guard-ok: recursively validates the complete JSON value + if value is None or isinstance(value, (bool, int, float, str)): + return True + if isinstance(value, list): + return all( + _is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # list items are validated recursively + for item in value # pyright: ignore[reportUnknownVariableType] # list items are validated recursively + ) + if isinstance(value, dict): + return all( + isinstance(key, str) and _is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # mapping entries are validated recursively + for key, item in value.items() # pyright: ignore[reportUnknownVariableType] # mapping entries are validated recursively + ) + return False + + +def project_response(response: object) -> JSONValue: + model_dump: Final = getattr(response, "model_dump", None) + projected: Final = model_dump(mode="json") if callable(model_dump) else response + json.dumps(projected, allow_nan=False) + if _is_json_value(projected): + return projected + raise TypeError(f"unsupported response type: {type(response).__qualname__}") + + +def new_call_id() -> str: + return str(uuid.uuid4()) + + +def report(name: str, event: str, error: BaseException) -> None: + from litellm._logging import verbose_logger + + exception: Final = (type(error), error, error.__traceback__) + verbose_logger.exception("callback %s failed while handling %s", name, event, exc_info=exception) diff --git a/litellm/rust_bridge/callbacks_v1_python.py b/litellm/rust_bridge/callbacks_v1_python.py deleted file mode 100644 index 2de77e9034e..00000000000 --- a/litellm/rust_bridge/callbacks_v1_python.py +++ /dev/null @@ -1,272 +0,0 @@ -import inspect -import json -import threading -import uuid -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from typing import Final, Literal, Protocol, TypeAlias, TypeGuard # noqa: TID251 # recursive JSON validation - -from typing_extensions import ReadOnly, TypeAliasType, TypedDict - -JSONScalar: TypeAlias = None | bool | int | float | str -JSONValue = TypeAliasType( # rebind-ok: named recursive alias required for runtime validation - "JSONValue", JSONScalar | Sequence["JSONValue"] | Mapping[str, "JSONValue"] -) - -SUPPORTED_SCHEMAS: Final = frozenset({1}) -EVENTS: Final = frozenset( - { - "call.started", - "request.sending", - "response.received", - "call.succeeded", - "call.failed", - } -) - - -class TimingFactsV1(TypedDict): - start_time: ReadOnly[float] - end_time: ReadOnly[float] - duration: ReadOnly[float] - - -class CallFactsV1(TypedDict): - start_time: ReadOnly[float] - asynchronous: ReadOnly[bool] - metadata: ReadOnly[Mapping[str, JSONValue]] - metadata_dropped: ReadOnly[Sequence[str]] - - -class RequestFactsV1(TypedDict): - model: ReadOnly[str] - custom_llm_provider: ReadOnly[str] - optional_params: ReadOnly[JSONValue] - url: ReadOnly[str] - headers: ReadOnly[Sequence[tuple[str, str]]] - body: ReadOnly[JSONValue] - - -ErrorFactsV1 = TypedDict( # rebind-ok: runtime TypedDict type for the reserved class field - "ErrorFactsV1", - { # mutable-ok: functional TypedDict syntax requires a dict for the reserved class field - "class": ReadOnly[str], - "message": ReadOnly[str], - "status_code": ReadOnly[int | None], - }, -) - - -class CallStartedV1(CallFactsV1): - type: ReadOnly[Literal["call.started"]] - - -class RequestSendingV1(RequestFactsV1): - type: ReadOnly[Literal["request.sending"]] - - -class ResponseReceivedV1(TypedDict): - type: ReadOnly[Literal["response.received"]] - body: ReadOnly[str] - - -class CallSucceededV1(TypedDict): - type: ReadOnly[Literal["call.succeeded"]] - timing: ReadOnly[TimingFactsV1] - streamed: ReadOnly[bool] - response: ReadOnly[JSONValue] - response_error: ReadOnly[str | None] - - -class CallFailedV1(TypedDict): - type: ReadOnly[Literal["call.failed"]] - timing: ReadOnly[TimingFactsV1] - streamed: ReadOnly[bool] - origin: ReadOnly[Literal["call", "host"]] - error: ReadOnly[ErrorFactsV1] - - -EventV1: TypeAlias = CallStartedV1 | RequestSendingV1 | ResponseReceivedV1 | CallSucceededV1 | CallFailedV1 - - -class EnvelopeV1(TypedDict): - schema: ReadOnly[Literal[1]] - call_id: ReadOnly[str] - call_type: ReadOnly[str] - seq: ReadOnly[int] - event: ReadOnly[EventV1] - - -class HeaderPatchV1(TypedDict, total=False): - remove: ReadOnly[Sequence[str]] - set: ReadOnly[Sequence[tuple[str, str]]] - - -class WirePatchV1(TypedDict, total=False): - headers: ReadOnly[HeaderPatchV1] - body: ReadOnly[JSONValue] - - -class Callback(Protocol): - name: str - schema: int - events: frozenset[str] - - def on_event(self, event: EnvelopeV1) -> None: ... - - async def async_on_event(self, event: EnvelopeV1) -> None: ... - - def before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ... - - async def async_before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ... - - -class RegistrationError(ValueError): - pass - - -class UnsupportedSchema(RegistrationError): - pass - - -class UnknownEvent(RegistrationError): - pass - - -class MissingHandler(RegistrationError): - pass - - -class EmptyObserverEvents(RegistrationError): - pass - - -class HandlerKindMismatch(RegistrationError): - pass - - -class DuplicateName(RegistrationError): - pass - - -@dataclass(frozen=True, slots=True) -class Subscriber: - name: str - schema: int - events: frozenset[str] - on_event: object | None - async_on_event: object | None - before_send: object | None - async_before_send: object | None - - -class _Registry: - def __init__(self) -> None: - self.subscribers: tuple[Subscriber, ...] = () - self.lock: Final = threading.Lock() - - -_REGISTRY: Final = _Registry() - - -def _handler(callback: Callback, name: str) -> object | None: - return getattr(callback, name, None) - - -def _validate_kind(name: str, handler: object | None, asynchronous: bool) -> None: - if handler is None: - return - if not callable(handler): - raise HandlerKindMismatch(f"{name} must be callable") - is_async: Final = inspect.iscoroutinefunction(handler) - if is_async != asynchronous: - expected: Final = "async" if asynchronous else "sync" - raise HandlerKindMismatch(f"{name} must be {expected}") - - -def register(callback: Callback) -> None: - schema: Final = callback.schema - if schema not in SUPPORTED_SCHEMAS: - raise UnsupportedSchema(f"unsupported callback schema: {schema}") - events: Final = frozenset(callback.events) - unknown: Final = events - EVENTS - if unknown: - raise UnknownEvent(f"unknown callback events: {sorted(unknown)}") - - on_event: Final = _handler(callback, "on_event") - async_on_event: Final = _handler(callback, "async_on_event") - before_send: Final = _handler(callback, "before_send") - async_before_send: Final = _handler(callback, "async_before_send") - handlers: Final = (on_event, async_on_event, before_send, async_before_send) - if all(handler is None for handler in handlers): - raise MissingHandler("callback must define at least one handler") - if (on_event is not None or async_on_event is not None) and not events: - raise EmptyObserverEvents("observer callback events cannot be empty") - - _validate_kind("on_event", on_event, False) - _validate_kind("async_on_event", async_on_event, True) - _validate_kind("before_send", before_send, False) - _validate_kind("async_before_send", async_before_send, True) - - subscriber: Final = Subscriber( - name=callback.name, - schema=schema, - events=events, - on_event=on_event, - async_on_event=async_on_event, - before_send=before_send, - async_before_send=async_before_send, - ) - with _REGISTRY.lock: - if any(existing.name == subscriber.name for existing in _REGISTRY.subscribers): - raise DuplicateName(f"duplicate callback name: {subscriber.name}") - _REGISTRY.subscribers = (*_REGISTRY.subscribers, subscriber) - - -def unregister(callback: Callback) -> None: - with _REGISTRY.lock: - _REGISTRY.subscribers = tuple( - subscriber for subscriber in _REGISTRY.subscribers if subscriber.name != callback.name - ) - - -def snapshot() -> tuple[Subscriber, ...]: - return _REGISTRY.subscribers - - -def _is_json_value( - value: object, -) -> TypeGuard[JSONValue]: # guard-ok: recursively validates the complete JSON value - if value is None or isinstance(value, (bool, int, float, str)): - return True - if isinstance(value, list): - return all( - _is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # list items are validated recursively - for item in value # pyright: ignore[reportUnknownVariableType] # list items are validated recursively - ) - if isinstance(value, dict): - return all( - isinstance(key, str) and _is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # mapping entries are validated recursively - for key, item in value.items() # pyright: ignore[reportUnknownVariableType] # mapping entries are validated recursively - ) - return False - - -def project_response(response: object) -> JSONValue: - model_dump: Final = getattr(response, "model_dump", None) - projected: Final = model_dump(mode="json") if callable(model_dump) else response - json.dumps(projected, allow_nan=False) - if _is_json_value(projected): - return projected - raise TypeError(f"unsupported response type: {type(response).__qualname__}") - - -def new_call_id() -> str: - return str(uuid.uuid4()) - - -def report(name: str, event: str, error: BaseException) -> None: - from litellm._logging import verbose_logger - - exception: Final = (type(error), error, error.__traceback__) - verbose_logger.exception("callback %s failed while handling %s", name, event, exc_info=exception) diff --git a/tests/test_litellm/callbacks_v1/__init__.py b/tests/test_litellm/callbacks_v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/callbacks_v1/builtin/__init__.py b/tests/test_litellm/callbacks_v1/builtin/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/callbacks_v1/builtin/support.py b/tests/test_litellm/callbacks_v1/builtin/support.py new file mode 100644 index 00000000000..51f88472e4d --- /dev/null +++ b/tests/test_litellm/callbacks_v1/builtin/support.py @@ -0,0 +1,152 @@ +"""What every port's tests share: golden envelopes, a fake transport, and the parity diff. + +The parity diff is the migration test. A legacy logger and its port handle the same call, +and every difference between what they send the vendor must be `Allowed` with a reason; +an unexplained one fails. An `Allowed` entry that no longer matches anything fails too, +and so does one citing a `Gap` its port no longer lists, so the allow-list and the gap +ledger shrink together as the contract grows instead of rotting apart. +""" + +import json +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from datetime import datetime, timezone +from fnmatch import fnmatchcase +from pathlib import Path +from typing import Final + +from litellm.callbacks_v1 import Callback, EnvelopeV1, EventName +from litellm.callbacks_v1.builtin import anthropic_cache_control, generic_api, openmeter +from litellm.callbacks_v1.builtin.manifest import Gap +from litellm.callbacks_v1.builtin.port import Delivery +from litellm.callbacks_v1.builtin.runtime import Patcher, Sink, Transport + +ROOT: Final = Path(__file__).parents[4] +GOLDEN_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1/golden/v1" +FIXED_NOW: Final = datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc) + + +def golden(kind: EventName) -> EnvelopeV1: + """The envelope Rust pins for `kind`; a port tested on it cannot drift from the contract. + `tests/test_litellm/test_callbacks_v1.py` is what validates these files against the types.""" + return json.loads((GOLDEN_PATH / f"{kind}.json").read_text()) + + +def golden_call(terminal: EventName, **metadata: str) -> tuple[EnvelopeV1, ...]: + """One call's envelopes in order, ending in `terminal`, with `metadata` on call.started.""" + started: Final = golden("call.started") + with_metadata: Final[EnvelopeV1] = { + **started, + "event": {**started["event"], "metadata": {**started["event"]["metadata"], **metadata}}, # pyright: ignore[reportTypedDictNotRequiredAccess, reportGeneralTypeIssues] # the golden call.started event always has metadata + } + return (with_metadata, golden("request.sending"), golden("response.received"), golden(terminal)) + + +@dataclass +class FakeTransport: + deliveries: list[Delivery] = field(default_factory=list) + + def send(self, delivery: Delivery) -> None: + self.deliveries.append(delivery) + + def bodies(self) -> list[object]: + return [json.loads(delivery.body) for delivery in self.deliveries] + + +def plain(value: object) -> object: + """A contract value as plain dicts and lists, so it compares and serialises like JSON.""" + if isinstance(value, Mapping): + return {key: plain(item) for key, item in value.items()} + if isinstance(value, Sequence) and not isinstance(value, (str, bytes)): + return [plain(item) for item in value] + return value + + +# One configured port per manifest entry; `test_conventions` requires one for every entry. +# A port is pure and unnamed, so an example is a value: the host, not the port, is what +# needs a transport and what decides the registration name. +OPENMETER: Final = openmeter.OpenMeter(openmeter.Config(api_key="test-key", endpoint="http://openmeter.test")) +GENERIC_API: Final = generic_api.GenericApi(generic_api.Config(endpoint="http://generic.test/logs", batch_size=1)) +ANTHROPIC_CACHE_CONTROL: Final = anthropic_cache_control.AnthropicCacheControl( + anthropic_cache_control.Config(points=(anthropic_cache_control.InjectionPoint(role="system"),)) +) +EXAMPLES: Final[Mapping[str, object]] = { + "openmeter": OPENMETER, + "generic_api": GENERIC_API, + "anthropic_cache_control": ANTHROPIC_CACHE_CONTROL, +} + +# The subscriber a host builds for each example port, under the manifest's name. Naming and +# wrapping are the host's job, which is why these live here and not in the ports; each call +# is also where the type checker holds a port to its protocol. The clock is fixed so a port +# that stamps a time is comparable run to run, and one that does not is handed it anyway: +# that is the point of the two kinds having one shape. +SUBSCRIBERS: Final[Mapping[str, Callable[[Transport], Callback]]] = { + "openmeter": lambda transport: Sink("openmeter", OPENMETER, transport, lambda: FIXED_NOW), + "generic_api": lambda transport: Sink("generic_api", GENERIC_API, transport, lambda: FIXED_NOW), + "anthropic_cache_control": lambda _transport: Patcher("anthropic_cache_control", ANTHROPIC_CACHE_CONTROL), +} + + +@dataclass(frozen=True, slots=True) +class Allowed: + """A difference from the legacy twin that is intended. `path` is an fnmatch pattern over + dotted paths (`data.cost`, `0.metadata.*`; a list index is a segment); `reason` says why. + + `gap` is the `Gap.key` this difference exists because of. A difference with no `gap` is + one the port intends to keep; a difference with one goes away when the contract grows.""" + + path: str + reason: str + gap: str | None = None + + +def _leaves(value: object, prefix: str = "") -> Mapping[str, object]: + if isinstance(value, Mapping) and value: + return { + path: leaf + for key, item in value.items() + for path, leaf in _leaves(item, f"{prefix}.{key}" if prefix else str(key)).items() + } + if isinstance(value, list) and value: + return { + path: leaf + for index, item in enumerate(value) + for path, leaf in _leaves(item, f"{prefix}.{index}" if prefix else str(index)).items() + } + return {prefix: value} + + +def unexplained(legacy: object, port: object, allowed: Sequence[Allowed], gaps: Sequence[Gap]) -> tuple[str, ...]: + """Every difference between two vendor bodies that `allowed` does not cover, plus every + `allowed` entry that covered nothing or cites a gap not in `gaps`. Empty means parity.""" + assert all(entry.reason.strip() for entry in allowed), "every Allowed difference needs a reason" + open_keys: Final = {gap.key for gap in gaps} + closed: Final = [entry for entry in allowed if entry.gap is not None and entry.gap not in open_keys] + before: Final = _leaves(plain(legacy)) + after: Final = _leaves(plain(port)) + differences: Final = { + **{path: f"missing in port: {path} (legacy sent {before[path]!r})" for path in before.keys() - after.keys()}, + **{path: f"only in port: {path} = {after[path]!r}" for path in after.keys() - before.keys()}, + **{ + path: f"differs: {path}: legacy {before[path]!r} != port {after[path]!r}" + for path in before.keys() & after.keys() + if before[path] != after[path] + }, + } + covered: Final = {path: [entry for entry in allowed if fnmatchcase(path, entry.path)] for path in differences} + stale: Final = [entry for entry in allowed if not any(entry in entries for entries in covered.values())] + return ( + *(message for path, message in sorted(differences.items()) if not covered[path]), + *(f"stale Allowed({entry.path!r}): it no longer matches any difference" for entry in stale), + *(f"Allowed({entry.path!r}) cites gap {entry.gap!r}, which the port does not list" for entry in closed), + ) + + +def assert_parity(legacy: bytes, port: bytes, allowed: Sequence[Allowed], gaps: Sequence[Gap]) -> None: + """Compares two raw vendor request bodies as JSON and fails with one line per unexplained + difference, so the fix is to port it or to allow it. Two empty bodies are not parity. + `gaps` is the port's own `GAPS`, which every `Allowed.gap` has to be found in.""" + assert legacy and port, "nothing to compare: a side sent no body" + problems: Final = unexplained(json.loads(legacy), json.loads(port), allowed, gaps) + assert not problems, f"{len(problems)} unexplained difference(s) from the legacy twin:\n " + "\n ".join(problems) diff --git a/tests/test_litellm/callbacks_v1/builtin/test_anthropic_cache_control.py b/tests/test_litellm/callbacks_v1/builtin/test_anthropic_cache_control.py new file mode 100644 index 00000000000..0c157fc59b4 --- /dev/null +++ b/tests/test_litellm/callbacks_v1/builtin/test_anthropic_cache_control.py @@ -0,0 +1,81 @@ +"""Function-level parity: the port's `marked` against the legacy hook's Messages-route function +on the same bodies. The legacy hook never runs on the native route, so there is no call to share.""" + +import copy + +import pytest + +from litellm.callbacks_v1.builtin import anthropic_cache_control as port +from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook +from tests.test_litellm.callbacks_v1.builtin.support import golden, plain + +EPHEMERAL = {"type": "ephemeral"} +TURNS = [ + {"role": "user", "content": "first"}, + {"role": "assistant", "content": [{"type": "text", "text": "second"}]}, + {"role": "user", "content": [{"type": "text", "text": "third"}, {"type": "text", "text": "fourth"}]}, +] +ALREADY_MARKED = [ + {"role": "user", "content": [{"type": "text", "text": "kept", "cache_control": {"type": "ephemeral", "ttl": "1h"}}]} +] +FIVE_USERS = [{"role": "user", "content": f"turn {index}"} for index in range(5)] + +CASES = { + "system string": ({"system": "be brief", "messages": TURNS}, [("system", None, None)]), + "system blocks": ( + {"system": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}], "messages": TURNS}, + [("system", None, None)], + ), + "no system to mark": ({"messages": TURNS}, [("system", None, None)]), + "every user turn": ({"messages": TURNS}, [("user", None, None)]), + "last message by index": ({"messages": TURNS}, [(None, -1, None)]), + "index out of range": ({"messages": TURNS}, [(None, 9, None)]), + "ttl": ({"system": "be brief", "messages": TURNS}, [("system", None, "1h"), (None, -1, "5m")]), + "client breakpoint is kept": ({"messages": ALREADY_MARKED}, [("user", None, None)]), + "stops at four breakpoints": ( + {"system": "be brief", "messages": FIVE_USERS}, + [("system", None, None), ("user", None, None)], + ), +} + + +def legacy(body: dict, points: list[tuple]) -> dict: + injection_points = [ + {"location": "message", "role": role, "index": index, "control": {**EPHEMERAL, **({"ttl": ttl} if ttl else {})}} + for role, index, ttl in points + ] + messages, system, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + copy.deepcopy(body["messages"]), + copy.deepcopy(body.get("system")), + injection_points, # pyright: ignore[reportArgumentType] # plain dicts stand in for the legacy TypedDicts + ) + return {**body, "messages": messages, **({"system": system} if "system" in body else {})} + + +@pytest.mark.parametrize("name", CASES) +def test_the_port_marks_a_messages_body_exactly_as_the_legacy_hook_does(name: str) -> None: + body, points = CASES[name] + config = port.Config(points=tuple(port.InjectionPoint(role, index, ttl) for role, index, ttl in points)) + original = copy.deepcopy(body) + + assert plain(port.AnthropicCacheControl(config).marked(body)) == legacy(body, points) + assert body == original + + +def test_the_patch_covers_only_the_configured_providers() -> None: + request = {**golden("request.sending")["event"], "body": {"system": "be brief", "messages": TURNS}} + config = port.Config(points=(port.InjectionPoint(role="system"),)) + + interceptor = port.AnthropicCacheControl(config) + + assert interceptor.payload(request) is None # pyright: ignore[reportArgumentType] # the golden event plus a Messages body + patch = interceptor.payload({**request, "custom_llm_provider": "anthropic"}) # pyright: ignore[reportArgumentType] # the golden event plus a Messages body + + assert patch is not None and set(patch) == {"body"} + assert plain(patch["body"])["system"] == [{"type": "text", "text": "be brief", "cache_control": EPHEMERAL}] # pyright: ignore[reportIndexIssue, reportTypedDictNotRequiredAccess, reportCallIssue, reportArgumentType] # the patch body is the Messages mapping + + +def test_a_body_that_is_not_a_messages_request_is_left_alone() -> None: + config = port.Config(points=(port.InjectionPoint(role="user"),)) + assert port.AnthropicCacheControl(config).marked({"input": "hello"}) is None + assert port.AnthropicCacheControl(port.Config(points=())).marked({"messages": TURNS}) is None diff --git a/tests/test_litellm/callbacks_v1/builtin/test_conventions.py b/tests/test_litellm/callbacks_v1/builtin/test_conventions.py new file mode 100644 index 00000000000..b2203a17606 --- /dev/null +++ b/tests/test_litellm/callbacks_v1/builtin/test_conventions.py @@ -0,0 +1,180 @@ +"""The rules every port obeys, checked for every manifest entry. A new port gets these for free.""" + +import ast +import importlib +import subprocess +import sys +from collections.abc import Generator +from dataclasses import is_dataclass +from pathlib import Path +from typing import Final + +import pytest + +from litellm import callbacks_v1 +from litellm.callbacks_v1.builtin import port as port_vocabulary +from litellm.callbacks_v1.builtin.manifest import PORTS, Entry, Gap, ledger +from litellm.callbacks_v1.builtin.port import InterceptorPort, SinkPort +from litellm.callbacks_v1.builtin.runtime import Sink +from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.integrations.custom_logger import CustomLogger +from tests.test_litellm.callbacks_v1.builtin.support import EXAMPLES, SUBSCRIBERS, FakeTransport, golden_call + +# The legacy hooks a v1 sink can stand in for. A legacy twin overriding any other +# CustomLogger hook does something the v1 contract has no place for. +SINK_HOOKS: Final = frozenset( + { + "log_pre_api_call", + "log_post_api_call", + "log_success_event", + "log_failure_event", + "async_log_success_event", + "async_log_failure_event", + "async_log_pre_api_call", + } +) +PUBLIC_HOOKS: Final = frozenset( + name + for name, value in vars(CustomLogger).items() + if callable(value) and (name.startswith(("log_", "async_")) or name.endswith("_hook")) +) + +# What a port may import. A port is values and pure methods, the shape a sandboxed custom +# callback is confined to; anything that can reach a socket, a thread, the environment or the +# rest of litellm belongs to `runtime.py`, which a port may not import at all. A new pure +# stdlib module is a reviewed line here. +PURE_MODULES: Final = frozenset( + { + "base64", + "collections.abc", + "dataclasses", + "datetime", + "hashlib", + "json", + "math", + "re", + "types", + "typing", + "typing_extensions", + "urllib.parse", + "litellm.callbacks_v1", + "litellm.callbacks_v1.builtin.port", + } +) +# What a port never carries. `runtime.py` names it and wraps it into a subscriber, and +# `manifest.py` holds everything the migration needs to say about it. +NOT_A_PORTS: Final = frozenset( + {"NAME", "GAPS", "events", "on_event", "async_on_event", "before_send", "async_before_send"} +) + +pytestmark = pytest.mark.parametrize("entry", PORTS, ids=[entry.name for entry in PORTS]) + + +@pytest.fixture(autouse=True) +def clean_registry() -> Generator[None]: + yield + for subscriber in callbacks_v1.snapshot(): + callbacks_v1.unregister(subscriber) + + +def _legacy(entry: Entry) -> type: + module, _, attribute = entry.legacy.partition(":") + return getattr(importlib.import_module(module), attribute) + + +def test_importing_a_port_registers_nothing(entry: Entry) -> None: + script: Final = ( + f"import {entry.module.__name__}; from litellm import callbacks_v1; " + "import sys; sys.exit(len(callbacks_v1.snapshot()))" + ) + assert subprocess.run([sys.executable, "-c", script], check=False).returncode == 0 + + +def _impure_imports(source: str) -> tuple[str, ...]: + nodes: Final = tuple(ast.walk(ast.parse(source))) + plain: Final = tuple(alias.name for node in nodes if isinstance(node, ast.Import) for alias in node.names) + froms: Final = tuple( + str(node.module) + for node in nodes + if isinstance(node, ast.ImportFrom) and (node.level or node.module not in PURE_MODULES) + ) + return (*(name for name in plain if name not in PURE_MODULES), *froms) + + +def test_a_port_imports_nothing_that_can_act(entry: Entry) -> None: + for module in (entry.module, port_vocabulary): + source = Path(str(module.__file__)).read_text() # rebind-ok: one source per checked module + assert _impure_imports(source) == (), f"{module.__name__} imports outside the pure allow-list" + + +def test_a_port_is_a_frozen_class_over_a_frozen_config(entry: Entry) -> None: + example: Final = EXAMPLES[entry.name] + assert all(isinstance(gap, Gap) for gap in entry.gaps) + assert is_dataclass(example) and type(example).__dataclass_params__.frozen + assert is_dataclass(entry.module.Config) and entry.module.Config.__dataclass_params__.frozen + assert "Legacy twin:" in (entry.module.__doc__ or "") + + +def test_a_port_declares_the_protocol_of_its_kind_and_nothing_of_the_contract(entry: Entry) -> None: + """A port names its protocol as a base, so the type checker holds it to the shape at the + class. This is the same claim at runtime, for a port whose module a type checker never saw.""" + declared: Final = type(EXAMPLES[entry.name]).__mro__ + expected: Final = SinkPort if entry.kind == "sink" else InterceptorPort + assert expected in declared, f"{entry.name} is a {entry.kind}; it should declare {expected.__name__} as its base" + assert (SinkPort in declared) == (entry.kind == "sink") + + example: Final = EXAMPLES[entry.name] + carried: Final = {member for member in NOT_A_PORTS if hasattr(example, member)} + assert not carried, f"{entry.name} carries {sorted(carried)}; that is runtime.py's or manifest.py's" + + +def test_a_port_registers_under_its_manifest_name_with_the_handlers_of_its_kind(entry: Entry) -> None: + callbacks_v1.register(SUBSCRIBERS[entry.name](FakeTransport())) + + (subscriber,) = callbacks_v1.snapshot() + assert subscriber.name == entry.name + observes: Final = subscriber.on_event is not None or subscriber.async_on_event is not None + intercepts: Final = subscriber.before_send is not None or subscriber.async_before_send is not None + assert (observes, intercepts) == ((True, False) if entry.kind == "sink" else (False, True)) + # A sink's handler only enqueues, so the sync one serves async calls too. + assert subscriber.async_on_event is None + + +def test_a_sink_handler_does_no_io_and_keeps_no_finished_call(entry: Entry) -> None: + if entry.kind != "sink": + pytest.skip("interceptors have no outbox") + transport: Final = FakeTransport() + callback: Final = SUBSCRIBERS[entry.name](transport) + assert isinstance(callback, Sink), "a sink port supplies pure methods; `runtime.Sink` is its only subscriber" + + for envelope in golden_call("call.succeeded", user_api_key_user_id="user-1"): + callback.on_event(envelope) + + assert callback.outbox.flush() + assert len(transport.deliveries) == 1 + assert callback.open_calls() == 0 + + +def test_the_legacy_twin_resolves_and_a_sink_twin_is_only_a_sink(entry: Entry) -> None: + legacy: Final = _legacy(entry) + if entry.kind != "sink": + return + overridden: Final = { + name + for cls in legacy.__mro__ + if cls not in (CustomLogger, CustomBatchLogger, object) + for name in vars(cls) + if name in PUBLIC_HOOKS + } + assert overridden <= SINK_HOOKS, f"{entry.legacy} also overrides {sorted(overridden - SINK_HOOKS)}" + + +def test_a_port_names_each_gap_once_by_what_the_contract_lacks(entry: Entry) -> None: + keys: Final = tuple(gap.key for gap in entry.gaps) + assert len(keys) == len(frozenset(keys)), f"{entry.name} lists a gap key twice: {sorted(keys)}" + assert all(key and key == key.lower() and key.replace("_", "").isalnum() for key in keys) + + +def test_ready_means_no_open_gap(entry: Entry) -> None: + assert entry.status != "ready" or entry.gaps == () + assert entry.name in ledger() diff --git a/tests/test_litellm/callbacks_v1/builtin/test_generic_api.py b/tests/test_litellm/callbacks_v1/builtin/test_generic_api.py new file mode 100644 index 00000000000..feff14b7e7b --- /dev/null +++ b/tests/test_litellm/callbacks_v1/builtin/test_generic_api.py @@ -0,0 +1,101 @@ +import json +from datetime import datetime, timezone + +import pytest + +from litellm.callbacks_v1.builtin import generic_api +from litellm.callbacks_v1.builtin.port import CallRecord +from litellm.callbacks_v1.builtin.runtime import CallJoin, Sink +from tests.test_litellm.callbacks_v1.builtin.support import golden_call + +PORT = generic_api.GenericApi(generic_api.Config(endpoint="http://generic.test")) +FAILURES_ONLY = generic_api.GenericApi( + generic_api.Config(endpoint="http://generic.test", event_types=frozenset({"llm_api_failure"})) +) +# A sink is handed the clock whether or not it stamps a time; this one does not. +UNUSED_NOW = datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc) + +COMMON = { + "litellm_call_id": "call-123", + "call_type": "ocr", + "startTime": 10.0, + "model": "model", + "custom_llm_provider": "provider", + "model_parameters": {"temperature": 0.2}, +} + + +def joined(terminal: str) -> CallRecord: + join = CallJoin() + records = [join.accept(envelope) for envelope in golden_call(terminal)] # pyright: ignore[reportArgumentType] # terminal is an EventName literal at every call site + (record,) = [record for record in records if record is not None] + return record + + +def test_a_succeeded_call_uses_the_standard_logging_payload_names() -> None: + assert PORT.payload(joined("call.succeeded"), UNUSED_NOW) == { + **COMMON, + "id": "response", + "stream": False, + "status": "success", + "endTime": 12.5, + "response_time": 2.5, + "response": {"id": "response"}, + } + + +def test_a_failed_call_carries_error_information_instead_of_a_response() -> None: + assert PORT.payload(joined("call.failed"), UNUSED_NOW) == { + **COMMON, + "id": "call-123", + "stream": True, + "status": "failure", + "endTime": 11.0, + "response_time": 1.0, + "error_str": "bad request", + "error_information": {"error_class": "ValueError", "error_message": "bad request", "error_code": "400"}, + } + + +@pytest.mark.parametrize( + ("log_format", "bodies"), + [ + ("json_array", ['[{"id": "a"}, {"id": "b"}]']), + ("ndjson", ['{"id": "a"}\n{"id": "b"}']), + ("single", ['{"id": "a"}', '{"id": "b"}']), + ], +) +def test_a_batch_is_framed_by_log_format(log_format: generic_api.LogFormat, bodies: list[str]) -> None: + port = generic_api.GenericApi(generic_api.Config(endpoint="http://generic.test", log_format=log_format)) + batch = ({"id": "a"}, {"id": "b"}) + + sent = port.deliveries(batch) # pyright: ignore[reportArgumentType] # framing does not depend on the record's fields + + assert [delivery.body.decode() for delivery in sent] == bodies + assert {delivery.url for delivery in sent} == {"http://generic.test"} + + +def test_an_outcome_left_out_of_event_types_has_no_record() -> None: + assert FAILURES_ONLY.payload(joined("call.succeeded"), UNUSED_NOW) is None + assert FAILURES_ONLY.payload(joined("call.failed"), UNUSED_NOW) is not None + + +def test_event_types_filter_what_is_queued() -> None: + from tests.test_litellm.callbacks_v1.builtin.support import FakeTransport + + transport = FakeTransport() + sink = Sink("generic_api", FAILURES_ONLY, transport) + for terminal in ("call.succeeded", "call.failed"): + for envelope in golden_call(terminal): + sink.on_event(envelope) + + assert sink.outbox.flush() + assert [record["status"] for body in transport.bodies() for record in body] == ["failure"] # pyright: ignore[reportGeneralTypeIssues, reportIndexIssue] # json bodies are lists of records here + + +def test_config_parses_the_legacy_header_variable() -> None: + config = generic_api.Config.from_env( + {"GENERIC_LOGGER_ENDPOINT": "http://e", "GENERIC_LOGGER_HEADERS": "a=1, b = 2"} + ) + assert config.headers == (("Content-Type", "application/json"), ("a", "1"), ("b", "2")) + assert json.loads(json.dumps(config.headers)) == [["Content-Type", "application/json"], ["a", "1"], ["b", "2"]] diff --git a/tests/test_litellm/callbacks_v1/builtin/test_openmeter.py b/tests/test_litellm/callbacks_v1/builtin/test_openmeter.py new file mode 100644 index 00000000000..791b0cab915 --- /dev/null +++ b/tests/test_litellm/callbacks_v1/builtin/test_openmeter.py @@ -0,0 +1,54 @@ +import pytest + +from litellm.callbacks_v1.builtin import openmeter +from litellm.callbacks_v1.builtin.port import CallRecord +from litellm.callbacks_v1.builtin.runtime import CallJoin +from tests.test_litellm.callbacks_v1.builtin.support import FIXED_NOW, golden_call + +PORT = openmeter.OpenMeter(openmeter.Config(api_key="test-key", endpoint="http://openmeter.test/")) + + +def joined(terminal: str, **metadata: str) -> CallRecord: + join = CallJoin() + records = [join.accept(envelope) for envelope in golden_call(terminal, **metadata)] # pyright: ignore[reportArgumentType] # terminal is an EventName literal at every call site + (record,) = [record for record in records if record is not None] + return record + + +def test_a_succeeded_call_becomes_one_cloud_event() -> None: + assert PORT.payload(joined("call.succeeded", user_api_key_user_id="user-1"), FIXED_NOW) == { + "specversion": "1.0", + "type": "litellm_tokens", + "id": "response", + "time": "2026-01-02T03:04:05+00:00", + "subject": "user-1", + "source": "litellm-proxy", + "data": {"model": "model", "cost": None}, + } + + +def test_a_failed_call_is_not_metered() -> None: + assert PORT.payload(joined("call.failed", user_api_key_user_id="user-1"), FIXED_NOW) is None + + +def test_a_call_without_a_subject_is_an_error_like_the_legacy_logger() -> None: + with pytest.raises(ValueError, match="user is required"): + PORT.payload(joined("call.succeeded"), FIXED_NOW) + + +def test_each_event_is_its_own_authorised_request() -> None: + (delivery,) = PORT.deliveries(({"id": "a"},)) # pyright: ignore[reportArgumentType] # framing does not depend on the event's fields + + assert delivery.url == "http://openmeter.test/api/v1/events" + assert dict(delivery.headers) == { + "Content-Type": "application/cloudevents+json", + "Authorization": "Bearer test-key", + } + assert delivery.body == b'{"id": "a"}' + + +def test_config_reads_the_legacy_environment_variables() -> None: + config = openmeter.Config.from_env({"OPENMETER_API_KEY": "k", "OPENMETER_API_ENDPOINT": "http://host/"}) + assert (config.api_key, config.url, config.event_type) == ("k", "http://host/api/v1/events", "litellm_tokens") + with pytest.raises(ValueError, match="OPENMETER_API_KEY"): + openmeter.Config.from_env({}) diff --git a/tests/test_litellm/callbacks_v1/builtin/test_outbox.py b/tests/test_litellm/callbacks_v1/builtin/test_outbox.py new file mode 100644 index 00000000000..3da4683fa28 --- /dev/null +++ b/tests/test_litellm/callbacks_v1/builtin/test_outbox.py @@ -0,0 +1,94 @@ +import threading +import time + +from litellm.callbacks_v1.builtin.outbox import Outbox +from litellm.callbacks_v1.builtin.runtime import CallJoin +from tests.test_litellm.callbacks_v1.builtin.support import golden, golden_call + + +def test_put_returns_before_a_slow_delivery_finishes() -> None: + release = threading.Event() + delivered: list[tuple[int, ...]] = [] + + def deliver(batch: tuple[int, ...]) -> None: + release.wait(2) + delivered.append(batch) + + outbox = Outbox("slow", deliver) + started = time.monotonic() + outbox.put(1) + + assert time.monotonic() - started < 0.5 + assert delivered == [] + release.set() + assert outbox.flush() + assert delivered == [(1,)] + + +def test_items_are_delivered_batch_size_at_a_time_and_flush_sends_the_rest() -> None: + delivered: list[tuple[int, ...]] = [] + outbox = Outbox("batched", delivered.append, batch_size=2, flush_interval=60) + + for item in (1, 2, 3): + outbox.put(item) + + assert outbox.flush() + assert delivered == [(1, 2), (3,)] + + +def test_a_failed_delivery_is_dropped_and_the_next_one_still_goes_out() -> None: + delivered: list[tuple[int, ...]] = [] + + def deliver(batch: tuple[int, ...]) -> None: + if batch == (1,): + raise RuntimeError("vendor is down") + delivered.append(batch) + + outbox = Outbox("flaky", deliver) + outbox.put(1) + outbox.put(2) + + assert outbox.flush() + assert delivered == [(2,)] + + +def test_a_full_outbox_drops_the_new_item_instead_of_blocking_the_call() -> None: + release = threading.Event() + outbox = Outbox("full", lambda batch: release.wait(2) and None, capacity=1) + outbox.put(1) + time.sleep(0.05) + outbox.put(2) + + started = time.monotonic() + outbox.put(3) + + assert time.monotonic() - started < 0.5 + release.set() + assert outbox.flush() + + +def test_the_join_returns_one_record_at_the_terminal_envelope_and_forgets_the_call() -> None: + join = CallJoin() + started, request, received, terminal = golden_call("call.succeeded") + + assert [join.accept(envelope) for envelope in (started, request, received)] == [None, None, None] + record = join.accept(terminal) + + assert record is not None + assert (record.call_id, record.call_type) == ("call-123", "ocr") + assert (record.started, record.request, record.terminal) == (started["event"], request["event"], terminal["event"]) + assert join.open_calls() == 0 + + +def test_a_call_that_failed_before_sending_has_no_request() -> None: + record = CallJoin().accept(golden("call.failed")) + assert record is not None and record.request is None and record.started is None + + +def test_calls_that_never_finish_are_bounded() -> None: + join = CallJoin(capacity=2) + started = golden("call.started") + for call_id in ("a", "b", "c"): + join.accept({**started, "call_id": call_id}) + + assert join.open_calls() == 2 diff --git a/tests/test_litellm/callbacks_v1/builtin/test_parity_diff.py b/tests/test_litellm/callbacks_v1/builtin/test_parity_diff.py new file mode 100644 index 00000000000..9789d6ed968 --- /dev/null +++ b/tests/test_litellm/callbacks_v1/builtin/test_parity_diff.py @@ -0,0 +1,34 @@ +"""The parity diff itself: what it reports is what keeps the allow-lists and the gap ledger honest.""" + +from types import MappingProxyType +from typing import Final + +from litellm.callbacks_v1.builtin.manifest import Gap +from tests.test_litellm.callbacks_v1.builtin.support import Allowed, unexplained + +COST: Final = Gap("fact", "cost", "kwargs['response_cost']", "no cost fact") +LEGACY: Final = MappingProxyType({"id": "call-1", "data": MappingProxyType({"cost": 0.25, "model": "claude"})}) +PORT: Final = MappingProxyType({"id": "call-1", "data": MappingProxyType({"cost": None, "model": "claude"})}) + + +def test_identical_bodies_need_no_allowance() -> None: + assert unexplained(LEGACY, LEGACY, (), ()) == () + + +def test_a_difference_nobody_allowed_is_reported_by_path() -> None: + assert unexplained(LEGACY, PORT, (), ()) == ("differs: data.cost: legacy 0.25 != port None",) + + +def test_a_difference_allowed_by_an_open_gap_is_parity() -> None: + assert unexplained(LEGACY, PORT, (Allowed("data.cost", "no cost fact", gap="cost"),), (COST,)) == () + + +def test_an_allowance_outlives_neither_its_difference_nor_its_gap() -> None: + allowed: Final = (Allowed("data.cost", "no cost fact", gap="cost"),) + + assert unexplained(LEGACY, LEGACY, allowed, (COST,)) == ( + "stale Allowed('data.cost'): it no longer matches any difference", + ) + assert unexplained(LEGACY, PORT, allowed, ()) == ( + "Allowed('data.cost') cites gap 'cost', which the port does not list", + ) diff --git a/tests/test_litellm/callbacks_v1/test_host.py b/tests/test_litellm/callbacks_v1/test_host.py new file mode 100644 index 00000000000..206fdda3f8c --- /dev/null +++ b/tests/test_litellm/callbacks_v1/test_host.py @@ -0,0 +1,299 @@ +"""The extension host worker, driven over a socketpair the way a gateway would drive it.""" + +import asyncio +import os +import socket +import subprocess +import sys +import threading +from collections.abc import Generator +from pathlib import Path +from types import MappingProxyType + +import pytest + +from litellm import callbacks_v1 +from litellm.callbacks_v1 import EnvelopeV1, RequestFactsV1, WirePatchV1 +from litellm.callbacks_v1.host import INTERCEPT_THREADS, ProtocolError, Worker, encode_frame, read_frame +from tests.test_litellm.callbacks_v1.builtin.support import golden + + +@pytest.fixture(autouse=True) +def clean_registry() -> Generator[None]: + yield + for subscriber in callbacks_v1.snapshot(): + callbacks_v1.unregister(subscriber) + + +class Gateway: + """The gateway's end of the socket: starts a worker on a thread and speaks frames to it.""" + + def __init__(self) -> None: + self.sock, worker_sock = socket.socketpair() + self.sock.settimeout(5) + self.thread = threading.Thread(target=Worker(worker_sock, callbacks_v1.snapshot()).serve, daemon=True) + self.thread.start() + self.hello = self.read() + + def send(self, **frame: object) -> None: + self.sock.sendall(encode_frame(frame)) + + def read(self) -> dict: + frame = read_frame(self.sock) + assert frame is not None + return dict(frame) + + def flush(self) -> list[dict]: + """Everything the worker said before it confirmed the flush.""" + self.send(type="flush", id="f") + frames = [] + while (frame := self.read()) != {"type": "flushed", "id": "f"}: + frames.append(frame) + return frames + + def close(self) -> None: + self.sock.close() + self.thread.join(5) + assert not self.thread.is_alive() + + +def test_hello_reports_every_subscription_as_data() -> None: + @callbacks_v1.on_event("call.succeeded", "call.failed", name="recorder") + def record(event: EnvelopeV1) -> None: ... + + @callbacks_v1.before_send + async def patch(request: RequestFactsV1) -> WirePatchV1 | None: ... + + gateway = Gateway() + + assert gateway.hello == { + "type": "hello", + "schema": 1, + "subscriptions": [ + {"name": "recorder", "events": ["call.failed", "call.succeeded"], "observes": True, "intercepts": False}, + {"name": f"{__name__}.{patch.__qualname__}", "events": [], "observes": False, "intercepts": True}, + ], + } + gateway.close() + + +def test_one_subscriber_gets_its_envelopes_in_the_order_they_were_sent() -> None: + seen: list[int] = [] + + @callbacks_v1.on_event("response.received", name="ordered") + async def record(event: EnvelopeV1) -> None: + seen.append(event["seq"]) + + gateway = Gateway() + for seq in range(50): + gateway.send(type="event", subscriber=0, envelope={**golden("response.received"), "seq": seq}) + + assert gateway.flush() == [] + assert seen == list(range(50)) + gateway.close() + + +def test_an_observer_that_raises_is_reported_and_the_next_envelope_still_arrives() -> None: + seen: list[str] = [] + + @callbacks_v1.on_event("call.started", "call.failed", name="flaky") + def record(event: EnvelopeV1) -> None: + seen.append(event["event"]["type"]) + if event["event"]["type"] == "call.started": + raise ValueError("boom") + + gateway = Gateway() + gateway.send(type="event", subscriber=0, envelope=golden("call.started")) + gateway.send(type="event", subscriber=0, envelope=golden("call.failed")) + + assert gateway.flush() == [{"type": "report", "subscriber": 0, "event": "call.started", "message": "boom"}] + assert seen == ["call.started", "call.failed"] + gateway.close() + + +def test_an_interceptor_answers_with_its_patch_even_from_a_read_only_view() -> None: + @callbacks_v1.before_send + def patch(request: RequestFactsV1) -> WirePatchV1 | None: + return {"body": MappingProxyType({"model": request["model"], "blocks": ({"a": 1},)})} + + gateway = Gateway() + gateway.send(type="intercept", id=7, subscriber=0, request=golden("request.sending")["event"]) + + assert gateway.read() == {"type": "patch", "id": 7, "patch": {"body": {"model": "model", "blocks": [{"a": 1}]}}} + gateway.close() + + +def test_an_interceptor_that_raises_is_an_error_for_the_gateway_to_act_on() -> None: + @callbacks_v1.before_send + async def patch(request: RequestFactsV1) -> WirePatchV1 | None: + raise PermissionError("not this model") + + gateway = Gateway() + gateway.send(type="intercept", id="a", subscriber=0, request=golden("request.sending")["event"]) + + assert gateway.read() == { + "type": "error", + "id": "a", + "error_class": "builtins.PermissionError", + "message": "not this model", + } + gateway.close() + + +def test_an_interception_does_not_wait_behind_a_slow_observer() -> None: + release = threading.Event() + + @callbacks_v1.on_event("call.started", name="slow") + def slow(event: EnvelopeV1) -> None: + release.wait(5) + + @callbacks_v1.before_send + def patch(request: RequestFactsV1) -> WirePatchV1 | None: + return None + + gateway = Gateway() + gateway.send(type="event", subscriber=0, envelope=golden("call.started")) + gateway.send(type="intercept", id=1, subscriber=1, request=golden("request.sending")["event"]) + + assert gateway.read() == {"type": "patch", "id": 1, "patch": None} + release.set() + gateway.close() + + +def test_every_async_interception_is_in_flight_at_once() -> None: + """Concurrency is the loop's, not a pool size: an async handler that only waits holds no + thread, so far more interceptions run together than there are intercept threads. Awaiting + each other is how they prove it — none of them can finish until all of them have started.""" + burst = INTERCEPT_THREADS * 4 + started = 0 + everyone: asyncio.Event | None = None + + @callbacks_v1.before_send + async def patch(request: RequestFactsV1) -> WirePatchV1 | None: + nonlocal started, everyone + if everyone is None: # every handler runs on the one loop, so this races with nothing + everyone = asyncio.Event() + started += 1 + if started == burst: + everyone.set() + await asyncio.wait_for(everyone.wait(), 3) + return None + + gateway = Gateway() + for frame_id in range(burst): + gateway.send(type="intercept", id=frame_id, subscriber=0, request=golden("request.sending")["event"]) + + answers = sorted((gateway.read() for _ in range(burst)), key=lambda frame: frame["id"]) + assert answers == [{"type": "patch", "id": frame_id, "patch": None} for frame_id in range(burst)] + gateway.close() + + +def test_a_frame_the_protocol_does_not_know_stops_the_worker() -> None: + sock, worker_sock = socket.socketpair() + worker = Worker(worker_sock, ()) + sock.sendall(encode_frame({"type": "shell", "command": "id"})) + + with pytest.raises(ProtocolError, match="unknown frame type"): + worker.serve() + sock.close() + + +CALLBACK_MODULE = """ +import os +from litellm.callbacks_v1 import before_send + +@before_send +def leak(request): + return {"body": {"env": sorted(os.environ), "cwd_files": sorted(os.listdir("."))}} +""" + + +def test_a_spawned_worker_sees_only_the_environment_it_was_given(tmp_path: Path) -> None: + modules = tmp_path / "modules" + modules.mkdir() + (modules / "leaky.py").write_text(CALLBACK_MODULE) + empty = tmp_path / "empty" + empty.mkdir() + gateway_sock, worker_sock = socket.socketpair() + gateway_sock.settimeout(60) + repo = Path(__file__).parents[3] + given = {"PYTHONPATH": os.pathsep.join((str(repo), str(modules))), "LITELLM_MODE": "PRODUCTION"} + + process = subprocess.Popen( + [sys.executable, "-m", "litellm.callbacks_v1.host", "--fd", str(worker_sock.fileno()), "--load", "leaky"], + pass_fds=(worker_sock.fileno(),), + env=given, + cwd=empty, + ) + worker_sock.close() + try: + hello = read_frame(gateway_sock) + assert hello is not None and [sub["name"] for sub in hello["subscriptions"]] == ["leaky.leak"] # pyright: ignore[reportIndexIssue, reportGeneralTypeIssues] # the hello frame's subscriptions are a list of objects + gateway_sock.sendall(encode_frame({"type": "intercept", "id": 1, "subscriber": 0, "request": {}})) + answer = read_frame(gateway_sock) + finally: + gateway_sock.close() + assert process.wait(30) == 0 + + assert answer is not None + leaked = answer["patch"]["body"] # pyright: ignore[reportIndexIssue, reportOptionalSubscript] # the patch frame carries the callback's body + assert set(leaked["env"]) - {"LC_CTYPE", "__CF_USER_TEXT_ENCODING"} <= set(given) + assert leaked["cwd_files"] == [] + + +GOLDEN_FRAMES = Path(__file__).parents[3] / "litellm-rust/crates/callbacks-v1-exthost/golden" + + +def golden_frame(name: str) -> dict: + import json + + return json.loads((GOLDEN_FRAMES / name).read_text()) + + +def test_the_worker_answers_the_golden_frames_rust_sends_with_the_golden_frames_rust_parses() -> None: + """Both sides are pinned to one directory: Rust serialises `to_worker` and parses + `from_worker`; here the worker is fed the first and must produce the shape of the second.""" + seen: list[object] = [] + + @callbacks_v1.on_event("call.started", name="recorder") + def record(event: EnvelopeV1) -> None: + seen.append(event) + raise ValueError("boom") + + @callbacks_v1.before_send + def patch(request: RequestFactsV1) -> WirePatchV1 | None: + seen.append(request) + return {"headers": {"remove": ["X-Trace"], "set": [("X-Team", "core")]}, "body": {"input": "patched"}} + + event, intercept, flush = (golden_frame(f"to_worker/{name}.json") for name in ("event", "intercept", "flush")) + gateway = Gateway() + assert set(gateway.hello) == set(golden_frame("from_worker/hello.json")) + assert {key for sub in gateway.hello["subscriptions"] for key in sub} == { + key for sub in golden_frame("from_worker/hello.json")["subscriptions"] for key in sub + } + + gateway.send(**intercept) + assert gateway.read() == golden_frame("from_worker/patch.json") + gateway.send(**event) + gateway.send(**flush) + assert gateway.read() == golden_frame("from_worker/report.json") + assert gateway.read() == golden_frame("from_worker/flushed.json") + assert seen == [intercept["request"], event["envelope"]] + gateway.close() + + +def test_the_authoring_surface_and_the_worker_import_nothing_but_the_standard_library() -> None: + """A worker that could run without the rest of litellm stays possible only while these two + files need nothing from it. `typing_extensions` is the one third-party import.""" + import ast + + allowed = set(sys.stdlib_module_names) | {"typing_extensions", "litellm.callbacks_v1"} + for path in ("litellm/callbacks_v1/__init__.py", "litellm/callbacks_v1/host.py"): + tree = ast.parse((Path(__file__).parents[3] / path).read_text()) + imported = { + alias.name if isinstance(node, ast.Import) else str(node.module) + for node in ast.walk(tree) + if isinstance(node, (ast.Import, ast.ImportFrom)) + for alias in node.names + } + assert {name for name in imported if name not in allowed and name.split(".")[0] not in allowed} == set(), path diff --git a/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py b/tests/test_litellm/rust_bridge/test_callbacks_legacy.py similarity index 95% rename from tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py rename to tests/test_litellm/rust_bridge/test_callbacks_legacy.py index cb3c6358fd0..10e69b56501 100644 --- a/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py +++ b/tests/test_litellm/rust_bridge/test_callbacks_legacy.py @@ -10,8 +10,8 @@ from pydantic import TypeAdapter import litellm from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.rust_bridge import callbacks_legacy_python as callbacks -from litellm.rust_bridge.callbacks_legacy_python import check_limits, setup +from litellm.rust_bridge import callbacks_legacy as callbacks +from litellm.rust_bridge.callbacks_legacy import check_limits, setup _OCR_KWARGS: Final = MappingProxyType( { diff --git a/tests/test_litellm/rust_bridge/test_callbacks_v1.py b/tests/test_litellm/rust_bridge/test_callbacks_v1.py new file mode 100644 index 00000000000..9b0edea9b32 --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_callbacks_v1.py @@ -0,0 +1,31 @@ +import inspect +from pathlib import Path +from typing import Final + +import pytest +from pydantic import BaseModel, TypeAdapter + +from litellm.rust_bridge import callbacks_v1 as callbacks + +ROOT: Final = Path(__file__).parents[3] +CONTRACT_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1-python/python_contract.json" + + +def test_the_rust_contract_matches_the_shim_signatures() -> None: + contract: Final = TypeAdapter(dict[str, list[str]]).validate_json(CONTRACT_PATH.read_text()) + assert contract == {name: list(inspect.signature(getattr(callbacks, name)).parameters) for name in contract} + + +class ResponseModel(BaseModel): + value: int + + +def test_project_response_supports_pydantic_and_json_values() -> None: + assert callbacks.project_response(ResponseModel(value=3)) == {"value": 3} + original: Final = {"nested": [1, True, None]} + assert callbacks.project_response(original) is original + + +def test_project_response_rejects_unsupported_objects() -> None: + with pytest.raises(TypeError): + callbacks.project_response(object()) diff --git a/tests/test_litellm/rust_bridge/test_callbacks_v1_python.py b/tests/test_litellm/rust_bridge/test_callbacks_v1_python.py deleted file mode 100644 index f854ea4b121..00000000000 --- a/tests/test_litellm/rust_bridge/test_callbacks_v1_python.py +++ /dev/null @@ -1,178 +0,0 @@ -import inspect -import warnings -from collections.abc import Generator -from pathlib import Path -from typing import Final, cast - -import pytest -from pydantic import BaseModel, TypeAdapter - -from litellm.rust_bridge import callbacks_v1_python as callbacks - -ROOT: Final = Path(__file__).parents[3] -CONTRACT_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1-python/python_contract.json" -GOLDEN_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1/golden/v1" - - -def _clear_registry() -> None: - for subscriber in callbacks.snapshot(): - callbacks.unregister(cast(callbacks.Callback, subscriber)) - - -@pytest.fixture(autouse=True) -def clear_registry() -> Generator[None, None, None]: - _clear_registry() - yield - _clear_registry() - - -def test_the_rust_contract_matches_the_shim_signatures() -> None: - contract: Final = TypeAdapter(dict[str, list[str]]).validate_json(CONTRACT_PATH.read_text()) - assert contract == {name: list(inspect.signature(getattr(callbacks, name)).parameters) for name in contract} - - -def test_v1_golden_envelopes_match_the_typed_contract() -> None: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", message="Items .*ReadOnly.*", category=UserWarning) - adapter: Final = TypeAdapter(callbacks.EnvelopeV1) - envelopes: Final = [adapter.validate_json(path.read_text(), strict=True) for path in GOLDEN_PATH.glob("*.json")] - assert {envelope["event"]["type"] for envelope in envelopes} == callbacks.EVENTS - assert {envelope["schema"] for envelope in envelopes} == callbacks.SUPPORTED_SCHEMAS - - -class ValidCallback: - name: str = "valid" - schema: int = 1 - events: frozenset[str] = frozenset({"call.started"}) - - def on_event(self, event: callbacks.EnvelopeV1) -> None: - del event - - -@pytest.mark.parametrize( - ("callback", "error"), - [ - ( - type( - "Unsupported", - (), - {"name": "unsupported", "schema": 2, "events": frozenset(), "before_send": lambda *_: None}, - )(), - callbacks.UnsupportedSchema, - ), - ( - type( - "Unknown", - (), - {"name": "unknown", "schema": 1, "events": frozenset({"new.event"}), "on_event": lambda *_: None}, - )(), - callbacks.UnknownEvent, - ), - ( - type("Missing", (), {"name": "missing", "schema": 1, "events": frozenset()})(), - callbacks.MissingHandler, - ), - ( - type("Empty", (), {"name": "empty", "schema": 1, "events": frozenset(), "on_event": lambda *_: None})(), - callbacks.EmptyObserverEvents, - ), - ( - type( - "AsyncFunction", - (), - { - "name": "async-function", - "schema": 1, - "events": frozenset({"call.started"}), - "async_on_event": lambda *_: None, - }, - )(), - callbacks.HandlerKindMismatch, - ), - ], - ids=["schema", "event", "missing", "empty-events", "async-kind"], -) -def test_register_rejects_invalid_callbacks(callback: object, error: type[ValueError]) -> None: - with pytest.raises(error): - callbacks.register(callback) # pyright: ignore[reportArgumentType] # invalid callback shapes are the subject of the test - - -class WrongSyncKind: - name: str = "wrong-sync-kind" - schema: int = 1 - events: frozenset[str] = frozenset({"call.started"}) - - async def on_event(self, event: callbacks.EnvelopeV1) -> None: - del event - - -def test_register_rejects_coroutine_in_sync_slot() -> None: - with pytest.raises(callbacks.HandlerKindMismatch): - callbacks.register(cast(callbacks.Callback, WrongSyncKind())) - - -class AsyncObserver: - name: str = "async-observer" - schema: int = 1 - events: frozenset[str] = frozenset({"call.started"}) - - async def async_on_event(self, event: callbacks.EnvelopeV1) -> None: - del event - - -class SyncInterceptor: - name: str = "sync-interceptor" - schema: int = 1 - events: frozenset[str] = frozenset() - - def before_send(self, request: callbacks.RequestFactsV1) -> callbacks.WirePatchV1 | None: - del request - return None - - -class AsyncInterceptor: - name: str = "async-interceptor" - schema: int = 1 - events: frozenset[str] = frozenset() - - async def async_before_send(self, request: callbacks.RequestFactsV1) -> callbacks.WirePatchV1 | None: - del request - return None - - -@pytest.mark.parametrize("callback", [ValidCallback(), AsyncObserver(), SyncInterceptor(), AsyncInterceptor()]) -def test_register_accepts_each_handler_family(callback: callbacks.Callback) -> None: - callbacks.register(callback) - assert len(callbacks.snapshot()) == 1 - - -def test_register_rejects_duplicate_names() -> None: - callbacks.register(cast(callbacks.Callback, ValidCallback())) - with pytest.raises(callbacks.DuplicateName): - callbacks.register(cast(callbacks.Callback, ValidCallback())) - - -def test_snapshot_is_stable_across_later_registry_writes() -> None: - first: Final = ValidCallback() - second: Final = AsyncObserver() - callbacks.register(cast(callbacks.Callback, first)) - captured: Final = callbacks.snapshot() - callbacks.register(cast(callbacks.Callback, second)) - callbacks.unregister(cast(callbacks.Callback, first)) - assert callbacks.snapshot() is not captured - assert tuple(subscriber.name for subscriber in captured) == ("valid",) - - -class ResponseModel(BaseModel): - value: int - - -def test_project_response_supports_pydantic_and_json_values() -> None: - assert callbacks.project_response(ResponseModel(value=3)) == {"value": 3} - original: Final = {"nested": [1, True, None]} - assert callbacks.project_response(original) is original - - -def test_project_response_rejects_unsupported_objects() -> None: - with pytest.raises(TypeError): - callbacks.project_response(object()) diff --git a/tests/test_litellm/test_callbacks_v1.py b/tests/test_litellm/test_callbacks_v1.py new file mode 100644 index 00000000000..7c2a35619f7 --- /dev/null +++ b/tests/test_litellm/test_callbacks_v1.py @@ -0,0 +1,236 @@ +import warnings +from collections.abc import Generator +from pathlib import Path +from typing import Final, get_args + +import pytest +from pydantic import TypeAdapter + +from litellm import callbacks_v1 +from litellm.callbacks_v1 import EnvelopeV1, EventName, RequestFactsV1, SchemaVersion, WirePatchV1 + +ROOT: Final = Path(__file__).parents[2] +GOLDEN_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1/golden/v1" + + +def _clear_registry() -> None: + for subscriber in callbacks_v1.snapshot(): + callbacks_v1.unregister(subscriber) + + +@pytest.fixture(autouse=True) +def clear_registry() -> Generator[None, None, None]: + _clear_registry() + yield + _clear_registry() + + +def test_the_event_names_and_schemas_are_the_literal_aliases() -> None: + assert callbacks_v1.EVENTS == frozenset(get_args(EventName)) + assert callbacks_v1.SUPPORTED_SCHEMAS == frozenset(get_args(SchemaVersion)) + + +def test_v1_golden_envelopes_match_the_typed_contract() -> None: + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="Items .*ReadOnly.*", category=UserWarning) + adapter: Final = TypeAdapter(callbacks_v1.EnvelopeV1) + envelopes: Final = [adapter.validate_json(path.read_text(), strict=True) for path in GOLDEN_PATH.glob("*.json")] + assert {envelope["event"]["type"] for envelope in envelopes} == callbacks_v1.EVENTS + assert {envelope["schema"] for envelope in envelopes} == callbacks_v1.SUPPORTED_SCHEMAS + + +class SyncObserver: + name: str = "valid" + events: frozenset[EventName] = frozenset({"call.started"}) + + def on_event(self, event: callbacks_v1.EnvelopeV1) -> None: + del event + + +class AsyncObserver: + name: str = "async-observer" + events: frozenset[EventName] = frozenset({"call.started"}) + + async def async_on_event(self, event: callbacks_v1.EnvelopeV1) -> None: + del event + + +class SyncInterceptor: + name: str = "sync-interceptor" + events: frozenset[EventName] = frozenset() + + def before_send(self, request: callbacks_v1.RequestFactsV1) -> callbacks_v1.WirePatchV1 | None: + del request + return None + + +class AsyncInterceptor: + name: str = "async-interceptor" + events: frozenset[EventName] = frozenset() + + async def async_before_send(self, request: callbacks_v1.RequestFactsV1) -> callbacks_v1.WirePatchV1 | None: + del request + return None + + +class WrongSyncKind: + name: str = "wrong-sync-kind" + events: frozenset[EventName] = frozenset({"call.started"}) + + async def on_event(self, event: callbacks_v1.EnvelopeV1) -> None: + del event + + +@pytest.mark.parametrize( + ("callback", "error"), + [ + ( + type( + "Unsupported", + (), + {"name": "unsupported", "schema": 2, "events": frozenset(), "before_send": lambda *_: None}, + )(), + callbacks_v1.UnsupportedSchema, + ), + ( + type( + "Unknown", + (), + {"name": "unknown", "schema": 1, "events": frozenset({"new.event"}), "on_event": lambda *_: None}, + )(), + callbacks_v1.UnknownEvent, + ), + ( + type("Missing", (), {"name": "missing", "schema": 1, "events": frozenset()})(), + callbacks_v1.MissingHandler, + ), + ( + type("Empty", (), {"name": "empty", "schema": 1, "events": frozenset(), "on_event": lambda *_: None})(), + callbacks_v1.EmptyObserverEvents, + ), + ( + type( + "AsyncFunction", + (), + { + "name": "async-function", + "schema": 1, + "events": frozenset({"call.started"}), + "async_on_event": lambda *_: None, + }, + )(), + callbacks_v1.HandlerKindMismatch, + ), + (WrongSyncKind(), callbacks_v1.HandlerKindMismatch), + ], + ids=["schema", "event", "missing", "empty-events", "async-kind", "coroutine-in-sync-slot"], +) +def test_register_rejects_invalid_callbacks(callback: object, error: type[ValueError]) -> None: + with pytest.raises(error): + callbacks_v1.register(callback) # pyright: ignore[reportArgumentType] # invalid callback shapes are the subject of the test + + +@pytest.mark.parametrize("callback", [SyncObserver(), AsyncObserver(), SyncInterceptor(), AsyncInterceptor()]) +def test_register_accepts_each_handler_family(callback: callbacks_v1.Callback) -> None: + callbacks_v1.register(callback) + assert len(callbacks_v1.snapshot()) == 1 + + +def test_register_rejects_duplicate_names() -> None: + callbacks_v1.register(SyncObserver()) + with pytest.raises(callbacks_v1.DuplicateName): + callbacks_v1.register(SyncObserver()) + + +def test_snapshot_is_stable_across_later_registry_writes() -> None: + first: Final = SyncObserver() + second: Final = AsyncObserver() + callbacks_v1.register(first) + captured: Final = callbacks_v1.snapshot() + callbacks_v1.register(second) + callbacks_v1.unregister(first) + assert callbacks_v1.snapshot() is not captured + assert tuple(subscriber.name for subscriber in captured) == ("valid",) + + +def test_register_defaults_the_schema_when_the_callback_declares_none() -> None: + callbacks_v1.register(SyncObserver()) + assert callbacks_v1.snapshot()[0].schema == callbacks_v1.SCHEMA + + +def test_register_still_rejects_a_declared_foreign_schema() -> None: + class Foreign(SyncObserver): + name: str = "foreign" + schema: int = 2 + + with pytest.raises(callbacks_v1.UnsupportedSchema): + callbacks_v1.register(Foreign()) + + +def test_on_event_registers_a_sync_function_in_the_sync_slot() -> None: + @callbacks_v1.on_event("call.started", "call.succeeded") + def record(event: EnvelopeV1) -> None: + del event + + subscriber: Final = callbacks_v1.snapshot()[0] + assert subscriber.on_event is record + assert subscriber.async_on_event is None + assert subscriber.events == frozenset({"call.started", "call.succeeded"}) + assert subscriber.name.endswith("record") + assert record( # the function is returned unchanged, so it stays directly callable + {"schema": 1, "call_id": "c", "call_type": "ocr", "seq": 0, "event": {"type": "response.received", "body": ""}} + ) is None + + +def test_on_event_routes_a_coroutine_function_to_the_async_slot() -> None: + @callbacks_v1.on_event("call.failed", name="named-observer") + async def record(event: EnvelopeV1) -> None: + del event + + subscriber: Final = callbacks_v1.snapshot()[0] + assert subscriber.async_on_event is record + assert subscriber.on_event is None + assert subscriber.name == "named-observer" + + +def test_before_send_routes_by_kind_and_subscribes_to_no_events() -> None: + @callbacks_v1.before_send + def sync_patch(request: RequestFactsV1) -> WirePatchV1 | None: + del request + return None + + @callbacks_v1.before_send + async def async_patch(request: RequestFactsV1) -> WirePatchV1 | None: + del request + return None + + first, second = callbacks_v1.snapshot() + assert (first.before_send, first.async_before_send) == (sync_patch, None) + assert (second.before_send, second.async_before_send) == (None, async_patch) + assert first.events == second.events == frozenset() + + +def test_on_event_rejects_an_unknown_event_at_decoration() -> None: + with pytest.raises(callbacks_v1.UnknownEvent): + + @callbacks_v1.on_event("call.startd") # pyright: ignore[reportArgumentType] # the typo is the subject of the test + def record(event: EnvelopeV1) -> None: + del event + + +def test_on_event_rejects_an_empty_event_set_at_decoration() -> None: + with pytest.raises(callbacks_v1.EmptyObserverEvents): + + @callbacks_v1.on_event() + def record(event: EnvelopeV1) -> None: + del event + + +def test_unregister_accepts_a_bare_name() -> None: + @callbacks_v1.on_event("call.started", name="by-name") + def record(event: EnvelopeV1) -> None: + del event + + assert len(callbacks_v1.snapshot()) == 1 + callbacks_v1.unregister("by-name") + assert callbacks_v1.snapshot() == () diff --git a/tests/test_litellm_rust/builtin/__init__.py b/tests/test_litellm_rust/builtin/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm_rust/builtin/test_parity.py b/tests/test_litellm_rust/builtin/test_parity.py new file mode 100644 index 00000000000..c9b5cc3c83a --- /dev/null +++ b/tests/test_litellm_rust/builtin/test_parity.py @@ -0,0 +1,215 @@ +"""Differential parity: one native call reaches a legacy logger and its v1 port, each posting +to its own recording vendor, and every difference between the two vendor bodies is `Allowed`. + +The native chain runs the legacy contract and then v1 on the same call, so nothing is +replayed or mocked: both sides see the one real call. Registration happens in fixtures only. +""" + +from collections.abc import Generator +from datetime import datetime, timezone +from typing import Final + +import pytest + +import litellm +from litellm import callbacks_v1 +from litellm.callbacks_v1.builtin import anthropic_cache_control, generic_api, openmeter +from litellm.callbacks_v1.builtin.manifest import entry +from litellm.callbacks_v1.builtin.runtime import HttpxTransport, Patcher, Sink +from litellm.integrations.generic_api.generic_api_callback import GenericAPILogger +from litellm.integrations.openmeter import OpenMeterLogger +from tests.test_litellm.callbacks_v1.builtin.support import Allowed, assert_parity +from tests.test_litellm_rust.support.callback_recorder import drain_logging +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec, recording_service +from tests.test_litellm_rust.support.requests import MESSAGES, MESSAGES_MODEL, MESSAGES_RESPONSE + +pytestmark = pytest.mark.requires_rust_extension + +METADATA: Final = {"user_api_key_user_id": "user-1"} + + +@pytest.fixture +def provider(recording_server: RecordingServer) -> RecordingServer: + recording_server.default_response = ResponseSpec(body=MESSAGES_RESPONSE) + return recording_server + + +@pytest.fixture +def legacy_vendor() -> Generator[RecordingServer]: + with recording_service() as server: + yield server + + +@pytest.fixture +def port_vendor() -> Generator[RecordingServer]: + with recording_service() as server: + yield server + + +@pytest.fixture(autouse=True) +def clean_registry() -> Generator[None]: + yield + for subscriber in callbacks_v1.snapshot(): + callbacks_v1.unregister(subscriber) + + +def arguments(server: RecordingServer, **kwargs: object) -> dict[str, object]: + # Identity rides in `metadata` here because call.started projects only that keyword; the proxy + # sends it as `litellm_metadata` on this route, which v1 does not see yet (see entry("openmeter").gaps). + return { + "model": MESSAGES_MODEL, + "messages": [dict(message) for message in MESSAGES], + "max_tokens": 64, + "api_key": "test-key", + "api_base": server.base_url, + "metadata": METADATA, + **kwargs, + } + + +OPENMETER_ALLOWED: Final = ( + Allowed("time", "each side stamps its own clock"), + Allowed("data.cost", "no cost fact on call.succeeded", gap="cost"), + Allowed("data.*_tokens", "no usage fact; legacy counts tokens off the OpenAI-normalised response", gap="usage"), +) + + +@pytest.mark.asyncio +async def test_openmeter_meters_a_native_call_like_its_legacy_twin( + provider: RecordingServer, + legacy_vendor: RecordingServer, + port_vendor: RecordingServer, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OPENMETER_API_KEY", "test-key") + monkeypatch.setenv("OPENMETER_API_ENDPOINT", legacy_vendor.base_url) + port: Final = openmeter.OpenMeter(openmeter.Config(api_key="test-key", endpoint=port_vendor.base_url)) + sink: Final = Sink("openmeter", port, HttpxTransport(), lambda: datetime.now(timezone.utc)) + callbacks_v1.register(sink) + + await litellm.anthropic.messages.acreate(**arguments(provider, callbacks=[OpenMeterLogger()])) + await drain_logging() + assert sink.outbox.flush() + + (legacy_request,) = legacy_vendor.requests + (port_request,) = port_vendor.requests + assert port_request.path == legacy_request.path == "/api/v1/events" + assert port_request.headers["authorization"] == legacy_request.headers["authorization"] + assert port_request.headers["content-type"] == legacy_request.headers["content-type"] + assert_parity(legacy_request.raw_body, port_request.raw_body, OPENMETER_ALLOWED, entry("openmeter").gaps) + + +# What the legacy StandardLoggingPayload has and v1 facts cannot fill yet, on any outcome. +GENERIC_API_GAP_ALLOWED: Final = ( + Allowed("0.stream", "legacy leaves stream null on a non-streaming call, v1 always says false"), + Allowed("0.completionStartTime", "no first-token time in timing", gap="first_token_time"), + Allowed("0.response_cost*", "no cost facts", gap="cost"), + Allowed("0.cost_breakdown*", "no cost facts", gap="cost"), + Allowed("0.saved_cache_cost", "no cost facts", gap="cost"), + Allowed("0.autorouter_savings", "no cost facts", gap="cost"), + Allowed("0.*_tokens", "no usage fact", gap="usage"), + Allowed("0.cache_*", "no cache fact", gap="cache"), + Allowed("0.model_id", "no routing fact", gap="routing"), + Allowed("0.model_group", "no routing fact", gap="routing"), + Allowed("0.model_map_information.*", "no routing fact", gap="routing"), + Allowed("0.api_base", "no routing fact", gap="routing"), + Allowed("0.metadata.*", "no identity fact", gap="identity"), + Allowed("0.end_user", "no identity fact", gap="identity"), + Allowed("0.requester_ip_address", "no identity fact", gap="identity"), + Allowed("0.user_agent", "no identity fact", gap="identity"), + Allowed("0.request_tags", "no tags fact", gap="tags"), + Allowed("0.request_model_access_groups", "no tags fact", gap="tags"), + Allowed("0.trace_id", "no correlation beyond call_id", gap="correlation"), + Allowed("0.session_id", "no correlation beyond call_id", gap="correlation"), + Allowed("0.hidden_params.*", "no equivalent", gap="no_equivalent"), + Allowed("0.guardrail_information", "no equivalent", gap="no_equivalent"), + Allowed("0.standard_built_in_tools_params.*", "no equivalent", gap="no_equivalent"), + Allowed("0.status_fields.*", "derived from guardrail_information, which v1 has no fact for", gap="no_equivalent"), + Allowed("0.messages.*", "no route-neutral prompt projection", gap="prompt"), + Allowed( + "0.model_parameters.metadata.*", + "this test sends identity as `metadata`, which Anthropic also accepts; legacy keeps it out of model_parameters", + ), +) + +GENERIC_API_ALLOWED: Final = ( + *GENERIC_API_GAP_ALLOWED, + Allowed( + "0.response.*", + "legacy logs the OpenAI-normalised response, v1 carries the route's own", + gap="normalised_response", + ), + Allowed("0.error_str", "legacy sends null on success, the port omits the key"), + Allowed("0.error_information.*", "legacy sends an empty shell on success, the port omits the key"), +) + + +@pytest.mark.asyncio +async def test_generic_api_logs_a_native_call_like_its_legacy_twin( + provider: RecordingServer, legacy_vendor: RecordingServer, port_vendor: RecordingServer +) -> None: + legacy: Final = GenericAPILogger(endpoint=legacy_vendor.base_url, batch_size=1) + port: Final = generic_api.GenericApi(generic_api.Config(endpoint=port_vendor.base_url, batch_size=1)) + sink: Final = Sink("generic_api", port, HttpxTransport()) + callbacks_v1.register(sink) + + await litellm.anthropic.messages.acreate(**arguments(provider, callbacks=[legacy])) + await drain_logging() + assert sink.outbox.flush() + + (legacy_request,) = legacy_vendor.requests + (port_request,) = port_vendor.requests + assert port_request.headers["content-type"] == legacy_request.headers["content-type"] + assert_parity(legacy_request.raw_body, port_request.raw_body, GENERIC_API_ALLOWED, entry("generic_api").gaps) + + +GENERIC_API_FAILURE_ALLOWED: Final = ( + *GENERIC_API_GAP_ALLOWED, + Allowed("0.response", "legacy sends an empty object on failure, the port omits the key"), + Allowed("0.error_information.error_budget_*", "ErrorFacts is class/message/status only", gap="error_detail"), + Allowed("0.error_information.error_rate_limit_*", "ErrorFacts is class/message/status only", gap="error_detail"), + Allowed( + "0.error_information.error_provider_request_id", "ErrorFacts is class/message/status only", gap="error_detail" + ), + Allowed("0.error_information.llm_provider", "ErrorFacts is class/message/status only", gap="error_detail"), + Allowed("0.error_information.traceback", "ErrorFacts is class/message/status only", gap="error_detail"), +) + + +@pytest.mark.asyncio +async def test_generic_api_logs_a_failed_native_call_like_its_legacy_twin( + provider: RecordingServer, legacy_vendor: RecordingServer, port_vendor: RecordingServer +) -> None: + provider.enqueue( + ResponseSpec(body={"type": "error", "error": {"type": "invalid_request_error", "message": "bad"}}, status=400) + ) + legacy: Final = GenericAPILogger(endpoint=legacy_vendor.base_url, batch_size=1) + port: Final = generic_api.GenericApi(generic_api.Config(endpoint=port_vendor.base_url, batch_size=1)) + sink: Final = Sink("generic_api", port, HttpxTransport()) + callbacks_v1.register(sink) + + with pytest.raises(litellm.BadRequestError): + await litellm.anthropic.messages.acreate(**arguments(provider, callbacks=[legacy])) + await drain_logging() + assert sink.outbox.flush() + + (legacy_request,) = legacy_vendor.requests + (port_request,) = port_vendor.requests + assert_parity( + legacy_request.raw_body, port_request.raw_body, GENERIC_API_FAILURE_ALLOWED, entry("generic_api").gaps + ) + + +def test_the_cache_control_patch_survives_the_contract_and_reaches_the_provider(provider: RecordingServer) -> None: + config: Final = anthropic_cache_control.Config( + points=(anthropic_cache_control.InjectionPoint(role="system"), anthropic_cache_control.InjectionPoint(index=-1)) + ) + callbacks_v1.register(Patcher("anthropic_cache_control", anthropic_cache_control.AnthropicCacheControl(config))) + + litellm.anthropic.messages.create(**arguments(provider, system="be brief")) + + sent: Final = provider.requests[-1].body + assert isinstance(sent, dict) + assert sent["system"] == [{"type": "text", "text": "be brief", "cache_control": {"type": "ephemeral"}}] + assert sent["messages"][-1]["content"][-1]["cache_control"] == {"type": "ephemeral"} + assert sent["model"] == MESSAGES_MODEL.split("/", 1)[1] diff --git a/tests/test_litellm_rust/messages/test_callbacks_v1.py b/tests/test_litellm_rust/messages/test_callbacks_v1.py new file mode 100644 index 00000000000..ecb16b02a61 --- /dev/null +++ b/tests/test_litellm_rust/messages/test_callbacks_v1.py @@ -0,0 +1,146 @@ +from collections.abc import AsyncIterator, Generator +from typing import Final + +import pytest + +import litellm +from litellm import callbacks_v1 +from litellm.callbacks_v1 import EVENTS, EnvelopeV1, EventName, RequestFactsV1, WirePatchV1 +from litellm.integrations.custom_logger import CustomLogger +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec +from tests.test_litellm_rust.support.requests import ( + MESSAGES, + MESSAGES_EVENTS, + MESSAGES_MODEL, + MESSAGES_RESPONSE, +) + +pytestmark = pytest.mark.requires_rust_extension + +STREAM: Final = ResponseSpec(body=None, events=MESSAGES_EVENTS) + + +@pytest.fixture +def messages_server(recording_server: RecordingServer) -> RecordingServer: + recording_server.default_response = ResponseSpec(body=MESSAGES_RESPONSE) + return recording_server + + +def arguments(server: RecordingServer, **kwargs: object) -> dict[str, object]: + return { + "model": MESSAGES_MODEL, + "messages": [dict(message) for message in MESSAGES], + "max_tokens": 64, + "api_key": "test-key", + "api_base": server.base_url, + **kwargs, + } + + +class BothObserver: + """Declares both kinds; an async call must select the async one and never the sync one.""" + + name: Final = "both-observer" + schema: Final = 1 + events: Final = EVENTS + + def __init__(self) -> None: + self.envelopes: Final[list[EnvelopeV1]] = [] + self.sync_calls: Final[list[EnvelopeV1]] = [] + + def on_event(self, event: EnvelopeV1) -> None: + self.sync_calls.append(event) + + async def async_on_event(self, event: EnvelopeV1) -> None: + self.envelopes.append(event) + + +class AsyncInterceptor: + name: Final = "async-interceptor" + schema: Final = 1 + events: Final[frozenset[EventName]] = frozenset() + + def __init__(self) -> None: + self.requests: Final[list[RequestFactsV1]] = [] + + async def async_before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: + self.requests.append(request) + return {"headers": {"set": [("x-v1-async-interceptor", "seen")]}} + + +@pytest.fixture +def observer() -> Generator[BothObserver]: + subscriber: Final = BothObserver() + callbacks_v1.register(subscriber) + try: + yield subscriber + finally: + callbacks_v1.unregister(subscriber) + + +@pytest.fixture +def interceptor() -> Generator[AsyncInterceptor]: + subscriber: Final = AsyncInterceptor() + callbacks_v1.register(subscriber) + try: + yield subscriber + finally: + callbacks_v1.unregister(subscriber) + + +class LegacyCallId(CustomLogger): + def __init__(self) -> None: + self.call_ids: Final[list[str]] = [] + + def log_pre_api_call(self, model, messages, kwargs): + self.call_ids.append(kwargs["litellm_call_id"]) + + +@pytest.mark.asyncio +async def test_native_messages_async_call_selects_the_async_observer_and_shares_the_legacy_call_id( + messages_server: RecordingServer, observer: BothObserver +) -> None: + legacy: Final = LegacyCallId() + + await litellm.anthropic.messages.acreate(**arguments(messages_server, callbacks=[legacy])) + + assert observer.sync_calls == [] + assert [envelope["event"]["type"] for envelope in observer.envelopes] == [ + "call.started", + "request.sending", + "response.received", + "call.succeeded", + ] + assert [envelope["seq"] for envelope in observer.envelopes] == [0, 1, 2, 3] + assert {envelope["call_type"] for envelope in observer.envelopes} == {"anthropic_messages"} + assert len(legacy.call_ids) == 1 + assert {envelope["call_id"] for envelope in observer.envelopes} == set(legacy.call_ids) + + +@pytest.mark.asyncio +async def test_native_messages_async_interceptor_patch_reaches_the_provider( + messages_server: RecordingServer, interceptor: AsyncInterceptor +) -> None: + await litellm.anthropic.messages.acreate(**arguments(messages_server)) + + assert len(interceptor.requests) == 1 + request: Final = interceptor.requests[0] + assert request["custom_llm_provider"] == "anthropic" + assert request["headers"] != [] + assert messages_server.requests[-1].headers["x-v1-async-interceptor"] == "seen" + + +@pytest.mark.asyncio +async def test_native_messages_stream_marks_the_terminal_envelope_streamed( + messages_server: RecordingServer, observer: BothObserver +) -> None: + messages_server.enqueue(STREAM) + + stream: Final = await litellm.anthropic.messages.acreate(**arguments(messages_server, stream=True)) + assert isinstance(stream, AsyncIterator) + delivered: Final = [chunk async for chunk in stream] + + assert delivered + terminal: Final = observer.envelopes[-1] + assert terminal["event"]["type"] == "call.succeeded" + assert terminal["event"]["streamed"] is True diff --git a/tests/test_litellm_rust/ocr/test_callbacks_v1.py b/tests/test_litellm_rust/ocr/test_callbacks_v1.py index 3da44a6c394..505e2a9e391 100644 --- a/tests/test_litellm_rust/ocr/test_callbacks_v1.py +++ b/tests/test_litellm_rust/ocr/test_callbacks_v1.py @@ -3,9 +3,9 @@ from typing import Final import pytest +from litellm import callbacks_v1 +from litellm.callbacks_v1 import EVENTS, EnvelopeV1, EventName, RequestFactsV1, WirePatchV1 from litellm.integrations.custom_logger import CustomLogger -from litellm.rust_bridge import callbacks_v1_python -from litellm.rust_bridge.callbacks_v1_python import EVENTS, EnvelopeV1, RequestFactsV1, WirePatchV1 from tests.test_litellm_rust.support.callback_recorder import drain_logging from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec from tests.test_litellm_rust.support.requests import ( @@ -39,7 +39,7 @@ class Observer: class Interceptor: name: Final = "interceptor" schema: Final = 1 - events: Final[frozenset[str]] = frozenset() + events: Final[frozenset[EventName]] = frozenset() def __init__(self) -> None: self.bodies: Final[list[object]] = [] @@ -52,21 +52,21 @@ class Interceptor: @pytest.fixture def observer() -> Generator[Observer]: subscriber: Final = Observer() - callbacks_v1_python.register(subscriber) + callbacks_v1.register(subscriber) try: yield subscriber finally: - callbacks_v1_python.unregister(subscriber) + callbacks_v1.unregister(subscriber) @pytest.fixture def interceptor() -> Generator[Interceptor]: subscriber: Final = Interceptor() - callbacks_v1_python.register(subscriber) + callbacks_v1.register(subscriber) try: yield subscriber finally: - callbacks_v1_python.unregister(subscriber) + callbacks_v1.unregister(subscriber) class LegacyCallId(CustomLogger): @@ -139,3 +139,28 @@ def test_native_ocr_failure_reaches_both_contracts_once( ] assert [envelope["event"]["type"] for envelope in terminal] == ["call.failed"] assert observer.envelopes[-1] is terminal[0] + + +def test_native_ocr_decorated_function_callbacks_run_through_the_native_call( + ocr_server: RecordingServer, +) -> None: + seen: Final[list[str]] = [] + + @callbacks_v1.on_event("call.started", "call.succeeded", name="decorated-observer") + def record(event: EnvelopeV1) -> None: + seen.append(event["event"]["type"]) + + @callbacks_v1.before_send + def patch(request: RequestFactsV1) -> WirePatchV1 | None: + del request + return {"headers": {"set": [("x-v1-decorated", "seen")]}} + + try: + call_native_ocr(ocr_server) + finally: + callbacks_v1.unregister("decorated-observer") + callbacks_v1.unregister(f"{patch.__module__}.{patch.__qualname__}") + + assert seen == ["call.started", "call.succeeded"] + assert ocr_server.requests[-1].headers["x-v1-decorated"] == "seen" + assert callbacks_v1.snapshot() == ()