feat(model-catalog): bind native refresher to Python

This commit is contained in:
Devin AI 2026-10-26 03:31:27 +00:00
parent 2cff68eee5
commit d3d33019a9
7 changed files with 1061 additions and 5 deletions

View file

@ -4309,6 +4309,7 @@ dependencies = [
name = "litellm-python-bridge"
version = "0.1.0"
dependencies = [
"async-trait",
"aws-sdk-secretsmanager",
"bytes",
"criterion",
@ -4335,6 +4336,7 @@ dependencies = [
"litellm-llms",
"litellm-llms-types",
"litellm-model-catalog",
"litellm-model-catalog-refresh",
"litellm-router-types",
"litellm-secrets",
"litellm-secrets-aws",

View file

@ -19,12 +19,14 @@ huggingface = ["litellm-token-counter/huggingface"]
tiktoken = ["litellm-token-counter/tiktoken"]
[dependencies]
async-trait = "0.1.89"
fancy-regex.workspace = true
litellm-tracing.workspace = true
litellm-spend-clickhouse.workspace = true
litellm-storage-clickhouse.workspace = true
litellm-host.workspace = true
litellm-model-catalog.workspace = true
litellm-model-catalog-refresh.workspace = true
litellm-cost.workspace = true
time.workspace = true
bytes.workspace = true

View file

@ -32,10 +32,11 @@ mod _native {
use crate::logger::NativeDiagnosticProcessor;
#[pymodule_export]
use crate::model_registry::{
ModelCostLayers, ModelInfoView, ModelNotFound, ModelRegistry, PriceLine, Priced,
RegistrationTarget, ResolvedModel, RowError, RowErrors, Unpriced, UsageInput, WhenInput,
invalidate_model_cost_index, model_info, model_row_errors, price, registration_target,
resolve_model, write_rows,
CatalogRefresher, FetchedRegistry, ModelCostLayers, ModelInfoView, ModelNotFound,
ModelRegistry, PriceLine, Priced, RegistrationTarget, RegistryNotModified,
RegistryUnavailable, ResolvedModel, RowError, RowErrors, Unpriced, UsageInput, WhenInput,
fetch_once, invalidate_model_cost_index, model_info, model_row_errors, price,
registration_target, resolve_model, write_rows,
};
#[pymodule_export]
use crate::routes::audio_transcription::{atranscription, transcription};
@ -123,6 +124,11 @@ assert live == {"model": {"litellm_provider": "openai", "input_cost_per_token":
Python::attach(|py| {
let mut expected = vec![
"NativeCacheHandle",
"CatalogRefresher",
"FetchedRegistry",
"RegistryNotModified",
"RegistryUnavailable",
"fetch_once",
"ModelRegistry",
"ModelCostLayers",
"RowError",

View file

@ -1,6 +1,8 @@
mod live;
mod refresh;
pub(crate) use live::*;
pub(crate) use refresh::*;
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::sync::{Arc, LazyLock, Mutex, PoisonError};
@ -102,6 +104,10 @@ pub(crate) struct RowError {
#[derive(Clone)]
pub(crate) struct ModelRegistry {
pub(crate) parsed: Arc<ParsedRegistry>,
#[pyo3(get)]
pub(crate) source_revision: Option<String>,
#[pyo3(get)]
pub(crate) source_validator: Option<String>,
}
#[pymethods]
@ -117,6 +123,8 @@ impl ModelRegistry {
.map_err(|error| PyValueError::new_err(error.to_string()))?;
Ok(Self {
parsed: Arc::new(parsed),
source_revision: None,
source_validator: None,
})
}
@ -127,6 +135,8 @@ impl ModelRegistry {
catalog: Arc::clone(&BUNDLED.catalog),
errors: Arc::clone(&BUNDLED.errors),
}),
source_revision: None,
source_validator: None,
})
}
@ -196,6 +206,8 @@ impl ModelRegistry {
catalog: Arc::new(catalog),
errors: Arc::clone(&self.parsed.errors),
}),
source_revision: self.source_revision.clone(),
source_validator: self.source_validator.clone(),
},
removed_errors,
)

View file

@ -0,0 +1,476 @@
use std::{
sync::{
Arc, Weak,
atomic::{AtomicBool, Ordering},
},
time::Duration,
};
use async_trait::async_trait;
use litellm_host_python::release_gil;
use litellm_model_catalog::{
FetchRequest, Installed, Origin, Reply, RetryPolicy, Revalidation, Step,
};
use litellm_model_catalog_refresh::{Adopted, Checked, Fetch, Install, Refresher, Source, Stale};
use pyo3::{
PyTraverseError, PyVisit,
exceptions::{PyRuntimeError, PyTypeError, PyValueError},
prelude::*,
types::PyDict,
};
use super::{BUNDLED, ModelRegistry, ParsedRegistry, RowError, row_error};
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct FetchedRegistry {
#[pyo3(get)]
registry: ModelRegistry,
#[pyo3(get)]
revision: String,
#[pyo3(get)]
validator: Option<String>,
}
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct RegistryNotModified;
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct RegistryUnavailable {
#[pyo3(get)]
reason: String,
#[pyo3(get)]
retryable: bool,
#[pyo3(get)]
retry_after: Option<f64>,
}
#[derive(IntoPyObject)]
enum FetchResult {
Fetched(FetchedRegistry),
NotModified(RegistryNotModified),
Unavailable(RegistryUnavailable),
}
fn installed(revision: Option<String>, validator: Option<String>, source: &Source) -> Installed {
Installed {
catalog: Arc::clone(&BUNDLED.catalog),
origin: if revision.is_some() {
Origin::Remote {
source: source.id(),
}
} else {
Origin::Bundled
},
revision: revision.unwrap_or_default(),
validator,
}
}
fn unavailable(reason: String, retryable: bool, after: Option<Duration>) -> FetchResult {
FetchResult::Unavailable(RegistryUnavailable {
reason,
retryable,
retry_after: after.map(|delay| delay.as_secs_f64()),
})
}
#[pyfunction]
#[pyo3(signature = (source, *, if_none_match, timeout_seconds))]
pub(crate) fn fetch_once(
py: Python<'_>,
source: &str,
if_none_match: Option<String>,
timeout_seconds: f64,
) -> PyResult<Py<PyAny>> {
let source = Source::parse(source).map_err(|error| PyValueError::new_err(error.to_string()))?;
let timeout = Duration::try_from_secs_f64(timeout_seconds)
.map_err(|_| PyValueError::new_err("timeout_seconds must be finite and non-negative"))?;
if tokio::runtime::Handle::try_current().is_ok() {
return Err(PyRuntimeError::new_err(
"fetch_once cannot run from a Tokio context",
));
}
let http = crate::http::call_config(py, &PyDict::new(py), false)?;
let result = release_gil(py, move || {
let fetch = match source.fetcher(&http) {
Ok(fetch) => fetch,
Err(_) => {
return unavailable(
"model catalog transport configuration failed".into(),
false,
None,
);
}
};
let runtime = match tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
{
Ok(runtime) => runtime,
Err(_) => return unavailable("model catalog runtime unavailable".into(), false, None),
};
let reply = runtime.block_on(fetch.fetch(FetchRequest {
if_none_match,
timeout,
}));
runtime.shutdown_background();
match reply {
Reply::Retry { after, reason } => unavailable(reason, true, after),
Reply::Failed { reason } => unavailable(reason, false, None),
Reply::NotModified => FetchResult::NotModified(RegistryNotModified),
reply @ Reply::Fetched { .. } => {
let (mut state, _) = Revalidation::new(
source.id(),
&installed(None, None, &source),
RetryPolicy {
max_attempts: 1,
timeout,
..RetryPolicy::default()
},
None,
);
match state.on_reply(reply) {
Step::Adopt { installed, errors } => FetchResult::Fetched(FetchedRegistry {
registry: ModelRegistry {
parsed: Arc::new(ParsedRegistry {
catalog: installed.catalog,
errors: Arc::from(errors),
}),
source_revision: Some(installed.revision.clone()),
source_validator: installed.validator.clone(),
},
revision: installed.revision,
validator: installed.validator,
}),
Step::Done { reason } => unavailable(reason.unwrap_or_default(), false, None),
Step::Fetch(_) | Step::Sleep(_) => {
unreachable!("a fetched reply never retries")
}
}
}
}
});
Ok(result.into_pyobject(py)?.unbind())
}
struct PythonInstall {
install: Py<PyAny>,
checked: Py<PyAny>,
generation: Py<PyAny>,
stopped: Arc<AtomicBool>,
}
struct CallbackFetch {
inner: Arc<dyn Fetch>,
stopped: Arc<AtomicBool>,
}
#[async_trait]
impl Fetch for CallbackFetch {
async fn fetch(&self, request: FetchRequest) -> Reply {
if self.stopped.load(Ordering::Acquire) {
return Reply::Failed {
reason: "model catalog callbacks stopped".into(),
};
}
self.inner.fetch(request).await
}
}
impl PythonInstall {
fn call<T>(&self, call: impl FnOnce(Python<'_>) -> PyResult<T>) -> Option<T> {
Python::try_attach(|py| {
if self.stopped.load(Ordering::Acquire) {
return None;
}
match call(py) {
Ok(value) => Some(value),
Err(error) => {
self.stopped.store(true, Ordering::Release);
error.write_unraisable(py, None);
None
}
}
})
.flatten()
}
}
impl Install for PythonInstall {
fn generation(&self) -> u64 {
self.call(|py| self.generation.call0(py)?.extract(py))
.unwrap_or_default()
}
fn install(&self, seen: u64, adopted: Adopted) -> Result<u64, Stale> {
let registry = ModelRegistry {
parsed: Arc::new(ParsedRegistry {
catalog: adopted.installed.catalog,
errors: Arc::from([]),
}),
source_revision: Some(adopted.installed.revision),
source_validator: adopted.installed.validator,
};
let left_out: Vec<RowError> = adopted.left_out.iter().map(row_error).collect();
self.call(|py| {
self.install
.call1(py, (seen, registry, left_out))?
.extract::<Option<u64>>(py)
})
.flatten()
.ok_or(Stale)
}
fn checked(&self, outcome: Checked) {
let reason = match outcome {
Checked::NotModified => None,
Checked::Failed { reason } => Some(reason),
};
self.call(|py| self.checked.call1(py, (reason,)).map(|_| ()));
}
}
#[pyclass(module = "litellm.rust_bridge._native")]
pub(crate) struct CatalogRefresher {
inner: Option<Refresher>,
callbacks: Weak<PythonInstall>,
}
#[pymethods]
impl CatalogRefresher {
#[new]
#[pyo3(signature = (source, *, installed_revision, installed_validator, install, checked, generation))]
fn new(
py: Python<'_>,
source: &str,
installed_revision: Option<String>,
installed_validator: Option<String>,
install: Py<PyAny>,
checked: Py<PyAny>,
generation: Py<PyAny>,
) -> PyResult<Self> {
let source =
Source::parse(source).map_err(|error| PyValueError::new_err(error.to_string()))?;
if !install.bind(py).is_callable()
|| !checked.bind(py).is_callable()
|| !generation.bind(py).is_callable()
{
return Err(PyTypeError::new_err(
"install, checked and generation must be callable",
));
}
let http = crate::http::call_config(py, &PyDict::new(py), false)?;
let logger = crate::logger::capture(py);
let callbacks = Arc::new(PythonInstall {
install,
checked,
generation,
stopped: Arc::new(AtomicBool::new(false)),
});
let weak = Arc::downgrade(&callbacks);
let inner = release_gil(py, move || {
let fetch = Arc::new(CallbackFetch {
inner: source.fetcher(&http).map_err(crate::http::client_error)?,
stopped: Arc::clone(&callbacks.stopped),
});
let installed = installed(installed_revision, installed_validator, &source);
Ok::<_, PyErr>(
logger.scope(|| Refresher::start(source.id(), fetch, &installed, callbacks)),
)
})?;
Ok(Self {
inner: Some(inner),
callbacks: weak,
})
}
fn stop(&self) {
if let Some(callbacks) = self.callbacks.upgrade() {
callbacks.stopped.store(true, Ordering::Release);
}
if let Some(inner) = &self.inner {
inner.stop();
}
}
fn after_fork(&self, py: Python<'_>) {
if self
.callbacks
.upgrade()
.is_some_and(|callbacks| callbacks.stopped.load(Ordering::Acquire))
{
return;
}
if let Some(inner) = &self.inner {
release_gil(py, || inner.after_fork());
}
}
fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> {
if let Some(callbacks) = self.callbacks.upgrade() {
// A running worker owns these edges independently of this Python object.
if Arc::strong_count(&callbacks) == 2 {
visit.call(&callbacks.install)?;
visit.call(&callbacks.checked)?;
visit.call(&callbacks.generation)?;
}
}
Ok(())
}
fn __clear__(&mut self) {
self.stop();
self.inner.take();
}
}
impl Drop for CatalogRefresher {
fn drop(&mut self) {
self.stop();
}
}
#[cfg(test)]
mod tests {
use std::ffi::CString;
use pyo3::types::PyList;
use rstest::{fixture, rstest};
use super::*;
#[fixture]
fn python() {
Python::initialize();
}
fn callbacks(py: Python<'_>, install_result: &str) -> (PythonInstall, Py<PyList>) {
let locals = PyDict::new(py);
py.run(&CString::new(format!(
"events = []\ndef install(seen, registry, errors):\n events.append((seen, registry, errors))\n return {install_result}\ndef checked(reason):\n events.append(reason)\ndef generation():\n return 7"
)).unwrap(), Some(&locals), None).unwrap();
let callback = |name| locals.get_item(name).unwrap().unwrap().unbind();
let result = PythonInstall {
install: callback("install"),
checked: callback("checked"),
generation: callback("generation"),
stopped: Arc::new(AtomicBool::new(false)),
};
(
result,
locals
.get_item("events")
.unwrap()
.unwrap()
.cast_into::<PyList>()
.unwrap()
.unbind(),
)
}
#[rstest]
#[case::adopted("8", Ok(8))]
#[case::stale("None", Err(Stale))]
fn install_return_maps_to_generation_or_stale(
#[from(python)] _python: (),
#[case] answer: &str,
#[case] expected: Result<u64, Stale>,
) {
Python::attach(|py| {
let (callbacks, events) = callbacks(py, answer);
let source = Source::parse("https://example.test/catalog").unwrap();
assert_eq!(callbacks.generation(), 7);
let adopted = Adopted {
installed: installed(Some("revision".into()), Some("validator".into()), &source),
left_out: vec![],
};
assert_eq!(callbacks.install(7, adopted), expected);
let event = events.bind(py).get_item(0).unwrap();
assert_eq!(event.get_item(0).unwrap().extract::<u64>().unwrap(), 7);
let registry = event
.get_item(1)
.unwrap()
.extract::<PyRef<'_, ModelRegistry>>()
.unwrap();
assert!(Arc::ptr_eq(&registry.parsed.catalog, &BUNDLED.catalog));
assert_eq!(registry.source_revision.as_deref(), Some("revision"));
assert_eq!(registry.source_validator.as_deref(), Some("validator"));
assert!(
event
.get_item(2)
.unwrap()
.cast::<PyList>()
.unwrap()
.is_empty()
);
});
}
#[rstest]
#[case::not_modified(Checked::NotModified, None)]
#[case::failed(Checked::Failed { reason: "offline".into() }, Some("offline"))]
fn checked_maps_reason(
#[from(python)] _python: (),
#[case] outcome: Checked,
#[case] expected: Option<&str>,
) {
Python::attach(|py| {
let (callbacks, events) = callbacks(py, "8");
callbacks.checked(outcome);
assert_eq!(
events
.bind(py)
.get_item(0)
.unwrap()
.extract::<Option<String>>()
.unwrap()
.as_deref(),
expected
);
});
}
#[rstest]
fn stopped_callbacks_are_not_invoked(#[from(python)] _python: ()) {
Python::attach(|py| {
let (callbacks, events) = callbacks(py, "8");
callbacks.stopped.store(true, Ordering::Release);
assert_eq!(callbacks.generation(), 0);
callbacks.checked(Checked::NotModified);
assert!(events.bind(py).is_empty());
});
}
#[rstest]
#[case::unsupported_scheme("ftp://example.test")]
#[case::relative("catalog.json")]
fn invalid_source_precedes_python_config(#[from(python)] _python: (), #[case] source: &str) {
Python::attach(|py| {
let error = fetch_once(py, source, None, 1.0).unwrap_err();
assert!(error.is_instance_of::<PyValueError>(py));
});
}
#[rstest]
#[case::negative(-1.0)]
#[case::nan(f64::NAN)]
#[case::infinite(f64::INFINITY)]
fn invalid_timeout_precedes_python_config(#[from(python)] _python: (), #[case] timeout: f64) {
Python::attach(|py| {
let error = fetch_once(py, "https://example.test", None, timeout).unwrap_err();
assert!(error.is_instance_of::<PyValueError>(py));
});
}
#[rstest]
fn nested_runtime_is_rejected(#[from(python)] _python: ()) {
let runtime = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
let _entered = runtime.enter();
Python::attach(|py| {
let error = fetch_once(py, "https://example.test", None, 1.0).unwrap_err();
assert!(error.is_instance_of::<PyRuntimeError>(py));
});
}
}

View file

@ -0,0 +1,510 @@
from __future__ import annotations
import gc
import hashlib
import json
import logging
import os
import queue
import ssl
import subprocess
import sys
import tempfile
import threading
import time
import unittest
import weakref
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Final
from unittest.mock import patch
import litellm
from litellm.rust_bridge import _native as native
class Response:
def __init__(self, status: int = 200, body: bytes | None = None, hold: bool = False) -> None:
self.status = status
self.body = native.ModelRegistry.bundled_bytes() if body is None else body
self.requests: queue.Queue[tuple[str, str | None]] = queue.Queue()
self.entered = threading.Event()
self.release = threading.Event()
if not hold:
self.release.set()
@contextmanager
def server(response: Response, tls: ssl.SSLContext | None = None) -> Iterator[str]:
class Handler(BaseHTTPRequestHandler):
def do_GET(self) -> None:
response.requests.put((self.path, self.headers.get("If-None-Match")))
response.entered.set()
response.release.wait(15)
self.send_response(response.status)
self.send_header("ETag", '"catalog-v1"')
self.send_header("Retry-After", "0")
self.send_header("Content-Length", str(len(response.body) if response.status == 200 else 0))
self.end_headers()
if response.status == 200:
try:
self.wfile.write(response.body)
except (BrokenPipeError, ConnectionResetError, ssl.SSLError):
pass
def log_message(self, format: str, *args: object) -> None:
pass
with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as http:
if tls is not None:
http.socket = tls.wrap_socket(http.socket, server_side=True)
worker: Final = threading.Thread(target=http.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True)
worker.start()
try:
yield f"{'https' if tls else 'http'}://127.0.0.1:{http.server_port}"
finally:
response.release.set()
http.shutdown()
worker.join(5)
def refresh_threads() -> frozenset[str]:
return frozenset(path.name for path in Path("/proc/self/task").iterdir() if catalog_thread(path))
def catalog_thread(path: Path) -> bool:
try:
return (path / "comm").read_text().startswith("litellm-catalog")
except FileNotFoundError:
return False
def wait_for(predicate: Callable[[], bool], timeout: float = 5) -> None:
deadline: Final = time.monotonic() + timeout
while not predicate():
if time.monotonic() >= deadline:
raise AssertionError("condition did not complete")
threading.Event().wait(0.01)
class Observer:
def __init__(self, accept: bool = True) -> None:
self.value = 7
self.accept = accept
self.events: queue.Queue[tuple[str, object]] = queue.Queue()
self.provenance: tuple[str | None, str | None] = (None, None)
def generation(self) -> int:
return self.value
def install(self, seen: int, registry: native.ModelRegistry, errors: list[native.RowError]) -> int | None:
if self.accept and seen == self.value:
self.provenance = (registry.source_revision, registry.source_validator)
self.value += 1
self.events.put(("install", (seen, registry, errors)))
return self.value
self.events.put(("install", (seen, registry, errors)))
return None
def checked(self, reason: str | None) -> None:
self.events.put(("checked", reason))
def start(self, url: str, validator: str | None = None, revision: str | None = None) -> native.CatalogRefresher:
return native.CatalogRefresher(
url,
installed_revision=revision,
installed_validator=validator,
install=self.install,
checked=self.checked,
generation=self.generation,
)
class CatalogRefreshTests(unittest.TestCase):
def setUp(self) -> None:
self.environment = patch.dict(os.environ, {"NO_PROXY": "*", "no_proxy": "*"})
self.environment.start()
self.addCleanup(self.environment.stop)
self.ssl_verify = litellm.ssl_verify
self.addCleanup(setattr, litellm, "ssl_verify", self.ssl_verify)
self.threads = refresh_threads()
self.assertFalse(native.process_state_started())
def tearDown(self) -> None:
wait_for(lambda: refresh_threads() == self.threads)
self.assertFalse(native.process_state_started())
def fetch(self, url: str, validator: str | None = None, timeout: float = 3) -> object:
return native.fetch_once(url, if_none_match=validator, timeout_seconds=timeout)
def test_fetch_installs_nothing_and_preserves_validator_revision(self) -> None:
response: Final = Response()
original: Final = litellm.model_cost
original_rows: Final = dict(original)
with server(response) as url:
result: Final = self.fetch(url, '"previous"')
self.assertIsInstance(result, native.FetchedRegistry)
assert isinstance(result, native.FetchedRegistry)
self.assertEqual(result.registry.to_dict(), native.ModelRegistry.bundled().to_dict())
self.assertEqual(result.validator, '"catalog-v1"')
self.assertEqual(result.registry.source_revision, result.revision)
self.assertEqual(result.registry.source_validator, result.validator)
filtered, _ = result.registry.without_rejected()
self.assertEqual(filtered.source_revision, result.revision)
self.assertEqual(filtered.source_validator, result.validator)
self.assertEqual(
result.revision,
hashlib.sha1(b"blob " + str(len(response.body)).encode() + b"\0" + response.body).hexdigest(),
)
self.assertEqual(response.requests.get_nowait(), ("/", '"previous"'))
self.assertTrue(response.requests.empty())
self.assertIs(litellm.model_cost, original)
self.assertEqual(litellm.model_cost, original_rows)
def test_parse_and_bundled_have_no_source_metadata(self) -> None:
bundled: Final = native.ModelRegistry.bundled()
parsed: Final = native.ModelRegistry.parse(native.ModelRegistry.bundled_bytes())
self.assertIsNone(bundled.source_revision)
self.assertIsNone(bundled.source_validator)
self.assertIsNone(parsed.source_revision)
self.assertIsNone(parsed.source_validator)
self.assertEqual(bundled.to_dict(), parsed.to_dict())
with self.assertRaises(AttributeError):
setattr(bundled, "source_revision", "not-writable")
def test_not_modified(self) -> None:
response: Final = Response(304)
with server(response) as url:
self.assertIsInstance(self.fetch(url, '"old"'), native.RegistryNotModified)
observer: Final = Observer()
refresher: Final = observer.start(url, '"old"')
self.addCleanup(refresher.stop)
self.assertEqual(observer.events.get(timeout=5), ("checked", None))
wait_for(lambda: refresh_threads() == self.threads)
self.assertTrue(observer.events.empty())
self.assertEqual(response.requests.get_nowait(), ("/", '"old"'))
self.assertEqual(response.requests.get_nowait(), ("/", '"old"'))
def test_refresher_filters_rejected_rows_and_reports_them(self) -> None:
body: Final = json.dumps(
{
**json.loads(native.ModelRegistry.bundled_bytes()),
"p3-invalid": {"litellm_provider": "openai", "input_cost_per_token": -1},
}
).encode()
response: Final = Response(body=body)
with server(response) as url:
observer: Final = Observer()
refresher: Final = observer.start(url)
self.addCleanup(refresher.stop)
kind, payload = observer.events.get(timeout=5)
self.assertEqual(kind, "install")
assert isinstance(payload, tuple)
seen, registry, errors = payload
self.assertEqual(seen, 7)
self.assertNotIn("p3-invalid", registry.to_dict())
self.assertTrue(any(error.key == "p3-invalid" for error in errors))
self.assertGreater(len(registry.to_dict()), 100)
self.assertEqual(
registry.source_revision,
hashlib.sha1(b"blob " + str(len(body)).encode() + b"\0" + body).hexdigest(),
)
self.assertEqual(registry.source_validator, '"catalog-v1"')
self.assertEqual(observer.provenance, (registry.source_revision, registry.source_validator))
wait_for(lambda: refresh_threads() == self.threads)
self.assertTrue(observer.events.empty())
self.assertEqual(response.requests.qsize(), 1)
def test_stale_is_logged_once_without_checked_or_retry(self) -> None:
response: Final = Response()
with self.assertLogs(litellm.verbose_logger, level=logging.DEBUG) as logs, server(response) as url:
observer: Final = Observer(accept=False)
refresher: Final = observer.start(url)
self.addCleanup(refresher.stop)
self.assertEqual(observer.events.get(timeout=5)[0], "install")
wait_for(lambda: refresh_threads() == self.threads)
self.assertTrue(observer.events.empty())
self.assertEqual(response.requests.qsize(), 1)
self.assertTrue(
any(getattr(record, "rust_fields", {}).get("outcome") == "stale" for record in logs.records), logs.output
)
self.assertEqual(observer.provenance, (None, None))
def test_stop_held_response_has_no_callback_and_exits_thread(self) -> None:
response: Final = Response(hold=True)
with server(response) as url:
observer: Final = Observer()
refresher: Final = observer.start(url)
self.addCleanup(refresher.stop)
self.assertTrue(response.entered.wait(5))
before: Final = time.monotonic()
refresher.stop()
self.assertLess(time.monotonic() - before, 0.5)
wait_for(lambda: refresh_threads() == self.threads)
response.release.set()
refresher.stop()
self.assertTrue(observer.events.empty())
def test_new_generation_fences_inflight_result(self) -> None:
response: Final = Response(hold=True)
with server(response) as url:
observer: Final = Observer()
refresher: Final = observer.start(url)
self.addCleanup(refresher.stop)
self.assertTrue(response.entered.wait(5))
observer.value = 8
response.release.set()
wait_for(lambda: refresh_threads() == self.threads)
self.assertTrue(observer.events.empty())
self.assertEqual(observer.provenance, (None, None))
def test_retryable_once_does_not_retry(self) -> None:
response: Final = Response(503)
with server(response) as url:
result: Final = self.fetch(url)
self.assertIsInstance(result, native.RegistryUnavailable)
assert isinstance(result, native.RegistryUnavailable)
self.assertTrue(result.retryable)
self.assertEqual(result.retry_after, 0)
self.assertIn("503", result.reason)
self.assertEqual(response.requests.qsize(), 1)
def test_failure_calls_checked_reason_once(self) -> None:
response: Final = Response(404)
with server(response) as url:
observer: Final = Observer()
refresher: Final = observer.start(url)
self.addCleanup(refresher.stop)
kind, reason = observer.events.get(timeout=5)
self.assertEqual(kind, "checked")
self.assertIn("404", str(reason))
wait_for(lambda: refresh_threads() == self.threads)
self.assertTrue(observer.events.empty())
def test_file_source_ignores_http_validator(self) -> None:
with tempfile.TemporaryDirectory() as directory:
path: Final = Path(directory) / "catalog.json"
path.write_bytes(native.ModelRegistry.bundled_bytes())
result: Final = self.fetch(path.as_uri())
self.assertIsInstance(result, native.FetchedRegistry)
assert isinstance(result, native.FetchedRegistry)
self.assertIsNone(result.validator)
self.assertIsInstance(self.fetch(path.as_uri(), '"old"'), native.FetchedRegistry)
def test_s3_endpoint_and_conditional_header(self) -> None:
response: Final = Response()
with (
server(response) as url,
patch.dict(
os.environ,
{
"AWS_ENDPOINT_URL_S3": url,
"AWS_ACCESS_KEY_ID": "local-test",
"AWS_SECRET_ACCESS_KEY": "local-test",
"AWS_REGION": "us-east-1",
"AWS_EC2_METADATA_DISABLED": "true",
},
),
):
result: Final = self.fetch("s3://bucket/catalog.json", '"old"')
self.assertIsInstance(result, native.FetchedRegistry)
assert isinstance(result, native.FetchedRegistry)
self.assertEqual(result.registry.to_dict(), native.ModelRegistry.bundled().to_dict())
self.assertEqual(response.requests.get_nowait(), ("/bucket/catalog.json?x-id=GetObject", '"old"'))
self.assertTrue(response.requests.empty())
def test_unsupported_source_and_invalid_timeout(self) -> None:
with self.assertRaises(ValueError):
self.fetch("ftp://x")
with self.assertRaises(ValueError):
Observer().start("ftp://x")
with self.assertRaises(ValueError):
self.fetch("http://127.0.0.1", timeout=float("nan"))
def test_timeout_releases_gil_and_performs_one_request(self) -> None:
response: Final = Response(hold=True)
with server(response) as url:
result: Final = self.fetch(url, timeout=0.05)
self.assertTrue(response.entered.is_set())
self.assertIsInstance(result, native.RegistryUnavailable)
self.assertEqual(response.requests.qsize(), 1)
def test_parse_and_integrity_failure(self) -> None:
response: Final = Response(body=b'{"small": {"litellm_provider": "openai"}}')
with server(response) as url:
result: Final = self.fetch(url)
self.assertIsInstance(result, native.RegistryUnavailable)
assert isinstance(result, native.RegistryUnavailable)
self.assertFalse(result.retryable)
self.assertTrue(result.reason)
def test_rejected_document_does_not_advance_provenance(self) -> None:
response: Final = Response(body=b"not-json")
observer: Final = Observer()
with server(response) as url:
refresher: Final = observer.start(url)
self.addCleanup(refresher.stop)
kind, reason = observer.events.get(timeout=5)
self.assertEqual(kind, "checked")
self.assertTrue(reason)
wait_for(lambda: refresh_threads() == self.threads)
self.assertTrue(observer.events.empty())
self.assertEqual(observer.provenance, (None, None))
def test_tls_verify_applies_to_fetch_and_refresher(self) -> None:
with tempfile.TemporaryDirectory() as directory:
key: Final = Path(directory) / "key.pem"
cert: Final = Path(directory) / "cert.pem"
subprocess.run(
[
"openssl",
"req",
"-x509",
"-newkey",
"rsa:2048",
"-nodes",
"-days",
"1",
"-subj",
"/CN=localhost",
"-addext",
"subjectAltName=IP:127.0.0.1",
"-keyout",
str(key),
"-out",
str(cert),
],
check=True,
capture_output=True,
)
context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
context.load_cert_chain(cert, key)
response: Final = Response()
with server(response, context) as url:
litellm.ssl_verify = False
self.assertIsInstance(self.fetch(url), native.FetchedRegistry)
observer: Final = Observer()
refresher: Final = observer.start(url)
self.addCleanup(refresher.stop)
litellm.ssl_verify = True
self.assertEqual(observer.events.get(timeout=5)[0], "install")
failed: Final = self.fetch(url)
self.assertIsInstance(failed, native.RegistryUnavailable)
assert isinstance(failed, native.RegistryUnavailable)
self.assertTrue(failed.reason)
failure_observer: Final = Observer()
failure_refresher: Final = failure_observer.start(url)
self.addCleanup(failure_refresher.stop)
kind, reason = failure_observer.events.get(timeout=15)
self.assertEqual(kind, "checked")
self.assertTrue(reason)
def test_callback_exception_is_unraisable_and_terminal(self) -> None:
failures: Final[queue.Queue[object]] = queue.Queue()
def install(seen: int, registry: native.ModelRegistry, errors: list[native.RowError]) -> int:
raise RuntimeError("callback-test")
response: Final = Response()
observer: Final = Observer()
with patch.object(sys, "unraisablehook", failures.put), server(response) as url:
refresher: Final = native.CatalogRefresher(
url,
installed_revision=None,
installed_validator=None,
install=install,
checked=observer.checked,
generation=observer.generation,
)
self.addCleanup(refresher.stop)
failure: Final = failures.get(timeout=5)
self.assertIn("callback-test", str(getattr(failure, "exc_value")))
wait_for(lambda: refresh_threads() == self.threads)
self.assertTrue(observer.events.empty())
self.assertTrue(failures.empty())
def test_generation_failure_starts_no_transport(self) -> None:
failures: Final[queue.Queue[object]] = queue.Queue()
def generation() -> int:
raise RuntimeError("generation-test")
response: Final = Response()
observer: Final = Observer()
with patch.object(sys, "unraisablehook", failures.put), server(response) as url:
refresher: Final = native.CatalogRefresher(
url,
installed_revision=None,
installed_validator=None,
install=observer.install,
checked=observer.checked,
generation=generation,
)
self.addCleanup(refresher.stop)
failure: Final = failures.get(timeout=5)
self.assertIn("generation-test", str(getattr(failure, "exc_value")))
wait_for(lambda: refresh_threads() == self.threads)
self.assertTrue(response.requests.empty())
self.assertTrue(observer.events.empty())
def test_callback_cycle_collected_after_worker_completes(self) -> None:
finalized: Final[queue.Queue[bool]] = queue.Queue()
class Owner(Observer):
def __init__(self, url: str) -> None:
super().__init__()
self.refresher = self.start(url)
def __del__(self) -> None:
self.refresher.stop()
self.refresher.after_fork()
finalized.put(True)
response: Final = Response(304)
with server(response) as url:
def make_cycle() -> weakref.ReferenceType[Owner]:
owner: Final = Owner(url)
self.assertEqual(owner.events.get(timeout=5), ("checked", None))
return weakref.ref(owner)
reference: Final = make_cycle()
wait_for(lambda: refresh_threads() == self.threads)
gc.collect()
self.assertIsNone(reference())
self.assertTrue(finalized.get(timeout=1))
@unittest.skipUnless(hasattr(os, "fork"), "requires fork")
def test_after_fork_restarts_unadopted_child(self) -> None:
response: Final = Response(hold=True)
with server(response) as url:
observer: Final = Observer()
refresher: Final = observer.start(url)
self.addCleanup(refresher.stop)
self.assertTrue(response.entered.wait(5))
read_fd, write_fd = os.pipe()
pid: Final = os.fork()
if pid == 0:
os.close(read_fd)
try:
refresher.after_fork()
kind, _ = observer.events.get(timeout=10)
refresher.stop()
os.write(write_fd, kind.encode())
finally:
os._exit(0)
os.close(write_fd)
try:
response.release.set()
self.assertEqual(os.read(read_fd, 32), b"install")
self.assertEqual(os.waitpid(pid, 0)[1], 0)
self.assertEqual(observer.events.get(timeout=5)[0], "install")
finally:
os.close(read_fd)
if __name__ == "__main__":
unittest.main()

View file

@ -1,5 +1,5 @@
from asyncio import Future
from collections.abc import Mapping, Sequence
from collections.abc import Callable, Mapping, Sequence
from typing import Literal, Never, final
import httpx
@ -55,6 +55,10 @@ class RowError:
@final
class ModelRegistry:
@property
def source_revision(self) -> str | None: ...
@property
def source_validator(self) -> str | None: ...
@staticmethod
def parse(body: bytes) -> ModelRegistry: ...
@staticmethod
@ -69,6 +73,45 @@ class ModelRegistry:
def check_size(self, reference: int) -> RowError | None: ...
def model_row_errors(key: str, row: dict[str, JsonValue]) -> list[RowError]: ...
@final
class FetchedRegistry:
@property
def registry(self) -> ModelRegistry: ...
@property
def revision(self) -> str: ...
@property
def validator(self) -> str | None: ...
@final
class RegistryNotModified: ...
@final
class RegistryUnavailable:
@property
def reason(self) -> str: ...
@property
def retryable(self) -> bool: ...
@property
def retry_after(self) -> float | None: ...
def fetch_once(
source: str, *, if_none_match: str | None, timeout_seconds: float
) -> FetchedRegistry | RegistryNotModified | RegistryUnavailable: ...
@final
class CatalogRefresher:
def __new__(
cls,
source: str,
*,
installed_revision: str | None,
installed_validator: str | None,
install: Callable[[int, ModelRegistry, list[RowError]], int | None],
checked: Callable[[str | None], None],
generation: Callable[[], int],
) -> CatalogRefresher: ...
def stop(self) -> None: ...
def after_fork(self) -> None: ...
def write_rows(target: dict[str, object], set: dict[str, object], remove: list[str]) -> None: ...
@final
class ModelCostLayers:
@ -384,6 +427,8 @@ def process_state_started() -> bool: ...
def reserve_process_for_forking() -> None: ...
__all__ = [
"CatalogRefresher",
"FetchedRegistry",
"ForkedAfterNativeRuntimeStarted",
"HuggingFaceEncoding",
"ModelCostLayers",
@ -397,6 +442,8 @@ __all__ = [
"Priced",
"ProcessReservedForForking",
"RegistrationTarget",
"RegistryNotModified",
"RegistryUnavailable",
"ResolvedModel",
"ResponsesWebSocketConnection",
"RowError",
@ -417,6 +464,7 @@ __all__ = [
"atranscription",
"completion",
"embedding",
"fetch_once",
"gil_stats",
"invalidate_model_cost_index",
"messages",