diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 3c4a4ce0ea3..e476bfd4072 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4309,6 +4309,7 @@ dependencies = [ name = "litellm-python-bridge" version = "0.1.0" dependencies = [ + "async-trait", "aws-sdk-secretsmanager", "bytes", "criterion", @@ -4335,6 +4336,7 @@ dependencies = [ "litellm-llms", "litellm-llms-types", "litellm-model-catalog", + "litellm-model-catalog-refresh", "litellm-router-types", "litellm-secrets", "litellm-secrets-aws", diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 221f32daabe..b6fee41db17 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -19,12 +19,14 @@ huggingface = ["litellm-token-counter/huggingface"] tiktoken = ["litellm-token-counter/tiktoken"] [dependencies] +async-trait = "0.1.89" fancy-regex.workspace = true litellm-tracing.workspace = true litellm-spend-clickhouse.workspace = true litellm-storage-clickhouse.workspace = true litellm-host.workspace = true litellm-model-catalog.workspace = true +litellm-model-catalog-refresh.workspace = true litellm-cost.workspace = true time.workspace = true bytes.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index fcec33cd4df..302036be581 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -32,10 +32,11 @@ mod _native { use crate::logger::NativeDiagnosticProcessor; #[pymodule_export] use crate::model_registry::{ - ModelCostLayers, ModelInfoView, ModelNotFound, ModelRegistry, PriceLine, Priced, - RegistrationTarget, ResolvedModel, RowError, RowErrors, Unpriced, UsageInput, WhenInput, - invalidate_model_cost_index, model_info, model_row_errors, price, registration_target, - resolve_model, write_rows, + CatalogRefresher, FetchedRegistry, ModelCostLayers, ModelInfoView, ModelNotFound, + ModelRegistry, PriceLine, Priced, RegistrationTarget, RegistryNotModified, + RegistryUnavailable, ResolvedModel, RowError, RowErrors, Unpriced, UsageInput, WhenInput, + fetch_once, invalidate_model_cost_index, model_info, model_row_errors, price, + registration_target, resolve_model, write_rows, }; #[pymodule_export] use crate::routes::audio_transcription::{atranscription, transcription}; @@ -123,6 +124,11 @@ assert live == {"model": {"litellm_provider": "openai", "input_cost_per_token": Python::attach(|py| { let mut expected = vec![ "NativeCacheHandle", + "CatalogRefresher", + "FetchedRegistry", + "RegistryNotModified", + "RegistryUnavailable", + "fetch_once", "ModelRegistry", "ModelCostLayers", "RowError", diff --git a/litellm-rust/crates/python-bridge/src/model_registry.rs b/litellm-rust/crates/python-bridge/src/model_registry.rs index 1ee38bc0cfb..4e59931cc47 100644 --- a/litellm-rust/crates/python-bridge/src/model_registry.rs +++ b/litellm-rust/crates/python-bridge/src/model_registry.rs @@ -1,6 +1,8 @@ mod live; +mod refresh; pub(crate) use live::*; +pub(crate) use refresh::*; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::{Arc, LazyLock, Mutex, PoisonError}; @@ -102,6 +104,10 @@ pub(crate) struct RowError { #[derive(Clone)] pub(crate) struct ModelRegistry { pub(crate) parsed: Arc, + #[pyo3(get)] + pub(crate) source_revision: Option, + #[pyo3(get)] + pub(crate) source_validator: Option, } #[pymethods] @@ -117,6 +123,8 @@ impl ModelRegistry { .map_err(|error| PyValueError::new_err(error.to_string()))?; Ok(Self { parsed: Arc::new(parsed), + source_revision: None, + source_validator: None, }) } @@ -127,6 +135,8 @@ impl ModelRegistry { catalog: Arc::clone(&BUNDLED.catalog), errors: Arc::clone(&BUNDLED.errors), }), + source_revision: None, + source_validator: None, }) } @@ -196,6 +206,8 @@ impl ModelRegistry { catalog: Arc::new(catalog), errors: Arc::clone(&self.parsed.errors), }), + source_revision: self.source_revision.clone(), + source_validator: self.source_validator.clone(), }, removed_errors, ) diff --git a/litellm-rust/crates/python-bridge/src/model_registry/refresh.rs b/litellm-rust/crates/python-bridge/src/model_registry/refresh.rs new file mode 100644 index 00000000000..b1b37ae1842 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/model_registry/refresh.rs @@ -0,0 +1,476 @@ +use std::{ + sync::{ + Arc, Weak, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use async_trait::async_trait; +use litellm_host_python::release_gil; +use litellm_model_catalog::{ + FetchRequest, Installed, Origin, Reply, RetryPolicy, Revalidation, Step, +}; +use litellm_model_catalog_refresh::{Adopted, Checked, Fetch, Install, Refresher, Source, Stale}; +use pyo3::{ + PyTraverseError, PyVisit, + exceptions::{PyRuntimeError, PyTypeError, PyValueError}, + prelude::*, + types::PyDict, +}; + +use super::{BUNDLED, ModelRegistry, ParsedRegistry, RowError, row_error}; + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct FetchedRegistry { + #[pyo3(get)] + registry: ModelRegistry, + #[pyo3(get)] + revision: String, + #[pyo3(get)] + validator: Option, +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct RegistryNotModified; + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct RegistryUnavailable { + #[pyo3(get)] + reason: String, + #[pyo3(get)] + retryable: bool, + #[pyo3(get)] + retry_after: Option, +} + +#[derive(IntoPyObject)] +enum FetchResult { + Fetched(FetchedRegistry), + NotModified(RegistryNotModified), + Unavailable(RegistryUnavailable), +} + +fn installed(revision: Option, validator: Option, source: &Source) -> Installed { + Installed { + catalog: Arc::clone(&BUNDLED.catalog), + origin: if revision.is_some() { + Origin::Remote { + source: source.id(), + } + } else { + Origin::Bundled + }, + revision: revision.unwrap_or_default(), + validator, + } +} + +fn unavailable(reason: String, retryable: bool, after: Option) -> FetchResult { + FetchResult::Unavailable(RegistryUnavailable { + reason, + retryable, + retry_after: after.map(|delay| delay.as_secs_f64()), + }) +} + +#[pyfunction] +#[pyo3(signature = (source, *, if_none_match, timeout_seconds))] +pub(crate) fn fetch_once( + py: Python<'_>, + source: &str, + if_none_match: Option, + timeout_seconds: f64, +) -> PyResult> { + let source = Source::parse(source).map_err(|error| PyValueError::new_err(error.to_string()))?; + let timeout = Duration::try_from_secs_f64(timeout_seconds) + .map_err(|_| PyValueError::new_err("timeout_seconds must be finite and non-negative"))?; + if tokio::runtime::Handle::try_current().is_ok() { + return Err(PyRuntimeError::new_err( + "fetch_once cannot run from a Tokio context", + )); + } + let http = crate::http::call_config(py, &PyDict::new(py), false)?; + let result = release_gil(py, move || { + let fetch = match source.fetcher(&http) { + Ok(fetch) => fetch, + Err(_) => { + return unavailable( + "model catalog transport configuration failed".into(), + false, + None, + ); + } + }; + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(_) => return unavailable("model catalog runtime unavailable".into(), false, None), + }; + let reply = runtime.block_on(fetch.fetch(FetchRequest { + if_none_match, + timeout, + })); + runtime.shutdown_background(); + match reply { + Reply::Retry { after, reason } => unavailable(reason, true, after), + Reply::Failed { reason } => unavailable(reason, false, None), + Reply::NotModified => FetchResult::NotModified(RegistryNotModified), + reply @ Reply::Fetched { .. } => { + let (mut state, _) = Revalidation::new( + source.id(), + &installed(None, None, &source), + RetryPolicy { + max_attempts: 1, + timeout, + ..RetryPolicy::default() + }, + None, + ); + match state.on_reply(reply) { + Step::Adopt { installed, errors } => FetchResult::Fetched(FetchedRegistry { + registry: ModelRegistry { + parsed: Arc::new(ParsedRegistry { + catalog: installed.catalog, + errors: Arc::from(errors), + }), + source_revision: Some(installed.revision.clone()), + source_validator: installed.validator.clone(), + }, + revision: installed.revision, + validator: installed.validator, + }), + Step::Done { reason } => unavailable(reason.unwrap_or_default(), false, None), + Step::Fetch(_) | Step::Sleep(_) => { + unreachable!("a fetched reply never retries") + } + } + } + } + }); + Ok(result.into_pyobject(py)?.unbind()) +} + +struct PythonInstall { + install: Py, + checked: Py, + generation: Py, + stopped: Arc, +} + +struct CallbackFetch { + inner: Arc, + stopped: Arc, +} + +#[async_trait] +impl Fetch for CallbackFetch { + async fn fetch(&self, request: FetchRequest) -> Reply { + if self.stopped.load(Ordering::Acquire) { + return Reply::Failed { + reason: "model catalog callbacks stopped".into(), + }; + } + self.inner.fetch(request).await + } +} + +impl PythonInstall { + fn call(&self, call: impl FnOnce(Python<'_>) -> PyResult) -> Option { + Python::try_attach(|py| { + if self.stopped.load(Ordering::Acquire) { + return None; + } + match call(py) { + Ok(value) => Some(value), + Err(error) => { + self.stopped.store(true, Ordering::Release); + error.write_unraisable(py, None); + None + } + } + }) + .flatten() + } +} + +impl Install for PythonInstall { + fn generation(&self) -> u64 { + self.call(|py| self.generation.call0(py)?.extract(py)) + .unwrap_or_default() + } + + fn install(&self, seen: u64, adopted: Adopted) -> Result { + let registry = ModelRegistry { + parsed: Arc::new(ParsedRegistry { + catalog: adopted.installed.catalog, + errors: Arc::from([]), + }), + source_revision: Some(adopted.installed.revision), + source_validator: adopted.installed.validator, + }; + let left_out: Vec = adopted.left_out.iter().map(row_error).collect(); + self.call(|py| { + self.install + .call1(py, (seen, registry, left_out))? + .extract::>(py) + }) + .flatten() + .ok_or(Stale) + } + + fn checked(&self, outcome: Checked) { + let reason = match outcome { + Checked::NotModified => None, + Checked::Failed { reason } => Some(reason), + }; + self.call(|py| self.checked.call1(py, (reason,)).map(|_| ())); + } +} + +#[pyclass(module = "litellm.rust_bridge._native")] +pub(crate) struct CatalogRefresher { + inner: Option, + callbacks: Weak, +} + +#[pymethods] +impl CatalogRefresher { + #[new] + #[pyo3(signature = (source, *, installed_revision, installed_validator, install, checked, generation))] + fn new( + py: Python<'_>, + source: &str, + installed_revision: Option, + installed_validator: Option, + install: Py, + checked: Py, + generation: Py, + ) -> PyResult { + let source = + Source::parse(source).map_err(|error| PyValueError::new_err(error.to_string()))?; + if !install.bind(py).is_callable() + || !checked.bind(py).is_callable() + || !generation.bind(py).is_callable() + { + return Err(PyTypeError::new_err( + "install, checked and generation must be callable", + )); + } + let http = crate::http::call_config(py, &PyDict::new(py), false)?; + let logger = crate::logger::capture(py); + let callbacks = Arc::new(PythonInstall { + install, + checked, + generation, + stopped: Arc::new(AtomicBool::new(false)), + }); + let weak = Arc::downgrade(&callbacks); + let inner = release_gil(py, move || { + let fetch = Arc::new(CallbackFetch { + inner: source.fetcher(&http).map_err(crate::http::client_error)?, + stopped: Arc::clone(&callbacks.stopped), + }); + let installed = installed(installed_revision, installed_validator, &source); + Ok::<_, PyErr>( + logger.scope(|| Refresher::start(source.id(), fetch, &installed, callbacks)), + ) + })?; + Ok(Self { + inner: Some(inner), + callbacks: weak, + }) + } + + fn stop(&self) { + if let Some(callbacks) = self.callbacks.upgrade() { + callbacks.stopped.store(true, Ordering::Release); + } + if let Some(inner) = &self.inner { + inner.stop(); + } + } + + fn after_fork(&self, py: Python<'_>) { + if self + .callbacks + .upgrade() + .is_some_and(|callbacks| callbacks.stopped.load(Ordering::Acquire)) + { + return; + } + if let Some(inner) = &self.inner { + release_gil(py, || inner.after_fork()); + } + } + + fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + if let Some(callbacks) = self.callbacks.upgrade() { + // A running worker owns these edges independently of this Python object. + if Arc::strong_count(&callbacks) == 2 { + visit.call(&callbacks.install)?; + visit.call(&callbacks.checked)?; + visit.call(&callbacks.generation)?; + } + } + Ok(()) + } + + fn __clear__(&mut self) { + self.stop(); + self.inner.take(); + } +} + +impl Drop for CatalogRefresher { + fn drop(&mut self) { + self.stop(); + } +} + +#[cfg(test)] +mod tests { + use std::ffi::CString; + + use pyo3::types::PyList; + use rstest::{fixture, rstest}; + + use super::*; + + #[fixture] + fn python() { + Python::initialize(); + } + + fn callbacks(py: Python<'_>, install_result: &str) -> (PythonInstall, Py) { + let locals = PyDict::new(py); + py.run(&CString::new(format!( + "events = []\ndef install(seen, registry, errors):\n events.append((seen, registry, errors))\n return {install_result}\ndef checked(reason):\n events.append(reason)\ndef generation():\n return 7" + )).unwrap(), Some(&locals), None).unwrap(); + let callback = |name| locals.get_item(name).unwrap().unwrap().unbind(); + let result = PythonInstall { + install: callback("install"), + checked: callback("checked"), + generation: callback("generation"), + stopped: Arc::new(AtomicBool::new(false)), + }; + ( + result, + locals + .get_item("events") + .unwrap() + .unwrap() + .cast_into::() + .unwrap() + .unbind(), + ) + } + + #[rstest] + #[case::adopted("8", Ok(8))] + #[case::stale("None", Err(Stale))] + fn install_return_maps_to_generation_or_stale( + #[from(python)] _python: (), + #[case] answer: &str, + #[case] expected: Result, + ) { + Python::attach(|py| { + let (callbacks, events) = callbacks(py, answer); + let source = Source::parse("https://example.test/catalog").unwrap(); + assert_eq!(callbacks.generation(), 7); + let adopted = Adopted { + installed: installed(Some("revision".into()), Some("validator".into()), &source), + left_out: vec![], + }; + assert_eq!(callbacks.install(7, adopted), expected); + let event = events.bind(py).get_item(0).unwrap(); + assert_eq!(event.get_item(0).unwrap().extract::().unwrap(), 7); + let registry = event + .get_item(1) + .unwrap() + .extract::>() + .unwrap(); + assert!(Arc::ptr_eq(®istry.parsed.catalog, &BUNDLED.catalog)); + assert_eq!(registry.source_revision.as_deref(), Some("revision")); + assert_eq!(registry.source_validator.as_deref(), Some("validator")); + assert!( + event + .get_item(2) + .unwrap() + .cast::() + .unwrap() + .is_empty() + ); + }); + } + + #[rstest] + #[case::not_modified(Checked::NotModified, None)] + #[case::failed(Checked::Failed { reason: "offline".into() }, Some("offline"))] + fn checked_maps_reason( + #[from(python)] _python: (), + #[case] outcome: Checked, + #[case] expected: Option<&str>, + ) { + Python::attach(|py| { + let (callbacks, events) = callbacks(py, "8"); + callbacks.checked(outcome); + assert_eq!( + events + .bind(py) + .get_item(0) + .unwrap() + .extract::>() + .unwrap() + .as_deref(), + expected + ); + }); + } + + #[rstest] + fn stopped_callbacks_are_not_invoked(#[from(python)] _python: ()) { + Python::attach(|py| { + let (callbacks, events) = callbacks(py, "8"); + callbacks.stopped.store(true, Ordering::Release); + assert_eq!(callbacks.generation(), 0); + callbacks.checked(Checked::NotModified); + assert!(events.bind(py).is_empty()); + }); + } + + #[rstest] + #[case::unsupported_scheme("ftp://example.test")] + #[case::relative("catalog.json")] + fn invalid_source_precedes_python_config(#[from(python)] _python: (), #[case] source: &str) { + Python::attach(|py| { + let error = fetch_once(py, source, None, 1.0).unwrap_err(); + assert!(error.is_instance_of::(py)); + }); + } + + #[rstest] + #[case::negative(-1.0)] + #[case::nan(f64::NAN)] + #[case::infinite(f64::INFINITY)] + fn invalid_timeout_precedes_python_config(#[from(python)] _python: (), #[case] timeout: f64) { + Python::attach(|py| { + let error = fetch_once(py, "https://example.test", None, timeout).unwrap_err(); + assert!(error.is_instance_of::(py)); + }); + } + + #[rstest] + fn nested_runtime_is_rejected(#[from(python)] _python: ()) { + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let _entered = runtime.enter(); + Python::attach(|py| { + let error = fetch_once(py, "https://example.test", None, 1.0).unwrap_err(); + assert!(error.is_instance_of::(py)); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/tests/catalog_refresh.py b/litellm-rust/crates/python-bridge/tests/catalog_refresh.py new file mode 100644 index 00000000000..0376f9dcf9c --- /dev/null +++ b/litellm-rust/crates/python-bridge/tests/catalog_refresh.py @@ -0,0 +1,510 @@ +from __future__ import annotations + +import gc +import hashlib +import json +import logging +import os +import queue +import ssl +import subprocess +import sys +import tempfile +import threading +import time +import unittest +import weakref +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Final +from unittest.mock import patch + +import litellm +from litellm.rust_bridge import _native as native + + +class Response: + def __init__(self, status: int = 200, body: bytes | None = None, hold: bool = False) -> None: + self.status = status + self.body = native.ModelRegistry.bundled_bytes() if body is None else body + self.requests: queue.Queue[tuple[str, str | None]] = queue.Queue() + self.entered = threading.Event() + self.release = threading.Event() + if not hold: + self.release.set() + + +@contextmanager +def server(response: Response, tls: ssl.SSLContext | None = None) -> Iterator[str]: + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + response.requests.put((self.path, self.headers.get("If-None-Match"))) + response.entered.set() + response.release.wait(15) + self.send_response(response.status) + self.send_header("ETag", '"catalog-v1"') + self.send_header("Retry-After", "0") + self.send_header("Content-Length", str(len(response.body) if response.status == 200 else 0)) + self.end_headers() + if response.status == 200: + try: + self.wfile.write(response.body) + except (BrokenPipeError, ConnectionResetError, ssl.SSLError): + pass + + def log_message(self, format: str, *args: object) -> None: + pass + + with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as http: + if tls is not None: + http.socket = tls.wrap_socket(http.socket, server_side=True) + worker: Final = threading.Thread(target=http.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True) + worker.start() + try: + yield f"{'https' if tls else 'http'}://127.0.0.1:{http.server_port}" + finally: + response.release.set() + http.shutdown() + worker.join(5) + + +def refresh_threads() -> frozenset[str]: + return frozenset(path.name for path in Path("/proc/self/task").iterdir() if catalog_thread(path)) + + +def catalog_thread(path: Path) -> bool: + try: + return (path / "comm").read_text().startswith("litellm-catalog") + except FileNotFoundError: + return False + + +def wait_for(predicate: Callable[[], bool], timeout: float = 5) -> None: + deadline: Final = time.monotonic() + timeout + while not predicate(): + if time.monotonic() >= deadline: + raise AssertionError("condition did not complete") + threading.Event().wait(0.01) + + +class Observer: + def __init__(self, accept: bool = True) -> None: + self.value = 7 + self.accept = accept + self.events: queue.Queue[tuple[str, object]] = queue.Queue() + self.provenance: tuple[str | None, str | None] = (None, None) + + def generation(self) -> int: + return self.value + + def install(self, seen: int, registry: native.ModelRegistry, errors: list[native.RowError]) -> int | None: + if self.accept and seen == self.value: + self.provenance = (registry.source_revision, registry.source_validator) + self.value += 1 + self.events.put(("install", (seen, registry, errors))) + return self.value + self.events.put(("install", (seen, registry, errors))) + return None + + def checked(self, reason: str | None) -> None: + self.events.put(("checked", reason)) + + def start(self, url: str, validator: str | None = None, revision: str | None = None) -> native.CatalogRefresher: + return native.CatalogRefresher( + url, + installed_revision=revision, + installed_validator=validator, + install=self.install, + checked=self.checked, + generation=self.generation, + ) + + +class CatalogRefreshTests(unittest.TestCase): + def setUp(self) -> None: + self.environment = patch.dict(os.environ, {"NO_PROXY": "*", "no_proxy": "*"}) + self.environment.start() + self.addCleanup(self.environment.stop) + self.ssl_verify = litellm.ssl_verify + self.addCleanup(setattr, litellm, "ssl_verify", self.ssl_verify) + self.threads = refresh_threads() + self.assertFalse(native.process_state_started()) + + def tearDown(self) -> None: + wait_for(lambda: refresh_threads() == self.threads) + self.assertFalse(native.process_state_started()) + + def fetch(self, url: str, validator: str | None = None, timeout: float = 3) -> object: + return native.fetch_once(url, if_none_match=validator, timeout_seconds=timeout) + + def test_fetch_installs_nothing_and_preserves_validator_revision(self) -> None: + response: Final = Response() + original: Final = litellm.model_cost + original_rows: Final = dict(original) + with server(response) as url: + result: Final = self.fetch(url, '"previous"') + self.assertIsInstance(result, native.FetchedRegistry) + assert isinstance(result, native.FetchedRegistry) + self.assertEqual(result.registry.to_dict(), native.ModelRegistry.bundled().to_dict()) + self.assertEqual(result.validator, '"catalog-v1"') + self.assertEqual(result.registry.source_revision, result.revision) + self.assertEqual(result.registry.source_validator, result.validator) + filtered, _ = result.registry.without_rejected() + self.assertEqual(filtered.source_revision, result.revision) + self.assertEqual(filtered.source_validator, result.validator) + self.assertEqual( + result.revision, + hashlib.sha1(b"blob " + str(len(response.body)).encode() + b"\0" + response.body).hexdigest(), + ) + self.assertEqual(response.requests.get_nowait(), ("/", '"previous"')) + self.assertTrue(response.requests.empty()) + self.assertIs(litellm.model_cost, original) + self.assertEqual(litellm.model_cost, original_rows) + + def test_parse_and_bundled_have_no_source_metadata(self) -> None: + bundled: Final = native.ModelRegistry.bundled() + parsed: Final = native.ModelRegistry.parse(native.ModelRegistry.bundled_bytes()) + self.assertIsNone(bundled.source_revision) + self.assertIsNone(bundled.source_validator) + self.assertIsNone(parsed.source_revision) + self.assertIsNone(parsed.source_validator) + self.assertEqual(bundled.to_dict(), parsed.to_dict()) + with self.assertRaises(AttributeError): + setattr(bundled, "source_revision", "not-writable") + + def test_not_modified(self) -> None: + response: Final = Response(304) + with server(response) as url: + self.assertIsInstance(self.fetch(url, '"old"'), native.RegistryNotModified) + observer: Final = Observer() + refresher: Final = observer.start(url, '"old"') + self.addCleanup(refresher.stop) + self.assertEqual(observer.events.get(timeout=5), ("checked", None)) + wait_for(lambda: refresh_threads() == self.threads) + self.assertTrue(observer.events.empty()) + self.assertEqual(response.requests.get_nowait(), ("/", '"old"')) + self.assertEqual(response.requests.get_nowait(), ("/", '"old"')) + + def test_refresher_filters_rejected_rows_and_reports_them(self) -> None: + body: Final = json.dumps( + { + **json.loads(native.ModelRegistry.bundled_bytes()), + "p3-invalid": {"litellm_provider": "openai", "input_cost_per_token": -1}, + } + ).encode() + response: Final = Response(body=body) + with server(response) as url: + observer: Final = Observer() + refresher: Final = observer.start(url) + self.addCleanup(refresher.stop) + kind, payload = observer.events.get(timeout=5) + self.assertEqual(kind, "install") + assert isinstance(payload, tuple) + seen, registry, errors = payload + self.assertEqual(seen, 7) + self.assertNotIn("p3-invalid", registry.to_dict()) + self.assertTrue(any(error.key == "p3-invalid" for error in errors)) + self.assertGreater(len(registry.to_dict()), 100) + self.assertEqual( + registry.source_revision, + hashlib.sha1(b"blob " + str(len(body)).encode() + b"\0" + body).hexdigest(), + ) + self.assertEqual(registry.source_validator, '"catalog-v1"') + self.assertEqual(observer.provenance, (registry.source_revision, registry.source_validator)) + wait_for(lambda: refresh_threads() == self.threads) + self.assertTrue(observer.events.empty()) + self.assertEqual(response.requests.qsize(), 1) + + def test_stale_is_logged_once_without_checked_or_retry(self) -> None: + response: Final = Response() + with self.assertLogs(litellm.verbose_logger, level=logging.DEBUG) as logs, server(response) as url: + observer: Final = Observer(accept=False) + refresher: Final = observer.start(url) + self.addCleanup(refresher.stop) + self.assertEqual(observer.events.get(timeout=5)[0], "install") + wait_for(lambda: refresh_threads() == self.threads) + self.assertTrue(observer.events.empty()) + self.assertEqual(response.requests.qsize(), 1) + self.assertTrue( + any(getattr(record, "rust_fields", {}).get("outcome") == "stale" for record in logs.records), logs.output + ) + self.assertEqual(observer.provenance, (None, None)) + + def test_stop_held_response_has_no_callback_and_exits_thread(self) -> None: + response: Final = Response(hold=True) + with server(response) as url: + observer: Final = Observer() + refresher: Final = observer.start(url) + self.addCleanup(refresher.stop) + self.assertTrue(response.entered.wait(5)) + before: Final = time.monotonic() + refresher.stop() + self.assertLess(time.monotonic() - before, 0.5) + wait_for(lambda: refresh_threads() == self.threads) + response.release.set() + refresher.stop() + self.assertTrue(observer.events.empty()) + + def test_new_generation_fences_inflight_result(self) -> None: + response: Final = Response(hold=True) + with server(response) as url: + observer: Final = Observer() + refresher: Final = observer.start(url) + self.addCleanup(refresher.stop) + self.assertTrue(response.entered.wait(5)) + observer.value = 8 + response.release.set() + wait_for(lambda: refresh_threads() == self.threads) + self.assertTrue(observer.events.empty()) + + self.assertEqual(observer.provenance, (None, None)) + + def test_retryable_once_does_not_retry(self) -> None: + response: Final = Response(503) + with server(response) as url: + result: Final = self.fetch(url) + self.assertIsInstance(result, native.RegistryUnavailable) + assert isinstance(result, native.RegistryUnavailable) + self.assertTrue(result.retryable) + self.assertEqual(result.retry_after, 0) + self.assertIn("503", result.reason) + self.assertEqual(response.requests.qsize(), 1) + + def test_failure_calls_checked_reason_once(self) -> None: + response: Final = Response(404) + with server(response) as url: + observer: Final = Observer() + refresher: Final = observer.start(url) + self.addCleanup(refresher.stop) + kind, reason = observer.events.get(timeout=5) + self.assertEqual(kind, "checked") + self.assertIn("404", str(reason)) + wait_for(lambda: refresh_threads() == self.threads) + self.assertTrue(observer.events.empty()) + + def test_file_source_ignores_http_validator(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path: Final = Path(directory) / "catalog.json" + path.write_bytes(native.ModelRegistry.bundled_bytes()) + result: Final = self.fetch(path.as_uri()) + self.assertIsInstance(result, native.FetchedRegistry) + assert isinstance(result, native.FetchedRegistry) + self.assertIsNone(result.validator) + self.assertIsInstance(self.fetch(path.as_uri(), '"old"'), native.FetchedRegistry) + + def test_s3_endpoint_and_conditional_header(self) -> None: + response: Final = Response() + with ( + server(response) as url, + patch.dict( + os.environ, + { + "AWS_ENDPOINT_URL_S3": url, + "AWS_ACCESS_KEY_ID": "local-test", + "AWS_SECRET_ACCESS_KEY": "local-test", + "AWS_REGION": "us-east-1", + "AWS_EC2_METADATA_DISABLED": "true", + }, + ), + ): + result: Final = self.fetch("s3://bucket/catalog.json", '"old"') + self.assertIsInstance(result, native.FetchedRegistry) + assert isinstance(result, native.FetchedRegistry) + self.assertEqual(result.registry.to_dict(), native.ModelRegistry.bundled().to_dict()) + self.assertEqual(response.requests.get_nowait(), ("/bucket/catalog.json?x-id=GetObject", '"old"')) + self.assertTrue(response.requests.empty()) + + def test_unsupported_source_and_invalid_timeout(self) -> None: + with self.assertRaises(ValueError): + self.fetch("ftp://x") + with self.assertRaises(ValueError): + Observer().start("ftp://x") + with self.assertRaises(ValueError): + self.fetch("http://127.0.0.1", timeout=float("nan")) + + def test_timeout_releases_gil_and_performs_one_request(self) -> None: + response: Final = Response(hold=True) + with server(response) as url: + result: Final = self.fetch(url, timeout=0.05) + self.assertTrue(response.entered.is_set()) + self.assertIsInstance(result, native.RegistryUnavailable) + self.assertEqual(response.requests.qsize(), 1) + + def test_parse_and_integrity_failure(self) -> None: + response: Final = Response(body=b'{"small": {"litellm_provider": "openai"}}') + with server(response) as url: + result: Final = self.fetch(url) + self.assertIsInstance(result, native.RegistryUnavailable) + assert isinstance(result, native.RegistryUnavailable) + self.assertFalse(result.retryable) + self.assertTrue(result.reason) + + def test_rejected_document_does_not_advance_provenance(self) -> None: + response: Final = Response(body=b"not-json") + observer: Final = Observer() + with server(response) as url: + refresher: Final = observer.start(url) + self.addCleanup(refresher.stop) + kind, reason = observer.events.get(timeout=5) + self.assertEqual(kind, "checked") + self.assertTrue(reason) + wait_for(lambda: refresh_threads() == self.threads) + self.assertTrue(observer.events.empty()) + self.assertEqual(observer.provenance, (None, None)) + + def test_tls_verify_applies_to_fetch_and_refresher(self) -> None: + with tempfile.TemporaryDirectory() as directory: + key: Final = Path(directory) / "key.pem" + cert: Final = Path(directory) / "cert.pem" + subprocess.run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=IP:127.0.0.1", + "-keyout", + str(key), + "-out", + str(cert), + ], + check=True, + capture_output=True, + ) + context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert, key) + response: Final = Response() + with server(response, context) as url: + litellm.ssl_verify = False + self.assertIsInstance(self.fetch(url), native.FetchedRegistry) + observer: Final = Observer() + refresher: Final = observer.start(url) + self.addCleanup(refresher.stop) + litellm.ssl_verify = True + self.assertEqual(observer.events.get(timeout=5)[0], "install") + failed: Final = self.fetch(url) + self.assertIsInstance(failed, native.RegistryUnavailable) + assert isinstance(failed, native.RegistryUnavailable) + self.assertTrue(failed.reason) + failure_observer: Final = Observer() + failure_refresher: Final = failure_observer.start(url) + self.addCleanup(failure_refresher.stop) + kind, reason = failure_observer.events.get(timeout=15) + self.assertEqual(kind, "checked") + self.assertTrue(reason) + + def test_callback_exception_is_unraisable_and_terminal(self) -> None: + failures: Final[queue.Queue[object]] = queue.Queue() + + def install(seen: int, registry: native.ModelRegistry, errors: list[native.RowError]) -> int: + raise RuntimeError("callback-test") + + response: Final = Response() + observer: Final = Observer() + with patch.object(sys, "unraisablehook", failures.put), server(response) as url: + refresher: Final = native.CatalogRefresher( + url, + installed_revision=None, + installed_validator=None, + install=install, + checked=observer.checked, + generation=observer.generation, + ) + self.addCleanup(refresher.stop) + failure: Final = failures.get(timeout=5) + self.assertIn("callback-test", str(getattr(failure, "exc_value"))) + wait_for(lambda: refresh_threads() == self.threads) + self.assertTrue(observer.events.empty()) + self.assertTrue(failures.empty()) + + def test_generation_failure_starts_no_transport(self) -> None: + failures: Final[queue.Queue[object]] = queue.Queue() + + def generation() -> int: + raise RuntimeError("generation-test") + + response: Final = Response() + observer: Final = Observer() + with patch.object(sys, "unraisablehook", failures.put), server(response) as url: + refresher: Final = native.CatalogRefresher( + url, + installed_revision=None, + installed_validator=None, + install=observer.install, + checked=observer.checked, + generation=generation, + ) + self.addCleanup(refresher.stop) + failure: Final = failures.get(timeout=5) + self.assertIn("generation-test", str(getattr(failure, "exc_value"))) + wait_for(lambda: refresh_threads() == self.threads) + self.assertTrue(response.requests.empty()) + self.assertTrue(observer.events.empty()) + + def test_callback_cycle_collected_after_worker_completes(self) -> None: + finalized: Final[queue.Queue[bool]] = queue.Queue() + + class Owner(Observer): + def __init__(self, url: str) -> None: + super().__init__() + self.refresher = self.start(url) + + def __del__(self) -> None: + self.refresher.stop() + self.refresher.after_fork() + finalized.put(True) + + response: Final = Response(304) + with server(response) as url: + + def make_cycle() -> weakref.ReferenceType[Owner]: + owner: Final = Owner(url) + self.assertEqual(owner.events.get(timeout=5), ("checked", None)) + return weakref.ref(owner) + + reference: Final = make_cycle() + wait_for(lambda: refresh_threads() == self.threads) + gc.collect() + self.assertIsNone(reference()) + self.assertTrue(finalized.get(timeout=1)) + + @unittest.skipUnless(hasattr(os, "fork"), "requires fork") + def test_after_fork_restarts_unadopted_child(self) -> None: + response: Final = Response(hold=True) + with server(response) as url: + observer: Final = Observer() + refresher: Final = observer.start(url) + self.addCleanup(refresher.stop) + self.assertTrue(response.entered.wait(5)) + read_fd, write_fd = os.pipe() + pid: Final = os.fork() + if pid == 0: + os.close(read_fd) + try: + refresher.after_fork() + kind, _ = observer.events.get(timeout=10) + refresher.stop() + os.write(write_fd, kind.encode()) + finally: + os._exit(0) + os.close(write_fd) + try: + response.release.set() + self.assertEqual(os.read(read_fd, 32), b"install") + self.assertEqual(os.waitpid(pid, 0)[1], 0) + self.assertEqual(observer.events.get(timeout=5)[0], "install") + finally: + os.close(read_fd) + + +if __name__ == "__main__": + unittest.main() diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 24718eee83f..d94461fccf0 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -1,5 +1,5 @@ from asyncio import Future -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from typing import Literal, Never, final import httpx @@ -55,6 +55,10 @@ class RowError: @final class ModelRegistry: + @property + def source_revision(self) -> str | None: ... + @property + def source_validator(self) -> str | None: ... @staticmethod def parse(body: bytes) -> ModelRegistry: ... @staticmethod @@ -69,6 +73,45 @@ class ModelRegistry: def check_size(self, reference: int) -> RowError | None: ... def model_row_errors(key: str, row: dict[str, JsonValue]) -> list[RowError]: ... +@final +class FetchedRegistry: + @property + def registry(self) -> ModelRegistry: ... + @property + def revision(self) -> str: ... + @property + def validator(self) -> str | None: ... + +@final +class RegistryNotModified: ... + +@final +class RegistryUnavailable: + @property + def reason(self) -> str: ... + @property + def retryable(self) -> bool: ... + @property + def retry_after(self) -> float | None: ... + +def fetch_once( + source: str, *, if_none_match: str | None, timeout_seconds: float +) -> FetchedRegistry | RegistryNotModified | RegistryUnavailable: ... +@final +class CatalogRefresher: + def __new__( + cls, + source: str, + *, + installed_revision: str | None, + installed_validator: str | None, + install: Callable[[int, ModelRegistry, list[RowError]], int | None], + checked: Callable[[str | None], None], + generation: Callable[[], int], + ) -> CatalogRefresher: ... + def stop(self) -> None: ... + def after_fork(self) -> None: ... + def write_rows(target: dict[str, object], set: dict[str, object], remove: list[str]) -> None: ... @final class ModelCostLayers: @@ -384,6 +427,8 @@ def process_state_started() -> bool: ... def reserve_process_for_forking() -> None: ... __all__ = [ + "CatalogRefresher", + "FetchedRegistry", "ForkedAfterNativeRuntimeStarted", "HuggingFaceEncoding", "ModelCostLayers", @@ -397,6 +442,8 @@ __all__ = [ "Priced", "ProcessReservedForForking", "RegistrationTarget", + "RegistryNotModified", + "RegistryUnavailable", "ResolvedModel", "ResponsesWebSocketConnection", "RowError", @@ -417,6 +464,7 @@ __all__ = [ "atranscription", "completion", "embedding", + "fetch_once", "gil_stats", "invalidate_model_cost_index", "messages",