mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
feat(model-catalog): bind native refresher to Python
This commit is contained in:
parent
2cff68eee5
commit
d3d33019a9
7 changed files with 1061 additions and 5 deletions
2
litellm-rust/Cargo.lock
generated
2
litellm-rust/Cargo.lock
generated
|
|
@ -4309,6 +4309,7 @@ dependencies = [
|
|||
name = "litellm-python-bridge"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"aws-sdk-secretsmanager",
|
||||
"bytes",
|
||||
"criterion",
|
||||
|
|
@ -4335,6 +4336,7 @@ dependencies = [
|
|||
"litellm-llms",
|
||||
"litellm-llms-types",
|
||||
"litellm-model-catalog",
|
||||
"litellm-model-catalog-refresh",
|
||||
"litellm-router-types",
|
||||
"litellm-secrets",
|
||||
"litellm-secrets-aws",
|
||||
|
|
|
|||
|
|
@ -19,12 +19,14 @@ huggingface = ["litellm-token-counter/huggingface"]
|
|||
tiktoken = ["litellm-token-counter/tiktoken"]
|
||||
|
||||
[dependencies]
|
||||
async-trait = "0.1.89"
|
||||
fancy-regex.workspace = true
|
||||
litellm-tracing.workspace = true
|
||||
litellm-spend-clickhouse.workspace = true
|
||||
litellm-storage-clickhouse.workspace = true
|
||||
litellm-host.workspace = true
|
||||
litellm-model-catalog.workspace = true
|
||||
litellm-model-catalog-refresh.workspace = true
|
||||
litellm-cost.workspace = true
|
||||
time.workspace = true
|
||||
bytes.workspace = true
|
||||
|
|
|
|||
|
|
@ -32,10 +32,11 @@ mod _native {
|
|||
use crate::logger::NativeDiagnosticProcessor;
|
||||
#[pymodule_export]
|
||||
use crate::model_registry::{
|
||||
ModelCostLayers, ModelInfoView, ModelNotFound, ModelRegistry, PriceLine, Priced,
|
||||
RegistrationTarget, ResolvedModel, RowError, RowErrors, Unpriced, UsageInput, WhenInput,
|
||||
invalidate_model_cost_index, model_info, model_row_errors, price, registration_target,
|
||||
resolve_model, write_rows,
|
||||
CatalogRefresher, FetchedRegistry, ModelCostLayers, ModelInfoView, ModelNotFound,
|
||||
ModelRegistry, PriceLine, Priced, RegistrationTarget, RegistryNotModified,
|
||||
RegistryUnavailable, ResolvedModel, RowError, RowErrors, Unpriced, UsageInput, WhenInput,
|
||||
fetch_once, invalidate_model_cost_index, model_info, model_row_errors, price,
|
||||
registration_target, resolve_model, write_rows,
|
||||
};
|
||||
#[pymodule_export]
|
||||
use crate::routes::audio_transcription::{atranscription, transcription};
|
||||
|
|
@ -123,6 +124,11 @@ assert live == {"model": {"litellm_provider": "openai", "input_cost_per_token":
|
|||
Python::attach(|py| {
|
||||
let mut expected = vec![
|
||||
"NativeCacheHandle",
|
||||
"CatalogRefresher",
|
||||
"FetchedRegistry",
|
||||
"RegistryNotModified",
|
||||
"RegistryUnavailable",
|
||||
"fetch_once",
|
||||
"ModelRegistry",
|
||||
"ModelCostLayers",
|
||||
"RowError",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
mod live;
|
||||
mod refresh;
|
||||
|
||||
pub(crate) use live::*;
|
||||
pub(crate) use refresh::*;
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::sync::{Arc, LazyLock, Mutex, PoisonError};
|
||||
|
|
@ -102,6 +104,10 @@ pub(crate) struct RowError {
|
|||
#[derive(Clone)]
|
||||
pub(crate) struct ModelRegistry {
|
||||
pub(crate) parsed: Arc<ParsedRegistry>,
|
||||
#[pyo3(get)]
|
||||
pub(crate) source_revision: Option<String>,
|
||||
#[pyo3(get)]
|
||||
pub(crate) source_validator: Option<String>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
|
|
@ -117,6 +123,8 @@ impl ModelRegistry {
|
|||
.map_err(|error| PyValueError::new_err(error.to_string()))?;
|
||||
Ok(Self {
|
||||
parsed: Arc::new(parsed),
|
||||
source_revision: None,
|
||||
source_validator: None,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -127,6 +135,8 @@ impl ModelRegistry {
|
|||
catalog: Arc::clone(&BUNDLED.catalog),
|
||||
errors: Arc::clone(&BUNDLED.errors),
|
||||
}),
|
||||
source_revision: None,
|
||||
source_validator: None,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -196,6 +206,8 @@ impl ModelRegistry {
|
|||
catalog: Arc::new(catalog),
|
||||
errors: Arc::clone(&self.parsed.errors),
|
||||
}),
|
||||
source_revision: self.source_revision.clone(),
|
||||
source_validator: self.source_validator.clone(),
|
||||
},
|
||||
removed_errors,
|
||||
)
|
||||
|
|
|
|||
476
litellm-rust/crates/python-bridge/src/model_registry/refresh.rs
Normal file
476
litellm-rust/crates/python-bridge/src/model_registry/refresh.rs
Normal file
|
|
@ -0,0 +1,476 @@
|
|||
use std::{
|
||||
sync::{
|
||||
Arc, Weak,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use litellm_host_python::release_gil;
|
||||
use litellm_model_catalog::{
|
||||
FetchRequest, Installed, Origin, Reply, RetryPolicy, Revalidation, Step,
|
||||
};
|
||||
use litellm_model_catalog_refresh::{Adopted, Checked, Fetch, Install, Refresher, Source, Stale};
|
||||
use pyo3::{
|
||||
PyTraverseError, PyVisit,
|
||||
exceptions::{PyRuntimeError, PyTypeError, PyValueError},
|
||||
prelude::*,
|
||||
types::PyDict,
|
||||
};
|
||||
|
||||
use super::{BUNDLED, ModelRegistry, ParsedRegistry, RowError, row_error};
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct FetchedRegistry {
|
||||
#[pyo3(get)]
|
||||
registry: ModelRegistry,
|
||||
#[pyo3(get)]
|
||||
revision: String,
|
||||
#[pyo3(get)]
|
||||
validator: Option<String>,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct RegistryNotModified;
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct RegistryUnavailable {
|
||||
#[pyo3(get)]
|
||||
reason: String,
|
||||
#[pyo3(get)]
|
||||
retryable: bool,
|
||||
#[pyo3(get)]
|
||||
retry_after: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(IntoPyObject)]
|
||||
enum FetchResult {
|
||||
Fetched(FetchedRegistry),
|
||||
NotModified(RegistryNotModified),
|
||||
Unavailable(RegistryUnavailable),
|
||||
}
|
||||
|
||||
fn installed(revision: Option<String>, validator: Option<String>, source: &Source) -> Installed {
|
||||
Installed {
|
||||
catalog: Arc::clone(&BUNDLED.catalog),
|
||||
origin: if revision.is_some() {
|
||||
Origin::Remote {
|
||||
source: source.id(),
|
||||
}
|
||||
} else {
|
||||
Origin::Bundled
|
||||
},
|
||||
revision: revision.unwrap_or_default(),
|
||||
validator,
|
||||
}
|
||||
}
|
||||
|
||||
fn unavailable(reason: String, retryable: bool, after: Option<Duration>) -> FetchResult {
|
||||
FetchResult::Unavailable(RegistryUnavailable {
|
||||
reason,
|
||||
retryable,
|
||||
retry_after: after.map(|delay| delay.as_secs_f64()),
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (source, *, if_none_match, timeout_seconds))]
|
||||
pub(crate) fn fetch_once(
|
||||
py: Python<'_>,
|
||||
source: &str,
|
||||
if_none_match: Option<String>,
|
||||
timeout_seconds: f64,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
let source = Source::parse(source).map_err(|error| PyValueError::new_err(error.to_string()))?;
|
||||
let timeout = Duration::try_from_secs_f64(timeout_seconds)
|
||||
.map_err(|_| PyValueError::new_err("timeout_seconds must be finite and non-negative"))?;
|
||||
if tokio::runtime::Handle::try_current().is_ok() {
|
||||
return Err(PyRuntimeError::new_err(
|
||||
"fetch_once cannot run from a Tokio context",
|
||||
));
|
||||
}
|
||||
let http = crate::http::call_config(py, &PyDict::new(py), false)?;
|
||||
let result = release_gil(py, move || {
|
||||
let fetch = match source.fetcher(&http) {
|
||||
Ok(fetch) => fetch,
|
||||
Err(_) => {
|
||||
return unavailable(
|
||||
"model catalog transport configuration failed".into(),
|
||||
false,
|
||||
None,
|
||||
);
|
||||
}
|
||||
};
|
||||
let runtime = match tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
{
|
||||
Ok(runtime) => runtime,
|
||||
Err(_) => return unavailable("model catalog runtime unavailable".into(), false, None),
|
||||
};
|
||||
let reply = runtime.block_on(fetch.fetch(FetchRequest {
|
||||
if_none_match,
|
||||
timeout,
|
||||
}));
|
||||
runtime.shutdown_background();
|
||||
match reply {
|
||||
Reply::Retry { after, reason } => unavailable(reason, true, after),
|
||||
Reply::Failed { reason } => unavailable(reason, false, None),
|
||||
Reply::NotModified => FetchResult::NotModified(RegistryNotModified),
|
||||
reply @ Reply::Fetched { .. } => {
|
||||
let (mut state, _) = Revalidation::new(
|
||||
source.id(),
|
||||
&installed(None, None, &source),
|
||||
RetryPolicy {
|
||||
max_attempts: 1,
|
||||
timeout,
|
||||
..RetryPolicy::default()
|
||||
},
|
||||
None,
|
||||
);
|
||||
match state.on_reply(reply) {
|
||||
Step::Adopt { installed, errors } => FetchResult::Fetched(FetchedRegistry {
|
||||
registry: ModelRegistry {
|
||||
parsed: Arc::new(ParsedRegistry {
|
||||
catalog: installed.catalog,
|
||||
errors: Arc::from(errors),
|
||||
}),
|
||||
source_revision: Some(installed.revision.clone()),
|
||||
source_validator: installed.validator.clone(),
|
||||
},
|
||||
revision: installed.revision,
|
||||
validator: installed.validator,
|
||||
}),
|
||||
Step::Done { reason } => unavailable(reason.unwrap_or_default(), false, None),
|
||||
Step::Fetch(_) | Step::Sleep(_) => {
|
||||
unreachable!("a fetched reply never retries")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(result.into_pyobject(py)?.unbind())
|
||||
}
|
||||
|
||||
struct PythonInstall {
|
||||
install: Py<PyAny>,
|
||||
checked: Py<PyAny>,
|
||||
generation: Py<PyAny>,
|
||||
stopped: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
struct CallbackFetch {
|
||||
inner: Arc<dyn Fetch>,
|
||||
stopped: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Fetch for CallbackFetch {
|
||||
async fn fetch(&self, request: FetchRequest) -> Reply {
|
||||
if self.stopped.load(Ordering::Acquire) {
|
||||
return Reply::Failed {
|
||||
reason: "model catalog callbacks stopped".into(),
|
||||
};
|
||||
}
|
||||
self.inner.fetch(request).await
|
||||
}
|
||||
}
|
||||
|
||||
impl PythonInstall {
|
||||
fn call<T>(&self, call: impl FnOnce(Python<'_>) -> PyResult<T>) -> Option<T> {
|
||||
Python::try_attach(|py| {
|
||||
if self.stopped.load(Ordering::Acquire) {
|
||||
return None;
|
||||
}
|
||||
match call(py) {
|
||||
Ok(value) => Some(value),
|
||||
Err(error) => {
|
||||
self.stopped.store(true, Ordering::Release);
|
||||
error.write_unraisable(py, None);
|
||||
None
|
||||
}
|
||||
}
|
||||
})
|
||||
.flatten()
|
||||
}
|
||||
}
|
||||
|
||||
impl Install for PythonInstall {
|
||||
fn generation(&self) -> u64 {
|
||||
self.call(|py| self.generation.call0(py)?.extract(py))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn install(&self, seen: u64, adopted: Adopted) -> Result<u64, Stale> {
|
||||
let registry = ModelRegistry {
|
||||
parsed: Arc::new(ParsedRegistry {
|
||||
catalog: adopted.installed.catalog,
|
||||
errors: Arc::from([]),
|
||||
}),
|
||||
source_revision: Some(adopted.installed.revision),
|
||||
source_validator: adopted.installed.validator,
|
||||
};
|
||||
let left_out: Vec<RowError> = adopted.left_out.iter().map(row_error).collect();
|
||||
self.call(|py| {
|
||||
self.install
|
||||
.call1(py, (seen, registry, left_out))?
|
||||
.extract::<Option<u64>>(py)
|
||||
})
|
||||
.flatten()
|
||||
.ok_or(Stale)
|
||||
}
|
||||
|
||||
fn checked(&self, outcome: Checked) {
|
||||
let reason = match outcome {
|
||||
Checked::NotModified => None,
|
||||
Checked::Failed { reason } => Some(reason),
|
||||
};
|
||||
self.call(|py| self.checked.call1(py, (reason,)).map(|_| ()));
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass(module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct CatalogRefresher {
|
||||
inner: Option<Refresher>,
|
||||
callbacks: Weak<PythonInstall>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl CatalogRefresher {
|
||||
#[new]
|
||||
#[pyo3(signature = (source, *, installed_revision, installed_validator, install, checked, generation))]
|
||||
fn new(
|
||||
py: Python<'_>,
|
||||
source: &str,
|
||||
installed_revision: Option<String>,
|
||||
installed_validator: Option<String>,
|
||||
install: Py<PyAny>,
|
||||
checked: Py<PyAny>,
|
||||
generation: Py<PyAny>,
|
||||
) -> PyResult<Self> {
|
||||
let source =
|
||||
Source::parse(source).map_err(|error| PyValueError::new_err(error.to_string()))?;
|
||||
if !install.bind(py).is_callable()
|
||||
|| !checked.bind(py).is_callable()
|
||||
|| !generation.bind(py).is_callable()
|
||||
{
|
||||
return Err(PyTypeError::new_err(
|
||||
"install, checked and generation must be callable",
|
||||
));
|
||||
}
|
||||
let http = crate::http::call_config(py, &PyDict::new(py), false)?;
|
||||
let logger = crate::logger::capture(py);
|
||||
let callbacks = Arc::new(PythonInstall {
|
||||
install,
|
||||
checked,
|
||||
generation,
|
||||
stopped: Arc::new(AtomicBool::new(false)),
|
||||
});
|
||||
let weak = Arc::downgrade(&callbacks);
|
||||
let inner = release_gil(py, move || {
|
||||
let fetch = Arc::new(CallbackFetch {
|
||||
inner: source.fetcher(&http).map_err(crate::http::client_error)?,
|
||||
stopped: Arc::clone(&callbacks.stopped),
|
||||
});
|
||||
let installed = installed(installed_revision, installed_validator, &source);
|
||||
Ok::<_, PyErr>(
|
||||
logger.scope(|| Refresher::start(source.id(), fetch, &installed, callbacks)),
|
||||
)
|
||||
})?;
|
||||
Ok(Self {
|
||||
inner: Some(inner),
|
||||
callbacks: weak,
|
||||
})
|
||||
}
|
||||
|
||||
fn stop(&self) {
|
||||
if let Some(callbacks) = self.callbacks.upgrade() {
|
||||
callbacks.stopped.store(true, Ordering::Release);
|
||||
}
|
||||
if let Some(inner) = &self.inner {
|
||||
inner.stop();
|
||||
}
|
||||
}
|
||||
|
||||
fn after_fork(&self, py: Python<'_>) {
|
||||
if self
|
||||
.callbacks
|
||||
.upgrade()
|
||||
.is_some_and(|callbacks| callbacks.stopped.load(Ordering::Acquire))
|
||||
{
|
||||
return;
|
||||
}
|
||||
if let Some(inner) = &self.inner {
|
||||
release_gil(py, || inner.after_fork());
|
||||
}
|
||||
}
|
||||
|
||||
fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
if let Some(callbacks) = self.callbacks.upgrade() {
|
||||
// A running worker owns these edges independently of this Python object.
|
||||
if Arc::strong_count(&callbacks) == 2 {
|
||||
visit.call(&callbacks.install)?;
|
||||
visit.call(&callbacks.checked)?;
|
||||
visit.call(&callbacks.generation)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn __clear__(&mut self) {
|
||||
self.stop();
|
||||
self.inner.take();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CatalogRefresher {
|
||||
fn drop(&mut self) {
|
||||
self.stop();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::ffi::CString;
|
||||
|
||||
use pyo3::types::PyList;
|
||||
use rstest::{fixture, rstest};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[fixture]
|
||||
fn python() {
|
||||
Python::initialize();
|
||||
}
|
||||
|
||||
fn callbacks(py: Python<'_>, install_result: &str) -> (PythonInstall, Py<PyList>) {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(&CString::new(format!(
|
||||
"events = []\ndef install(seen, registry, errors):\n events.append((seen, registry, errors))\n return {install_result}\ndef checked(reason):\n events.append(reason)\ndef generation():\n return 7"
|
||||
)).unwrap(), Some(&locals), None).unwrap();
|
||||
let callback = |name| locals.get_item(name).unwrap().unwrap().unbind();
|
||||
let result = PythonInstall {
|
||||
install: callback("install"),
|
||||
checked: callback("checked"),
|
||||
generation: callback("generation"),
|
||||
stopped: Arc::new(AtomicBool::new(false)),
|
||||
};
|
||||
(
|
||||
result,
|
||||
locals
|
||||
.get_item("events")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.cast_into::<PyList>()
|
||||
.unwrap()
|
||||
.unbind(),
|
||||
)
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::adopted("8", Ok(8))]
|
||||
#[case::stale("None", Err(Stale))]
|
||||
fn install_return_maps_to_generation_or_stale(
|
||||
#[from(python)] _python: (),
|
||||
#[case] answer: &str,
|
||||
#[case] expected: Result<u64, Stale>,
|
||||
) {
|
||||
Python::attach(|py| {
|
||||
let (callbacks, events) = callbacks(py, answer);
|
||||
let source = Source::parse("https://example.test/catalog").unwrap();
|
||||
assert_eq!(callbacks.generation(), 7);
|
||||
let adopted = Adopted {
|
||||
installed: installed(Some("revision".into()), Some("validator".into()), &source),
|
||||
left_out: vec![],
|
||||
};
|
||||
assert_eq!(callbacks.install(7, adopted), expected);
|
||||
let event = events.bind(py).get_item(0).unwrap();
|
||||
assert_eq!(event.get_item(0).unwrap().extract::<u64>().unwrap(), 7);
|
||||
let registry = event
|
||||
.get_item(1)
|
||||
.unwrap()
|
||||
.extract::<PyRef<'_, ModelRegistry>>()
|
||||
.unwrap();
|
||||
assert!(Arc::ptr_eq(®istry.parsed.catalog, &BUNDLED.catalog));
|
||||
assert_eq!(registry.source_revision.as_deref(), Some("revision"));
|
||||
assert_eq!(registry.source_validator.as_deref(), Some("validator"));
|
||||
assert!(
|
||||
event
|
||||
.get_item(2)
|
||||
.unwrap()
|
||||
.cast::<PyList>()
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::not_modified(Checked::NotModified, None)]
|
||||
#[case::failed(Checked::Failed { reason: "offline".into() }, Some("offline"))]
|
||||
fn checked_maps_reason(
|
||||
#[from(python)] _python: (),
|
||||
#[case] outcome: Checked,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
Python::attach(|py| {
|
||||
let (callbacks, events) = callbacks(py, "8");
|
||||
callbacks.checked(outcome);
|
||||
assert_eq!(
|
||||
events
|
||||
.bind(py)
|
||||
.get_item(0)
|
||||
.unwrap()
|
||||
.extract::<Option<String>>()
|
||||
.unwrap()
|
||||
.as_deref(),
|
||||
expected
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn stopped_callbacks_are_not_invoked(#[from(python)] _python: ()) {
|
||||
Python::attach(|py| {
|
||||
let (callbacks, events) = callbacks(py, "8");
|
||||
callbacks.stopped.store(true, Ordering::Release);
|
||||
assert_eq!(callbacks.generation(), 0);
|
||||
callbacks.checked(Checked::NotModified);
|
||||
assert!(events.bind(py).is_empty());
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::unsupported_scheme("ftp://example.test")]
|
||||
#[case::relative("catalog.json")]
|
||||
fn invalid_source_precedes_python_config(#[from(python)] _python: (), #[case] source: &str) {
|
||||
Python::attach(|py| {
|
||||
let error = fetch_once(py, source, None, 1.0).unwrap_err();
|
||||
assert!(error.is_instance_of::<PyValueError>(py));
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::negative(-1.0)]
|
||||
#[case::nan(f64::NAN)]
|
||||
#[case::infinite(f64::INFINITY)]
|
||||
fn invalid_timeout_precedes_python_config(#[from(python)] _python: (), #[case] timeout: f64) {
|
||||
Python::attach(|py| {
|
||||
let error = fetch_once(py, "https://example.test", None, timeout).unwrap_err();
|
||||
assert!(error.is_instance_of::<PyValueError>(py));
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn nested_runtime_is_rejected(#[from(python)] _python: ()) {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.build()
|
||||
.unwrap();
|
||||
let _entered = runtime.enter();
|
||||
Python::attach(|py| {
|
||||
let error = fetch_once(py, "https://example.test", None, 1.0).unwrap_err();
|
||||
assert!(error.is_instance_of::<PyRuntimeError>(py));
|
||||
});
|
||||
}
|
||||
}
|
||||
510
litellm-rust/crates/python-bridge/tests/catalog_refresh.py
Normal file
510
litellm-rust/crates/python-bridge/tests/catalog_refresh.py
Normal file
|
|
@ -0,0 +1,510 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
import weakref
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
import litellm
|
||||
from litellm.rust_bridge import _native as native
|
||||
|
||||
|
||||
class Response:
|
||||
def __init__(self, status: int = 200, body: bytes | None = None, hold: bool = False) -> None:
|
||||
self.status = status
|
||||
self.body = native.ModelRegistry.bundled_bytes() if body is None else body
|
||||
self.requests: queue.Queue[tuple[str, str | None]] = queue.Queue()
|
||||
self.entered = threading.Event()
|
||||
self.release = threading.Event()
|
||||
if not hold:
|
||||
self.release.set()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def server(response: Response, tls: ssl.SSLContext | None = None) -> Iterator[str]:
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None:
|
||||
response.requests.put((self.path, self.headers.get("If-None-Match")))
|
||||
response.entered.set()
|
||||
response.release.wait(15)
|
||||
self.send_response(response.status)
|
||||
self.send_header("ETag", '"catalog-v1"')
|
||||
self.send_header("Retry-After", "0")
|
||||
self.send_header("Content-Length", str(len(response.body) if response.status == 200 else 0))
|
||||
self.end_headers()
|
||||
if response.status == 200:
|
||||
try:
|
||||
self.wfile.write(response.body)
|
||||
except (BrokenPipeError, ConnectionResetError, ssl.SSLError):
|
||||
pass
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None:
|
||||
pass
|
||||
|
||||
with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as http:
|
||||
if tls is not None:
|
||||
http.socket = tls.wrap_socket(http.socket, server_side=True)
|
||||
worker: Final = threading.Thread(target=http.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True)
|
||||
worker.start()
|
||||
try:
|
||||
yield f"{'https' if tls else 'http'}://127.0.0.1:{http.server_port}"
|
||||
finally:
|
||||
response.release.set()
|
||||
http.shutdown()
|
||||
worker.join(5)
|
||||
|
||||
|
||||
def refresh_threads() -> frozenset[str]:
|
||||
return frozenset(path.name for path in Path("/proc/self/task").iterdir() if catalog_thread(path))
|
||||
|
||||
|
||||
def catalog_thread(path: Path) -> bool:
|
||||
try:
|
||||
return (path / "comm").read_text().startswith("litellm-catalog")
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
def wait_for(predicate: Callable[[], bool], timeout: float = 5) -> None:
|
||||
deadline: Final = time.monotonic() + timeout
|
||||
while not predicate():
|
||||
if time.monotonic() >= deadline:
|
||||
raise AssertionError("condition did not complete")
|
||||
threading.Event().wait(0.01)
|
||||
|
||||
|
||||
class Observer:
|
||||
def __init__(self, accept: bool = True) -> None:
|
||||
self.value = 7
|
||||
self.accept = accept
|
||||
self.events: queue.Queue[tuple[str, object]] = queue.Queue()
|
||||
self.provenance: tuple[str | None, str | None] = (None, None)
|
||||
|
||||
def generation(self) -> int:
|
||||
return self.value
|
||||
|
||||
def install(self, seen: int, registry: native.ModelRegistry, errors: list[native.RowError]) -> int | None:
|
||||
if self.accept and seen == self.value:
|
||||
self.provenance = (registry.source_revision, registry.source_validator)
|
||||
self.value += 1
|
||||
self.events.put(("install", (seen, registry, errors)))
|
||||
return self.value
|
||||
self.events.put(("install", (seen, registry, errors)))
|
||||
return None
|
||||
|
||||
def checked(self, reason: str | None) -> None:
|
||||
self.events.put(("checked", reason))
|
||||
|
||||
def start(self, url: str, validator: str | None = None, revision: str | None = None) -> native.CatalogRefresher:
|
||||
return native.CatalogRefresher(
|
||||
url,
|
||||
installed_revision=revision,
|
||||
installed_validator=validator,
|
||||
install=self.install,
|
||||
checked=self.checked,
|
||||
generation=self.generation,
|
||||
)
|
||||
|
||||
|
||||
class CatalogRefreshTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.environment = patch.dict(os.environ, {"NO_PROXY": "*", "no_proxy": "*"})
|
||||
self.environment.start()
|
||||
self.addCleanup(self.environment.stop)
|
||||
self.ssl_verify = litellm.ssl_verify
|
||||
self.addCleanup(setattr, litellm, "ssl_verify", self.ssl_verify)
|
||||
self.threads = refresh_threads()
|
||||
self.assertFalse(native.process_state_started())
|
||||
|
||||
def tearDown(self) -> None:
|
||||
wait_for(lambda: refresh_threads() == self.threads)
|
||||
self.assertFalse(native.process_state_started())
|
||||
|
||||
def fetch(self, url: str, validator: str | None = None, timeout: float = 3) -> object:
|
||||
return native.fetch_once(url, if_none_match=validator, timeout_seconds=timeout)
|
||||
|
||||
def test_fetch_installs_nothing_and_preserves_validator_revision(self) -> None:
|
||||
response: Final = Response()
|
||||
original: Final = litellm.model_cost
|
||||
original_rows: Final = dict(original)
|
||||
with server(response) as url:
|
||||
result: Final = self.fetch(url, '"previous"')
|
||||
self.assertIsInstance(result, native.FetchedRegistry)
|
||||
assert isinstance(result, native.FetchedRegistry)
|
||||
self.assertEqual(result.registry.to_dict(), native.ModelRegistry.bundled().to_dict())
|
||||
self.assertEqual(result.validator, '"catalog-v1"')
|
||||
self.assertEqual(result.registry.source_revision, result.revision)
|
||||
self.assertEqual(result.registry.source_validator, result.validator)
|
||||
filtered, _ = result.registry.without_rejected()
|
||||
self.assertEqual(filtered.source_revision, result.revision)
|
||||
self.assertEqual(filtered.source_validator, result.validator)
|
||||
self.assertEqual(
|
||||
result.revision,
|
||||
hashlib.sha1(b"blob " + str(len(response.body)).encode() + b"\0" + response.body).hexdigest(),
|
||||
)
|
||||
self.assertEqual(response.requests.get_nowait(), ("/", '"previous"'))
|
||||
self.assertTrue(response.requests.empty())
|
||||
self.assertIs(litellm.model_cost, original)
|
||||
self.assertEqual(litellm.model_cost, original_rows)
|
||||
|
||||
def test_parse_and_bundled_have_no_source_metadata(self) -> None:
|
||||
bundled: Final = native.ModelRegistry.bundled()
|
||||
parsed: Final = native.ModelRegistry.parse(native.ModelRegistry.bundled_bytes())
|
||||
self.assertIsNone(bundled.source_revision)
|
||||
self.assertIsNone(bundled.source_validator)
|
||||
self.assertIsNone(parsed.source_revision)
|
||||
self.assertIsNone(parsed.source_validator)
|
||||
self.assertEqual(bundled.to_dict(), parsed.to_dict())
|
||||
with self.assertRaises(AttributeError):
|
||||
setattr(bundled, "source_revision", "not-writable")
|
||||
|
||||
def test_not_modified(self) -> None:
|
||||
response: Final = Response(304)
|
||||
with server(response) as url:
|
||||
self.assertIsInstance(self.fetch(url, '"old"'), native.RegistryNotModified)
|
||||
observer: Final = Observer()
|
||||
refresher: Final = observer.start(url, '"old"')
|
||||
self.addCleanup(refresher.stop)
|
||||
self.assertEqual(observer.events.get(timeout=5), ("checked", None))
|
||||
wait_for(lambda: refresh_threads() == self.threads)
|
||||
self.assertTrue(observer.events.empty())
|
||||
self.assertEqual(response.requests.get_nowait(), ("/", '"old"'))
|
||||
self.assertEqual(response.requests.get_nowait(), ("/", '"old"'))
|
||||
|
||||
def test_refresher_filters_rejected_rows_and_reports_them(self) -> None:
|
||||
body: Final = json.dumps(
|
||||
{
|
||||
**json.loads(native.ModelRegistry.bundled_bytes()),
|
||||
"p3-invalid": {"litellm_provider": "openai", "input_cost_per_token": -1},
|
||||
}
|
||||
).encode()
|
||||
response: Final = Response(body=body)
|
||||
with server(response) as url:
|
||||
observer: Final = Observer()
|
||||
refresher: Final = observer.start(url)
|
||||
self.addCleanup(refresher.stop)
|
||||
kind, payload = observer.events.get(timeout=5)
|
||||
self.assertEqual(kind, "install")
|
||||
assert isinstance(payload, tuple)
|
||||
seen, registry, errors = payload
|
||||
self.assertEqual(seen, 7)
|
||||
self.assertNotIn("p3-invalid", registry.to_dict())
|
||||
self.assertTrue(any(error.key == "p3-invalid" for error in errors))
|
||||
self.assertGreater(len(registry.to_dict()), 100)
|
||||
self.assertEqual(
|
||||
registry.source_revision,
|
||||
hashlib.sha1(b"blob " + str(len(body)).encode() + b"\0" + body).hexdigest(),
|
||||
)
|
||||
self.assertEqual(registry.source_validator, '"catalog-v1"')
|
||||
self.assertEqual(observer.provenance, (registry.source_revision, registry.source_validator))
|
||||
wait_for(lambda: refresh_threads() == self.threads)
|
||||
self.assertTrue(observer.events.empty())
|
||||
self.assertEqual(response.requests.qsize(), 1)
|
||||
|
||||
def test_stale_is_logged_once_without_checked_or_retry(self) -> None:
|
||||
response: Final = Response()
|
||||
with self.assertLogs(litellm.verbose_logger, level=logging.DEBUG) as logs, server(response) as url:
|
||||
observer: Final = Observer(accept=False)
|
||||
refresher: Final = observer.start(url)
|
||||
self.addCleanup(refresher.stop)
|
||||
self.assertEqual(observer.events.get(timeout=5)[0], "install")
|
||||
wait_for(lambda: refresh_threads() == self.threads)
|
||||
self.assertTrue(observer.events.empty())
|
||||
self.assertEqual(response.requests.qsize(), 1)
|
||||
self.assertTrue(
|
||||
any(getattr(record, "rust_fields", {}).get("outcome") == "stale" for record in logs.records), logs.output
|
||||
)
|
||||
self.assertEqual(observer.provenance, (None, None))
|
||||
|
||||
def test_stop_held_response_has_no_callback_and_exits_thread(self) -> None:
|
||||
response: Final = Response(hold=True)
|
||||
with server(response) as url:
|
||||
observer: Final = Observer()
|
||||
refresher: Final = observer.start(url)
|
||||
self.addCleanup(refresher.stop)
|
||||
self.assertTrue(response.entered.wait(5))
|
||||
before: Final = time.monotonic()
|
||||
refresher.stop()
|
||||
self.assertLess(time.monotonic() - before, 0.5)
|
||||
wait_for(lambda: refresh_threads() == self.threads)
|
||||
response.release.set()
|
||||
refresher.stop()
|
||||
self.assertTrue(observer.events.empty())
|
||||
|
||||
def test_new_generation_fences_inflight_result(self) -> None:
|
||||
response: Final = Response(hold=True)
|
||||
with server(response) as url:
|
||||
observer: Final = Observer()
|
||||
refresher: Final = observer.start(url)
|
||||
self.addCleanup(refresher.stop)
|
||||
self.assertTrue(response.entered.wait(5))
|
||||
observer.value = 8
|
||||
response.release.set()
|
||||
wait_for(lambda: refresh_threads() == self.threads)
|
||||
self.assertTrue(observer.events.empty())
|
||||
|
||||
self.assertEqual(observer.provenance, (None, None))
|
||||
|
||||
def test_retryable_once_does_not_retry(self) -> None:
|
||||
response: Final = Response(503)
|
||||
with server(response) as url:
|
||||
result: Final = self.fetch(url)
|
||||
self.assertIsInstance(result, native.RegistryUnavailable)
|
||||
assert isinstance(result, native.RegistryUnavailable)
|
||||
self.assertTrue(result.retryable)
|
||||
self.assertEqual(result.retry_after, 0)
|
||||
self.assertIn("503", result.reason)
|
||||
self.assertEqual(response.requests.qsize(), 1)
|
||||
|
||||
def test_failure_calls_checked_reason_once(self) -> None:
|
||||
response: Final = Response(404)
|
||||
with server(response) as url:
|
||||
observer: Final = Observer()
|
||||
refresher: Final = observer.start(url)
|
||||
self.addCleanup(refresher.stop)
|
||||
kind, reason = observer.events.get(timeout=5)
|
||||
self.assertEqual(kind, "checked")
|
||||
self.assertIn("404", str(reason))
|
||||
wait_for(lambda: refresh_threads() == self.threads)
|
||||
self.assertTrue(observer.events.empty())
|
||||
|
||||
def test_file_source_ignores_http_validator(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path: Final = Path(directory) / "catalog.json"
|
||||
path.write_bytes(native.ModelRegistry.bundled_bytes())
|
||||
result: Final = self.fetch(path.as_uri())
|
||||
self.assertIsInstance(result, native.FetchedRegistry)
|
||||
assert isinstance(result, native.FetchedRegistry)
|
||||
self.assertIsNone(result.validator)
|
||||
self.assertIsInstance(self.fetch(path.as_uri(), '"old"'), native.FetchedRegistry)
|
||||
|
||||
def test_s3_endpoint_and_conditional_header(self) -> None:
|
||||
response: Final = Response()
|
||||
with (
|
||||
server(response) as url,
|
||||
patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"AWS_ENDPOINT_URL_S3": url,
|
||||
"AWS_ACCESS_KEY_ID": "local-test",
|
||||
"AWS_SECRET_ACCESS_KEY": "local-test",
|
||||
"AWS_REGION": "us-east-1",
|
||||
"AWS_EC2_METADATA_DISABLED": "true",
|
||||
},
|
||||
),
|
||||
):
|
||||
result: Final = self.fetch("s3://bucket/catalog.json", '"old"')
|
||||
self.assertIsInstance(result, native.FetchedRegistry)
|
||||
assert isinstance(result, native.FetchedRegistry)
|
||||
self.assertEqual(result.registry.to_dict(), native.ModelRegistry.bundled().to_dict())
|
||||
self.assertEqual(response.requests.get_nowait(), ("/bucket/catalog.json?x-id=GetObject", '"old"'))
|
||||
self.assertTrue(response.requests.empty())
|
||||
|
||||
def test_unsupported_source_and_invalid_timeout(self) -> None:
|
||||
with self.assertRaises(ValueError):
|
||||
self.fetch("ftp://x")
|
||||
with self.assertRaises(ValueError):
|
||||
Observer().start("ftp://x")
|
||||
with self.assertRaises(ValueError):
|
||||
self.fetch("http://127.0.0.1", timeout=float("nan"))
|
||||
|
||||
def test_timeout_releases_gil_and_performs_one_request(self) -> None:
|
||||
response: Final = Response(hold=True)
|
||||
with server(response) as url:
|
||||
result: Final = self.fetch(url, timeout=0.05)
|
||||
self.assertTrue(response.entered.is_set())
|
||||
self.assertIsInstance(result, native.RegistryUnavailable)
|
||||
self.assertEqual(response.requests.qsize(), 1)
|
||||
|
||||
def test_parse_and_integrity_failure(self) -> None:
|
||||
response: Final = Response(body=b'{"small": {"litellm_provider": "openai"}}')
|
||||
with server(response) as url:
|
||||
result: Final = self.fetch(url)
|
||||
self.assertIsInstance(result, native.RegistryUnavailable)
|
||||
assert isinstance(result, native.RegistryUnavailable)
|
||||
self.assertFalse(result.retryable)
|
||||
self.assertTrue(result.reason)
|
||||
|
||||
def test_rejected_document_does_not_advance_provenance(self) -> None:
|
||||
response: Final = Response(body=b"not-json")
|
||||
observer: Final = Observer()
|
||||
with server(response) as url:
|
||||
refresher: Final = observer.start(url)
|
||||
self.addCleanup(refresher.stop)
|
||||
kind, reason = observer.events.get(timeout=5)
|
||||
self.assertEqual(kind, "checked")
|
||||
self.assertTrue(reason)
|
||||
wait_for(lambda: refresh_threads() == self.threads)
|
||||
self.assertTrue(observer.events.empty())
|
||||
self.assertEqual(observer.provenance, (None, None))
|
||||
|
||||
def test_tls_verify_applies_to_fetch_and_refresher(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
key: Final = Path(directory) / "key.pem"
|
||||
cert: Final = Path(directory) / "cert.pem"
|
||||
subprocess.run(
|
||||
[
|
||||
"openssl",
|
||||
"req",
|
||||
"-x509",
|
||||
"-newkey",
|
||||
"rsa:2048",
|
||||
"-nodes",
|
||||
"-days",
|
||||
"1",
|
||||
"-subj",
|
||||
"/CN=localhost",
|
||||
"-addext",
|
||||
"subjectAltName=IP:127.0.0.1",
|
||||
"-keyout",
|
||||
str(key),
|
||||
"-out",
|
||||
str(cert),
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
context.load_cert_chain(cert, key)
|
||||
response: Final = Response()
|
||||
with server(response, context) as url:
|
||||
litellm.ssl_verify = False
|
||||
self.assertIsInstance(self.fetch(url), native.FetchedRegistry)
|
||||
observer: Final = Observer()
|
||||
refresher: Final = observer.start(url)
|
||||
self.addCleanup(refresher.stop)
|
||||
litellm.ssl_verify = True
|
||||
self.assertEqual(observer.events.get(timeout=5)[0], "install")
|
||||
failed: Final = self.fetch(url)
|
||||
self.assertIsInstance(failed, native.RegistryUnavailable)
|
||||
assert isinstance(failed, native.RegistryUnavailable)
|
||||
self.assertTrue(failed.reason)
|
||||
failure_observer: Final = Observer()
|
||||
failure_refresher: Final = failure_observer.start(url)
|
||||
self.addCleanup(failure_refresher.stop)
|
||||
kind, reason = failure_observer.events.get(timeout=15)
|
||||
self.assertEqual(kind, "checked")
|
||||
self.assertTrue(reason)
|
||||
|
||||
def test_callback_exception_is_unraisable_and_terminal(self) -> None:
|
||||
failures: Final[queue.Queue[object]] = queue.Queue()
|
||||
|
||||
def install(seen: int, registry: native.ModelRegistry, errors: list[native.RowError]) -> int:
|
||||
raise RuntimeError("callback-test")
|
||||
|
||||
response: Final = Response()
|
||||
observer: Final = Observer()
|
||||
with patch.object(sys, "unraisablehook", failures.put), server(response) as url:
|
||||
refresher: Final = native.CatalogRefresher(
|
||||
url,
|
||||
installed_revision=None,
|
||||
installed_validator=None,
|
||||
install=install,
|
||||
checked=observer.checked,
|
||||
generation=observer.generation,
|
||||
)
|
||||
self.addCleanup(refresher.stop)
|
||||
failure: Final = failures.get(timeout=5)
|
||||
self.assertIn("callback-test", str(getattr(failure, "exc_value")))
|
||||
wait_for(lambda: refresh_threads() == self.threads)
|
||||
self.assertTrue(observer.events.empty())
|
||||
self.assertTrue(failures.empty())
|
||||
|
||||
def test_generation_failure_starts_no_transport(self) -> None:
|
||||
failures: Final[queue.Queue[object]] = queue.Queue()
|
||||
|
||||
def generation() -> int:
|
||||
raise RuntimeError("generation-test")
|
||||
|
||||
response: Final = Response()
|
||||
observer: Final = Observer()
|
||||
with patch.object(sys, "unraisablehook", failures.put), server(response) as url:
|
||||
refresher: Final = native.CatalogRefresher(
|
||||
url,
|
||||
installed_revision=None,
|
||||
installed_validator=None,
|
||||
install=observer.install,
|
||||
checked=observer.checked,
|
||||
generation=generation,
|
||||
)
|
||||
self.addCleanup(refresher.stop)
|
||||
failure: Final = failures.get(timeout=5)
|
||||
self.assertIn("generation-test", str(getattr(failure, "exc_value")))
|
||||
wait_for(lambda: refresh_threads() == self.threads)
|
||||
self.assertTrue(response.requests.empty())
|
||||
self.assertTrue(observer.events.empty())
|
||||
|
||||
def test_callback_cycle_collected_after_worker_completes(self) -> None:
|
||||
finalized: Final[queue.Queue[bool]] = queue.Queue()
|
||||
|
||||
class Owner(Observer):
|
||||
def __init__(self, url: str) -> None:
|
||||
super().__init__()
|
||||
self.refresher = self.start(url)
|
||||
|
||||
def __del__(self) -> None:
|
||||
self.refresher.stop()
|
||||
self.refresher.after_fork()
|
||||
finalized.put(True)
|
||||
|
||||
response: Final = Response(304)
|
||||
with server(response) as url:
|
||||
|
||||
def make_cycle() -> weakref.ReferenceType[Owner]:
|
||||
owner: Final = Owner(url)
|
||||
self.assertEqual(owner.events.get(timeout=5), ("checked", None))
|
||||
return weakref.ref(owner)
|
||||
|
||||
reference: Final = make_cycle()
|
||||
wait_for(lambda: refresh_threads() == self.threads)
|
||||
gc.collect()
|
||||
self.assertIsNone(reference())
|
||||
self.assertTrue(finalized.get(timeout=1))
|
||||
|
||||
@unittest.skipUnless(hasattr(os, "fork"), "requires fork")
|
||||
def test_after_fork_restarts_unadopted_child(self) -> None:
|
||||
response: Final = Response(hold=True)
|
||||
with server(response) as url:
|
||||
observer: Final = Observer()
|
||||
refresher: Final = observer.start(url)
|
||||
self.addCleanup(refresher.stop)
|
||||
self.assertTrue(response.entered.wait(5))
|
||||
read_fd, write_fd = os.pipe()
|
||||
pid: Final = os.fork()
|
||||
if pid == 0:
|
||||
os.close(read_fd)
|
||||
try:
|
||||
refresher.after_fork()
|
||||
kind, _ = observer.events.get(timeout=10)
|
||||
refresher.stop()
|
||||
os.write(write_fd, kind.encode())
|
||||
finally:
|
||||
os._exit(0)
|
||||
os.close(write_fd)
|
||||
try:
|
||||
response.release.set()
|
||||
self.assertEqual(os.read(read_fd, 32), b"install")
|
||||
self.assertEqual(os.waitpid(pid, 0)[1], 0)
|
||||
self.assertEqual(observer.events.get(timeout=5)[0], "install")
|
||||
finally:
|
||||
os.close(read_fd)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
from asyncio import Future
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from typing import Literal, Never, final
|
||||
|
||||
import httpx
|
||||
|
|
@ -55,6 +55,10 @@ class RowError:
|
|||
|
||||
@final
|
||||
class ModelRegistry:
|
||||
@property
|
||||
def source_revision(self) -> str | None: ...
|
||||
@property
|
||||
def source_validator(self) -> str | None: ...
|
||||
@staticmethod
|
||||
def parse(body: bytes) -> ModelRegistry: ...
|
||||
@staticmethod
|
||||
|
|
@ -69,6 +73,45 @@ class ModelRegistry:
|
|||
def check_size(self, reference: int) -> RowError | None: ...
|
||||
|
||||
def model_row_errors(key: str, row: dict[str, JsonValue]) -> list[RowError]: ...
|
||||
@final
|
||||
class FetchedRegistry:
|
||||
@property
|
||||
def registry(self) -> ModelRegistry: ...
|
||||
@property
|
||||
def revision(self) -> str: ...
|
||||
@property
|
||||
def validator(self) -> str | None: ...
|
||||
|
||||
@final
|
||||
class RegistryNotModified: ...
|
||||
|
||||
@final
|
||||
class RegistryUnavailable:
|
||||
@property
|
||||
def reason(self) -> str: ...
|
||||
@property
|
||||
def retryable(self) -> bool: ...
|
||||
@property
|
||||
def retry_after(self) -> float | None: ...
|
||||
|
||||
def fetch_once(
|
||||
source: str, *, if_none_match: str | None, timeout_seconds: float
|
||||
) -> FetchedRegistry | RegistryNotModified | RegistryUnavailable: ...
|
||||
@final
|
||||
class CatalogRefresher:
|
||||
def __new__(
|
||||
cls,
|
||||
source: str,
|
||||
*,
|
||||
installed_revision: str | None,
|
||||
installed_validator: str | None,
|
||||
install: Callable[[int, ModelRegistry, list[RowError]], int | None],
|
||||
checked: Callable[[str | None], None],
|
||||
generation: Callable[[], int],
|
||||
) -> CatalogRefresher: ...
|
||||
def stop(self) -> None: ...
|
||||
def after_fork(self) -> None: ...
|
||||
|
||||
def write_rows(target: dict[str, object], set: dict[str, object], remove: list[str]) -> None: ...
|
||||
@final
|
||||
class ModelCostLayers:
|
||||
|
|
@ -384,6 +427,8 @@ def process_state_started() -> bool: ...
|
|||
def reserve_process_for_forking() -> None: ...
|
||||
|
||||
__all__ = [
|
||||
"CatalogRefresher",
|
||||
"FetchedRegistry",
|
||||
"ForkedAfterNativeRuntimeStarted",
|
||||
"HuggingFaceEncoding",
|
||||
"ModelCostLayers",
|
||||
|
|
@ -397,6 +442,8 @@ __all__ = [
|
|||
"Priced",
|
||||
"ProcessReservedForForking",
|
||||
"RegistrationTarget",
|
||||
"RegistryNotModified",
|
||||
"RegistryUnavailable",
|
||||
"ResolvedModel",
|
||||
"ResponsesWebSocketConnection",
|
||||
"RowError",
|
||||
|
|
@ -417,6 +464,7 @@ __all__ = [
|
|||
"atranscription",
|
||||
"completion",
|
||||
"embedding",
|
||||
"fetch_once",
|
||||
"gil_stats",
|
||||
"invalidate_model_cost_index",
|
||||
"messages",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue