feat(router): add the Rust router core with the host as its only boundary

Adds the litellm-router-core crate, its step-driven PyO3 bridge
(RouterCoreHandle / RouterCall) and RustCore, served behind LITELLM_RUST
through the ROUTER route rule. The host call-out is the only boundary:
every attempt goes through Host::invoke, there is no native executor path,
and RustCore never builds, wraps or forwards to PythonCore, so
router_python_core.py stays deletable. Attempt preparation lives in
litellm/router_attempt.py so it outlives router_python_core.py.

RustCore matches PythonCore's behavior, including its known bugs (fixed in
follow-up PRs). Routing decisions (strategy picks, routing groups, PTU,
pre-routing, pre-call checks, fallback walk, health routing, access groups,
weights, cooldowns, allowed fails, usage) live in the crate; work that needs
live Python objects (request shaping, endpoints, streaming, registry, model
info, callbacks, alerting, prompt management) lives in host-side
litellm/router_rust_*.py modules driven by Rust steps.

tests/unit/test_router/core characterizes PythonCore's public behavior and
runs every test on both cores; the remaining Rust differences (the router's
own outcome callbacks) are listed in rust_parity_pending.txt.

litellm-rust/Cargo.lock

@@ -4331,6 +4331,7 @@ dependencies = [
 name = "litellm-python-bridge"
 version = "0.1.0"
 dependencies = [
+ "async-trait",
  "aws-sdk-secretsmanager",
  "bytes",
  "criterion",
@@ -4357,6 +4358,7 @@ dependencies = [
  "litellm-llms",
  "litellm-llms-types",
  "litellm-model-catalog",
+ "litellm-router-core",
  "litellm-router-types",
  "litellm-secret-value",
  "litellm-secrets",
@@ -4439,6 +4441,24 @@ dependencies = [
 [[package]]
 name = "litellm-router-core"
 version = "0.1.0"
+dependencies = [
+ "arc-swap",
+ "async-trait",
+ "jiff",
+ "litellm-cache",
+ "litellm-router-types",
+ "parking_lot",
+ "rand 0.8.7",
+ "rstest",
+ "serde",
+ "serde_json",
+ "serde_with",
+ "sha2 0.10.9",
+ "strum",
+ "thiserror 2.0.19",
+ "tokio",
+ "tracing",
+]
 
 [[package]]
 name = "litellm-router-types"

litellm-rust/Cargo.toml

@@ -13,6 +13,7 @@ litellm-config = { path = "crates/config" }
 litellm-error = { path = "crates/error" }
 litellm-router = { path = "crates/router" }
 litellm-router-types = { path = "crates/router-types" }
+litellm-router-core = { path = "crates/router-core" }
 litellm-tracing = { path = "crates/tracing" }
 litellm-spend-clickhouse = { path = "crates/spend-clickhouse" }
 litellm-storage-clickhouse = { path = "crates/storage-clickhouse" }
@@ -78,6 +79,8 @@ litellm-cost = { path = "crates/cost" }
 litellm-model-catalog = { path = "crates/model-catalog" }
 litellm-model-catalog-refresh = { path = "crates/model-catalog-refresh" }
 
+arc-swap = "1.9.2"
+async-trait = "0.1.91"
 askama = { version = "0.16.1", default-features = false, features = ["derive", "std"] }
 tracing = "0.1"
 axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio", "multipart"] }

litellm-rust/crates/python-bridge/Cargo.toml

@@ -25,6 +25,7 @@ litellm-tracing.workspace = true
 litellm-spend-clickhouse.workspace = true
 litellm-storage-clickhouse.workspace = true
 litellm-host.workspace = true
+async-trait.workspace = true
 bytes.workspace = true
 futures-util.workspace = true
 litellm-cache.workspace = true
@@ -49,6 +50,7 @@ litellm-secrets = { workspace = true, features = ["aws", "azure", "google", "has
 litellm-secrets-types.workspace = true
 litellm-llms-types.workspace = true
 litellm-router-types.workspace = true
+litellm-router-core.workspace = true
 litellm-host-python.workspace = true
 litellm-model-catalog.workspace = true
 litellm-token-counter = { path = "../token-counter", default-features = false }

litellm-rust/crates/python-bridge/src/lib.rs

@@ -12,6 +12,7 @@ mod marshal;
 mod model_registry;
 mod preflight;
 mod python_settings;
+mod router;
 mod routes;
 mod secrets;
 mod tokenizer;
@@ -36,6 +37,8 @@ mod _native {
     #[pymodule_export]
     use crate::model_registry::bundled_model_cost_map;
     #[pymodule_export]
+    use crate::router::{RouterCall, RouterCoreHandle};
+    #[pymodule_export]
     use crate::routes::audio_transcription::{atranscription, transcription};
     #[pymodule_export]
     use crate::routes::chat_completions::{acompletion, completion};
@@ -120,6 +123,8 @@ mod tests {
                 "bundled_model_cost_map",
                 "TokenCounter",
                 "Tokenizer",
+                "RouterCoreHandle",
+                "RouterCall",
                 "gil_stats",
                 "process_state_started",
                 "reserve_process_for_forking",

litellm-rust/crates/python-bridge/src/router/config.rs

@@ -0,0 +1,357 @@
+//! The documents `RouterCoreHandle` reads: the router's static and dynamic config, a call's
+//! request and an `apply` change set. A config key router-core does not read declines the
+//! handle, so Python keeps the router on its own core.
+
+use std::collections::BTreeMap;
+
+use litellm_router_core::select::PassThrough;
+use litellm_router_core::{
+    ChangeSet, ConfigError, DeploymentSpec, RouterConfig, RouterDynamicConfig, RouterRequest,
+    config::{Fallbacks, RetryPolicy},
+    mock::MockFailure,
+    registry::IndexOverrides,
+};
+use serde::{Deserialize, de::DeserializeOwned};
+use serde_json::{Map, Value};
+
+/// The `RouterConfig` keys router-core reads.
+const STATIC_KEYS: &[&str] = &[
+    "model_list",
+    "routing_strategy",
+    "routing_groups",
+    "routing_group_members",
+    "plugins",
+    "cooldown_time",
+    "disable_cooldowns",
+    "allowed_fails_policy",
+    "retry_policy",
+    "model_group_retry_policy",
+    "retry_after",
+    "timeout",
+    "request_timeout",
+    "drop_params",
+    "enable_pre_call_checks",
+    "routing_plugins",
+    "callback_filters",
+    "enable_health_check_routing",
+    "health_check_staleness_threshold",
+    "background_health_check_model_groups",
+    "enable_weighted_failover",
+    "fallback_on_unsupported_params",
+    "provider_default_deployment_ids",
+];
+
+/// The `RouterDynamicConfig` keys router-core reads.
+const DYNAMIC_KEYS: &[&str] = &[
+    "version",
+    "model_alias_map",
+    "model_group_alias",
+    "fallbacks",
+    "default_fallbacks",
+    "context_window_fallbacks",
+    "content_policy_fallbacks",
+    "num_retries",
+    "max_fallbacks",
+    "allowed_fails",
+    "fallback_keys",
+    "default_deployment",
+    "default_deployment_raises",
+];
+
+/// Router arguments that do not change routing, accepted in either config and dropped.
+const IGNORED_KEYS: &[&str] = &[
+    "set_verbose",
+    "debug_level",
+    "alerting_config",
+    "client_ttl",
+];
+
+/// Fallback lists Python treats as unset when empty (`context_window_fallbacks or None`).
+const EMPTY_MEANS_UNSET: &[&str] = &["context_window_fallbacks", "content_policy_fallbacks"];
+
+/// Which document a failure is about.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display)]
+pub(crate) enum Document {
+    #[strum(serialize = "router config")]
+    Static,
+    #[strum(serialize = "dynamic router config")]
+    Dynamic,
+    #[strum(serialize = "router request")]
+    Request,
+    #[strum(serialize = "change set")]
+    Changes,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub(crate) enum Error {
+    #[error("the {document} must be a JSON object")]
+    NotObject { document: Document },
+    #[error("router-core does not read `{key}` in the {document}")]
+    UnknownKey { document: Document, key: String },
+    #[error("the {document} does not have the shape router-core reads")]
+    Shape {
+        document: Document,
+        #[source]
+        source: serde_json::Error,
+    },
+    #[error("the request names no model")]
+    MissingModel,
+    #[error("router-core rejected the router config")]
+    Core(#[from] ConfigError),
+}
+
+impl Error {
+    /// Whether router-core cannot serve this config, so Python keeps its own core.
+    pub(crate) fn declined(&self) -> bool {
+        match self {
+            Self::UnknownKey { .. } | Self::Core(_) => true,
+            Self::Shape { document, .. } => {
+                matches!(document, Document::Static | Document::Dynamic)
+            }
+            Self::NotObject { .. } | Self::MissingModel => false,
+        }
+    }
+}
+
+fn object(document: Document, value: Value) -> Result<Map<String, Value>, Error> {
+    match value {
+        Value::Object(object) => Ok(object),
+        _ => Err(Error::NotObject { document }),
+    }
+}
+
+fn shaped<T: DeserializeOwned>(document: Document, object: Map<String, Value>) -> Result<T, Error> {
+    serde_json::from_value(Value::Object(object))
+        .map_err(|source| Error::Shape { document, source })
+}
+
+/// Drop unset and ignored keys, then reject any key outside `known`.
+fn routing_keys(
+    document: Document,
+    value: Value,
+    known: &[&str],
+) -> Result<Map<String, Value>, Error> {
+    let mut object = object(document, value)?;
+    object.retain(|key, value| {
+        let empty_unset = EMPTY_MEANS_UNSET.contains(&key.as_str())
+            && value.as_array().is_some_and(Vec::is_empty);
+        !(value.is_null() || empty_unset || IGNORED_KEYS.contains(&key.as_str()))
+    });
+    if let Some(key) = object.keys().find(|key| !known.contains(&key.as_str())) {
+        return Err(Error::UnknownKey {
+            document,
+            key: key.clone(),
+        });
+    }
+    Ok(object)
+}
+
+pub(crate) fn router_config(value: Value) -> Result<RouterConfig, Error> {
+    let object = routing_keys(Document::Static, value, STATIC_KEYS)?;
+    shaped(Document::Static, object)
+}
+
+pub(crate) fn dynamic_config(value: Value) -> Result<RouterDynamicConfig, Error> {
+    let object = routing_keys(Document::Dynamic, value, DYNAMIC_KEYS)?;
+    shaped(Document::Dynamic, object)
+}
+
+/// A call's request as Python spells it.
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct RequestSpec {
+    model: Option<String>,
+    team_id: Option<String>,
+    proxy_admin: Option<bool>,
+    allowed_models: Option<Vec<String>>,
+    admin_model: Option<String>,
+    specific_deployment: Option<bool>,
+    num_retries: Option<u32>,
+    disable_fallbacks: Option<bool>,
+    router_weights: Option<Value>,
+    pinned_deployment_id: Option<String>,
+    routing_strategy: Option<String>,
+    filter_tags: Option<bool>,
+    /// `listed` or `healthy`: which pass-through pick the request makes.
+    pass_through: Option<String>,
+    target_order: Option<i64>,
+    excluded_ids: Option<Vec<String>>,
+    retry_skipped_ids: Option<Vec<String>>,
+    pre_routes: Option<bool>,
+    has_image: Option<bool>,
+    unrouted: Option<bool>,
+    model_group_retry_policy: Option<BTreeMap<String, RetryPolicy>>,
+    retry_policy_raises: Option<Vec<String>>,
+    #[serde(default, deserialize_with = "present")]
+    fallbacks: Option<Option<Fallbacks>>,
+    #[serde(default, deserialize_with = "present")]
+    context_window_fallbacks: Option<Option<Fallbacks>>,
+    #[serde(default, deserialize_with = "present")]
+    content_policy_fallbacks: Option<Option<Fallbacks>>,
+    max_fallbacks: Option<u32>,
+    mock_failure: Option<MockFailure>,
+    cooldown_time: Option<f64>,
+    health_check_probe: Option<bool>,
+    body: Option<Value>,
+}
+
+/// A key the request passed, even as `None`: `Some(None)`.
+fn present<'de, D, T>(deserializer: D) -> Result<Option<Option<T>>, D::Error>
+where
+    D: serde::Deserializer<'de>,
+    T: Deserialize<'de>,
+{
+    Option::<T>::deserialize(deserializer).map(Some)
+}
+
+/// Read a request; `model` replaces the request's own when given.
+pub(crate) fn request(value: Value, model: Option<&str>) -> Result<RouterRequest, Error> {
+    let spec: RequestSpec = shaped(Document::Request, object(Document::Request, value)?)?;
+    let model = model
+        .map(str::to_owned)
+        .or(spec.model)
+        .ok_or(Error::MissingModel)?;
+    Ok(RouterRequest {
+        model,
+        body: spec.body.unwrap_or_default(),
+        team_id: spec.team_id,
+        proxy_admin: spec.proxy_admin.unwrap_or(false),
+        allowed_models: spec.allowed_models.unwrap_or_default(),
+        admin_model: spec.admin_model,
+        specific_deployment: spec.specific_deployment.unwrap_or(false),
+        num_retries: spec.num_retries,
+        disable_fallbacks: spec.disable_fallbacks.unwrap_or(false),
+        router_weights: spec.router_weights,
+        pinned_deployment_id: spec.pinned_deployment_id,
+        routing_strategy: spec.routing_strategy,
+        filter_tags: spec.filter_tags.unwrap_or(false),
+        pass_through: match spec.pass_through.as_deref() {
+            None => None,
+            Some("healthy") => Some(PassThrough::Healthy),
+            Some(_) => Some(PassThrough::Listed),
+        },
+        target_order: spec.target_order,
+        excluded_ids: spec.excluded_ids.unwrap_or_default(),
+        retry_skipped_ids: spec.retry_skipped_ids.unwrap_or_default(),
+        pre_routes: spec.pre_routes.unwrap_or(false),
+        has_image: spec.has_image.unwrap_or(false),
+        unrouted: spec.unrouted.unwrap_or(false),
+        model_group_retry_policy: spec.model_group_retry_policy,
+        retry_policy_raises: spec.retry_policy_raises.unwrap_or_default(),
+        fallbacks: spec.fallbacks,
+        context_window_fallbacks: spec.context_window_fallbacks,
+        content_policy_fallbacks: spec.content_policy_fallbacks,
+        max_fallbacks: spec.max_fallbacks,
+        mock_failure: spec.mock_failure,
+        cooldown_time: spec.cooldown_time,
+        health_check_probe: spec.health_check_probe.unwrap_or(false),
+        payload: None,
+    })
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct ChangesSpec {
+    /// Every deployment, replacing the current list before the upserts and deletes.
+    #[serde(default)]
+    replace: Option<Vec<DeploymentSpec>>,
+    #[serde(default)]
+    upsert: Vec<DeploymentSpec>,
+    #[serde(default)]
+    delete: Vec<String>,
+    /// A new dynamic document (the same keys `RouterCoreHandle(static, dynamic)` takes).
+    #[serde(default)]
+    settings: Option<Value>,
+    /// The registry indexes the host wrote (`IndexOverrides`); `{}` drops them.
+    #[serde(default)]
+    index: Option<IndexOverrides>,
+}
+
+pub(crate) fn changes(value: Value) -> Result<ChangeSet, Error> {
+    let spec: ChangesSpec = shaped(Document::Changes, object(Document::Changes, value)?)?;
+    Ok(ChangeSet {
+        replace: spec.replace,
+        upserts: spec.upsert,
+        deletes: spec.delete,
+        settings: spec.settings.map(dynamic_config).transpose()?,
+        index: spec.index,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+
+    #[rstest]
+    #[case::unknown_key(json!({"routing_strategy_args": {"ttl": 1}}), "routing_strategy_args")]
+    #[case::dynamic_key_in_static(json!({"num_retries": 2}), "num_retries")]
+    fn an_unread_static_key_declines_naming_the_key(#[case] json: Value, #[case] key: &str) {
+        let error = router_config(json).unwrap_err();
+        assert!(error.declined());
+        assert!(matches!(&error, Error::UnknownKey { key: named, .. } if named == key));
+    }
+
+    #[rstest]
+    #[case::null_unknown(json!({"routing_strategy_args": null}))]
+    #[case::ignored(json!({"set_verbose": true, "debug_level": "INFO"}))]
+    #[case::pre_call_checks_off(json!({"enable_pre_call_checks": false}))]
+    #[case::pre_call_checks_on(json!({"enable_pre_call_checks": true}))]
+    fn unset_and_ignored_keys_are_accepted(#[case] json: Value) {
+        router_config(json).unwrap();
+    }
+
+    #[rstest]
+    #[case::not_object(json!([]), false)]
+    #[case::bad_shape(json!({"retry_after": "soon"}), true)]
+    fn a_config_that_is_not_an_object_is_a_value_error_and_a_wrong_shape_declines(
+        #[case] json: Value,
+        #[case] declined: bool,
+    ) {
+        assert_eq!(router_config(json).unwrap_err().declined(), declined);
+    }
+
+    #[rstest]
+    fn python_router_defaults_read_as_unset() {
+        let dynamic = dynamic_config(json!({
+            "num_retries": null, "fallbacks": [], "context_window_fallbacks": [],
+            "content_policy_fallbacks": [], "model_group_alias": {"gpt": "gpt-4"},
+        }))
+        .unwrap();
+        assert_eq!(dynamic.num_retries, None);
+        assert!(dynamic.context_window_fallbacks.is_none());
+        assert!(dynamic.content_policy_fallbacks.is_none());
+        assert_eq!(dynamic.model_alias_map["gpt"], "gpt-4");
+    }
+
+    #[rstest]
+    #[case::own_model(json!({"model": "gpt", "num_retries": 1}), None, "gpt")]
+    #[case::override_model(json!({"model": "gpt"}), Some("other"), "other")]
+    fn a_request_reads_its_model(
+        #[case] json: Value,
+        #[case] model: Option<&str>,
+        #[case] expected: &str,
+    ) {
+        assert_eq!(request(json, model).unwrap().model, expected);
+    }
+
+    #[rstest]
+    #[case::unknown_field(json!({"model": "gpt", "temperature": 1}))]
+    #[case::no_model(json!({"team_id": "t"}))]
+    fn a_bad_request_is_a_value_error(#[case] json: Value) {
+        assert!(!request(json, None).unwrap_err().declined());
+    }
+
+    #[rstest]
+    fn a_change_set_reads_upserts_and_deletes() {
+        let changes = changes(json!({
+            "upsert": [{"model_name": "m", "litellm_params": {"model": "openai/x"}}],
+            "delete": ["old"],
+        }))
+        .unwrap();
+        assert_eq!(changes.upserts.len(), 1);
+        assert_eq!(changes.deletes, ["old"]);
+    }
+}

litellm-rust/crates/python-bridge/src/router/retries.rs

@@ -0,0 +1,55 @@
+//! The retry decisions `RouterCoreHandle` exposes as members: `get_num_retries_from_retry_policy`
+//! and `should_retry_this_error`, decided by router-core for a failure Python describes the way it
+//! describes a failed attempt.
+
+use litellm_router_core::{
+    AttemptError, Router,
+    retry::{RetryContext, resolve_retry_policy, should_retry as core_should_retry},
+};
+use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict};
+
+fn failure(outcome: &Bound<'_, PyDict>) -> PyResult<AttemptError> {
+    match super::attempt_outcome(outcome)? {
+        Err(failure) => Ok(failure),
+        Ok(_) => Err(PyValueError::new_err("a failure needs `ok: False`")),
+    }
+}
+
+pub(crate) fn policy_retries(
+    router: &Router,
+    outcome: &Bound<'_, PyDict>,
+    model_group: Option<&str>,
+) -> PyResult<Option<u32>> {
+    let failure = failure(outcome)?;
+    let settings = router.settings();
+    let policy = match model_group {
+        Some(group) => resolve_retry_policy(
+            settings.retry_policy.as_ref(),
+            group,
+            Some(&settings.model_group_retry_policy),
+        ),
+        None => settings.retry_policy.as_ref(),
+    };
+    Ok(policy.and_then(|policy| policy.retries_for(&failure)))
+}
+
+pub(crate) fn allowed_fails(router: &Router, outcome: &Bound<'_, PyDict>) -> PyResult<Option<u32>> {
+    Ok(router.settings().allowed_fails_for(&failure(outcome)?))
+}
+
+pub(crate) fn should_retry(
+    outcome: &Bound<'_, PyDict>,
+    (healthy, all): (usize, usize),
+    (context_window, content_policy, regular): (bool, bool, bool),
+) -> PyResult<bool> {
+    Ok(core_should_retry(
+        &failure(outcome)?,
+        RetryContext {
+            healthy,
+            all,
+            has_context_window_fallbacks: context_window,
+            has_content_policy_fallbacks: content_policy,
+            has_regular_fallbacks: regular,
+        },
+    ))
+}

litellm-rust/crates/python-bridge/src/router/stream.rs

@@ -0,0 +1,94 @@
+//! The stream-recovery entry points of `RouterCoreHandle`: what may still take over a stream that
+//! failed before content, the same-group stream retry, and the fallback walk re-entered from a
+//! stream. The host's stream wrappers call them; the calls are driven like `start`.
+
+use litellm_router_core::{
+    FailureKind, Op, Router,
+    call::{StreamOrigin, StreamPolicy},
+};
+use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict};
+
+use super::{RouterCall, attempt_outcome, config, config_error_to_pyerr, document, host::Finish};
+
+/// A stream recovery a host wrapper asks for.
+#[derive(Clone, Copy)]
+pub(super) enum Recovery {
+    /// `_astream_retry_same_group`.
+    Retry,
+    /// The fallback walk re-entered from the stream wrapper.
+    Fallback,
+}
+
+fn origin(value: &Bound<'_, PyAny>) -> PyResult<StreamOrigin> {
+    serde_json::from_value(document(value)?)
+        .map_err(|error| PyValueError::new_err(format!("a stream origin: {error}")))
+}
+
+fn kind(kind: Option<&str>) -> PyResult<Option<FailureKind>> {
+    kind.map(|kind| {
+        kind.parse()
+            .map_err(|_| PyValueError::new_err(format!("`{kind}` is not a router failure kind")))
+    })
+    .transpose()
+}
+
+/// `{"can_retry", "can_fall_back", "refusal_fallback", "retry_budget": int | None,
+/// "policy_applies": bool}`.
+pub(super) fn policy<'py>(
+    py: Python<'py>,
+    router: &Router,
+    request: &Bound<'py, PyAny>,
+    stream_origin: &Bound<'py, PyAny>,
+    failure_kind: Option<&str>,
+) -> PyResult<Bound<'py, PyDict>> {
+    let request = config::request(document(request)?, None).map_err(config_error_to_pyerr)?;
+    let StreamPolicy {
+        can_retry,
+        can_fall_back,
+        refusal_fallback,
+        retry_budget,
+    } = router.stream_policy(&request, &origin(stream_origin)?, kind(failure_kind)?);
+    let dict = PyDict::new(py);
+    dict.set_item("can_retry", can_retry)?;
+    dict.set_item("can_fall_back", can_fall_back)?;
+    dict.set_item("refusal_fallback", refusal_fallback)?;
+    dict.set_item("retry_budget", retry_budget.map(|(budget, _)| budget))?;
+    dict.set_item(
+        "policy_applies",
+        retry_budget.is_some_and(|(_, applies)| applies),
+    )?;
+    Ok(dict)
+}
+
+/// A recovery call for a stream from `stream_origin` that failed with `failure`.
+pub(super) fn recover(
+    router: &Router,
+    recovery: Recovery,
+    op: &str,
+    request: &Bound<'_, PyAny>,
+    stream_origin: &Bound<'_, PyAny>,
+    failure: &Bound<'_, PyDict>,
+) -> PyResult<RouterCall> {
+    let op: Op = op.parse().unwrap_or_else(|_| Op::Other(op.to_owned()));
+    let request = config::request(document(request)?, None).map_err(config_error_to_pyerr)?;
+    let stream_origin = origin(stream_origin)?;
+    let Err(failure) = attempt_outcome(failure)? else {
+        return Err(PyValueError::new_err("a stream failure needs `ok: False`"));
+    };
+    let router = router.clone();
+    Ok(RouterCall::new(async move {
+        let response = match recovery {
+            Recovery::Retry => {
+                router
+                    .retry_stream(op, request, stream_origin, failure)
+                    .await
+            }
+            Recovery::Fallback => {
+                router
+                    .fallback_stream(op, request, stream_origin, failure)
+                    .await
+            }
+        };
+        response.map(Finish::Call)
+    }))
+}

litellm-rust/crates/router-core/Cargo.toml

@@ -7,3 +7,21 @@ repository.workspace = true
 description = "The router core behind litellm.Router: registry, settings, health, selection and the call loop"
 
 [dependencies]
+arc-swap.workspace = true
+async-trait.workspace = true
+litellm-cache.workspace = true
+litellm-router-types.workspace = true
+jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-zoneinfo", "tzdb-bundle-always"] }
+parking_lot.workspace = true
+rand.workspace = true
+serde.workspace = true
+serde_with.workspace = true
+serde_json = { workspace = true, features = ["preserve_order"] }
+sha2.workspace = true
+strum.workspace = true
+thiserror.workspace = true
+tracing.workspace = true
+
+[dev-dependencies]
+rstest.workspace = true
+tokio.workspace = true

litellm-rust/crates/router-core/src/access.rs

@@ -0,0 +1,236 @@
+//! Who may reach a deployment: model access windows (`litellm_core_utils/access_windows.py`,
+//! a deployment reserved for some teams during a daily wall-clock window) and model access
+//! groups (`_filter_deployments_by_model_access_groups`, a caller allowed only through access
+//! groups is kept to its groups' deployments).
+
+use std::{collections::BTreeSet, sync::Arc};
+
+use jiff::{Timestamp, tz::TimeZone};
+use serde_json::Value;
+
+use crate::registry::Deployment;
+
+/// One `model_info.access_windows` entry (`ModelAccessWindow`), validated when the deployment
+/// was added.
+#[derive(Clone, Debug, PartialEq)]
+pub struct AccessWindow {
+    /// Microseconds after midnight.
+    pub start: u64,
+    pub end: u64,
+    pub timezone: String,
+    pub team_ids: Vec<String>,
+}
+
+const MICROS_PER_SECOND: u64 = 1_000_000;
+
+/// A `datetime.time` as pydantic reads it: `HH:MM[:SS[.ffffff]]`, or seconds after midnight.
+fn wall_clock(value: &Value) -> Option<u64> {
+    match value {
+        Value::Number(seconds) => {
+            let seconds = seconds.as_f64()?;
+            (0.0..86_400.0)
+                .contains(&seconds)
+                .then(|| (seconds * MICROS_PER_SECOND as f64).round() as u64)
+        }
+        Value::String(text) => {
+            let mut parts = text.splitn(3, ':');
+            let hour: u64 = parts.next()?.parse().ok()?;
+            let minute: u64 = parts.next()?.parse().ok()?;
+            let (second, micros) = match parts.next() {
+                None => (0, 0),
+                Some(rest) => {
+                    let (whole, fraction) = rest.split_once('.').unwrap_or((rest, ""));
+                    let digits: String = fraction.chars().take(6).collect();
+                    let micros = if digits.is_empty() {
+                        0
+                    } else {
+                        format!("{digits:0<6}").parse().ok()?
+                    };
+                    (whole.parse().ok()?, micros)
+                }
+            };
+            (hour < 24 && minute < 60 && second < 60)
+                .then(|| ((hour * 60 + minute) * 60 + second) * MICROS_PER_SECOND + micros)
+        }
+        _ => None,
+    }
+}
+
+impl AccessWindow {
+    fn parse(value: &Value) -> Option<Self> {
+        let window = value.as_object()?;
+        Some(Self {
+            start: wall_clock(window.get("start")?)?,
+            end: wall_clock(window.get("end")?)?,
+            timezone: window.get("timezone")?.as_str()?.to_owned(),
+            team_ids: window
+                .get("team_ids")?
+                .as_array()?
+                .iter()
+                .filter_map(Value::as_str)
+                .map(str::to_owned)
+                .collect(),
+        })
+    }
+
+    /// `is_window_active`: the window's wall clock contains `now`, wrapping past midnight when
+    /// it ends before it starts.
+    pub fn is_active(&self, now: Timestamp) -> bool {
+        let Ok(zone) = TimeZone::get(&self.timezone) else {
+            return false;
+        };
+        let local = now.to_zoned(zone).time();
+        let micros = ((u64::from(local.hour().unsigned_abs()) * 60
+            + u64::from(local.minute().unsigned_abs()))
+            * 60
+            + u64::from(local.second().unsigned_abs()))
+            * MICROS_PER_SECOND
+            + u64::from(local.subsec_nanosecond().unsigned_abs()) / 1_000;
+        if self.start < self.end {
+            self.start <= micros && micros < self.end
+        } else {
+            micros >= self.start || micros < self.end
+        }
+    }
+
+    /// The window's end as `{end:%H:%M}`.
+    pub fn end_hhmm(&self) -> String {
+        let minutes = self.end / MICROS_PER_SECOND / 60;
+        format!("{:02}:{:02}", minutes / 60, minutes % 60)
+    }
+}
+
+/// A deployment's access windows.
+pub fn access_windows(deployment: &Deployment) -> Vec<AccessWindow> {
+    deployment
+        .model_info
+        .raw
+        .get("access_windows")
+        .and_then(Value::as_array)
+        .map(|windows| windows.iter().filter_map(AccessWindow::parse).collect())
+        .unwrap_or_default()
+}
+
+/// `_reservation_blocking_window`: the first active window, unless the caller's team is named
+/// by one of the active windows.
+fn blocking_window(
+    deployment: &Deployment,
+    team_id: Option<&str>,
+    now: Timestamp,
+) -> Option<AccessWindow> {
+    let active: Vec<AccessWindow> = access_windows(deployment)
+        .into_iter()
+        .filter(|window| window.is_active(now))
+        .collect();
+    if let Some(team) = team_id
+        && active
+            .iter()
+            .any(|window| window.team_ids.iter().any(|id| id == team))
+    {
+        return None;
+    }
+    active.into_iter().next()
+}
+
+/// `filter_reserved_deployments`: the deployments no active window withholds from the caller,
+/// and the first window that withheld one.
+pub fn filter_reserved(
+    deployments: Vec<Arc<Deployment>>,
+    team_id: Option<&str>,
+    now: Timestamp,
+) -> (Vec<Arc<Deployment>>, Option<AccessWindow>) {
+    let mut blocking = None;
+    let kept = deployments
+        .into_iter()
+        .filter(
+            |deployment| match blocking_window(deployment, team_id, now) {
+                None => true,
+                Some(window) => {
+                    blocking.get_or_insert(window);
+                    false
+                }
+            },
+        )
+        .collect();
+    (kept, blocking)
+}
+
+/// `_filter_deployments_by_model_access_groups`: a caller whose allowed models (`models` plus
+/// `team_models`) reach `model` only through access groups is kept to the deployments in those
+/// groups. `model_groups` are the access groups the rows listed for `model` carry.
+pub fn filter_by_access_groups(
+    model: &str,
+    deployments: Vec<Arc<Deployment>>,
+    allowed_models: &[String],
+    model_groups: impl FnOnce() -> BTreeSet<String>,
+) -> Vec<Arc<Deployment>> {
+    if deployments.is_empty() || allowed_models.is_empty() {
+        return deployments;
+    }
+    let allowed = |name: &str| allowed_models.iter().any(|m| m == name);
+    if allowed(model) || allowed("*") || allowed("all-proxy-models") {
+        return deployments;
+    }
+    let groups: BTreeSet<String> = model_groups()
+        .into_iter()
+        .filter(|group| allowed(group))
+        .collect();
+    if groups.is_empty() {
+        return deployments;
+    }
+    deployments
+        .into_iter()
+        .filter(|d| {
+            d.model_info
+                .access_groups
+                .iter()
+                .any(|g| groups.contains(g))
+        })
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+
+    #[rstest]
+    #[case::minutes(json!("09:30"), Some(34_200_000_000))]
+    #[case::seconds(json!("09:30:15"), Some(34_215_000_000))]
+    #[case::fraction(json!("00:00:00.5"), Some(500_000))]
+    #[case::number(json!(60), Some(60_000_000))]
+    #[case::bad(json!("25:00"), None)]
+    fn wall_clock_reads_pydantic_times(#[case] raw: Value, #[case] expected: Option<u64>) {
+        assert_eq!(wall_clock(&raw), expected);
+    }
+
+    #[rstest]
+    #[case::inside("2026-01-01T10:00:00Z", true)]
+    #[case::at_end("2026-01-01T17:00:00Z", false)]
+    #[case::before("2026-01-01T08:59:59Z", false)]
+    fn a_window_is_active_inside_its_wall_clock(#[case] now: &str, #[case] expected: bool) {
+        let window = AccessWindow {
+            start: wall_clock(&json!("09:00")).unwrap(),
+            end: wall_clock(&json!("17:00")).unwrap(),
+            timezone: "UTC".to_owned(),
+            team_ids: vec!["t".to_owned()],
+        };
+        assert_eq!(window.is_active(now.parse().unwrap()), expected);
+        assert_eq!(window.end_hhmm(), "17:00");
+    }
+
+    #[test]
+    fn an_overnight_window_wraps_past_midnight() {
+        let window = AccessWindow {
+            start: wall_clock(&json!("22:00")).unwrap(),
+            end: wall_clock(&json!("02:00")).unwrap(),
+            timezone: "UTC".to_owned(),
+            team_ids: vec!["t".to_owned()],
+        };
+        assert!(window.is_active("2026-01-01T23:00:00Z".parse().unwrap()));
+        assert!(window.is_active("2026-01-01T01:00:00Z".parse().unwrap()));
+        assert!(!window.is_active("2026-01-01T12:00:00Z".parse().unwrap()));
+    }
+}

litellm-rust/crates/router-core/src/cache.rs

@@ -0,0 +1,145 @@
+//! [`RouterCache`]: the DualCache-style store all routing state lives in, under the keys,
+//! encodings and TTLs the Python router uses, so Python and Rust workers share it. Method names
+//! follow `litellm-cache`'s `BaseCache`, `BatchCache`, `CounterCache` and `DeleteCache`.
+
+use std::{
+    collections::HashMap,
+    sync::Arc,
+    time::{Duration, Instant},
+};
+
+use async_trait::async_trait;
+use litellm_cache::Error;
+use parking_lot::Mutex;
+use serde_json::Value;
+
+#[async_trait]
+pub trait RouterCache: Send + Sync {
+    async fn async_get_cache(&self, key: &str) -> Result<Option<Value>, Error>;
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: Value,
+        ttl: Option<Duration>,
+    ) -> Result<(), Error>;
+
+    /// Add `amount` to the number at `key` (0 when missing) and return the new value. The TTL
+    /// applies when the key is created, as Redis `INCRBYFLOAT` + `EXPIRE NX` does.
+    async fn async_increment(
+        &self,
+        key: &str,
+        amount: f64,
+        ttl: Option<Duration>,
+    ) -> Result<f64, Error>;
+
+    async fn async_batch_get_cache(&self, keys: &[String]) -> Result<Vec<Option<Value>>, Error>;
+
+    async fn async_delete_cache(&self, key: &str) -> Result<(), Error>;
+
+    /// Read this worker's in-memory tier only (`local_only=True`).
+    async fn async_get_local(&self, key: &str) -> Result<Option<Value>, Error> {
+        self.async_get_cache(key).await
+    }
+
+    /// Add to a counter in this worker's in-memory tier only (`local_only=True`).
+    async fn async_increment_local(
+        &self,
+        key: &str,
+        amount: f64,
+        ttl: Option<Duration>,
+    ) -> Result<f64, Error> {
+        self.async_increment(key, amount, ttl).await
+    }
+}
+
+#[derive(Default)]
+struct Entry {
+    value: Value,
+    expires: Option<Instant>,
+}
+
+/// An in-process [`RouterCache`], for tests and for state Python keeps `local_only`.
+#[derive(Clone, Default)]
+pub struct InMemoryRouterCache {
+    entries: Arc<Mutex<HashMap<String, Entry>>>,
+}
+
+impl InMemoryRouterCache {
+    fn live(entries: &mut HashMap<String, Entry>, key: &str) -> Option<Value> {
+        let expired = entries
+            .get(key)
+            .is_some_and(|entry| entry.expires.is_some_and(|at| at <= Instant::now()));
+        if expired {
+            entries.remove(key);
+        }
+        entries.get(key).map(|entry| entry.value.clone())
+    }
+
+    pub fn get(&self, key: &str) -> Option<Value> {
+        Self::live(&mut self.entries.lock(), key)
+    }
+
+    pub fn increment(&self, key: &str, amount: f64, ttl: Option<Duration>) -> f64 {
+        let mut entries = self.entries.lock();
+        let current = Self::live(&mut entries, key).and_then(|value| value.as_f64());
+        let next = current.unwrap_or(0.0) + amount;
+        let value = serde_json::Number::from_f64(next)
+            .map(|number| {
+                if next.fract() == 0.0 && next.abs() < i64::MAX as f64 {
+                    Value::from(next as i64)
+                } else {
+                    Value::Number(number)
+                }
+            })
+            .unwrap_or(Value::Null);
+        let expires = match (current, entries.get(key)) {
+            (Some(_), Some(entry)) => entry.expires,
+            _ => ttl.map(|ttl| Instant::now() + ttl),
+        };
+        entries.insert(key.to_owned(), Entry { value, expires });
+        next
+    }
+}
+
+#[async_trait]
+impl RouterCache for InMemoryRouterCache {
+    async fn async_get_cache(&self, key: &str) -> Result<Option<Value>, Error> {
+        Ok(self.get(key))
+    }
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: Value,
+        ttl: Option<Duration>,
+    ) -> Result<(), Error> {
+        let expires = ttl.map(|ttl| Instant::now() + ttl);
+        self.entries
+            .lock()
+            .insert(key.to_owned(), Entry { value, expires });
+        Ok(())
+    }
+
+    async fn async_increment(
+        &self,
+        key: &str,
+        amount: f64,
+        ttl: Option<Duration>,
+    ) -> Result<f64, Error> {
+        Ok(self.increment(key, amount, ttl))
+    }
+
+    async fn async_batch_get_cache(&self, keys: &[String]) -> Result<Vec<Option<Value>>, Error> {
+        let mut entries = self.entries.lock();
+        Ok(keys
+            .iter()
+            .map(|key| Self::live(&mut entries, key))
+            .collect())
+    }
+
+    async fn async_delete_cache(&self, key: &str) -> Result<(), Error> {
+        self.entries.lock().remove(key);
+        Ok(())
+    }
+}

litellm-rust/crates/router-core/src/call/stream.rs

@@ -0,0 +1,413 @@
+//! Recovering a stream that failed after its attempt returned it. The host wraps every streamed
+//! response; when the stream fails before the caller received content, the wrapper asks the core
+//! what may still take over and hands the failure back here. Mirrors the decisions of the Python
+//! router's stream wrappers: `_stream_can_fall_back`, `_refusal_fallback_available`,
+//! `_anthropic_messages_stream_can_retry`, `_stream_retry_budget`, `_astream_retry_same_group`,
+//! and the `async_function_with_fallbacks_common_utils` re-entry of the mid-stream fallback.
+
+use std::sync::Arc;
+
+use serde::Deserialize;
+
+use super::{Hop, Target, Tried, Walk};
+use crate::{
+    Router, Snapshot,
+    config::{Op, RouterDynamicConfig},
+    error::{AttemptError, Error, FailureKind},
+    fallback::{FallbackTarget, Fallbacks},
+    health::should_retry_status,
+    host::RouterRequest,
+    select::{Candidates, SelectInputs},
+};
+
+/// Where a streamed response came from: the attempt that returned it.
+#[derive(Clone, Debug, Default, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct StreamOrigin {
+    /// The name the attempt served: the requested group, or the fallback target.
+    pub group: String,
+    /// The `order` level an order fallback targeted.
+    pub order: Option<i64>,
+    /// `fallback_depth` of the attempt.
+    pub depth: u32,
+    pub level: u32,
+    /// The deployment the attempt ran on.
+    pub deployment_id: Option<String>,
+    /// Same-group stream retries this request already spent (`attempted_retries_for_request`).
+    pub attempted_retries: u32,
+    /// The budget the first stream retry committed to (`committed_retry_budget_for_request`).
+    pub committed_budget: Option<u32>,
+}
+
+/// What may still take over a stream that fails before content.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct StreamPolicy {
+    /// `_anthropic_messages_stream_can_retry`.
+    pub can_retry: bool,
+    /// `_stream_can_fall_back`.
+    pub can_fall_back: bool,
+    /// `_refusal_fallback_available`.
+    pub refusal_fallback: bool,
+    /// `_stream_retry_budget` for the failure kind asked about: the budget, and whether a retry
+    /// policy governs the retry.
+    pub retry_budget: Option<(u32, bool)>,
+}
+
+impl StreamOrigin {
+    fn target(&self) -> Target {
+        Target {
+            group: self.group.clone(),
+            order: self.order,
+            params: None,
+            excluded: Arc::from([]),
+            failover_excluded: Arc::from([]),
+        }
+    }
+
+    /// `fallback_lookup_groups`: the served group, then the group the request was for.
+    fn lookup<'a>(&'a self, request: &'a RouterRequest) -> Vec<&'a str> {
+        let mut lookup = vec![self.group.as_str()];
+        if request.model != self.group {
+            lookup.push(request.model.as_str());
+        }
+        lookup
+    }
+
+    /// `has_unattempted_fallback_target`. The first attempt carries no attempted targets, so any
+    /// resolved chain counts; a fallback hop has already tried its own group and the original one.
+    fn has_unattempted(&self, request: &RouterRequest, chain: Option<Vec<String>>) -> bool {
+        let Some(chain) = chain else {
+            return false;
+        };
+        if self.depth == 0 {
+            return true;
+        }
+        chain
+            .iter()
+            .any(|target| *target != self.group && *target != request.model)
+    }
+}
+
+/// The group names a chain targets (a dict target counts by its `model`).
+fn names(chain: Option<Vec<FallbackTarget>>) -> Option<Vec<String>> {
+    chain.map(|targets| {
+        targets
+            .iter()
+            .filter_map(|target| target.model().map(str::to_owned))
+            .collect()
+    })
+}
+
+/// The regular fallback targets for the lookup groups, then `default_fallbacks`, which Python
+/// folds into `fallbacks` as `{"*": [...]}`.
+fn regular_targets(dynamic: &RouterDynamicConfig, lookup: &[&str]) -> Option<Vec<String>> {
+    names(
+        dynamic
+            .fallbacks
+            .lookup_groups(lookup, &dynamic.fallback_keys),
+    )
+    .or_else(|| (!dynamic.default_fallbacks.is_empty()).then(|| dynamic.default_fallbacks.clone()))
+}
+
+fn has_default_fallbacks(fallbacks: &Fallbacks, default_fallbacks: &[String]) -> bool {
+    fallbacks.has_star() || !default_fallbacks.is_empty()
+}
+
+impl Router {
+    /// What may still take over a stream from `origin` that fails before content, and the retry
+    /// budget for a failure of `kind`.
+    pub fn stream_policy(
+        &self,
+        request: &RouterRequest,
+        origin: &StreamOrigin,
+        kind: Option<FailureKind>,
+    ) -> StreamPolicy {
+        let snapshot = self.core.snapshot.load_full();
+        StreamPolicy {
+            can_retry: stream_can_retry(&snapshot, request, origin),
+            can_fall_back: stream_can_fall_back(&snapshot, request, origin),
+            refusal_fallback: refusal_fallback_available(&snapshot, request, origin),
+            retry_budget: kind.map(|kind| stream_retry_budget(&snapshot, request, origin, kind)),
+        }
+    }
+
+    /// The fallback walk for a stream from `origin` that failed with `failure` before content:
+    /// `async_function_with_fallbacks_common_utils` re-entered from the stream wrapper, so the
+    /// group's own retry loop does not run again.
+    pub async fn fallback_stream(
+        &self,
+        op: Op,
+        request: RouterRequest,
+        origin: StreamOrigin,
+        failure: AttemptError,
+    ) -> Result<super::RouterResponse, Error> {
+        self.live()?;
+        let snapshot = self.core.snapshot.load_full();
+        let request = Arc::new(request);
+        let mut walk = Walk::default();
+        if request.model != origin.group {
+            walk.attempted
+                .push(Tried::Target(FallbackTarget::Name(request.model.clone())));
+        }
+        self.core
+            .with_fallbacks(
+                &snapshot,
+                &op,
+                &request,
+                origin.target(),
+                Hop {
+                    depth: origin.depth,
+                    level: origin.level,
+                    scope: 0,
+                },
+                &mut walk,
+                Some(Error::Attempt(failure)),
+            )
+            .await
+    }
+
+    /// `_astream_retry_same_group`: re-run the attempt in the stream's own group until a retry
+    /// returns a response or the budget runs out. The failed result carries the newest failure.
+    pub async fn retry_stream(
+        &self,
+        op: Op,
+        request: RouterRequest,
+        origin: StreamOrigin,
+        failure: AttemptError,
+    ) -> Result<super::RouterResponse, Error> {
+        self.live()?;
+        let snapshot = self.core.snapshot.load_full();
+        let request = Arc::new(request);
+        let target = origin.target();
+        let at = Hop {
+            depth: origin.depth,
+            level: origin.level,
+            scope: 0,
+        };
+        let counting = SelectInputs {
+            count_only: true,
+            ..SelectInputs::default()
+        };
+        let counted = |c: &Candidates, n: usize| if c.kind.is_single() { 0 } else { n };
+        let (healthy, all) = self
+            .core
+            .candidates(&snapshot, &op, &request, &target.group, &counting)
+            .await
+            .map(|c| (counted(&c, c.healthy.len()), counted(&c, c.all.len())))
+            .unwrap_or((0, 0));
+        let (budget, policy_applies) =
+            stream_retry_budget(&snapshot, &request, &origin, failure.kind);
+        let inputs = SelectInputs {
+            target_order: target.order,
+            ..SelectInputs::default()
+        };
+        let mut last = failure;
+        for attempt in origin.attempted_retries..budget {
+            if !policy_applies
+                && !self
+                    .core
+                    .should_retry(&snapshot, &request, &last, healthy, all)
+            {
+                break;
+            }
+            let delay = self.core.time_to_sleep(
+                &snapshot,
+                last.retry_after,
+                budget - attempt,
+                budget,
+                healthy,
+                all,
+            );
+            self.core.host.sleep(delay).await;
+            match self
+                .core
+                .attempt(
+                    &snapshot,
+                    &op,
+                    &request,
+                    &target,
+                    &inputs,
+                    (attempt + 1, budget),
+                    (at, &[]),
+                )
+                .await
+            {
+                Ok(mut response) => {
+                    response.attempted_retries = attempt + 1;
+                    response.max_retries = Some(budget);
+                    return Ok(response);
+                }
+                Err(failed) => match failed.error {
+                    // `_stream_fallback_error_for_raised`: a status the pre-stream retry rule
+                    // would not retry ends the retries with that failure.
+                    Error::Attempt(failure) => {
+                        let stop = failure.status.is_some_and(|s| !should_retry_status(s));
+                        last = failure;
+                        if stop {
+                            break;
+                        }
+                    }
+                    other => return Err(other),
+                },
+            }
+        }
+        Err(Error::Attempt(last))
+    }
+}
+
+/// The request's `num_retries`, then the routed deployment's, then the router's
+/// (`_anthropic_messages_plain_retry_budget`).
+fn plain_retry_budget(snapshot: &Snapshot, request: &RouterRequest, origin: &StreamOrigin) -> u32 {
+    if let Some(retries) = request.num_retries {
+        return retries;
+    }
+    let deployment = origin
+        .deployment_id
+        .as_deref()
+        .and_then(|id| snapshot.registry.deployment(id));
+    let configured = deployment.and_then(|d| match d.raw_params.get("num_retries") {
+        Some(serde_json::Value::Number(n)) => n.as_u64().and_then(|n| u32::try_from(n).ok()),
+        Some(serde_json::Value::String(s)) if s.chars().all(|c| c.is_ascii_digit()) => {
+            s.parse().ok()
+        }
+        _ => None,
+    });
+    configured.unwrap_or(snapshot.settings.num_retries)
+}
+
+/// The retry policy for the request's model group (`_anthropic_messages_resolved_retry_policy`),
+/// unless the request opted out with `num_retries=0`.
+fn retry_policy<'a>(
+    snapshot: &'a Snapshot,
+    request: &'a RouterRequest,
+    origin: &StreamOrigin,
+) -> Option<&'a crate::config::RetryPolicy> {
+    if request.num_retries == Some(0) {
+        return None;
+    }
+    let settings = &snapshot.settings;
+    crate::retry::resolve_retry_policy(
+        settings.retry_policy.as_ref(),
+        &origin.group,
+        Some(
+            request
+                .model_group_retry_policy
+                .as_ref()
+                .unwrap_or(&settings.model_group_retry_policy),
+        ),
+    )
+}
+
+fn stream_retry_budget(
+    snapshot: &Snapshot,
+    request: &RouterRequest,
+    origin: &StreamOrigin,
+    kind: FailureKind,
+) -> (u32, bool) {
+    let failure = AttemptError::new(kind, None);
+    let policy_retries =
+        retry_policy(snapshot, request, origin).and_then(|p| p.retries_for(&failure));
+    if origin.attempted_retries > 0
+        && let Some(committed) = origin.committed_budget
+    {
+        return (committed, policy_retries.is_some());
+    }
+    match policy_retries {
+        Some(retries) => (retries, true),
+        None => (plain_retry_budget(snapshot, request, origin), false),
+    }
+}
+
+fn stream_can_retry(snapshot: &Snapshot, request: &RouterRequest, origin: &StreamOrigin) -> bool {
+    let attempted = origin.attempted_retries;
+    if attempted > 0
+        && let Some(committed) = origin.committed_budget
+    {
+        return committed > attempted;
+    }
+    let plain = plain_retry_budget(snapshot, request, origin);
+    // `_retry_policy_ceiling`: every field the policy dumps.
+    let ceiling = retry_policy(snapshot, request, origin).map_or(plain, |policy| {
+        let granted = [
+            policy.BadRequestErrorRetries,
+            policy.AuthenticationErrorRetries,
+            policy.TimeoutErrorRetries,
+            policy.RateLimitErrorRetries,
+            policy.ContentPolicyViolationErrorRetries,
+            policy.InternalServerErrorRetries,
+            policy.ServiceUnavailableErrorRetries,
+            policy.NotFoundErrorRetries,
+            policy.DefaultRetries,
+        ];
+        plain.max(granted.into_iter().flatten().max().unwrap_or(0))
+    });
+    ceiling > attempted
+}
+
+fn stream_can_fall_back(
+    snapshot: &Snapshot,
+    request: &RouterRequest,
+    origin: &StreamOrigin,
+) -> bool {
+    if request.disable_fallbacks {
+        return false;
+    }
+    if snapshot.settings.enable_weighted_failover {
+        return true;
+    }
+    let mut orders: Vec<i64> = snapshot
+        .registry
+        .group(&origin.group)
+        .unwrap_or_default()
+        .iter()
+        .filter(|d| d.usable_by_team(request.team_id.as_deref()))
+        .filter_map(|d| d.order())
+        .collect();
+    orders.sort_unstable();
+    orders.dedup();
+    if orders.len() > 1 {
+        let current = origin.order.unwrap_or(orders[0]);
+        if orders.iter().any(|order| *order > current) {
+            return true;
+        }
+    }
+    let dynamic = &snapshot.settings.dynamic;
+    let lookup = origin.lookup(request);
+    if let Some(content_policy) = &dynamic.content_policy_fallbacks
+        && origin.has_unattempted(
+            request,
+            names(content_policy.lookup_groups(&lookup, &dynamic.fallback_keys)),
+        )
+    {
+        return true;
+    }
+    if !dynamic.has_regular_fallbacks() {
+        return false;
+    }
+    origin.has_unattempted(request, regular_targets(dynamic, &lookup))
+}
+
+fn refusal_fallback_available(
+    snapshot: &Snapshot,
+    request: &RouterRequest,
+    origin: &StreamOrigin,
+) -> bool {
+    if request.disable_fallbacks {
+        return false;
+    }
+    let dynamic = &snapshot.settings.dynamic;
+    let lookup = origin.lookup(request);
+    if let Some(content_policy) = &dynamic.content_policy_fallbacks {
+        return origin.has_unattempted(
+            request,
+            names(content_policy.lookup_groups(&lookup, &dynamic.fallback_keys)),
+        );
+    }
+    if has_default_fallbacks(&dynamic.fallbacks, &dynamic.default_fallbacks) {
+        return true;
+    }
+    if !dynamic.has_regular_fallbacks() {
+        return false;
+    }
+    origin.has_unattempted(request, regular_targets(dynamic, &lookup))
+}

litellm-rust/crates/router-core/src/config.rs

@@ -0,0 +1,361 @@
+//! What the router is told: [`RouterConfig`], fixed for the router's life, and
+//! [`RouterDynamicConfig`], a versioned snapshot read once per call. The router never reads
+//! process globals.
+
+use std::{collections::BTreeMap, time::Duration};
+
+use serde::Deserialize;
+
+use crate::{
+    error::AttemptError,
+    registry::{DeploymentSpec, RoutingGroupSpec},
+};
+
+/// `DEFAULT_COOLDOWN_TIME_SECONDS` in `litellm/constants.py`.
+pub const DEFAULT_COOLDOWN_TIME: Duration = Duration::from_secs(5);
+/// `ROUTER_MAX_FALLBACKS` in `litellm/constants.py`.
+pub const DEFAULT_MAX_FALLBACKS: u32 = 5;
+
+/// The router operation a call runs, spelled as the Python router's endpoint method.
+#[derive(
+    Clone,
+    Debug,
+    PartialEq,
+    Eq,
+    Hash,
+    strum::EnumString,
+    strum::Display,
+    serde_with::DeserializeFromStr,
+)]
+pub enum Op {
+    #[strum(serialize = "acompletion")]
+    Completion,
+    #[strum(serialize = "atext_completion")]
+    TextCompletion,
+    #[strum(serialize = "aembedding")]
+    Embedding,
+    #[strum(serialize = "aresponses")]
+    Responses,
+    #[strum(serialize = "aanthropic_messages")]
+    Messages,
+    #[strum(serialize = "aimage_generation")]
+    ImageGeneration,
+    #[strum(serialize = "atranscription")]
+    Transcription,
+    #[strum(serialize = "aspeech")]
+    Speech,
+    #[strum(serialize = "aocr")]
+    Ocr,
+    #[strum(default, transparent)]
+    Other(String),
+}
+
+/// `routing_strategy`. Only simple-shuffle is native; the rest go through `Host::select`.
+#[derive(
+    Clone,
+    Debug,
+    Default,
+    PartialEq,
+    Eq,
+    strum::EnumString,
+    strum::Display,
+    serde_with::DeserializeFromStr,
+)]
+pub enum RoutingStrategy {
+    #[default]
+    #[strum(serialize = "simple-shuffle")]
+    SimpleShuffle,
+    #[strum(serialize = "least-busy")]
+    LeastBusy,
+    #[strum(serialize = "usage-based-routing")]
+    UsageBased,
+    #[strum(serialize = "usage-based-routing-v2")]
+    UsageBasedV2,
+    #[strum(serialize = "latency-based-routing")]
+    LatencyBased,
+    #[strum(serialize = "cost-based-routing")]
+    CostBased,
+    /// `routing_strategy=None` passed explicitly (sent as `"None"`): stored as None, so no
+    /// strategy arm matches and every pick finds no deployment.
+    #[strum(serialize = "None")]
+    Unset,
+    #[strum(default, transparent)]
+    Other(String),
+}
+
+impl RoutingStrategy {
+    pub fn is_native(&self) -> bool {
+        matches!(self, Self::SimpleShuffle | Self::Unset)
+    }
+}
+
+/// `PluginKind` in `litellm/router_api.py`.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, strum::Display)]
+pub enum PluginKind {
+    #[serde(rename = "auto")]
+    #[strum(serialize = "auto")]
+    Auto,
+    #[serde(rename = "complexity")]
+    #[strum(serialize = "complexity")]
+    Complexity,
+    #[serde(rename = "quality")]
+    #[strum(serialize = "quality")]
+    Quality,
+    #[serde(rename = "adaptive")]
+    #[strum(serialize = "adaptive")]
+    Adaptive,
+    #[serde(rename = "routing")]
+    #[strum(serialize = "routing")]
+    Routing,
+    #[serde(rename = "budget_limiter")]
+    #[strum(serialize = "budget_limiter")]
+    BudgetLimiter,
+    #[serde(rename = "prompt_caching_pins")]
+    #[strum(serialize = "prompt_caching_pins")]
+    PromptCachingPins,
+}
+
+/// One registered plugin. Its behaviour lives in the host until it is ported.
+#[derive(Clone, Debug, PartialEq, Deserialize)]
+pub struct PluginSpec {
+    pub kind: PluginKind,
+    pub name: String,
+    #[serde(default)]
+    pub config: serde_json::Value,
+}
+
+/// `RetryPolicy` in `litellm/types/router.py`: retries per exception type.
+#[derive(Clone, Debug, Default, PartialEq, Deserialize)]
+#[allow(non_snake_case)]
+pub struct RetryPolicy {
+    pub BadRequestErrorRetries: Option<u32>,
+    pub AuthenticationErrorRetries: Option<u32>,
+    pub TimeoutErrorRetries: Option<u32>,
+    pub RateLimitErrorRetries: Option<u32>,
+    pub ContentPolicyViolationErrorRetries: Option<u32>,
+    pub InternalServerErrorRetries: Option<u32>,
+    pub ServiceUnavailableErrorRetries: Option<u32>,
+    pub NotFoundErrorRetries: Option<u32>,
+    pub DefaultRetries: Option<u32>,
+}
+
+/// `AllowedFailsPolicy` in `litellm/types/router.py`: allowed fails per minute per exception type.
+#[derive(Clone, Debug, Default, PartialEq, Deserialize)]
+#[allow(non_snake_case)]
+pub struct AllowedFailsPolicy {
+    pub BadRequestErrorAllowedFails: Option<u32>,
+    pub AuthenticationErrorAllowedFails: Option<u32>,
+    pub TimeoutErrorAllowedFails: Option<u32>,
+    pub RateLimitErrorAllowedFails: Option<u32>,
+    pub ContentPolicyViolationErrorAllowedFails: Option<u32>,
+    pub InternalServerErrorAllowedFails: Option<u32>,
+    pub ServiceUnavailableErrorAllowedFails: Option<u32>,
+    pub BadGatewayErrorAllowedFails: Option<u32>,
+    pub NotFoundErrorAllowedFails: Option<u32>,
+}
+
+impl AllowedFailsPolicy {
+    /// `_core_get_allowed_fails_from_policy`: the first set field whose exception class the
+    /// failure is an instance of, in Python's order.
+    pub fn allowed_fails_for(&self, failure: &AttemptError) -> Option<u32> {
+        [
+            ("AuthenticationError", self.AuthenticationErrorAllowedFails),
+            ("Timeout", self.TimeoutErrorAllowedFails),
+            ("RateLimitError", self.RateLimitErrorAllowedFails),
+            (
+                "ContentPolicyViolationError",
+                self.ContentPolicyViolationErrorAllowedFails,
+            ),
+            ("BadRequestError", self.BadRequestErrorAllowedFails),
+            ("InternalServerError", self.InternalServerErrorAllowedFails),
+            (
+                "ServiceUnavailableError",
+                self.ServiceUnavailableErrorAllowedFails,
+            ),
+            ("BadGatewayError", self.BadGatewayErrorAllowedFails),
+            ("NotFoundError", self.NotFoundErrorAllowedFails),
+        ]
+        .into_iter()
+        .find_map(|(class, allowed)| allowed.filter(|_| failure.is_a(class)))
+    }
+}
+
+pub use crate::fallback::{FallbackKeys, Fallbacks};
+
+/// Static config, fixed for the router's life.
+#[derive(Clone, Debug, Default, Deserialize)]
+#[serde(default)]
+pub struct RouterConfig {
+    pub model_list: Vec<DeploymentSpec>,
+    pub routing_strategy: RoutingStrategy,
+    pub routing_groups: Vec<RoutingGroupSpec>,
+    /// `_model_to_group` as Python last built it (member model group to routing group), set
+    /// only when a raw `routing_groups` write replaced the groups since: member calls keep
+    /// routing by it, as the Python router's stale map does.
+    pub routing_group_members: Option<BTreeMap<String, String>>,
+    pub plugins: Vec<PluginSpec>,
+    /// Router default `cooldown_time`; `None` is `DEFAULT_COOLDOWN_TIME_SECONDS`.
+    pub cooldown_time: Option<f64>,
+    pub disable_cooldowns: bool,
+    pub allowed_fails_policy: Option<AllowedFailsPolicy>,
+    pub retry_policy: Option<RetryPolicy>,
+    /// `model_group_retry_policy`: a group's own policy, which wins over `retry_policy`.
+    pub model_group_retry_policy: BTreeMap<String, RetryPolicy>,
+    /// `retry_after`: the minimum backoff between retries, in seconds.
+    pub retry_after: f64,
+    pub timeout: Option<f64>,
+    pub request_timeout: Option<f64>,
+    pub drop_params: Option<bool>,
+    pub enable_pre_call_checks: bool,
+    /// `Router(plugins=[...])` is set: the host runs the routing-plugin pipeline.
+    pub routing_plugins: bool,
+    /// The host's `CustomLogger` deployment filters run on every async candidate list
+    /// (`async_callback_filter_deployments`).
+    pub callback_filters: bool,
+    pub enable_health_check_routing: bool,
+    /// How old (seconds) a background health check result may be and still count
+    /// (`health_check_staleness_threshold`, else the default interval times its multiplier).
+    pub health_check_staleness_threshold: Option<f64>,
+    /// `background_health_check_model_groups`: when set, only these groups are health-filtered.
+    pub background_health_check_model_groups: Option<Vec<String>>,
+    /// `enable_weighted_failover`: a failed deployment's group is re-picked without it before
+    /// any cross-group fallback (simple-shuffle only).
+    pub enable_weighted_failover: bool,
+    /// `fallback_on_unsupported_params`; `None` is Python's default, `True`.
+    pub fallback_on_unsupported_params: Option<bool>,
+    /// Deployment ids that are a provider's default and never cool down
+    /// (`provider_default_deployment_ids`).
+    pub provider_default_deployment_ids: Vec<String>,
+    /// The operations this router must serve: `Host::supports` must accept each one for every
+    /// configured provider, or the router fails to build.
+    pub served_ops: Vec<Op>,
+}
+
+/// Dynamic config: a versioned snapshot read once per call.
+#[derive(Clone, Debug, Deserialize)]
+#[serde(default)]
+pub struct RouterDynamicConfig {
+    pub version: u64,
+    /// `model_group_alias` as Python spells it: each alias maps to a group name, or to
+    /// `{"model": <group>, "hidden": <bool>}`.
+    #[serde(alias = "model_group_alias", deserialize_with = "alias_targets")]
+    pub model_alias_map: BTreeMap<String, String>,
+    pub fallbacks: Fallbacks,
+    pub default_fallbacks: Vec<String>,
+    pub context_window_fallbacks: Option<Fallbacks>,
+    pub content_policy_fallbacks: Option<Fallbacks>,
+    pub num_retries: Option<u32>,
+    pub max_fallbacks: u32,
+    /// `allowed_fails`. `Some` means it was set explicitly, which switches cooldowns from the
+    /// failure-ratio rule to the allowed-fails counter (`_is_allowed_fails_set_on_router`).
+    pub allowed_fails: Option<u32>,
+    /// Other spellings a fallback key matches.
+    pub fallback_keys: FallbackKeys,
+    /// `default_deployment`: what a name no deployment, alias, routing group, team route or
+    /// wildcard serves is routed to, as a copy whose `litellm_params.model` is the name
+    /// (`_try_early_resolve_deployments_for_model_not_in_names`).
+    pub default_deployment: Option<DeploymentSpec>,
+    /// The `default_deployment` is a value Python cannot copy for a name: resolving to it
+    /// raises the copy's error ([`crate::Error::DefaultDeploymentRaises`]).
+    pub default_deployment_raises: bool,
+}
+
+/// `model_group_alias` as `resolve_model_group_alias` reads it: an alias maps to a non-empty
+/// group name, or to `{"model": <non-empty group name>, ...}`; any other value rewrites nothing.
+fn alias_targets<'de, D>(deserializer: D) -> Result<BTreeMap<String, String>, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    let aliases = BTreeMap::<String, serde_json::Value>::deserialize(deserializer)?;
+    Ok(aliases
+        .into_iter()
+        .filter_map(|(alias, target)| {
+            let group = match &target {
+                serde_json::Value::Object(entry) => entry.get("model")?,
+                other => other,
+            };
+            group
+                .as_str()
+                .filter(|group| !group.is_empty())
+                .map(|group| (alias, group.to_owned()))
+        })
+        .collect())
+}
+
+impl Default for RouterDynamicConfig {
+    fn default() -> Self {
+        Self {
+            version: 0,
+            model_alias_map: BTreeMap::new(),
+            fallbacks: Fallbacks::default(),
+            default_fallbacks: Vec::new(),
+            context_window_fallbacks: None,
+            content_policy_fallbacks: None,
+            num_retries: None,
+            max_fallbacks: DEFAULT_MAX_FALLBACKS,
+            allowed_fails: None,
+            fallback_keys: FallbackKeys::default(),
+            default_deployment: None,
+            default_deployment_raises: false,
+        }
+    }
+}
+
+impl RouterDynamicConfig {
+    /// The `default_deployment` as resolution serves it.
+    pub fn default_ref(&self) -> Option<crate::resolve::DefaultDeployment<'_>> {
+        if self.default_deployment_raises {
+            return Some(crate::resolve::DefaultDeployment::Raises);
+        }
+        self.default_deployment
+            .as_ref()
+            .map(crate::resolve::DefaultDeployment::Spec)
+    }
+
+    /// The regular fallback targets for the lookup groups, then `default_fallbacks`, which
+    /// Python folds into `fallbacks` as `{"*": [...]}`: what `_has_fallback_group` reads.
+    pub fn fallback_targets(&self, groups: &[&str]) -> Option<Vec<String>> {
+        if groups
+            .iter()
+            .any(|group| self.fallbacks.has_keyed_chain(group, &self.fallback_keys))
+        {
+            return Some(Vec::new());
+        }
+        (!self.default_fallbacks.is_empty()).then(|| self.default_fallbacks.clone())
+    }
+
+    pub fn has_regular_fallbacks(&self) -> bool {
+        !self.fallbacks.is_empty() || !self.default_fallbacks.is_empty()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+
+    #[rstest]
+    #[case::group_name(json!({"model_group_alias": {"gpt": "gpt-4"}}))]
+    #[case::hidden_entry(json!({"model_group_alias": {"gpt": {"model": "gpt-4", "hidden": true}}}))]
+    #[case::native_name(json!({"model_alias_map": {"gpt": "gpt-4"}}))]
+    fn model_group_alias_reads_both_python_spellings(#[case] wire: serde_json::Value) {
+        let dynamic: RouterDynamicConfig = serde_json::from_value(wire).unwrap();
+        assert_eq!(
+            dynamic.model_alias_map,
+            BTreeMap::from([("gpt".to_owned(), "gpt-4".to_owned())])
+        );
+    }
+
+    #[rstest]
+    #[case::native("simple-shuffle", RoutingStrategy::SimpleShuffle)]
+    #[case::unknown("my-strategy", RoutingStrategy::Other("my-strategy".to_owned()))]
+    fn routing_strategy_parses_its_wire_name(
+        #[case] wire: &str,
+        #[case] expected: RoutingStrategy,
+    ) {
+        assert_eq!(wire.parse::<RoutingStrategy>().unwrap(), expected);
+        assert_eq!(expected.to_string(), wire);
+    }
+}

litellm-rust/crates/router-core/src/error.rs

@@ -0,0 +1,399 @@
+//! The router's errors: [`ConfigError`] when a router is built or changed, [`Error`] when a call
+//! or a selection fails, and [`AttemptError`], the typed facts of one failed attempt.
+
+use std::{any::Any, fmt, sync::Arc, time::Duration};
+
+use crate::config::{Op, RoutingStrategy};
+
+/// What a host cannot serve, found when the router is built rather than partway through a call.
+#[derive(Clone, Debug, PartialEq, thiserror::Error)]
+pub enum Unsupported {
+    #[error("routing strategy `{0}` needs a host that can select")]
+    Strategy(RoutingStrategy),
+    #[error("routing plugin `{0}` needs a host that can select")]
+    Plugin(String),
+    #[error("the host cannot run operation `{op}` on provider `{provider}`")]
+    Op { op: Op, provider: String },
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum ConfigError {
+    #[error("the router config asks for something this host cannot serve")]
+    Unsupported(#[from] Unsupported),
+    #[error("deployment `{model_name}` has invalid litellm_params")]
+    InvalidDeployment {
+        model_name: String,
+        #[source]
+        source: serde_json::Error,
+    },
+    #[error("deployment id `{id}` is configured more than once")]
+    DuplicateId { id: String },
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("the router config is invalid")]
+    Config(#[from] ConfigError),
+    #[error("no deployment serves model `{model}`")]
+    UnknownModel { model: String },
+    #[error("no deployment of `{model}` is available")]
+    NoHealthyDeployments { model: String },
+    #[error("the attempt failed")]
+    Attempt(#[from] AttemptError),
+    #[error("the routing cache failed")]
+    Cache(#[from] litellm_cache::Error),
+    #[error("the router was retired")]
+    Retired,
+    /// A blocked deployment named by its id (`model` is its litellm model).
+    #[error("deployment model `{model}` is paused")]
+    Paused { model: String },
+    /// A pass-through pick named a deployment id without `use_in_pass_through`.
+    #[error("deployment `{id}` does not support pass-through")]
+    NotPassThrough { id: String },
+    /// A pass-through pick of a group with healthy deployments, none of them pass-through.
+    #[error("model `{model}` has no pass-through deployment")]
+    NoPassThroughDeployments { model: String },
+    /// Every deployment `model` resolved to is a pre-routing strategy marker (`auto_router/...`).
+    #[error("every deployment of `{model}` is a strategy router marker")]
+    OnlyMarkers { model: String },
+    /// Every deployment `model` resolved to is split into PTU shares the caller holds none of.
+    #[error("deployment `{model}` is reserved for the teams holding a PTU share of it")]
+    PtuReserved { model: String },
+    /// A proxy admin without a team named a public model name several teams use.
+    #[error("model name `{model}` matches deployments from multiple teams")]
+    MultipleTeams { model: String },
+    /// Every deployment `model` resolved to is reserved for other teams by an active access
+    /// window, the first of which ends at `until` (`HH:MM`) in `timezone`.
+    #[error("deployment `{model}` is reserved for another team until {until} {timezone}")]
+    Reserved {
+        model: String,
+        until: String,
+        timezone: String,
+    },
+    /// The pre-call checks found no deployment whose context window fits the prompt; `detail`
+    /// names each one dropped for it.
+    #[error("no deployment of `{model}` has a context window large enough for this call")]
+    ContextWindowExceeded { model: String, detail: String },
+    /// The pre-call checks found no deployment of `model` that takes the requested `params`.
+    #[error("no deployment of `{model}` supports {params:?}")]
+    UnsupportedParams { model: String, params: Vec<String> },
+    /// The routing plugins left none of the healthy deployments' models.
+    #[error("no deployments left after routing-plugin filtering: {models:?}")]
+    RoutingPluginsLeftNothing { models: Vec<String> },
+    /// The request's `_router_weights` failed `validate_router_weights` when simple-shuffle picked.
+    #[error("the request's router weights are invalid")]
+    InvalidRouterWeights,
+    /// A deployment's `litellm_params` `metric` (weight, rpm or tpm) is not a number.
+    #[error("deployment {metric} must be numeric")]
+    NonNumericWeight { metric: &'static str },
+    /// A router with routing plugins was asked for a pick on the sync path, which never runs them.
+    #[error("routing plugins only run on the async selection path")]
+    RoutingPluginsNeedAsync,
+    /// The stale routing-group member map (`_model_to_group`, kept as it was when a raw
+    /// `routing_groups` write replaced the groups) names a group that is no longer configured:
+    /// Python's `_get_routing_context` raises `KeyError(group)` for every pick of the name.
+    #[error("routing group `{group}` is no longer configured")]
+    MissingRoutingGroup { group: String },
+    /// The router's `default_deployment` is not a value Python can copy for `model`
+    /// (`_try_early_resolve_deployments_for_model_not_in_names` raises the copy's error).
+    #[error("the default deployment cannot be served for `{model}`")]
+    DefaultDeploymentRaises { model: String },
+    /// A written model-name or team index names a position past the end of the model list:
+    /// Python's lookup of `model` raises `IndexError`.
+    #[error("a written index of `{model}` names a position past the model list")]
+    IndexOutOfRange { model: String },
+}
+
+impl Error {
+    /// The failed attempt behind this error, when a deployment was called.
+    pub fn attempt(&self) -> Option<&AttemptError> {
+        match self {
+            Self::Attempt(attempt) => Some(attempt),
+            Self::Config(_)
+            | Self::UnknownModel { .. }
+            | Self::NoHealthyDeployments { .. }
+            | Self::Cache(_)
+            | Self::Retired
+            | Self::Paused { .. }
+            | Self::NotPassThrough { .. }
+            | Self::NoPassThroughDeployments { .. }
+            | Self::OnlyMarkers { .. }
+            | Self::PtuReserved { .. }
+            | Self::MultipleTeams { .. }
+            | Self::Reserved { .. }
+            | Self::ContextWindowExceeded { .. }
+            | Self::UnsupportedParams { .. }
+            | Self::RoutingPluginsLeftNothing { .. }
+            | Self::InvalidRouterWeights
+            | Self::NonNumericWeight { .. }
+            | Self::RoutingPluginsNeedAsync
+            | Self::MissingRoutingGroup { .. }
+            | Self::DefaultDeploymentRaises { .. }
+            | Self::IndexOutOfRange { .. } => None,
+        }
+    }
+
+    /// The Python exception a pick that failed with this error raises, as the retry loop
+    /// decides by it (its class names, most specific first, and its status): `None` for an
+    /// attempt's failure and for errors no pick raises.
+    pub fn pick_failure(&self) -> Option<AttemptError> {
+        const BAD_REQUEST: &[&str] = &["BadRequestError"];
+        let (classes, status): (&[&str], Option<u16>) = match self {
+            Self::UnknownModel { .. }
+            | Self::OnlyMarkers { .. }
+            | Self::PtuReserved { .. }
+            | Self::NotPassThrough { .. }
+            | Self::NoPassThroughDeployments { .. }
+            | Self::MultipleTeams { .. }
+            | Self::Reserved { .. } => (BAD_REQUEST, Some(400)),
+            Self::ContextWindowExceeded { .. } => (
+                &["ContextWindowExceededError", "BadRequestError"],
+                Some(400),
+            ),
+            Self::UnsupportedParams { .. } => {
+                (&["UnsupportedParamsError", "BadRequestError"], Some(400))
+            }
+            Self::Paused { .. } => (&["ServiceUnavailableError"], Some(503)),
+            Self::NoHealthyDeployments { .. } => (&["RouterRateLimitError", "ValueError"], None),
+            Self::RoutingPluginsLeftNothing { .. } | Self::RoutingPluginsNeedAsync => {
+                (&["ValueError"], None)
+            }
+            Self::InvalidRouterWeights => (&["ValidationError", "ValueError"], None),
+            Self::NonNumericWeight { .. } => (&["TypeError"], None),
+            Self::MissingRoutingGroup { .. } => (&["KeyError", "LookupError"], None),
+            Self::IndexOutOfRange { .. } => (&["IndexError", "LookupError"], None),
+            Self::DefaultDeploymentRaises { .. } => (&["Exception"], None),
+            Self::Attempt(_) | Self::Config(_) | Self::Cache(_) | Self::Retired => return None,
+        };
+        let mut failure = AttemptError::host_raised(classes);
+        failure.status = status;
+        Some(failure)
+    }
+
+    /// A copy of a pick's error, for the events that name it; `None` for any other error.
+    pub fn pick_copy(&self) -> Option<Self> {
+        Some(match self {
+            Self::UnknownModel { model } => Self::UnknownModel {
+                model: model.clone(),
+            },
+            Self::NoHealthyDeployments { model } => Self::NoHealthyDeployments {
+                model: model.clone(),
+            },
+            Self::Paused { model } => Self::Paused {
+                model: model.clone(),
+            },
+            Self::NotPassThrough { id } => Self::NotPassThrough { id: id.clone() },
+            Self::NoPassThroughDeployments { model } => Self::NoPassThroughDeployments {
+                model: model.clone(),
+            },
+            Self::OnlyMarkers { model } => Self::OnlyMarkers {
+                model: model.clone(),
+            },
+            Self::PtuReserved { model } => Self::PtuReserved {
+                model: model.clone(),
+            },
+            Self::MultipleTeams { model } => Self::MultipleTeams {
+                model: model.clone(),
+            },
+            Self::Reserved {
+                model,
+                until,
+                timezone,
+            } => Self::Reserved {
+                model: model.clone(),
+                until: until.clone(),
+                timezone: timezone.clone(),
+            },
+            Self::ContextWindowExceeded { model, detail } => Self::ContextWindowExceeded {
+                model: model.clone(),
+                detail: detail.clone(),
+            },
+            Self::UnsupportedParams { model, params } => Self::UnsupportedParams {
+                model: model.clone(),
+                params: params.clone(),
+            },
+            Self::RoutingPluginsLeftNothing { models } => Self::RoutingPluginsLeftNothing {
+                models: models.clone(),
+            },
+            Self::InvalidRouterWeights => Self::InvalidRouterWeights,
+            Self::NonNumericWeight { metric } => Self::NonNumericWeight { metric },
+            Self::RoutingPluginsNeedAsync => Self::RoutingPluginsNeedAsync,
+            Self::MissingRoutingGroup { group } => Self::MissingRoutingGroup {
+                group: group.clone(),
+            },
+            Self::DefaultDeploymentRaises { model } => Self::DefaultDeploymentRaises {
+                model: model.clone(),
+            },
+            Self::IndexOutOfRange { model } => Self::IndexOutOfRange {
+                model: model.clone(),
+            },
+            Self::Attempt(_) | Self::Config(_) | Self::Cache(_) | Self::Retired => return None,
+        })
+    }
+}
+
+/// The class of a failed attempt, as litellm's exception mapping names it.
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    PartialEq,
+    Eq,
+    Hash,
+    strum::Display,
+    strum::EnumString,
+    strum::IntoStaticStr,
+    strum::VariantArray,
+)]
+pub enum FailureKind {
+    #[strum(serialize = "RateLimitError")]
+    RateLimit,
+    #[strum(serialize = "ContextWindowExceededError")]
+    ContextWindowExceeded,
+    #[strum(serialize = "ContentPolicyViolationError")]
+    ContentPolicyViolation,
+    #[strum(serialize = "AuthenticationError")]
+    Authentication,
+    #[strum(serialize = "PermissionDeniedError")]
+    PermissionDenied,
+    #[strum(serialize = "NotFoundError")]
+    NotFound,
+    #[strum(serialize = "BadRequestError")]
+    BadRequest,
+    #[strum(serialize = "Timeout")]
+    Timeout,
+    #[strum(serialize = "APIConnectionError")]
+    ApiConnection,
+    #[strum(serialize = "InternalServerError")]
+    InternalServer,
+    #[strum(serialize = "ServiceUnavailableError")]
+    ServiceUnavailable,
+    #[strum(serialize = "BadGatewayError")]
+    BadGateway,
+    /// A guardrail blocked the request: never retried, never a fallback.
+    #[strum(serialize = "GuardrailIntervention")]
+    GuardrailIntervention,
+    /// litellm rejected the request before calling the deployment (`MalformedRequestFieldError`,
+    /// `UnsupportedEndpointError`): never cools the deployment down.
+    #[strum(serialize = "Declined")]
+    Declined,
+    /// The caller's own credential failed, not the deployment's.
+    #[strum(serialize = "CallerCredentialError")]
+    CallerCredential,
+    #[strum(serialize = "APIError")]
+    Other,
+}
+
+/// An opaque value the host hands through the router unchanged, such as the original Python
+/// exception or response object.
+#[derive(Clone)]
+pub struct Payload(pub Arc<dyn Any + Send + Sync>);
+
+impl fmt::Debug for Payload {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str("Payload(..)")
+    }
+}
+
+/// One failed attempt, as typed facts the retry, fallback and cooldown rules decide by.
+#[derive(Clone, Debug, thiserror::Error)]
+#[error("the deployment call failed with {kind}")]
+pub struct AttemptError {
+    pub kind: FailureKind,
+    pub status: Option<u16>,
+    /// The provider's `retry-after` (or `retry-after-ms`) header.
+    pub retry_after: Option<Duration>,
+    /// A deployment-level `num_retries` the failure carries (Python's `exception.num_retries`).
+    pub num_retries: Option<u32>,
+    /// The provider's error text, kept only for the cooldown record Python writes
+    /// (`exception_received`, masked past 50 characters).
+    pub detail: Option<Arc<str>>,
+    /// The exception's class names, most specific first (its `__mro__`), for the rules Python
+    /// decides by `isinstance`. Empty when the host reports none; the kind then stands in.
+    pub classes: Vec<Arc<str>>,
+    /// The host already recorded what this failure does to the deployment's health (a routing
+    /// pre-call check's failure, which Python's attempt handles itself: a rate limit cools the
+    /// deployment down, anything else records nothing), so the core records nothing.
+    pub recorded: bool,
+    /// The exception names its provider (`llm_provider` is set), which marks a provider's own
+    /// 400 for `fallback_on_unsupported_params`.
+    pub names_provider: bool,
+    pub payload: Option<Payload>,
+}
+
+impl AttemptError {
+    pub fn new(kind: FailureKind, status: Option<u16>) -> Self {
+        Self {
+            kind,
+            status,
+            retry_after: None,
+            num_retries: None,
+            detail: None,
+            classes: Vec::new(),
+            recorded: false,
+            names_provider: false,
+            payload: None,
+        }
+    }
+
+    /// The exception's class names, most specific first: the host's, else the kind's.
+    pub fn class_names(&self) -> Vec<&str> {
+        if self.classes.is_empty() {
+            return crate::retry::kind_classes(self.kind).to_vec();
+        }
+        self.classes.iter().map(AsRef::as_ref).collect()
+    }
+
+    /// `isinstance(error, <class named name>)`.
+    pub fn is_a(&self, name: &str) -> bool {
+        self.class_names().contains(&name)
+    }
+
+    #[must_use]
+    pub fn with_retry_after(self, retry_after: Duration) -> Self {
+        Self {
+            retry_after: Some(retry_after),
+            ..self
+        }
+    }
+
+    /// A plain Python exception the host raised outside any attempt (a `KeyError` from a stale
+    /// routing-group lookup, an error a retry policy lookup raised): no status, no provider,
+    /// nothing recorded against a deployment. `classes` are its class names, most specific first.
+    pub fn host_raised(classes: &[&str]) -> Self {
+        Self {
+            classes: classes.iter().map(|class| (*class).into()).collect(),
+            recorded: true,
+            ..Self::new(FailureKind::Other, None)
+        }
+    }
+
+    /// The status Python's cooldown rules read: the provider's, else 500.
+    pub fn cooldown_status(&self) -> u16 {
+        self.status.unwrap_or(500)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+
+    #[rstest]
+    #[case::rate_limit("RateLimitError", FailureKind::RateLimit)]
+    #[case::context_window("ContextWindowExceededError", FailureKind::ContextWindowExceeded)]
+    #[case::caller_credential("CallerCredentialError", FailureKind::CallerCredential)]
+    #[case::other("APIError", FailureKind::Other)]
+    fn failure_kind_round_trips_its_exception_name(#[case] wire: &str, #[case] kind: FailureKind) {
+        assert_eq!(wire.parse::<FailureKind>().unwrap(), kind);
+        assert_eq!(<&'static str>::from(kind), wire);
+    }
+
+    #[rstest]
+    #[case::empty("")]
+    #[case::lowercase("ratelimiterror")]
+    fn failure_kind_rejects_unknown_names(#[case] wire: &str) {
+        assert!(wire.parse::<FailureKind>().is_err());
+    }
+}

litellm-rust/crates/router-core/src/fallback.rs

@@ -0,0 +1,310 @@
+//! Fallback lists as Python spells them, and how a failed group's chain is found in one.
+//! Mirrors `get_fallback_model_group`, `get_fallback_model_group_for_lookup_groups`,
+//! `check_non_standard_fallback_format` and `fallback_attempt_key` in
+//! `router_utils/fallback_event_handlers.py`.
+//!
+//! A list holds keyed entries (`{"gpt-4": ["gpt-3.5", {"model": "x", "temperature": 0}]}`),
+//! bare names (`"claude"`, a chain for every group) and, in a client-side list, dict targets
+//! (`{"model": "x", "messages": [...]}`), which override request params for their attempt.
+
+use serde::Deserialize;
+use serde_json::{Map, Value};
+
+use std::collections::BTreeMap;
+
+/// One item of a fallback list, kept as Python holds it.
+#[derive(Clone, Debug, PartialEq)]
+enum Entry {
+    Name(String),
+    Dict(Map<String, Value>),
+    /// Anything else: Python skips it when it looks a chain up.
+    Other,
+}
+
+/// A fallback list, in order. `"*"` keys the default chain.
+#[derive(Clone, Debug, Default, PartialEq, Deserialize)]
+#[serde(from = "Vec<Value>")]
+pub struct Fallbacks(Vec<Entry>);
+
+impl From<Vec<Value>> for Fallbacks {
+    fn from(entries: Vec<Value>) -> Self {
+        Self(
+            entries
+                .into_iter()
+                .map(|entry| match entry {
+                    Value::String(name) => Entry::Name(name),
+                    Value::Object(dict) => Entry::Dict(dict),
+                    _ => Entry::Other,
+                })
+                .collect(),
+        )
+    }
+}
+
+/// One target of a chain: a group name, or a dict target that also overrides request params.
+#[derive(Clone, Debug, PartialEq)]
+pub enum FallbackTarget {
+    Name(String),
+    Params(Map<String, Value>),
+}
+
+impl FallbackTarget {
+    fn of(value: &Value) -> Option<Self> {
+        match value {
+            Value::String(name) => Some(Self::Name(name.clone())),
+            Value::Object(dict) => Some(Self::Params(dict.clone())),
+            _ => None,
+        }
+    }
+
+    /// `_get_fallback_target_model_group`: the group the target names, if any.
+    pub fn model(&self) -> Option<&str> {
+        match self {
+            Self::Name(name) => Some(name),
+            Self::Params(dict) => dict.get("model").and_then(Value::as_str),
+        }
+    }
+
+    /// `fallback_attempt_key`: a bare name and `{"model": name}` are the same attempt; a dict
+    /// carrying anything else is its own.
+    pub fn identity(&self) -> FallbackTarget {
+        match self {
+            Self::Params(dict) if dict.len() == 1 => match dict.get("model") {
+                Some(Value::String(name)) => Self::Name(name.clone()),
+                _ => self.clone(),
+            },
+            _ => self.clone(),
+        }
+    }
+
+    /// The params a dict target merges into the request (`kwargs.update(target)`).
+    pub fn params(&self) -> Option<&Map<String, Value>> {
+        match self {
+            Self::Params(dict) => Some(dict),
+            Self::Name(_) => None,
+        }
+    }
+}
+
+/// What fallback keys match besides the exact group name: Python's `provider_list`, for
+/// `"openai/gpt-4o"` to use a `"gpt-4o"` entry, the provider-prefixed spelling of each bare
+/// group name (`_provider_prefixed_model_group`), for `"gpt-4o"` to use an `"openai/gpt-4o"`
+/// entry, and the `LiteLLMParamsTypedDict` keys that mark a client-side dict target.
+#[derive(Clone, Debug, Default, PartialEq, Deserialize)]
+#[serde(default)]
+pub struct FallbackKeys {
+    pub providers: Vec<String>,
+    pub provider_prefixed: BTreeMap<String, String>,
+    pub param_keys: Vec<String>,
+}
+
+impl FallbackKeys {
+    /// `_check_stripped_model_group`.
+    fn is_stripped(&self, group: &str, key: &str) -> bool {
+        self.providers.iter().any(|provider| {
+            let prefix = format!("{provider}/");
+            group.starts_with(&prefix) && group.replace(&prefix, "") == key
+        })
+    }
+}
+
+/// A chain found for a group, and whether it is the `"*"` chain itself.
+struct Found {
+    targets: Vec<FallbackTarget>,
+    generic: bool,
+}
+
+/// The targets of a keyed entry's value: a list of names and dicts. A bare string is walked
+/// character by character, as Python iterates it.
+fn chain_of(value: &Value) -> Vec<FallbackTarget> {
+    match value {
+        Value::Array(items) => items.iter().filter_map(FallbackTarget::of).collect(),
+        Value::String(text) => text
+            .chars()
+            .map(|c| FallbackTarget::Name(c.to_string()))
+            .collect(),
+        _ => Vec::new(),
+    }
+}
+
+impl Fallbacks {
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+
+    /// `check_non_standard_fallback_format`: a client-side list of names, or of dicts that set
+    /// a litellm param, is one chain for every group.
+    pub fn is_client_side(&self, keys: &FallbackKeys) -> bool {
+        if self.0.is_empty() {
+            return false;
+        }
+        if self.0.iter().all(|entry| matches!(entry, Entry::Name(_))) {
+            return true;
+        }
+        let dicts: Vec<_> = self
+            .0
+            .iter()
+            .filter_map(|entry| match entry {
+                Entry::Dict(dict) => Some(dict),
+                _ => None,
+            })
+            .collect();
+        dicts.len() == self.0.len()
+            && dicts.iter().any(|dict| {
+                keys.param_keys
+                    .iter()
+                    .any(|key| dict.get(key).is_some_and(|value| !value.is_array()))
+            })
+    }
+
+    /// The whole list as one chain (a client-side list).
+    pub fn as_chain(&self) -> Vec<FallbackTarget> {
+        self.0
+            .iter()
+            .filter_map(|entry| match entry {
+                Entry::Name(name) => Some(FallbackTarget::Name(name.clone())),
+                Entry::Dict(dict) => Some(FallbackTarget::Params(dict.clone())),
+                Entry::Other => None,
+            })
+            .collect()
+    }
+
+    /// `_get_first_default_fallback`: the first target of the first `"*"` entry.
+    pub fn default_target(&self) -> Option<&str> {
+        self.0.iter().find_map(|entry| match entry {
+            Entry::Dict(dict) => match dict.get("*") {
+                Some(Value::Array(targets)) => targets.first().and_then(Value::as_str),
+                _ => None,
+            },
+            _ => None,
+        })
+    }
+
+    /// `_has_default_fallbacks`: some entry keys `"*"`.
+    pub fn has_star(&self) -> bool {
+        self.0
+            .iter()
+            .any(|entry| matches!(entry, Entry::Dict(dict) if dict.contains_key("*")))
+    }
+
+    /// `get_fallback_model_group`.
+    fn lookup(&self, group: &str, keys: &FallbackKeys) -> Option<Found> {
+        let prefixed = keys.provider_prefixed.get(group).map(String::as_str);
+        let mut generic = None;
+        let mut stripped = None;
+        let mut found = None;
+        for entry in &self.0 {
+            match entry {
+                Entry::Dict(dict) => {
+                    let Some((key, value)) = dict.iter().next() else {
+                        continue;
+                    };
+                    if key == group {
+                        found = Some(chain_of(value));
+                        break;
+                    } else if Some(key.as_str()) == prefixed || keys.is_stripped(group, key) {
+                        stripped = Some(chain_of(value));
+                    } else if key == "*" {
+                        generic = Some(value);
+                    }
+                }
+                Entry::Name(name) => found = Some(vec![FallbackTarget::Name(name.clone())]),
+                Entry::Other => {}
+            }
+        }
+        found
+            .or(stripped)
+            .map(|targets| Found {
+                targets,
+                generic: false,
+            })
+            .or_else(|| {
+                generic.map(|value| Found {
+                    targets: chain_of(value),
+                    generic: true,
+                })
+            })
+    }
+
+    /// `get_fallback_model_group_for_lookup_groups`: the first lookup group with its own chain
+    /// wins; `"*"` applies only after every group missed.
+    pub fn lookup_groups(
+        &self,
+        groups: &[&str],
+        keys: &FallbackKeys,
+    ) -> Option<Vec<FallbackTarget>> {
+        let results: Vec<Found> = groups
+            .iter()
+            .filter_map(|group| self.lookup(group, keys))
+            .collect();
+        let index = results
+            .iter()
+            .position(|found| !found.generic)
+            .or_else(|| (!results.is_empty()).then_some(0))?;
+        results.into_iter().nth(index).map(|found| found.targets)
+    }
+
+    /// `_has_fallback_group`'s view: only keyed entries whose targets are names or dicts count.
+    pub fn has_keyed_chain(&self, group: &str, keys: &FallbackKeys) -> bool {
+        let keyed = Fallbacks(
+            self.0
+                .iter()
+                .filter(|entry| matches!(entry, Entry::Dict(_)))
+                .cloned()
+                .collect(),
+        );
+        keyed
+            .lookup(group, keys)
+            .is_some_and(|found| !found.targets.is_empty())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+
+    fn names(targets: Option<Vec<FallbackTarget>>) -> Option<Vec<String>> {
+        targets.map(|targets| {
+            targets
+                .iter()
+                .map(|t| t.model().unwrap_or_default().to_owned())
+                .collect()
+        })
+    }
+
+    #[rstest]
+    #[case::own_entry(json!([{"a": ["b"]}, {"*": ["d"]}]), "a", Some(vec!["b"]))]
+    #[case::star_entry(json!([{"a": ["b"]}, {"*": ["d"]}]), "x", Some(vec!["d"]))]
+    #[case::string_is_every_groups_chain(json!(["b", {"x": ["c"]}]), "a", Some(vec!["b"]))]
+    #[case::keyed_entry_wins(json!(["c", {"a": ["b"]}]), "a", Some(vec!["b"]))]
+    #[case::string_value_is_iterated(json!([{"a": "bc"}]), "a", Some(vec!["b", "c"]))]
+    #[case::none(json!([{"x": ["c"]}]), "a", None)]
+    fn chains_follow_python(
+        #[case] list: Value,
+        #[case] group: &str,
+        #[case] expected: Option<Vec<&str>>,
+    ) {
+        let fallbacks: Fallbacks = serde_json::from_value(list).unwrap();
+        assert_eq!(
+            names(fallbacks.lookup_groups(&[group], &FallbackKeys::default())),
+            expected.map(|e| e.into_iter().map(str::to_owned).collect())
+        );
+    }
+
+    #[rstest]
+    #[case::names(json!(["a", "b"]), true)]
+    #[case::param_dicts(json!([{"model": "a", "temperature": 1}]), true)]
+    #[case::keyed(json!([{"a": ["b"]}]), false)]
+    #[case::mixed(json!(["a", {"a": ["b"]}]), false)]
+    fn client_side_lists_follow_python(#[case] list: Value, #[case] expected: bool) {
+        let fallbacks: Fallbacks = serde_json::from_value(list).unwrap();
+        let keys = FallbackKeys {
+            param_keys: vec!["model".to_owned(), "temperature".to_owned()],
+            ..FallbackKeys::default()
+        };
+        assert_eq!(fallbacks.is_client_side(&keys), expected);
+    }
+}

litellm-rust/crates/router-core/src/health.rs

@@ -0,0 +1,644 @@
+//! [`Health`]: cooldowns, allowed-fails counters and per-minute usage, kept in the shared
+//! [`RouterCache`] under the Python router's keys, encodings and TTLs:
+//!
+//! | state | key | value | TTL |
+//! |---|---|---|---|
+//! | cooldown | `deployment:{id}:cooldown` | `{exception_received, status_code, timestamp, cooldown_time}` | cooldown time |
+//! | allowed fails | `deployment:{id}:allowed_fails[:{suffix}]` | int counter | cooldown time |
+//! | usage | `global_router:{id}:{model}:tpm:{HH-MM}` and `:rpm:` | int counter | 60s |
+//! | outcomes (local only) | `{id}:successes`, `{id}:fails` | int counter | 60s |
+//!
+//! From `router_utils/cooldown_cache.py`, `router_utils/cooldown_handlers.py`,
+//! `router_utils/router_callbacks/track_deployment_metrics.py`, `RouterCacheEnum` in
+//! `types/router.py` and `Router._increment_usage` / `deployment_callback_on_failure` in
+//! `router.py`.
+
+use std::{
+    sync::Arc,
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+
+use serde::{Deserialize, Serialize};
+use tracing::debug;
+
+use crate::{
+    cache::RouterCache,
+    error::{AttemptError, Error, FailureKind},
+    registry::Deployment,
+    settings::Settings,
+};
+
+/// `RoutingArgs.ttl`: per-minute usage keys expire after a minute.
+pub const USAGE_TTL: Duration = Duration::from_secs(60);
+/// `DEFAULT_FAILURE_THRESHOLD_PERCENT`.
+pub const FAILURE_THRESHOLD_PERCENT: f64 = 0.5;
+/// `DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS`.
+pub const FAILURE_THRESHOLD_MINIMUM_REQUESTS: u64 = 5;
+/// `SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD`.
+pub const SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD: u64 = 1000;
+/// `exception_masker` in `CooldownCache`: the visible prefix of `exception_received`.
+const EXCEPTION_VISIBLE_PREFIX: usize = 50;
+
+pub trait Clock: Send + Sync {
+    fn now(&self) -> SystemTime;
+}
+
+pub struct SystemClock;
+
+impl Clock for SystemClock {
+    fn now(&self) -> SystemTime {
+        SystemTime::now()
+    }
+}
+
+fn unix_seconds(time: SystemTime) -> f64 {
+    time.duration_since(UNIX_EPOCH)
+        .map(|elapsed| elapsed.as_secs_f64())
+        .unwrap_or_default()
+}
+
+/// `get_utc_datetime().strftime("%H-%M")`.
+pub fn current_minute(time: SystemTime) -> String {
+    let seconds = time
+        .duration_since(UNIX_EPOCH)
+        .map(|elapsed| elapsed.as_secs())
+        .unwrap_or_default()
+        % 86_400;
+    format!("{:02}-{:02}", seconds / 3600, (seconds % 3600) / 60)
+}
+
+pub fn cooldown_key(id: &str) -> String {
+    format!("deployment:{id}:cooldown")
+}
+
+pub fn allowed_fails_key(id: &str, suffix: Option<&str>) -> String {
+    match suffix {
+        Some(suffix) => format!("deployment:{id}:allowed_fails:{suffix}"),
+        None => format!("deployment:{id}:allowed_fails"),
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display)]
+pub enum UsageMetric {
+    #[strum(serialize = "tpm")]
+    Tpm,
+    #[strum(serialize = "rpm")]
+    Rpm,
+}
+
+/// `RouterCacheEnum.TPM` / `RPM`: `model` is the deployment's `litellm_params.model`.
+pub fn usage_key(id: &str, model: &str, metric: UsageMetric, minute: &str) -> String {
+    format!("global_router:{id}:{model}:{metric}:{minute}")
+}
+
+/// `CooldownCacheValue`.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct CooldownValue {
+    pub exception_received: String,
+    pub status_code: String,
+    pub timestamp: f64,
+    /// Whole seconds are written as an int, as Python's default `cooldown_time` is, so the
+    /// "Try again in N seconds" text reads the same.
+    #[serde(serialize_with = "whole_as_int")]
+    pub cooldown_time: f64,
+}
+
+fn whole_as_int<S: serde::Serializer>(value: &f64, serializer: S) -> Result<S::Ok, S::Error> {
+    if value.fract() == 0.0 && value.abs() < i64::MAX as f64 {
+        serializer.serialize_i64(*value as i64)
+    } else {
+        serializer.serialize_f64(*value)
+    }
+}
+
+/// `litellm.should_retry`.
+pub fn should_retry_status(status: u16) -> bool {
+    matches!(status, 408 | 409 | 429) || status >= 500
+}
+
+/// `_is_cooldown_required`: a 4xx cools down only for 429, 401, 402, 404 and 408, every other
+/// status does, and a connection error never does.
+pub fn is_cooldown_required(failure: &AttemptError) -> bool {
+    if failure.kind == FailureKind::ApiConnection {
+        return false;
+    }
+    match failure.cooldown_status() {
+        status @ 400..500 => matches!(status, 429 | 401 | 402 | 408 | 404),
+        _ => true,
+    }
+}
+
+/// `SensitiveDataMasker(visible_prefix=50, visible_suffix=0, mask_short_values=False)`.
+fn mask_exception(text: &str) -> String {
+    let length = text.chars().count();
+    if length <= EXCEPTION_VISIBLE_PREFIX {
+        return text.to_owned();
+    }
+    let mut masked: String = text.chars().take(EXCEPTION_VISIBLE_PREFIX).collect();
+    masked.extend(std::iter::repeat_n('*', length - EXCEPTION_VISIBLE_PREFIX));
+    masked
+}
+
+/// `_EXCEPTION_POLICY_FIELDS`: a deployment policy's fields in the order Python matches them
+/// (a content-policy violation is a bad request, so it comes first).
+const DEPLOYMENT_POLICY_FIELDS: [(&str, &str); 9] = [
+    (
+        "ContentPolicyViolationError",
+        "ContentPolicyViolationErrorAllowedFails",
+    ),
+    ("BadRequestError", "BadRequestErrorAllowedFails"),
+    ("AuthenticationError", "AuthenticationErrorAllowedFails"),
+    ("Timeout", "TimeoutErrorAllowedFails"),
+    ("RateLimitError", "RateLimitErrorAllowedFails"),
+    ("InternalServerError", "InternalServerErrorAllowedFails"),
+    (
+        "ServiceUnavailableError",
+        "ServiceUnavailableErrorAllowedFails",
+    ),
+    ("BadGatewayError", "BadGatewayErrorAllowedFails"),
+    ("NotFoundError", "NotFoundErrorAllowedFails"),
+];
+
+/// `_resolve_allowed_fails_from_policy` over a deployment's `model_info.allowed_fails_policy`:
+/// the first field set for a class the failure is an instance of.
+pub fn deployment_allowed_fails(
+    policy: Option<&serde_json::Map<String, serde_json::Value>>,
+    failure: &AttemptError,
+) -> Option<f64> {
+    let policy = policy?;
+    DEPLOYMENT_POLICY_FIELDS
+        .iter()
+        .filter(|(class, _)| failure.is_a(class))
+        .find_map(|(_, field)| policy.get(*field).and_then(serde_json::Value::as_f64))
+}
+
+/// What the cooldown rules need to know about the deployment's group.
+#[derive(Clone, Copy, Debug, Default)]
+pub struct GroupShape {
+    /// The deployment is the only one in its model group.
+    pub single_deployment: bool,
+    /// The requested group has a fallback configured.
+    pub has_fallback_group: bool,
+}
+
+/// An outcome the router did not see itself (`Health.report`).
+#[derive(Clone, Debug)]
+pub enum Outcome {
+    Success { total_tokens: u64 },
+    Failure(AttemptError),
+}
+
+#[derive(Clone)]
+pub struct Health {
+    cache: Arc<dyn RouterCache>,
+    clock: Arc<dyn Clock>,
+}
+
+impl Health {
+    pub fn new(cache: Arc<dyn RouterCache>, clock: Arc<dyn Clock>) -> Self {
+        Self { cache, clock }
+    }
+
+    pub fn cache(&self) -> &Arc<dyn RouterCache> {
+        &self.cache
+    }
+
+    pub(crate) fn now_time(&self) -> SystemTime {
+        self.clock.now()
+    }
+
+    fn now(&self) -> f64 {
+        unix_seconds(self.clock.now())
+    }
+
+    /// The ids among `ids` with an unexpired cooldown (`async_get_active_cooldowns`).
+    pub async fn active_cooldowns(
+        &self,
+        ids: &[String],
+    ) -> Result<Vec<(String, CooldownValue)>, Error> {
+        let keys: Vec<String> = ids.iter().map(|id| cooldown_key(id)).collect();
+        let values = self.cache.async_batch_get_cache(&keys).await?;
+        let now = self.now();
+        Ok(ids
+            .iter()
+            .zip(values)
+            .filter_map(|(id, value)| {
+                let value: CooldownValue = serde_json::from_value(value?).ok()?;
+                (value.timestamp + value.cooldown_time - now > 0.0).then(|| (id.clone(), value))
+            })
+            .collect())
+    }
+
+    /// `add_deployment_to_cooldown`.
+    pub async fn add_cooldown(
+        &self,
+        id: &str,
+        failure: &AttemptError,
+        cooldown: Duration,
+    ) -> Result<(), Error> {
+        let value = CooldownValue {
+            exception_received: mask_exception(failure.detail.as_deref().unwrap_or_default()),
+            status_code: failure.cooldown_status().to_string(),
+            timestamp: self.now(),
+            cooldown_time: cooldown.as_secs_f64(),
+        };
+        let value = serde_json::to_value(value).unwrap_or_default();
+        self.cache
+            .async_set_cache(&cooldown_key(id), value, Some(cooldown))
+            .await?;
+        Ok(())
+    }
+
+    async fn local_count(&self, key: &str) -> Result<u64, Error> {
+        Ok(self
+            .cache
+            .async_get_local(key)
+            .await?
+            .and_then(|value| value.as_f64())
+            .map_or(0, |count| count as u64))
+    }
+
+    /// The per-minute (successes, fails) this worker saw (`{id}:successes`, `{id}:fails`, kept
+    /// `local_only`).
+    pub async fn outcomes_this_minute(&self, id: &str) -> Result<(u64, u64), Error> {
+        Ok((
+            self.local_count(&format!("{id}:successes")).await?,
+            self.local_count(&format!("{id}:fails")).await?,
+        ))
+    }
+
+    /// `increment_deployment_successes_for_current_minute`: returns the key it counted under.
+    pub async fn count_success(&self, id: &str) -> Result<String, Error> {
+        let key = format!("{id}:successes");
+        self.cache
+            .async_increment_local(&key, 1.0, Some(USAGE_TTL))
+            .await?;
+        Ok(key)
+    }
+
+    /// `increment_deployment_failures_for_current_minute`.
+    pub async fn count_failure(&self, id: &str) -> Result<(), Error> {
+        self.cache
+            .async_increment_local(&format!("{id}:fails"), 1.0, Some(USAGE_TTL))
+            .await?;
+        Ok(())
+    }
+
+    /// Cooldown duration precedence from `deployment_callback_on_failure`: the deployment's
+    /// `cooldown_time`, then the provider's retry-after, then the router default.
+    pub fn cooldown_duration(
+        settings: &Settings,
+        deployment: &Deployment,
+        failure: &AttemptError,
+    ) -> Duration {
+        Self::cooldown_for_call(settings, deployment, None, failure)
+    }
+
+    /// `cooldown_duration` for a call whose own params carry `cooldown_time` (a request's
+    /// value wins over the deployment's `litellm_params`, not over its `model_info`).
+    pub fn cooldown_for_call(
+        settings: &Settings,
+        deployment: &Deployment,
+        call_cooldown: Option<f64>,
+        failure: &AttemptError,
+    ) -> Duration {
+        let configured = deployment
+            .model_info
+            .cooldown_time
+            .or(call_cooldown)
+            .or_else(|| {
+                deployment
+                    .raw_params
+                    .get("cooldown_time")
+                    .and_then(|v| v.as_f64())
+            })
+            .filter(|seconds| *seconds >= 0.0)
+            .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok());
+        configured
+            .or(failure.retry_after)
+            .unwrap_or(settings.cooldown_time)
+    }
+
+    /// The deployment's own `cooldown_time`: `model_info` first, then `litellm_params`
+    /// (`_first_present(model_info, litellm_params, key="cooldown_time")`). It is both the
+    /// cooldown and the TTL of the deployment's `allowed_fails:generic` counter.
+    fn configured_cooldown(deployment: &Deployment) -> Option<Duration> {
+        deployment
+            .model_info
+            .cooldown_time
+            .or_else(|| {
+                deployment
+                    .raw_params
+                    .get("cooldown_time")
+                    .and_then(|v| v.as_f64())
+            })
+            .filter(|seconds| *seconds >= 0.0)
+            .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok())
+    }
+
+    /// Record a failed attempt and cool the deployment down when the rules say so. Returns the
+    /// cooldown applied, if any.
+    pub async fn record_failure(
+        &self,
+        settings: &Settings,
+        deployment: &Deployment,
+        shape: GroupShape,
+        call_cooldown: Option<f64>,
+        failure: &AttemptError,
+    ) -> Result<Option<Duration>, Error> {
+        self.count_failure(&deployment.id).await?;
+        let cooldown = Self::cooldown_for_call(settings, deployment, call_cooldown, failure);
+        self.cool_down_after(settings, deployment, shape, failure, cooldown)
+            .await
+    }
+
+    /// `set_cooldown_deployments` for a failure already counted: cool the deployment down for
+    /// `cooldown` when the rules say so. Returns the cooldown applied, if any.
+    pub async fn cool_down_after(
+        &self,
+        settings: &Settings,
+        deployment: &Deployment,
+        shape: GroupShape,
+        failure: &AttemptError,
+        cooldown: Duration,
+    ) -> Result<Option<Duration>, Error> {
+        if !Self::should_run_cooldown_logic(settings, deployment, failure, cooldown) {
+            return Ok(None);
+        }
+        if !self
+            .should_cooldown(settings, deployment, shape, failure)
+            .await?
+        {
+            return Ok(None);
+        }
+        debug!(deployment_id = %deployment.id, cooldown_seconds = cooldown.as_secs_f64(), "cooling deployment down");
+        self.add_cooldown(&deployment.id, failure, cooldown).await?;
+        Ok(Some(cooldown))
+    }
+
+    /// `_should_run_cooldown_logic`.
+    fn should_run_cooldown_logic(
+        settings: &Settings,
+        deployment: &Deployment,
+        failure: &AttemptError,
+        cooldown: Duration,
+    ) -> bool {
+        !matches!(
+            failure.kind,
+            FailureKind::Declined | FailureKind::CallerCredential
+        ) && !cooldown.is_zero()
+            && !settings.disable_cooldowns
+            && (is_cooldown_required(failure)
+                || deployment_allowed_fails(
+                    deployment.model_info.allowed_fails_policy.as_ref(),
+                    failure,
+                )
+                .is_some())
+            && !settings
+                .provider_default_deployment_ids
+                .contains(&deployment.id)
+    }
+
+    /// `_should_cooldown_deployment`.
+    async fn should_cooldown(
+        &self,
+        settings: &Settings,
+        deployment: &Deployment,
+        shape: GroupShape,
+        failure: &AttemptError,
+    ) -> Result<bool, Error> {
+        let status = failure.cooldown_status();
+        let policy_fails = settings.allowed_fails_for(failure);
+        let lone = shape.single_deployment && !shape.has_fallback_group;
+        let deployment_policy = deployment.model_info.allowed_fails_policy.as_ref();
+        let deployment_fails = deployment_allowed_fails(deployment_policy, failure);
+        if shape.single_deployment
+            && status == 402
+            && deployment_fails.is_none()
+            && policy_fails.is_none()
+        {
+            return Ok(false);
+        }
+        let deployment_allowed = deployment.model_info.allowed_fails;
+        if deployment_policy.is_some() || deployment_allowed.is_some() {
+            // `_should_cooldown_based_on_deployment_policy`: a deployment-wide `allowed_fails`
+            // keeps the lone-deployment safety net; a policy entry naming the error does not.
+            if deployment_fails.is_none()
+                && lone
+                && (deployment_allowed.is_some() || policy_fails.is_none())
+            {
+                return Ok(false);
+            }
+            let (allowed, suffix) = match deployment_fails {
+                Some(allowed) => (
+                    Some(allowed),
+                    failure.class_names().first().map(|name| (*name).to_owned()),
+                ),
+                None => (
+                    deployment_allowed.map(f64::from),
+                    deployment_allowed.map(|_| "generic".to_owned()),
+                ),
+            };
+            let allowed = allowed
+                .or_else(|| policy_fails.map(f64::from))
+                .unwrap_or_else(|| f64::from(settings.effective_allowed_fails()));
+            let ttl =
+                Self::configured_cooldown(deployment).unwrap_or_else(|| settings.fails_window());
+            return self
+                .exceeds_allowed_fails(&deployment.id, suffix.as_deref(), allowed, ttl)
+                .await;
+        }
+        if settings.allowed_fails_policy.is_none() && settings.allowed_fails.is_none() {
+            let (successes, fails) = self.outcomes_this_minute(&deployment.id).await?;
+            let total = successes + fails;
+            let percent_fails = if total > 0 {
+                fails as f64 / total as f64
+            } else {
+                0.0
+            };
+            return Ok((status == 429 && !shape.single_deployment)
+                || (percent_fails == 1.0 && total >= SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD)
+                || (percent_fails > FAILURE_THRESHOLD_PERCENT
+                    && total >= FAILURE_THRESHOLD_MINIMUM_REQUESTS
+                    && !shape.single_deployment)
+                || !should_retry_status(status));
+        }
+        if lone && policy_fails.is_none() {
+            return Ok(false);
+        }
+        let allowed = policy_fails.unwrap_or_else(|| settings.effective_allowed_fails());
+        self.exceeds_allowed_fails(
+            &deployment.id,
+            None,
+            f64::from(allowed),
+            settings.fails_window(),
+        )
+        .await
+    }
+
+    /// `should_cooldown_based_on_allowed_fails_policy`: count this failure, then compare.
+    async fn exceeds_allowed_fails(
+        &self,
+        id: &str,
+        suffix: Option<&str>,
+        allowed: f64,
+        ttl: Duration,
+    ) -> Result<bool, Error> {
+        let fails = self
+            .cache
+            .async_increment(&allowed_fails_key(id, suffix), 1.0, Some(ttl))
+            .await?;
+        Ok(fails > allowed)
+    }
+
+    /// Count a success, and the tokens and request against the deployment's per-minute usage
+    /// when it has limits to track.
+    pub async fn record_success(
+        &self,
+        deployment: &Deployment,
+        total_tokens: u64,
+    ) -> Result<(), Error> {
+        self.count_success(&deployment.id).await?;
+        self.add_usage(deployment, total_tokens as f64, 1.0).await
+    }
+
+    /// `_increment_deployment_usage`: tokens and requests against a deployment that tracks
+    /// usage, unless there is nothing to add.
+    pub async fn add_usage(
+        &self,
+        deployment: &Deployment,
+        total_tokens: f64,
+        requests: f64,
+    ) -> Result<(), Error> {
+        if (total_tokens <= 0.0 && requests <= 0.0) || !deployment.tracks_usage() {
+            return Ok(());
+        }
+        let minute = current_minute(self.clock.now());
+        let model = &deployment.params.model;
+        self.cache
+            .async_increment(
+                &usage_key(&deployment.id, model, UsageMetric::Tpm, &minute),
+                total_tokens,
+                Some(USAGE_TTL),
+            )
+            .await?;
+        self.cache
+            .async_increment(
+                &usage_key(&deployment.id, model, UsageMetric::Rpm, &minute),
+                requests,
+                Some(USAGE_TTL),
+            )
+            .await?;
+        Ok(())
+    }
+
+    /// `async_deployment_callback_on_failure`: a failed request counts against the
+    /// deployment's per-minute requests, whether or not it tracks usage.
+    pub async fn count_failed_request(&self, deployment: &Deployment) -> Result<(), Error> {
+        let minute = current_minute(self.clock.now());
+        self.cache
+            .async_increment(
+                &usage_key(
+                    &deployment.id,
+                    &deployment.params.model,
+                    UsageMetric::Rpm,
+                    &minute,
+                ),
+                1.0,
+                Some(USAGE_TTL),
+            )
+            .await?;
+        Ok(())
+    }
+
+    /// This minute's (tpm, rpm) for a deployment.
+    pub async fn usage(&self, deployment: &Deployment) -> Result<(u64, u64), Error> {
+        let minute = current_minute(self.clock.now());
+        let model = &deployment.params.model;
+        let keys = [UsageMetric::Tpm, UsageMetric::Rpm]
+            .map(|metric| usage_key(&deployment.id, model, metric, &minute));
+        let values = self.cache.async_batch_get_cache(&keys).await?;
+        let count = |index: usize| {
+            values
+                .get(index)
+                .cloned()
+                .flatten()
+                .and_then(|value| value.as_f64())
+                .map_or(0, |count| count as u64)
+        };
+        Ok((count(0), count(1)))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+
+    #[rstest]
+    #[case::rate_limit(FailureKind::RateLimit, Some(429), true)]
+    #[case::auth(FailureKind::Authentication, Some(401), true)]
+    #[case::payment(FailureKind::Other, Some(402), true)]
+    #[case::not_found(FailureKind::NotFound, Some(404), true)]
+    #[case::timeout(FailureKind::Timeout, Some(408), true)]
+    #[case::bad_request(FailureKind::BadRequest, Some(400), false)]
+    #[case::forbidden(FailureKind::PermissionDenied, Some(403), false)]
+    #[case::server(FailureKind::InternalServer, Some(500), true)]
+    #[case::no_status(FailureKind::Other, None, true)]
+    #[case::connection(FailureKind::ApiConnection, Some(500), false)]
+    fn cooldown_required_follows_python(
+        #[case] kind: FailureKind,
+        #[case] status: Option<u16>,
+        #[case] expected: bool,
+    ) {
+        assert_eq!(
+            is_cooldown_required(&AttemptError::new(kind, status)),
+            expected
+        );
+    }
+
+    #[rstest]
+    #[case::midnight(0, "00-00")]
+    #[case::afternoon(13 * 3600 + 7 * 60 + 59, "13-07")]
+    fn current_minute_is_utc_hours_and_minutes(#[case] seconds: u64, #[case] expected: &str) {
+        let time = UNIX_EPOCH + Duration::from_secs(86_400 * 3 + seconds);
+        assert_eq!(current_minute(time), expected);
+    }
+
+    #[rstest]
+    #[case::content_policy_first(&["ContentPolicyViolationError", "BadRequestError"], Some(1.0))]
+    #[case::bad_request(&["BadRequestError"], Some(2.0))]
+    #[case::unnamed(&["Timeout"], None)]
+    fn a_deployment_policy_matches_in_python_order(
+        #[case] classes: &[&str],
+        #[case] expected: Option<f64>,
+    ) {
+        let policy = serde_json::json!({
+            "BadRequestErrorAllowedFails": 2,
+            "ContentPolicyViolationErrorAllowedFails": 1,
+        });
+        let mut failure = AttemptError::new(FailureKind::BadRequest, Some(400));
+        failure.classes = classes.iter().map(|class| (*class).into()).collect();
+        assert_eq!(
+            deployment_allowed_fails(policy.as_object(), &failure),
+            expected
+        );
+    }
+
+    #[rstest]
+    fn keys_match_python() {
+        assert_eq!(
+            (
+                cooldown_key("abc"),
+                allowed_fails_key("abc", Some("generic")),
+                usage_key("abc", "openai/gpt-4o", UsageMetric::Tpm, "09-05"),
+            ),
+            (
+                "deployment:abc:cooldown".to_owned(),
+                "deployment:abc:allowed_fails:generic".to_owned(),
+                "global_router:abc:openai/gpt-4o:tpm:09-05".to_owned(),
+            )
+        );
+    }
+}

litellm-rust/crates/router-core/src/health_routing.rs

@@ -0,0 +1,126 @@
+//! Health-check-driven routing: the background health checks' results, kept in the shared
+//! [`RouterCache`](crate::cache::RouterCache) under one key as Python's `DeploymentHealthCache`
+//! writes them, and the filter `_async_filter_health_check_unhealthy_deployments` /
+//! `_filter_health_check_unhealthy_deployments` run before the cooldown filter.
+
+use std::{collections::BTreeSet, sync::Arc, time::UNIX_EPOCH};
+
+use serde_json::Value;
+
+use crate::{health::Health, registry::Deployment, settings::Settings};
+
+/// `DeploymentHealthCache.CACHE_KEY`.
+pub const HEALTH_STATE_KEY: &str = "litellm:health_check:deployment_health_state";
+
+/// `_extract_unhealthy_ids`: the ids marked unhealthy whose result is younger than `staleness`.
+pub fn unhealthy_ids(raw: Option<&Value>, now: f64, staleness: f64) -> BTreeSet<String> {
+    let Some(Value::Object(states)) = raw else {
+        return BTreeSet::new();
+    };
+    states
+        .iter()
+        .filter(|(_, state)| {
+            let Value::Object(state) = state else {
+                return false;
+            };
+            // `not state.get("is_healthy", True)`: any falsy value is unhealthy.
+            let unhealthy = match state.get("is_healthy") {
+                None => false,
+                Some(Value::Null) => true,
+                Some(Value::Bool(value)) => !value,
+                Some(Value::Number(value)) => value.as_f64() == Some(0.0),
+                Some(Value::String(value)) => value.is_empty(),
+                Some(Value::Array(value)) => value.is_empty(),
+                Some(Value::Object(value)) => value.is_empty(),
+            };
+            let timestamp = state
+                .get("timestamp")
+                .and_then(Value::as_f64)
+                .unwrap_or(0.0);
+            unhealthy && now - timestamp < staleness
+        })
+        .map(|(id, _)| id.clone())
+        .collect()
+}
+
+/// What the filter does when it would leave nothing.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum WhenAllUnhealthy {
+    /// The safety net: every candidate is kept.
+    KeepAll,
+    /// A health check probe sees no deployment.
+    Nothing,
+}
+
+impl Health {
+    /// The deployments the background health checks have not marked unhealthy, as the
+    /// router's health-check routing settings say.
+    pub async fn health_check_filter(
+        &self,
+        settings: &Settings,
+        deployments: Vec<Arc<Deployment>>,
+        when_all_unhealthy: WhenAllUnhealthy,
+    ) -> Vec<Arc<Deployment>> {
+        if !settings.enable_health_check_routing {
+            return deployments;
+        }
+        let scoped = settings.background_health_check_model_groups.as_deref();
+        // With an allowed_fails_policy cooldowns are the only exclusion, unless the filter is
+        // scoped to listed groups.
+        if settings.allowed_fails_policy.is_some() && scoped.is_none() {
+            return deployments;
+        }
+        // A read that fails counts as no health state.
+        let raw = self
+            .cache()
+            .async_get_cache(HEALTH_STATE_KEY)
+            .await
+            .ok()
+            .flatten();
+        let now = self
+            .now_time()
+            .duration_since(UNIX_EPOCH)
+            .map(|elapsed| elapsed.as_secs_f64())
+            .unwrap_or_default();
+        let unhealthy = unhealthy_ids(
+            raw.as_ref(),
+            now,
+            settings.health_check_staleness.as_secs_f64(),
+        );
+        if unhealthy.is_empty() {
+            return deployments;
+        }
+        let filtered: Vec<Arc<Deployment>> = deployments
+            .iter()
+            .filter(|d| {
+                !unhealthy.contains(&d.id)
+                    || scoped.is_some_and(|groups| !groups.contains(&d.model_name))
+            })
+            .cloned()
+            .collect();
+        match (filtered.is_empty(), when_all_unhealthy) {
+            (false, _) => filtered,
+            (true, WhenAllUnhealthy::KeepAll) => deployments,
+            (true, WhenAllUnhealthy::Nothing) => Vec::new(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+
+    #[rstest]
+    #[case::fresh_unhealthy(json!({"a": {"is_healthy": false, "timestamp": 95.0}}), vec!["a"])]
+    #[case::stale(json!({"a": {"is_healthy": false, "timestamp": 50.0}}), vec![])]
+    #[case::healthy(json!({"a": {"is_healthy": true, "timestamp": 95.0}}), vec![])]
+    #[case::no_flag(json!({"a": {"timestamp": 95.0}}), vec![])]
+    #[case::not_a_dict(json!({"a": 1}), vec![])]
+    fn unhealthy_ids_follow_python(#[case] raw: Value, #[case] expected: Vec<&str>) {
+        let ids = unhealthy_ids(Some(&raw), 100.0, 10.0);
+        assert_eq!(ids.iter().map(String::as_str).collect::<Vec<_>>(), expected);
+    }
+}

litellm-rust/crates/router-core/src/host.rs

@@ -0,0 +1,402 @@
+//! [`Host`]: the router core's only boundary. Everything the core needs from the embedding
+//! process goes through it, and every attempt runs through [`Host::invoke`]: the core never runs
+//! an operation itself. `PythonHost` serves it in a Python process (attempts call `litellm.<op>`);
+//! `NativeHost` serves it in the Rust server (attempts call the `inference-*` crates).
+
+use std::{collections::BTreeMap, sync::Arc, time::Duration};
+
+use async_trait::async_trait;
+use serde_json::{Map, Value};
+
+use crate::{
+    cache::RouterCache,
+    config::{Op, RetryPolicy, RoutingStrategy},
+    error::{AttemptError, Error, FailureKind, Payload},
+    fallback::Fallbacks,
+    health::Health,
+    mock::MockFailure,
+    precall::PreCallFacts,
+    prerouting::{PreRoute, PreRouted},
+    registry::{Deployment, Registry},
+    settings::Settings,
+    strategy::StrategyScope,
+};
+
+/// One call into the router.
+#[derive(Clone, Debug, Default)]
+pub struct RouterRequest {
+    /// The requested name: a model group, alias, routing group, deployment id or wildcard match.
+    pub model: String,
+    /// The request params, for native executors and plugins that read them.
+    pub body: Value,
+    /// `user_api_key_team_id`.
+    pub team_id: Option<String>,
+    /// The caller's `user_role` is `proxy_admin`.
+    pub proxy_admin: bool,
+    /// The caller's `models` plus `team_models` (`user_api_key_auth`), which narrow a caller
+    /// allowed only through access groups to those groups' deployments.
+    pub allowed_models: Vec<String>,
+    /// For a proxy admin, the model the request names (`model`, else the metadata
+    /// `model_group`), which keeps one team's deployments listed under it (`filter_team_based_models`).
+    pub admin_model: Option<String>,
+    pub specific_deployment: bool,
+    /// A per-request `num_retries`, which wins over the router's.
+    pub num_retries: Option<u32>,
+    pub disable_fallbacks: bool,
+    /// `_router_weights` as the request passed it: model group to deployment id to weight,
+    /// validated when simple-shuffle picks ([`crate::weights::validate`]).
+    pub router_weights: Option<Value>,
+    /// The deployment a call on an existing resource (a video id) names: picked whenever the
+    /// filters keep it (`_narrow_to_video_deployment`).
+    pub pinned_deployment_id: Option<String>,
+    /// A per-request `routing_strategy` override, as Python normalizes it.
+    pub routing_strategy: Option<String>,
+    /// Whether tag routing may apply to this request (router, request or model setting), so the
+    /// host's tag stage runs.
+    pub filter_tags: bool,
+    /// A pass-through pick: only `use_in_pass_through` deployments may serve it.
+    pub pass_through: Option<crate::select::PassThrough>,
+    /// `_target_order`: the order level to pick from instead of the lowest.
+    pub target_order: Option<i64>,
+    /// `_excluded_deployment_ids`: never picked, even when that leaves nothing.
+    pub excluded_ids: Vec<String>,
+    /// `_retry_skipped_deployment_ids`: skipped unless that leaves nothing.
+    pub retry_skipped_ids: Vec<String>,
+    /// The request carries an image (`_request_carries_image`), so deployments that declare no
+    /// vision are skipped while any other remains.
+    pub has_image: bool,
+    /// An async pick: the host's pre-routing step runs before each pick resolves its name.
+    pub pre_routes: bool,
+    /// The requested name is no model group (a search tool name): nothing is selected, every
+    /// attempt is handed to the host as is, and no outcome is recorded for a deployment
+    /// (`_asearch_with_fallbacks`, whose helper picks among the router's search tools).
+    pub unrouted: bool,
+    /// Fallback lists the request passed (`Some(None)` when it passed `None`), which replace
+    /// the router's.
+    pub fallbacks: Option<Option<Fallbacks>>,
+    pub context_window_fallbacks: Option<Option<Fallbacks>>,
+    pub content_policy_fallbacks: Option<Option<Fallbacks>>,
+    /// A per-request `max_fallbacks`, which replaces the router's.
+    pub max_fallbacks: Option<u32>,
+    /// A per-request `cooldown_time`: the call's own params carry it, so it cools a failed
+    /// deployment down for that long unless its `model_info` sets one.
+    pub cooldown_time: Option<f64>,
+    /// A `mock_testing_*` flag the request set.
+    pub mock_failure: Option<MockFailure>,
+    /// A per-request `model_group_retry_policy`, which replaces the router's.
+    pub model_group_retry_policy: Option<BTreeMap<String, RetryPolicy>>,
+    /// The model groups whose `model_group_retry_policy` lookup raises (the policy is no mapping
+    /// of policies, or the group's entry is not one): the first failure of a hop serving one ends
+    /// the hop with the host's error instead ([`RouterEvent::RetryPolicyRaised`]), no retry.
+    pub retry_policy_raises: Vec<String>,
+    /// `health_check_probe`: a health check's own pick, which sees no deployment when every
+    /// one is marked unhealthy (the safety net keeps them all for any other request).
+    pub health_check_probe: bool,
+    /// What the host needs to run the call itself, such as the Python kwargs.
+    pub payload: Option<Payload>,
+}
+
+/// One attempt against one deployment.
+#[derive(Clone, Debug)]
+pub struct Attempt {
+    pub op: Op,
+    pub request: Arc<RouterRequest>,
+    pub deployment: Arc<Deployment>,
+    /// The name this attempt resolved: the requested name, or the fallback target.
+    pub name: String,
+    /// The model group being served, after fallbacks.
+    pub model_group: String,
+    /// The `order` level an order fallback targets in the same group.
+    pub target_order: Option<i64>,
+    /// Every deployment `name` resolved to for this caller, the picked one included.
+    pub candidate_ids: Vec<String>,
+    /// 0 for the first attempt in this group, then the retry number.
+    pub retry: u32,
+    pub fallback_depth: u32,
+    /// How many fallbacks deep the walk is: 0 for the requested name.
+    pub fallback_level: u32,
+    pub timeout: Option<Duration>,
+    /// Request params the fallback walk's dict targets merged in (`kwargs.update(target)`).
+    pub params: Option<Arc<Map<String, Value>>>,
+    /// The walk level whose request metadata this attempt carries (see [`RouterEvent::Fallback`]).
+    pub scope: u32,
+    /// The retry budget the level's loop stamps on the request before this attempt.
+    pub max_retries: u32,
+    /// How many deployments `get_model_list` lists for the level's group (`model_group_size`).
+    pub model_group_size: usize,
+}
+
+/// What a successful attempt returns.
+#[derive(Clone, Debug)]
+pub struct Output {
+    pub value: Payload,
+    /// Prompt plus completion tokens, counted against the deployment's per-minute usage.
+    pub total_tokens: u64,
+    /// A response the provider blocked, which the router may raise to fall back.
+    pub blocked: Option<Blocked>,
+}
+
+/// How a provider blocked a response the call otherwise served.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::IntoStaticStr, strum::EnumString)]
+#[strum(serialize_all = "snake_case")]
+pub enum Blocked {
+    /// A chat response whose first choice was content-filtered (or that has no choices).
+    ContentFilter,
+    /// An Anthropic /v1/messages safeguard refusal carrying `stop_details`.
+    Refusal,
+}
+
+/// What a plugin or strategy sees: never the router itself.
+pub struct RoutingContext<'a> {
+    pub registry: &'a Registry,
+    pub settings: &'a Settings,
+    pub health: &'a Health,
+    pub request: &'a RouterRequest,
+    pub op: &'a Op,
+    pub model_group: &'a str,
+    /// The name the request resolved from (Python's `model`), which strategies key state by.
+    pub name: &'a str,
+    /// Whose selector a strategy stage reads.
+    pub scope: StrategyScope,
+    /// The deployments the name resolved to for this caller, before health filters
+    /// (`request_deployments`), for the tag stage.
+    pub pool: &'a [Arc<Deployment>],
+}
+
+/// Which selection step the host is asked to run.
+#[derive(Clone, Debug, PartialEq)]
+pub enum SelectStage {
+    /// Pre-call checks and sub-routers registered as plugins: return the deployments that remain.
+    Plugins,
+    /// A strategy not yet ported: return the candidates with the pick first.
+    Strategy(RoutingStrategy),
+    /// Tag routing (`get_deployments_for_tag`), not yet ported: return the deployments that remain.
+    Tags,
+    /// `_deployment_declares_no_vision`: return the candidates that declare no vision support.
+    TextOnly,
+}
+
+/// Routing events, delivered through [`Host::observe`].
+#[derive(Clone, Debug, PartialEq)]
+pub enum RouterEvent {
+    Selected {
+        model_group: String,
+        deployment_id: String,
+    },
+    Succeeded {
+        deployment_id: String,
+    },
+    AttemptFailed {
+        deployment_id: String,
+        kind: FailureKind,
+        status: Option<u16>,
+    },
+    CooldownSet {
+        deployment_id: String,
+        cooldown: Duration,
+    },
+    Retry {
+        model_group: String,
+        retry: u32,
+        delay: Duration,
+    },
+    Fallback {
+        from: String,
+        to: String,
+        /// The order level of `to` a same-group hop walks to.
+        order: Option<i64>,
+        depth: u32,
+        /// The fallback level `to` is tried at: 1 for the request's own fallbacks.
+        level: u32,
+        /// The new level's request metadata scope, copied from `parent`'s at the hop.
+        scope: u32,
+        parent: u32,
+        /// The `max_fallbacks` the walk reads, stamped on the hop's request.
+        max_fallbacks: u32,
+    },
+    /// A hop of the fallback walk returned: `log_success_fallback_event` or
+    /// `log_failure_fallback_event` for the level at `parent`.
+    FallbackDone {
+        from: String,
+        to: String,
+        level: u32,
+        scope: u32,
+        parent: u32,
+        succeeded: bool,
+    },
+    /// The router raised a `ContentPolicyViolationError` for a blocked response, since a
+    /// fallback would serve it (`_should_raise_content_policy_error`,
+    /// `_should_raise_anthropic_refusal_error`).
+    Blocked {
+        level: u32,
+        model: String,
+        blocked: Blocked,
+    },
+    /// `log_retry`: the router recorded a failure in `scope`'s request metadata
+    /// (`previous_models`, `request_retry_count`) before retrying or falling back.
+    RetryLogged {
+        scope: u32,
+        level: u32,
+        /// The group the failed level served (`kwargs["model"]`).
+        model: String,
+        /// The retry number stamped before the failed try; `None` keeps the scope's own.
+        attempted_retries: Option<u32>,
+        cause: RetryCause,
+    },
+    /// A level of the fallback walk serving `model` ended on an error a pick raised (no
+    /// deployment for the caller, none available, a stale routing group, ...): Python raises
+    /// that error for the level, whatever its earlier attempts raised.
+    LevelFailed {
+        level: u32,
+        model: String,
+        error: PickError,
+        /// Raised by the level's fallback handling, while the level's own error was handled
+        /// (so it chains to that error), rather than by the level's retry loop.
+        handling: bool,
+    },
+    /// The router raised a `mock_testing_*` failure for `model` instead of calling it.
+    MockFailed {
+        mock: MockFailure,
+        level: u32,
+        model: String,
+        num_retries: Option<u32>,
+    },
+    /// The first failure of a hop serving `model` looked up a `model_group_retry_policy` that
+    /// raises for it: the hop ends with the error the lookup raised (the host raises it, from the
+    /// failure `cause` names), without retrying, and falls back as from any plain exception.
+    RetryPolicyRaised {
+        level: u32,
+        model: String,
+        cause: RetryCause,
+    },
+    /// A group's retry loop ran every retry and the last one failed: Python stamps
+    /// `num_retries` and `max_retries` on that error.
+    RetriesExhausted {
+        model_group: String,
+        attempted: u32,
+        max: u32,
+        /// The level the retry loop served.
+        level: u32,
+        /// The last failure was a pick's error, which the level then raises
+        /// ([`RouterEvent::LevelFailed`]), rather than an attempt's.
+        picked: bool,
+    },
+}
+
+/// A pick's error, as the events that name it carry it.
+#[derive(Clone, Debug)]
+pub struct PickError(pub Arc<Error>);
+
+impl PickError {
+    /// The copy of `error` events carry, when a pick raised it.
+    pub fn of(error: &Error) -> Option<Self> {
+        error.pick_copy().map(|copy| Self(Arc::new(copy)))
+    }
+}
+
+impl PartialEq for PickError {
+    fn eq(&self, other: &Self) -> bool {
+        format!("{:?}", self.0) == format!("{:?}", other.0)
+    }
+}
+
+/// Which error a [`RouterEvent::RetryLogged`] records.
+#[derive(Clone, Debug, PartialEq)]
+pub enum RetryCause {
+    /// The last attempt's failure.
+    Attempt,
+    /// The error the level ended on (`run_async_fallback`'s `original_exception`).
+    Level,
+    /// A pick raised this error: the group was unknown or had none available, the stale
+    /// routing-group member map named a group no longer configured, ...
+    Pick(PickError),
+    /// Any other error no attempt raised.
+    Other,
+}
+
+impl RetryCause {
+    /// The cause that names `error`.
+    pub fn of(error: &Error) -> Self {
+        match error {
+            Error::Attempt(_) => Self::Attempt,
+            _ => PickError::of(error).map_or(Self::Other, Self::Pick),
+        }
+    }
+}
+
+#[async_trait]
+pub trait Host: Send + Sync {
+    /// Run one attempt against the picked deployment. The core records its outcome (cooldowns,
+    /// allowed-fails counters, usage) itself; the host only runs it.
+    async fn invoke(&self, attempt: &Attempt) -> Result<Output, AttemptError>;
+
+    /// Run a selection stage the core does not implement.
+    async fn select(
+        &self,
+        stage: &SelectStage,
+        ctx: &RoutingContext<'_>,
+        candidates: Vec<Arc<Deployment>>,
+    ) -> Result<Vec<Arc<Deployment>>, Error>;
+
+    /// Pre-routing before a pick resolves `ask.name`: the model group to resolve instead, a
+    /// custom strategy's deployment to serve, or `None` to resolve the name itself. Asked only
+    /// for picks that pre-route (`pre_routes`).
+    async fn pre_route(&self, ask: &PreRoute<'_>) -> Result<Option<PreRouted>, Error> {
+        let _ = ask;
+        Ok(None)
+    }
+
+    /// Access and budget checks before a cross-group fallback target.
+    async fn allow_fallback(&self, request: &RouterRequest, target: &str) -> bool;
+
+    async fn sleep(&self, duration: Duration);
+
+    fn observe(&self, event: &RouterEvent);
+
+    /// The shared store routing state lives in.
+    fn cache(&self) -> Arc<dyn RouterCache>;
+
+    /// Whether [`Self::invoke`] can run `op` on `provider`. Checked for every served op and
+    /// deployment when the router is built or changed, so an unsupported pair fails the config,
+    /// never a request. `PythonHost` runs every op; `NativeHost` only those with a Rust route.
+    fn supports(&self, _op: &Op, _provider: &str) -> bool {
+        true
+    }
+
+    /// Whether [`Self::select`] can run strategies and plugins.
+    fn can_select(&self) -> bool {
+        true
+    }
+
+    /// The facts the `enable_pre_call_checks` filter decides by (token count, context windows,
+    /// param support), for `candidates` and the deployments in `ctx.pool`. A host without them
+    /// reports no input, so nothing is checked.
+    /// Run the routing plugins for `ctx.name` when `run` (the async pick's pre-routing hook), and
+    /// report the candidate models they left on the request, if any.
+    async fn routing_plugin_models(
+        &self,
+        _ctx: &RoutingContext<'_>,
+        _run: bool,
+    ) -> Result<Option<Vec<String>>, Error> {
+        Ok(None)
+    }
+
+    async fn pre_call_facts(
+        &self,
+        _ctx: &RoutingContext<'_>,
+        _candidates: &[Arc<Deployment>],
+    ) -> Result<PreCallFacts, Error> {
+        Ok(PreCallFacts::default())
+    }
+
+    /// `async_callback_filter_deployments`: the host's routing callbacks narrow `candidates` in
+    /// turn. A filter that raises fails the try with [`Error::Attempt`], which the call retries and
+    /// falls back from like any attempt failure. A host without filters keeps every candidate.
+    async fn callback_filters(
+        &self,
+        _ctx: &RoutingContext<'_>,
+        candidates: Vec<Arc<Deployment>>,
+    ) -> Result<Vec<Arc<Deployment>>, Error> {
+        Ok(candidates)
+    }
+}

litellm-rust/crates/router-core/src/lib.rs

@@ -43,21 +43,460 @@
 //! - `call(op, request)`: select, run the attempt, and retry or fall back until it succeeds or the
 //!   policy is exhausted.
 //!
-//! # Execution
-//! Each attempt resolves an `Executor` after a deployment is picked. A Rust-native operation runs
-//! its `inference-*` route machine in-process; anything else goes through `Host::invoke`. A
-//! started attempt is never replayed on a different executor.
+//! # Hosts: the only boundary
+//! The core reaches outside itself only through [`Host`], and every attempt runs through
+//! `Host::invoke`: the core picks the deployment, the host runs the call, the core records the
+//! outcome. There is no second, "native" attempt path in the core. Which code runs an attempt is
+//! the host's business:
+//! - `PythonHost` (inside a Python process) builds the request for the picked deployment and calls
+//!   `litellm.<op>`. It never calls back into a Python router.
+//! - `NativeHost` (the Rust server) calls the `inference-*` crates. An op or provider it cannot run
+//!   fails `Host::supports` when the router is built or changed, never partway through a request.
 //!
-//! # Hosts
-//! `Host` is what the embedding process provides: `invoke` for non-native operations, `select`
-//! for strategies and sub-routers not yet ported, `allow_fallback` for access and budget checks,
-//! `sleep`, `observe` for routing events, and `cache`, a DualCache-style store that holds all
-//! routing state under the same keys, encodings and TTLs the Python router uses. `PythonHost`
-//! serves them from the existing Python code when the router runs inside Python; `NativeHost`
-//! serves them in the Rust server, where an operation without a Rust implementation makes the
-//! config fail at build time, never partway through a request.
+//! Besides `invoke` and `supports`, a host provides `select` for strategies and plugins not yet
+//! ported, `allow_fallback` for access and budget checks, `sleep`, `observe` for routing events,
+//! and `cache`, a DualCache-style store that holds all routing state under the same keys,
+//! encodings and TTLs the Python router uses. The core writes cooldowns, allowed-fails counters
+//! and usage itself, through that cache.
 //!
 //! # Plugins
 //! Routing strategies, pre-call checks and sub-routers (auto, complexity, quality, adaptive)
 //! register through `register_plugin` and see a `RoutingContext` (registry, settings, health and
 //! the current request), never the router itself.
+//!
+//! # Modules
+//! - [`registry`]: deployments and their indices, `deployment_id`, `ChangeSet`.
+//! - [`config`] / [`settings`]: static and dynamic config, and the frozen `Settings` view.
+//! - [`cache`] / [`health`]: the shared routing cache, cooldowns and usage.
+//! - [`select`]: resolve, filter chain, simple-shuffle.
+//! - [`call`]: retries and fallbacks.
+//! - [`precall`]: the `enable_pre_call_checks` filter.
+//! - [`host`]: the `Host` boundary.
+
+pub mod access;
+pub mod cache;
+pub mod call;
+pub mod config;
+pub mod error;
+pub mod fallback;
+pub mod health;
+pub mod health_routing;
+pub mod host;
+pub mod mock;
+pub mod precall;
+pub mod prerouting;
+pub mod ptu;
+pub mod registry;
+pub mod resolve;
+pub mod retry;
+pub mod routing_groups;
+pub mod routing_plugins;
+pub mod select;
+pub mod settings;
+pub mod strategy;
+pub mod weights;
+
+use std::sync::{
+    Arc,
+    atomic::{AtomicBool, Ordering},
+};
+
+use arc_swap::ArcSwap;
+use parking_lot::Mutex;
+
+pub use crate::{
+    cache::{InMemoryRouterCache, RouterCache},
+    call::RouterResponse,
+    config::{Op, RouterConfig, RouterDynamicConfig, RoutingStrategy},
+    error::{AttemptError, ConfigError, Error, FailureKind, Payload, Unsupported},
+    health::{Clock, Health, Outcome, SystemClock},
+    host::{
+        Attempt, Blocked, Host, Output, RetryCause, RouterEvent, RouterRequest, RoutingContext,
+        SelectStage,
+    },
+    prerouting::{PreRoute, PreRouted},
+    registry::{ChangeSet, Deployment, DeploymentSpec, Registry, RegistryVersion},
+    select::{Randomness, SeededRandomness, ThreadRandomness},
+    settings::Settings,
+};
+use crate::{health::GroupShape, registry::DeploymentFilter, select::SelectInputs};
+
+/// The registry and settings one call reads, swapped atomically by `apply`.
+pub(crate) struct Snapshot {
+    pub registry: Arc<Registry>,
+    pub settings: Arc<Settings>,
+}
+
+impl Snapshot {
+    /// `_get_routing_context` for a pick of `name`: unless the request overrides the strategy,
+    /// a stale member map naming a group no longer configured raises `KeyError(group)`.
+    pub(crate) fn missing_routing_group(
+        &self,
+        request: &RouterRequest,
+        name: &str,
+    ) -> Result<(), Error> {
+        if strategy::request_override(request.routing_strategy.as_deref()).is_some() {
+            return Ok(());
+        }
+        match self.registry.missing_member_group(name) {
+            Some(group) => Err(Error::MissingRoutingGroup {
+                group: group.to_owned(),
+            }),
+            None => Ok(()),
+        }
+    }
+}
+
+pub(crate) struct Core {
+    host: Arc<dyn Host>,
+    rng: Arc<dyn Randomness>,
+    health: Health,
+    config: RouterConfig,
+    snapshot: ArcSwap<Snapshot>,
+    writer: Mutex<()>,
+    retired: AtomicBool,
+}
+
+impl Core {
+    /// Reject what this host cannot serve, before any request.
+    fn validate(&self, registry: &Registry, settings: &Settings) -> Result<(), ConfigError> {
+        let can_select = self.host.can_select();
+        let strategies = std::iter::once(&settings.routing_strategy).chain(
+            registry
+                .routing_groups()
+                .iter()
+                .map(|group| &group.routing_strategy),
+        );
+        for strategy in strategies {
+            if !strategy.is_native() && !can_select {
+                return Err(Unsupported::Strategy(strategy.clone()).into());
+            }
+        }
+        if let Some(plugin) = settings.plugins.first()
+            && !can_select
+        {
+            return Err(Unsupported::Plugin(plugin.name.clone()).into());
+        }
+        for deployment in registry.deployments(&DeploymentFilter::default()) {
+            for op in &self.config.served_ops {
+                if !self.host.supports(op, deployment.provider()) {
+                    return Err(Unsupported::Op {
+                        op: op.clone(),
+                        provider: deployment.provider().to_owned(),
+                    }
+                    .into());
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+/// How a reported failure is recorded.
+#[derive(Clone, Debug, Default)]
+pub struct FailureReport {
+    /// Cool down for the deployment's own `cooldown_time` (then the provider's retry-after),
+    /// as the failure callback does, rather than the router default.
+    pub deployment_cooldown: bool,
+    /// The `cooldown_time` the failed call's own params carried (`model_info`, then
+    /// `litellm_params`), which wins over the provider's retry-after for a callback report.
+    pub cooldown_time: Option<f64>,
+    /// Count the failure only: its status is one no cooldown rule reads (an empty status).
+    pub count_only: bool,
+    /// The model group the request was for, whose fallbacks keep a lone deployment cooling.
+    pub requested_group: Option<String>,
+    /// Cool down without counting the failure (`set_cooldown_deployments` alone).
+    pub uncounted: bool,
+}
+
+/// Builds a [`Router`], with test seams.
+pub struct RouterBuilder {
+    config: RouterConfig,
+    dynamic: RouterDynamicConfig,
+    host: Arc<dyn Host>,
+    rng: Arc<dyn Randomness>,
+    clock: Arc<dyn Clock>,
+}
+
+impl RouterBuilder {
+    #[must_use]
+    pub fn randomness(self, rng: Arc<dyn Randomness>) -> Self {
+        Self { rng, ..self }
+    }
+
+    #[must_use]
+    pub fn clock(self, clock: Arc<dyn Clock>) -> Self {
+        Self { clock, ..self }
+    }
+
+    pub fn build(self) -> Result<Router, ConfigError> {
+        let registry = Registry::build(
+            self.config.model_list.clone(),
+            self.dynamic.model_alias_map.clone(),
+            self.config.routing_groups.clone(),
+        )?
+        .with_member_groups(self.config.routing_group_members.clone());
+        let settings = Settings::new(&self.config, Arc::new(self.dynamic));
+        let core = Core {
+            health: Health::new(self.host.cache(), self.clock),
+            host: self.host,
+            rng: self.rng,
+            config: self.config,
+            snapshot: ArcSwap::from_pointee(Snapshot {
+                registry: Arc::new(Registry::default()),
+                settings: Arc::new(settings.clone()),
+            }),
+            writer: Mutex::new(()),
+            retired: AtomicBool::new(false),
+        };
+        core.validate(&registry, &settings)?;
+        core.snapshot.store(Arc::new(Snapshot {
+            registry: Arc::new(registry),
+            settings: Arc::new(settings),
+        }));
+        Ok(Router {
+            core: Arc::new(core),
+        })
+    }
+}
+
+/// The router: the entry points `litellm.Router` delegates to.
+#[derive(Clone)]
+pub struct Router {
+    core: Arc<Core>,
+}
+
+impl Router {
+    pub fn new(
+        config: RouterConfig,
+        dynamic: RouterDynamicConfig,
+        host: Arc<dyn Host>,
+    ) -> Result<Self, ConfigError> {
+        Self::builder(config, dynamic, host).build()
+    }
+
+    pub fn builder(
+        config: RouterConfig,
+        dynamic: RouterDynamicConfig,
+        host: Arc<dyn Host>,
+    ) -> RouterBuilder {
+        RouterBuilder {
+            config,
+            dynamic,
+            host,
+            rng: Arc::new(ThreadRandomness),
+            clock: Arc::new(SystemClock),
+        }
+    }
+
+    fn live(&self) -> Result<(), Error> {
+        if self.core.retired.load(Ordering::Acquire) {
+            return Err(Error::Retired);
+        }
+        Ok(())
+    }
+
+    /// Select, run the attempt, and retry or fall back until it succeeds or the policy is
+    /// exhausted.
+    pub async fn call(&self, op: Op, request: RouterRequest) -> Result<RouterResponse, Error> {
+        self.live()?;
+        self.core.call(op, Arc::new(request)).await
+    }
+
+    /// Resolve, filter and pick a deployment without calling it.
+    pub async fn select(&self, op: &Op, request: &RouterRequest) -> Result<Arc<Deployment>, Error> {
+        self.live()?;
+        let snapshot = self.core.snapshot.load_full();
+        routing_plugins::refuse_sync_pick(
+            snapshot.settings.routing_plugins,
+            &snapshot.settings.routing_strategy,
+            request,
+        )?;
+        let routed = self
+            .core
+            .pre_routed(PreRoute {
+                op,
+                request,
+                name: &request.model,
+                fallback_depth: 0,
+                retry: 0,
+            })
+            .await?;
+        let name = match routed {
+            PreRouted::Model(name) => name,
+            PreRouted::Deployment(deployment) => return Ok(deployment),
+        };
+        snapshot.missing_routing_group(request, &name)?;
+        let candidates = self
+            .core
+            .candidates(&snapshot, op, request, &name, &SelectInputs::default())
+            .await?;
+        self.core.pick(&snapshot, op, request, &candidates).await
+    }
+
+    /// The deployments a pick for `request` would choose among: resolved, team- and
+    /// cooldown-filtered, through the plugins and the lowest `order`.
+    pub async fn healthy(
+        &self,
+        op: &Op,
+        request: &RouterRequest,
+    ) -> Result<select::Candidates, Error> {
+        self.live()?;
+        let snapshot = self.core.snapshot.load_full();
+        let candidates = self
+            .core
+            .candidates(
+                &snapshot,
+                op,
+                request,
+                &request.model,
+                &SelectInputs::default(),
+            )
+            .await?;
+        // `_core_async_get_healthy_deployments` raises when nothing is left.
+        if candidates.healthy.is_empty() {
+            return Err(Error::NoHealthyDeployments {
+                model: candidates.model_group,
+            });
+        }
+        Ok(candidates)
+    }
+
+    pub fn registry(&self) -> Arc<Registry> {
+        self.core.snapshot.load().registry.clone()
+    }
+
+    pub fn settings(&self) -> Arc<Settings> {
+        self.core.snapshot.load().settings.clone()
+    }
+
+    pub fn health(&self) -> &Health {
+        &self.core.health
+    }
+
+    /// Record an outcome the router did not see itself, such as a health check. Returns whether
+    /// the deployment was cooled down.
+    pub async fn report(&self, id: &str, outcome: Outcome) -> Result<bool, Error> {
+        match outcome {
+            Outcome::Success { total_tokens } => self
+                .report_success(id, total_tokens as f64, 1.0)
+                .await
+                .map(|_| false),
+            Outcome::Failure(failure) => {
+                self.report_failure(id, &failure, FailureReport::default())
+                    .await
+            }
+        }
+    }
+
+    /// A success a logging callback saw (`deployment_callback_on_success`): counted for a
+    /// configured deployment, with its tokens and requests when it tracks usage. Returns the
+    /// success counter's key, or `None` for an unknown deployment.
+    pub async fn report_success(
+        &self,
+        id: &str,
+        total_tokens: f64,
+        requests: f64,
+    ) -> Result<Option<String>, Error> {
+        let snapshot = self.core.snapshot.load_full();
+        let Some(deployment) = snapshot.registry.deployment(id) else {
+            return Ok(None);
+        };
+        let key = self.core.health.count_success(id).await?;
+        self.core
+            .health
+            .add_usage(&deployment, total_tokens, requests)
+            .await?;
+        Ok(Some(key))
+    }
+
+    /// `sync_deployment_callback_on_success`: count a success against `id`, configured or not,
+    /// and return the counter's key.
+    pub async fn count_success(&self, id: &str) -> Result<String, Error> {
+        self.core.health.count_success(id).await
+    }
+
+    /// A failure the router did not see itself: counted against `id` (known or not), then
+    /// cooled down by the rules (`record_failure`, `deployment_callback_on_failure`).
+    pub async fn report_failure(
+        &self,
+        id: &str,
+        failure: &AttemptError,
+        report: FailureReport,
+    ) -> Result<bool, Error> {
+        if !report.uncounted {
+            self.core.health.count_failure(id).await?;
+        }
+        let snapshot = self.core.snapshot.load_full();
+        let Some(deployment) = snapshot.registry.deployment(id) else {
+            return Ok(false);
+        };
+        // An empty status is no cooldown (`_is_cooldown_required`), unless the deployment's
+        // own policy names the error.
+        if report.count_only
+            && crate::health::deployment_allowed_fails(
+                deployment.model_info.allowed_fails_policy.as_ref(),
+                failure,
+            )
+            .is_none()
+        {
+            return Ok(false);
+        }
+        let cooldown = if report.deployment_cooldown {
+            report
+                .cooldown_time
+                .filter(|seconds| *seconds >= 0.0)
+                .and_then(|seconds| std::time::Duration::try_from_secs_f64(seconds).ok())
+                .or(failure.retry_after)
+                .unwrap_or(snapshot.settings.cooldown_time)
+        } else {
+            snapshot.settings.cooldown_time
+        };
+        let shape = GroupShape {
+            single_deployment: snapshot
+                .registry
+                .group(&deployment.model_name)
+                .is_some_and(|group| group.len() == 1),
+            has_fallback_group: snapshot
+                .settings
+                .dynamic
+                .fallback_targets(&[report
+                    .requested_group
+                    .as_deref()
+                    .unwrap_or(&deployment.model_name)])
+                .is_some(),
+        };
+        self.core
+            .health
+            .cool_down_after(&snapshot.settings, &deployment, shape, failure, cooldown)
+            .await
+            .map(|cooldown| cooldown.is_some())
+    }
+
+    /// Apply one change set atomically and return the new registry version.
+    pub fn apply(&self, changes: ChangeSet) -> Result<RegistryVersion, ConfigError> {
+        let _writer = self.core.writer.lock();
+        let current = self.core.snapshot.load_full();
+        let aliases = changes
+            .settings
+            .as_ref()
+            .map(|dynamic| dynamic.model_alias_map.clone());
+        let registry = current.registry.apply(&changes, aliases)?;
+        let settings = match changes.settings {
+            Some(dynamic) => Settings::new(&self.core.config, Arc::new(dynamic)),
+            None => Settings::clone(&current.settings),
+        };
+        self.core.validate(&registry, &settings)?;
+        let version = registry.version();
+        self.core.snapshot.store(Arc::new(Snapshot {
+            registry: Arc::new(registry),
+            settings: Arc::new(settings),
+        }));
+        Ok(version)
+    }
+
+    /// Stop serving: every later call and selection fails with [`Error::Retired`].
+    pub fn retire(&self) {
+        self.core.retired.store(true, Ordering::Release);
+    }
+}

litellm-rust/crates/router-core/src/mock.rs

@@ -0,0 +1,44 @@
+//! The request's `mock_testing_*` flags: a failure the router raises itself, without calling a
+//! deployment (`_handle_mock_testing_fallbacks` and `_handle_mock_testing_rate_limit_error`).
+
+use serde::Deserialize;
+
+use crate::error::{AttemptError, FailureKind};
+
+/// Which mock failure a request asks for.
+#[derive(
+    Clone, Copy, Debug, PartialEq, Eq, Deserialize, strum::IntoStaticStr, strum::EnumString,
+)]
+#[serde(rename_all = "snake_case")]
+#[strum(serialize_all = "snake_case")]
+pub enum MockFailure {
+    /// `mock_testing_fallbacks`: an `InternalServerError` before the first level's retries.
+    Fallbacks,
+    /// `mock_testing_context_fallbacks`: a `ContextWindowExceededError`, likewise.
+    ContextWindow,
+    /// `mock_testing_content_policy_fallbacks`: a `ContentPolicyViolationError`, likewise.
+    ContentPolicy,
+    /// `mock_testing_rate_limit_error`: a `RateLimitError` in place of every level's first
+    /// attempt, which the retry loop handles like a failed attempt.
+    RateLimit,
+}
+
+impl MockFailure {
+    /// Whether the failure replaces a level's walk (raised before its retries).
+    pub fn before_retries(self) -> bool {
+        !matches!(self, Self::RateLimit)
+    }
+
+    /// The failure as the rules see it.
+    pub fn failure(self, num_retries: Option<u32>) -> AttemptError {
+        let (kind, status) = match self {
+            Self::Fallbacks => (FailureKind::InternalServer, 500),
+            Self::ContextWindow => (FailureKind::ContextWindowExceeded, 400),
+            Self::ContentPolicy => (FailureKind::ContentPolicyViolation, 400),
+            Self::RateLimit => (FailureKind::RateLimit, 429),
+        };
+        let mut failure = AttemptError::new(kind, Some(status));
+        failure.num_retries = num_retries;
+        failure
+    }
+}

litellm-rust/crates/router-core/src/precall.rs

@@ -0,0 +1,300 @@
+//! `enable_pre_call_checks` (`_pre_call_checks`): before a pick, drop the deployments whose
+//! context window the prompt exceeds, those outside the request's `allowed_model_region`, and
+//! those that cannot take a requested param, then decide which error a group with nothing left
+//! raises. The facts that need live model data (the prompt's token count, each deployment's
+//! context window, which params it takes) come from [`Host::pre_call_facts`]; every decision is
+//! made here.
+//!
+//! [`Host::pre_call_facts`]: crate::host::Host::pre_call_facts
+
+use std::{collections::BTreeMap, sync::Arc};
+
+use serde::Deserialize;
+use serde_json::Value;
+
+use crate::{error::Error, registry::Deployment};
+
+/// What the host knows about one deployment for the pre-call checks.
+#[derive(Clone, Debug, Default, Deserialize, PartialEq)]
+#[serde(default)]
+pub struct DeploymentFacts {
+    /// The deployment's context window (`get_router_model_info`'s `max_input_tokens`), when it
+    /// declares one and resolves.
+    pub max_input_tokens: Option<i64>,
+    /// `_deployment_lacks_params`: it does not support one of the requested params.
+    pub lacks_params: bool,
+    /// `_deployment_rejects_params`: it lacks one the request and deployment neither drop nor allow.
+    pub rejects_params: bool,
+}
+
+/// What the host knows about a request for the pre-call checks.
+#[derive(Clone, Debug, Default, Deserialize, PartialEq)]
+#[serde(default)]
+pub struct PreCallFacts {
+    /// The request carries `messages` or `input`: without either no check runs.
+    pub has_input: bool,
+    /// The prompt may be counted against a context window (no compaction is pending).
+    pub countable: bool,
+    /// The prompt's token count, when it was counted.
+    pub input_tokens: Option<i64>,
+    /// The request's `allowed_model_region`.
+    pub allowed_model_region: Option<Value>,
+    /// The requested params the check filters on, sorted (`_pre_call_check_requested_chat_params`).
+    pub requested_params: Vec<String>,
+    /// Facts by deployment id, for the candidates and the group's deployments.
+    pub deployments: BTreeMap<String, DeploymentFacts>,
+}
+
+impl PreCallFacts {
+    fn of(&self, deployment: &Deployment) -> DeploymentFacts {
+        self.deployments
+            .get(&deployment.id)
+            .cloned()
+            .unwrap_or_default()
+    }
+}
+
+/// The name a context-window error gives a deployment: its base model, else its model.
+fn deployment_model(deployment: &Deployment) -> String {
+    let truthy = |value: Option<&Value>| match value {
+        Some(Value::String(text)) if !text.is_empty() => Some(text.clone()),
+        Some(Value::Null | Value::Bool(false)) | None => None,
+        Some(Value::String(_)) => None,
+        Some(other) => Some(python_str(other)),
+    };
+    truthy(deployment.model_info.raw.get("base_model"))
+        .or_else(|| truthy(deployment.raw_params.get("base_model")))
+        .unwrap_or_else(|| deployment.params.model.clone())
+}
+
+/// `str(value)` for the scalar values a base model may hold.
+fn python_str(value: &Value) -> String {
+    match value {
+        Value::Bool(true) => "True".to_owned(),
+        Value::Number(number) => number.to_string(),
+        other => other.to_string(),
+    }
+}
+
+/// `is_region_allowed`: the deployment's `region_name` equals the allowed region.
+fn in_region(deployment: &Deployment, allowed: &Value) -> bool {
+    deployment.raw_params.get("region_name") == Some(allowed)
+}
+
+/// `_pre_call_checks` over `candidates`; `pool` is the group's deployments before health filters
+/// (`request_deployments`), which decide whether nothing left is an unsupported-params error.
+pub fn filter(
+    model: &str,
+    candidates: Vec<Arc<Deployment>>,
+    pool: &[Arc<Deployment>],
+    facts: &PreCallFacts,
+) -> Result<Vec<Arc<Deployment>>, Error> {
+    if !facts.has_input {
+        return Ok(candidates);
+    }
+    let mut invalid = vec![false; candidates.len()];
+    let mut context_window_error = false;
+    let mut detail = String::new();
+    for (index, deployment) in candidates.iter().enumerate() {
+        let found = facts.of(deployment);
+        if let (Some(limit), true, Some(tokens)) =
+            (found.max_input_tokens, facts.countable, facts.input_tokens)
+            && tokens > limit
+        {
+            invalid[index] = true;
+            context_window_error = true;
+            detail.push_str(&format!(
+                "Model={}, Max Input Tokens={limit}, Got={tokens}",
+                deployment_model(deployment)
+            ));
+            continue;
+        }
+        if let Some(allowed) = &facts.allowed_model_region
+            && !allowed.is_null()
+            && !in_region(deployment, allowed)
+        {
+            invalid[index] = true;
+        }
+    }
+    let params_checked = !facts.requested_params.is_empty();
+    let param_candidates: Vec<usize> = (0..candidates.len()).filter(|i| !invalid[*i]).collect();
+    let lacking: Vec<bool> = param_candidates
+        .iter()
+        .map(|i| params_checked && facts.of(&candidates[*i]).lacks_params)
+        .collect();
+    let none_takes = !lacking.is_empty() && lacking.iter().all(|lacks| *lacks);
+    let failing: Vec<bool> = if none_takes {
+        param_candidates
+            .iter()
+            .map(|i| params_checked && facts.of(&candidates[*i]).rejects_params)
+            .collect()
+    } else {
+        lacking
+    };
+    for (index, fails) in param_candidates.iter().zip(failing) {
+        if fails {
+            invalid[*index] = true;
+        }
+    }
+    let excluded = invalid.iter().filter(|x| **x).count();
+    if excluded == candidates.len() {
+        if !context_window_error
+            && !pool.is_empty()
+            && pool
+                .iter()
+                .all(|d| params_checked && facts.of(d).rejects_params)
+        {
+            return Err(Error::UnsupportedParams {
+                model: model.to_owned(),
+                params: facts.requested_params.clone(),
+            });
+        }
+        if context_window_error {
+            return Err(Error::ContextWindowExceeded {
+                model: model.to_owned(),
+                detail,
+            });
+        }
+    }
+    Ok(candidates
+        .into_iter()
+        .zip(invalid)
+        .filter_map(|(deployment, invalid)| (!invalid).then_some(deployment))
+        .collect())
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+    use crate::registry::DeploymentSpec;
+
+    fn deployment(id: &str, params: Value, model_info: Value) -> Arc<Deployment> {
+        let mut info = model_info;
+        info["id"] = json!(id);
+        let spec: DeploymentSpec = serde_json::from_value(json!({
+            "model_name": "g", "litellm_params": params, "model_info": info,
+        }))
+        .unwrap();
+        Arc::new(Deployment::from_spec(spec).unwrap())
+    }
+
+    fn facts(entries: &[(&str, DeploymentFacts)]) -> PreCallFacts {
+        PreCallFacts {
+            has_input: true,
+            countable: true,
+            deployments: entries
+                .iter()
+                .map(|(id, f)| ((*id).to_owned(), f.clone()))
+                .collect(),
+            ..PreCallFacts::default()
+        }
+    }
+
+    fn window(limit: i64) -> DeploymentFacts {
+        DeploymentFacts {
+            max_input_tokens: Some(limit),
+            ..DeploymentFacts::default()
+        }
+    }
+
+    #[rstest]
+    fn a_prompt_over_a_context_window_drops_that_deployment() {
+        let small = deployment("small", json!({"model": "openai/a"}), json!({}));
+        let big = deployment("big", json!({"model": "openai/b"}), json!({}));
+        let mut found = facts(&[("small", window(5)), ("big", window(10_000))]);
+        found.input_tokens = Some(100);
+        let kept = filter("g", vec![small, big], &[], &found).unwrap();
+        assert_eq!(
+            kept.iter().map(|d| d.id.as_str()).collect::<Vec<_>>(),
+            ["big"]
+        );
+    }
+
+    #[rstest]
+    fn no_window_left_names_each_deployment_back_to_back() {
+        let one = deployment("one", json!({"model": "openai/one"}), json!({}));
+        let two = deployment(
+            "two",
+            json!({"model": "openai/two"}),
+            json!({"base_model": "azure/base"}),
+        );
+        let mut found = facts(&[("one", window(5)), ("two", window(6))]);
+        found.input_tokens = Some(9);
+        let error = filter("g", vec![one, two], &[], &found).unwrap_err();
+        let Error::ContextWindowExceeded { model, detail } = error else {
+            panic!("expected a context window error, got {error:?}");
+        };
+        assert_eq!(model, "g");
+        assert_eq!(
+            detail,
+            "Model=openai/one, Max Input Tokens=5, Got=9Model=azure/base, Max Input Tokens=6, Got=9"
+        );
+    }
+
+    #[rstest]
+    fn without_input_nothing_is_checked() {
+        let small = deployment("small", json!({"model": "openai/a"}), json!({}));
+        let mut found = facts(&[("small", window(5))]);
+        found.input_tokens = Some(100);
+        found.has_input = false;
+        assert_eq!(filter("g", vec![small], &[], &found).unwrap().len(), 1);
+    }
+
+    #[rstest]
+    #[case::in_region("eu", 1)]
+    #[case::out_of_region("ap", 0)]
+    fn the_allowed_region_keeps_only_its_deployments(#[case] region: &str, #[case] kept: usize) {
+        let eu = deployment(
+            "eu",
+            json!({"model": "openai/a", "region_name": "eu"}),
+            json!({}),
+        );
+        let none = deployment("none", json!({"model": "openai/a"}), json!({}));
+        let mut found = facts(&[]);
+        found.allowed_model_region = Some(json!(region));
+        assert_eq!(
+            filter("g", vec![eu, none], &[], &found).unwrap().len(),
+            kept
+        );
+    }
+
+    #[rstest]
+    fn a_group_where_every_deployment_rejects_the_param_is_unsupported() {
+        let old = deployment("old", json!({"model": "openai/old"}), json!({}));
+        let mut found = facts(&[(
+            "old",
+            DeploymentFacts {
+                lacks_params: true,
+                rejects_params: true,
+                ..DeploymentFacts::default()
+            },
+        )]);
+        found.requested_params = vec!["response_format".to_owned()];
+        let error = filter("g", vec![old.clone()], &[old], &found).unwrap_err();
+        assert!(
+            matches!(error, Error::UnsupportedParams { params, .. } if params == ["response_format"])
+        );
+    }
+
+    #[rstest]
+    fn a_deployment_that_takes_the_param_wins_over_one_that_would_drop_it() {
+        let old = deployment("old", json!({"model": "openai/old"}), json!({}));
+        let new = deployment("new", json!({"model": "openai/new"}), json!({}));
+        let mut found = facts(&[(
+            "old",
+            DeploymentFacts {
+                lacks_params: true,
+                ..DeploymentFacts::default()
+            },
+        )]);
+        found.requested_params = vec!["response_format".to_owned()];
+        let kept = filter("g", vec![old, new], &[], &found).unwrap();
+        assert_eq!(
+            kept.iter().map(|d| d.id.as_str()).collect::<Vec<_>>(),
+            ["new"]
+        );
+    }
+}

litellm-rust/crates/router-core/src/prerouting.rs

@@ -0,0 +1,53 @@
+//! Pre-routing: before an async pick resolves its name, the host may route the request to another
+//! model group (Python's `_core_async_pre_routing_hook`). The complexity, auto, adaptive and
+//! quality routers registered from `auto_router/...` marker deployments, and the Claude Code
+//! session binding, live host-side; the core asks once per pick, before resolution, and resolves
+//! the name the host answers. The attempt still serves the requested name (its `model_group`):
+//! only the deployments come from the routed group.
+//!
+//! A custom routing strategy (`register_plugin`, `set_custom_routing_strategy`, lar1) replaces the
+//! whole pick (`_core_async_get_available_deployment` and `get_available_deployment`): the host
+//! answers the same step with the deployment document the strategy returned, which may name a
+//! deployment the registry does not hold, and the core serves that deployment as is.
+
+use std::sync::Arc;
+
+use crate::{Core, config::Op, error::Error, host::RouterRequest, registry::Deployment};
+
+/// What the host's pre-routing step sees.
+#[derive(Clone, Copy, Debug)]
+pub struct PreRoute<'a> {
+    pub op: &'a Op,
+    pub request: &'a RouterRequest,
+    /// The name this pick resolves: the requested name or the fallback target.
+    pub name: &'a str,
+    /// How many fallbacks deep the walk is (Python's `fallback_depth`): only a first attempt
+    /// binds a Claude Code session.
+    pub fallback_depth: u32,
+    /// 0 for the first attempt in this group, then the retry number.
+    pub retry: u32,
+}
+
+/// The host's answer to a pre-routing step.
+#[derive(Clone, Debug)]
+pub enum PreRouted {
+    /// Resolve this model group instead of the requested name.
+    Model(String),
+    /// A custom strategy's pick: serve this deployment, without resolving or filtering.
+    Deployment(Arc<Deployment>),
+}
+
+impl Core {
+    /// What a pick of `ask.name` serves: the host's pre-routed group or custom pick, else the
+    /// name itself. Only picks that ask for it pre-route (`request.pre_routes`).
+    pub(crate) async fn pre_routed(&self, ask: PreRoute<'_>) -> Result<PreRouted, Error> {
+        if !ask.request.pre_routes {
+            return Ok(PreRouted::Model(ask.name.to_owned()));
+        }
+        Ok(self
+            .host
+            .pre_route(&ask)
+            .await?
+            .unwrap_or_else(|| PreRouted::Model(ask.name.to_owned())))
+    }
+}

litellm-rust/crates/router-core/src/ptu.rs

@@ -0,0 +1,96 @@
+//! PTU shares (`litellm_core_utils/ptu_shares.py`): a deployment whose `model_info.ptu_shares`
+//! splits it across teams serves only those teams, and the teams holding a share are kept to
+//! their share.
+
+use std::{collections::BTreeSet, sync::Arc};
+
+use serde_json::{Map, Value};
+
+use crate::registry::Deployment;
+
+/// `parsed_ptu_shares`: the teams of a non-empty `ptu_shares` map whose every entry is a team id
+/// and a positive whole number of PTUs (`2.0` reads as 2, a bool is no number), else None.
+pub fn share_teams(model_info: &Map<String, Value>) -> Option<BTreeSet<String>> {
+    let shares = model_info.get("ptu_shares")?.as_object()?;
+    if shares.is_empty() {
+        return None;
+    }
+    shares
+        .iter()
+        .map(|(team, share)| (!team.is_empty() && whole_ptus(share)).then(|| team.clone()))
+        .collect()
+}
+
+/// `_whole_ptus`: a positive whole number.
+fn whole_ptus(share: &Value) -> bool {
+    match share {
+        Value::Number(number) => number.as_i64().map_or_else(
+            || {
+                number
+                    .as_u64()
+                    .map(|whole| whole > 0)
+                    .or_else(|| number.as_f64().map(|f| f.fract() == 0.0 && f > 0.0))
+                    .unwrap_or(false)
+            },
+            |whole| whole > 0,
+        ),
+        _ => false,
+    }
+}
+
+/// `filter_ptu_shared_deployments`: the deployments `team_id` holds a share on when it holds one,
+/// else the ones split into no shares; a caller without a team holds none. Also whether any
+/// deployment was withheld.
+pub fn filter_shared(
+    deployments: Vec<Arc<Deployment>>,
+    team_id: Option<&str>,
+) -> (Vec<Arc<Deployment>>, bool) {
+    let team = team_id.unwrap_or("");
+    let checks: Vec<(Arc<Deployment>, Option<BTreeSet<String>>)> = deployments
+        .into_iter()
+        .map(|deployment| {
+            let shares = share_teams(&deployment.model_info.raw);
+            (deployment, shares)
+        })
+        .collect();
+    let holds_share = checks
+        .iter()
+        .any(|(_, shares)| shares.as_ref().is_some_and(|teams| teams.contains(team)));
+    let total = checks.len();
+    let kept: Vec<Arc<Deployment>> = checks
+        .into_iter()
+        .filter(|(_, shares)| match shares {
+            Some(teams) => teams.contains(team),
+            None => !holds_share,
+        })
+        .map(|(deployment, _)| deployment)
+        .collect();
+    let withheld = kept.len() < total;
+    (kept, withheld)
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+
+    #[rstest]
+    #[case(json!({"TA": 50}), Some(vec!["TA"]))]
+    #[case(json!({"TA": 2.0, "TB": 1}), Some(vec!["TA", "TB"]))]
+    #[case(json!({"TA": 2.5}), None)]
+    #[case(json!({"TA": true}), None)]
+    #[case(json!({"TA": 0}), None)]
+    #[case(json!({}), None)]
+    #[case(json!({"": 1}), None)]
+    #[case(json!("TA"), None)]
+    fn reads_the_share_teams(#[case] shares: Value, #[case] expected: Option<Vec<&str>>) {
+        let mut model_info = Map::new();
+        model_info.insert("ptu_shares".to_owned(), shares);
+        assert_eq!(
+            share_teams(&model_info),
+            expected.map(|teams| teams.into_iter().map(str::to_owned).collect())
+        );
+    }
+}

litellm-rust/crates/router-core/src/resolve.rs

@@ -0,0 +1,411 @@
+//! A requested name to the deployments that may serve this caller: a port of
+//! `_common_checks_available_deployment`. In order: a specific deployment, a deployment id, the
+//! alias, a routing group, then for a name no deployment has, a team's public name, every team's
+//! public name for a proxy admin, a wildcard; else the group (team-first), kept to the caller's
+//! access groups, then deployments whose litellm model is the name, then the first default
+//! fallback. Every arm drops strategy markers and withholds deployments reserved for other teams.
+
+use std::{collections::BTreeSet, sync::Arc};
+
+use jiff::Timestamp;
+
+use serde_json::Value;
+
+use crate::{
+    access::{filter_by_access_groups, filter_reserved},
+    error::Error,
+    registry::{Deployment, DeploymentSpec, Registry, Resolution, ResolveKind},
+    select::is_strategy_marker,
+};
+
+/// Who is asking, as the request's auth metadata says.
+#[derive(Clone, Copy, Debug, Default)]
+pub struct Caller<'a> {
+    /// `user_api_key_team_id`.
+    pub team_id: Option<&'a str>,
+    /// The caller's `user_role` is `proxy_admin`.
+    pub proxy_admin: bool,
+    /// The caller's `models` plus `team_models`.
+    pub allowed_models: &'a [String],
+}
+
+/// `_core_should_include_deployment`: the team's own deployment under its public name, or a
+/// deployment named `model_name` the team may use.
+fn included(deployment: &Deployment, model_name: &str, team_id: Option<&str>) -> bool {
+    let info = &deployment.model_info;
+    if team_id.is_some()
+        && info.team_id.as_deref() == team_id
+        && info.team_public_model_name.as_deref() == Some(model_name)
+    {
+        return true;
+    }
+    deployment.model_name == model_name
+        && (team_id.is_none() || info.team_id.is_none() || info.team_id.as_deref() == team_id)
+}
+
+/// `_core_get_all_deployments(model_name, team_id)`: the team's deployments under that public
+/// name when it has any, else the deployments named `model_name` the team may use; a name the
+/// model-name index does not hold is looked for in every deployment, for a team.
+pub fn all_deployments(
+    registry: &Registry,
+    model_name: &str,
+    team_id: Option<&str>,
+) -> Result<Vec<Arc<Deployment>>, Error> {
+    let keep = |deployments: Vec<Arc<Deployment>>| -> Vec<Arc<Deployment>> {
+        deployments
+            .into_iter()
+            .filter(|d| included(d, model_name, team_id))
+            .collect()
+    };
+    // A written index position past the model list raises `IndexError` where it is read.
+    let out_of_range = || Error::IndexOutOfRange {
+        model: model_name.to_owned(),
+    };
+    if let Some(team) = team_id {
+        if registry.team_index_overflows(team, model_name) {
+            return Err(out_of_range());
+        }
+        if let Some(own) = registry.team_named(team, model_name) {
+            let own = keep(own);
+            if !own.is_empty() {
+                return Ok(own);
+            }
+        }
+    }
+    if registry.name_index_overflows(model_name) {
+        return Err(out_of_range());
+    }
+    Ok(match registry.named(model_name) {
+        Some(named) => keep(named),
+        None if team_id.is_some() => keep(registry.all().to_vec()),
+        None => Vec::new(),
+    })
+}
+
+/// `_team_deployments_across_teams`: every team's deployments under public name `model`, for a
+/// proxy admin calling without a team; a name several teams share is refused.
+fn across_teams(registry: &Registry, model: &str) -> Result<Vec<Arc<Deployment>>, Error> {
+    if registry.across_teams_index_overflows(model) {
+        return Err(Error::IndexOutOfRange {
+            model: model.to_owned(),
+        });
+    }
+    let deployments = registry.team_named_across_teams(model);
+    let teams: BTreeSet<&str> = deployments
+        .iter()
+        .filter_map(|d| d.model_info.team_id.as_deref())
+        .collect();
+    if teams.len() > 1 {
+        return Err(Error::MultipleTeams {
+            model: model.to_owned(),
+        });
+    }
+    Ok(deployments)
+}
+
+/// The access groups of the rows `_core_get_model_list(model_name, team_id)` lists: the
+/// group's deployments, an alias's target deployments, a routing group's rows (which carry no
+/// access groups), and only when there are none of those, the wildcard's deployments.
+fn model_access_groups(
+    registry: &Registry,
+    model: &str,
+    team_id: Option<&str>,
+) -> BTreeSet<String> {
+    let mut rows = all_deployments(registry, model, team_id).unwrap_or_default();
+    let mut stripped_rows = 0;
+    if let Some(target) = registry.alias_of(model) {
+        match registry.routing_group_named(target) {
+            Some(group) => stripped_rows += routing_group_rows(registry, group, None).len(),
+            None => rows.extend(all_deployments(registry, target, None).unwrap_or_default()),
+        }
+    }
+    if let Some(group) = registry.routing_group_named(model) {
+        stripped_rows += routing_group_rows(registry, group, None).len();
+    }
+    if rows.is_empty() && stripped_rows == 0 {
+        rows = registry
+            .wildcard_for(model)
+            .map(|(_, deployments)| deployments)
+            .unwrap_or_default();
+    }
+    rows.iter()
+        .flat_map(|d| d.model_info.access_groups.iter().cloned())
+        .collect()
+}
+
+fn routing_group_rows(
+    registry: &Registry,
+    group: &crate::registry::RoutingGroupSpec,
+    team_id: Option<&str>,
+) -> Vec<Arc<Deployment>> {
+    crate::routing_groups::members(group, |member| {
+        all_deployments(registry, member, team_id).unwrap_or_default()
+    })
+}
+
+/// `_filter_reserved_deployments` for deployments resolved as `model`: markers dropped (only
+/// markers is refused), then the deployments an active access window withholds; when that
+/// leaves nothing, the window is named in the refusal.
+fn reserved(
+    model: &str,
+    deployments: Vec<Arc<Deployment>>,
+    team_id: Option<&str>,
+    now: Timestamp,
+) -> Result<Vec<Arc<Deployment>>, Error> {
+    let any = !deployments.is_empty();
+    let selectable: Vec<Arc<Deployment>> = deployments
+        .into_iter()
+        .filter(|d| !is_strategy_marker(d))
+        .collect();
+    if any && selectable.is_empty() {
+        return Err(Error::OnlyMarkers {
+            model: model.to_owned(),
+        });
+    }
+    let (kept, blocking) = filter_reserved(selectable, team_id, now);
+    match blocking {
+        Some(window) if kept.is_empty() => Err(Error::Reserved {
+            model: model.to_owned(),
+            until: window.end_hhmm(),
+            timezone: window.timezone,
+        }),
+        _ => Ok(kept),
+    }
+}
+
+/// The first default fallback a name no deployment serves is routed to.
+pub fn default_target(dynamic: &crate::config::RouterDynamicConfig) -> Option<&str> {
+    dynamic
+        .fallbacks
+        .default_target()
+        .or_else(|| dynamic.default_fallbacks.first().map(String::as_str))
+}
+
+/// The router's `default_deployment`, as resolution serves it.
+#[derive(Clone, Copy, Debug)]
+pub enum DefaultDeployment<'a> {
+    Spec(&'a DeploymentSpec),
+    /// A value Python cannot copy for a name: resolving to it raises.
+    Raises,
+}
+
+/// The default deployment as served for `model`: a copy whose `litellm_params.model` is the name.
+fn default_for(spec: &DeploymentSpec, model: &str) -> Option<Arc<Deployment>> {
+    let mut spec = spec.clone();
+    spec.litellm_params
+        .insert("model".to_owned(), Value::String(model.to_owned()));
+    Deployment::from_spec(spec).ok().map(Arc::new)
+}
+
+/// `_core_get_routable_deployments`: the deployments resolution would consider for `request`,
+/// before any team, cooldown or strategy filter; nothing when it resolves to nothing or refuses.
+pub fn routable(
+    registry: &Registry,
+    dynamic: &crate::config::RouterDynamicConfig,
+    request: &crate::host::RouterRequest,
+) -> Vec<Arc<Deployment>> {
+    let caller = Caller {
+        team_id: request.team_id.as_deref(),
+        proxy_admin: request.proxy_admin,
+        allowed_models: &request.allowed_models,
+    };
+    match resolve_request(
+        registry,
+        &request.model,
+        false,
+        caller,
+        default_target(dynamic),
+        dynamic.default_ref(),
+        Timestamp::now(),
+    ) {
+        Ok(Some(resolution)) => resolution.deployments,
+        Ok(None) | Err(_) => Vec::new(),
+    }
+}
+
+/// Resolve `name` for `caller`. `None` when nothing serves it (`no_healthy_deployments`).
+pub fn resolve_request(
+    registry: &Registry,
+    name: &str,
+    specific_deployment: bool,
+    caller: Caller<'_>,
+    default_target: Option<&str>,
+    default_deployment: Option<DefaultDeployment<'_>>,
+    now: Timestamp,
+) -> Result<Option<Resolution>, Error> {
+    let team_id = caller.team_id;
+    let resolved = |kind, group: &str, deployments| {
+        Ok(Some(Resolution {
+            kind,
+            group: group.to_owned(),
+            deployments,
+        }))
+    };
+    if specific_deployment {
+        let matching = registry
+            .all()
+            .iter()
+            .filter(|d| d.params.model == name)
+            .cloned()
+            .collect();
+        let kept = reserved(name, matching, team_id, now)?;
+        return resolved(ResolveKind::SpecificDeployment, name, kept);
+    }
+    if !registry.has_name(name)
+        && let Some(deployment) = registry.deployment(name)
+    {
+        let group = deployment.model_name.clone();
+        let litellm_model = deployment.params.model.clone();
+        let kept = reserved(&litellm_model, vec![deployment], team_id, now)?;
+        return resolved(ResolveKind::Id, &group, kept);
+    }
+    let aliased = registry.alias_of(name);
+    let model = aliased.unwrap_or(name);
+    let routing_group = registry
+        .routing_group_named(model)
+        .map(|group| routing_group_rows(registry, group, team_id));
+    if routing_group.is_none() && !registry.has_name(model) {
+        let early = match team_id {
+            Some(team) => Some(all_deployments(registry, model, Some(team))?),
+            None if caller.proxy_admin => Some(across_teams(registry, model)?),
+            None => None,
+        }
+        .filter(|deployments| !deployments.is_empty());
+        if let Some(deployments) = early {
+            let kept = reserved(model, deployments, team_id, now)?;
+            return resolved(ResolveKind::Group, model, kept);
+        }
+        if let Some((pattern, deployments)) = registry.wildcard_for(model) {
+            let kept = reserved(model, deployments, team_id, now)?;
+            return resolved(ResolveKind::Wildcard, &pattern, kept);
+        }
+        match default_deployment {
+            Some(DefaultDeployment::Raises) => {
+                return Err(Error::DefaultDeploymentRaises {
+                    model: model.to_owned(),
+                });
+            }
+            Some(DefaultDeployment::Spec(spec)) => {
+                if let Some(deployment) = default_for(spec, model) {
+                    let kept = reserved(model, vec![deployment], team_id, now)?;
+                    return resolved(ResolveKind::Default, model, kept);
+                }
+            }
+            None => {}
+        }
+    }
+    let (kind, listed) = match routing_group {
+        Some(deployments) => (ResolveKind::RoutingGroup, deployments),
+        None if aliased.is_some() => (
+            ResolveKind::Alias,
+            all_deployments(registry, model, team_id)?,
+        ),
+        None => (
+            ResolveKind::Group,
+            all_deployments(registry, model, team_id)?,
+        ),
+    };
+    let narrow = |model: &str, deployments: Vec<Arc<Deployment>>| {
+        let allowed = filter_by_access_groups(model, deployments, caller.allowed_models, || {
+            model_access_groups(registry, model, team_id)
+        });
+        reserved(model, allowed, team_id, now)
+    };
+    let listed_any = !listed.is_empty();
+    let deployments = narrow(model, listed)?;
+    if !deployments.is_empty() {
+        return resolved(kind, model, deployments);
+    }
+    let mut access_emptied = listed_any;
+    if !listed_any {
+        // The caller may have named a deployment's litellm model instead.
+        let by_model: Vec<Arc<Deployment>> = registry
+            .all()
+            .iter()
+            .filter(|d| d.params.model == model)
+            .cloned()
+            .collect();
+        let found = !by_model.is_empty();
+        let deployments = narrow(model, by_model)?;
+        if !deployments.is_empty() {
+            return resolved(ResolveKind::Group, model, deployments);
+        }
+        access_emptied = found;
+    }
+    if let Some(target) = default_target.filter(|_| !access_emptied) {
+        let deployments = narrow(target, all_deployments(registry, target, team_id)?)?;
+        if !deployments.is_empty() {
+            return resolved(ResolveKind::Group, target, deployments);
+        }
+    }
+    Ok(None)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::BTreeMap;
+
+    use serde_json::{Value, json};
+
+    use super::*;
+    use crate::registry::DeploymentSpec;
+
+    fn registry(entries: Value) -> Registry {
+        let specs: Vec<DeploymentSpec> = serde_json::from_value(entries).unwrap();
+        Registry::build(specs, BTreeMap::new(), Vec::new()).unwrap()
+    }
+
+    fn ids(resolution: Option<Resolution>) -> Vec<String> {
+        resolution
+            .map(|r| r.deployments.iter().map(|d| d.id.clone()).collect())
+            .unwrap_or_default()
+    }
+
+    #[test]
+    fn an_access_group_only_caller_is_kept_to_its_groups() {
+        let registry = registry(json!([
+            {"model_name": "g", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a", "access_groups": ["A"]}},
+            {"model_name": "g", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "b", "access_groups": ["B"]}},
+        ]));
+        let allowed = vec!["A".to_owned()];
+        let caller = Caller {
+            allowed_models: &allowed,
+            ..Caller::default()
+        };
+        let resolved = resolve_request(&registry, "g", false, caller, None, None, Timestamp::now());
+        assert_eq!(ids(resolved.unwrap()), ["a"]);
+        // The litellm model string is not narrowed (python-fixes-to-make.md).
+        let by_model = resolve_request(
+            &registry,
+            "openai/b",
+            false,
+            caller,
+            None,
+            None,
+            Timestamp::now(),
+        );
+        assert_eq!(ids(by_model.unwrap()), ["b"]);
+    }
+
+    #[test]
+    fn a_proxy_admin_is_refused_a_public_name_several_teams_share() {
+        let registry = registry(json!([
+            {"model_name": "x", "litellm_params": {"model": "openai/x"}, "model_info": {"id": "x", "team_id": "tx", "team_public_model_name": "pub"}},
+            {"model_name": "y", "litellm_params": {"model": "openai/y"}, "model_info": {"id": "y", "team_id": "ty", "team_public_model_name": "pub"}},
+        ]));
+        let caller = Caller {
+            proxy_admin: true,
+            ..Caller::default()
+        };
+        let refused = resolve_request(
+            &registry,
+            "pub",
+            false,
+            caller,
+            None,
+            None,
+            Timestamp::now(),
+        );
+        assert!(matches!(refused, Err(Error::MultipleTeams { model }) if model == "pub"));
+    }
+}

litellm-rust/crates/router-core/src/retry.rs

@@ -0,0 +1,216 @@
+//! Retry decisions: how many retries a `RetryPolicy` grants for a failure, and whether a failure
+//! is retried in the same group at all. Mirrors `get_num_retries_from_retry_policy` and
+//! `resolve_retry_policy` in `litellm_core_utils/get_retry_from_policy.py`, and
+//! `Router.should_retry_this_error`.
+
+use std::collections::BTreeMap;
+
+use crate::{
+    config::RetryPolicy,
+    error::{AttemptError, FailureKind},
+    health::should_retry_status,
+};
+
+/// `resolve_retry_policy`: the group's own policy when `model_group_retry_policy` has one, else
+/// the router's `retry_policy`.
+pub fn resolve_retry_policy<'a>(
+    retry_policy: Option<&'a RetryPolicy>,
+    model_group: &str,
+    model_group_retry_policy: Option<&'a BTreeMap<String, RetryPolicy>>,
+) -> Option<&'a RetryPolicy> {
+    model_group_retry_policy
+        .and_then(|policies| policies.get(model_group))
+        .or(retry_policy)
+}
+
+impl RetryPolicy {
+    /// `get_num_retries_from_retry_policy`: `DefaultRetries` for an `UnsupportedEndpointError`;
+    /// else `NotFoundErrorRetries` for any 404 answer, then the most specific exception class
+    /// the policy names, then `DefaultRetries`.
+    pub fn retries_for(&self, failure: &AttemptError) -> Option<u32> {
+        if failure.is_a("UnsupportedEndpointError") {
+            return self.DefaultRetries;
+        }
+        let not_found = (failure.status == Some(404))
+            .then_some(self.NotFoundErrorRetries)
+            .flatten();
+        not_found
+            .or_else(|| {
+                failure
+                    .class_names()
+                    .iter()
+                    .find_map(|class| self.field_for(class))
+            })
+            .or(self.DefaultRetries)
+    }
+
+    /// `_RETRIES_BY_EXCEPTION_TYPE`: the policy field an exception class reads.
+    fn field_for(&self, class: &str) -> Option<u32> {
+        match class {
+            "AuthenticationError" => self.AuthenticationErrorRetries,
+            "Timeout" => self.TimeoutErrorRetries,
+            "RateLimitError" => self.RateLimitErrorRetries,
+            "ContentPolicyViolationError" => self.ContentPolicyViolationErrorRetries,
+            "BadRequestError" => self.BadRequestErrorRetries,
+            "ServiceUnavailableError" => self.ServiceUnavailableErrorRetries,
+            "InternalServerError" => self.InternalServerErrorRetries,
+            _ => None,
+        }
+    }
+}
+
+/// What `should_retry_this_error` knows besides the failure.
+#[derive(Clone, Copy, Debug, Default)]
+pub struct RetryContext {
+    pub healthy: usize,
+    pub all: usize,
+    /// `context_window_fallbacks is not None`.
+    pub has_context_window_fallbacks: bool,
+    /// `content_policy_fallbacks is not None`.
+    pub has_content_policy_fallbacks: bool,
+    /// `regular_fallbacks is not None and len(regular_fallbacks) > 0`.
+    pub has_regular_fallbacks: bool,
+}
+
+/// `should_retry_this_error`, as a predicate: `false` where Python raises the error.
+pub fn should_retry(failure: &AttemptError, ctx: RetryContext) -> bool {
+    if failure.is_a("ContextWindowExceededError") && ctx.has_context_window_fallbacks {
+        return false;
+    }
+    if failure.is_a("ContentPolicyViolationError") && ctx.has_content_policy_fallbacks {
+        return false;
+    }
+    if let Some(status) = failure.status
+        && !should_retry_status(status)
+        && !matches!(status, 401 | 403)
+        && !failure.is_a("UnsupportedEndpointError")
+    {
+        return false;
+    }
+    if failure.is_a("NotFoundError") {
+        return false;
+    }
+    if failure.is_a("RateLimitError") && ctx.healthy == 0 && ctx.has_regular_fallbacks {
+        return false;
+    }
+    if (failure.is_a("AuthenticationError") || failure.is_a("PermissionDeniedError"))
+        && ctx.all <= 1
+    {
+        return false;
+    }
+    ctx.healthy > 0
+}
+
+/// `ContentPolicyViolationError`'s classes, most specific first.
+pub(crate) fn content_policy_classes() -> Vec<std::sync::Arc<str>> {
+    [
+        "ContentPolicyViolationError",
+        "BadRequestError",
+        "APIStatusError",
+        "APIError",
+        "OpenAIError",
+        "Exception",
+        "BaseException",
+        "object",
+    ]
+    .into_iter()
+    .map(Into::into)
+    .collect()
+}
+
+/// The exception classes a failure kind stands for, most specific first, for a host that
+/// reports no class names (`litellm/exceptions.py`).
+pub(crate) fn kind_classes(kind: FailureKind) -> &'static [&'static str] {
+    match kind {
+        FailureKind::RateLimit => &["RateLimitError"],
+        FailureKind::ContextWindowExceeded => &["ContextWindowExceededError", "BadRequestError"],
+        FailureKind::ContentPolicyViolation => &["ContentPolicyViolationError", "BadRequestError"],
+        FailureKind::Authentication => &["AuthenticationError"],
+        FailureKind::PermissionDenied => &["PermissionDeniedError"],
+        FailureKind::NotFound => &["NotFoundError"],
+        FailureKind::BadRequest => &["BadRequestError"],
+        FailureKind::Timeout => &["Timeout", "APITimeoutError", "APIConnectionError"],
+        FailureKind::ApiConnection => &["APIConnectionError"],
+        FailureKind::InternalServer => &["InternalServerError"],
+        FailureKind::ServiceUnavailable => &["ServiceUnavailableError"],
+        FailureKind::BadGateway => &["BadGatewayError"],
+        FailureKind::GuardrailIntervention
+        | FailureKind::Declined
+        | FailureKind::CallerCredential
+        | FailureKind::Other => &[],
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+
+    fn failure(kind: FailureKind, status: Option<u16>, classes: &[&str]) -> AttemptError {
+        let mut failure = AttemptError::new(kind, status);
+        failure.classes = classes.iter().map(|class| (*class).into()).collect();
+        failure
+    }
+
+    #[rstest]
+    #[case::own_field(failure(FailureKind::InternalServer, Some(500), &[]), Some(4))]
+    #[case::default(failure(FailureKind::BadGateway, Some(502), &[]), Some(7))]
+    #[case::not_found_status(failure(FailureKind::Other, Some(404), &["APIError"]), Some(1))]
+    #[case::unsupported_endpoint(
+        failure(FailureKind::Declined, Some(400), &["UnsupportedEndpointError", "BadRequestError"]),
+        Some(7)
+    )]
+    #[case::malformed_reads_bad_request(
+        failure(FailureKind::Declined, Some(400), &["MalformedRequestFieldError", "BadRequestError"]),
+        Some(3)
+    )]
+    fn retries_follow_the_most_specific_class(
+        #[case] failure: AttemptError,
+        #[case] expected: Option<u32>,
+    ) {
+        let policy = RetryPolicy {
+            InternalServerErrorRetries: Some(4),
+            BadRequestErrorRetries: Some(3),
+            NotFoundErrorRetries: Some(1),
+            DefaultRetries: Some(7),
+            ..RetryPolicy::default()
+        };
+        assert_eq!(policy.retries_for(&failure), expected);
+    }
+
+    #[rstest]
+    #[case::rate_limit_with_healthy(FailureKind::RateLimit, Some(429), 1, 2, false, true)]
+    #[case::rate_limit_none_healthy_with_fallbacks(
+        FailureKind::RateLimit,
+        Some(429),
+        0,
+        2,
+        true,
+        false
+    )]
+    #[case::bad_request(FailureKind::BadRequest, Some(400), 2, 2, false, false)]
+    #[case::auth_lone_deployment(FailureKind::Authentication, Some(401), 1, 1, false, false)]
+    #[case::auth_with_siblings(FailureKind::Authentication, Some(401), 1, 2, false, true)]
+    #[case::not_found(FailureKind::NotFound, Some(404), 2, 2, false, false)]
+    #[case::server_error_no_healthy(FailureKind::InternalServer, Some(500), 0, 2, false, false)]
+    fn should_retry_follows_python(
+        #[case] kind: FailureKind,
+        #[case] status: Option<u16>,
+        #[case] healthy: usize,
+        #[case] all: usize,
+        #[case] has_regular_fallbacks: bool,
+        #[case] expected: bool,
+    ) {
+        let ctx = RetryContext {
+            healthy,
+            all,
+            has_regular_fallbacks,
+            ..RetryContext::default()
+        };
+        assert_eq!(
+            should_retry(&AttemptError::new(kind, status), ctx),
+            expected
+        );
+    }
+}

litellm-rust/crates/router-core/src/routing_groups.rs

@@ -0,0 +1,48 @@
+//! Routing groups: a callable name over the deployments of its member model groups
+//! (`_get_routing_group_deployments`). A priority group serves each member's priority as the
+//! member deployments' `order` (`apply_routing_group_priority`), so the order filter and order
+//! fallbacks walk the priorities.
+
+use std::sync::Arc;
+
+use serde_json::Value;
+
+use crate::registry::{Deployment, RoutingGroupSpec};
+
+/// The member deployments of `group`, in member order, with a priority group's priorities as
+/// their `order`.
+pub fn members(
+    group: &RoutingGroupSpec,
+    deployments_of: impl Fn(&str) -> Vec<Arc<Deployment>>,
+) -> Vec<Arc<Deployment>> {
+    let priorities = group
+        .model_priorities
+        .as_ref()
+        .filter(|_| is_priority(group));
+    group
+        .models
+        .iter()
+        .flat_map(|member| {
+            let priority = priorities.and_then(|p| p.get(member)).copied();
+            deployments_of(member)
+                .into_iter()
+                .map(move |deployment| match priority {
+                    Some(order) => with_order(&deployment, order),
+                    None => deployment,
+                })
+        })
+        .collect()
+}
+
+/// Whether `group` is a priority group.
+pub fn is_priority(group: &RoutingGroupSpec) -> bool {
+    matches!(&group.routing_strategy, crate::config::RoutingStrategy::Other(name) if name == "priority")
+}
+
+fn with_order(deployment: &Arc<Deployment>, order: i64) -> Arc<Deployment> {
+    let mut prioritized = Deployment::clone(deployment);
+    prioritized
+        .raw_params
+        .insert("order".to_owned(), Value::from(order));
+    Arc::new(prioritized)
+}

litellm-rust/crates/router-core/src/routing_plugins.rs

@@ -0,0 +1,114 @@
+//! `Router(plugins=[...])`: the routing-plugin pipeline. The host runs the plugins (live Python
+//! objects) in the async pick's pre-routing hook and reports the candidate models they left on
+//! the request (`_routing_plugin_candidate_models`); router-core decides which deployments that
+//! keeps (`_filter_by_routing_plugin_candidates`) and refuses the paths that never run the
+//! pipeline (`_get_available_deployment_unresolved`).
+
+use std::sync::Arc;
+
+use crate::{config::RoutingStrategy, error::Error, host::RouterRequest, registry::Deployment};
+
+/// Whether the router's own strategy has an async selector: the others pick on the sync path
+/// even for async callers (`_async_get_available_deployment`).
+fn picks_async(strategy: &RoutingStrategy) -> bool {
+    matches!(
+        strategy,
+        RoutingStrategy::UsageBasedV2
+            | RoutingStrategy::SimpleShuffle
+            | RoutingStrategy::CostBased
+            | RoutingStrategy::LatencyBased
+            | RoutingStrategy::LeastBusy
+    )
+}
+
+/// The sync pick never runs the pipeline, so a router with plugins refuses it rather than
+/// letting a policy plugin be bypassed. `request` is a pick or a call, never a listing.
+pub(crate) fn refuse_sync_pick(
+    has_plugins: bool,
+    strategy: &RoutingStrategy,
+    request: &RouterRequest,
+) -> Result<(), Error> {
+    if has_plugins
+        && request.pass_through.is_none()
+        && (!request.pre_routes || !picks_async(strategy))
+    {
+        return Err(Error::RoutingPluginsNeedAsync);
+    }
+    Ok(())
+}
+
+/// `_filter_by_routing_plugin_candidates`: keep the deployments whose litellm model the plugins
+/// left; nothing left is an error, never a fallback to the unfiltered pool.
+pub(crate) fn narrow(
+    healthy: Vec<Arc<Deployment>>,
+    models: Option<&[String]>,
+) -> Result<Vec<Arc<Deployment>>, Error> {
+    let Some(models) = models else {
+        return Ok(healthy);
+    };
+    let kept: Vec<Arc<Deployment>> = healthy
+        .into_iter()
+        .filter(|d| models.contains(&d.params.model))
+        .collect();
+    if kept.is_empty() {
+        return Err(Error::RoutingPluginsLeftNothing {
+            models: models.to_vec(),
+        });
+    }
+    Ok(kept)
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+    use crate::registry::DeploymentSpec;
+
+    fn deployment(id: &str, model: &str) -> Arc<Deployment> {
+        let spec: DeploymentSpec = serde_json::from_value(json!({
+            "model_name": "g", "litellm_params": {"model": model}, "model_info": {"id": id},
+        }))
+        .unwrap();
+        Arc::new(Deployment::from_spec(spec).unwrap())
+    }
+
+    #[rstest]
+    fn the_plugins_models_narrow_the_pool() {
+        let pool = vec![deployment("a", "openai/a"), deployment("b", "openai/b")];
+        let kept = narrow(pool, Some(&["openai/b".to_owned()])).unwrap();
+        assert_eq!(
+            kept.iter().map(|d| d.id.as_str()).collect::<Vec<_>>(),
+            ["b"]
+        );
+    }
+
+    #[rstest]
+    fn narrowing_to_nothing_is_an_error() {
+        let pool = vec![deployment("a", "openai/a")];
+        assert!(matches!(
+            narrow(pool, Some(&[])),
+            Err(Error::RoutingPluginsLeftNothing { models }) if models.is_empty()
+        ));
+    }
+
+    #[rstest]
+    #[case::async_pick(true, RoutingStrategy::SimpleShuffle, false)]
+    #[case::sync_pick(false, RoutingStrategy::SimpleShuffle, true)]
+    #[case::v1_strategy(true, RoutingStrategy::UsageBased, true)]
+    fn only_the_async_pick_runs_plugins(
+        #[case] pre_routes: bool,
+        #[case] strategy: RoutingStrategy,
+        #[case] refused: bool,
+    ) {
+        let request = RouterRequest {
+            pre_routes,
+            ..RouterRequest::default()
+        };
+        assert_eq!(
+            refuse_sync_pick(true, &strategy, &request).is_err(),
+            refused
+        );
+    }
+}

litellm-rust/crates/router-core/src/select.rs

@@ -0,0 +1,673 @@
+//! Selection: resolve a name, filter its deployments in the order of
+//! `Router._core_async_get_healthy_deployments` (team, cooldown, blocked, plugins, pre-call
+//! checks, tags, order, retry skips), then pick one with the strategy. Simple-shuffle is native
+//! (`router_strategy/simple_shuffle.py`); other strategies go through `Host::select`.
+
+use std::{collections::BTreeMap, sync::Arc};
+
+use parking_lot::Mutex;
+use rand::{Rng, SeedableRng, rngs::StdRng};
+
+use crate::{
+    Core, Snapshot,
+    config::{Op, RoutingStrategy},
+    error::Error,
+    health_routing::WhenAllUnhealthy,
+    host::{RouterRequest, RoutingContext, SelectStage},
+    registry::{Deployment, ResolveKind},
+    resolve::{Caller, resolve_request},
+    strategy::{StrategyScope, routing_context},
+};
+
+/// Randomness for the shuffle and retry jitter, injectable so tests are deterministic.
+pub trait Randomness: Send + Sync {
+    /// A float in `[0, 1)`, as Python's `random.random()`.
+    fn next_f64(&self) -> f64;
+}
+
+pub struct ThreadRandomness;
+
+impl Randomness for ThreadRandomness {
+    fn next_f64(&self) -> f64 {
+        rand::thread_rng().r#gen()
+    }
+}
+
+pub struct SeededRandomness(Mutex<StdRng>);
+
+impl SeededRandomness {
+    pub fn new(seed: u64) -> Self {
+        Self(Mutex::new(StdRng::seed_from_u64(seed)))
+    }
+}
+
+impl Randomness for SeededRandomness {
+    fn next_f64(&self) -> f64 {
+        self.0.lock().r#gen()
+    }
+}
+
+/// Which pass-through pick a request makes.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum PassThrough {
+    /// `get_available_deployment_for_pass_through` (and `select(kind="pass_through")`): the
+    /// pass-through deployments are chosen first, then cooldown and blocked filters, and only a
+    /// priority group filters by order.
+    Listed,
+    /// `async_get_available_deployment_for_pass_through`: the full healthy chain, then the
+    /// pass-through filter.
+    Healthy,
+}
+
+/// A group's deployments before and after filtering.
+#[derive(Clone, Debug)]
+pub struct Candidates {
+    /// The name that was resolved (Python's `model`).
+    pub name: String,
+    pub model_group: String,
+    pub kind: ResolveKind,
+    /// Every deployment the caller may use (after the team filter).
+    pub all: Vec<Arc<Deployment>>,
+    pub healthy: Vec<Arc<Deployment>>,
+}
+
+/// Per-attempt selection inputs the call loop carries.
+#[derive(Clone, Debug, Default)]
+pub(crate) struct SelectInputs {
+    pub target_order: Option<i64>,
+    pub skipped_ids: Vec<String>,
+    /// A weighted failover hop's `_excluded_deployment_ids`: never picked.
+    pub excluded_ids: Arc<[String]>,
+    /// Stop before the plugins, order and skip filters: the group less cooldowns and blocked
+    /// deployments, which `_async_get_healthy_deployments` counts when deciding whether to retry.
+    pub count_only: bool,
+}
+
+/// `simple_shuffle`: weighted by `_router_weights[group]`, else `weight`, else `rpm`, else
+/// `tpm`, each normalized by its maximum, else uniform. A metric is read only when every set
+/// before it weighed nothing, and a non-numeric one fails the pick then (`_metric_weight`).
+pub fn simple_shuffle(
+    deployments: &[Arc<Deployment>],
+    model_group: &str,
+    router_weights: Option<&BTreeMap<String, BTreeMap<String, f64>>>,
+    rng: &dyn Randomness,
+) -> Result<Option<Arc<Deployment>>, Error> {
+    if deployments.is_empty() {
+        return Ok(None);
+    }
+    let scoped = router_weights
+        .and_then(|weights| weights.get(model_group))
+        .filter(|weights| !weights.is_empty())
+        .map(|weights| {
+            deployments
+                .iter()
+                .map(|d| weights.get(&d.id).copied().unwrap_or(0.0))
+                .collect::<Vec<_>>()
+        });
+    let metric_sets = ["weight", "rpm", "tpm"].into_iter().map(|metric| {
+        deployments
+            .iter()
+            .map(|d| d.metric_weight(metric))
+            .collect::<Option<Vec<_>>>()
+            .ok_or(Error::NonNumericWeight { metric })
+    });
+    for weights in scoped.into_iter().map(Ok).chain(metric_sets) {
+        let weights = weights?;
+        let largest = weights.iter().copied().fold(0.0, f64::max);
+        if largest <= 0.0 {
+            continue;
+        }
+        let normalized: Vec<f64> = weights.iter().map(|weight| weight / largest).collect();
+        let total: f64 = normalized.iter().sum();
+        if total <= 0.0 {
+            continue;
+        }
+        // `random.choices`: bisect_right over the cumulative weights.
+        let target = rng.next_f64() * total;
+        let mut cumulative = 0.0;
+        let index = normalized
+            .iter()
+            .position(|weight| {
+                cumulative += weight;
+                target < cumulative
+            })
+            .unwrap_or(deployments.len() - 1);
+        return Ok(Some(deployments[index].clone()));
+    }
+    let index = ((rng.next_f64() * deployments.len() as f64) as usize).min(deployments.len() - 1);
+    Ok(Some(deployments[index].clone()))
+}
+
+/// A pre-routing strategy marker (`classify_strategy_router_model`): its model is `auto_router/...`.
+pub fn is_strategy_marker(deployment: &Deployment) -> bool {
+    deployment.params.model.starts_with("auto_router/")
+}
+
+/// `litellm_params.use_in_pass_through`, read for truthiness as Python does.
+fn uses_pass_through(deployment: &Deployment) -> bool {
+    match deployment.raw_params.get("use_in_pass_through") {
+        None | Some(serde_json::Value::Null) => false,
+        Some(serde_json::Value::Bool(value)) => *value,
+        Some(serde_json::Value::Number(value)) => value.as_f64().is_some_and(|n| n != 0.0),
+        Some(serde_json::Value::String(value)) => !value.is_empty(),
+        Some(serde_json::Value::Array(value)) => !value.is_empty(),
+        Some(serde_json::Value::Object(value)) => !value.is_empty(),
+    }
+}
+
+/// `filter_team_based_models`: a team-owned deployment serves only its team, except that a proxy
+/// admin without a team keeps deployments that are all team-owned and listed under the model the
+/// request names, unless they belong to several teams.
+fn filter_team_based(
+    deployments: Vec<Arc<Deployment>>,
+    request: &RouterRequest,
+) -> Result<Vec<Arc<Deployment>>, Error> {
+    let team_id = request.team_id.as_deref();
+    if team_id.is_none()
+        && request.proxy_admin
+        && let Some(requested) = request.admin_model.as_deref()
+        && !deployments.is_empty()
+        && deployments.iter().all(|d| {
+            d.model_info.team_id.is_some()
+                && (d.model_name == requested
+                    || d.model_info.team_public_model_name.as_deref() == Some(requested))
+        })
+    {
+        let teams: std::collections::BTreeSet<&str> = deployments
+            .iter()
+            .filter_map(|d| d.model_info.team_id.as_deref())
+            .collect();
+        if teams.len() > 1 {
+            return Err(Error::MultipleTeams {
+                model: requested.to_owned(),
+            });
+        }
+        return Ok(deployments);
+    }
+    Ok(deployments
+        .into_iter()
+        .filter(|d| d.usable_by_team(team_id))
+        .collect())
+}
+
+impl Core {
+    fn context<'a>(
+        &'a self,
+        snapshot: &'a Snapshot,
+        op: &'a Op,
+        request: &'a RouterRequest,
+        model_group: &'a str,
+        name: &'a str,
+        scope: StrategyScope,
+    ) -> RoutingContext<'a> {
+        RoutingContext {
+            registry: &snapshot.registry,
+            settings: &snapshot.settings,
+            health: &self.health,
+            request,
+            op,
+            model_group,
+            name,
+            scope,
+            pool: &[],
+        }
+    }
+
+    /// The eligible deployments for `name`, in filter-chain order.
+    pub(crate) async fn candidates(
+        &self,
+        snapshot: &Snapshot,
+        op: &Op,
+        request: &RouterRequest,
+        name: &str,
+        inputs: &SelectInputs,
+    ) -> Result<Candidates, Error> {
+        let registry = &snapshot.registry;
+        // The async pick's pre-routing hook runs the routing plugins before anything resolves;
+        // every pick but a listed pass-through one reads what they left on the request.
+        let plugin_models = if snapshot.settings.routing_plugins
+            && !inputs.count_only
+            && request.pass_through != Some(PassThrough::Listed)
+        {
+            let ctx = self.context(snapshot, op, request, name, name, StrategyScope::Default);
+            self.host
+                .routing_plugin_models(&ctx, request.pre_routes)
+                .await?
+        } else {
+            None
+        };
+        // No deployment for the name: route to the first default fallback.
+        let default_target = crate::resolve::default_target(&snapshot.settings.dynamic);
+        let caller = Caller {
+            team_id: request.team_id.as_deref(),
+            proxy_admin: request.proxy_admin,
+            allowed_models: &request.allowed_models,
+        };
+        let resolution = resolve_request(
+            registry,
+            name,
+            request.specific_deployment,
+            caller,
+            default_target,
+            snapshot.settings.dynamic.default_ref(),
+            jiff::Timestamp::now(),
+        )?
+        .ok_or_else(|| Error::UnknownModel {
+            model: name.to_owned(),
+        })?;
+        let first_model = resolution
+            .deployments
+            .first()
+            .map(|d| d.params.model.clone());
+        let team_id = request.team_id.as_deref();
+        let usable = filter_team_based(resolution.deployments.clone(), request)?;
+        // `_filter_reserved_deployments`: PTU shares keep holders to their share and everyone
+        // else to the deployments split into none.
+        let (mut all, withheld) = crate::ptu::filter_shared(usable, team_id);
+        if withheld && all.is_empty() {
+            return Err(Error::PtuReserved {
+                model: match resolution.kind {
+                    ResolveKind::Id => first_model.unwrap_or_else(|| name.to_owned()),
+                    ResolveKind::Wildcard | ResolveKind::SpecificDeployment => name.to_owned(),
+                    _ => resolution.group.clone(),
+                },
+            });
+        }
+        let listed =
+            request.pass_through == Some(PassThrough::Listed) && !resolution.kind.is_single();
+        if listed {
+            all.retain(|d| uses_pass_through(d));
+            if all.is_empty() {
+                return Err(Error::NoPassThroughDeployments {
+                    model: name.to_owned(),
+                });
+            }
+        }
+
+        // A deployment named by its id is served even while it cools down
+        // (`get_deployment_by_model_id` returns before the cooldown filter).
+        let settings = &snapshot.settings;
+        let cooling = if resolution.kind.is_single() {
+            Vec::new()
+        } else {
+            let ids: Vec<String> = all.iter().map(|d| d.id.clone()).collect();
+            self.health.active_cooldowns(&ids).await?
+        };
+        // Health-check filtering runs before the cooldown filter, except for a deployment named
+        // by its id and the healthy count a retry decision reads.
+        let checked = if resolution.kind.is_single() || inputs.count_only {
+            all.clone()
+        } else {
+            let when_all_unhealthy = if request.health_check_probe {
+                WhenAllUnhealthy::Nothing
+            } else {
+                WhenAllUnhealthy::KeepAll
+            };
+            self.health
+                .health_check_filter(settings, all.clone(), when_all_unhealthy)
+                .await
+        };
+        let mut healthy: Vec<Arc<Deployment>> = checked
+            .iter()
+            .filter(|d| !cooling.iter().any(|(id, _)| *id == d.id))
+            .cloned()
+            .collect();
+        // The safety net when health-check routing drives cooldowns (an allowed_fails_policy):
+        // never on a listed pass-through pick or the healthy count.
+        if healthy.is_empty()
+            && !listed
+            && !inputs.count_only
+            && settings.enable_health_check_routing
+            && settings.allowed_fails_policy.is_some()
+        {
+            healthy = checked;
+        }
+        if resolution.kind.is_single()
+            && let Some(deployment) = all.first()
+        {
+            // `_core_async_get_healthy_deployments`: a deployment named by id is checked alone.
+            if deployment.model_info.blocked {
+                return Err(Error::Paused {
+                    model: deployment.params.model.clone(),
+                });
+            }
+            if request.pass_through.is_some() && !uses_pass_through(deployment) {
+                return Err(Error::NotPassThrough {
+                    id: deployment.id.clone(),
+                });
+            }
+        }
+        healthy.retain(|d| !d.model_info.blocked);
+        // `continuation_deployments`: a mid-stream continuation never lands on a deployment the host
+        // named as unable to take its prefill.
+        if let Some(excluded) = request
+            .body
+            .get("continuation_excluded_ids")
+            .and_then(serde_json::Value::as_array)
+        {
+            healthy.retain(|d| !excluded.iter().any(|id| id.as_str() == Some(d.id.as_str())));
+        }
+        // `async_callback_filter_deployments` runs after the cooldown and blocked filters on every
+        // async candidate list but a deployment named by id and a listed pass-through pick.
+        if settings.callback_filters
+            && !inputs.count_only
+            && !resolution.kind.is_single()
+            && request.pass_through != Some(PassThrough::Listed)
+        {
+            let ctx = self.context(
+                snapshot,
+                op,
+                request,
+                &resolution.group,
+                name,
+                StrategyScope::Default,
+            );
+            healthy = self.host.callback_filters(&ctx, healthy).await?;
+        }
+        if inputs.count_only {
+            return Ok(Candidates {
+                name: name.to_owned(),
+                model_group: resolution.group,
+                kind: resolution.kind,
+                all,
+                healthy,
+            });
+        }
+
+        if listed {
+            if settings.enable_pre_call_checks && !resolution.kind.is_single() {
+                healthy = self
+                    .pre_call_checks(
+                        snapshot,
+                        op,
+                        request,
+                        &resolution.group,
+                        name,
+                        &all,
+                        healthy,
+                    )
+                    .await?;
+            }
+            // Only a priority group narrows a listed pass-through pick by order.
+            let priority = resolution.kind == ResolveKind::RoutingGroup
+                && registry.routing_groups().iter().any(|g| {
+                    g.group_name == resolution.group && crate::routing_groups::is_priority(g)
+                });
+            if priority {
+                let order = request
+                    .target_order
+                    .or_else(|| healthy.iter().filter_map(|d| d.order()).min());
+                if let Some(order) = order {
+                    healthy.retain(|d| d.order() == Some(order));
+                }
+            }
+            return Ok(Candidates {
+                name: name.to_owned(),
+                model_group: resolution.group,
+                kind: resolution.kind,
+                all,
+                healthy,
+            });
+        }
+
+        if !settings.plugins.is_empty() && !healthy.is_empty() {
+            let ctx = self.context(
+                snapshot,
+                op,
+                request,
+                &resolution.group,
+                name,
+                StrategyScope::Default,
+            );
+            healthy = self
+                .host
+                .select(&SelectStage::Plugins, &ctx, healthy)
+                .await?;
+        }
+
+        if settings.enable_pre_call_checks && !resolution.kind.is_single() {
+            healthy = self
+                .pre_call_checks(
+                    snapshot,
+                    op,
+                    request,
+                    &resolution.group,
+                    name,
+                    &all,
+                    healthy,
+                )
+                .await?;
+        }
+
+        if request.filter_tags && !healthy.is_empty() {
+            let mut ctx = self.context(
+                snapshot,
+                op,
+                request,
+                &resolution.group,
+                name,
+                StrategyScope::Default,
+            );
+            ctx.pool = &all;
+            healthy = self.host.select(&SelectStage::Tags, &ctx, healthy).await?;
+        }
+
+        if !resolution.kind.is_single() {
+            healthy = crate::routing_plugins::narrow(healthy, plugin_models.as_deref())?;
+        }
+
+        // `_without_declared_text_only_deployments`: an image request skips the deployments that
+        // declare no vision, unless that leaves nothing.
+        if request.has_image && !resolution.kind.is_single() && !healthy.is_empty() {
+            let ctx = self.context(
+                snapshot,
+                op,
+                request,
+                &resolution.group,
+                name,
+                StrategyScope::Default,
+            );
+            let text_only = self
+                .host
+                .select(&SelectStage::TextOnly, &ctx, healthy.clone())
+                .await?;
+            let vision: Vec<Arc<Deployment>> = healthy
+                .iter()
+                .filter(|d| !text_only.iter().any(|t| t.id == d.id))
+                .cloned()
+                .collect();
+            if !vision.is_empty() {
+                healthy = vision;
+            }
+        }
+
+        if let Some(pinned) = request.pinned_deployment_id.as_deref()
+            && healthy.iter().any(|d| d.id == pinned)
+        {
+            healthy.retain(|d| d.id == pinned);
+        }
+
+        let order = inputs
+            .target_order
+            .or(request.target_order)
+            .or_else(|| healthy.iter().filter_map(|d| d.order()).min());
+        if let Some(order) = order {
+            healthy.retain(|d| d.order() == Some(order));
+        }
+
+        healthy.retain(|d| {
+            !request.excluded_ids.contains(&d.id) && !inputs.excluded_ids.contains(&d.id)
+        });
+
+        if !inputs.skipped_ids.is_empty() || !request.retry_skipped_ids.is_empty() {
+            let kept: Vec<Arc<Deployment>> = healthy
+                .iter()
+                .filter(|d| {
+                    !inputs.skipped_ids.contains(&d.id)
+                        && !request.retry_skipped_ids.contains(&d.id)
+                })
+                .cloned()
+                .collect();
+            if !kept.is_empty() {
+                healthy = kept;
+            }
+        }
+
+        if request.pass_through == Some(PassThrough::Healthy)
+            && !resolution.kind.is_single()
+            && !healthy.is_empty()
+        {
+            healthy.retain(|d| uses_pass_through(d));
+            if healthy.is_empty() {
+                return Err(Error::NoPassThroughDeployments {
+                    model: name.to_owned(),
+                });
+            }
+        }
+
+        Ok(Candidates {
+            name: name.to_owned(),
+            model_group: resolution.group,
+            kind: resolution.kind,
+            all,
+            healthy,
+        })
+    }
+
+    /// `_pre_call_checks` over `healthy`, with the host's facts.
+    #[allow(clippy::too_many_arguments)]
+    async fn pre_call_checks(
+        &self,
+        snapshot: &Snapshot,
+        op: &Op,
+        request: &RouterRequest,
+        model_group: &str,
+        name: &str,
+        pool: &[Arc<Deployment>],
+        healthy: Vec<Arc<Deployment>>,
+    ) -> Result<Vec<Arc<Deployment>>, Error> {
+        let mut ctx = self.context(
+            snapshot,
+            op,
+            request,
+            model_group,
+            name,
+            StrategyScope::Default,
+        );
+        ctx.pool = pool;
+        let facts = self.host.pre_call_facts(&ctx, &healthy).await?;
+        crate::precall::filter(model_group, healthy, pool, &facts)
+    }
+
+    /// Pick one of `candidates.healthy` with the group's strategy.
+    pub(crate) async fn pick(
+        &self,
+        snapshot: &Snapshot,
+        op: &Op,
+        request: &RouterRequest,
+        candidates: &Candidates,
+    ) -> Result<Arc<Deployment>, Error> {
+        let no_deployment = || Error::NoHealthyDeployments {
+            model: candidates.model_group.clone(),
+        };
+        // A deployment named by its id is returned as is, before any strategy runs.
+        if candidates.kind.is_single() {
+            return candidates
+                .healthy
+                .first()
+                .cloned()
+                .ok_or_else(no_deployment);
+        }
+        let (strategy, scope) = routing_context(
+            &snapshot.registry,
+            &snapshot.settings.routing_strategy,
+            request.routing_strategy.as_deref(),
+            &candidates.kind,
+            &candidates.model_group,
+        );
+        if strategy == RoutingStrategy::Unset {
+            return Err(no_deployment());
+        }
+        if strategy == RoutingStrategy::SimpleShuffle {
+            let weights = crate::weights::validate(request.router_weights.as_ref())
+                .map_err(|_| Error::InvalidRouterWeights)?;
+            return simple_shuffle(
+                &candidates.healthy,
+                &candidates.model_group,
+                weights.as_ref(),
+                self.rng.as_ref(),
+            )?
+            .ok_or_else(no_deployment);
+        }
+        let ctx = self.context(
+            snapshot,
+            op,
+            request,
+            &candidates.model_group,
+            &candidates.name,
+            scope,
+        );
+        self.host
+            .select(
+                &SelectStage::Strategy(strategy),
+                &ctx,
+                candidates.healthy.clone(),
+            )
+            .await?
+            .into_iter()
+            .next()
+            .ok_or_else(no_deployment)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+    use crate::registry::DeploymentSpec;
+
+    struct Fixed(f64);
+
+    impl Randomness for Fixed {
+        fn next_f64(&self) -> f64 {
+            self.0
+        }
+    }
+
+    fn deployment(id: &str, params: serde_json::Value) -> Arc<Deployment> {
+        let spec: DeploymentSpec = serde_json::from_value(json!({
+            "model_name": "g", "litellm_params": params, "model_info": {"id": id},
+        }))
+        .unwrap();
+        Arc::new(Deployment::from_spec(spec).unwrap())
+    }
+
+    #[rstest]
+    #[case::weight_low_draw(0.1, "a")]
+    #[case::weight_high_draw(0.9, "b")]
+    fn shuffle_uses_weight_before_rpm(#[case] draw: f64, #[case] expected: &str) {
+        let deployments = [
+            deployment("a", json!({"model": "openai/a", "weight": 1, "rpm": 1000})),
+            deployment("b", json!({"model": "openai/b", "weight": 3})),
+        ];
+        let picked = simple_shuffle(&deployments, "g", None, &Fixed(draw))
+            .unwrap()
+            .unwrap();
+        assert_eq!(picked.id, expected);
+    }
+
+    #[rstest]
+    fn shuffle_prefers_request_weights() {
+        let deployments = [
+            deployment("a", json!({"model": "openai/a", "weight": 100})),
+            deployment("b", json!({"model": "openai/b"})),
+        ];
+        let weights = BTreeMap::from([("g".to_owned(), BTreeMap::from([("b".to_owned(), 1.0)]))]);
+        let picked = simple_shuffle(&deployments, "g", Some(&weights), &Fixed(0.99))
+            .unwrap()
+            .unwrap();
+        assert_eq!(picked.id, "b");
+    }
+}

litellm-rust/crates/router-core/src/settings.rs

@@ -0,0 +1,103 @@
+//! [`Settings`]: one typed, frozen view of the configuration a caller may read, built from the
+//! static config and the current dynamic snapshot.
+
+use std::{collections::BTreeMap, sync::Arc, time::Duration};
+
+use crate::{
+    config::{
+        AllowedFailsPolicy, DEFAULT_COOLDOWN_TIME, PluginSpec, RetryPolicy, RouterConfig,
+        RouterDynamicConfig, RoutingStrategy,
+    },
+    error::AttemptError,
+};
+
+#[derive(Clone, Debug)]
+pub struct Settings {
+    pub routing_strategy: RoutingStrategy,
+    pub plugins: Arc<[PluginSpec]>,
+    pub cooldown_time: Duration,
+    pub disable_cooldowns: bool,
+    pub allowed_fails: Option<u32>,
+    pub allowed_fails_policy: Option<AllowedFailsPolicy>,
+    pub retry_policy: Option<RetryPolicy>,
+    pub model_group_retry_policy: Arc<BTreeMap<String, RetryPolicy>>,
+    pub num_retries: u32,
+    pub max_fallbacks: u32,
+    pub retry_after: Duration,
+    pub timeout: Option<Duration>,
+    pub drop_params: Option<bool>,
+    pub enable_pre_call_checks: bool,
+    pub routing_plugins: bool,
+    pub callback_filters: bool,
+    pub enable_health_check_routing: bool,
+    pub health_check_staleness: Duration,
+    pub background_health_check_model_groups: Option<Arc<[String]>>,
+    pub enable_weighted_failover: bool,
+    pub fallback_on_unsupported_params: bool,
+    pub provider_default_deployment_ids: Arc<[String]>,
+    pub dynamic: Arc<RouterDynamicConfig>,
+}
+
+/// `DEFAULT_HEALTH_CHECK_INTERVAL * DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER`.
+pub const DEFAULT_HEALTH_CHECK_STALENESS: Duration = Duration::from_secs(300 * 2);
+
+fn seconds(value: f64) -> Duration {
+    Duration::try_from_secs_f64(value).unwrap_or_default()
+}
+
+impl Settings {
+    pub fn new(config: &RouterConfig, dynamic: Arc<RouterDynamicConfig>) -> Self {
+        Self {
+            routing_strategy: config.routing_strategy.clone(),
+            plugins: config.plugins.clone().into(),
+            cooldown_time: config.cooldown_time.map_or(DEFAULT_COOLDOWN_TIME, seconds),
+            disable_cooldowns: config.disable_cooldowns,
+            allowed_fails: dynamic.allowed_fails,
+            allowed_fails_policy: config.allowed_fails_policy.clone(),
+            retry_policy: config.retry_policy.clone(),
+            model_group_retry_policy: Arc::new(config.model_group_retry_policy.clone()),
+            num_retries: dynamic.num_retries.unwrap_or(0),
+            max_fallbacks: dynamic.max_fallbacks,
+            retry_after: seconds(config.retry_after),
+            timeout: config.request_timeout.or(config.timeout).map(seconds),
+            drop_params: config.drop_params,
+            enable_pre_call_checks: config.enable_pre_call_checks,
+            routing_plugins: config.routing_plugins,
+            callback_filters: config.callback_filters,
+            enable_health_check_routing: config.enable_health_check_routing,
+            health_check_staleness: config
+                .health_check_staleness_threshold
+                .map_or(DEFAULT_HEALTH_CHECK_STALENESS, seconds),
+            background_health_check_model_groups: config
+                .background_health_check_model_groups
+                .clone()
+                .map(Into::into),
+            enable_weighted_failover: config.enable_weighted_failover,
+            fallback_on_unsupported_params: config.fallback_on_unsupported_params.unwrap_or(true),
+            provider_default_deployment_ids: config.provider_default_deployment_ids.clone().into(),
+            dynamic,
+        }
+    }
+
+    /// `allowed_fails_for`: the policy's count for this failure.
+    pub fn allowed_fails_for(&self, failure: &AttemptError) -> Option<u32> {
+        self.allowed_fails_policy
+            .as_ref()
+            .and_then(|policy| policy.allowed_fails_for(failure))
+    }
+
+    /// How long an allowed-fails counter lives: `settings().cooldown_time or
+    /// DEFAULT_COOLDOWN_TIME_SECONDS`, so a router cooldown of 0 still counts over the default.
+    pub fn fails_window(&self) -> Duration {
+        if self.cooldown_time.is_zero() {
+            DEFAULT_COOLDOWN_TIME
+        } else {
+            self.cooldown_time
+        }
+    }
+
+    /// Python's `allowed_fails` default when set without a value (`litellm.allowed_fails`).
+    pub fn effective_allowed_fails(&self) -> u32 {
+        self.allowed_fails.unwrap_or(3)
+    }
+}

litellm-rust/crates/router-core/src/strategy.rs

@@ -0,0 +1,110 @@
+//! Which routing strategy picks for a request, and whose state it reads: a port of
+//! `Router._get_routing_context`. A per-request `routing_strategy` override wins, then the
+//! routing group the resolved model belongs to (called as the group, or as a member of a
+//! non-priority group), then the router's own strategy.
+
+use crate::{
+    config::RoutingStrategy,
+    registry::{Registry, ResolveKind},
+};
+
+/// The per-request overrides `_OVERRIDABLE_ROUTING_STRATEGIES` honours: simple-shuffle and every
+/// strategy with a default selector. Anything else is ignored.
+pub const OVERRIDABLE: [&str; 6] = [
+    "simple-shuffle",
+    "least-busy",
+    "usage-based-routing",
+    "usage-based-routing-v2",
+    "latency-based-routing",
+    "cost-based-routing",
+];
+
+/// Whose selector state a strategy pick reads.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum StrategyScope {
+    /// The router's own strategy (the implicit `"default"` group).
+    Default,
+    /// A routing group's own selector.
+    Group(String),
+    /// A per-request override: the default selector when it names the router's strategy, else
+    /// one selector per strategy kept for the router's life.
+    Override,
+}
+
+impl StrategyScope {
+    /// The scope as the host step spells it.
+    pub fn wire(&self) -> (&'static str, Option<&str>) {
+        match self {
+            Self::Default => ("default", None),
+            Self::Group(name) => ("group", Some(name)),
+            Self::Override => ("override", None),
+        }
+    }
+}
+
+/// A request's `routing_strategy` override, when it is one Python honours.
+pub fn request_override(raw: Option<&str>) -> Option<RoutingStrategy> {
+    let raw = raw?;
+    OVERRIDABLE
+        .contains(&raw)
+        .then(|| raw.parse().unwrap_or_default())
+}
+
+/// The strategy and selector scope for a pick of `model_group` (the resolved group, after
+/// aliases), reached through a resolution of `kind`.
+pub fn routing_context(
+    registry: &Registry,
+    default: &RoutingStrategy,
+    override_strategy: Option<&str>,
+    kind: &ResolveKind,
+    model_group: &str,
+) -> (RoutingStrategy, StrategyScope) {
+    if let Some(strategy) = request_override(override_strategy) {
+        return (strategy, StrategyScope::Override);
+    }
+    let group = match kind {
+        ResolveKind::RoutingGroup => registry
+            .routing_groups()
+            .iter()
+            .find(|group| group.group_name == model_group),
+        ResolveKind::Alias | ResolveKind::Group => registry.member_routing_group(model_group),
+        ResolveKind::SpecificDeployment
+        | ResolveKind::Id
+        | ResolveKind::Default
+        | ResolveKind::Wildcard => None,
+    };
+    match group {
+        None => (default.clone(), StrategyScope::Default),
+        Some(group) if is_priority(&group.routing_strategy) => {
+            (RoutingStrategy::SimpleShuffle, StrategyScope::Default)
+        }
+        Some(group) => (
+            group.routing_strategy.clone(),
+            StrategyScope::Group(group.group_name.clone()),
+        ),
+    }
+}
+
+fn is_priority(strategy: &RoutingStrategy) -> bool {
+    matches!(strategy, RoutingStrategy::Other(name) if name == "priority")
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+
+    #[rstest]
+    #[case::honoured(Some("least-busy"), Some(RoutingStrategy::LeastBusy))]
+    #[case::shuffle(Some("simple-shuffle"), Some(RoutingStrategy::SimpleShuffle))]
+    #[case::lar1_ignored(Some("lar1"), None)]
+    #[case::unknown_ignored(Some("bogus"), None)]
+    #[case::unset(None, None)]
+    fn only_overridable_strategies_override(
+        #[case] raw: Option<&str>,
+        #[case] expected: Option<RoutingStrategy>,
+    ) {
+        assert_eq!(request_override(raw), expected);
+    }
+}

litellm-rust/crates/router-core/src/weights.rs

@@ -0,0 +1,81 @@
+//! `_router_weights` as `validate_router_weights` reads it (`litellm/types/router_weights.py`):
+//! model group to deployment id to a finite weight of at least 0 (an int or a float, never a
+//! bool or a string), with non-blank ids and at least one positive weight in every non-empty group.
+
+use std::collections::BTreeMap;
+
+use serde_json::Value;
+
+/// Validated request weights.
+pub type RouterWeights = BTreeMap<String, BTreeMap<String, f64>>;
+
+/// The request's weights didn't validate; the host raises the validator's own error.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct InvalidRouterWeights;
+
+fn identifier(text: &str) -> bool {
+    text.chars().any(|c| !c.is_whitespace())
+}
+
+fn weight(value: &Value) -> Option<f64> {
+    match value {
+        Value::Number(number) => number.as_f64().filter(|w| w.is_finite() && *w >= 0.0),
+        _ => None,
+    }
+}
+
+/// `validate_router_weights(raw)`: `None` when the request passed none.
+pub fn validate(raw: Option<&Value>) -> Result<Option<RouterWeights>, InvalidRouterWeights> {
+    let groups = match raw {
+        None | Some(Value::Null) => return Ok(None),
+        Some(Value::Object(groups)) => groups,
+        Some(_) => return Err(InvalidRouterWeights),
+    };
+    let mut validated = RouterWeights::new();
+    for (group, weights) in groups {
+        let Value::Object(weights) = weights else {
+            return Err(InvalidRouterWeights);
+        };
+        if !identifier(group) {
+            return Err(InvalidRouterWeights);
+        }
+        let mut entries = BTreeMap::new();
+        for (id, value) in weights {
+            if !identifier(id) {
+                return Err(InvalidRouterWeights);
+            }
+            entries.insert(id.clone(), weight(value).ok_or(InvalidRouterWeights)?);
+        }
+        if !entries.is_empty() && !entries.values().any(|w| *w > 0.0) {
+            return Err(InvalidRouterWeights);
+        }
+        validated.insert(group.clone(), entries);
+    }
+    Ok(Some(validated))
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+
+    #[rstest]
+    #[case::all_zero(json!({"g": {"a": 0.0, "b": 0.0}}))]
+    #[case::negative(json!({"g": {"a": -1.0}}))]
+    #[case::text(json!({"g": {"a": "1"}}))]
+    #[case::boolean(json!({"g": {"a": true}}))]
+    #[case::blank_id(json!({"g": {" ": 1.0}}))]
+    #[case::not_a_mapping(json!(["g"]))]
+    fn invalid_weights_are_refused(#[case] raw: Value) {
+        assert_eq!(validate(Some(&raw)), Err(InvalidRouterWeights));
+    }
+
+    #[rstest]
+    #[case::empty_group(json!({"g": {}}))]
+    #[case::int_weight(json!({"g": {"a": 1, "b": 0}}))]
+    fn valid_weights_are_kept(#[case] raw: Value) {
+        assert!(validate(Some(&raw)).unwrap().is_some());
+    }
+}

litellm-rust/crates/router-core/tests/call_loop.rs

@@ -0,0 +1,341 @@
+//! A call through the router end to end: selection, retries, fallbacks and cooldowns against a
+//! scripted host.
+
+use std::{collections::VecDeque, sync::Arc, time::Duration};
+
+use async_trait::async_trait;
+use litellm_router_core::{
+    Attempt, AttemptError, Deployment, Error, FailureKind, Host, InMemoryRouterCache, Op, Output,
+    Payload, Router, RouterCache, RouterConfig, RouterDynamicConfig, RouterEvent, RouterRequest,
+    RoutingContext, SeededRandomness, SelectStage,
+};
+use parking_lot::Mutex;
+use rstest::{fixture, rstest};
+use serde_json::json;
+
+#[derive(Default)]
+struct ScriptedHost {
+    /// Failures handed out in order, keyed by model group; a group with none left succeeds.
+    script: Mutex<Vec<(String, VecDeque<AttemptError>)>>,
+    events: Mutex<Vec<RouterEvent>>,
+    cache: InMemoryRouterCache,
+}
+
+impl ScriptedHost {
+    fn failing(group: &str, failures: Vec<AttemptError>) -> Self {
+        Self {
+            script: Mutex::new(vec![(group.to_owned(), failures.into())]),
+            ..Self::default()
+        }
+    }
+}
+
+#[async_trait]
+impl Host for ScriptedHost {
+    async fn invoke(&self, attempt: &Attempt) -> Result<Output, AttemptError> {
+        let next = self
+            .script
+            .lock()
+            .iter_mut()
+            .find(|(group, _)| *group == attempt.model_group)
+            .and_then(|(_, failures)| failures.pop_front());
+        match next {
+            Some(failure) => Err(failure),
+            None => Ok(Output {
+                value: Payload(Arc::new(attempt.deployment.id.clone())),
+                total_tokens: 10,
+                blocked: None,
+            }),
+        }
+    }
+
+    async fn select(
+        &self,
+        _stage: &SelectStage,
+        _ctx: &RoutingContext<'_>,
+        candidates: Vec<Arc<Deployment>>,
+    ) -> Result<Vec<Arc<Deployment>>, Error> {
+        Ok(candidates)
+    }
+
+    async fn allow_fallback(&self, _request: &RouterRequest, _target: &str) -> bool {
+        true
+    }
+
+    async fn sleep(&self, _duration: Duration) {}
+
+    fn observe(&self, event: &RouterEvent) {
+        self.events.lock().push(event.clone());
+    }
+
+    fn cache(&self) -> Arc<dyn RouterCache> {
+        Arc::new(self.cache.clone())
+    }
+}
+
+#[fixture]
+fn config() -> RouterConfig {
+    serde_json::from_value(json!({
+        "model_list": [
+            {"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}},
+            {"model_name": "primary", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "a2"}},
+            {"model_name": "backup", "litellm_params": {"model": "anthropic/c"}, "model_info": {"id": "b1"}},
+        ]
+    }))
+    .unwrap()
+}
+
+fn router(config: RouterConfig, dynamic: serde_json::Value, host: Arc<ScriptedHost>) -> Router {
+    let dynamic: RouterDynamicConfig = serde_json::from_value(dynamic).unwrap();
+    Router::builder(config, dynamic, host)
+        .randomness(Arc::new(SeededRandomness::new(7)))
+        .build()
+        .unwrap()
+}
+
+fn request() -> RouterRequest {
+    RouterRequest {
+        model: "primary".to_owned(),
+        ..RouterRequest::default()
+    }
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_rate_limit_retries_in_the_same_group(config: RouterConfig) {
+    let host = Arc::new(ScriptedHost::failing(
+        "primary",
+        vec![AttemptError::new(FailureKind::RateLimit, Some(429))],
+    ));
+    let router = router(config, json!({"num_retries": 2}), host.clone());
+    let response = router.call(Op::Completion, request()).await.unwrap();
+    assert_eq!(
+        (
+            response.model_group.as_str(),
+            response.attempted_retries,
+            response.attempted_fallbacks
+        ),
+        ("primary", 1, 0)
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_bad_request_falls_back_without_retrying(config: RouterConfig) {
+    let host = Arc::new(ScriptedHost::failing(
+        "primary",
+        vec![AttemptError::new(FailureKind::BadRequest, Some(400))],
+    ));
+    let router = router(
+        config,
+        json!({"num_retries": 3, "fallbacks": [{"primary": ["backup"]}]}),
+        host.clone(),
+    );
+    let response = router.call(Op::Completion, request()).await.unwrap();
+    let retried = host
+        .events
+        .lock()
+        .iter()
+        .any(|event| matches!(event, RouterEvent::Retry { .. }));
+    assert_eq!(
+        (
+            response.deployment_id.as_str(),
+            response.attempted_fallbacks,
+            retried
+        ),
+        ("b1", 1, false)
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_rate_limit_cools_the_deployment_down(config: RouterConfig) {
+    let host = Arc::new(ScriptedHost::failing(
+        "primary",
+        vec![AttemptError::new(FailureKind::RateLimit, Some(429))],
+    ));
+    let router = router(config, json!({}), host.clone());
+    let error = router.call(Op::Completion, request()).await.unwrap_err();
+    let cooled = router
+        .health()
+        .active_cooldowns(&["a1".to_owned(), "a2".to_owned()])
+        .await
+        .unwrap();
+    assert_eq!(
+        (error.attempt().map(|failure| failure.kind), cooled.len()),
+        (Some(FailureKind::RateLimit), 1)
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_raising_retry_policy_lookup_ends_the_hop_and_falls_back(config: RouterConfig) {
+    let host = Arc::new(ScriptedHost::failing(
+        "primary",
+        vec![AttemptError::new(FailureKind::RateLimit, Some(429))],
+    ));
+    let router = router(
+        config,
+        json!({"num_retries": 3, "fallbacks": [{"primary": ["backup"]}]}),
+        host.clone(),
+    );
+    let request = RouterRequest {
+        retry_policy_raises: vec!["primary".to_owned()],
+        ..request()
+    };
+    let response = router.call(Op::Completion, request).await.unwrap();
+    let events = host.events.lock();
+    assert!(
+        !events
+            .iter()
+            .any(|e| matches!(e, RouterEvent::Retry { .. }))
+    );
+    assert!(events.iter().any(|e| matches!(
+        e,
+        RouterEvent::RetryPolicyRaised { level: 0, model, .. } if model == "primary"
+    )));
+    assert_eq!(response.model_group, "backup");
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_name_nothing_serves_routes_to_the_default_deployment(config: RouterConfig) {
+    let host = Arc::new(ScriptedHost::default());
+    let router = router(
+        config,
+        json!({"default_deployment": {
+            "model_name": "dflt", "litellm_params": {"model": "openai/x"}, "model_info": {"id": "dd"}
+        }}),
+        host.clone(),
+    );
+    let request = RouterRequest {
+        model: "other".to_owned(),
+        ..RouterRequest::default()
+    };
+    let response = router.call(Op::Completion, request.clone()).await.unwrap();
+    assert_eq!(
+        (
+            response.deployment_id.as_str(),
+            response.model_group.as_str()
+        ),
+        ("dd", "other")
+    );
+    let picked = router.select(&Op::Completion, &request).await.unwrap();
+    assert_eq!(picked.params.model, "other");
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_stale_member_map_naming_a_dropped_group_raises_on_every_pick(mut config: RouterConfig) {
+    config.routing_group_members = Some([("primary".to_owned(), "gone".to_owned())].into());
+    let router = router(config, json!({}), Arc::new(ScriptedHost::default()));
+    let refused = router.select(&Op::Completion, &request()).await;
+    assert!(matches!(refused, Err(Error::MissingRoutingGroup { group }) if group == "gone"));
+    let other = RouterRequest {
+        model: "backup".to_owned(),
+        ..RouterRequest::default()
+    };
+    assert!(router.select(&Op::Completion, &other).await.is_ok());
+}
+
+#[rstest]
+#[tokio::test]
+async fn written_indexes_replace_the_ones_the_deployments_build(config: RouterConfig) {
+    let router = router(config, json!({}), Arc::new(ScriptedHost::default()));
+    let index: litellm_router_core::registry::IndexOverrides = serde_json::from_value(json!({
+        "model_names": ["backup"],
+        "model_name": {"primary": [1], "backup": [2]},
+    }))
+    .unwrap();
+    router
+        .apply(litellm_router_core::ChangeSet {
+            index: Some(index),
+            ..litellm_router_core::ChangeSet::default()
+        })
+        .unwrap();
+    for _ in 0..4 {
+        let picked = router.select(&Op::Completion, &request()).await.unwrap();
+        assert_eq!(picked.id, "a2");
+    }
+    router
+        .apply(litellm_router_core::ChangeSet {
+            index: Some(litellm_router_core::registry::IndexOverrides::default()),
+            ..litellm_router_core::ChangeSet::default()
+        })
+        .unwrap();
+    assert_eq!(router.registry().group("primary").unwrap().len(), 2);
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_retry_policy_retries_a_pick_that_found_no_deployment(config: RouterConfig) {
+    let host = Arc::new(ScriptedHost::default());
+    let mut config = config;
+    config.retry_policy = serde_json::from_value(json!({"DefaultRetries": 2})).unwrap();
+    let router = router(config, json!({"num_retries": 0}), host.clone());
+    let request = RouterRequest {
+        model: "nothing".to_owned(),
+        ..RouterRequest::default()
+    };
+    let refused = router.call(Op::Completion, request).await;
+    assert!(matches!(refused, Err(Error::UnknownModel { .. })));
+    let events = host.events.lock();
+    let retries = events
+        .iter()
+        .filter(|event| matches!(event, RouterEvent::Retry { .. }))
+        .count();
+    assert_eq!(retries, 2);
+    assert!(events.iter().any(|event| matches!(
+        event,
+        RouterEvent::RetriesExhausted {
+            picked: true,
+            level: 0,
+            ..
+        }
+    )));
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_default_deployment_python_cannot_copy_raises_for_a_name_nothing_serves(
+    config: RouterConfig,
+) {
+    let router = router(
+        config,
+        json!({"default_deployment_raises": true}),
+        Arc::new(ScriptedHost::default()),
+    );
+    let request = RouterRequest {
+        model: "other".to_owned(),
+        ..RouterRequest::default()
+    };
+    let refused = router.select(&Op::Completion, &request).await;
+    assert!(matches!(refused, Err(Error::DefaultDeploymentRaises { model }) if model == "other"));
+    assert!(
+        router
+            .select(&Op::Completion, &self::request())
+            .await
+            .is_ok()
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_written_index_position_past_the_list_raises_for_that_name(config: RouterConfig) {
+    let router = router(config, json!({}), Arc::new(ScriptedHost::default()));
+    let index: litellm_router_core::registry::IndexOverrides =
+        serde_json::from_value(json!({"model_name": {"primary": [0, 9], "backup": [2]}})).unwrap();
+    router
+        .apply(litellm_router_core::ChangeSet {
+            index: Some(index),
+            ..litellm_router_core::ChangeSet::default()
+        })
+        .unwrap();
+    let refused = router.select(&Op::Completion, &request()).await;
+    assert!(matches!(refused, Err(Error::IndexOutOfRange { model }) if model == "primary"));
+    let backup = RouterRequest {
+        model: "backup".to_owned(),
+        ..RouterRequest::default()
+    };
+    assert!(router.select(&Op::Completion, &backup).await.is_ok());
+}

litellm/router_api.py

@@ -45,6 +45,7 @@ from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, overload
 from litellm.router_config import detached
 
 if TYPE_CHECKING:
+    from functools import _lru_cache_wrapper
     from typing import Any  # noqa: TID251  # only to spell the cores' untyped (Unknown) containers and one Any type
 
     from opentelemetry.trace import Span
@@ -178,9 +179,9 @@ def deployment_usable_by_team(deployment: "Mapping[str, object] | Deployment", t
     Whether ``team_id`` may use ``deployment``: a team-scoped deployment (``model_info.team_id``
     set) only by that team, any other deployment by everyone. Needs no router instance.
     """
-    from litellm.router_python_core import PythonCore
+    from litellm.router_rust_registry import HostRegistry
 
-    return PythonCore._core_deployment_usable_by_team(deployment, team_id)
+    return HostRegistry._core_deployment_usable_by_team(deployment, team_id)
 
 
 def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str:
@@ -200,11 +201,77 @@ def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str
     return hashlib.sha256("".join(parts).encode()).hexdigest()
 
 
+# The arguments ``Router.__init__`` accepts, in its signature's order. Every core reads this list
+# (the Python core's ``get_valid_args`` reads its constructor's signature, and the core suite checks
+# that the two agree).
+ROUTER_INIT_ARGS: Final[tuple[str, ...]] = (
+    "model_list",
+    "assistants_config",
+    "search_tools",
+    "guardrail_list",
+    "redis_url",
+    "redis_host",
+    "redis_port",
+    "redis_password",
+    "redis_db",
+    "cache_responses",
+    "cache_kwargs",
+    "caching_groups",
+    "client_ttl",
+    "polling_interval",
+    "default_priority",
+    "num_retries",
+    "max_fallbacks",
+    "timeout",
+    "stream_timeout",
+    "default_litellm_params",
+    "default_max_parallel_requests",
+    "set_verbose",
+    "debug_level",
+    "default_fallbacks",
+    "fallbacks",
+    "context_window_fallbacks",
+    "content_policy_fallbacks",
+    "model_group_alias",
+    "enable_pre_call_checks",
+    "enable_tag_filtering",
+    "tag_filtering_match_any",
+    "tag_routing_prefix",
+    "plugins",
+    "retry_after",
+    "retry_policy",
+    "model_group_retry_policy",
+    "allowed_fails",
+    "allowed_fails_policy",
+    "cooldown_time",
+    "disable_cooldowns",
+    "fallback_on_unsupported_params",
+    "routing_strategy",
+    "optional_pre_call_checks",
+    "routing_strategy_args",
+    "routing_groups",
+    "provider_budget_config",
+    "alerting_config",
+    "router_general_settings",
+    "deployment_affinity_ttl_seconds",
+    "model_group_affinity_config",
+    "ignore_invalid_deployments",
+    "enable_health_check_routing",
+    "health_check_staleness_threshold",
+    "health_check_ignore_transient_errors",
+    "background_health_check_model_groups",
+    "enable_weighted_failover",
+    "enable_mid_stream_fallback_continuation",
+    "fallback_access_check",
+    "fallback_budget_check",
+    "auto_router_capability_limit",
+    "session_affinity_repin_on_failover",
+)
+
+
 def router_init_args() -> tuple[str, ...]:
     """The arguments ``Router.__init__`` accepts (was ``Router.get_valid_args``). Needs no router instance."""
-    from litellm.router_python_core import PythonCore
-
-    return tuple(PythonCore.get_valid_args())
+    return ROUTER_INIT_ARGS
 
 
 def mark_advisor_orchestration_failure(exception: BaseException) -> None:
@@ -268,9 +335,9 @@ def deployment_metadata(request_kwargs: Mapping[str, object]) -> Mapping[str, ob
 
 
 def _json_default_stable_id(obj: object) -> str:
-    from litellm.router_python_core import PythonCore
+    from litellm.router_rust_prerouting import _json_default_stable_id as stable_id
 
-    return PythonCore._json_default_stable_id(obj)
+    return stable_id(obj)
 
 
 class RouterRegistry:
@@ -315,6 +382,13 @@ class RouterRegistry:
             return self._router._core_cached_deployment_model_info(id, model_name)
         return self._router._core_get_deployment_model_info(model_id=id, model_name=model_name)
 
+    def deployment_model_info_cache(self) -> "_lru_cache_wrapper[ModelMapInfo | None]":
+        """
+        The lru cache ``deployment_model_info(..., cached=True)`` reads, called as ``(id, model_name)``
+        and cleared with ``cache_clear()`` (was the ``cached_deployment_model_info`` attribute).
+        """
+        return self._router._core_cached_deployment_model_info
+
     def router_model_info(
         self,
         deployment: "dict[str, object] | Deployment | None",  # mutable-ok: the core takes a dict
@@ -531,10 +605,8 @@ class RouterRegistry:
         ``should_include_deployment``), or, without ``model_name``, whether the team may use it at
         all (was ``deployment_usable_by_team``).
         """
-        from litellm.router_python_core import PythonCore
-
         if model_name is None:
-            return PythonCore._core_deployment_usable_by_team(deployment, team_id)
+            return deployment_usable_by_team(deployment, team_id)
         return self._router._core_should_include_deployment(model_name=model_name, model=deployment, team_id=team_id)
 
     @overload
@@ -1176,16 +1248,16 @@ class RouterHealth:
     @staticmethod
     def is_blocked(deployment: "Deployment") -> bool:
         """Whether an admin paused this deployment (was ``is_deployment_blocked``)."""
-        from litellm.router_python_core import PythonCore
+        from litellm.router_rust_health_routing import is_blocked as deployment_is_blocked
 
-        return PythonCore._core_is_deployment_blocked(deployment)
+        return deployment_is_blocked(deployment)
 
     @staticmethod
     def all_blocked(deployments: "Sequence[DeploymentTypedDict]") -> bool:
         """Whether every deployment in the list is paused (was ``are_all_deployments_blocked``)."""
-        from litellm.router_python_core import PythonCore
+        from litellm.router_rust_health_routing import all_blocked as deployments_all_blocked
 
-        return PythonCore._core_are_all_deployments_blocked(deployments)
+        return deployments_all_blocked(deployments)
 
     def fully_blocked_model_names(self) -> set[str]:  # mutable-ok: the core's type, a fresh set
         """Groups whose every deployment is paused (was ``get_fully_blocked_model_names``)."""

litellm/router_attempt.py

@@ -0,0 +1,208 @@
+"""
+One router attempt, as the host runs it: the request for the deployment the router core picked.
+
+The Rust router core picks a deployment and hands the attempt to its host; ``PythonHost`` builds
+the keyword arguments for that one deployment here and calls ``litellm.<op>`` with them. Nothing
+here selects, retries, falls back or records cooldowns: the core does all of that. Nothing here
+reads a Python router either, so this module outlives ``router_python_core.py``.
+"""
+
+import re
+import uuid
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final
+
+from litellm.litellm_core_utils.batch_utils import get_router_metadata_variable_name
+from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
+from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
+from litellm.litellm_core_utils.routing_common_utils import without_router_only_kwargs
+
+# Request keys the router reads to route the call; the deployment call never receives them.
+ROUTING_KEYS: Final = frozenset(
+    {
+        "model",
+        "num_retries",
+        "disable_fallbacks",
+        "specific_deployment",
+        "_router_weights",
+        "_retry_skipped_deployment_ids",
+        "_target_order",
+        "model_group_retry_policy",
+        "fallbacks",
+        "context_window_fallbacks",
+        "content_policy_fallbacks",
+        "mock_testing_fallbacks",
+        "mock_testing_context_fallbacks",
+        "mock_testing_content_policy_fallbacks",
+        "mock_testing_rate_limit_error",
+    }
+)
+
+# The positional parameters of each endpoint, in litellm's order: a call that passes them
+# positionally is read back into keywords before it is routed.
+POSITIONAL: Final = MappingProxyType(
+    {
+        "acompletion": ("model", "messages"),
+        "completion": ("model", "messages"),
+        "atext_completion": ("model", "prompt"),
+        "text_completion": ("model", "prompt"),
+        "aembedding": ("model", "input"),
+        "embedding": ("model", "input"),
+        "aimage_generation": ("prompt", "model"),
+        "image_generation": ("prompt", "model"),
+        "atranscription": ("model", "file"),
+        "aspeech": ("model", "input", "voice"),
+        "arerank": ("model", "query", "documents"),
+        "amoderation": ("input", "model"),
+        "aresponses": ("input", "model"),
+        "aanthropic_messages": (),
+    }
+)
+
+
+def request_from_call(op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> Mapping[str, object]:
+    """The call's arguments as one keyword request. Raises ``TypeError`` like the endpoint would."""
+    names: Final = POSITIONAL.get(op, ())
+    if len(args) > len(names):
+        raise TypeError(f"{op}() takes {len(names)} positional arguments but {len(args)} were given")
+    request: Final = dict(kwargs)
+    for name, value in zip(names, args, strict=False):
+        if name in request:
+            raise TypeError(f"{op}() got multiple values for argument {name!r}")
+        request[name] = value
+    return request
+
+
+# Endpoints the Python router serves through ``_ageneric_api_call_with_fallbacks``, which writes
+# the routing metadata under ``litellm_metadata``.
+_GENERIC_API_OPS: Final = frozenset({"aanthropic_messages", "aresponses"})
+
+
+def metadata_field(op: str) -> str:
+    """``metadata`` or ``litellm_metadata``: where the router writes what it routed the call as."""
+    return "litellm_metadata" if op in _GENERIC_API_OPS else get_router_metadata_variable_name(function_name=op)
+
+
+def prepare_call(op: str, request: Mapping[str, object], model_group_alias: str | None) -> Mapping[str, object]:
+    """What the router adds to a request once, before its first attempt
+    (``_update_kwargs_before_fallbacks``): a trace id and the model group it was asked for."""
+    field: Final = metadata_field(op)
+    metadata: Final = dict(_as_mapping(request.get(field)))
+    metadata.update({"model_group": request.get("model"), "model_group_alias": model_group_alias})
+    return {**request, "litellm_trace_id": request.get("litellm_trace_id") or str(uuid.uuid4()), field: metadata}
+
+
+def for_requested_model(deployment: Mapping[str, object], requested: str | None) -> Mapping[str, object]:
+    """A wildcard deployment as the router serves ``requested`` through it: its ``model`` with
+    the wildcards filled in from the requested name (``PatternMatchRouter.route``). Any other
+    deployment comes back unchanged."""
+    model_name: Final = deployment.get("model_name")
+    params: Final = _as_mapping(deployment.get("litellm_params"))
+    model: Final = params.get("model")
+    if requested is None or not isinstance(model_name, str) or "*" not in model_name or not isinstance(model, str):
+        return deployment
+    regex: Final = PatternMatchRouter().pattern_to_regex(model_name)
+    match: Final = re.match(regex, requested) or re.match(regex, f"{inferred_provider(requested)}/{requested}")
+    if match is None:
+        return deployment
+    return {
+        **deployment,
+        "litellm_params": {**params, "model": PatternMatchRouter.set_deployment_model_name(match, model)},
+    }
+
+
+def attempt_kwargs(
+    op: str,
+    request: Mapping[str, object],
+    deployment: Mapping[str, object],
+    *,
+    model_group: str,
+    default_litellm_params: Mapping[str, object],
+    timeout: float | None,
+) -> Mapping[str, object]:
+    """The keyword arguments for ``litellm.<op>`` against one deployment
+    (``_update_kwargs_with_deployment`` and the call sites that merge ``litellm_params``)."""
+    params: Final = _as_mapping(deployment.get("litellm_params"))
+    model_info: Final = _model_info_for(request, deployment, model_group)
+    field: Final = metadata_field(op)
+    metadata: Final = dict(_as_mapping(request.get(field)))
+    metadata.update(
+        {
+            "model_group": model_group,
+            "deployment": params.get("model"),
+            "model_info": model_info,
+            "api_base": params.get("api_base"),
+            "deployment_model_name": deployment.get("model_name"),
+        }
+    )
+    tags: Final = _as_list(params.get("tags"))
+    if tags:
+        merged: Final = list(_as_list(metadata.get("tags")))
+        merged.extend(tag for tag in tags if tag not in merged)
+        metadata["tags"] = merged
+    call: Final = {key: value for key, value in request.items() if key not in ROUTING_KEYS}
+    for key, value in default_litellm_params.items():
+        call.setdefault(key, value)
+    call[field] = metadata
+    call["model_info"] = model_info
+    if "timeout" not in call:
+        deployment_timeout: Final = params.get("timeout")
+        chosen: Final = deployment_timeout if deployment_timeout is not None else timeout
+        if chosen is not None:
+            call["timeout"] = chosen
+    return without_router_only_kwargs({**params, **call, "model": params.get("model")})
+
+
+def _model_info_for(
+    request: Mapping[str, object], deployment: Mapping[str, object], model_group: str
+) -> dict[str, object]:  # mutable-ok: a fresh dict for the call
+    """The deployment's ``model_info``, or, for a request carrying its own credentials, the
+    request-scoped deployment's (``_handle_clientside_credential``): its own stable id, and the
+    configured deployment's id as ``original_model_id``."""
+    from litellm.litellm_core_utils import clientside_credential_handler as clientside
+    from litellm.router_api import deployment_id
+
+    model_info: Final = dict(_as_mapping(deployment.get("model_info")))
+    if not clientside.is_clientside_credential(request_kwargs=dict(request)):  # pyright: ignore[reportUnknownMemberType]  # an untyped helper
+        return model_info
+    dynamic: Final = _as_mapping(
+        clientside.get_dynamic_litellm_params(  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # an untyped helper
+            litellm_params=dict(_as_mapping(deployment.get("litellm_params"))), request_kwargs=dict(request)
+        )
+    )
+    original: Final = model_info.get("id")
+    model_info["id"] = deployment_id(model_group, dynamic)
+    model_info["original_model_id"] = original
+    return model_info
+
+
+def stamp_attempt_error(error: BaseException, deployment: Mapping[str, object]) -> None:
+    """What the router stamps on a failed attempt's error: the deployment's own ``num_retries``
+    (``_set_deployment_num_retries_on_exception``) and its id (``failed_deployment_id``)."""
+    params: Final = _as_mapping(deployment.get("litellm_params"))
+    if getattr(error, "num_retries", None) is None and params.get("num_retries") is not None:
+        try:
+            error.num_retries = int(str(params["num_retries"]))  # pyright: ignore[reportAttributeAccessIssue]  # the router stamps the same attribute  # rebind-ok: the router stamps the error in place
+        except ValueError:
+            pass
+    deployment_id: Final = _as_mapping(deployment.get("model_info")).get("id")
+    if deployment_id and not getattr(error, "failed_deployment_id", None):
+        try:
+            error.failed_deployment_id = deployment_id  # pyright: ignore[reportAttributeAccessIssue]  # the router stamps the same attribute  # rebind-ok: see above
+        except AttributeError:
+            pass
+
+
+def _as_mapping(value: object) -> Mapping[str, object]:
+    if isinstance(value, Mapping):
+        entries: Final[Mapping[object, object]] = value  # pyright: ignore[reportUnknownVariableType]  # a request's untyped mapping
+        return {str(key): item for key, item in entries.items()}
+    return MappingProxyType({})
+
+
+def _as_list(value: object) -> list[object]:  # mutable-ok: a fresh list
+    if isinstance(value, list):
+        items: Final[list[object]] = value  # pyright: ignore[reportUnknownVariableType]  # a request's untyped list  # mutable-ok: read, then copied
+        return list(items)
+    return []

litellm/router_core.py

@@ -1,14 +1,16 @@
 """
 ``RouterCore``: the interface ``litellm.Router`` delegates to, and ``make_core``, which picks one.
 
-``PythonCore`` (``litellm/router_python_core.py``) is today's routing logic. The Rust core
-(litellm-rust/crates/router-core) implements the same operations. The core is picked once, when
-the router is built, from its ``RouterConfig``.
+``PythonCore`` (``litellm/router_python_core.py``) is today's routing logic. ``RustCore``
+(``litellm/router_rust_core.py`` over litellm-rust/crates/router-core) is a separate implementation
+that reaches outside itself only through its host; it does not use ``PythonCore``. The core is
+picked once, when the router is built, from its ``RouterConfig``.
 """
 
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Protocol, runtime_checkable
+from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable
 
+from litellm._logging import verbose_router_logger
 from litellm.router_config import RouterConfig
 
 if TYPE_CHECKING:
@@ -51,6 +53,26 @@ class RouterCore(Protocol):
 
 
 def make_core(config: RouterConfig, host: "PythonHost", core_class: "type[PythonCore] | None" = None) -> RouterCore:
+    """
+    The Rust core when the route catalog's router rule allows it and the Rust core serves this
+    config, else the Python core. Decided once, for the router's life, from the config alone: the
+    Rust core never builds, wraps or falls back to a Python core.
+    """
     from litellm.router_python_core import PythonCore
+    from litellm.rust_bridge.catalog import Route, RouteContext, decision
+    from litellm.rust_bridge.configuration import Decision
 
-    return (core_class or PythonCore)(**config.arguments, _host=host)  # pyright: ignore[reportArgumentType]  # the arguments were bound against PythonCore's own signature
+    python_class: Final = core_class or PythonCore
+    chosen: Final = decision(RouteContext(Route.ROUTER))
+    if chosen is not Decision.PYTHON:
+        from litellm.router_rust_core import RustCore
+        from litellm.rust_bridge._native import RustBridgeDeclined
+
+        try:
+            return RustCore.build(config, host)  # pyright: ignore[reportReturnType]  # RustCore serves the RouterCore entry points; other members raise RustCoreUnsupported
+        except RustBridgeDeclined as declined:
+            # A config the Rust core cannot take keeps the router on Python while both cores exist.
+            if chosen is Decision.RUST_REQUIRED:
+                raise
+            verbose_router_logger.info("Router served by the Python core: %s", declined)
+    return python_class(**config.arguments, _host=host)  # pyright: ignore[reportArgumentType]  # the arguments were bound against PythonCore's own signature

litellm/router_deprecated.py

@@ -5,8 +5,10 @@ DeprecatedRouterMembers: the Router members that outside code used before the en
 entry points (``registry()``, ``settings()``, ``health()``, ``select()``, ``apply()``,
 ``observe()``, ``register_plugin()``, ``retire()``; see litellm/router_api.py). Every member here
 returns what it returned before, with the type it had before, and names its replacement in a
-``DeprecationWarning``. Reads go to the same Router implementation (``Router._core_<name>``) the
-entry point uses; writes go through ``apply`` / ``aapply`` so the registry version moves.
+``DeprecationWarning``. Reads go through that entry point, so they work on either router core
+(Python or Rust) and, on the Python core, return the same ``_core_<name>`` object as before.
+Settings writes go through ``apply`` / ``aapply`` so the registry version moves; any other old
+attribute write sets the core's ``_core_<name>``, which each core keeps where its reads find it.
 
 Access from the router itself and from its own modules (``litellm.router*``) is not counted:
 those are the router's internals and move behind the core interface separately. Every other access
@@ -17,7 +19,7 @@ deleted (planned about a month after the core swap).
 # pyright: reportPrivateUsage=false, reportMissingTypeArgument=false, reportUnknownParameterType=false
 # pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false
 # pyright: reportUnknownLambdaType=false
-# This shim is part of the router (it reads Router's ``_core_*`` members, as Router's own methods do)
+# This shim is part of the router (it writes Router's ``_core_*`` members, as Router's own methods do)
 # and repeats each old member's signature exactly, untyped parts included (``dict``, ``list``), so
 # callers keep the types they had. Typing those parts is the core's job, not the shim's.
 
@@ -27,11 +29,11 @@ import inspect
 import sys
 import warnings
 from collections import Counter
-from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
+from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, Generic, TypeVar, overload
 
-from litellm.router_api import ASSIGNABLE_ALIASES, SETTING_FIELDS, ChangeSet
+from litellm.router_api import ASSIGNABLE_ALIASES, SETTING_FIELDS, ChangeSet, RoutingContext
 
 if TYPE_CHECKING:
     from typing import Any  # noqa: TID251  # only to repeat two old signatures that had Any, unchanged
@@ -39,6 +41,7 @@ if TYPE_CHECKING:
     from opentelemetry.trace import Span
 
     from litellm.caching.caching import RedisCache
+    from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
     from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling
     from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
@@ -129,6 +132,9 @@ class _DeprecatedAttribute(Generic[_T]):
         _note(self.name, self.replacement)
         router: Final = _as_router(obj)
         try:
+            if obj.__dict__.get("_core") is None:
+                # No core yet (a Router made without __init__): the value written through this name.
+                return getattr(obj, "_core_" + self.name)  # pyright: ignore[reportAny]  # the pending write, as given
             return self.read(router)
         except AttributeError as e:
             raise _public_attribute_error(router, "_core_" + self.name, self.name, e) from None
@@ -164,6 +170,55 @@ def _attr(replacement: str, read: Callable[["Router"], _T]) -> _DeprecatedAttrib
     return _DeprecatedAttribute(replacement, read)
 
 
+def _async_picker(router: "Router") -> Callable[..., Awaitable[object]]:
+    """
+    The router's async pick (the old ``async_get_available_deployment`` attribute). A core that
+    keeps its pick as a member (the Python core's, or one assigned through this attribute) returns
+    that member, so a patched pick reads back as patched; any other core picks through ``select``.
+    """
+    member: Final[object] = getattr(router, "_core_async_get_available_deployment", None)
+    if callable(member):
+        return member  # pyright: ignore[reportReturnType]  # the core's own async pick
+
+    async def async_get_available_deployment(
+        model: str,
+        request_kwargs: dict,  # mutable-ok: the old parameter type, unchanged
+        messages: list[dict[str, str]] | None = None,  # mutable-ok: the old parameter type, unchanged
+        input: str | list | None = None,  # mutable-ok: the old parameter type, unchanged
+        specific_deployment: bool | None = False,
+    ) -> object:
+        return await router.select(
+            model, request_kwargs, messages=messages, input=input, specific_deployment=specific_deployment
+        )
+
+    return async_get_available_deployment
+
+
+def _slack_alerting_logger(router: "Router") -> "SlackAlerting":
+    """``observe().slack_alerting_logger``; a router without one has no such attribute, as before."""
+    logger: Final = router.observe().slack_alerting_logger
+    if logger is None:
+        raise AttributeError(
+            f"'{type(router).__name__}' object has no attribute 'slack_alerting_logger'",
+            name="slack_alerting_logger",
+            obj=router,
+        )
+    return logger
+
+
+def _run_ready(step: Coroutine[object, object, _T]) -> _T:
+    """
+    The result of a coroutine that finishes without suspending: ``select(kind='pass_through')``
+    picks synchronously on every core, so the old synchronous member can return its result.
+    """
+    try:
+        step.send(None)
+    except StopIteration as done:
+        return done.value  # pyright: ignore[reportAny]  # the coroutine's own result
+    step.close()
+    raise RuntimeError("select(kind='pass_through') suspended; it must pick without awaiting")
+
+
 # (id(router), member name) pairs whose subclass override is running in this context.
 _OVERRIDES_RUNNING: Final[contextvars.ContextVar[frozenset[tuple[int, str]]]] = contextvars.ContextVar(
     "router_overrides_running", default=frozenset()
@@ -298,109 +353,112 @@ class DeprecatedRouterMembers:
                 setattr(cls, name, _enter_override(name, value))
 
     # -- registry: attributes --------------------------------------------------------------------
-    model_list = _attr("registry().all()", lambda r: r._core_model_list)
-    model_names = _attr("registry().names(kind='group_set')", lambda r: r._core_model_names)
-    deployment_names = _attr("registry().names(kind='deployment_names')", lambda r: r._core_deployment_names)
-    team_public_model_names = _attr("registry().names(kind='team_public')", lambda r: r._core_team_public_model_names)
+    model_list = _attr("registry().all()", lambda r: r.registry().all())
+    model_names = _attr("registry().names(kind='group_set')", lambda r: r.registry().names(kind="group_set"))
+    deployment_names = _attr(
+        "registry().names(kind='deployment_names')", lambda r: r.registry().names(kind="deployment_names")
+    )
+    team_public_model_names = _attr(
+        "registry().names(kind='team_public')", lambda r: r.registry().names(kind="team_public")
+    )
     model_name_to_deployment_indices = _attr(
-        "registry().index('model_name')", lambda r: r._core_model_name_to_deployment_indices
+        "registry().index('model_name')", lambda r: r.registry().index("model_name")
     )
     team_model_to_deployment_indices = _attr(
-        "registry().index('team_model')", lambda r: r._core_team_model_to_deployment_indices
+        "registry().index('team_model')", lambda r: r.registry().index("team_model")
     )
     cached_deployment_model_info = _attr(
         "registry().deployment_model_info(id, model_name, cached=True)",
-        lambda r: r._core_cached_deployment_model_info,
+        lambda r: r.registry().deployment_model_info_cache(),
     )
-    model_group_alias = _attr("registry().aliases()", lambda r: r._core_model_group_alias)
-    pattern_router = _attr("registry().patterns()", lambda r: r._core_pattern_router)
-    team_pattern_routers = _attr("registry().patterns(team=True)", lambda r: r._core_team_pattern_routers)
-    routing_groups = _attr("registry().routing_groups()", lambda r: r._core_routing_groups)
+    model_group_alias = _attr("registry().aliases()", lambda r: r.registry().aliases())
+    pattern_router = _attr("registry().patterns()", lambda r: r.registry().patterns())
+    team_pattern_routers = _attr("registry().patterns(team=True)", lambda r: r.registry().patterns(team=True))
+    routing_groups = _attr("registry().routing_groups()", lambda r: r.registry().routing_groups())
     _routing_groups = routing_groups
-    auto_routers = _attr("registry().plugins('auto')", lambda r: r._core_auto_routers)
-    complexity_routers = _attr("registry().plugins('complexity')", lambda r: r._core_complexity_routers)
-    quality_routers = _attr("registry().plugins('quality')", lambda r: r._core_quality_routers)
-    adaptive_routers = _attr("registry().plugins('adaptive')", lambda r: r._core_adaptive_routers)
-    routing_plugins = _attr("registry().plugins('routing')", lambda r: r._core_routing_plugins)
+    auto_routers = _attr("registry().plugins('auto')", lambda r: r.registry().plugins("auto"))
+    complexity_routers = _attr("registry().plugins('complexity')", lambda r: r.registry().plugins("complexity"))
+    quality_routers = _attr("registry().plugins('quality')", lambda r: r.registry().plugins("quality"))
+    adaptive_routers = _attr("registry().plugins('adaptive')", lambda r: r.registry().plugins("adaptive"))
+    routing_plugins = _attr("registry().plugins('routing')", lambda r: r.registry().plugins("routing"))
     prompt_caching_pin_cache = _attr(
-        "registry().plugins('prompt_caching_pins')", lambda r: r._core_prompt_caching_pin_cache
+        "registry().plugins('prompt_caching_pins')", lambda r: r.registry().plugins("prompt_caching_pins")
     )
 
     # -- health and selection: attributes --------------------------------------------------------
-    cache = _attr("health().store", lambda r: r._core_cache)
-    cooldown_cache = _attr("health().cooldowns", lambda r: r._core_cooldown_cache)
-    health_state_cache = _attr("health().health_checks", lambda r: r._core_health_state_cache)
-    deployment_latency_map = _attr("health().latency", lambda r: r._core_deployment_latency_map)
-    async_get_available_deployment = _attr(
-        "select(model, request_kwargs)",
-        lambda r: r._core_async_get_available_deployment,
-    )
+    cache = _attr("health().store", lambda r: r.health().store)
+    cooldown_cache = _attr("health().cooldowns", lambda r: r.health().cooldowns)
+    health_state_cache = _attr("health().health_checks", lambda r: r.health().health_checks)
+    deployment_latency_map = _attr("health().latency", lambda r: r.health().latency)
+    async_get_available_deployment = _attr("select(model, request_kwargs)", _async_picker)
 
     # -- observers ------------------------------------------------------------------------------
-    slack_alerting_logger = _attr("observe().slack_alerting_logger", lambda r: r._core_slack_alerting_logger)
+    slack_alerting_logger = _attr("observe().slack_alerting_logger", _slack_alerting_logger)
 
     # -- settings -------------------------------------------------------------------------------
-    routing_strategy = _setting("routing_strategy", lambda r: r._core_routing_strategy)
-    context_window_fallbacks = _setting("context_window_fallbacks", lambda r: r._core_context_window_fallbacks)
-    content_policy_fallbacks = _setting("content_policy_fallbacks", lambda r: r._core_content_policy_fallbacks)
-    default_deployment = _setting("default_deployment", lambda r: r._core_default_deployment)
-    default_litellm_params = _setting("default_litellm_params", lambda r: r._core_default_litellm_params)
+    routing_strategy = _setting("routing_strategy", lambda r: r.settings().routing_strategy)
+    context_window_fallbacks = _setting("context_window_fallbacks", lambda r: r.settings().context_window_fallbacks)
+    content_policy_fallbacks = _setting("content_policy_fallbacks", lambda r: r.settings().content_policy_fallbacks)
+    default_deployment = _setting("default_deployment", lambda r: r.settings().default_deployment)
+    default_litellm_params = _setting("default_litellm_params", lambda r: r.settings().default_litellm_params)
     default_max_parallel_requests = _setting(
-        "default_max_parallel_requests", lambda r: r._core_default_max_parallel_requests
+        "default_max_parallel_requests", lambda r: r.settings().default_max_parallel_requests
+    )
+    cooldown_time = _setting("cooldown_time", lambda r: r.settings().cooldown_time)
+    allowed_fails = _setting("allowed_fails", lambda r: r.settings().allowed_fails)
+    allowed_fails_policy = _setting("allowed_fails_policy", lambda r: r.settings().allowed_fails_policy)
+    disable_cooldowns = _setting("disable_cooldowns", lambda r: r.settings().disable_cooldowns)
+    enable_pre_call_checks = _setting("enable_pre_call_checks", lambda r: r.settings().enable_pre_call_checks)
+    enable_health_check_routing = _setting(
+        "enable_health_check_routing", lambda r: r.settings().enable_health_check_routing
     )
-    cooldown_time = _setting("cooldown_time", lambda r: r._core_cooldown_time)
-    allowed_fails = _setting("allowed_fails", lambda r: r._core_allowed_fails)
-    allowed_fails_policy = _setting("allowed_fails_policy", lambda r: r._core_allowed_fails_policy)
-    disable_cooldowns = _setting("disable_cooldowns", lambda r: r._core_disable_cooldowns)
-    enable_pre_call_checks = _setting("enable_pre_call_checks", lambda r: r._core_enable_pre_call_checks)
-    enable_health_check_routing = _setting("enable_health_check_routing", lambda r: r._core_enable_health_check_routing)
     health_check_ignore_transient_errors = _setting(
-        "health_check_ignore_transient_errors", lambda r: r._core_health_check_ignore_transient_errors
+        "health_check_ignore_transient_errors", lambda r: r.settings().health_check_ignore_transient_errors
     )
     background_health_check_model_groups = _setting(
-        "background_health_check_model_groups", lambda r: r._core_background_health_check_model_groups
+        "background_health_check_model_groups", lambda r: r.settings().background_health_check_model_groups
     )
-    enable_tag_filtering = _setting("enable_tag_filtering", lambda r: r._core_enable_tag_filtering)
-    tag_filtering_match_any = _setting("tag_filtering_match_any", lambda r: r._core_tag_filtering_match_any)
-    tag_routing_prefix = _setting("tag_routing_prefix", lambda r: r._core_tag_routing_prefix)
-    router_general_settings = _setting("router_general_settings", lambda r: r._core_router_general_settings)
-    provider_budget_config = _setting("provider_budget_config", lambda r: r._core_provider_budget_config)
+    enable_tag_filtering = _setting("enable_tag_filtering", lambda r: r.settings().enable_tag_filtering)
+    tag_filtering_match_any = _setting("tag_filtering_match_any", lambda r: r.settings().tag_filtering_match_any)
+    tag_routing_prefix = _setting("tag_routing_prefix", lambda r: r.settings().tag_routing_prefix)
+    router_general_settings = _setting("router_general_settings", lambda r: r.settings().router_general_settings)
+    provider_budget_config = _setting("provider_budget_config", lambda r: r.settings().provider_budget_config)
     provider_default_deployment_ids = _setting(
-        "provider_default_deployment_ids", lambda r: r._core_provider_default_deployment_ids
+        "provider_default_deployment_ids", lambda r: r.settings().provider_default_deployment_ids
     )
-    fallback_access_check = _setting("fallback_access_check", lambda r: r._core_fallback_access_check)
-    fallback_budget_check = _setting("fallback_budget_check", lambda r: r._core_fallback_budget_check)
+    fallback_access_check = _setting("fallback_access_check", lambda r: r.settings().fallback_access_check)
+    fallback_budget_check = _setting("fallback_budget_check", lambda r: r.settings().fallback_budget_check)
     # These three are declared with the attribute type Router gave them rather than the descriptor's:
     # protocols in router_utils and the proxy match Router on them, and protocol matching does not
     # see through a descriptor. Reads and writes still go through _setting at runtime.
     fallbacks: list | list[dict[str, list[str] | list | None]] | None = _setting(  # pyright: ignore[reportAssignmentType]  # see above  # mutable-ok: the old type
-        "fallbacks", lambda r: r._core_fallbacks
+        "fallbacks", lambda r: r.settings().fallbacks
     )
     kubernetes_pod_discovery: "KubernetesPodDiscovery" = _setting(  # pyright: ignore[reportAssignmentType]  # see above
-        "kubernetes_pod_discovery", lambda r: r._core_kubernetes_pod_discovery
+        "kubernetes_pod_discovery", lambda r: r.settings().kubernetes_pod_discovery
     )
     search_tools: "list[SearchToolTypedDict]" = _setting(  # pyright: ignore[reportAssignmentType]  # see above  # mutable-ok: the old type
-        "search_tools", lambda r: r._core_search_tools
+        "search_tools", lambda r: r.settings().search_tools
     )
-    guardrail_list = _setting("guardrail_list", lambda r: r._core_guardrail_list)
-    cache_responses = _setting("cache_responses", lambda r: r._core_cache_responses)
+    guardrail_list = _setting("guardrail_list", lambda r: r.settings().guardrail_list)
+    cache_responses = _setting("cache_responses", lambda r: r.settings().cache_responses)
 
     # -- registry: one deployment ---------------------------------------------------------------
     def get_deployment(self, model_id: str) -> "Deployment | None":
         _note("get_deployment", "registry().deployment(id)")
-        return _as_router(self)._core_get_deployment(model_id=model_id)
+        return _as_router(self).registry().deployment(model_id)
 
     def has_model_id(self, candidate_id: str) -> bool:
         _note("has_model_id", "registry().has_deployment(id)")
-        return _as_router(self)._core_has_model_id(candidate_id)
+        return _as_router(self).registry().has_deployment(candidate_id)
 
     def get_model_info(self, id: str) -> dict | None:  # mutable-ok: the old return type, unchanged
         _note("get_model_info", "registry().deployment_dict(id)")
-        return _as_router(self)._core_get_model_info(id=id)
+        return _as_router(self).registry().deployment_dict(id)
 
     def get_deployment_model_info(self, model_id: str, model_name: str) -> "ModelMapInfo | None":
         _note("get_deployment_model_info", "registry().deployment_model_info(id, model_name)")
-        return _as_router(self)._core_get_deployment_model_info(model_id=model_id, model_name=model_name)
+        return _as_router(self).registry().deployment_model_info(model_id, model_name)
 
     @overload
     def get_router_model_info(
@@ -420,62 +478,56 @@ class DeprecatedRouterMembers:
         id: str | None = None,
     ) -> "ModelMapInfo":
         _note("get_router_model_info", "registry().router_model_info(...)")
-        router: Final = _as_router(self)
-        # The core's overloads split (deployment, id) by which one is None; this forwards either form unchanged.
-        return router._core_get_router_model_info(  # pyright: ignore[reportCallIssue]  # see above
-            deployment=deployment,  # pyright: ignore[reportArgumentType]  # see above
-            received_model_name=received_model_name,
-            id=id,  # pyright: ignore[reportArgumentType]  # see above
-        )
+        return _as_router(self).registry().router_model_info(deployment, received_model_name, id)
 
     def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]:
         _note("get_discovered_model_info", "registry().discovered_model_info(id)")
-        return _as_router(self)._core_get_discovered_model_info(model_id)
+        return _as_router(self).registry().discovered_model_info(model_id)
 
     # -- registry: many deployments -------------------------------------------------------------
     def get_model_list(
         self, model_name: str | None = None, team_id: str | None = None
     ) -> "list[DeploymentTypedDict] | None":  # mutable-ok: the old return type, unchanged
         _note("get_model_list", "registry().deployments(model_name, team_id=...)")
-        return _as_router(self)._core_get_model_list(model_name=model_name, team_id=team_id)
+        return _as_router(self).registry().deployments(model_name, team_id=team_id)
 
     def get_all_deployments(
         self, model_name: str, model_alias: str | None = None, team_id: str | None = None
     ) -> "list[DeploymentTypedDict]":  # mutable-ok: the old return type, unchanged
         _note("get_all_deployments", "registry().deployments(model_name, source='all')")
-        return _as_router(self)._core_get_all_deployments(
-            model_name=model_name, model_alias=model_alias, team_id=team_id
+        return (
+            _as_router(self).registry().deployments(model_name, source="all", model_alias=model_alias, team_id=team_id)
         )
 
     _get_all_deployments = _private_alias("get_all_deployments", get_all_deployments)
 
     def config_deployments(self) -> Iterator[Mapping[str, object]]:
         _note("config_deployments", "registry().deployments(source='config')")
-        return _as_router(self)._core_config_deployments()
+        return _as_router(self).registry().deployments(source="config")
 
     def deployments_named(self, model_name: str) -> "tuple[DeploymentTypedDict, ...] | None":
         _note("deployments_named", "registry().deployments(model_name, source='named')")
-        return _as_router(self)._core_deployments_named(model_name)
+        return _as_router(self).registry().deployments(model_name, source="named")
 
     def get_model_list_from_model_alias(
         self, model_name: str | None = None
     ) -> "list[DeploymentTypedDict]":  # mutable-ok: the old return type, unchanged
         _note("get_model_list_from_model_alias", "registry().deployments(model_name, source='alias')")
-        return _as_router(self)._core_get_model_list_from_model_alias(model_name=model_name)
+        return _as_router(self).registry().deployments(model_name, source="alias")
 
     def get_model_list_from_routing_groups(self, model_name: str | None = None) -> "Sequence[DeploymentTypedDict]":
         _note("get_model_list_from_routing_groups", "registry().deployments(model_name, source='routing_groups')")
-        return _as_router(self)._core_get_model_list_from_routing_groups(model_name=model_name)
+        return _as_router(self).registry().deployments(model_name, source="routing_groups")
 
     def get_model_list_of_routed_group(
         self, model_group: str
     ) -> "list[DeploymentTypedDict]":  # mutable-ok: the old return type, unchanged
         _note("get_model_list_of_routed_group", "registry().deployments(model_group, source='routed_group')")
-        return _as_router(self)._core_get_model_list_of_routed_group(model_group)
+        return _as_router(self).registry().deployments(model_group, source="routed_group")
 
     def wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]:
         _note("wildcard_route_deployments", "registry().deployments(model, source='wildcard')")
-        return _as_router(self)._core_wildcard_route_deployments(model, team_id)
+        return _as_router(self).registry().deployments(model, source="wildcard", team_id=team_id)
 
     def get_routable_deployments(
         self,
@@ -486,17 +538,21 @@ class DeprecatedRouterMembers:
         _note(
             "get_routable_deployments", "registry().deployments(model, source='routable', request=..., call_type=...)"
         )
-        return _as_router(self)._core_get_routable_deployments(model, request_kwargs, call_type)
+        return (
+            _as_router(self)
+            .registry()
+            .deployments(model, source="routable", request=request_kwargs, call_type=call_type)
+        )
 
     def get_deployment_by_model_group_name(self, model_group_name: str) -> "Deployment | None":
         _note("get_deployment_by_model_group_name", "registry().deployments(name, source='first_in_group')")
-        return _as_router(self)._core_get_deployment_by_model_group_name(model_group_name=model_group_name)
+        return _as_router(self).registry().deployments(model_group_name, source="first_in_group")
 
     def deployments_for_request(
         self, model: str, request_kwargs: Mapping[str, object]
     ) -> "Sequence[DeploymentTypedDict]":
         _note("deployments_for_request", "registry().deployments(model, source='for_request', request=...)")
-        return _as_router(self)._core_deployments_for_request(model, request_kwargs)
+        return _as_router(self).registry().deployments(model, source="for_request", request=request_kwargs)
 
     def should_include_deployment(
         self,
@@ -505,92 +561,96 @@ class DeprecatedRouterMembers:
         team_id: str | None = None,
     ) -> bool:
         _note("should_include_deployment", "registry().includes(deployment, model_name=..., team_id=...)")
-        return _as_router(self)._core_should_include_deployment(model_name=model_name, model=model, team_id=team_id)
+        return _as_router(self).registry().includes(model, model_name=model_name, team_id=team_id)
 
     @staticmethod
     def deployment_usable_by_team(model: "Mapping[str, object] | Deployment", team_id: str | None) -> bool:
         _note("deployment_usable_by_team", "registry().includes(deployment, team_id=...)")
-        from litellm.router_python_core import PythonCore
+        from litellm.router_api import deployment_usable_by_team as usable_by_team
 
-        return PythonCore._core_deployment_usable_by_team(model, team_id)
+        return usable_by_team(model, team_id)
 
     _deployment_usable_by_team = deployment_usable_by_team
 
     # -- registry: names -------------------------------------------------------------------------
     def get_model_names(self, team_id: str | None = None) -> list[str]:  # mutable-ok: the old return type, unchanged
         _note("get_model_names", "registry().names(team_id=...)")
-        return _as_router(self)._core_get_model_names(team_id=team_id)
+        return _as_router(self).registry().names(team_id=team_id)
 
     def get_model_ids(
         self, model_name: str | None = None, exclude_team_models: bool = False
     ) -> list[str]:  # mutable-ok: the old return type, unchanged
         _note("get_model_ids", "registry().names(model_name, kind='ids')")
-        return _as_router(self)._core_get_model_ids(model_name=model_name, exclude_team_models=exclude_team_models)
+        return _as_router(self).registry().names(model_name, kind="ids", exclude_team_models=exclude_team_models)
 
     def resolved_litellm_models(
         self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False
     ) -> tuple[str, ...]:
         _note("resolved_litellm_models", "registry().names(model_name, kind='litellm_models' or 'litellm_base_models')")
-        return _as_router(self)._core_resolved_litellm_models(
-            model_name, team_id=team_id, use_base_model=use_base_model
+        return (
+            _as_router(self)
+            .registry()
+            .names(model_name, kind="litellm_base_models" if use_base_model else "litellm_models", team_id=team_id)
         )
 
     def wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]:
         _note("wildcard_routed_models", "registry().names(model, kind='wildcard')")
-        return _as_router(self)._core_wildcard_routed_models(model, team_id)
+        return _as_router(self).registry().names(model, kind="wildcard", team_id=team_id)
 
     def get_candidate_model_ids_for_route(self, model: str, team_id: str | None = None) -> frozenset[str]:
         _note("get_candidate_model_ids_for_route", "registry().names(model, kind='route_candidates')")
-        return _as_router(self)._core_get_candidate_model_ids_for_route(model, team_id=team_id)
+        return _as_router(self).registry().names(model, kind="route_candidates", team_id=team_id)
 
     # -- registry: resolving names ---------------------------------------------------------------
     def routable_model_group(self, model_name: str) -> str:
         _note("routable_model_group", "registry().resolve(name)")
-        return _as_router(self)._core_routable_model_group(model_name)
+        return _as_router(self).registry().resolve(model_name)
 
     def get_model_from_alias(self, model: str) -> str | None:
         _note("get_model_from_alias", "registry().resolve(name, kind='alias')")
-        return _as_router(self)._core_get_model_from_alias(model=model)
+        return _as_router(self).registry().resolve(model, kind="alias")
 
     _get_model_from_alias = _private_alias("get_model_from_alias", get_model_from_alias)
 
     def resolve_served_model_name(self, model_name: str, served_model: str) -> str:
         _note("resolve_served_model_name", "registry().resolve(name, kind='served', served_model=...)")
-        return _as_router(self)._core_resolve_served_model_name(model_name, served_model)
+        return _as_router(self).registry().resolve(model_name, kind="served", served_model=served_model)
 
     def resolve_model_name_from_model_id(
         self, model_id: str | None, custom_llm_provider: str | None = None
     ) -> str | None:
         _note("resolve_model_name_from_model_id", "registry().resolve(id, kind='from_id')")
-        return _as_router(self)._core_resolve_model_name_from_model_id(
-            model_id, custom_llm_provider=custom_llm_provider
-        )
+        return _as_router(self).registry().resolve(model_id, kind="from_id", custom_llm_provider=custom_llm_provider)
 
     def map_team_model(
         self, team_model_name: str | None, team_id: str, custom_llm_provider: str | None = None
     ) -> str | None:
         _note("map_team_model", "registry().resolve(name, kind='team_model', team_id=...)")
-        return _as_router(self)._core_map_team_model(team_model_name, team_id, custom_llm_provider=custom_llm_provider)
+        return (
+            _as_router(self)
+            .registry()
+            .resolve(team_model_name, kind="team_model", team_id=team_id, custom_llm_provider=custom_llm_provider)
+        )
 
     def is_recognized_model(self, model: str) -> bool:
         _note("is_recognized_model", "registry().resolve(name, kind='recognized')")
-        return _as_router(self)._core_is_recognized_model(model)
+        return _as_router(self).registry().resolve(model, kind="recognized")
 
     def get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None:
         _note("get_routable_upstream_model", "registry().resolve(name, kind='upstream')")
-        return _as_router(self)._core_get_routable_upstream_model(model_name, team_id=team_id)
+        return _as_router(self).registry().resolve(model_name, kind="upstream", team_id=team_id)
 
     def get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]:
         _note("get_search_tool_support", "registry().resolve(name, kind='search_tool_support')")
-        return _as_router(self)._core_get_search_tool_support(model_name, team_id=team_id)
+        return _as_router(self).registry().resolve(model_name, kind="search_tool_support", team_id=team_id)
 
     def get_model_group(self, id: str) -> list | None:  # mutable-ok: the old return type, unchanged
         _note("get_model_group", "registry().resolve(id, kind='group_of_id')")
-        return _as_router(self)._core_get_model_group(id=id)
+        return _as_router(self).registry().resolve(id, kind="group_of_id")
 
     def is_model_access_group_for_wildcard_route(self, model_access_group: str) -> bool:
         _note("is_model_access_group_for_wildcard_route", "registry().resolve(group, kind='wildcard_access_group')")
-        return _as_router(self)._core_is_model_access_group_for_wildcard_route(model_access_group=model_access_group)
+        return _as_router(self).registry().resolve(model_access_group, kind="wildcard_access_group")
 
     _is_model_access_group_for_wildcard_route = _private_alias(
         "is_model_access_group_for_wildcard_route", is_model_access_group_for_wildcard_route
@@ -598,42 +658,42 @@ class DeprecatedRouterMembers:
 
     def team_model_has_alternatives(self, deployment_id: str) -> bool:
         _note("team_model_has_alternatives", "registry().resolve(id, kind='team_model_alternatives')")
-        return _as_router(self)._core_team_model_has_alternatives(deployment_id)
+        return _as_router(self).registry().resolve(deployment_id, kind="team_model_alternatives")
 
     def routing_group_has_alternatives(self, model_group: str | None) -> bool:
         _note("routing_group_has_alternatives", "registry().resolve(group, kind='routing_group_alternatives')")
-        return _as_router(self)._core_routing_group_has_alternatives(model_group)
+        return _as_router(self).registry().resolve(model_group, kind="routing_group_alternatives")
 
     # -- registry: groups, access groups, credentials, plugins -----------------------------------
     def get_model_group_info(
         self, model_group: str, *, include_hidden: bool = False, team_id: str | None = None
     ) -> "ModelGroupInfo | None":
         _note("get_model_group_info", "registry().group(name)")
-        return _as_router(self)._core_get_model_group_info(model_group, include_hidden=include_hidden, team_id=team_id)
+        return _as_router(self).registry().group(model_group, include_hidden=include_hidden, team_id=team_id)
 
     def cached_model_group_info(self, model_group: str) -> "ModelGroupInfo | None":
         _note("cached_model_group_info", "registry().group(name, field='cached_info')")
-        return _as_router(self)._core_cached_model_group_info(model_group)
+        return _as_router(self).registry().group(model_group, field="cached_info")
 
     def get_configured_display_name(self, model_name: str) -> str | None:
         _note("get_configured_display_name", "registry().group(name, field='display_name')")
-        return _as_router(self)._core_get_configured_display_name(model_name)
+        return _as_router(self).registry().group(model_name, field="display_name")
 
     def get_configured_mode(self, model_name: str) -> str | None:
         _note("get_configured_mode", "registry().group(name, field='mode')")
-        return _as_router(self)._core_get_configured_mode(model_name)
+        return _as_router(self).registry().group(model_name, field="mode")
 
     def get_configured_service_tiers(self, model_name: str, team_id: str | None = None) -> tuple[object, ...]:
         _note("get_configured_service_tiers", "registry().group(name, field='service_tiers')")
-        return _as_router(self)._core_get_configured_service_tiers(model_name, team_id=team_id)
+        return _as_router(self).registry().group(model_name, field="service_tiers", team_id=team_id)
 
     def get_configured_token_limits(self, model_name: str) -> tuple[int | None, int | None]:
         _note("get_configured_token_limits", "registry().group(name, field='token_limits')")
-        return _as_router(self)._core_get_configured_token_limits(model_name)
+        return _as_router(self).registry().group(model_name, field="token_limits")
 
     def get_model_listing_info(self, model_name: str) -> "DeploymentModelListingInfo | None":
         _note("get_model_listing_info", "registry().group(name, field='listing_info')")
-        return _as_router(self)._core_get_model_listing_info(model_name)
+        return _as_router(self).registry().group(model_name, field="listing_info")
 
     def get_model_access_groups(
         self,
@@ -642,27 +702,29 @@ class DeprecatedRouterMembers:
         team_id: str | None = None,
     ) -> dict[str, list[str]]:  # mutable-ok: the old return type, unchanged
         _note("get_model_access_groups", "registry().access_groups(...)")
-        return _as_router(self)._core_get_model_access_groups(
-            model_name=model_name, model_access_group=model_access_group, team_id=team_id
+        return (
+            _as_router(self)
+            .registry()
+            .access_groups(model_name=model_name, model_access_group=model_access_group, team_id=team_id)
         )
 
     def get_deployment_credentials(self, model_id: str) -> dict | None:  # mutable-ok: the old return type, unchanged
         _note("get_deployment_credentials", "registry().credentials(id)")
-        return _as_router(self)._core_get_deployment_credentials(model_id=model_id)
+        return _as_router(self).registry().credentials(model_id)
 
     def get_deployment_credentials_with_provider(
         self, model_id: str, team_id: str | None = None
     ) -> "dict[str, Any] | None":  # pyright: ignore[reportExplicitAny]  # the old return type, unchanged  # mutable-ok: same
         _note("get_deployment_credentials_with_provider", "registry().credentials(id, form='with_provider')")
-        return _as_router(self)._core_get_deployment_credentials_with_provider(model_id=model_id, team_id=team_id)
+        return _as_router(self).registry().credentials(model_id, form="with_provider", team_id=team_id)
 
     def get_credential_deployment(self, model_id: str, team_id: str | None = None) -> "Deployment | None":
         _note("get_credential_deployment", "registry().credentials(id, form='deployment')")
-        return _as_router(self)._core_get_credential_deployment(model_id=model_id, team_id=team_id)
+        return _as_router(self).registry().credentials(model_id, form="deployment", team_id=team_id)
 
     def get_router_deployment_budget_limiter(self) -> "RouterBudgetLimiting | None":
         _note("get_router_deployment_budget_limiter", "registry().plugins('budget_limiter')")
-        return _as_router(self)._core_get_router_deployment_budget_limiter()
+        return _as_router(self).registry().plugins("budget_limiter")
 
     _get_router_deployment_budget_limiter = _private_alias(
         "get_router_deployment_budget_limiter", get_router_deployment_budget_limiter
@@ -678,15 +740,15 @@ class DeprecatedRouterMembers:
     # -- settings: methods -----------------------------------------------------------------------
     def get_settings(self) -> dict[str, object]:  # mutable-ok: Router._core_get_settings's type, a fresh dict
         _note("get_settings", "settings().as_dict()")
-        return _as_router(self)._core_get_settings()
+        return dict(_as_router(self).settings().as_dict())
 
     def get_allowed_fails_from_policy(self, exception: Exception) -> int | None:
         _note("get_allowed_fails_from_policy", "settings().allowed_fails_for(exception)")
-        return _as_router(self)._core_get_allowed_fails_from_policy(exception=exception)
+        return _as_router(self).settings().allowed_fails_for(exception)
 
     def get_available_guardrail(self, guardrail_name: str) -> "GuardrailTypedDict":
         _note("get_available_guardrail", "settings().guardrail(name)")
-        return _as_router(self)._core_get_available_guardrail(guardrail_name)
+        return _as_router(self).settings().guardrail(guardrail_name)
 
     # -- health ----------------------------------------------------------------------------------
     @staticmethod
@@ -709,31 +771,31 @@ class DeprecatedRouterMembers:
 
     def get_fully_blocked_model_names(self) -> set[str]:  # mutable-ok: the old return type, unchanged
         _note("get_fully_blocked_model_names", "health().fully_blocked_model_names()")
-        return _as_router(self)._core_get_fully_blocked_model_names()
+        return _as_router(self).health().fully_blocked_model_names()
 
     async def async_get_fully_unhealthy_model_names(self) -> set[str]:  # mutable-ok: the old return type, unchanged
         _note("async_get_fully_unhealthy_model_names", "health().fully_unhealthy_model_names()")
-        return await _as_router(self)._core_async_get_fully_unhealthy_model_names()
+        return await _as_router(self).health().fully_unhealthy_model_names()
 
     async def get_model_group_usage(self, model_group: str) -> tuple[int | None, int | None]:
         _note("get_model_group_usage", "health().usage(group)")
-        return await _as_router(self)._core_get_model_group_usage(model_group=model_group)
+        return await _as_router(self).health().usage(model_group)
 
     async def get_remaining_model_group_usage(
         self, model_group: str
     ) -> dict[str, int]:  # mutable-ok: the old return type, unchanged
         _note("get_remaining_model_group_usage", "health().usage(group, kind='remaining')")
-        return await _as_router(self)._core_get_remaining_model_group_usage(model_group=model_group)
+        return await _as_router(self).health().usage(model_group, kind="remaining")
 
     def team_ptu_ceiling(self, team_id: str, model_group: str) -> "PTUTeamCeiling | None":
         _note("team_ptu_ceiling", "health().team_ptu_ceiling(team_id, group)")
-        return _as_router(self)._core_team_ptu_ceiling(team_id, model_group)
+        return _as_router(self).health().team_ptu_ceiling(team_id, model_group)
 
     def split_request_priority(
         self, model: str, kwargs: Mapping[str, object]
     ) -> tuple[int | None, Mapping[str, object]]:
         _note("split_request_priority", "health().split_priority(model, kwargs)")
-        return _as_router(self)._core_split_request_priority(model, kwargs)
+        return _as_router(self).health().split_priority(model, kwargs)
 
     # -- selection -------------------------------------------------------------------------------
     async def async_get_healthy_deployments(
@@ -747,9 +809,10 @@ class DeprecatedRouterMembers:
         health_check_probe: bool = False,
     ) -> list[dict] | dict:  # mutable-ok: the old return type, unchanged
         _note("async_get_healthy_deployments", "select(model, request_kwargs, kind='healthy')")
-        return await _as_router(self)._core_async_get_healthy_deployments(
-            model=model,
-            request_kwargs=request_kwargs,
+        return await _as_router(self).select(
+            model,
+            request_kwargs,
+            kind="healthy",
             messages=messages,
             input=input,
             specific_deployment=specific_deployment,
@@ -757,7 +820,7 @@ class DeprecatedRouterMembers:
             health_check_probe=health_check_probe,
         )
 
-    def get_available_deployment_for_pass_through(  # noqa: ANN201  # the old member had no return annotation; its inferred type is kept by inferring it from the core
+    def get_available_deployment_for_pass_through(  # noqa: ANN201  # the old member had no return annotation; its inferred type is kept by inferring it from select
         self,
         model: str,
         messages: list[dict[str, str]] | None = None,  # mutable-ok: the old parameter type, unchanged
@@ -766,12 +829,15 @@ class DeprecatedRouterMembers:
         request_kwargs: dict | None = None,  # mutable-ok: the old parameter type, unchanged
     ):
         _note("get_available_deployment_for_pass_through", "select(model, kind='pass_through')")
-        return _as_router(self)._core_get_available_deployment_for_pass_through(
-            model=model,
-            messages=messages,
-            input=input,
-            specific_deployment=specific_deployment,
-            request_kwargs=request_kwargs,
+        return _run_ready(
+            _as_router(self).select(
+                model,
+                request_kwargs,
+                kind="pass_through",
+                messages=messages,
+                input=input,
+                specific_deployment=specific_deployment,
+            )
         )
 
     # -- writes ----------------------------------------------------------------------------------
@@ -808,8 +874,15 @@ class DeprecatedRouterMembers:
 
     def bind_to_request_logging(self, logging_obj: "LiteLLMLogging") -> None:
         _note("bind_to_request_logging", "observe(logging_obj)")
-        # Straight to the core, as before: observe() skips binding when logging_obj is None, the old member did not.
-        _as_router(self)._core_bind_to_request_logging(logging_obj)
+        router: Final = _as_router(self)
+        if logging_obj is None:  # pyright: ignore[reportUnnecessaryComparison]  # callers pass None despite the type
+            # observe() skips binding when logging_obj is None; the old member bound it, and failed on None.
+            raise AttributeError(
+                "'NoneType' object has no attribute 'add_dynamic_function_callbacks'",
+                name="add_dynamic_function_callbacks",
+                obj=None,
+            )
+        router.observe(logging_obj)
 
     def discard(self) -> None:
         _note("discard", "retire()")
@@ -822,7 +895,7 @@ class DeprecatedRouterMembers:
         request_kwargs: dict[str, object] | None = None,  # mutable-ok: the old parameter type, unchanged
     ) -> None:
         _note("arm_routing_read_prefetch", "RoutingContext.arm_read_prefetch")
-        _as_router(self)._core_arm_routing_read_prefetch(model, request_kwargs)
+        RoutingContext(_as_router(self)).arm_read_prefetch(model, request_kwargs)
 
     async def async_pre_routing_hook(
         self,
@@ -833,7 +906,7 @@ class DeprecatedRouterMembers:
         specific_deployment: bool | None = False,
     ) -> "PreRoutingHookResponse | None":
         _note("async_pre_routing_hook", "RoutingContext.pre_routing_hook")
-        return await _as_router(self)._core_async_pre_routing_hook(
+        return await RoutingContext(_as_router(self)).pre_routing_hook(
             model=model,
             request_kwargs=request_kwargs,
             messages=messages,
@@ -843,7 +916,7 @@ class DeprecatedRouterMembers:
 
     def log_retry(self, kwargs: dict, e: Exception) -> dict:  # mutable-ok: the old signature, unchanged
         _note("log_retry", "RoutingContext.log_retry")
-        return _as_router(self)._core_log_retry(kwargs=kwargs, e=e)
+        return RoutingContext(_as_router(self)).log_retry(kwargs=kwargs, e=e)
 
 
 # Router-family functions and classes that took the Router (``litellm_router_instance=``,

litellm/router_python_core.py

@@ -353,6 +353,11 @@ from litellm.router_utils.fallback_event_handlers import (
     routed_deployment_id,
     run_async_fallback,
 )
+from litellm.router_utils.global_callback_holders import (
+    live_check_owners,
+    routers_holding_global_callbacks,
+    routers_holding_global_callbacks_lock,
+)
 from litellm.router_utils.handle_error import (
     async_raise_no_deployment_exception,
     send_llm_exception_alert,
@@ -1199,8 +1204,9 @@ class RoutingArgs(enum.Enum):
 # as the per-request one built from a caller-supplied user_config, drops out on its
 # own rather than leaving entries behind that nothing can withdraw.
 _live_routers: Final["weakref.WeakSet[PythonCore]"] = weakref.WeakSet()
-_routers_holding_global_callbacks: Final["weakref.WeakSet[PythonCore]"] = weakref.WeakSet()
-_routers_holding_global_callbacks_lock: Final = threading.Lock()
+# Shared with the other router cores of the process (litellm/router_utils/global_callback_holders.py).
+_routers_holding_global_callbacks: Final["weakref.WeakSet[PythonCore]"] = routers_holding_global_callbacks  # pyright: ignore[reportAssignmentType]  # every member serves the holder members
+_routers_holding_global_callbacks_lock: Final = routers_holding_global_callbacks_lock
 
 
 def _replay_live_router_model_cost() -> None:
@@ -1839,6 +1845,7 @@ class PythonCore(DeprecatedRouterMembers):
         # cost-map entries from the list it is serving at that moment, whether
         # those deployments came from model_list or were added later from the DB.
         _live_routers.add(self)
+        live_check_owners.add(self)
 
         if allowed_fails is not None:
             self._core_allowed_fails = allowed_fails
@@ -2064,6 +2071,7 @@ class PythonCore(DeprecatedRouterMembers):
         # Stop contributing to cost-map rebuilds straight away rather than waiting
         # for this router to be collected.
         _live_routers.discard(self)
+        live_check_owners.discard(self)
         self.retire_routing_state()
         self._release_global_callbacks()
 
@@ -3359,7 +3367,7 @@ class PythonCore(DeprecatedRouterMembers):
         self.optional_callbacks = [cb for cb in self.optional_callbacks if type(cb) is not callback_cls]
         if any(
             router is not self and any(type(cb) is callback_cls for cb in (router.optional_callbacks or []))
-            for router in tuple(_live_routers)
+            for router in (*_live_routers, *(owner for owner in live_check_owners if not isinstance(owner, PythonCore)))
         ):
             return
         for cb in tuple(self._host.callback_list("callbacks")):

litellm/router_python_host.py

@@ -7,7 +7,10 @@ come from ``dynamic()`` (``litellm/router_config.py``). A Rust core gets the sam
 """
 
 import asyncio
-from collections.abc import Awaitable, Callable, Sequence
+import importlib
+import time
+import types
+from collections.abc import Awaitable, Callable, Mapping, Sequence
 from typing import (
     TYPE_CHECKING,
     Final,
@@ -19,12 +22,13 @@ from typing import (
 from typing_extensions import overload
 
 import litellm
-from litellm.router_config import RouterDynamicConfig, live_dynamic_config
+from litellm.router_config import RouterConfig, RouterDynamicConfig, live_dynamic_config
 
 if TYPE_CHECKING:
     from pydantic import BaseModel
 
-    from litellm.caching.caching import Cache
+    from litellm.caching.caching import Cache, RedisCache
+    from litellm.caching.dual_cache import DualCache
     from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
     from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
     from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
@@ -97,7 +101,7 @@ class PythonHost:
     def invoke(self, op: str) -> Endpoint: ...
     def invoke(self, op: str) -> Endpoint:
         """The litellm function named ``op`` (``acompletion``, ``get_model_info``, ...), looked up at call time."""
-        endpoint: Final[object] = getattr(litellm, op, None)
+        endpoint: Final[object] = getattr(litellm, op, None) if "." not in op else _dotted(op)
         if endpoint is None:
             raise AttributeError(f"litellm has no endpoint {op!r}")
         if not callable(endpoint):
@@ -107,6 +111,10 @@ class PythonHost:
     def sleep(self, seconds: float) -> Awaitable[None]:
         return asyncio.sleep(seconds)
 
+    def sleep_sync(self, seconds: float) -> None:
+        """The backoff of a sync endpoint's retry."""
+        time.sleep(seconds)
+
     def dynamic(self) -> RouterDynamicConfig:
         """The process settings, read live on each access (``current_dynamic_config()`` takes a snapshot)."""
         return cast(RouterDynamicConfig, live_dynamic_config)  # cast-ok: the live view reads the same fields
@@ -116,7 +124,7 @@ class PythonHost:
         return litellm.logging_callback_manager
 
     def callback_list(self, name: CallbackList) -> list[object]:  # mutable-ok: the process callback list itself
-        value: Final = getattr(litellm, name)
+        value: Final[object] = getattr(litellm, name)  # pyright: ignore[reportAny]  # an untyped litellm global
         return cast(list[object], value)  # cast-ok: the process callback list is an untyped litellm global
 
     def set_callback_list(self, name: CallbackList, value: list[object]) -> None:  # mutable-ok: assigned as given
@@ -131,5 +139,78 @@ class PythonHost:
     def suppress_debug_info(self) -> None:
         litellm.suppress_debug_info = True
 
+    def routing_cache(self, config: RouterConfig) -> "DualCache":
+        """The store a router keeps its routing state in (cooldowns, fail counts, usage): in memory,
+        plus Redis when the router's arguments name one. The host builds and owns it; a Rust core
+        reads and writes it through cache steps."""
+        from litellm.caching.caching import DualCache, InMemoryCache
+
+        return DualCache(redis_cache=redis_cache_for(config), in_memory_cache=InMemoryCache())
+
+
+def redis_cache_for(config: RouterConfig) -> "RedisCache | None":
+    """The Redis cache the router's ``redis_*`` and ``cache_kwargs`` arguments describe, or None."""
+    import json
+
+    from litellm.caching.caching import RedisCache, RedisClusterCache
+    from litellm.utils import get_secret
+
+    url: Final = config.get("redis_url")
+    host: Final = config.get("redis_host")
+    port: Final = config.get("redis_port")
+    if url is None and (host is None or port is None):
+        return None
+    settings: Final[dict[str, object]] = {}  # mutable-ok: built here, then passed as keywords
+    if url is not None:
+        settings["url"] = url
+    if host is not None:
+        settings["host"] = host
+    if port is not None:
+        settings["port"] = str(port)
+    if config.get("redis_password") is not None:
+        settings["password"] = config.get("redis_password")
+    if config.get("redis_db") is not None:
+        settings["db"] = str(config.get("redis_db"))
+    extra: Final = config.get("cache_kwargs")
+    if isinstance(extra, dict):
+        settings.update({str(key): value for key, value in extra.items()})  # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType]  # the cache_kwargs argument is untyped
+    nodes: object = settings.get("startup_nodes")  # rebind-ok: the env cluster nodes when the arguments name none
+    if not nodes:
+        env_nodes: Final[object] = get_secret("REDIS_CLUSTER_NODES")  # pyright: ignore[reportUnknownVariableType]  # get_secret is untyped
+        if isinstance(env_nodes, str):
+            parsed: Final[object] = json.loads(env_nodes)  # pyright: ignore[reportAny]  # plain JSON
+            nodes = parsed  # rebind-ok: see above
+    if nodes:
+        return RedisClusterCache(**{**settings, "startup_nodes": nodes})  # pyright: ignore[reportArgumentType]  # RedisCache's keyword arguments
+    return RedisCache(**settings)  # pyright: ignore[reportArgumentType]  # RedisCache's keyword arguments
+
 
 python_host: Final = PythonHost()
+
+
+def _dotted(op: str) -> object:
+    """``op`` named by its path under ``litellm`` (``vector_stores.main.asearch``), read at call time.
+
+    Each segment is a module attribute, a submodule, or a key of a mapping such as a module's
+    table of generated endpoints.
+    """
+    node: object = litellm  # rebind-ok: walks the path one segment at a time
+    path: str = "litellm"  # rebind-ok: the module path walked so far
+    for segment in op.split("."):
+        path = f"{path}.{segment}"
+        if isinstance(node, Mapping):
+            node = cast(Mapping[str, object], node).get(segment)  # cast-ok: endpoint tables are keyed by name
+        elif isinstance(node, types.ModuleType) and (submodule := _submodule(path)) is not None:
+            node = submodule
+        else:
+            node = cast(object, getattr(node, segment, None))  # cast-ok: any attribute
+        if node is None:
+            return None
+    return node
+
+
+def _submodule(path: str) -> types.ModuleType | None:
+    try:
+        return importlib.import_module(path)
+    except ModuleNotFoundError:
+        return None

litellm/router_rust_access.py

@@ -0,0 +1,85 @@
+"""The caller facts the Rust core routes on, read from the request's auth metadata: whether the
+caller is a proxy admin, the models its key and team allow (which narrow an access-group-only
+caller to its groups' deployments) and, for a proxy admin, the model the request names."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Final
+
+from pydantic import TypeAdapter
+
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.litellm_core_utils.routing_common_utils import is_proxy_admin_request, request_user_api_key_auth
+
+_NAMES: Final = TypeAdapter(tuple[object, ...])
+
+
+def _names(value: object) -> tuple[str, ...]:
+    names: Final = _NAMES.validate_python(value) if isinstance(value, list | tuple | set | frozenset) else ()
+    return tuple(str(name) for name in names)
+
+
+def _admin_model(request: Mapping[str, object]) -> object:
+    """The model ``filter_team_based_models`` compares team deployments against."""
+    metadata: Final = as_str_mapping(request.get("metadata")) or {}
+    litellm_metadata: Final = as_str_mapping(request.get("litellm_metadata")) or {}
+    return request.get("model") or metadata.get("model_group") or litellm_metadata.get("model_group")
+
+
+def caller_fields(request: Mapping[str, object]) -> Mapping[str, object]:
+    """``proxy_admin``, ``allowed_models`` and ``admin_model`` for the Rust request document."""
+    auth: Final[object] = request_user_api_key_auth(request)
+    if auth is None:
+        return {}
+    allowed: Final = sorted({*_names(getattr(auth, "models", None)), *_names(getattr(auth, "team_models", None))})
+    admin: Final = is_proxy_admin_request(request)
+    model: Final = _admin_model(request) if admin else None
+    return {
+        **({"allowed_models": allowed} if allowed else {}),
+        **({"proxy_admin": True} if admin else {}),
+        **({"admin_model": model} if isinstance(model, str) else {}),
+    }
+
+
+def _router_weights_error(raw: object) -> BaseException:
+    """The error ``validate_router_weights`` raises for weights the Rust core refused."""
+    from pydantic import ValidationError
+
+    from litellm.types.router_weights import validate_router_weights
+
+    try:
+        _ = validate_router_weights(raw)
+    except ValidationError as error:
+        return error
+    return RuntimeError(f"router-core refused router weights the validator accepts: {raw!r}")
+
+
+def selection_refusal(step: Mapping[str, object], model: str, request: Mapping[str, object]) -> BaseException | None:
+    """The error for a pick the Rust core refused over who the caller is or what it asked for:
+    a public name several teams share, deployments reserved for other teams, invalid request
+    router weights or a non-numeric deployment weight. None for any other failure."""
+    import litellm.exceptions as litellm_exceptions
+
+    kind: Final = step.get("error_kind")
+    named: Final = str(step.get("error_model") or model)
+    if kind == "multiple_teams":
+        return litellm_exceptions.BadRequestError(
+            message=(
+                f"Model name '{named}' matches deployments from multiple teams. "
+                "Specify the deployment ID directly to disambiguate."
+            ),
+            model=named,
+            llm_provider="",
+        )
+    if kind == "reserved":
+        return litellm_exceptions.BadRequestError(
+            message=f"Deployment {named} is reserved for another team until {step.get('error_detail')}",
+            model=named,
+            llm_provider="",
+        )
+    if kind == "invalid_router_weights":
+        return _router_weights_error(request.get("_router_weights"))
+    if kind == "non_numeric_weight":
+        return TypeError(f"Deployment {step.get('error_detail')} must be numeric")
+    return None

litellm/router_rust_alerting.py

@@ -0,0 +1,33 @@
+"""The Rust core's Slack alerting: ``alerting_config`` built into a ``SlackAlerting`` logger on the
+host, as the Python core's ``_initialize_alerting`` builds it."""
+
+from typing import TYPE_CHECKING, Final
+
+from litellm._logging import verbose_router_logger
+
+if TYPE_CHECKING:
+    from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
+    from litellm.router_python_host import PythonHost
+    from litellm.types.router import AlertingConfig
+
+
+def initialize_alerting(host: "PythonHost", alerting_config: "AlertingConfig | None") -> "SlackAlerting | None":
+    """``_initialize_alerting``: a ``SlackAlerting`` logger with the config's threshold and webhook,
+    added to the host's callbacks, and its slow-response check to the host's success callbacks.
+    No config builds nothing."""
+    from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
+
+    if alerting_config is None:
+        return None
+    logger: Final = SlackAlerting(
+        alerting_threshold=alerting_config.alerting_threshold,
+        alerting=["slack"],
+        default_webhook_url=alerting_config.webhook_url,
+    )
+    host.callbacks.add_litellm_callback(logger)
+    host.callbacks.add_litellm_success_callback(logger.response_taking_too_long_callback)  # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType]  # SlackAlerting's untyped callback
+    verbose_router_logger.info("\033[94m\nInitialized Alerting for litellm.Router\033[0m\n")
+    return logger
+
+
+__all__ = ["initialize_alerting"]

litellm/router_rust_callbacks.py

@@ -0,0 +1,558 @@
+"""
+Host-side optional pre-call checks for ``RustCore``: the ``CustomLogger`` checks a router builds
+from ``optional_pre_call_checks``, ``provider_budget_config``, per-deployment budgets and
+``model_group_affinity_config``, registered in the host's ``callbacks`` list exactly as the Python
+router's ``add_optional_pre_call_checks`` registers them, since they are features users configure.
+
+The Rust core asks the host to run every routing callback's ``async_filter_deployments`` through a
+``callback_filters`` select stage (``async_callback_filter_deployments``) and decides itself what
+the surviving deployments or the raised error mean: a pick with nothing left, a retry, a fallback.
+
+This module also keeps the process-wide holder rules (``litellm/router_utils/global_callback_holders``):
+a retiring router hands the checks it vacates to a survivor, a runtime removal keeps a check
+another live router still has, and ``observe(request_scoped=True)`` takes every routing callback
+off the host lists while the router keeps running them (``_routing_callbacks``). It never reaches a
+Python router core.
+"""
+
+from collections.abc import Iterable, Mapping, Sequence
+from itertools import chain
+from typing import TYPE_CHECKING, Final
+
+from litellm.caching.dual_cache import DualCache
+from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.constants import PROMPT_CACHE_PIN_CACHE_MAX_SIZE
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy
+from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
+from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
+from litellm.router_utils.global_callback_holders import (
+    live_check_owners,
+    routers_holding_global_callbacks,
+    routers_holding_global_callbacks_lock,
+)
+from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
+    DeploymentAffinityCheck,
+    warn_on_unknown_model_group_affinity_flags,
+)
+from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck
+from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
+from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging
+    from litellm.router_python_host import PythonHost
+    from litellm.router_rust_registry import HostRegistry
+    from litellm.router_rust_selection import StrategySelectors
+    from litellm.router_utils.global_callback_holders import GlobalCallbackHolder
+    from litellm.types.router import Deployment
+
+_AFFINITY_CHECKS: Final = (
+    "deployment_affinity",
+    "responses_api_deployment_check",
+    "session_affinity",
+    "encrypted_content_affinity",
+)
+# `_RUNTIME_TOGGLEABLE_PRE_CALL_CHECKS`: the checks a runtime list drops when it leaves them out.
+_RUNTIME_TOGGLEABLE: Final[Mapping[str, type[CustomLogger]]] = {
+    "prompt_caching": PromptCachingDeploymentCheck,
+    "enforce_model_rate_limits": ModelRateLimitingCheck,
+    "encrypted_content_affinity": EncryptedContentAffinityCheck,
+}
+
+
+def _callbacks(host: "PythonHost") -> list[object]:  # mutable-ok: the process list itself
+    return host.callback_list("callbacks")  # pyright: ignore[reportArgumentType]  # not in the CallbackList literal
+
+
+def register_router_selector(host: "PythonHost", selector: CustomLogger) -> None:
+    """``_register_router_selector``: least-busy also counts calls as they start."""
+    if isinstance(selector, LeastBusyLoggingHandler):
+        if isinstance(host.callback_list("input_callback"), list):  # pyright: ignore[reportUnnecessaryIsInstance]  # a process list a user may have reassigned
+            host.callbacks.add_litellm_input_callback(selector)
+        else:
+            host.set_callback_list("input_callback", [selector])
+    if isinstance(_callbacks(host), list):  # pyright: ignore[reportUnnecessaryIsInstance]  # see above
+        host.callbacks.add_litellm_callback(selector)
+
+
+def _move_before_deployment_affinity(
+    callback_list: list[object],  # mutable-ok: reordered in place
+    callback: object,
+) -> None:
+    if callback not in callback_list:
+        return
+    callback_list.remove(callback)
+    index: Final = next(
+        (i for i, entry in enumerate(callback_list) if isinstance(entry, DeploymentAffinityCheck)),
+        len(callback_list),
+    )
+    callback_list.insert(index, callback)
+
+
+class OptionalChecks:
+    """One router's optional pre-call checks (``optional_callbacks``) and the stores they read."""
+
+    def __init__(
+        self,
+        owner: "GlobalCallbackHolder",
+        host: "PythonHost",
+        cache: DualCache,
+        hosted: "HostRegistry",
+        ctx: object,
+        arguments: Mapping[str, object],
+    ) -> None:
+        self._owner: Final = owner
+        self._host: Final = host
+        self._cache: Final = cache
+        self._hosted: Final = hosted
+        self._ctx: Final = ctx
+        self.optional_callbacks: list[object] | None = None  # mutable-ok: the router's live list, as Python keeps it
+        self.router_budget_logger: RouterBudgetLimiting | None = None
+        self.provider_budget_config: object = arguments.get("provider_budget_config")
+        ttl: Final = arguments.get("deployment_affinity_ttl_seconds")
+        self.deployment_affinity_ttl_seconds: object = 3600 if ttl is None else ttl
+        self.session_affinity_repin_on_failover: object = bool(arguments.get("session_affinity_repin_on_failover"))
+        affinity_config: Final = arguments.get("model_group_affinity_config")
+        self.model_group_affinity_config: dict[str, list[str]] | None = (  # mutable-ok: the argument as passed
+            affinity_config if isinstance(affinity_config, dict) else None  # pyright: ignore[reportAttributeAccessIssue]  # the argument as passed
+        )
+        warn_on_unknown_model_group_affinity_flags(self.model_group_affinity_config)
+        self.prompt_caching_pin_cache: DualCache = DualCache(
+            redis_cache=cache.redis_cache,
+            in_memory_cache=InMemoryCache(max_size_in_memory=PROMPT_CACHE_PIN_CACHE_MAX_SIZE),
+        )
+        self._request_scoped_selectors: tuple[CustomLogger, ...] = ()
+        self._request_scoped_policies: tuple[CustomLogger, ...] = ()
+        # ``_slack_alerting_loggers``: the Slack alerting logger ``alerting_config`` builds, if any.
+        self.slack_alerting_loggers: tuple[CustomLogger, ...] = ()
+
+    # -- construction ---------------------------------------------------------------------------
+
+    def build(self, checks: object, model_list: object) -> None:
+        """The constructor's checks: a budget appends ``router_budget_limiting`` to the list given (in
+        place, as Python does), the list is added, a per-group affinity config adds the affinity
+        check, and ``apply_default_settings`` adds the empty default list."""
+        listed: list[object] | None = checks if isinstance(checks, list) else None  # pyright: ignore[reportUnknownVariableType]  # mutable-ok: appended in place, as Python does  # rebind-ok: the argument as passed
+        if RouterBudgetLimiting.should_init_router_budget_limiter(  # pyright: ignore[reportUnknownMemberType]  # untyped model_list
+            model_list=model_list if isinstance(model_list, list) else None,  # pyright: ignore[reportUnknownArgumentType]  # the argument as passed
+            provider_budget_config=self.provider_budget_config,  # pyright: ignore[reportArgumentType]  # the argument as passed
+        ):
+            if listed is not None:
+                listed.append("router_budget_limiting")
+            else:
+                listed = ["router_budget_limiting"]  # rebind-ok: the list Python builds
+        if listed is not None:
+            self.add(listed)  # pyright: ignore[reportUnknownArgumentType]  # names as given
+        if self.model_group_affinity_config:
+            self.ensure_deployment_affinity_callback()
+        self.add([])
+
+    def _is_priority_group(self, model: str) -> bool:
+        """``_is_priority_routing_group``."""
+        hosted: Final = self._hosted
+        resolved: Final = hosted._core_get_model_from_alias(model=model) or model  # pyright: ignore[reportPrivateUsage]  # the host registry's alias lookup
+        group: Final = hosted.get_routing_group(resolved)
+        return group is not None and group.routing_strategy == "priority"
+
+    def _affinity_check(self, user_key: bool, responses_api: bool, session_id: bool) -> DeploymentAffinityCheck:
+        return DeploymentAffinityCheck(
+            cache=self._cache,
+            ttl_seconds=self.deployment_affinity_ttl_seconds,  # pyright: ignore[reportArgumentType]  # the argument as passed
+            enable_user_key_affinity=user_key,
+            enable_responses_api_affinity=responses_api,
+            enable_session_id_affinity=session_id,
+            model_group_affinity_config=self.model_group_affinity_config,
+            is_priority_group=self._is_priority_group,
+            session_affinity_repin_on_failover=self.session_affinity_repin_on_failover,  # pyright: ignore[reportArgumentType]  # same
+        )
+
+    def _own(self, callback: CustomLogger) -> None:
+        if self.optional_callbacks is None:
+            self.optional_callbacks = []
+        self.optional_callbacks.append(callback)
+        self._host.callbacks.add_litellm_callback(callback)
+
+    def ensure_deployment_affinity_callback(self) -> None:
+        """``_ensure_deployment_affinity_callback``: the affinity check with every global flag off."""
+        if any(isinstance(cb, DeploymentAffinityCheck) for cb in self.optional_callbacks or ()):
+            return
+        self._own(self._affinity_check(False, False, False))
+
+    def _add_affinity(self, checks: Sequence[object]) -> None:
+        user_key: Final = "deployment_affinity" in checks
+        responses_api: Final = "responses_api_deployment_check" in checks
+        session_id: Final = "session_affinity" in checks
+        if not (user_key or responses_api or session_id):
+            return
+        if self.optional_callbacks is None:
+            self.optional_callbacks = []
+        existing: Final = next(
+            (cb for cb in self.optional_callbacks if isinstance(cb, DeploymentAffinityCheck)),
+            None,
+        )
+        if existing is None:
+            self._own(self._affinity_check(user_key, responses_api, session_id))
+            return
+        existing.enable_user_key_affinity = existing.enable_user_key_affinity or user_key
+        existing.enable_responses_api_affinity = existing.enable_responses_api_affinity or responses_api
+        existing.enable_session_id_affinity = existing.enable_session_id_affinity or session_id
+        existing.ttl_seconds = self.deployment_affinity_ttl_seconds  # pyright: ignore[reportAttributeAccessIssue]  # the argument as passed
+        if self.model_group_affinity_config:
+            existing.model_group_affinity_config = self.model_group_affinity_config
+
+    def _add_encrypted_content_affinity(self, enable_global_affinity: bool) -> None:
+        """``_add_encrypted_content_affinity_check``."""
+        if not (
+            enable_global_affinity
+            or EncryptedContentAffinityCheck.has_model_group_affinity_enabled(self.model_group_affinity_config)
+        ):
+            return
+        if self.optional_callbacks is None:
+            self.optional_callbacks = []
+        existing: Final = next(
+            (cb for cb in self.optional_callbacks if isinstance(cb, EncryptedContentAffinityCheck)), None
+        )
+        if existing is not None:
+            existing.ctx = self._ctx  # pyright: ignore[reportAttributeAccessIssue]  # the core's RoutingContext serves registry()
+            existing.enable_global_affinity = existing.enable_global_affinity or enable_global_affinity
+            existing.model_group_affinity_config = self.model_group_affinity_config or {}
+            check: EncryptedContentAffinityCheck = existing  # rebind-ok: the check to move
+        else:
+            check = EncryptedContentAffinityCheck(  # rebind-ok: the check to move
+                ctx=self._ctx,  # pyright: ignore[reportArgumentType]  # see above
+                enable_global_affinity=enable_global_affinity,
+                model_group_affinity_config=self.model_group_affinity_config,
+            )
+            self._own(check)
+        _move_before_deployment_affinity(self.optional_callbacks, check)
+        _move_before_deployment_affinity(_callbacks(self._host), check)
+
+    def add(self, checks: Sequence[object] | None) -> None:
+        """``add_optional_pre_call_checks``."""
+        if checks is None:
+            return
+        self._add_affinity(checks)
+        self._add_encrypted_content_affinity("encrypted_content_affinity" in checks)
+        for name in checks:
+            if name in _AFFINITY_CHECKS:
+                continue
+            callback: CustomLogger | None = None
+            if name == "prompt_caching":
+                callback = PromptCachingDeploymentCheck(
+                    cache=self.prompt_caching_pin_cache, is_priority_group=self._is_priority_group
+                )
+            elif name == "router_budget_limiting":
+                if self.budget_limiter() is not None:
+                    continue
+                callback = RouterBudgetLimiting(
+                    dual_cache=self._cache,
+                    provider_budget_config=self.provider_budget_config,  # pyright: ignore[reportArgumentType]  # the argument as passed
+                    model_list=self._hosted._core_model_list,  # pyright: ignore[reportPrivateUsage, reportArgumentType]  # the live model_list
+                )
+                self.router_budget_logger = callback
+            elif name == "enforce_model_rate_limits":
+                callback = ModelRateLimitingCheck(dual_cache=self._cache)
+            if callback is None:
+                continue
+            if self.optional_callbacks is not None and any(
+                isinstance(existing, type(callback)) for existing in self.optional_callbacks
+            ):
+                continue
+            self._own(callback)
+
+    def set(self, checks: object) -> None:
+        """``set_optional_pre_call_checks``: drop the toggleable checks the list leaves out, then add it."""
+        if checks is None:
+            return
+        listed: Final[tuple[object, ...]] = tuple(checks) if isinstance(checks, list | tuple) else ()  # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType]  # names as given
+        requested: Final = frozenset(listed)
+        for name, callback_cls in _RUNTIME_TOGGLEABLE.items():
+            if name not in requested:
+                self._remove_of_type(callback_cls)
+        self.add(listed)
+
+    def _remove_of_type(self, callback_cls: type[CustomLogger]) -> None:
+        """``_remove_optional_callbacks_of_type``: kept on the host while any other live router of
+        the process has a check of that type."""
+        if self.optional_callbacks is None or not any(type(cb) is callback_cls for cb in self.optional_callbacks):
+            return
+        self.optional_callbacks = [cb for cb in self.optional_callbacks if type(cb) is not callback_cls]
+        others: Final = tuple(owner for owner in live_check_owners if id(owner) != id(self._owner))
+        if any(any(type(cb) is callback_cls for cb in (owner.optional_callbacks or ())) for owner in others):
+            return
+        callbacks: Final = _callbacks(self._host)
+        for cb in tuple(callbacks):
+            if type(cb) is callback_cls:
+                self._host.callbacks.remove_callback_from_list_by_object(callbacks, cb, require_self=False)  # pyright: ignore[reportUnknownMemberType]  # the manager's untyped helper
+
+    # -- budgets --------------------------------------------------------------------------------
+
+    def budget_limiter(self) -> RouterBudgetLimiting | None:
+        """``_core_get_router_deployment_budget_limiter``: exact type, so a proxy subclass is not taken."""
+        if self.router_budget_logger is not None:
+            return self.router_budget_logger
+        for cb in self.optional_callbacks or ():
+            if isinstance(cb, RouterBudgetLimiting) and type(cb) is RouterBudgetLimiting:
+                self.router_budget_logger = cb
+                return cb
+        return None
+
+    def sync_deployment_budget(self, deployment: "Deployment") -> None:
+        """``_sync_deployment_budget_config``, after a deployment is added."""
+        model_id: Final = deployment.model_info.id
+        if model_id is None:
+            return
+        limiter: RouterBudgetLimiting | None = self.budget_limiter()  # rebind-ok: built when first needed
+        params: Final = deployment.litellm_params
+        if params.max_budget is None or params.budget_duration is None:
+            if limiter is not None:
+                limiter.unregister_deployment_budget(model_id=model_id)
+            return
+        if limiter is None:
+            self.add(["router_budget_limiting"])
+            limiter = self.budget_limiter()  # rebind-ok: the one just built
+        if limiter is not None:
+            limiter.register_deployment_budget(deployment=deployment.to_json(exclude_none=True))  # pyright: ignore[reportUnknownMemberType]  # pydantic's untyped dump
+
+    def unregister_deployment_budget(self, model_id: str) -> None:
+        limiter: Final = self.budget_limiter()
+        if limiter is not None:
+            limiter.unregister_deployment_budget(model_id=model_id)
+
+    # -- the routing callbacks ------------------------------------------------------------------
+
+    def loggers(self) -> tuple[CustomLogger, ...]:
+        """``_optional_loggers``."""
+        return tuple(cb for cb in self.optional_callbacks or () if isinstance(cb, CustomLogger))
+
+    def _policies_not_set_globally(self) -> tuple[CustomLogger, ...]:
+        keys: Final = self._host.callbacks.get_custom_logger_key
+        global_keys: Final = frozenset(keys(cb) for cb in _callbacks(self._host) if isinstance(cb, CustomLogger))
+        return tuple(policy for policy in self._request_scoped_policies if keys(policy) not in global_keys)
+
+    def bind_to_request_logging(self, logging_obj: "Logging") -> None:
+        """``_core_bind_to_request_logging``'s routing half: the request-scoped selectors replace the
+        host's same-keyed callbacks for the request, and the policies the host does not run are
+        added to it. The outcome callbacks are not bound: the Rust core records every attempt's
+        outcome itself, as it registers no outcome callbacks on the host."""
+        for selector in self._request_scoped_selectors:
+            logging_obj.replace_global_callback(selector)
+        for policy in self._policies_not_set_globally():
+            logging_obj.add_dynamic_callback(policy)
+
+    def routing_callbacks(self) -> Iterable[object]:
+        """``_routing_callbacks``: the host's callbacks, or after ``observe(request_scoped=True)`` the
+        host's callbacks with the router's own selectors and policies in place of theirs."""
+        if not (self._request_scoped_selectors or self._request_scoped_policies):
+            return tuple(_callbacks(self._host))
+        keys: Final = self._host.callbacks.get_custom_logger_key
+        replaced: Final = frozenset(map(keys, self._request_scoped_selectors))
+        return (
+            *self._host.callbacks.without_custom_logger_keys(_callbacks(self._host), replaced),  # pyright: ignore[reportArgumentType]  # the process list holds any callback
+            *self._request_scoped_selectors,
+            *self._policies_not_set_globally(),
+        )
+
+    async def filter_deployments(
+        self,
+        model: str,
+        deployments: list[dict[str, object]],  # mutable-ok: CustomLogger deployment-list contract
+        messages: object,
+        request: Mapping[str, object],
+    ) -> list[object]:  # mutable-ok: as the filters return it
+        """``async_callback_filter_deployments``: each routing callback narrows the deployments in
+        turn; a filter's error is raised for the Rust core to decide on."""
+        remaining: list[object] = list(deployments)  # mutable-ok: filter contract  # rebind-ok: narrowed
+        base: Final[object] = CustomLogger.async_filter_deployments  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # untyped deployment dicts
+        request_kwargs: Final = dict(request)
+        for callback in tuple(self.routing_callbacks()):
+            # The base filter returns its input unchanged: only overriding filters are run.
+            if not isinstance(callback, CustomLogger) or type(callback).async_filter_deployments is base:  # pyright: ignore[reportUnknownMemberType]  # see above
+                continue
+            remaining = await callback.async_filter_deployments(  # pyright: ignore[reportUnknownMemberType, reportAssignmentType, reportUnknownVariableType]  # see above
+                model=model,
+                healthy_deployments=remaining,
+                messages=messages,  # pyright: ignore[reportArgumentType]  # the request's messages, as given
+                request_kwargs=request_kwargs,
+                parent_otel_span=None,
+            )
+        return remaining
+
+    # -- release --------------------------------------------------------------------------------
+
+    def serve_through_request_logging(self, selectors: Sequence[CustomLogger], dropped: Sequence[object]) -> None:
+        """``_core_serve_through_request_logging``: keep running ``selectors`` (and request-scoped
+        adaptive hooks) and the optional checks, but take everything off the host lists."""
+        from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
+
+        adaptive: Final = tuple(
+            AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy)  # pyright: ignore[reportArgumentType]  # an adaptive router
+            for tagged in chain.from_iterable(self._hosted._core_adaptive_routers.values())  # pyright: ignore[reportPrivateUsage]  # the host registry's live registry
+        )
+        self._request_scoped_selectors = (*selectors, *adaptive)
+        self._request_scoped_policies = (*self.loggers(), *self.slack_alerting_loggers)
+        self.release(dropped)
+
+    def own_adaptive_hooks(self) -> tuple[CustomLogger, ...]:
+        """``_adaptive_router_hooks``: the host's adaptive post-call hooks of this router's routers."""
+        from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
+
+        own: Final = frozenset(
+            id(tagged.strategy)
+            for tagged in chain.from_iterable(self._hosted._core_adaptive_routers.values())  # pyright: ignore[reportPrivateUsage]  # see above
+        )
+        return tuple(
+            hook
+            for hook in self._host.callbacks.get_custom_loggers_for_type(AdaptiveRouterPostCallHook)
+            if isinstance(hook, AdaptiveRouterPostCallHook) and id(hook.adaptive_router) in own
+        )
+
+    def retire_state(self, selectors: Sequence[object]) -> None:
+        """``retire_routing_state``: stop the background work of the selectors and the budget limiter."""
+        for strategy in (*selectors, *self.loggers()):
+            if isinstance(strategy, BaseRoutingStrategy | RouterBudgetLimiting):
+                strategy.retire()
+
+    def release(self, dropped: Sequence[object]) -> None:
+        """``_release_global_callbacks``: drop ``dropped``, the optional checks and the adaptive hooks
+        from the host lists, then hand each vacated check to a surviving router's same-keyed one."""
+        with routers_holding_global_callbacks_lock:
+            routers_holding_global_callbacks.discard(self._owner)
+        adaptive: Final = self.own_adaptive_hooks()
+        slack: Final = self.slack_alerting_loggers
+        vacated: Final = self._host.callbacks.drop_callbacks(
+            (*dropped, *(self.optional_callbacks or ()), *slack, *adaptive), method_owners=(self._owner, *slack)
+        )
+        with routers_holding_global_callbacks_lock:
+            survivors: Final = tuple(routers_holding_global_callbacks)
+        candidates: Final = chain.from_iterable(router._global_callback_candidates() for router in survivors)  # pyright: ignore[reportPrivateUsage]  # the holder protocol
+        for candidate in self._host.callbacks.with_custom_logger_keys(candidates, vacated):
+            register_router_selector(self._host, candidate)
+        adaptive_keys: Final = frozenset(map(self._host.callbacks.get_custom_logger_key, adaptive))
+        survivor: Final = next((router for router in survivors if router._core_adaptive_routers), None)  # pyright: ignore[reportPrivateUsage]  # see above
+        if survivor is not None and vacated & adaptive_keys:
+            survivor._sync_adaptive_router_hooks()  # pyright: ignore[reportPrivateUsage]  # see above
+
+
+def filtered_ids(remaining: Iterable[object], candidates: Sequence[str]) -> list[str]:  # mutable-ok: a JSON list
+    """The ids of the deployments the filters left, in their order, among the candidates."""
+    ids: Final[list[str]] = []  # mutable-ok: built here
+    for deployment in remaining:
+        found = (as_str_mapping((as_str_mapping(deployment) or {}).get("model_info")) or {}).get("id")
+        if isinstance(found, str) and found in candidates and found not in ids:
+            ids.append(found)
+    return ids
+
+
+class RustOptionalChecks:
+    """``RustCore`` members for its optional pre-call checks and routing callbacks."""
+
+    # RustCore sets these as Final attributes in its constructor.
+    @property
+    def _opt(self) -> OptionalChecks:
+        checks: Final[OptionalChecks] = self.__dict__["_checks"]  # pyright: ignore[reportAny]  # RustCore's own attribute
+        return checks
+
+    @property
+    def _sel(self) -> "StrategySelectors":
+        selectors: Final[StrategySelectors] = self.__dict__["_selectors"]  # pyright: ignore[reportAny]  # same
+        return selectors
+
+    @property
+    def _reg(self) -> "HostRegistry":
+        hosted: Final[HostRegistry] = self.__dict__["_hosted"]  # pyright: ignore[reportAny]  # same
+        return hosted
+
+    @property
+    def optional_callbacks(self) -> "list[object] | None":  # mutable-ok: the router's live list
+        return self._opt.optional_callbacks
+
+    @property
+    def router_budget_logger(self) -> RouterBudgetLimiting | None:
+        return self._opt.router_budget_logger
+
+    @property
+    def deployment_affinity_ttl_seconds(self) -> object:
+        return self._opt.deployment_affinity_ttl_seconds
+
+    @property
+    def session_affinity_repin_on_failover(self) -> object:
+        return self._opt.session_affinity_repin_on_failover
+
+    @property
+    def model_group_affinity_config(self) -> "dict[str, list[str]] | None":  # mutable-ok: the argument as passed
+        return self._opt.model_group_affinity_config
+
+    def add_optional_pre_call_checks(self, optional_pre_call_checks: "Sequence[object] | None") -> None:
+        self._opt.add(optional_pre_call_checks)
+
+    def apply_default_settings(self) -> None:
+        """Add the (empty) default checks; an enabled continuation registers the prefill check."""
+        from litellm.router_rust_streaming import register_continuation_prefill_check
+
+        self._opt.add([])
+        if getattr(self, "enable_mid_stream_fallback_continuation", False):
+            register_continuation_prefill_check(self._opt._host)  # pyright: ignore[reportPrivateUsage]  # the checks' host
+
+    # The members another router's retire reads (`global_callback_holders`).
+
+    def _global_callback_candidates(self) -> tuple[CustomLogger, ...]:
+        return (*self._sel.callback_selectors(), *self._opt.loggers(), *self._opt.slack_alerting_loggers)
+
+    @property
+    def _core_adaptive_routers(self) -> Mapping[str, object]:
+        return self._reg._core_adaptive_routers  # pyright: ignore[reportPrivateUsage]  # the host registry's live registry
+
+    def _sync_adaptive_router_hooks(self) -> None:
+        strategies: Final = self._reg.strategies
+        if strategies is not None:
+            strategies._sync_adaptive_router_hooks()  # pyright: ignore[reportPrivateUsage]  # the strategies' own hook sync
+
+    def _join_callback_holders(self) -> None:
+        with routers_holding_global_callbacks_lock:
+            routers_holding_global_callbacks.add(self)
+        live_check_owners.add(self)
+
+    def _serve_through_request_logging(self) -> None:
+        selectors: Final = self._sel
+        self._opt.serve_through_request_logging(
+            selectors.callback_selectors(), (*selectors.callback_selectors(), *selectors.override_selectors())
+        )
+
+    def _retire_callbacks(self) -> None:
+        """``_core_discard``'s callback half: retire the routing state, release the callbacks."""
+        live_check_owners.discard(self)
+        selectors: Final = (*self._sel.callback_selectors(), *self._sel.override_selectors())
+        self._opt.retire_state(selectors)
+        self._opt.release(selectors)
+        self._sel.clear()
+
+    async def _answer_callback_filters(
+        self, step: Mapping[str, object], request: Mapping[str, object], messages: object
+    ) -> tuple[object, BaseException | None]:
+        """A ``callback_filters`` step: the remaining ids, or a filter's error as an attempt
+        failure with the error itself, which the caller keeps to raise."""
+        from litellm.router_rust_core import attempt_failure
+
+        candidates: Final = [str(id) for id in step.get("candidates") or ()]  # pyright: ignore[reportGeneralTypeIssues, reportUnknownVariableType, reportUnknownArgumentType]  # the step's id list
+        records: Final = self._records_of(candidates)  # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType]  # RustCore's deployment dicts
+        try:
+            remaining: Final = await self._opt.filter_deployments(
+                str(step.get("model_group")),
+                records,  # pyright: ignore[reportUnknownArgumentType]  # see above
+                messages,
+                request,
+            )
+        except Exception as error:  # noqa: BLE001  # handed to the Rust core as the try's failure
+            return attempt_failure(error), error
+        return filtered_ids(remaining, candidates), None  # pyright: ignore[reportUnknownArgumentType]  # see above
+
+
+__all__ = [
+    "OptionalChecks",
+    "RustOptionalChecks",
+    "filtered_ids",
+    "register_router_selector",
+]

litellm/router_rust_fallbacks.py

@@ -0,0 +1,132 @@
+"""
+Fallback configuration checks ``RustCore`` runs on the host: ``validate_fallbacks``, with the
+Python router's error texts. router-core walks the fallbacks; this module only refuses a list it
+could not walk the way the Python router would, before the core is built.
+"""
+
+from collections.abc import Mapping, Sequence
+from types import MappingProxyType
+from typing import Final
+
+from litellm.router_utils.fallback_event_handlers import check_non_standard_fallback_format
+
+
+def refuse_malformed_fallbacks(fallback_param: Sequence[object] | None) -> None:
+    """``Router.validate_fallbacks``: every entry is a one-key dict whose targets are a list (or a
+    client-side dict target)."""
+    if fallback_param is None:
+        return
+    for fallback_dict in fallback_param:
+        if not isinstance(fallback_dict, dict):
+            raise ValueError(f"Item '{fallback_dict}' is not a dictionary.")  # noqa: TRY004  # the Python router's error class
+        if len(fallback_dict) != 1:  # pyright: ignore[reportUnknownArgumentType]  # only the size is read
+            raise ValueError(
+                f"Dictionary '{fallback_dict}' must have exactly one key, but has {len(fallback_dict)} keys."  # pyright: ignore[reportUnknownArgumentType]  # see above
+            )
+        ((model_group, targets),) = fallback_dict.items()  # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType]  # an entry as configured
+        entry: list[object] = [fallback_dict]  # mutable-ok: the one-entry list the check reads
+        if isinstance(targets, list) or check_non_standard_fallback_format(fallbacks=entry):
+            continue
+        raise TypeError(
+            f"Fallbacks for '{model_group}' must be a list of model names, got '{targets}'. "
+            f"Use {{'{model_group}': ['{targets}']}}"
+        )
+
+
+def rejected_fallback_target(request: Mapping[str, object]) -> Exception | None:
+    """The error ``run_async_fallback`` raises before its walk when a client-side request list
+    carries a dict target the request may not set (a server-owned federation field, an ``oidc/``
+    api key): every target of that walk is refused."""
+    from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference
+    from litellm.types.router import reject_server_owned_wif_params
+
+    fallbacks: Final = request.get("fallbacks")
+    if not isinstance(fallbacks, list) or not check_non_standard_fallback_format(fallbacks=fallbacks):  # pyright: ignore[reportUnknownArgumentType]  # a request list as given
+        return None
+    for target in fallbacks:  # pyright: ignore[reportUnknownVariableType]  # see above
+        if isinstance(target, dict):
+            try:
+                reject_server_owned_wif_params(target)  # pyright: ignore[reportUnknownArgumentType]  # see above
+                reject_api_key_secret_reference(target)  # pyright: ignore[reportUnknownArgumentType]  # see above
+            except Exception as error:  # noqa: BLE001  # the walk's own refusal, reported as its failure
+                return error
+    return None
+
+
+# The `mock_testing_*` flags the router reads, and the mock failure each asks router-core for.
+MOCK_FLAGS: Final = MappingProxyType(
+    {
+        "mock_testing_fallbacks": "fallbacks",
+        "mock_testing_context_fallbacks": "context_window",
+        "mock_testing_content_policy_fallbacks": "content_policy",
+        "mock_testing_rate_limit_error": "rate_limit",
+    }
+)
+
+
+def mock_failure(request: Mapping[str, object]) -> str | None:
+    """The mock failure a request asks for (``MockRouterTestingParams.from_kwargs``, then
+    ``_handle_mock_testing_rate_limit_error``): the first fallback flag that is ``True`` (a string
+    is read as a bool), else the rate limit flag when it is ``True`` itself."""
+    from litellm.secret_managers.main import str_to_bool
+
+    for flag in ("mock_testing_fallbacks", "mock_testing_context_fallbacks", "mock_testing_content_policy_fallbacks"):
+        value = request.get(flag)
+        if (str_to_bool(value) if isinstance(value, str) else value) is True:
+            return MOCK_FLAGS[flag]
+    return "rate_limit" if request.get("mock_testing_rate_limit_error") is True else None
+
+
+def mock_error(
+    mock: str, model: str, num_retries: object, request: Mapping[str, object], configured: Mapping[str, object]
+) -> Exception:
+    """The exception the Python router raises for a mock failure; ``configured`` holds the
+    router's fallback lists, which the request's own replace."""
+    import litellm.exceptions as litellm_exceptions
+
+    def listed(name: str) -> object:
+        return request.get(name, configured.get(name))
+
+    if mock == "rate_limit":
+        return litellm_exceptions.RateLimitError(
+            model=model,
+            llm_provider="",
+            message=f"This is a mock exception for model={model}, to trigger a rate limit error.",
+            num_retries=num_retries if isinstance(num_retries, int) else None,
+        )
+    if mock == "context_window":
+        return litellm_exceptions.ContextWindowExceededError(
+            model=model,
+            llm_provider="",
+            message=f"This is a mock exception for model={model}, to trigger a fallback. \
+                    Context_Window_Fallbacks={listed('context_window_fallbacks')}",
+        )
+    if mock == "content_policy":
+        return litellm_exceptions.ContentPolicyViolationError(
+            model=model,
+            llm_provider="",
+            message=f"This is a mock exception for model={model}, to trigger a fallback. \
+                    Context_Policy_Fallbacks={listed('content_policy_fallbacks')}",
+        )
+    return litellm_exceptions.InternalServerError(
+        model=model,
+        llm_provider="",
+        message=f"This is a mock exception for model={model}, to trigger a fallback. Fallbacks={listed('fallbacks')}",
+    )
+
+
+class RustFallbacks:
+    """Fallback members served on the host."""
+
+    def validate_fallbacks(self, fallback_param: Sequence[object] | None) -> None:
+        refuse_malformed_fallbacks(fallback_param)
+
+
+__all__ = [
+    "MOCK_FLAGS",
+    "RustFallbacks",
+    "mock_error",
+    "mock_failure",
+    "refuse_malformed_fallbacks",
+    "rejected_fallback_target",
+]

litellm/router_rust_health.py

@@ -0,0 +1,199 @@
+"""
+The deployment outcome callbacks ``RustCore`` serves as members: ``deployment_callback_on_failure``,
+``deployment_callback_on_success`` and ``sync_deployment_callback_on_success``. router-core counts
+the outcome and decides the cooldown; this module reads litellm's logging kwargs, drops the
+failures the Python router does not hold against the deployment, and drives the Rust call.
+"""
+
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Final
+
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
+
+if TYPE_CHECKING:
+    from litellm.rust_bridge._native import (  # pyright: ignore[reportPrivateUsage]  # the stub's handle
+        RouterCall,
+        RouterCoreHandle,
+    )
+
+
+def _cast_status(exception_status: object) -> int | None:
+    if isinstance(exception_status, int) and not isinstance(exception_status, bool):
+        return exception_status
+    if isinstance(exception_status, str):
+        try:
+            return int(exception_status)
+        except ValueError:
+            return 500
+    return None
+
+
+def failure_status(exception_status: object) -> int | None:
+    """``cast_exception_status_to_int``: a number as given, a numeric string read as one, any
+    other string 500. A number no status can hold reads as no status."""
+    status: Final = _cast_status(exception_status)
+    return status if status is not None and 0 <= status <= 65535 else None
+
+
+def cooled_in_running_loop(cooled: bool) -> bool:
+    """``set_cooldown_deployments`` schedules the cooldown event with ``asyncio.create_task``
+    after writing the cooldown, so outside a running loop it raises ``RuntimeError`` then."""
+    import asyncio
+
+    if cooled:
+        asyncio.get_running_loop()
+    return cooled
+
+
+def _configured_cooldown(model_info: object, litellm_params: Mapping[str, object]) -> float | None:
+    """``_first_present(model_info, litellm_params, key="cooldown_time")``."""
+    for source in (as_str_mapping(model_info), litellm_params):
+        value = source.get("cooldown_time") if source is not None else None
+        if isinstance(value, int | float) and not isinstance(value, bool):
+            return float(value)
+    return None
+
+
+def _success_target(kwargs: Mapping[str, object]) -> tuple[str, Mapping[str, object]] | None:
+    """The deployment id and metadata a success callback counts under, or None."""
+    params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
+    metadata: Final = as_str_mapping(params.get("metadata"))
+    if metadata is None:
+        return None
+    model_info: Final = as_str_mapping(params.get("model_info")) or {}
+    deployment_id: Final = model_info.get("id")
+    if metadata.get("model_group") is None or deployment_id is None:
+        return None
+    return str(deployment_id), metadata
+
+
+class RustHealthCallbacks:
+    """Outcome callbacks read into the Rust core's health state."""
+
+    _handle: "RouterCoreHandle"
+
+    def _drive_sync_steps(self, call: "RouterCall") -> Mapping[str, object]:
+        raise NotImplementedError
+
+    async def _drive_cache_steps(self, call: "RouterCall") -> Mapping[str, object]:
+        raise NotImplementedError
+
+    def _core_get_allowed_fails_from_policy(self, exception: Exception) -> int | None:
+        """The router ``allowed_fails_policy``'s allowance for this exception, decided by router-core."""
+        from litellm.router_rust_core import attempt_failure
+
+        return self._handle.allowed_fails_for(dict(attempt_failure(exception)))
+
+    def deployment_callback_on_failure(
+        self,
+        kwargs: Mapping[str, object],
+        completion_response: object,
+        start_time: object,
+        end_time: object,
+    ) -> bool:
+        """Count the failure and cool the deployment down when the rules say so."""
+        from litellm.litellm_core_utils import core_helpers
+        from litellm.router_rust_core import attempt_failure
+        from litellm.router_utils.cooldown_handlers import (
+            is_background_response_poll_not_found,
+            is_caller_scoped_auth_failure,
+            is_caller_timeout_408,
+            is_failure_outside_selected_deployment,
+            is_video_lookup_not_found,
+        )
+        from litellm.types.router import Deployment
+
+        exception: Final = kwargs.get("exception")
+        if is_failure_outside_selected_deployment(exception if isinstance(exception, BaseException) else None):
+            return False
+        litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
+        model_info: Final = litellm_params.get("model_info", {})
+        if isinstance(exception, Exception) and is_background_response_poll_not_found(exception, litellm_params):
+            return False
+        if is_video_lookup_not_found(exception, kwargs):
+            return False
+        exception_status: Final[object] = getattr(exception, "status_code", "")
+        if is_caller_timeout_408(kwargs, exception_status):  # pyright: ignore[reportArgumentType]  # the status as raised
+            return False
+        info: Final = as_str_mapping(model_info)
+        if info is None:
+            return False
+        deployment_id: Final = info.get("id")
+        if isinstance(deployment_id, str):
+            record: Final = self._handle.deployment(deployment_id)
+            deployment: Final = Deployment(**dict(record)) if record is not None else None  # pyright: ignore[reportArgumentType]  # a model_list entry
+            if is_caller_scoped_auth_failure(deployment, exception_status):  # pyright: ignore[reportArgumentType]  # see above
+                return False
+        if deployment_id is None:
+            return False
+        described: Final[Mapping[str, object]] = (
+            attempt_failure(exception) if isinstance(exception, BaseException) else {"ok": False, "kind": "APIError"}
+        )
+        failure: Final = {**described, "status": failure_status(exception_status)}
+        metadata: Final[object] = core_helpers.get_litellm_metadata_from_kwargs(dict(kwargs))  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # an untyped helper's dict
+        group: Final = (as_str_mapping(metadata) or {}).get("model_group")  # pyright: ignore[reportUnknownArgumentType]  # read as a mapping
+        call: Final = self._handle.report_failure(
+            str(deployment_id),
+            failure,
+            deployment_cooldown=True,
+            count_only=exception_status == "",
+            requested_group=group if isinstance(group, str) else None,
+            cooldown_time=_configured_cooldown(model_info, litellm_params),
+        )
+        return cooled_in_running_loop(bool(self._drive_sync_steps(call).get("cooled")))
+
+    def sync_deployment_callback_on_success(
+        self,
+        kwargs: Mapping[str, object],
+        completion_response: object,
+        start_time: object,
+        end_time: object,
+    ) -> str | None:
+        """Count a success this minute; returns the counter's key."""
+        from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
+
+        if is_batch_retrieve_call_type(kwargs.get("call_type")):
+            return None
+        target: Final = _success_target(kwargs)
+        if target is None:
+            return None
+        key: Final = self._drive_sync_steps(self._handle.count_success(target[0])).get("key")
+        return key if isinstance(key, str) else None
+
+    async def deployment_callback_on_success(
+        self,
+        kwargs: Mapping[str, object],
+        completion_response: object,
+        start_time: object,
+        end_time: object,
+    ) -> None:
+        """Count a success and the call's tokens and request against a tracked deployment."""
+        from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
+        from litellm.router_utils.router_callbacks.track_deployment_metrics import get_counted_usage_tokens
+
+        call_type: Final = kwargs.get("call_type")
+        if call_type == "_arealtime" or is_batch_retrieve_call_type(call_type):
+            return
+        logged: Final = as_str_mapping(kwargs.get("standard_logging_object"))
+        if logged is None:
+            return
+        target: Final = _success_target(kwargs)
+        if target is None:
+            return
+        tokens: Final[object] = logged.get("total_tokens", 0)
+        total: Final = float(tokens) if isinstance(tokens, int | float) else 0.0
+        counted: Final = get_counted_usage_tokens(as_str_mapping(kwargs.get("litellm_params")) or {})
+        await self._drive_cache_steps(
+            self._handle.report_success(
+                target[0],
+                total if counted is None else max(0.0, total - counted),
+                1.0 if counted is None else 0.0,
+            )
+        )
+
+
+__all__ = [
+    "RustHealthCallbacks",
+    "cooled_in_running_loop",
+    "failure_status",
+]

litellm/router_rust_health_routing.py

@@ -0,0 +1,157 @@
+"""
+Health-check routing, allowed-fails policies and the failover flags on the host side of the Rust
+core: how the constructor resolves them, the static document router-core reads them from, and the
+``health()`` members that read the registry's live model list or the background health checks'
+results. router-core makes every routing decision (the health-check filter, the cooldown rules,
+weighted failover); this module only reads Python values into its documents and back.
+
+The rules are the Python router core's, ported (never imported).
+"""
+
+import dataclasses
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Final
+
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.router_utils.health_state_cache import DeploymentHealthCache
+from litellm.types.router import AllowedFailsPolicy
+
+if TYPE_CHECKING:
+    from litellm.caching.caching import DualCache
+    from litellm.router_config import RouterConfig
+    from litellm.types.router import Deployment, DeploymentTypedDict
+
+HEALTH_STATE_KEY: Final = DeploymentHealthCache.CACHE_KEY
+# The failover flags the Python core keeps as plain router attributes, with their defaults.
+FAILOVER_FLAGS: Final = ("enable_weighted_failover", "fallback_on_unsupported_params")
+
+
+def resolved_allowed_fails_policy(value: object) -> AllowedFailsPolicy | None:
+    """The router's ``allowed_fails_policy`` as the constructor keeps it: a dict is read into an
+    ``AllowedFailsPolicy`` (a bad field raises the model's own error), a policy is kept, anything
+    else is no policy."""
+    if isinstance(value, AllowedFailsPolicy):
+        return value
+    if isinstance(value, dict):
+        return AllowedFailsPolicy.model_validate(value)
+    return None
+
+
+def _staleness(threshold: object) -> float:
+    """``health_check_staleness_threshold or DEFAULT_HEALTH_CHECK_INTERVAL * multiplier``."""
+    from litellm.constants import DEFAULT_HEALTH_CHECK_INTERVAL, DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER
+
+    given: Final = threshold if isinstance(threshold, int | float) and not isinstance(threshold, bool) else None
+    return float(given or (DEFAULT_HEALTH_CHECK_INTERVAL * DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER))
+
+
+@dataclasses.dataclass(frozen=True, slots=True)
+class HealthRouting:
+    """The health-check routing settings, as the constructor resolves them."""
+
+    enable_health_check_routing: bool = False
+    health_check_staleness: float = 600.0
+    health_check_ignore_transient_errors: bool = False
+    background_health_check_model_groups: frozenset[str] | None = None
+
+    @classmethod
+    def of(cls, config: "RouterConfig") -> "HealthRouting":
+        groups: Final = config.get("background_health_check_model_groups")
+        return cls(
+            enable_health_check_routing=bool(config.get("enable_health_check_routing")),
+            health_check_staleness=_staleness(config.get("health_check_staleness_threshold")),
+            health_check_ignore_transient_errors=bool(config.get("health_check_ignore_transient_errors")),
+            background_health_check_model_groups=frozenset(groups) if isinstance(groups, Sequence | set) else None,  # pyright: ignore[reportUnknownArgumentType]  # the group names as passed
+        )
+
+    def static_document(self) -> Mapping[str, object]:
+        groups: Final = self.background_health_check_model_groups
+        return {
+            "enable_health_check_routing": True if self.enable_health_check_routing else None,
+            "health_check_staleness_threshold": self.health_check_staleness,
+            "background_health_check_model_groups": sorted(groups) if groups is not None else None,
+        }
+
+    def health_checks(self, cache: "DualCache") -> DeploymentHealthCache:
+        return DeploymentHealthCache(cache=cache, staleness_threshold=self.health_check_staleness)
+
+
+def failover_document(plain: Mapping[str, object]) -> Mapping[str, object]:
+    """The failover flags router-core reads, from the router's plain attributes (read for
+    truthiness, as the Python core's checks read them)."""
+    return {
+        "enable_weighted_failover": True if plain.get("enable_weighted_failover") else None,
+        "fallback_on_unsupported_params": False if not plain.get("fallback_on_unsupported_params", True) else None,
+    }
+
+
+def _model_info(deployment: Mapping[str, object]) -> Mapping[str, object]:
+    return as_str_mapping(deployment.get("model_info")) or {}
+
+
+def is_blocked(deployment: "Deployment") -> bool:
+    """``_core_is_deployment_blocked``."""
+    model_info: Final[object | None] = getattr(deployment, "model_info", None)
+    if model_info is None:
+        return False
+    return getattr(model_info, "blocked", None) is True
+
+
+def all_blocked(deployments: "Sequence[DeploymentTypedDict]") -> bool:
+    """``_core_are_all_deployments_blocked``."""
+    return len(deployments) > 0 and all(
+        _model_info(deployment).get("blocked") is True  # pyright: ignore[reportArgumentType]  # a deployment dict
+        for deployment in deployments
+    )
+
+
+def fully_blocked_model_names(
+    model_list: Sequence[Mapping[str, object]],
+) -> set[str]:  # mutable-ok: the core's type, a fresh set
+    """``_core_get_fully_blocked_model_names``: groups whose every deployment is blocked."""
+    blocked_by_name: Final[dict[str, bool]] = {}  # mutable-ok: built here
+    for deployment in model_list:
+        name = deployment.get("model_name") or ""
+        if not isinstance(name, str) or not name:
+            continue
+        blocked = _model_info(deployment).get("blocked") is True
+        blocked_by_name[name] = blocked_by_name.get(name, True) and blocked
+    return {name for name, fully in blocked_by_name.items() if fully}
+
+
+async def fully_unhealthy_model_names(
+    model_list: Sequence[Mapping[str, object]], health_checks: DeploymentHealthCache, policy: object
+) -> set[str]:  # mutable-ok: the core's type, a fresh set
+    """``_core_async_get_fully_unhealthy_model_names``: groups (and team public names) whose
+    every deployment failed its health check; nothing under an allowed_fails_policy."""
+    if policy is not None:
+        return set()
+    unhealthy_ids: Final = await health_checks.async_get_unhealthy_deployment_ids()
+    if not unhealthy_ids:
+        return set()
+    unhealthy_by_name: Final[dict[str, bool]] = {}  # mutable-ok: built here
+    for deployment in model_list:
+        model_info = _model_info(deployment)
+        names = [deployment.get("model_name") or ""]
+        team_public_model_name = model_info.get("team_public_model_name")
+        if team_public_model_name:
+            names.append(team_public_model_name)
+        unhealthy = model_info.get("id") in unhealthy_ids
+        for name in names:
+            if not isinstance(name, str) or not name:
+                continue
+            unhealthy_by_name[name] = unhealthy_by_name.get(name, True) and unhealthy
+    return {name for name, fully in unhealthy_by_name.items() if fully}
+
+
+__all__ = [
+    "FAILOVER_FLAGS",
+    "HEALTH_STATE_KEY",
+    "HealthRouting",
+    "all_blocked",
+    "failover_document",
+    "fully_blocked_model_names",
+    "fully_unhealthy_model_names",
+    "is_blocked",
+    "resolved_allowed_fails_policy",
+]

litellm/router_rust_members.py

@@ -0,0 +1,97 @@
+"""
+``RustRegistryMembers``: the Rust core's public registry members outside ``registry()`` and
+``apply``: ``add_deployment``, ``set_model_list``, the ``model_list`` attribute, the environment
+gate, the deprecated id hash and ``print_deployment``.
+
+Each one works on the core's host registry (``litellm/router_rust_registry.py``), which hands every
+change to the Rust core. The rules are the Python router core's, ported (never imported).
+"""
+
+import copy
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Final
+
+from litellm._logging import verbose_router_logger
+from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
+from litellm.router_api import deployment_id
+from litellm.router_rust_registry import HostRegistry, model_info_is_active_for_environment
+from litellm.types.router import Deployment
+
+if TYPE_CHECKING:
+    from litellm.router_python_host import PythonHost
+
+
+class RustRegistryMembers:
+    """Mixed into ``RustCore``, which supplies its host registry and host."""
+
+    @property
+    def _registry_state(self) -> HostRegistry:
+        raise NotImplementedError
+
+    @property
+    def _python_host(self) -> "PythonHost":
+        raise NotImplementedError
+
+    def add_deployment(self, deployment: Deployment) -> Deployment | None:
+        """Add a deployment unless its id exists (``add_deployment``)."""
+        return self._registry_state.add_deployment(deployment)
+
+    def set_model_list(self, model_list: Sequence[object]) -> None:
+        """Replace every deployment (``set_model_list``); aliases and wildcard routes stay."""
+        self._registry_state.set_model_list(model_list)
+
+    @property
+    def model_list(self) -> Sequence[object]:
+        """The live model list (the deprecated ``model_list`` attribute)."""
+        return self._registry_state._core_model_list  # pyright: ignore[reportPrivateUsage]  # the core reads its own host registry
+
+    def _assign_model_list(self, model_list: Sequence[object]) -> None:
+        """``router.model_list = [...]``: the list is swapped as given, without reindexing, and the
+        registry version moves once."""
+        self._registry_state._core_model_list = model_list  # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue]  # the list as assigned, as the core keeps it
+        self._registry_state._registry_version += 1  # pyright: ignore[reportPrivateUsage]  # see above
+        self._registry_state._on_change()  # pyright: ignore[reportPrivateUsage]  # see above
+
+    @staticmethod
+    def deployment_is_active_for_environment(deployment: Deployment) -> bool:
+        """Whether LITELLM_ENVIRONMENT admits the deployment (``deployment_is_active_for_environment``)."""
+        return model_info_is_active_for_environment(model_info=deployment.model_info)  # pyright: ignore[reportArgumentType]  # ModelInfo reads like a mapping
+
+    @staticmethod
+    def generate_model_id(model_group: str, litellm_params: Mapping[str, object]) -> str:
+        """The deprecated id hash (``generate_model_id``)."""
+        from litellm.router_deprecated import (
+            _note,  # pyright: ignore[reportPrivateUsage]  # the router's deprecation notice
+        )
+
+        _note("generate_model_id", "litellm.router_api.deployment_id(model_group, litellm_params)")
+        return deployment_id(model_group, litellm_params)
+
+    async def set_response_headers(
+        self, response: object, model_group: str | None = None, request_kwargs: Mapping[str, object] | None = None
+    ) -> object:
+        """Stamp the group's remaining usage and the model id on a response (``set_response_headers``)."""
+        headed: Final[object] = await self._registry_state.set_response_headers(  # pyright: ignore[reportAny]  # the response as the core hands it back
+            response, model_group, None if request_kwargs is None else dict(request_kwargs)
+        )
+        return headed
+
+    def replay_model_cost_registrations(self) -> None:
+        """Re-register the served deployments' cost-map entries (``replay_model_cost_registrations``)."""
+        self._registry_state.replay_model_cost_registrations()
+
+    def print_deployment(self, deployment: Mapping[str, object]) -> Mapping[str, object]:
+        """A copy of the deployment with its api key masked (``print_deployment``)."""
+        try:
+            printed: Final = dict(copy.deepcopy(deployment))
+            litellm_params: Final = printed["litellm_params"]
+            if self._python_host.dynamic().redact_user_api_key_info:
+                masker: Final = SensitiveDataMasker(visible_prefix=2, visible_suffix=0)
+                printed["litellm_params"] = masker.mask_dict(litellm_params)  # pyright: ignore[reportArgumentType]  # the deployment's params dict
+            elif isinstance(litellm_params, dict) and "api_key" in litellm_params:
+                litellm_params["api_key"] = litellm_params["api_key"][:2] + "*" * 10
+        except Exception as e:
+            verbose_router_logger.debug("Error occurred while printing deployment - %s", e)
+            raise
+        else:
+            return printed

litellm/router_rust_plugins.py

@@ -0,0 +1,87 @@
+"""
+Host side of the Rust core's routing-plugin pipeline (``Router(plugins=[...])``).
+
+The Rust core decides where the pipeline applies (only the async pick runs it; the sync pick
+refuses a router with plugins) and which deployments the plugins' candidate models keep
+(``router-core/src/routing_plugins.rs``). The plugins themselves are live Python objects, so this
+module runs them on a ``routing_plugins`` select step, the way the Python router core's
+``_run_routing_plugins`` does, and reports the candidate models left on the request.
+"""
+
+from collections.abc import Mapping, Sequence
+from typing import Final, Protocol
+
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata_variable_name_from_kwargs
+from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
+from litellm.types.router import RoutingContext, RoutingPlugin
+
+SYNC_PATH_MESSAGE: Final = (
+    "Router(plugins=[...]) is configured, but this call resolved to the synchronous "
+    "deployment-selection path, which never runs the routing-plugin pipeline. This "
+    "happens for sync Router methods (e.g. Router.completion()) and for async calls "
+    "with a routing_strategy that has no async-native selector (e.g. legacy "
+    "'usage-based-routing', v1). Silently skipping "
+    "configured plugins would let a policy plugin (e.g. a deny-all rule) be bypassed. "
+    "Use an async Router method with a supported routing_strategy (simple-shuffle, "
+    "usage-based-routing-v2, cost-based-routing, latency-based-routing, least-busy), "
+    "or remove `plugins` from the Router config."
+)
+_CANDIDATES_KEY: Final = "_routing_plugin_candidate_models"
+
+
+class PluginRegistry(Protocol):
+    def _core_get_model_list(
+        self, model_name: str | None = None, team_id: str | None = None
+    ) -> Sequence[Mapping[str, object]] | None: ...
+
+
+def _resolved_litellm_models(
+    registry: PluginRegistry, model: str
+) -> list[str]:  # mutable-ok: the plugins' candidate list
+    """``_core_resolved_litellm_models``: the provider models a call to ``model`` is served by."""
+    return [
+        litellm_model
+        for deployment in registry._core_get_model_list(model_name=model) or ()  # pyright: ignore[reportPrivateUsage]  # the host registry's listing
+        if isinstance(litellm_model := (as_str_mapping(deployment.get("litellm_params")) or {}).get("model"), str)
+        and litellm_model
+    ]
+
+
+def candidate_models(request: Mapping[str, object]) -> Mapping[str, object]:
+    """The answer to a ``routing_plugins`` step: the candidate models the plugins left on the
+    request, ``None`` when none narrowed it."""
+    metadata: Final = as_str_mapping(request.get(get_metadata_variable_name_from_kwargs(dict(request))))
+    return {"models": metadata.get(_CANDIDATES_KEY) if metadata is not None else None}
+
+
+async def run_plugins(
+    plugins: Sequence[RoutingPlugin],
+    registry: PluginRegistry,
+    model_group_alias: Mapping[str, object],
+    model: str,
+    request: dict[str, object],  # mutable-ok: the plugins stamp their outcome on the caller's request
+    messages: object,
+) -> None:
+    """``_run_routing_plugins``: run ``plugins`` in order over ``model``'s candidate models, then
+    stamp their signals (and a narrowed candidate list) on the request's metadata."""
+    from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages
+
+    registered: Final = resolve_model_group_alias(model_group_alias, model) or model
+    models: Final = _resolved_litellm_models(registry, registered)
+    metadata_key: Final = get_metadata_variable_name_from_kwargs(request)
+    metadata: Final = request.setdefault(metadata_key, {})
+    raw_messages: Final[object] = messages or list[object]()
+    context = RoutingContext(  # rebind-ok: each plugin returns the context the next one reads
+        raw_messages=raw_messages,  # pyright: ignore[reportArgumentType]  # the request's messages
+        structured_messages=resolve_structured_messages(messages=messages, request_kwargs=request) or [],  # pyright: ignore[reportArgumentType]  # the request's messages
+        candidate_models=list(models),
+        metadata=metadata,  # pyright: ignore[reportArgumentType]  # the request's metadata dict
+    )
+    for plugin in plugins:
+        context = await plugin.run(context)
+    metadata["routing_plugin_signals"] = context.signals  # pyright: ignore[reportIndexIssue]  # the request's metadata dict
+    if len(context.candidate_models) < len(models):
+        metadata[_CANDIDATES_KEY] = context.candidate_models  # pyright: ignore[reportIndexIssue]  # see above
+
+
+__all__ = ["SYNC_PATH_MESSAGE", "PluginRegistry", "candidate_models", "run_plugins"]

litellm/router_rust_precall.py

@@ -0,0 +1,318 @@
+"""
+Host-side facts for the Rust core's ``enable_pre_call_checks`` filter.
+
+The Rust core decides which deployments the pre-call checks drop and which error a group with
+nothing left raises (``router-core/src/precall.rs``). What needs live Python objects is answered
+here, on a ``pre_call`` select step: the prompt's token count (messages, Responses input and
+instructions, tools and an Anthropic ``system`` block), each deployment's context window from
+its router model info, and which requested params each deployment takes. Ported from the Python
+router core's ``_pre_call_checks`` helpers; it never reaches a Python router core.
+"""
+
+from collections.abc import Callable, Mapping, Sequence
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, Protocol
+
+from litellm._logging import verbose_router_logger
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
+from litellm.litellm_core_utils.token_counter import (
+    offload_token_count,
+    served_openai_models,
+    single_encoding_openai_model,
+)
+from litellm.router_python_host import python_host
+from litellm.router_strategy.complexity_router.context_compaction import compaction_pending
+from litellm.types.router import LiteLLM_Params
+
+if TYPE_CHECKING:
+    from litellm.router_python_host import PythonHost
+
+# The chat params a deployment must take when the request sends them (`_PRE_CALL_CHECK_FILTERED_PARAMS`).
+_FILTERED_PARAMS: Final = frozenset({"response_format"})
+
+
+class PreCallRegistry(Protocol):
+    """The host registry reads the facts need."""
+
+    def _core_get_model_info(self, id: str) -> Mapping[str, object] | None: ...  # the registry's name
+
+    def _core_deployments_named(self, model_name: str) -> Sequence[Mapping[str, object]] | None: ...
+
+    def _core_get_router_model_info(
+        self, deployment: Mapping[str, object], received_model_name: str
+    ) -> Mapping[str, object]: ...
+
+
+def count_tokens(
+    host: "PythonHost",
+    messages: object,
+    input: object,  # the request field's name
+    request_kwargs: Mapping[str, object] | None,
+    tokenizer_model: str,
+) -> int:
+    """``_count_pre_call_check_tokens``: messages, or Responses input normalized to messages, plus
+    the tools and Anthropic ``system`` block the message list never carries."""
+    from litellm.llms.anthropic.messages.native.utils import anthropic_system_to_openai_message
+
+    extras: Final[Mapping[str, object]] = request_kwargs if request_kwargs is not None else MappingProxyType({})
+    raw_instructions: Final = extras.get("instructions")
+    instructions: Final = raw_instructions if isinstance(raw_instructions, str) else None
+    raw_tools: Final = extras.get("tools")
+    tools: Final = raw_tools if _is_list(raw_tools) and raw_tools else None
+    system_message: Final = anthropic_system_to_openai_message(extras.get("system"))
+    counter: Final = host.invoke("token_counter")  # pyright: ignore[reportUnknownMemberType]  # the host's overloads name litellm's untyped returns
+    if messages is not None:
+        counted_messages: Final = (system_message, *_items(messages)) if system_message is not None else messages
+        return counter(model=tokenizer_model, messages=counted_messages, tools=tools)
+    if input is not None:
+        from litellm.responses.litellm_completion_transformation.transformation import (
+            LiteLLMCompletionResponsesConfig,
+        )
+
+        input_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(  # pyright: ignore[reportUnknownMemberType]  # its input type is untyped
+            input=input,  # pyright: ignore[reportArgumentType]  # str | list, as the request carries it
+            responses_api_request={"instructions": instructions} if instructions is not None else {},
+            only_since_compaction=True,
+        )
+        return counter(model=tokenizer_model, messages=input_messages, tools=tools)
+    raise ValueError("Either messages or input must be provided to count tokens")
+
+
+def _max_input_tokens(registry: PreCallRegistry, model: str, deployment: Mapping[str, object]) -> int | None:
+    """``_deployment_max_input_tokens``: the declared context window, or None when there is none or
+    the deployment cannot be resolved."""
+    try:
+        model_info: Final = registry._core_get_router_model_info(  # pyright: ignore[reportPrivateUsage]  # the host registry's lookup
+            deployment=deployment, received_model_name=model
+        )
+    except Exception as e:  # noqa: BLE001  # best-effort: an unmappable deployment must not hide the others
+        verbose_router_logger.debug("litellm.router.py::_deployment_max_input_tokens: skipping deployment. Got - %s", e)
+        return None
+    max_input_tokens: Final = model_info.get("max_input_tokens")
+    return max_input_tokens if isinstance(max_input_tokens, int) else None
+
+
+def _items(value: object) -> tuple[object, ...]:
+    """The items of a list, tuple or set; nothing for any other value."""
+    return tuple(value) if isinstance(value, list | tuple | set | frozenset) else ()  # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType]  # the items are read as objects
+
+
+def _is_list(value: object) -> bool:
+    return isinstance(value, list)
+
+
+def _declared_param_allowlist(params: Mapping[str, object]) -> frozenset[str]:
+    declared: Final = params.get("allowed_openai_params")
+    return frozenset(entry for entry in _items(declared) if isinstance(entry, str))
+
+
+def _capability_model_and_provider(deployment_params: Mapping[str, object], group: str) -> tuple[str, str]:
+    model_label: Final = str(deployment_params.get("model") or group)
+    custom_llm_provider: Final = deployment_params.get("custom_llm_provider")
+    authenticating_provider: Final = declared_authenticating_provider(
+        model_label,
+        custom_llm_provider,  # pyright: ignore[reportArgumentType]  # passed as configured, as the Python core does
+    )
+    if authenticating_provider is not None:
+        return model_label.removeprefix(f"{authenticating_provider}/"), authenticating_provider
+    model, provider, _, _ = python_host.invoke("get_llm_provider")(  # pyright: ignore[reportUnknownMemberType]  # see count_tokens
+        model=model_label,
+        custom_llm_provider=custom_llm_provider,
+        litellm_params=LiteLLM_Params.model_validate(dict(deployment_params)),
+    )
+    return model, provider
+
+
+def _supports_param(deployment: Mapping[str, object], group: str, param: str) -> bool:
+    """``_deployment_supports_param``."""
+    deployment_params: Final = as_str_mapping(deployment.get("litellm_params"))
+    if not deployment_params:
+        return True
+    deployment_model_info: Final = as_str_mapping(deployment.get("model_info"))
+    base_model: Final = (
+        deployment_model_info.get("base_model") if deployment_model_info else None
+    ) or deployment_params.get("base_model")
+    try:
+        model, custom_llm_provider = _capability_model_and_provider(deployment_params, group)
+        supported: Final = python_host.invoke("get_supported_openai_params")(  # pyright: ignore[reportUnknownMemberType]  # see count_tokens
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            base_model=base_model if isinstance(base_model, str) else None,
+        )
+    except Exception as e:  # noqa: BLE001  # best-effort filter: an unresolvable provider must not narrow the request
+        verbose_router_logger.debug(
+            "litellm.router.py::_deployment_supports_param: keeping %s for model=%s. Got - %s", param, group, e
+        )
+        return True
+    return supported is None or param in supported
+
+
+def _lacks_params(deployment: Mapping[str, object], group: str, params: frozenset[str]) -> bool:
+    return any(not _supports_param(deployment, group, param) for param in params)
+
+
+def _rejects_params(
+    deployment: Mapping[str, object], group: str, params: frozenset[str], request_kwargs: Mapping[str, object]
+) -> bool:
+    """``_deployment_rejects_params``: lacks a param neither the deployment nor the request drops or allows."""
+    call_settings: Final[Mapping[str, object]] = {
+        **(as_str_mapping(deployment.get("litellm_params")) or {}),
+        **request_kwargs,
+    }
+    if call_settings.get("drop_params") is True:
+        return False
+    dropped: Final = call_settings.get("additional_drop_params")
+    dropped_names: Final = frozenset(
+        entry for entry in (_items(dropped) if _is_list(dropped) else ()) if isinstance(entry, str)
+    )
+    return _lacks_params(deployment, group, params - _declared_param_allowlist(call_settings) - dropped_names)
+
+
+def _requested_params(request_kwargs: Mapping[str, object]) -> frozenset[str]:
+    """``_pre_call_check_requested_chat_params``."""
+    from litellm import utils as litellm_utils
+
+    return _filtered_params(
+        litellm_utils.get_non_default_params(passed_params=dict(request_kwargs))  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # an untyped helper
+    )
+
+
+def _filtered_params(requested: Mapping[object, object]) -> frozenset[str]:
+    return frozenset(
+        param
+        for param, value in requested.items()
+        if isinstance(param, str) and param in _FILTERED_PARAMS and not isinstance(value, str)
+    )
+
+
+class PreCallFacts:
+    """The facts of one ``pre_call`` step."""
+
+    def __init__(
+        self,
+        host: "PythonHost",
+        registry: PreCallRegistry,
+        step: Mapping[str, object],
+        records: Callable[[str], Mapping[str, object]],
+        request: Mapping[str, object] | None,
+        messages: object,
+        input: object,  # the request field's name
+    ) -> None:
+        self._host: Final = host
+        self._registry: Final = registry
+        self._model: Final = str(step.get("model_group"))
+        self._name: Final = str(step.get("model"))
+        self._candidates: Final = [str(id) for id in _items(step.get("candidates"))]
+        self._pool: Final = [str(id) for id in _items(step.get("pool"))]
+        self._records: Final = records
+        self._request: Final = request
+        self._messages: Final = messages
+        self._input: Final = input
+        self._has_input: Final = messages is not None or input is not None
+        self._windows: Final = {id: _max_input_tokens(registry, self._model, records(id)) for id in self._candidates}
+
+    def _record(self, id: str) -> Mapping[str, object]:  # a deployment id
+        return self._records(id)
+
+    def _needs_count(self) -> bool:
+        return self._has_input and any(limit is not None for limit in self._windows.values())
+
+    def _tokenizer_model(self) -> str:
+        named: Final = self._registry._core_deployments_named(self._model) or ()  # pyright: ignore[reportPrivateUsage]  # the host registry's index
+        return (
+            single_encoding_openai_model(
+                served_openai_models(named) or served_openai_models(self._record(id) for id in self._candidates)
+            )
+            or ""
+        )
+
+    def _count(self) -> int | None:
+        try:
+            return count_tokens(self._host, self._messages, self._input, self._request, self._tokenizer_model())
+        except Exception as e:  # noqa: BLE001  # best-effort: an uncountable prompt must not fail the request
+            verbose_router_logger.error(
+                "litellm.router.py::_acount_pre_call_check_tokens: failed to count tokens. Got - %s", e
+            )
+            return None
+
+    async def _count_async(self) -> int | None:
+        try:
+            return await offload_token_count(count_tokens)(
+                self._host, self._messages, self._input, self._request, self._tokenizer_model()
+            )
+        except Exception as e:  # noqa: BLE001  # best-effort: an uncountable prompt must not fail the request
+            verbose_router_logger.error(
+                "litellm.router.py::_acount_pre_call_check_tokens: failed to count tokens. Got - %s", e
+            )
+            return None
+
+    def _answer(self, input_tokens: int | None) -> Mapping[str, object]:
+        if not self._has_input:
+            return {"has_input": False}
+        request: Final[Mapping[str, object]] = self._request if self._request is not None else {}
+        params: Final = (
+            _requested_params(request)
+            if self._request is not None and self._host.dynamic().drop_params is False
+            else frozenset[str]()
+        )
+        deployments: Final[dict[str, dict[str, object]]] = {}  # mutable-ok: built here, then returned
+        for id in dict.fromkeys([*self._candidates, *self._pool]):
+            record = self._record(id)
+            deployments[id] = {
+                "max_input_tokens": self._windows.get(id),
+                "lacks_params": bool(params) and id in self._candidates and _lacks_params(record, self._model, params),
+                "rejects_params": bool(params) and _rejects_params(record, self._model, params, request),
+            }
+        allowed_region: Final = request.get("allowed_model_region")
+        return {
+            "has_input": True,
+            "countable": not compaction_pending(self._request),
+            "input_tokens": input_tokens,
+            "allowed_model_region": allowed_region,
+            "requested_params": sorted(params),
+            "deployments": deployments,
+        }
+
+    def answer(self) -> Mapping[str, object]:
+        """The facts on the sync path: the prompt is counted inline."""
+        return self._answer(self._count() if self._needs_count() else None)
+
+    async def answer_async(self) -> Mapping[str, object]:
+        """The facts on the async path: the prompt is counted off the event loop."""
+        return self._answer(await self._count_async() if self._needs_count() else None)
+
+
+def request_carries_image(messages: object, request: Mapping[str, object]) -> bool:
+    """``_request_carries_image``: the messages (or the request's own messages or input) hold an image."""
+    from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content
+
+    return request_contains_image_content(messages or request.get("messages") or request.get("input"))
+
+
+def declares_no_vision(deployment: Mapping[str, object], group: str) -> bool:
+    """``_deployment_declares_no_vision``: ``model_info.supports_vision`` is ``False``, or, when it
+    is unset, the cost map says the model has vision explicitly disabled."""
+    from litellm import utils as litellm_utils
+
+    model_info: Final = as_str_mapping(deployment.get("model_info")) or {}
+    declared: Final = model_info.get("supports_vision")
+    if declared is not None:
+        return declared is False
+    params: Final = as_str_mapping(deployment.get("litellm_params")) or {}
+    if not params:
+        return False
+    base_model: Final = model_info.get("base_model") or params.get("base_model")
+    try:
+        model, custom_llm_provider = _capability_model_and_provider(params, group)
+        return litellm_utils.is_vision_explicitly_disabled(
+            base_model if isinstance(base_model, str) else model, custom_llm_provider
+        )
+    except Exception as e:  # noqa: BLE001  # best-effort filter: an unresolvable provider must not narrow the request
+        verbose_router_logger.debug(
+            "litellm.router.py::_deployment_declares_no_vision: keeping deployment for model=%s. Got - %s", group, e
+        )
+        return False
+
+
+__all__ = ["PreCallFacts", "PreCallRegistry", "count_tokens", "declares_no_vision", "request_carries_image"]

litellm/router_rust_prompt_management.py

@@ -0,0 +1,131 @@
+"""
+Prompt-management groups on the Rust router core (``_is_prompt_management_model`` and
+``_prompt_management_factory``): a group of exactly one deployment whose model names a prompt
+management integration (``langfuse/...``) or ``litellm_agent/...``. The request is not routed with
+retries and fallbacks: the deployment is picked, its prompt rendered by the request's logging
+object, and the rendered model called directly, or routed when it names a group.
+"""
+
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Final
+
+from litellm.router_rust_endpoints import PLANS
+from litellm.router_rust_shaping import as_dict, update_kwargs_with_deployment
+
+if TYPE_CHECKING:
+    from litellm.router_rust_core import RustCore
+
+_PROMPT_MANAGEMENT_PARAMS: Final = frozenset(
+    {"bitbucket_config", "dotprompt_config", "prompt_id", "prompt_variables", "prompt_label", "prompt_version"}
+)
+
+
+def is_prompt_management_model(core: "RustCore", model: str) -> bool:
+    """`_is_prompt_management_model`: one deployment, whose model's provider prefix is a known
+    custom-logger-compatible callback."""
+    model_list: Final = core._hosted._core_get_model_list(model_name=model)  # pyright: ignore[reportPrivateUsage]  # the core's host registry
+    if model_list is None or len(model_list) != 1:
+        return False
+    litellm_model: Final = as_dict(model_list[0].get("litellm_params")).get("model")
+    if not isinstance(litellm_model, str) or "/" not in litellm_model:
+        return False
+    known: Final = core._host.dynamic().known_custom_logger_compatible_callbacks  # pyright: ignore[reportPrivateUsage]  # the core's host
+    return litellm_model.split("/")[0] in known
+
+
+def _logging_and_kwargs(
+    kwargs: dict[str, object],  # mutable-ok: the request kwargs the Python router mutates in place
+) -> tuple[object, dict[str, object]]:  # mutable-ok: same
+    """The request's logging object, else the one `function_setup` builds (with the kwargs it returns)."""
+    from litellm.utils import Rules, function_setup, get_utc_datetime
+
+    logging_obj: Final = kwargs.get("litellm_logging_obj")
+    if logging_obj is not None:
+        return logging_obj, kwargs
+    return function_setup(  # pyright: ignore[reportUnknownVariableType]  # litellm's untyped helpers
+        **{  # pyright: ignore[reportArgumentType]  # the request's keyword payload
+            "original_function": "acompletion",
+            "rules_obj": Rules(),
+            "start_time": get_utc_datetime(),
+            **kwargs,
+        }
+    )
+
+
+async def prompt_management_acompletion(
+    core: "RustCore",
+    model: str,
+    messages: object,
+    request: Mapping[str, object],
+) -> object:
+    """`_prompt_management_factory`, on the request as `acompletion` prepared it."""
+    from litellm.utils import get_non_default_completion_params
+
+    # `acompletion` hands the routing function along; function_setup consumes it.
+    logging_obj, kwargs = _logging_and_kwargs({**request, "original_function": core.acompletion})
+    caller_timeout: Final = kwargs.get("timeout") or kwargs.get("request_timeout")
+    caller_stream_timeout: Final = kwargs.get("stream_timeout")
+    specific_deployment: Final = kwargs.pop("specific_deployment", None)
+    picked: Final = await core.select(
+        model,
+        kwargs,  # pyright: ignore[reportUnknownArgumentType]  # the request kwargs
+        kind="pick",
+        specific_deployment=bool(specific_deployment),
+        messages=messages,
+    )
+    deployment: Final = as_dict(picked)
+    update_kwargs_with_deployment(
+        core._shaping,  # pyright: ignore[reportPrivateUsage]  # the core's own settings
+        deployment,
+        kwargs,  # pyright: ignore[reportUnknownArgumentType]  # the request kwargs
+        cache=core._cache,  # pyright: ignore[reportPrivateUsage]  # the routing cache the shaping refunds into
+        splits_stream_timeout=True,
+    )
+    params: Final = as_dict(deployment.get("litellm_params"))
+    data: Final = dict(params)
+    litellm_model: Final = data.get("model")
+    is_litellm_agent_model: Final = isinstance(litellm_model, str) and litellm_model.startswith("litellm_agent/")
+    prompt_id: Final = kwargs.get("prompt_id") or params.get("prompt_id")
+    prompt_variables: Final = kwargs.get("prompt_variables") or params.get("prompt_variables")
+    prompt_label: Final = kwargs.get("prompt_label") or params.get("prompt_label")
+    if not is_litellm_agent_model and (prompt_id is None or not isinstance(prompt_id, str)):
+        raise ValueError(f"Prompt ID is not set or not a string. Got={prompt_id}, type={type(prompt_id)}")
+    if prompt_variables is not None and not isinstance(prompt_variables, dict):
+        raise ValueError(
+            f"Prompt variables is set but not a dictionary. Got={prompt_variables}, type={type(prompt_variables)}"
+        )
+    rendered_model, rendered_messages, optional_params = logging_obj.get_chat_completion_prompt(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue, reportOptionalMemberAccess]  # the request's Logging object
+        model=litellm_model,  # pyright: ignore[reportArgumentType]  # a request value, passed as the Python router passes it
+        messages=messages,  # pyright: ignore[reportArgumentType]  # a request value, passed as the Python router passes it
+        non_default_params=get_non_default_completion_params(kwargs=kwargs),  # pyright: ignore[reportArgumentType]  # the request kwargs
+        prompt_id=prompt_id,  # pyright: ignore[reportArgumentType]  # a request value, passed as the Python router passes it
+        prompt_variables=prompt_variables,
+        prompt_label=prompt_label,  # pyright: ignore[reportArgumentType]  # a request value, passed as the Python router passes it
+        request_kwargs=kwargs,
+        injected_for_every_deployment=True,
+    )
+    filtered_data: Final = {key: value for key, value in data.items() if key not in _PROMPT_MANAGEMENT_PARAMS}
+    final: Final[dict[str, object]] = {**filtered_data, **kwargs, **optional_params}  # pyright: ignore[reportUnknownVariableType]  # mutable-ok: the call's kwargs
+    final["model"] = rendered_model
+    final["messages"] = rendered_messages
+    final["litellm_logging_obj"] = logging_obj
+    final["prompt_id"] = prompt_id
+    final["prompt_variables"] = prompt_variables
+    final["prompt_label"] = prompt_label
+    target_list: Final = core._hosted._core_get_model_list(model_name=str(rendered_model))  # pyright: ignore[reportPrivateUsage, reportUnknownArgumentType]  # the core's host registry
+    if target_list is None or len(target_list) == 0:  # a direct call to the rendered model
+        _ = final.pop("original_function")
+        return await core._host.invoke("acompletion")(**final)  # pyright: ignore[reportPrivateUsage, reportUnknownMemberType]  # the core's host
+    timeouts: Final = {
+        key: value
+        for key, value in (
+            ("timeout", caller_timeout if caller_timeout is not None else filtered_data.get("timeout")),
+            (
+                "stream_timeout",
+                caller_stream_timeout if caller_stream_timeout is not None else filtered_data.get("stream_timeout"),
+            ),
+        )
+        if value is not None
+    }
+    routed: Final = {key: value for key, value in final.items() if key not in ("timeout", "stream_timeout")}
+    return await core._drive(PLANS["acompletion"], {**routed, **timeouts}, prepared_already=True)  # pyright: ignore[reportPrivateUsage]  # the core drives its own call

litellm/router_rust_replay.py

@@ -0,0 +1,30 @@
+"""The Rust core's live routers, whose deployments a price data reload re-registers in
+``litellm.model_cost`` until they retire (``_live_routers`` / ``_replay_live_router_model_cost``)."""
+
+from __future__ import annotations
+
+import weakref
+from typing import TYPE_CHECKING, Final
+
+from litellm.utils import add_live_deployment_replay
+
+if TYPE_CHECKING:
+    from litellm.router_rust_registry import HostRegistry
+
+_live_registries: Final[weakref.WeakSet[HostRegistry]] = weakref.WeakSet()
+
+
+def track_live_registry(registry: HostRegistry) -> None:
+    _live_registries.add(registry)
+
+
+def untrack_live_registry(registry: HostRegistry) -> None:
+    _live_registries.discard(registry)
+
+
+def _replay_live_registries() -> None:
+    for registry in tuple(_live_registries):
+        registry.replay_model_cost_registrations()
+
+
+add_live_deployment_replay(_replay_live_registries)

litellm/router_rust_retries.py

@@ -0,0 +1,243 @@
+"""
+The retry members ``RustCore`` serves: ``should_retry_this_error`` and
+``get_num_retries_from_retry_policy``. router-core decides both; this module only describes the
+Python exception the way an attempt failure is described and reads the answer back.
+"""
+
+import dataclasses
+from collections.abc import Iterable, Mapping, Sequence
+from typing import TYPE_CHECKING, Final
+
+from litellm.types.router import RetryPolicy
+
+if TYPE_CHECKING:
+    from litellm.router_rust_core import Resolved
+    from litellm.rust_bridge._native import (  # pyright: ignore[reportPrivateUsage]  # the stub's handle
+        RouterCoreHandle,
+    )
+
+
+def resolved_retry_policy(value: object) -> RetryPolicy | None:
+    """The router's ``retry_policy`` as the Python router's constructor keeps it: a dict is read
+    into a ``RetryPolicy``, a ``RetryPolicy`` is kept, anything else is no policy."""
+    if isinstance(value, RetryPolicy):
+        return value
+    if isinstance(value, Mapping):
+        fields: Final[Mapping[str, int | None]] = value  # pyright: ignore[reportUnknownVariableType]  # the policy's fields
+        return RetryPolicy(**fields)  # pyright: ignore[reportArgumentType]  # validated by the model
+    return None
+
+
+def _is_policy(entry: object) -> bool:
+    return isinstance(entry, RetryPolicy | Mapping)
+
+
+def _reads_as_policy(entry: object) -> bool:
+    """Whether ``resolve_retry_policy`` reads ``entry`` without raising: no entry, a ``RetryPolicy``,
+    or a mapping ``RetryPolicy(**entry)`` accepts."""
+    if entry is None or isinstance(entry, RetryPolicy):
+        return True
+    if not isinstance(entry, Mapping):
+        return False
+    try:
+        _ = RetryPolicy(**entry)  # pyright: ignore[reportUnknownArgumentType]  # validated by the model
+    except Exception:  # noqa: BLE001  # any error the lookup would raise
+        return False
+    return True
+
+
+def is_policy_per_group(policies: object) -> bool:
+    """Whether ``model_group_retry_policy`` is what router-core's settings hold: a mapping of
+    group names to policies (or nothing)."""
+    if policies is None:
+        return True
+    if not isinstance(policies, Mapping):
+        return False
+    entries: Final[Mapping[object, object]] = policies  # pyright: ignore[reportUnknownVariableType]  # read as objects
+    return all(isinstance(group, str) and _is_policy(entry) for group, entry in entries.items())
+
+
+@dataclasses.dataclass(frozen=True, slots=True)
+class GroupPolicies:
+    """A ``model_group_retry_policy`` as router-core reads it, resolved the way
+    ``resolve_retry_policy`` reads it for each group the call may evaluate.
+
+    ``document`` holds a group's policy where the lookup finds one, and an empty policy (which
+    grants nothing) where it finds ``None``; a group it does not find is absent, so the router's
+    ``retry_policy`` applies. ``raising`` names the groups whose lookup raises (an ``in``/``[]``
+    the value does not support, or an entry that is not a policy, whose fields
+    ``get_num_retries_from_retry_policy`` then fails to read): the Python retry loop raises that
+    error in place of the attempt's on a hop's first failure.
+    """
+
+    document: Mapping[str, object]
+    raising: frozenset[str]
+
+
+def _looked_up(policies: object, group: str) -> tuple[bool, object]:
+    """``(found, entry)`` as ``resolve_retry_policy`` looks ``group`` up; raises as it raises."""
+    found: Final = group in policies  # pyright: ignore[reportOperatorIssue]  # any value, as the Python router reads it
+    return (True, policies[group]) if found else (False, None)  # pyright: ignore[reportIndexIssue, reportUnknownVariableType]  # see above
+
+
+def _policy_document(entry: object) -> object:
+    """A readable entry as router-core reads it: no entry grants nothing, a mapping is read the way
+    ``RetryPolicy(**entry)`` reads it."""
+    if entry is None:
+        return {}
+    if isinstance(entry, Mapping):
+        return RetryPolicy(**entry)  # pyright: ignore[reportUnknownArgumentType]  # checked by _reads_as_policy
+    return entry
+
+
+def group_policies(policies: object, groups: Iterable[str]) -> GroupPolicies:
+    """``policies`` resolved for ``groups`` (a mapping is read by its own keys)."""
+    if policies is None:
+        return GroupPolicies({}, raising=frozenset())
+    if isinstance(policies, Mapping):
+        entries: Final[Mapping[object, object]] = policies  # pyright: ignore[reportUnknownVariableType]  # read as objects
+        # A key that is no string never matches a group name.
+        named: Final = {group: entry for group, entry in entries.items() if isinstance(group, str)}
+        readable: Final = {group: entry for group, entry in named.items() if _reads_as_policy(entry)}
+        return GroupPolicies(
+            {group: _policy_document(entry) for group, entry in readable.items()},
+            raising=frozenset(named) - frozenset(readable),
+        )
+    document: Final[dict[str, object]] = {}  # mutable-ok: built here; router-core reads it as plain JSON
+    raising: Final[set[str]] = set()  # mutable-ok: the groups whose lookup raises
+    for group in groups:
+        try:
+            found, entry = _looked_up(policies, group)
+        except Exception:  # noqa: BLE001  # the Python retry loop raises it on a hop's first failure
+            raising.add(group)
+            continue
+        if not found:
+            continue
+        if not _reads_as_policy(entry):
+            raising.add(group)
+            continue
+        document[group] = _policy_document(entry)
+    return GroupPolicies(document, raising=frozenset(raising))
+
+
+def static_group_policies(policies: object) -> object:
+    """The router's ``model_group_retry_policy`` as router-core's static settings hold it. A value
+    that is not a mapping of policies is resolved per request instead (``group_policies``)."""
+    return group_policies(policies, ()).document if isinstance(policies, Mapping) else None  # pyright: ignore[reportUnknownArgumentType]  # read as objects
+
+
+_STREAM_POLICY_ENDPOINTS: Final = frozenset({"aanthropic_messages", "anthropic_messages"})
+
+
+def raising_groups(
+    endpoint: str, request: Mapping[str, object], policies: object, resolved: GroupPolicies
+) -> Sequence[str]:
+    """The groups whose ``model_group_retry_policy`` lookup raises on a hop's first failure, unless
+    the retry loop never reads it: the request opted out with ``num_retries=0``, or it is a mapping
+    on a streamed Anthropic messages call, which resolves such an entry to no policy at all
+    (``_anthropic_messages_resolved_retry_policy``)."""
+    if not resolved.raising or request.get("num_retries") == 0:
+        return ()
+    if endpoint in _STREAM_POLICY_ENDPOINTS and request.get("stream") is True and isinstance(policies, Mapping):
+        return ()
+    return sorted(resolved.raising)
+
+
+def policy_lookup_error(original: BaseException, model_group: str, retry_policy: object, policies: object) -> Exception:
+    """The error ``get_num_retries_from_retry_policy`` raises for ``model_group`` after ``original``
+    failed (the Python retry loop raises it in the attempt's place, chained to it); a lookup that
+    unexpectedly reads is reported as such."""
+    from litellm.litellm_core_utils.get_retry_from_policy import get_num_retries_from_retry_policy
+
+    traceback: Final = original.__traceback__
+    try:
+        # Looked up while `original` is being handled, as in the retry loop's `except` block, so
+        # the error chains to it the way Python chains it.
+        try:
+            raise original
+        except BaseException:  # noqa: BLE001  # re-raised only to be handled here
+            _ = get_num_retries_from_retry_policy(
+                exception=original,  # pyright: ignore[reportArgumentType]  # any exception the hop raised
+                retry_policy=retry_policy,  # pyright: ignore[reportArgumentType]  # the router's resolved policy
+                model_group=model_group,
+                model_group_retry_policy=policies,  # pyright: ignore[reportArgumentType]  # read as the Python router reads it
+            )
+    except Exception as raised:  # noqa: BLE001  # the error the hop ends with
+        return raised
+    finally:
+        original.__traceback__ = traceback  # rebind-ok: put back as the hop left it
+    return RuntimeError(f"the model_group_retry_policy lookup for {model_group!r} did not raise")
+
+
+def router_policy_retries(
+    exception: Exception, retry_policy: object, policies: object, model_group: str | None
+) -> int | None:
+    """``get_num_retries_from_retry_policy`` for a router ``model_group_retry_policy`` that is not
+    a mapping of policies, read as the Python router reads it (raising as it raises)."""
+    from litellm.litellm_core_utils.get_retry_from_policy import get_num_retries_from_retry_policy
+
+    return get_num_retries_from_retry_policy(
+        exception=exception,
+        retry_policy=retry_policy,  # pyright: ignore[reportArgumentType]  # the router's resolved policy
+        model_group=model_group,
+        model_group_retry_policy=policies,  # pyright: ignore[reportArgumentType]  # read as the Python router reads it
+    )
+
+
+def _count(deployments: object) -> int:
+    return len(deployments) if isinstance(deployments, list) else 0  # pyright: ignore[reportUnknownArgumentType]  # only the length is read
+
+
+def _filled(fallbacks: object) -> bool:
+    return isinstance(fallbacks, Sequence) and len(fallbacks) > 0  # pyright: ignore[reportUnknownArgumentType]  # only the length is read
+
+
+class RustRetries:
+    """Retry members read from the Rust core's handle."""
+
+    _handle: "RouterCoreHandle"
+    _resolved: "Resolved"
+
+    def should_retry_this_error(
+        self,
+        error: Exception,
+        healthy_deployments: list[object] | None = None,  # mutable-ok: the Python router's signature
+        all_deployments: list[object] | None = None,  # mutable-ok: same
+        context_window_fallbacks: list[object] | None = None,  # mutable-ok: same
+        content_policy_fallbacks: list[object] | None = None,  # mutable-ok: same
+        regular_fallbacks: list[object] | None = None,  # mutable-ok: same
+    ) -> bool:
+        """``True`` when the error is retried in the same group; otherwise the error is raised."""
+        from litellm.router_rust_core import attempt_failure
+
+        retried: Final = self._handle.should_retry(
+            dict(attempt_failure(error)),
+            (_count(healthy_deployments), _count(all_deployments)),
+            (context_window_fallbacks is not None, content_policy_fallbacks is not None, _filled(regular_fallbacks)),
+        )
+        if not retried:
+            raise error
+        return True
+
+    def get_num_retries_from_retry_policy(self, exception: Exception, model_group: str | None = None) -> int | None:
+        from litellm.router_rust_core import attempt_failure
+
+        resolved: Final = self._resolved
+        if not is_policy_per_group(resolved.model_group_retry_policy):
+            return router_policy_retries(
+                exception, resolved.retry_policy, resolved.model_group_retry_policy, model_group
+            )
+        return self._handle.retry_policy_retries(dict(attempt_failure(exception)), model_group)
+
+
+__all__ = [
+    "GroupPolicies",
+    "RustRetries",
+    "group_policies",
+    "is_policy_per_group",
+    "policy_lookup_error",
+    "raising_groups",
+    "resolved_retry_policy",
+    "router_policy_retries",
+    "static_group_policies",
+]

litellm/router_rust_selection.py

@@ -0,0 +1,405 @@
+"""
+Host-side strategy selectors for ``RustCore``.
+
+The Rust core decides which strategy picks and whose state it reads (``strategy::routing_context``:
+a request override, then the routing group, then the router's own strategy) and asks the host to
+run the stateful strategies it does not implement natively through a ``select`` step. The
+strategies themselves are the standalone ``litellm/router_strategy`` loggers, which double as
+``CustomLogger`` callbacks that record traffic in the host's routing cache. This module builds,
+registers and retires them the way the Python router's ``routing_strategy_init``,
+``_init_routing_groups`` and ``_get_override_strategy_selector`` do. It never reaches a Python
+router core.
+"""
+
+import threading
+from collections.abc import Iterable, Mapping, Sequence
+from itertools import chain
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, TypeAlias
+
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy
+from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
+from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
+from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler
+from litellm.router_strategy.lowest_tpm_rpm import LowestTPMLoggingHandler
+from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2
+from litellm.types.router import RoutingGroup, RoutingStrategy
+
+if TYPE_CHECKING:
+    from litellm.caching.dual_cache import DualCache
+    from litellm.router_python_host import PythonHost
+
+Selector: TypeAlias = (
+    LeastBusyLoggingHandler
+    | LowestCostLoggingHandler
+    | LowestLatencyLoggingHandler
+    | LowestTPMLoggingHandler
+    | LowestTPMLoggingHandler_v2
+)
+
+# The legacy router attribute that holds the default group's selector, per strategy.
+SELECTOR_ATTRIBUTES: Final = MappingProxyType(
+    {
+        "least-busy": "leastbusy_logger",
+        "usage-based-routing": "lowesttpm_logger",
+        "usage-based-routing-v2": "lowesttpm_logger_v2",
+        "latency-based-routing": "lowestlatency_logger",
+        "cost-based-routing": "lowestcost_logger",
+    }
+)
+
+
+def normalize_strategy(strategy: object) -> str | None:
+    """A strategy as the string Python compares against (``_normalize_strategy``)."""
+    if isinstance(strategy, RoutingStrategy):
+        return strategy.value
+    return strategy if isinstance(strategy, str) else None
+
+
+def build_selector(strategy: str | None, args: Mapping[str, object], cache: "DualCache") -> Selector | None:
+    """``_build_strategy_selector`` without registering: ``None`` for simple-shuffle and every
+    strategy without a selector."""
+    routing_args: Final = dict(args)
+    match strategy:
+        case "least-busy":
+            return LeastBusyLoggingHandler(router_cache=cache)
+        case "usage-based-routing":
+            return LowestTPMLoggingHandler(router_cache=cache, routing_args=routing_args)
+        case "usage-based-routing-v2":
+            return LowestTPMLoggingHandler_v2(router_cache=cache, routing_args=routing_args)
+        case "latency-based-routing":
+            return LowestLatencyLoggingHandler(router_cache=cache, routing_args=routing_args)
+        case "cost-based-routing":
+            return LowestCostLoggingHandler(router_cache=cache, routing_args={})
+        case _:
+            return None
+
+
+def _callbacks(host: "PythonHost") -> list[object]:  # mutable-ok: the process list itself
+    """``litellm.callbacks``, which the host serves like its other callback lists."""
+    return host.callback_list("callbacks")  # pyright: ignore[reportArgumentType]  # not in the CallbackList literal
+
+
+class StrategySelectors:
+    """The selectors of one router: the default group's, each routing group's, and the lazily
+    built per-request override ones."""
+
+    def __init__(self, host: "PythonHost", cache: "DualCache") -> None:
+        self._host: Final = host
+        self._cache: Final = cache
+        self._strategy: str | None = None
+        self._default: Selector | None = None
+        self._groups: dict[str, Selector] = {}  # mutable-ok: replaced as routing groups change
+        self._group_strategies: frozenset[str] = frozenset()
+        self._overrides: dict[str, Selector | None] = {}  # mutable-ok: filled lazily per strategy
+        self._lock: Final = threading.Lock()
+
+    # -- building ------------------------------------------------------------------------------
+
+    def _register(self, selector: CustomLogger) -> None:
+        """``_register_router_selector``: least-busy also counts calls as they start."""
+        host: Final = self._host
+        if isinstance(selector, LeastBusyLoggingHandler):
+            if isinstance(host.callback_list("input_callback"), list):  # pyright: ignore[reportUnnecessaryIsInstance]  # a process list a user may have reassigned
+                host.callbacks.add_litellm_input_callback(selector)
+            else:
+                host.set_callback_list("input_callback", [selector])
+        if isinstance(_callbacks(host), list):  # pyright: ignore[reportUnnecessaryIsInstance]  # see above
+            host.callbacks.add_litellm_callback(selector)
+
+    def _unregister(self, selectors: Sequence[object]) -> None:
+        live: Final = [selector for selector in selectors if selector is not None]
+        for selector in live:
+            if isinstance(selector, BaseRoutingStrategy):
+                selector.retire()
+        if live:
+            self._host.callbacks.drop_callbacks(live)
+
+    def init_default(self, strategy: object, args: Mapping[str, object]) -> None:
+        """``routing_strategy_init``: drop the old default and override selectors, build the new one."""
+        self._unregister([self._default, *self._overrides.values()])
+        self._overrides = {}
+        self._strategy = normalize_strategy(strategy)
+        self._default = build_selector(self._strategy, args, self._cache)
+        if self._default is not None:
+            self._register(self._default)
+
+    def rebuild_default(self, args: Mapping[str, object]) -> None:
+        """``_apply_updated_routing_strategy_args``: relink the default selector to new args, keeping
+        the old one when they are invalid."""
+        from pydantic import ValidationError
+
+        from litellm._logging import verbose_router_logger
+
+        if self._default is None:
+            return
+        try:
+            rebuilt: Final = build_selector(self._strategy, args, self._cache)
+        except (TypeError, ValidationError):
+            verbose_router_logger.exception(
+                "Invalid routing_strategy_args %s for '%s'; keeping the previous ones", dict(args), self._strategy
+            )
+            return
+        self._unregister([self._default])
+        self._default = rebuilt
+        if rebuilt is not None:
+            self._register(rebuilt)
+
+    def init_groups(self, groups: Sequence[tuple[str, str, Mapping[str, object]]]) -> None:
+        """``_replace_routing_groups``: one selector per group, from (name, strategy, args)."""
+        self._unregister(list(self._groups.values()))
+        built: Final = {
+            name: selector
+            for name, strategy, args in groups
+            if (selector := build_selector(None if strategy == "priority" else strategy, args, self._cache)) is not None
+        }
+        for selector in built.values():
+            self._register(selector)
+        self._groups = built
+        self._group_strategies = frozenset(strategy for _, strategy, _ in groups)
+
+    def retire(self) -> None:
+        self._unregister([self._default, *self._groups.values(), *self._overrides.values()])
+        self.clear()
+
+    def clear(self) -> None:
+        """Forget every selector, after the caller released them from the host lists."""
+        self._default = None
+        self._groups = {}
+        self._overrides = {}
+
+    def callback_selectors(self) -> tuple[Selector, ...]:
+        """``_callback_selectors``: the default group's selector and each routing group's."""
+        return tuple(selector for selector in (self._default, *self._groups.values()) if selector is not None)
+
+    def override_selectors(self) -> tuple[Selector, ...]:
+        """The per-request override selectors built so far."""
+        return tuple(selector for selector in self._overrides.values() if selector is not None)
+
+    @property
+    def default(self) -> Selector | None:
+        return self._default
+
+    def attribute(self, name: str) -> Selector | None:
+        """A legacy ``*_logger`` attribute: the default selector when it is that strategy's."""
+        return self._default if SELECTOR_ATTRIBUTES.get(self._strategy or "") == name else None
+
+    def latency_args(self) -> Mapping[str, object] | None:
+        """The default latency selector's full routing args, as ``get_settings`` reports them."""
+        if not isinstance(self._default, LowestLatencyLoggingHandler):
+            return None
+        reported: Final[object] = self._default.routing_args.json()  # pyright: ignore[reportUnknownMemberType]  # RoutingArgs.json is untyped
+        return as_str_mapping(reported)
+
+    # -- picking -------------------------------------------------------------------------------
+
+    def selector_for(
+        self, strategy: str, scope: str, group: str | None, request: Mapping[str, object] | None
+    ) -> Selector | None:
+        """The selector a Rust ``select`` step names (``_get_routing_context``)."""
+        if scope == "group":
+            return self._groups.get(group or "")
+        if scope == "override":
+            return self._override(strategy, request)
+        return self._default if strategy == self._strategy else None
+
+    def _override(self, strategy: str, request: Mapping[str, object] | None) -> Selector | None:
+        """``_get_override_strategy_selector`` and ``_bind_override_selector_to_request``."""
+        if strategy == self._strategy:
+            return self._default
+        with self._lock:
+            if strategy not in self._overrides:
+                self._overrides[strategy] = build_selector(strategy, {}, self._cache)
+            selector: Final = self._overrides[strategy]
+        if selector is not None and request is not None and strategy not in self._registered_strategies():
+            from litellm.litellm_core_utils.litellm_logging import Logging
+
+            logging_obj: Final = request.get("litellm_logging_obj")
+            if isinstance(logging_obj, Logging):
+                logging_obj.add_dynamic_callback(selector)
+        return selector
+
+    def _registered_strategies(self) -> frozenset[str]:
+        return frozenset({*((self._strategy,) if self._strategy else ()), *self._group_strategies})
+
+    def is_override(self, strategy: str, selector: object) -> bool:
+        """Whether ``selector`` is a per-request override one, whose pre-call check the router runs
+        itself because it is not in the callback lists."""
+        return selector is not None and self._overrides.get(strategy) is selector
+
+
+async def pick_async(
+    strategy: str,
+    selector: Selector,
+    model: str,
+    deployments: list[dict[str, object]],  # mutable-ok: the selectors take a list of deployment dicts
+    messages: object,
+    input: object,
+    request_kwargs: dict[str, object] | None,  # mutable-ok: the latency selector reads it as given
+) -> object:
+    """``_select_deployment_async``."""
+    match selector:
+        case LeastBusyLoggingHandler() if strategy == "least-busy":
+            return await selector.async_get_available_deployments(model_group=model, healthy_deployments=deployments)  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # the selectors return untyped deployment dicts
+        case LowestTPMLoggingHandler() if strategy == "usage-based-routing":
+            return selector.get_available_deployments(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # the selectors return untyped deployment dicts
+                model_group=model,
+                healthy_deployments=deployments,
+                messages=messages,  # pyright: ignore[reportArgumentType]  # the request's messages, as given
+                input=input,  # pyright: ignore[reportArgumentType]  # same
+            )
+        case LowestTPMLoggingHandler_v2() | LowestCostLoggingHandler() if strategy in (
+            "usage-based-routing-v2",
+            "cost-based-routing",
+        ):
+            return await selector.async_get_available_deployments(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # the selectors return untyped deployment dicts
+                model_group=model,
+                healthy_deployments=deployments,
+                messages=messages,  # pyright: ignore[reportArgumentType]  # same
+                input=input,  # pyright: ignore[reportArgumentType]  # same
+            )
+        case LowestLatencyLoggingHandler() if strategy == "latency-based-routing":
+            return await selector.async_get_available_deployments(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # the selectors return untyped deployment dicts
+                model_group=model,
+                healthy_deployments=deployments,
+                messages=messages,  # pyright: ignore[reportArgumentType]  # same
+                input=input,  # pyright: ignore[reportArgumentType]  # same
+                request_kwargs=request_kwargs,
+            )
+        case _:
+            return None
+
+
+def pick_sync(
+    strategy: str,
+    selector: Selector,
+    model: str,
+    deployments: list[dict[str, object]],  # mutable-ok: see pick_async
+    messages: object,
+    input: object,
+    request_kwargs: dict[str, object] | None,  # mutable-ok: see pick_async
+) -> object:
+    """``_select_deployment_sync``: cost-based routing has no sync pick."""
+    match selector:
+        case LeastBusyLoggingHandler() if strategy == "least-busy":
+            return selector.get_available_deployments(model_group=model, healthy_deployments=deployments)  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # the selectors return untyped deployment dicts
+        case LowestTPMLoggingHandler() | LowestTPMLoggingHandler_v2() if strategy in (
+            "usage-based-routing",
+            "usage-based-routing-v2",
+        ):
+            return selector.get_available_deployments(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # the selectors return untyped deployment dicts
+                model_group=model,
+                healthy_deployments=deployments,
+                messages=messages,  # pyright: ignore[reportArgumentType]  # same
+                input=input,  # pyright: ignore[reportArgumentType]  # same
+            )
+        case LowestLatencyLoggingHandler() if strategy == "latency-based-routing":
+            return selector.get_available_deployments(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # the selectors return untyped deployment dicts
+                model_group=model,
+                healthy_deployments=deployments,
+                messages=messages,  # pyright: ignore[reportArgumentType]  # same
+                input=input,  # pyright: ignore[reportArgumentType]  # same
+                request_kwargs=request_kwargs,
+            )
+        case _:
+            return None
+
+
+async def routing_pre_call_checks(callbacks: Iterable[object], deployment: Mapping[str, object]) -> None:
+    """``async_routing_strategy_pre_call_checks``: every ``CustomLogger`` among the router's routing
+    callbacks (``_routing_callbacks``) checks the picked deployment before the call
+    (usage-based-routing-v2 counts its rpm here). A check's error fails the attempt; the core
+    records that failure like any other."""
+    for callback in tuple(callbacks):
+        if isinstance(callback, CustomLogger):
+            await callback.async_pre_call_check(dict(deployment), None)  # pyright: ignore[reportUnknownMemberType]  # CustomLogger takes an untyped deployment dict
+
+
+def routing_pre_call_checks_sync(callbacks: Iterable[object], deployment: Mapping[str, object]) -> None:
+    """``routing_strategy_pre_call_checks``: the sync check of each ``CustomLogger`` callback."""
+    for callback in tuple(callbacks):
+        if isinstance(callback, CustomLogger):
+            callback.pre_call_check(dict(deployment))  # pyright: ignore[reportUnknownMemberType]  # see above
+
+
+def picked_id(picked: object) -> str | None:
+    """The id of the deployment a selector returned."""
+    info: Final = as_str_mapping((as_str_mapping(picked) or {}).get("model_info")) or {}
+    found: Final = info.get("id")
+    return str(found) if found is not None else None
+
+
+__all__ = [
+    "SELECTOR_ATTRIBUTES",
+    "Selector",
+    "StrategySelectors",
+    "build_selector",
+    "group_documents",
+    "group_settings",
+    "member_map",
+    "normalize_strategy",
+    "parse_groups",
+    "pick_async",
+    "pick_sync",
+    "picked_id",
+    "routing_pre_call_checks",
+    "routing_pre_call_checks_sync",
+]
+
+
+def parse_groups(
+    groups_input: object, model_names: frozenset[str], aliases: frozenset[str]
+) -> tuple[RoutingGroup, ...]:
+    """``_init_routing_groups``' validation: ``parse_routing_groups``, then a priority group may not
+    share its name with a model group or alias (a plain group only warns: its name is shadowed)."""
+    from litellm._logging import verbose_router_logger
+    from litellm.litellm_core_utils.routing_groups import (
+        parse_routing_groups,  # pyright: ignore[reportUnknownVariableType]  # takes untyped dicts
+    )
+
+    if not groups_input or not isinstance(groups_input, list | tuple):
+        return ()
+    groups: Final = parse_routing_groups(  # pyright: ignore[reportUnknownVariableType]  # returns RoutingGroup models
+        list(groups_input),  # pyright: ignore[reportUnknownArgumentType]  # validated by parse_routing_groups
+        known_model_names=model_names,
+    )
+    for group in groups:
+        if group.group_name in model_names or group.group_name in aliases:
+            if group.routing_strategy == "priority":
+                raise ValueError("Priority routing group names must not shadow a model or alias")
+            verbose_router_logger.warning(
+                "routing_groups: group_name '%s' is shadowed by an existing model_name or model_group_alias; "
+                "the group's strategy still applies to its members, but the name is not callable until renamed.",
+                group.group_name,
+            )
+    return groups
+
+
+def group_documents(groups: Sequence[RoutingGroup]) -> list[Mapping[str, object]]:  # mutable-ok: a JSON list
+    """The routing groups as the Rust core reads them."""
+    return [
+        {
+            "group_name": group.group_name,
+            "models": list(group.models),
+            "routing_strategy": group.routing_strategy,
+            **({"model_priorities": dict(group.model_priorities)} if group.model_priorities is not None else {}),
+        }
+        for group in groups
+    ]
+
+
+def group_settings(groups: Sequence[RoutingGroup]) -> list[object]:  # mutable-ok: the settings snapshot's list
+    """``get_settings()['routing_groups']``."""
+    return [
+        group.model_dump(exclude={"model_priorities"}) if group.model_priorities is None else group.model_dump()
+        for group in groups
+    ]
+
+
+def member_map(groups: Sequence[RoutingGroup]) -> Mapping[str, str]:
+    """``_model_to_group``: each member model group of a non-priority routing group, to the group
+    (a model listed by two groups maps to the last)."""
+    kept: Final = [group for group in groups if group.routing_strategy != "priority"]
+    return dict(chain.from_iterable(dict.fromkeys(group.models, group.group_name).items() for group in kept))

litellm/router_rust_settings.py

@@ -0,0 +1,218 @@
+"""
+The Rust core's settings semantics on the host side: how ``ChangeSet.settings`` is read (the rules
+of ``update_settings``), the ``settings().as_dict()`` view (``get_settings``), the router attributes
+the Python core keeps as plain attributes, deleted settings, and the construction-time side effects
+of the logging arguments.
+
+The rules are the Python router core's, ported (never imported). Settings change only through a
+changeset; a direct attribute write is turned into one by ``RustCore.__setattr__``.
+"""
+
+import logging
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final
+
+from litellm._logging import verbose_router_logger
+from litellm.constants import RUNTIME_UPDATABLE_ROUTER_SETTINGS
+from litellm.types.router import RetryPolicy
+
+if TYPE_CHECKING:
+    from litellm.router_config import RouterConfig
+    from litellm.router_python_host import PythonHost
+
+MISSING: Final = object()
+
+# `update_settings` casts these with `int(value)`.
+_INT_SETTINGS: Final = frozenset({"timeout", "num_retries", "retry_after", "allowed_fails", "cooldown_time"})
+CONTINUATION_SETTING: Final = "enable_mid_stream_fallback_continuation"
+
+# Settings the Python core keeps as plain router attributes and `update_settings` sets as given,
+# with their constructor defaults. ``max_retries`` has none: it exists once a changeset sets it.
+PLAIN_DEFAULTS: Final[Mapping[str, object]] = MappingProxyType(
+    {
+        "retry_policy": None,
+        "model_group_retry_policy": {},
+        "enable_weighted_failover": False,
+        "fallback_on_unsupported_params": True,
+    }
+)
+# `get_settings`: the settings it lists, in its order (`routing_groups` comes last).
+_VIEW_ORDER: Final = (
+    "routing_strategy_args",
+    "routing_strategy",
+    "allowed_fails",
+    "cooldown_time",
+    "num_retries",
+    "timeout",
+    "max_retries",
+    "retry_after",
+    "fallbacks",
+    "context_window_fallbacks",
+    "model_group_retry_policy",
+    "retry_policy",
+    "model_group_alias",
+    "enable_weighted_failover",
+    "enable_mid_stream_fallback_continuation",
+    "enable_tag_filtering",
+    "tag_routing_prefix",
+    "fallback_on_unsupported_params",
+)
+
+# Router attributes that are their constructor argument as given, else this default. Arguments
+# the Rust core declines are only ever at their default here.
+_ARGUMENT_ATTRIBUTES: Final[Mapping[str, object]] = MappingProxyType(
+    {
+        "set_verbose": False,
+        "debug_level": "INFO",
+        "client_ttl": 3600,
+        "stream_timeout": None,
+        "assistants_config": None,
+        "ignore_invalid_deployments": False,
+        "alerting_config": None,
+        "auto_router_capability_limit": None,
+        "deployment_affinity_ttl_seconds": 3600,
+        "session_affinity_repin_on_failover": False,
+        "model_group_affinity_config": None,
+    }
+)
+# Class-level and constant attributes of the Python core.
+_CONSTANT_ATTRIBUTES: Final[Mapping[str, object]] = MappingProxyType(
+    {"default_cache_time_seconds": 1 * 60 * 60, "tenacity": None, "access_groups": None}
+)
+
+
+@dataclass(frozen=True)
+class SettingsUpdate:
+    """What a ``ChangeSet.settings`` mapping changes, read with ``update_settings``' rules."""
+
+    values: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({}))  # fields and plain settings
+    plain: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({}))  # plain attributes, as stored
+    continuation: object = MISSING  # the continuation flag, when given
+    routing_groups: object = MISSING  # the routing groups input, when given
+    optional_pre_call_checks: object = MISSING  # the optional pre-call checks, when given
+
+
+def read_settings_update(settings: Mapping[str, object]) -> SettingsUpdate:
+    """``_core_update_settings``' reading of each setting: names that are not runtime-updatable are
+    ignored, the integer settings are cast with ``int``, a retry policy dict becomes a
+    ``RetryPolicy`` (any other type but ``None`` is dropped)."""
+    values: Final[dict[str, object]] = {}  # mutable-ok: built here, then frozen into the update
+    plain: Final[dict[str, object]] = {}  # mutable-ok: same
+    continuation: object = MISSING  # rebind-ok: set when the flag is given
+    routing_groups: object = MISSING  # rebind-ok: set when routing groups are given
+    optional_checks: object = MISSING  # rebind-ok: set when the optional pre-call checks are given
+    for name, value in settings.items():
+        if name not in RUNTIME_UPDATABLE_ROUTER_SETTINGS:
+            verbose_router_logger.debug("Setting %s is not allowed", name)
+        elif name in _INT_SETTINGS:
+            values[name] = int(value)  # pyright: ignore[reportArgumentType]  # `int(kwargs[var])`, as given: a bad value raises int's own error
+        elif name == "routing_groups":
+            routing_groups = value
+        elif name == "optional_pre_call_checks":
+            optional_checks = value
+        elif name == CONTINUATION_SETTING:
+            continuation = value
+        elif name == "retry_policy":
+            policy = RetryPolicy.model_validate(value) if isinstance(value, dict) else value
+            if policy is None or isinstance(policy, RetryPolicy):
+                plain[name] = policy
+        elif name in PLAIN_DEFAULTS or name == "max_retries":
+            plain[name] = value
+        else:
+            values[name] = value
+    return SettingsUpdate(
+        values=MappingProxyType(values),
+        plain=MappingProxyType(plain),
+        continuation=continuation,
+        routing_groups=routing_groups,
+        optional_pre_call_checks=optional_checks,
+    )
+
+
+def initial_plain(config: "RouterConfig") -> dict[str, object]:  # mutable-ok: the core's own plain attributes
+    """The plain attributes as the constructor sets them: a ``retry_policy`` dict is read into a
+    ``RetryPolicy`` and any other type but a policy is no policy."""
+    from litellm.router_rust_retries import resolved_retry_policy
+
+    plain: Final = {name: config.get(name, default) for name, default in PLAIN_DEFAULTS.items()}
+    plain["retry_policy"] = resolved_retry_policy(plain["retry_policy"])
+    return plain
+
+
+def settings_view(
+    values: Mapping[str, object], plain: Mapping[str, object], deleted: frozenset[str]
+) -> dict[str, object]:  # mutable-ok: the settings view is built fresh per call, as `get_settings` builds it
+    """``get_settings``: the listed settings the router has, then the routing groups. ``values``
+    holds the settings fields and plain attributes the Rust core resolves."""
+    view: Final[dict[str, object]] = {}  # mutable-ok: see above
+    for name in _VIEW_ORDER:
+        if name in deleted:
+            continue
+        if name in plain:
+            view[name] = plain[name]
+        elif name in values:
+            view[name] = values[name]
+    view["routing_groups"] = values.get("routing_groups", [])
+    return view
+
+
+def argument_attribute(config: "RouterConfig", name: str) -> object:
+    """A router attribute that is its constructor argument (or a constant), else ``MISSING``."""
+    if name in _ARGUMENT_ATTRIBUTES:
+        return config.get(name, _ARGUMENT_ATTRIBUTES[name])
+    if name in _CONSTANT_ATTRIBUTES:
+        return _CONSTANT_ATTRIBUTES[name]
+    return MISSING
+
+
+def default_litellm_params(config: "RouterConfig") -> dict[str, object]:  # mutable-ok: the router's own dict
+    """``default_litellm_params`` as the constructor completes it: the router timeout, no client
+    retries and the caching groups."""
+    given: Final = _as_dict(config.get("default_litellm_params")) or {}
+    params: Final[dict[str, object]] = dict(given)  # mutable-ok: see above
+    params.setdefault("timeout", config.get("timeout"))
+    params.setdefault("max_retries", 0)
+    metadata: Final = _as_dict(params.setdefault("metadata", {}))
+    if metadata is not None:
+        params["metadata"] = {**metadata, "caching_groups": config.get("caching_groups")}
+    return params
+
+
+def _as_dict(value: object) -> Mapping[str, object] | None:
+    return value if isinstance(value, dict) else None  # pyright: ignore[reportUnknownVariableType]  # the argument dict, as passed
+
+
+def request_timeout(config: "RouterConfig") -> float | None:
+    """The per-attempt ``request_timeout``: only stored when a router timeout is also set."""
+    from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout
+
+    return get_configured_request_timeout() if config.get("timeout") is not None else None
+
+
+def apply_construction_effects(config: "RouterConfig", host: "PythonHost") -> None:
+    """What the constructor's logging arguments do: the host stops its debug info, and
+    ``set_verbose`` sets the router logger to ``debug_level``."""
+    host.suppress_debug_info()
+    if config.get("set_verbose", False) is True:
+        level: Final = config.get("debug_level", "INFO")
+        if level == "INFO":
+            verbose_router_logger.setLevel(logging.INFO)
+        elif level == "DEBUG":
+            verbose_router_logger.setLevel(logging.DEBUG)
+
+
+__all__ = [
+    "CONTINUATION_SETTING",
+    "MISSING",
+    "PLAIN_DEFAULTS",
+    "SettingsUpdate",
+    "apply_construction_effects",
+    "argument_attribute",
+    "default_litellm_params",
+    "initial_plain",
+    "read_settings_update",
+    "request_timeout",
+    "settings_view",
+]

litellm/router_rust_shaping.py

@@ -0,0 +1,634 @@
+"""
+The keyword arguments one router attempt sends to one deployment, as the Python router builds them.
+
+The Rust router core picks the deployment; the host shapes the request for it here
+(``_update_kwargs_with_deployment``, ``_update_kwargs_with_default_litellm_params``,
+``_merge_tools_from_deployment``, the timeout resolvers and the reasoning-effort override), with
+the same precedence and the same oddities. Nothing here reads a Python router.
+"""
+
+import dataclasses
+from collections.abc import Mapping, MutableMapping, Sequence
+from functools import partial
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final
+
+from pydantic import TypeAdapter, ValidationError
+
+from litellm import exceptions as litellm_exceptions
+from litellm.constants import (
+    BAGGAGE_MAPPED_TAGS_METADATA_KEY,
+    INTERNAL_CALL_ORIGIN_METADATA_KEY,
+    KUBERNETES_POD_ROUTING_KEY,
+    ROUTING_REQUEST_TAGS_METADATA_KEY,
+)
+from litellm.litellm_core_utils.batch_utils import get_router_metadata_variable_name
+from litellm.litellm_core_utils.clientside_credential_handler import (
+    get_dynamic_litellm_params,  # pyright: ignore[reportUnknownVariableType]  # litellm's untyped helpers and request values
+    is_clientside_credential,  # pyright: ignore[reportUnknownVariableType]  # litellm's untyped helpers and request values
+)
+from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
+from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS
+from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, inferred_provider
+from litellm.litellm_core_utils.request_timeout_resolver import get_service_tier_timeout
+from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin
+from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, drop_policy
+from litellm.responses.retrieve_stream_query import requests_streamed_retrieval, without_retrieve_stream_defaults
+from litellm.router_api import deployment_id as generate_deployment_id
+from litellm.router_strategy.tag_based_routing import (
+    baggage_stamp_for_serving_deployment,
+    get_tags_from_request_kwargs,
+    tags_with_serving_deployment_tags,
+)
+from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
+    deployment_has_io_token_limits,  # pyright: ignore[reportUnknownVariableType]  # litellm's untyped helpers and request values
+    refund_stale_reservation_before_retry,
+    set_io_token_rate_limit_request_kwargs,
+)
+from litellm.router_utils.stream_usage_injection import router_config_requests_stream_usage
+from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
+from litellm.types.router import Deployment, GenericLiteLLMParams, LiteLLM_Params, validate_litellm_params_strictly
+from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN
+
+if TYPE_CHECKING:
+    from litellm.caching.dual_cache import DualCache
+    from litellm.router_python_host import PythonHost
+
+# The SDK-client and max_parallel_requests reads of the routing cache: process-local, so the family
+# only names a Redis span when a read misses through to Redis.
+ROUTER_CLIENTS_TARGET: Final = "router_clients"
+_MAPPING: Final = TypeAdapter(Mapping[str, object])
+_FORWARDED_CREDENTIAL_ISOLATION_PROVIDERS: Final = frozenset({"chatgpt", "anthropic"})
+_ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params"
+# Generic handlers whose stream timeout is split from the request timeout.
+STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"})
+
+
+@dataclasses.dataclass(frozen=True, slots=True)
+class Shaping:
+    """What the request shaping reads from the router, resolved once when the core is built.
+
+    ``timeout`` is the router's ``timeout`` (its own setting, else the process ``request_timeout``),
+    ``request_timeout`` the explicitly configured process ``request_timeout``, kept only when the
+    router has a timeout of its own, as the Python router keeps them."""
+
+    default_litellm_params: Mapping[str, object]
+    timeout: object
+    explicit_timeout: object
+    request_timeout: float | None
+    stream_timeout: object
+    cache_responses: bool
+
+
+def as_dict(value: object) -> dict[str, object]:  # mutable-ok: a fresh dict of a request mapping
+    try:
+        return dict(_MAPPING.validate_python(value))
+    except ValidationError:
+        return {}
+
+
+def _mapping(value: object) -> Mapping[str, object] | None:
+    try:
+        return _MAPPING.validate_python(value)
+    except ValidationError:
+        return None
+
+
+def metadata_bucket(
+    kwargs: MutableMapping[str, object],  # mutable-ok: the request kwargs the Python router mutates in place
+    name: str,
+) -> dict[str, object]:  # mutable-ok: the request's own bucket
+    """``kwargs.setdefault(name, {})``: the request's own metadata dict, created when absent."""
+    bucket: Final = kwargs.get(name)
+    if isinstance(bucket, dict):
+        return bucket  # pyright: ignore[reportUnknownVariableType]  # the request's untyped metadata dict
+    if name not in kwargs:
+        fresh: Final[dict[str, object]] = {}  # mutable-ok: the request kwargs the Python router mutates in place
+        kwargs[name] = fresh  # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
+        return fresh
+    # A non-dict value under the key: `setdefault` returns it as is, and `.update` would raise.
+    raise AttributeError(f"'{type(bucket).__name__}' object has no attribute 'update'")
+
+
+# --- request params the deployment sets --------------------------------------------------------
+
+
+def _drops_malformed_request_tools(
+    deployment_params: Mapping[str, object], defaults: Mapping[str, object], request: Mapping[str, object]
+) -> bool:
+    if isinstance(request.get("tools") or [], (list, tuple)) or not deployment_params.get("tools"):
+        return False
+    set_defaults: Final = {name: value for name, value in defaults.items() if value is not None}
+    return drop_policy({**deployment_params, **set_defaults, **request}).drops("tools")
+
+
+def _malformed_request_params_the_deployment_sets(
+    deployment_params: Mapping[str, object], defaults: Mapping[str, object], request: Mapping[str, object]
+) -> frozenset[str]:
+    overrides: Final = {
+        key: value
+        for key, value in request.items()
+        if key in SUPPLEMENTAL_KWARGS_KEYS and deployment_params.get(key) is not None
+    }
+    if not overrides:
+        return frozenset()
+    try:
+        validate_litellm_params_strictly(GenericLiteLLMParams, overrides)
+    except ValidationError as error:
+        set_defaults: Final = {name: value for name, value in defaults.items() if value is not None}
+        policy: Final = drop_policy({**deployment_params, **set_defaults, **request})
+        malformed: Final = (str(detail["loc"][0]) for detail in error.errors() if detail["loc"])
+        return frozenset(key for key in malformed if policy.drops(key))
+    return frozenset()
+
+
+def _merged_tools(
+    deployment_tools: Sequence[object], request_tools: object
+) -> list[object] | None:  # mutable-ok: sent as the request's tools
+    if not isinstance(request_tools, (list, tuple)):
+        return None
+    items: Final[Sequence[object]] = request_tools  # pyright: ignore[reportUnknownVariableType]  # a request's untyped tools
+    return [*deployment_tools, *items] if deployment_tools or items else None
+
+
+def merge_tools_from_deployment(
+    deployment_params: Mapping[str, object],
+    kwargs: MutableMapping[str, object],  # mutable-ok: the request kwargs the Python router mutates in place
+    defaults: Mapping[str, object],
+) -> None:
+    """``_merge_tools_from_deployment``: deployment tools first, then the request's; the request's
+    ``tool_choice`` else the deployment's."""
+    if _drops_malformed_request_tools(deployment_params, defaults, kwargs):
+        for name in TOOL_FIELDS:
+            kwargs.pop(name, None)
+    deployment_tools: Final = deployment_params.get("tools") or []  # pyright: ignore[reportUnknownVariableType]  # litellm's untyped helpers and request values
+    merged: Final = _merged_tools(
+        deployment_tools if isinstance(deployment_tools, list) else [],  # pyright: ignore[reportUnknownArgumentType]  # the deployment's untyped tools
+        kwargs.get("tools") or [],
+    )
+    if merged is not None:
+        kwargs["tools"] = merged  # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
+    if "tool_choice" not in kwargs and deployment_params.get("tool_choice") is not None:
+        kwargs["tool_choice"] = (  # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
+            deployment_params["tool_choice"]
+        )
+
+
+def _deployment_sets_litellm_param(deployment_params: Mapping[str, object], key: str) -> bool:
+    value: Final = deployment_params.get(key)
+    if value is None:
+        return False
+    field: Final = LiteLLM_Params.model_fields.get(key)
+    return field is None or value != field.default  # pyright: ignore[reportAny]  # litellm's untyped helpers and request values
+
+
+def _forwarded_alias_marker_keys_the_deployment_sets(
+    deployment_params: Mapping[str, object], forwarded_keys: object
+) -> tuple[str, ...]:
+    if not isinstance(forwarded_keys, tuple):
+        return ()
+    keys: Final[tuple[object, ...]] = forwarded_keys  # pyright: ignore[reportUnknownVariableType]  # the router's own marker
+    return tuple(key for key in keys if isinstance(key, str) and _deployment_sets_litellm_param(deployment_params, key))
+
+
+# --- reasoning effort -------------------------------------------------------------------------
+
+
+def _pop_effort_from_nested_carrier(params: dict[str, object], carrier: str) -> None:  # mutable-ok: copy-on-write
+    nested: Final = _mapping(params.get(carrier)) if isinstance(params.get(carrier), Mapping) else None
+    if nested is None:
+        return
+    sanitized: Final = {key: value for key, value in nested.items() if key != "effort"}
+    if sanitized:
+        params[carrier] = (  # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
+            sanitized
+        )
+    else:
+        params.pop(carrier, None)
+
+
+def deployment_params_with_request_reasoning_override(
+    deployment_params: Mapping[str, object], request_kwargs: Mapping[str, object]
+) -> dict[str, object]:  # mutable-ok: the call's own copy of the deployment params
+    """``_deployment_params_with_request_reasoning_override``: a request ``reasoning_effort`` drops
+    the deployment's ``thinking`` and nested ``effort`` carriers."""
+    sanitized: Final = dict(deployment_params)
+    if request_kwargs.get("reasoning_effort") is None:
+        return sanitized
+    sanitized.pop("thinking", None)
+    _pop_effort_from_nested_carrier(sanitized, "output_config")
+    _pop_effort_from_nested_carrier(sanitized, "reasoning")
+    extra_body: Final = sanitized.get("extra_body")
+    if isinstance(extra_body, Mapping):
+        sanitized_extra_body: Final = as_dict(extra_body)  # pyright: ignore[reportUnknownArgumentType]  # litellm's untyped helpers and request values
+        sanitized_extra_body.pop("reasoning_effort", None)
+        sanitized_extra_body.pop("thinking", None)
+        _pop_effort_from_nested_carrier(sanitized_extra_body, "output_config")
+        _pop_effort_from_nested_carrier(sanitized_extra_body, "reasoning")
+        if sanitized_extra_body:
+            sanitized["extra_body"] = sanitized_extra_body
+        else:
+            sanitized.pop("extra_body", None)
+    return sanitized
+
+
+def _is_classifier_internal_call(kwargs: Mapping[str, object]) -> bool:
+    return any(
+        isinstance(candidate, Mapping)
+        and (_mapping(candidate) or {}).get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == AUTOROUTER_CLASSIFIER_CALL_ORIGIN  # pyright: ignore[reportUnknownArgumentType]  # litellm's untyped helpers and request values
+        for candidate in (kwargs.get("metadata"), kwargs.get("litellm_metadata"))
+    )
+
+
+def _declared_param_allowlist(params: Mapping[str, object]) -> frozenset[str]:
+    declared: Final = params.get("allowed_openai_params")
+    if not isinstance(declared, (list, tuple, set, frozenset)):
+        return frozenset()
+    entries: Final[Sequence[object]] = list(declared)  # pyright: ignore[reportUnknownArgumentType]  # the deployment's untyped allowlist
+    return frozenset(entry for entry in entries if isinstance(entry, str))
+
+
+def _deployment_supports_param(deployment: Mapping[str, object], group: str, param: str, host: "PythonHost") -> bool:
+    params: Final = as_dict(deployment.get("litellm_params"))
+    if not params:
+        return True
+    model_info: Final = as_dict(deployment.get("model_info"))
+    base_model: Final = model_info.get("base_model") or params.get("base_model")
+    try:
+        model_label: Final = str(params.get("model") or group)
+        provider_param: Final = params.get("custom_llm_provider")
+        authenticating: Final = declared_authenticating_provider(
+            model_label, provider_param if isinstance(provider_param, str) else None
+        )
+        if authenticating is not None:
+            model, custom_llm_provider = model_label.removeprefix(f"{authenticating}/"), authenticating
+        else:
+            model, custom_llm_provider, _, _ = host.invoke(  # rebind-ok: rebound as the Python router rebinds it
+                "get_llm_provider"
+            )(
+                model=model_label,
+                custom_llm_provider=provider_param,
+                litellm_params=LiteLLM_Params.model_validate(params),
+            )
+        supported: Final = host.invoke("get_supported_openai_params")(
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            base_model=base_model if isinstance(base_model, str) else None,
+        )
+    except Exception:  # noqa: BLE001  # best-effort filter: an unresolvable provider must not narrow the request
+        return True
+    return supported is None or param in supported
+
+
+def deployment_accepts_param(deployment: Mapping[str, object], group: str, param: str, host: "PythonHost") -> bool:
+    params: Final = as_dict(deployment.get("litellm_params"))
+    if not params:
+        return True
+    if param in _declared_param_allowlist(params):
+        return True
+    provider_param: Final = params.get("custom_llm_provider")
+    if declared_authenticating_provider(
+        str(params.get("model") or ""), provider_param if isinstance(provider_param, str) else None
+    ):
+        return True
+    return _deployment_supports_param(deployment, group, param, host)
+
+
+def drop_unsupported_classifier_reasoning_effort(
+    deployment: Mapping[str, object],
+    model: str,
+    kwargs: MutableMapping[str, object],  # mutable-ok: the request kwargs the Python router mutates in place
+    host: "PythonHost",
+) -> None:
+    """``_drop_unsupported_classifier_reasoning_effort``."""
+    if kwargs.get("reasoning_effort") is None or not _is_classifier_internal_call(kwargs):
+        return
+    if deployment_accepts_param(deployment, model, "reasoning_effort", host):
+        return
+    kwargs.pop("reasoning_effort", None)
+    proxy_server_request: Final = kwargs.get("proxy_server_request")
+    if not isinstance(proxy_server_request, dict):
+        return
+    body: Final[object] = proxy_server_request.get("body")  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # the proxy's untyped request
+    if isinstance(body, dict):
+        body.pop("reasoning_effort", None)  # pyright: ignore[reportUnknownMemberType]  # same
+
+
+# --- timeouts ---------------------------------------------------------------------------------
+
+
+def stream_timeout(shaping: Shaping, kwargs: Mapping[str, object], data: Mapping[str, object]) -> object:
+    """``_get_stream_timeout``."""
+    return (
+        kwargs.get("stream_timeout", None)
+        or get_service_tier_timeout(kwargs.get("service_tier"), data, "stream_timeout_by_service_tier")
+        or data.get("stream_timeout", None)
+        or shaping.stream_timeout
+        or shaping.request_timeout
+        or shaping.default_litellm_params.get("stream_timeout", None)
+    )
+
+
+def non_stream_timeout(shaping: Shaping, kwargs: Mapping[str, object], data: Mapping[str, object]) -> object:
+    """``_get_non_stream_timeout``."""
+    return (
+        kwargs.get("timeout", None)
+        or kwargs.get("request_timeout", None)
+        or get_service_tier_timeout(kwargs.get("service_tier"), data, "timeout_by_service_tier")
+        or data.get("timeout", None)
+        or data.get("request_timeout", None)
+        or shaping.request_timeout
+        or shaping.timeout
+        or shaping.default_litellm_params.get("timeout", None)
+    )
+
+
+def call_timeout(shaping: Shaping, kwargs: Mapping[str, object], data: Mapping[str, object]) -> object:
+    """``_get_timeout``."""
+    stream: Final = stream_timeout(shaping, kwargs, data) if kwargs.get("stream") else None
+    return stream if stream is not None else non_stream_timeout(shaping, kwargs, data)
+
+
+# --- default litellm params -------------------------------------------------------------------
+
+
+def update_kwargs_with_default_litellm_params(
+    shaping: Shaping,
+    kwargs: MutableMapping[str, object],  # mutable-ok: the request kwargs the Python router mutates in place
+    metadata_variable_name: str,
+) -> None:
+    """``_update_kwargs_with_default_litellm_params``: unset non-None defaults fill the request;
+    the default metadata is merged over the request's."""
+    defaults: Final = dict(without_retrieve_stream_defaults(shaping.default_litellm_params, kwargs))
+    metadata_defaults: Final = defaults.pop("metadata", {}) or {}  # pyright: ignore[reportUnknownVariableType]  # litellm's untyped helpers and request values
+    for key, value in defaults.items():
+        if value is None:
+            continue
+        kwargs.setdefault(key, value)
+    metadata_bucket(kwargs, metadata_variable_name).update(as_dict(metadata_defaults))  # pyright: ignore[reportUnknownArgumentType]  # litellm's untyped helpers and request values
+
+
+# --- clientside credentials -------------------------------------------------------------------
+
+
+def _clientside_deployment(
+    deployment: Mapping[str, object], kwargs: Mapping[str, object], function_name: str | None
+) -> tuple[Mapping[str, object], str, object]:
+    """``_handle_clientside_credential``: the request-scoped model_info, model and api_base."""
+    model_info: Final = as_dict(deployment.get("model_info"))
+    litellm_params: Final = as_dict(deployment.get("litellm_params"))
+    dynamic_litellm_params: Final = get_dynamic_litellm_params(  # pyright: ignore[reportUnknownVariableType]  # litellm's untyped helpers and request values
+        litellm_params=litellm_params, request_kwargs=dict(kwargs)
+    )
+    metadata_variable_name: Final = get_router_metadata_variable_name(function_name=function_name)
+    model_group: Final = as_dict(kwargs.get(metadata_variable_name)).get("model_group")
+    model_id: Final = generate_deployment_id(model_group, dynamic_litellm_params)  # pyright: ignore[reportArgumentType, reportUnknownArgumentType]  # the group as the request carries it
+    original_model_id: Final = model_info.get("id")
+    model_info["id"] = model_id
+    model_info["original_model_id"] = original_model_id
+    built: Final = Deployment(
+        model_name=model_group,  # pyright: ignore[reportArgumentType]  # the group as the request carries it
+        litellm_params=LiteLLM_Params.model_validate(dynamic_litellm_params),
+        model_info=model_info,  # pyright: ignore[reportArgumentType]  # model_info as configured
+    )
+    return built.model_info.model_dump(), built.litellm_params.model, built.litellm_params.api_base
+
+
+# --- the deployment --------------------------------------------------------------------------
+
+
+def update_kwargs_with_deployment(
+    shaping: Shaping,
+    deployment: Mapping[str, object],
+    kwargs: MutableMapping[str, object],  # mutable-ok: the request kwargs the Python router mutates in place
+    *,
+    cache: "DualCache | None",
+    function_name: str | None = None,
+    splits_stream_timeout: bool = False,
+    opens_responses_socket: bool = False,
+) -> None:
+    """``_update_kwargs_with_deployment``, in place on the attempt's kwargs."""
+    deployment_params: Final = as_dict(deployment.get("litellm_params"))
+    for key in _forwarded_alias_marker_keys_the_deployment_sets(
+        deployment_params, kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ())
+    ):
+        kwargs.pop(key, None)
+    kwargs.pop("daily_reset_time", None)
+    defaults: Final = shaping.default_litellm_params
+    merge_tools_from_deployment(deployment_params, kwargs, defaults)
+
+    model_info: Mapping[str, object] = as_dict(  # rebind-ok: rebound as the Python router rebinds it
+        deployment.get("model_info")
+    )
+    deployment_model: object = deployment_params.get("model")  # rebind-ok: same
+    deployment_api_base: object = deployment_params.get("api_base")  # rebind-ok: same
+    deployment_model_name: Final = deployment.get("model_name")
+    from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
+
+    for key in _malformed_request_params_the_deployment_sets(deployment_params, defaults, kwargs):
+        kwargs.pop(key)
+    declared_provider: Final = deployment_params.get("custom_llm_provider")
+    declared_model: Final = deployment_params.get("model")
+    deployment_provider: Final = (
+        declared_provider
+        if isinstance(declared_provider, str)
+        else inferred_provider(declared_model if isinstance(declared_model, str) else None)
+    )
+    forwarded_authorization: Final = (
+        ProviderSpecificHeaderUtils.forwarded_authorization(kwargs.get("provider_specific_header"), deployment_provider)
+        if deployment_provider in _FORWARDED_CREDENTIAL_ISOLATION_PROVIDERS
+        else None
+    )
+    if is_clientside_credential(request_kwargs=dict(kwargs)) or forwarded_authorization is not None:
+        model_info, deployment_model, deployment_api_base = (  # rebind-ok: rebound as the Python router rebinds it
+            _clientside_deployment(
+                deployment,
+                {**kwargs, **({"api_key": forwarded_authorization} if forwarded_authorization is not None else {})},
+                function_name,
+            )
+        )
+
+    metadata_variable_name: Final = get_router_metadata_variable_name(function_name=function_name)
+    routing: Final = deployment.get(KUBERNETES_POD_ROUTING_KEY)
+    metadata: Final = metadata_bucket(kwargs, metadata_variable_name)
+    metadata.update(
+        {
+            "deployment": deployment_model,
+            "model_info": model_info,
+            "api_base": deployment_api_base,
+            "deployment_model_name": deployment_model_name,
+            KUBERNETES_POD_ROUTING_KEY: routing,
+        }
+    )
+    other_bucket: Final = "metadata" if metadata_variable_name == "litellm_metadata" else "litellm_metadata"
+    other_bucket_metadata: Final = kwargs.get(other_bucket)
+    if isinstance(other_bucket_metadata, Mapping) and KUBERNETES_POD_ROUTING_KEY in other_bucket_metadata:
+        kwargs[other_bucket] = {**as_dict(other_bucket_metadata), KUBERNETES_POD_ROUTING_KEY: routing}  # pyright: ignore[reportUnknownArgumentType]  # litellm's untyped helpers and request values  # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
+
+    if cache is not None:
+        refund_stale_reservation_before_retry(cache, kwargs)  # pyright: ignore[reportArgumentType]  # the request's kwargs dict
+    set_io_token_rate_limit_request_kwargs(
+        kwargs,  # pyright: ignore[reportArgumentType]  # the request's kwargs dict
+        store_in_context=deployment_has_io_token_limits(deployment),  # pyright: ignore[reportArgumentType]  # a deployment record
+        opens_responses_socket=opens_responses_socket,
+        metadata_channel=metadata_variable_name,
+    )
+    metadata.setdefault(
+        ROUTING_REQUEST_TAGS_METADATA_KEY,
+        tuple(get_tags_from_request_kwargs(kwargs, metadata_variable_name=metadata_variable_name)),  # pyright: ignore[reportArgumentType]  # the request's kwargs dict
+    )
+    deployment_tags: Final = deployment_params.get("tags")
+    merged_tags: Final = tags_with_serving_deployment_tags(metadata, deployment_tags)
+    if merged_tags is not None:
+        metadata["tags"] = list(merged_tags)
+    baggage_stamp: Final = baggage_stamp_for_serving_deployment(metadata, deployment_tags)
+    if baggage_stamp is not None:
+        metadata[BAGGAGE_MAPPED_TAGS_METADATA_KEY] = baggage_stamp
+
+    credential_name: Final = deployment_params.get("litellm_credential_name")
+    if credential_name:
+        credential_tag: Final = f"Credential: {credential_name}"
+        existing_tags: Final = metadata.get("tags") or []  # pyright: ignore[reportUnknownVariableType]  # litellm's untyped helpers and request values
+        if isinstance(existing_tags, list) and credential_tag not in existing_tags:
+            existing_tags.append(credential_tag)  # pyright: ignore[reportUnknownMemberType]  # the request's own tags list
+        metadata["tags"] = existing_tags
+
+    from litellm.llms.github_copilot.per_user_auth import github_copilot_per_user_credential_name
+
+    configured_per_user_name: Final = github_copilot_per_user_credential_name(deployment_params)
+    caller_credential_name: Final = kwargs.get("litellm_credential_name")
+    if (
+        configured_per_user_name is not None
+        and isinstance(caller_credential_name, str)
+        and caller_credential_name != configured_per_user_name
+    ):
+        raise litellm_exceptions.BadRequestError(
+            message="litellm_credential_name cannot be overridden on a deployment that uses per-user GitHub OAuth",
+            model=str(deployment_model_name),
+            llm_provider="",
+        )
+
+    kwargs["model_info"] = (  # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
+        model_info
+    )
+    _set_timeouts(shaping, deployment_params, kwargs, function_name, splits_stream_timeout)
+
+    if router_config_requests_stream_usage(kwargs, (deployment_params, defaults)):
+        kwargs[  # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
+            STREAM_USAGE_INJECTED_FIELD
+        ] = True
+    update_kwargs_with_default_litellm_params(shaping, kwargs, metadata_variable_name)
+    pinned_model_id: Final = deployment_model_id_pin(deployment_params)
+    if pinned_model_id is not None:
+        kwargs["model_id"] = (  # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
+            pinned_model_id
+        )
+
+
+def _set_timeouts(
+    shaping: Shaping,
+    deployment_params: Mapping[str, object],
+    kwargs: MutableMapping[str, object],  # mutable-ok: the request kwargs the Python router mutates in place
+    function_name: str | None,
+    splits_stream_timeout: bool,
+) -> None:
+    if function_name == "_ageneric_api_call_with_fallbacks":
+        from litellm.passthrough.timeout_utils import resolve_llm_passthrough_timeout
+
+        router_timeout: Final = (
+            shaping.request_timeout
+            if shaping.request_timeout is not None
+            else float(shaping.explicit_timeout)
+            if isinstance(shaping.explicit_timeout, (int, float))
+            else None
+        )
+        router_stream_timeout: Final = (
+            shaping.stream_timeout
+            if shaping.stream_timeout is not None
+            else shaping.request_timeout
+            if shaping.request_timeout is not None
+            else shaping.default_litellm_params.get("stream_timeout")
+        )
+        passthrough_timeout: Final = partial(
+            resolve_llm_passthrough_timeout,
+            litellm_params=deployment_params,
+            router_timeout=router_timeout,
+            router_stream_timeout=router_stream_timeout,  # pyright: ignore[reportArgumentType]  # a configured default, passed as the Python router passes it
+        )
+        timeout_request: Final = {**kwargs, "stream": True} if requests_streamed_retrieval(kwargs) else dict(kwargs)
+        kwargs.update(
+            CompletionTimeout.stream_request_kwargs(
+                passthrough_timeout(kwargs={**kwargs, "stream": False}), passthrough_timeout(kwargs=timeout_request)
+            )
+            if splits_stream_timeout and timeout_request.get("stream")
+            else {"timeout": passthrough_timeout(kwargs=timeout_request)}
+        )
+    elif splits_stream_timeout:
+        kwargs.update(
+            CompletionTimeout.stream_request_kwargs(
+                non_stream_timeout(shaping, kwargs, deployment_params),
+                stream_timeout(shaping, kwargs, deployment_params) if kwargs.get("stream") else None,
+            )
+        )
+    else:
+        kwargs["timeout"] = (  # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
+            call_timeout(shaping, kwargs, deployment_params)
+        )
+
+
+def stamp_deployment_num_retries(error: BaseException, deployment: Mapping[str, object]) -> None:
+    """``_set_deployment_num_retries_on_exception``."""
+    if getattr(error, "num_retries", None) is not None:
+        return
+    value: Final = as_dict(deployment.get("litellm_params")).get("num_retries")
+    if value is None:
+        return
+    try:
+        error.num_retries = int(str(value)) if not isinstance(value, int) else value  # pyright: ignore[reportAttributeAccessIssue]  # the router stamps the same attribute  # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
+    except (ValueError, TypeError):
+        pass
+
+
+def stamp_failed_deployment_id(error: BaseException, model_info: object) -> None:
+    """``_set_failed_deployment_id_on_exception`` with the effective model_info."""
+    if getattr(error, "failed_deployment_id", None):
+        return
+    deployment_id: Final = as_dict(model_info).get("id")
+    if deployment_id:
+        try:
+            error.failed_deployment_id = deployment_id  # pyright: ignore[reportAttributeAccessIssue]  # the router stamps the same attribute  # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
+        except Exception:  # noqa: BLE001, S110  # as the Python router: a read-only error keeps its attributes
+            pass
+
+
+def stamp_retry_skip_deployment_id(error: BaseException, kwargs: Mapping[str, object]) -> None:
+    """``_stamp_retry_skip_deployment_id``."""
+    model_info: Final = kwargs.get("model_info")
+    if not isinstance(model_info, Mapping):
+        return
+    info: Final = as_dict(model_info)  # pyright: ignore[reportUnknownArgumentType]  # litellm's untyped helpers and request values
+    configured: Final = (
+        info.get("original_model_id") if isinstance(error, litellm_exceptions.UnsupportedEndpointError) else None
+    )
+    deployment_id: Final = configured or info.get("id")
+    if isinstance(deployment_id, str) and deployment_id:
+        error.retry_skip_deployment_id = deployment_id  # pyright: ignore[reportAttributeAccessIssue]  # dynamic stamp, read on retry  # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
+
+
+NO_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
+
+__all__ = [
+    "STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS",
+    "Shaping",
+    "as_dict",
+    "call_timeout",
+    "deployment_params_with_request_reasoning_override",
+    "drop_unsupported_classifier_reasoning_effort",
+    "merge_tools_from_deployment",
+    "metadata_bucket",
+    "stamp_deployment_num_retries",
+    "stamp_failed_deployment_id",
+    "stamp_retry_skip_deployment_id",
+    "update_kwargs_with_default_litellm_params",
+    "update_kwargs_with_deployment",
+]

litellm/router_rust_slots.py

@@ -0,0 +1,108 @@
+"""
+A deployment's ``max_parallel_requests`` slot on the Rust router core (``_deployment_slot`` and
+``_get_client(client_type="max_parallel_requests")``): the attempt holds one of the deployment's
+slots around its provider call, and an open chat stream keeps it until the stream is exhausted,
+closed, or falls back. A caller arriving while every slot is taken gets a 429, which the core
+records as that attempt's failure.
+
+The limit object is the one the Python router caches (``MaxParallelRequestsLimit``), kept in the
+core's routing cache under the same key, so its in-flight count is shared by every attempt.
+"""
+
+import contextlib
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Final
+
+from litellm._internal_context import with_service_target
+from litellm.router_rust_shaping import ROUTER_CLIENTS_TARGET, as_dict
+
+if TYPE_CHECKING:
+    from litellm.router_rust_core import RustCore
+    from litellm.router_rust_endpoints import Plan
+
+# The attempt helpers that run inside `_deployment_slot` (`_acompletion` holds the same slot
+# itself). Sync helpers, rerank, moderation and the search helper hold none.
+_SLOTTED_KINDS: Final = frozenset(
+    {
+        "chat",
+        "text",
+        "embedding",
+        "image",
+        "transcription",
+        "speech",
+        "adapter",
+        "generic",
+        "batch",
+        "cancel_batch",
+        "vector_store_create",
+    }
+)
+
+
+@with_service_target(ROUTER_CLIENTS_TARGET)
+def max_parallel_requests_limit(
+    core: "RustCore", deployment: Mapping[str, object], kwargs: Mapping[str, object]
+) -> object:
+    """`_get_client(client_type="max_parallel_requests")`: the cached limit, built on first use
+    (`InitalizeCachedClient.set_max_parallel_requests_client`)."""
+    from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs
+    from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit
+    from litellm.utils import calculate_max_parallel_requests
+
+    model_id: Final = as_dict(deployment.get("model_info")).get("id")
+    key: Final = f"{model_id}_max_parallel_requests_client"
+    span: Final = get_parent_otel_span_from_kwargs(dict(kwargs))
+    cache: Final = core._cache  # pyright: ignore[reportPrivateUsage]  # the core's routing cache
+    cached: Final[object] = cache.get_cache(key=key, local_only=True, parent_otel_span=span)  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportArgumentType]  # the cache holds untyped values
+    if cached is not None:
+        return cached  # pyright: ignore[reportUnknownVariableType]  # the cache holds untyped values
+    params: Final = as_dict(deployment.get("litellm_params"))
+    calculated: Final = calculate_max_parallel_requests(
+        rpm=params.get("rpm"),  # pyright: ignore[reportArgumentType]  # the configured value, as the Python router reads it
+        max_parallel_requests=params.get("max_parallel_requests"),  # pyright: ignore[reportArgumentType]  # same
+        tpm=params.get("tpm"),  # pyright: ignore[reportArgumentType]  # same
+        default_max_parallel_requests=core._endpoints.default_max_parallel_requests,  # pyright: ignore[reportPrivateUsage, reportArgumentType]  # the setting as given, as the Python router passes it
+    )
+    if calculated:
+        cache.set_cache(  # pyright: ignore[reportUnknownMemberType]  # the routing cache
+            key=key,
+            value=MaxParallelRequestsLimit(
+                max_parallel_requests=calculated,
+                model_id=str(model_id),
+                model_group=str(deployment.get("model_name", "")),
+            ),
+            local_only=True,
+        )
+    return cache.get_cache(key=key, local_only=True, parent_otel_span=span)  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportArgumentType]  # see above
+
+
+def enter_deployment_slot(
+    core: "RustCore", plan: "Plan", deployment: Mapping[str, object], kwargs: Mapping[str, object]
+) -> contextlib.ExitStack:
+    """The attempt's slot, entered (a full deployment raises its 429 here); an empty stack when
+    the plan's helper holds none or the deployment has no limit."""
+    from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit
+
+    slot: Final = contextlib.ExitStack()
+    if plan.sync or plan.kind not in _SLOTTED_KINDS:
+        return slot
+    limit: Final = max_parallel_requests_limit(core, deployment, kwargs)
+    if isinstance(limit, MaxParallelRequestsLimit):
+        slot.enter_context(limit)
+    return slot
+
+
+def deployment_slot(
+    core: "RustCore", deployment: Mapping[str, object], kwargs: Mapping[str, object]
+) -> contextlib.ExitStack:
+    """`_deployment_slot` outside a plan's attempt (a file created on every deployment)."""
+    from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit
+
+    slot: Final = contextlib.ExitStack()
+    limit: Final = max_parallel_requests_limit(core, deployment, kwargs)
+    if isinstance(limit, MaxParallelRequestsLimit):
+        slot.enter_context(limit)
+    return slot
+
+
+__all__ = ["deployment_slot", "enter_deployment_slot", "max_parallel_requests_limit"]

litellm/router_rust_walk.py

@@ -0,0 +1,292 @@
+"""
+The request metadata the Python router stamps as it walks retries and fallbacks, kept for a
+``RustCore`` call. router-core decides the walk and reports it (each attempt's ``scope``,
+``max_retries`` and ``model_group_size``, and the ``fallback`` and ``retry_logged`` events); this
+module only holds what each level's metadata dict holds in the Python router and lays it over the
+attempt's kwargs.
+
+In the Python router one metadata dict serves a level's retries, and every fallback hop copies
+its parent's (``run_async_fallback``'s ``_hop_metadata``), so a hop's breadcrumbs never reach
+its parent. Each scope here is one such dict.
+"""
+
+from collections.abc import Callable, Coroutine, Mapping, Sequence
+from typing import TYPE_CHECKING, Final
+
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
+
+if TYPE_CHECKING:
+    from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets
+
+# `RETRY_BREADCRUMB_LIMIT`: how many failures `previous_models` keeps.
+RETRY_BREADCRUMB_LIMIT: Final = 4
+
+# The keys the walk stamps on each attempt's metadata.
+_WALK_KEYS: Final = (
+    "original_model_group",
+    "attempted_fallbacks",
+    "attempted_retries",
+    "max_retries",
+    "model_group_size",
+    "previous_models",
+    "request_retry_count",
+)
+# What a breadcrumb reads from the last attempt a scope made.
+_SERVED_KEYS: Final = ("model_info", "deployment")
+# The walk state `run_async_fallback` hands every hop as kwargs.
+_HOP_KWARGS: Final = ("fallback_depth", "max_fallbacks", "attempted_targets", "include_fallback_errors")
+
+
+class _Scope:
+    __slots__ = ("kwargs", "metadata")
+
+    def __init__(
+        self,
+        metadata: dict[str, object],  # mutable-ok: the level's metadata, stamped as the walk runs
+        kwargs: Mapping[str, object],
+    ) -> None:
+        self.metadata: Final = metadata
+        self.kwargs: Final = kwargs
+
+
+def _breadcrumb(
+    metadata: Mapping[str, object], model: str, error: BaseException, params_base_model: Callable[[str], str | None]
+) -> dict[str, object]:  # mutable-ok: a fresh RetryAttemptRecord
+    """``_core_log_retry``'s record of the failure."""
+    from litellm.litellm_core_utils.get_litellm_params import get_base_model_from_litellm_call_metadata
+
+    model_info: Final = as_str_mapping(metadata.get("model_info")) or {}
+    deployment_id: Final = model_info.get("id")
+    known_id: Final = deployment_id if isinstance(deployment_id, str) else None
+    base_model: Final = get_base_model_from_litellm_call_metadata(metadata=metadata) or (
+        params_base_model(known_id) if known_id is not None else None
+    )
+    deployment: Final = metadata.get("deployment")
+    attempted: Final = metadata.get("attempted_retries")
+    return {
+        "model_group": model,
+        "deployment_id": known_id,
+        "model": deployment if isinstance(deployment, str) else None,
+        "base_model": base_model,
+        "exception_type": type(error).__name__,
+        "exception_string": str(error),
+        "attempted_retries": attempted if type(attempted) is int else None,
+    }
+
+
+def _kept_breadcrumbs(earlier: object) -> tuple[object, ...]:
+    """The breadcrumbs a new one joins: the latest ``RETRY_BREADCRUMB_LIMIT - 1``."""
+    if not isinstance(earlier, list | tuple):
+        return ()
+    stamped: Final[tuple[object, ...]] = tuple(earlier)  # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType]  # the breadcrumbs as stamped
+    return stamped[-(RETRY_BREADCRUMB_LIMIT - 1) :]
+
+
+def log_retry_into(
+    kwargs: dict[str, object],  # mutable-ok: the request kwargs whose metadata the record joins
+    error: Exception,
+    params_base_model: Callable[[str], str | None],
+) -> dict[str, object]:  # mutable-ok: the same kwargs, as ``_core_log_retry`` returns them
+    """
+    ``_core_log_retry`` (``RoutingContext.log_retry``): the failure joins the request metadata's
+    ``previous_models`` and counts toward ``request_retry_count``, with a ``litellm.routing.retry``
+    phase event when a pre-routing strategy picked the group.
+    """
+    from litellm.integrations.otel.runtime import phase_event
+
+    field: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"
+    metadata: Final = kwargs[field]
+    view: Final = as_str_mapping(metadata)
+    if view is None:
+        # Python reads it with ``.get``.
+        raise AttributeError(f"{type(metadata).__name__!r} object has no attribute 'get'")
+    model_group: Final = kwargs.get("model")
+    record: Final = _breadcrumb(view, "", error, params_base_model)
+    record["model_group"] = model_group if isinstance(model_group, str) else None
+    deployment_id: Final = record["deployment_id"]
+    breadcrumbs: Final = (*_kept_breadcrumbs(view.get("previous_models")), record)
+    earlier: Final = view.get("request_retry_count")
+    request_retry_count: Final = (earlier if type(earlier) is int and 0 <= earlier else 0) + 1
+    decision: Final = view.get("routing_decision")
+    router_name: Final = (as_str_mapping(decision) or {}).get("router_model_name")
+    if isinstance(router_name, str):
+        phase_event(
+            "litellm.routing.retry",
+            {
+                "litellm.routing.router_model_name": router_name,
+                "litellm.retry.count": request_retry_count,
+                "error.type": type(error).__name__,
+                **({"litellm.deployment.model_group": model_group} if isinstance(model_group, str) else {}),
+                **({"litellm.deployment.id": deployment_id} if isinstance(deployment_id, str) else {}),
+            },
+        )
+    metadata["previous_models"] = breadcrumbs  # pyright: ignore[reportIndexIssue]  # the logging object already holds this dict
+    metadata["request_retry_count"] = request_retry_count  # pyright: ignore[reportIndexIssue]  # same dict, read by the cap
+    return kwargs
+
+
+class WalkMetadata:
+    """The metadata each level of one call's walk holds, by scope."""
+
+    __slots__ = ("_attempted", "_field", "_params_base_model", "_request", "_scopes")
+
+    def __init__(
+        self, request: Mapping[str, object], field: str, params_base_model: Callable[[str], str | None]
+    ) -> None:
+        model: Final = request.get("model")
+        root: Final[dict[str, object]] = {"attempted_fallbacks": 0}  # mutable-ok: the root level's metadata
+        if model is not None:
+            root["original_model_group"] = model
+        self._request: Final = request
+        self._field: Final = field
+        self._params_base_model: Final = params_base_model
+        self._scopes: Final[dict[int, _Scope]] = {0: _Scope(root, {})}  # mutable-ok: filled as the walk hops
+        self._attempted: AttemptedFallbackTargets | None = None
+
+    def _scope(self, value: object) -> _Scope:
+        found: Final = self._scopes.get(value) if isinstance(value, int) else None
+        return found if found is not None else self._scopes[0]
+
+    def targets(self) -> "AttemptedFallbackTargets":
+        """The walk's ``attempted_targets``: the targets tried and the ones skipped, and why."""
+        from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets
+
+        if self._attempted is None:
+            self._attempted = AttemptedFallbackTargets()
+        return self._attempted
+
+    def hop(self, event: Mapping[str, object]) -> None:
+        """A ``fallback`` event: the new level copies its parent's metadata and kwargs."""
+        parent: Final = self._scope(event.get("parent"))
+        attempted: Final = self.targets()
+        for key in (event.get("from"), event.get("to")):
+            if isinstance(key, str):
+                attempted.record(key)
+        metadata: Final = dict(parent.metadata)
+        original: Final = metadata.pop("original_model_group", self._request.get("model"))
+        metadata.pop("model_group", None)
+        metadata.pop("attempted_fallbacks", None)
+        metadata["original_model_group"] = original
+        metadata["model_group"] = event.get("to")
+        metadata["attempted_fallbacks"] = event.get("depth")
+        kwargs: Final = {
+            **parent.kwargs,
+            "fallback_depth": event.get("depth"),
+            "max_fallbacks": event.get("max_fallbacks"),
+            "attempted_targets": attempted,
+        }
+        if self._request.get("include_fallback_errors") is True:
+            kwargs["include_fallback_errors"] = True
+        scope: Final = event.get("scope")
+        if isinstance(scope, int):
+            self._scopes[scope] = _Scope(metadata, kwargs)
+
+    def hop_kwargs(self, event: Mapping[str, object]) -> dict[str, object]:  # mutable-ok: a fresh dict for the loggers
+        """The kwargs the level held after a hop (``kwargs["model"] = target``), as the
+        fallback loggers see them."""
+        scope: Final = self._scope(event.get("scope"))
+        return {**self._request, **scope.kwargs, "model": event.get("to"), self._field: dict(scope.metadata)}
+
+    def log_retry(self, event: Mapping[str, object], error: BaseException | None) -> None:
+        """A ``retry_logged`` event: the failure joins the scope's ``previous_models``."""
+        scope: Final = self._scope(event.get("scope"))
+        metadata: Final = scope.metadata
+        attempted: Final = event.get("attempted_retries")
+        if isinstance(attempted, int):
+            metadata["attempted_retries"] = attempted
+        if error is None:
+            return
+        record: Final = _breadcrumb(metadata, str(event.get("model")), error, self._params_base_model)
+        earlier: Final = metadata.get("previous_models")
+        kept: Final = _kept_breadcrumbs(earlier)
+        count: Final = metadata.get("request_retry_count")
+        metadata["previous_models"] = (*kept, record)
+        metadata["request_retry_count"] = (count if type(count) is int and count >= 0 else 0) + 1
+
+    def request_retry_count(self, step: Mapping[str, object]) -> object:
+        """The level's ``request_retry_count``, as the retry loop has counted it so far."""
+        return self._scope(step.get("scope")).metadata.get("request_retry_count")
+
+    def stamp(self, step: Mapping[str, object], kwargs: Mapping[str, object]) -> Mapping[str, object]:
+        """The attempt's kwargs with its level's walk metadata and hop kwargs laid over them."""
+        scope: Final = self._scope(step.get("scope"))
+        metadata: Final = scope.metadata
+        metadata["attempted_retries"] = step.get("retry")
+        metadata["max_retries"] = step.get("max_retries")
+        metadata["model_group_size"] = step.get("model_group_size")
+        served: Final = dict(as_str_mapping(kwargs.get(self._field)) or {})
+        served.update({key: metadata[key] for key in _WALK_KEYS if key in metadata})
+        metadata.update({key: served.get(key) for key in _SERVED_KEYS})
+        return {**kwargs, **{key: scope.kwargs[key] for key in _HOP_KWARGS if key in scope.kwargs}, self._field: served}
+
+
+class FallbackOutcomes:
+    """What ``run_async_fallback`` does when a hop returns: it logs the hop to the process's
+    custom loggers and, under ``include_fallback_errors``, lists the level's errors (its own
+    first, then each hop that failed) in the response's ``x-litellm-fallback-errors``."""
+
+    __slots__ = ("_failed", "_header_errors", "_include", "_pending")
+
+    def __init__(self, request: Mapping[str, object]) -> None:
+        self._include: Final = request.get("include_fallback_errors", False) is True
+        self._failed: Final[dict[object, list[object]]] = {}  # mutable-ok: each level's failed hops, in order
+        self._header_errors: Final[list[list[object]]] = []  # mutable-ok: each successful hop's errors, innermost first
+        self._pending: Final[list[Coroutine[object, object, None]]] = []  # mutable-ok: logger calls not yet awaited
+
+    def done(
+        self,
+        event: Mapping[str, object],
+        level_error: BaseException | None,
+        hop_error: BaseException | None,
+        kwargs: Mapping[str, object],
+    ) -> None:
+        from litellm.litellm_core_utils.add_retry_fallback_headers import get_fallback_error_info
+        from litellm.router_utils import fallback_event_handlers as handlers
+
+        if level_error is None or not isinstance(level_error, Exception):
+            return
+        parent: Final = event.get("parent")
+        failed: Final = self._failed.setdefault(parent, [])
+        original: Final = str(event.get("from"))
+        if event.get("succeeded") is True:
+            if self._include:
+                self._header_errors.append([get_fallback_error_info(level_error), *failed])
+            self._pending.append(
+                handlers.log_success_fallback_event(  # pyright: ignore[reportUnknownMemberType]  # takes the kwargs as an untyped dict
+                    original_model_group=original, kwargs=dict(kwargs), original_exception=level_error
+                )
+            )
+            return
+        if isinstance(hop_error, Exception):
+            failed.append(get_fallback_error_info(hop_error))
+        self._pending.append(
+            handlers.log_failure_fallback_event(  # pyright: ignore[reportUnknownMemberType]  # see above
+                original_model_group=original, kwargs=dict(kwargs), original_exception=level_error
+            )
+        )
+
+    def take_pending(self) -> tuple[Coroutine[object, object, None], ...]:
+        pending: Final = tuple(self._pending)
+        self._pending.clear()
+        return pending
+
+    def add_headers(self, response: object, attempted_fallbacks: int) -> object:
+        """``add_fallback_headers_to_response`` for each successful hop, innermost first."""
+        from litellm.litellm_core_utils.add_retry_fallback_headers import add_fallback_headers_to_response
+
+        headed: object = response  # rebind-ok: gains each level's errors
+        for errors in self._header_errors:
+            headed = add_fallback_headers_to_response(
+                response=headed,
+                attempted_fallbacks=attempted_fallbacks,
+                fallback_errors=errors,  # pyright: ignore[reportArgumentType]  # FallbackErrorInfo dicts
+            )
+        return headed
+
+
+async def await_all(pending: Sequence[Coroutine[object, object, None]]) -> None:
+    for call in pending:
+        await call
+
+
+__all__ = ["RETRY_BREADCRUMB_LIMIT", "FallbackOutcomes", "WalkMetadata", "await_all"]

litellm/router_rust_websocket.py

@@ -0,0 +1,131 @@
+"""
+Responses websocket turns on the Rust router core (``async_admit_deployment_turn``): the socket is
+opened on one deployment, and the core hands the handler a ``turn_admission`` callable that re-runs
+that deployment's admission before every later turn: the io-token turn budget, the deployment
+budget limiter, the routing strategy (on that deployment alone) and the pre-call checks.
+
+Which strategy and selector apply is the Rust core's routing context (``routing_context``); the
+selectors, limiters and checks are the live Python objects the host keeps.
+"""
+
+from collections.abc import Callable, Mapping
+from functools import partial
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final
+
+from litellm.router_rust_shaping import as_dict
+
+if TYPE_CHECKING:
+    from litellm.router_rust_core import RustCore
+
+
+def turn_admission(
+    core: "RustCore", model_group: str, deployment: Mapping[str, object], kwargs: Mapping[str, object]
+) -> Callable[[], object]:
+    """The ``turn_admission`` the handler calls before each later turn."""
+    return partial(admit_deployment_turn, core, model_group, deployment, kwargs)
+
+
+def _deployment_as_configured_now(core: "RustCore", deployment: Mapping[str, object]) -> Mapping[str, object]:
+    model_info: Final = deployment.get("model_info")
+    model_id: Final = as_dict(model_info).get("id") if isinstance(model_info, dict) else None  # pyright: ignore[reportUnknownArgumentType]  # the configured model_info dict
+    current: Final = core._hosted._core_get_deployment(model_id) if isinstance(model_id, str) else None  # pyright: ignore[reportPrivateUsage]  # the core's host registry
+    return deployment if current is None else current.model_dump(exclude_none=True)
+
+
+def _budget_limiter(core: "RustCore") -> object:
+    """The router's deployment-budget callback; the Rust core declines every config that has one."""
+    from litellm.router_rust_core import RustCoreUnsupported
+
+    try:
+        return core.registry().plugins("budget_limiter")
+    except RustCoreUnsupported:
+        return None
+
+
+async def admit_deployment_turn(
+    core: "RustCore", model_group: str, opened_on: Mapping[str, object], request_kwargs: Mapping[str, object]
+) -> Mapping[str, object]:
+    """`async_admit_deployment_turn`."""
+    from litellm import exceptions as litellm_exceptions
+    from litellm.router_rust_selection import routing_pre_call_checks
+    from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
+        async_admit_io_token_turn,
+        without_io_token_limits,
+    )
+    from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
+    from litellm.utils import get_utc_datetime
+
+    deployment: Final = _deployment_as_configured_now(core, opened_on)
+    rate_limit_check: Final = next(
+        (
+            callback
+            for callback in core._checks.routing_callbacks()  # pyright: ignore[reportPrivateUsage]  # `_routing_callbacks`
+            if isinstance(callback, ModelRateLimitingCheck)
+        ),
+        None,
+    )
+    io_token_admission: Final[Mapping[str, object]] = (
+        MappingProxyType({})
+        if rate_limit_check is None
+        else await async_admit_io_token_turn(rate_limit_check.dual_cache, deployment, get_utc_datetime())  # pyright: ignore[reportArgumentType]  # the configured deployment dict
+    )
+    budget_limiter: Final[object] = _budget_limiter(core)
+    if budget_limiter is not None:
+        try:
+            await budget_limiter.async_filter_deployments(  # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]  # RouterBudgetLimiting
+                model=model_group,
+                healthy_deployments=[dict(deployment)],
+                messages=None,
+                request_kwargs=dict(request_kwargs),
+            )
+        except ValueError as e:
+            raise litellm_exceptions.RateLimitError(
+                message=str(e),
+                llm_provider="",
+                model=model_group,
+                category=litellm_exceptions.RateLimitErrorCategory.LITELLM_RATE_LIMIT,
+            ) from e
+    await _admit_turn_through_routing_strategy(core, model_group, deployment, request_kwargs)
+    await routing_pre_call_checks(core._checks.routing_callbacks(), dict(without_io_token_limits(deployment)))  # pyright: ignore[reportPrivateUsage]  # `_routing_callbacks`
+    return io_token_admission
+
+
+async def _admit_turn_through_routing_strategy(
+    core: "RustCore", model_group: str, deployment: Mapping[str, object], request_kwargs: Mapping[str, object]
+) -> None:
+    """`_admit_turn_through_routing_strategy`: the group's strategy picks among this deployment alone."""
+    from litellm import exceptions as litellm_exceptions
+    from litellm.router_api import RoutingContext
+    from litellm.router_rust_selection import pick_async
+    from litellm.router_utils.handle_error import async_raise_no_deployment_exception
+
+    routing_kwargs: Final = {key: value for key, value in request_kwargs.items() if key != "litellm_logging_obj"}
+    override: Final = routing_kwargs.get("routing_strategy")
+    strategy, scope, group = core._handle.routing_context(  # pyright: ignore[reportPrivateUsage]  # the core's Rust handle
+        model_group, override if isinstance(override, str) else None
+    )
+    selectors: Final = core._selectors  # pyright: ignore[reportPrivateUsage]  # the core's strategy selectors
+    selector: Final = selectors.selector_for(strategy, scope, group, routing_kwargs)
+    if strategy == "simple-shuffle" or selector is None:
+        return
+    selected: Final = await pick_async(
+        strategy, selector, model_group, [dict(deployment)], None, None, dict(routing_kwargs)
+    )
+    if selected is None:
+        no_deployment: Final = await async_raise_no_deployment_exception(
+            ctx=RoutingContext(core),  # pyright: ignore[reportArgumentType]  # the error reads registry() and health()
+            model=model_group,
+            parent_otel_span=None,
+        )
+        raise litellm_exceptions.RateLimitError(
+            message=str(no_deployment),
+            llm_provider="",
+            model=model_group,
+            category=litellm_exceptions.RateLimitErrorCategory.LITELLM_RATE_LIMIT,
+        ) from no_deployment
+    if selectors.is_override(strategy, selector):
+        await selector.async_pre_call_check(dict(deployment), None)  # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]  # every selector is a CustomLogger
+
+
+__all__ = ["admit_deployment_turn", "turn_admission"]

litellm/router_utils/global_callback_holders.py

@@ -0,0 +1,53 @@
+"""
+The routers of this process that hold optional pre-call checks and routing callbacks, whatever core
+serves them.
+
+A router that retires hands the checks it vacates in its host's ``callbacks`` list to a surviving
+router's same-keyed check (``routers_holding_global_callbacks``), and a router that drops a check at
+runtime keeps it registered while another live router still has one of that type
+(``live_check_owners``). Both rules look at every router in the process, so the sets live here,
+outside any one router core.
+
+A member of ``routers_holding_global_callbacks`` serves ``_global_callback_candidates()``,
+``_core_adaptive_routers`` and ``_sync_adaptive_router_hooks()``; a member of ``live_check_owners``
+serves ``optional_callbacks``.
+"""
+
+import threading
+import weakref
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Final, Protocol
+
+if TYPE_CHECKING:
+    from litellm.integrations.custom_logger import CustomLogger
+
+
+class GlobalCallbackHolder(Protocol):
+    """A router whose callbacks a retiring router may hand its vacated checks to."""
+
+    @property
+    def _core_adaptive_routers(self) -> Mapping[str, object]: ...
+
+    def _global_callback_candidates(self) -> "tuple[CustomLogger, ...]": ...
+
+    def _sync_adaptive_router_hooks(self) -> None: ...
+
+
+class CheckOwner(Protocol):
+    """A live router with optional pre-call checks."""
+
+    @property
+    def optional_callbacks(self) -> "Sequence[object] | None": ...
+
+
+routers_holding_global_callbacks: Final["weakref.WeakSet[GlobalCallbackHolder]"] = weakref.WeakSet()
+routers_holding_global_callbacks_lock: Final = threading.Lock()
+live_check_owners: Final["weakref.WeakSet[CheckOwner]"] = weakref.WeakSet()
+
+__all__ = [
+    "CheckOwner",
+    "GlobalCallbackHolder",
+    "live_check_owners",
+    "routers_holding_global_callbacks",
+    "routers_holding_global_callbacks_lock",
+]

litellm/rust_bridge/_native.pyi

@@ -1,6 +1,6 @@
 from asyncio import Future
 from collections.abc import Mapping, Sequence
-from typing import Never, final
+from typing import Literal, Never, NotRequired, TypedDict, final
 
 import httpx
 from pydantic import JsonValue
@@ -218,6 +218,8 @@ __all__ = [
     "NativeDiagnosticProcessor",
     "ProcessReservedForForking",
     "ResponsesWebSocketConnection",
+    "RouterCall",
+    "RouterCoreHandle",
     "RustBridgeDeclined",
     "RustUpstreamError",
     "TokenCounter",
@@ -325,3 +327,184 @@ class NativeCacheHandle:
     def ping(self) -> Future[bool]: ...
     def disconnect(self) -> Future[None]: ...
     def delete(self, keys: Sequence[str]) -> Future[None]: ...
+
+# The FailureKind names an attempt outcome's `kind` takes (router-core `FailureKind`).
+_FailureKind = Literal[
+    "RateLimitError",
+    "ContextWindowExceededError",
+    "ContentPolicyViolationError",
+    "AuthenticationError",
+    "PermissionDeniedError",
+    "NotFoundError",
+    "BadRequestError",
+    "Timeout",
+    "APIConnectionError",
+    "InternalServerError",
+    "ServiceUnavailableError",
+    "GuardrailIntervention",
+    "Declined",
+    "CallerCredentialError",
+    "APIError",
+]
+
+class _AttemptStep(TypedDict):
+    kind: Literal["attempt"]
+    deployment_id: str
+    # The deployment as a `model_list` entry, with its id in `model_info`.
+    deployment: dict[str, JsonValue]
+    # The name this attempt serves: the requested model or the fallback target.
+    model: str
+    model_group: str
+    target_order: int | None
+    candidate_ids: list[str]
+    retry: int
+    fallback_depth: int
+    fallback_level: int
+    timeout: float | None
+
+class _SleepStep(TypedDict):
+    kind: Literal["sleep"]
+    seconds: float
+
+class _AllowFallbackStep(TypedDict):
+    kind: Literal["allow_fallback"]
+    target: str
+
+class _SelectStep(TypedDict):
+    kind: Literal["select"]
+    stage: Literal["plugins", "strategy"]
+    strategy: str | None
+    model_group: str
+    candidates: list[str]
+
+# One RouterCache op against the router's DualCache. `key` is set for every op but `batch_get`,
+# which sets `keys`; `value` is the JSON to store (`set`) or the amount (`increment`); `ttl` is
+# in seconds. Outcomes: get -> the JSON value or None; batch_get -> one JSON value or None per
+# key, in order; increment -> the new int or float; set and delete -> None.
+class _CacheStep(TypedDict):
+    kind: Literal["cache"]
+    op: Literal["get", "set", "increment", "batch_get", "delete"]
+    key: str | None
+    keys: list[str] | None
+    value: JsonValue
+    ttl: float | None
+
+class _DoneStep(TypedDict):
+    kind: Literal["done"]
+    deployment_id: str
+    model_group: str
+    attempted_retries: int
+    attempted_fallbacks: int
+    total_tokens: int
+
+class _FailedStep(TypedDict):
+    kind: Literal["failed"]
+    error: str
+    error_kind: Literal["config", "unknown_model", "no_healthy_deployments", "attempt", "cache", "retired"]
+    attempt_failed: bool
+    status: int | None
+    failure_kind: _FailureKind | None
+
+# The end of a `RouterCoreHandle.select`.
+class _SelectedStep(TypedDict):
+    kind: Literal["done"]
+    deployment_id: str
+    model_group: str
+
+# The end of a `RouterCoreHandle.healthy`.
+class _HealthyStep(TypedDict):
+    kind: Literal["done"]
+    deployment_ids: list[str]
+
+# The end of a `RouterCoreHandle.report_failure`.
+class _ReportedStep(TypedDict):
+    kind: Literal["done"]
+    cooled: bool
+
+# `RouterCoreHandle.resolve`: what a name resolves to.
+class _Resolution(TypedDict):
+    model_group: str
+    kind: str
+    ids: list[str]
+
+_RouterStep = _AttemptStep | _SleepStep | _AllowFallbackStep | _SelectStep | _CacheStep | _DoneStep | _FailedStep
+
+class _AttemptSucceeded(TypedDict):
+    ok: Literal[True]
+    total_tokens: NotRequired[int]
+
+class _AttemptFailed(TypedDict):
+    ok: Literal[False]
+    kind: _FailureKind
+    status: NotRequired[int | None]
+    retry_after: NotRequired[float | None]
+    num_retries: NotRequired[int | None]
+    detail: NotRequired[str | None]
+
+# Each document is a mapping or its JSON text.
+_Document = Mapping[str, object] | str
+
+@final
+class RouterCoreHandle:
+    def __new__(cls, config: _Document, dynamic: _Document) -> RouterCoreHandle: ...
+    def start(self, op: str, request: _Document) -> RouterCall: ...
+    # Driven like `start`; its `done` step is a `_SelectedStep`.
+    def select(self, model: str, request: _Document) -> RouterCall: ...
+    # Driven like `select`; its `done` step is a `_HealthyStep`.
+    def healthy(self, model: str, request: _Document) -> RouterCall: ...
+    # `{"upsert": [deployments], "delete": [ids], "settings": {dynamic document}}`.
+    def apply(self, changes: _Document) -> int: ...
+    def registry_version(self) -> int: ...
+    # A failure the router did not see itself, driven like `start`: cache steps, then a
+    # `_ReportedStep`.
+    def report_failure(
+        self,
+        id: str,
+        failure: Mapping[str, object],
+        deployment_cooldown: bool = False,
+        count_only: bool = False,
+        requested_group: str | None = None,
+        cooldown_time: float | None = None,
+        uncounted: bool = False,
+    ) -> RouterCall: ...
+    # A success a logging callback saw, driven like `report_failure`; `done` carries `key`.
+    def report_success(self, id: str, total_tokens: float, requests: float) -> RouterCall: ...
+    # Count a success against any id, driven like `report_failure`; `done` carries `key`.
+    def count_success(self, id: str) -> RouterCall: ...
+    # [(id, tpm_key, rpm_key)] for this minute, or None for a group with no deployment.
+    def usage_keys(self, model_group: str) -> list[tuple[str, str, str]] | None: ...
+    # Every deployment as a `model_list` entry, in config order.
+    def deployments(self) -> list[dict[str, JsonValue]]: ...
+    def deployment(self, id: str) -> dict[str, JsonValue] | None: ...
+    def resolve(self, name: str, specific_deployment: bool = False) -> _Resolution | None: ...
+    def routable(self, model: str, request: _Document) -> list[dict[str, JsonValue]]: ...
+    # `_get_routing_context` for a pick of `model`: (strategy, scope, group).
+    def routing_context(self, model: str, routing_strategy: str | None = None) -> tuple[str, str, str | None]: ...
+    def retire(self) -> None: ...
+    # A stream that failed before content: what may still take over it, and the retry budget for
+    # a failure of `kind`: {"can_retry", "can_fall_back", "refusal_fallback": bool,
+    # "retry_budget": int | None, "policy_applies": bool}.
+    def stream_policy(
+        self, request: _Document, origin: _Document, kind: str | None = None
+    ) -> dict[str, bool | int | None]: ...
+    # The same-group retry and the fallback walk for such a stream, driven like `start`.
+    def retry_stream(
+        self, op: str, request: _Document, origin: _Document, failure: Mapping[str, object]
+    ) -> RouterCall: ...
+    def fallback_stream(
+        self, op: str, request: _Document, origin: _Document, failure: Mapping[str, object]
+    ) -> RouterCall: ...
+    def retry_policy_retries(self, failure: Mapping[str, object], model_group: str | None = None) -> int | None: ...
+    def allowed_fails_for(self, failure: Mapping[str, object]) -> int | None: ...
+    def should_retry(
+        self, failure: Mapping[str, object], counts: tuple[int, int], fallbacks: tuple[bool, bool, bool]
+    ) -> bool: ...
+
+@final
+class RouterCall:
+    # attempt: _AttemptSucceeded | _AttemptFailed; sleep: None; allow_fallback: bool;
+    # select: the remaining candidate ids, pick first; cache: see _CacheStep; first call: None.
+    # A call from `select` steps through select and cache only, and ends in _SelectedStep or failed.
+    def next(
+        self, outcome: _AttemptSucceeded | _AttemptFailed | bool | Sequence[str] | JsonValue | None
+    ) -> _RouterStep | _SelectedStep | _HealthyStep | _ReportedStep: ...

litellm/rust_bridge/catalog.py

@@ -23,6 +23,7 @@ class Route(str, Enum):
     OCR = "ocr"
     TOKEN_COUNTER = "token_counter"
     TOKENIZER = "tokenizer"
+    ROUTER = "router"
 
 
 class Target(str, Enum):
@@ -221,6 +222,7 @@ RULES: Final[Rules] = (
     RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY),
     RouteRule(Route.TOKEN_COUNTER, Rollout.PYTHON_ONLY),
     RouteRule(Route.TOKENIZER, Rollout.PYTHON_ONLY),
+    RouteRule(Route.ROUTER, Rollout.RUST_OPT_IN),
     RouteRule(Route.TRANSCRIPTION, Rollout.RUST_REQUIRED, providers=frozenset({"bedrock"})),
 )
 

litellm/utils.py

@@ -4165,6 +4165,13 @@ class _LiveDeploymentReplay:
     """
 
     callback: Callable[[], None] | None = None
+    # Replays other router cores add beside the slot, run after it.
+    extra: tuple[Callable[[], None], ...] = ()
+
+
+def add_live_deployment_replay(replay: Callable[[], None]) -> None:
+    """Add a callback that re-asserts another router core's live deployments after a refresh."""
+    _LiveDeploymentReplay.extra = (*_LiveDeploymentReplay.extra, replay)
 
 
 def set_live_deployment_replay(replay: Callable[[], None]) -> None:
@@ -4198,6 +4205,8 @@ def reapply_runtime_model_cost_registrations() -> None:
     """
     if _LiveDeploymentReplay.callback is not None:
         _LiveDeploymentReplay.callback()
+    for extra_replay in _LiveDeploymentReplay.extra:
+        extra_replay()
     if _runtime_registered_model_cost:
         register_model(model_cost=dict(_runtime_registered_model_cost))
 

tests/code_coverage_tests/router_litellm_globals_check.py

@@ -21,6 +21,7 @@ CHECKED = (
     "litellm/router_core.py",
     "litellm/router_deprecated.py",
     "litellm/router_python_core.py",
+    "litellm/router_rust_core.py",
 )
 
 

tests/code_coverage_tests/router_surface_check.py

@@ -11,7 +11,8 @@ modules listed there. The lists hold what is used today and may only shrink:
   also fails, so the entry is removed in the same change.
 
 "Outside" means every non-test file under litellm/ and enterprise/ except
-litellm/router.py and the router's own modules. Uses from the router family
+litellm/router.py and the router's own modules (including every
+litellm/router_rust_*.py module, the Rust core's host-side glue). Uses from the router family
 (litellm/router_utils, litellm/router_strategy) are listed separately, because
 those modules become private parts of the router core in a later step.
 
@@ -42,9 +43,12 @@ ROUTER_FILES = {
     "litellm/router_core.py",
     "litellm/router_python_core.py",
     "litellm/router_python_host.py",
+    "litellm/router_rust_core.py",
     "litellm/router_deprecated.py",
     "litellm/router_host.py",
 }
+# The Rust core's host-side glue modules are part of the router.
+ROUTER_FILE_PREFIXES = ("litellm/router_rust_",)
 ROUTER_CLASS_FILES = ["litellm/router.py", "litellm/router_python_core.py", "litellm/router_deprecated.py"]
 ROUTER_CLASS_NAMES = {"Router", "PythonCore", "DeprecatedRouterMembers"}
 
@@ -128,7 +132,7 @@ def scan() -> Dict[str, Dict[str, Set[str]]]:
     uses: Dict[str, Dict[str, Set[str]]] = {"outside": {}, "router_family": {}}
     imports: Dict[str, Set[str]] = {}
     for rel, path in _python_files():
-        if rel in ROUTER_FILES or _is_test(rel):
+        if rel in ROUTER_FILES or rel.startswith(ROUTER_FILE_PREFIXES) or _is_test(rel):
             continue
         with open(path, encoding="utf-8") as f:
             src = f.read()

tests/unit/test_router/core/conftest.py

@@ -1,4 +1,6 @@
-from collections.abc import Callable
+from collections.abc import Callable, Generator
+from pathlib import Path
+from typing import Final
 
 import pytest
 
@@ -15,12 +17,53 @@ def host() -> ScriptedHost:
 def make_core(request: pytest.FixtureRequest, host: ScriptedHost) -> Callable[..., RouterCore]:
     """``make_core(**router_arguments)``: the core under test, on the ``host`` fixture. Every test that
     uses it runs once per core in ``CORE_BUILDERS`` (``-k python`` or ``-k rust`` picks one)."""
-    builder = CORE_BUILDERS[request.param]
+    name: Final[str] = str(request.param)  # pyright: ignore[reportAny]  # the fixture parameter
+    builder: Final = CORE_BUILDERS[name]
 
     def build(**arguments: object) -> RouterCore:  # kwargs-ok: the router's constructor arguments
         try:
             return builder(host, **arguments)
         except CoreDeclined as declined:
-            pytest.skip(f"{request.param} core declines this configuration: {declined}")
+            pytest.skip(f"{name} core declines this configuration: {declined}")
 
     return build
+
+
+_RUST_PARITY_PENDING: Final = frozenset(
+    line.strip()
+    for line in (Path(__file__).parent / "rust_parity_pending.txt").read_text().splitlines()
+    if line.strip() and not line.startswith("#")
+)
+
+
+def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
+    """Characterization tests the Rust core does not match yet are listed in
+    ``rust_parity_pending.txt`` and expected to fail (strictly: one that starts passing must leave the list)."""
+    here: Final = Path(__file__).parent
+    for item in items:
+        path: Final = Path(str(item.path))
+        if not path.is_relative_to(here):
+            continue  # the hook sees every collected item, not only this folder's
+        name: Final = "::".join((path.relative_to(here).as_posix(), *item.nodeid.split("::")[1:]))
+        if name in _RUST_PARITY_PENDING:
+            item.add_marker(pytest.mark.xfail(reason="Rust core parity pending", strict=True))
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_call(item: pytest.Item) -> Generator[None, object, None]:
+    """A test that reaches a member or request the Rust core does not serve yet is skipped with
+    that member named, so the run separates "not served yet" from a wrong answer."""
+    outcome: Final = yield
+    excinfo: Final[object] = getattr(outcome, "excinfo", None)
+    if not isinstance(excinfo, tuple):
+        return
+    info: Final[tuple[object, ...]] = excinfo  # pyright: ignore[reportUnknownVariableType]  # pluggy's (type, value, traceback)
+    if len(info) < 2:
+        return
+    from litellm.router_rust_core import RustCoreUnsupported
+
+    error: Final[object] = info[1]
+    cause: Final[object] = getattr(error, "__cause__", None)
+    force_exception: Final[object] = getattr(outcome, "force_exception", None)
+    if (isinstance(error, RustCoreUnsupported) or isinstance(cause, RustCoreUnsupported)) and callable(force_exception):
+        force_exception(pytest.skip.Exception(f"not served by the Rust core yet: {error}", _use_item_location=True))

tests/unit/test_router/core/construction_precall_helpers.py

@@ -0,0 +1,98 @@
+"""
+Helpers for the construction / pre-call characterization tests.
+
+``CallbackHost`` is a ``ScriptedHost`` whose callback manager also has
+``remove_callback_from_list_by_object`` (litellm's ``LoggingCallbackManager`` has it; ``HostCallbacks``
+does not), so the path that switches optional pre-call checks off at runtime can run.
+"""
+
+import copy
+import gc
+import uuid
+from collections.abc import Generator, Mapping
+from typing import Final
+
+import litellm
+from tests.unit.test_router.core.scripted_host import HostCallbacks, ScriptedHost, deployment
+
+
+class RemovingHostCallbacks(HostCallbacks):
+    def remove_callback_from_list_by_object(
+        self,
+        callback_list: object,
+        obj: object,
+        require_self: bool = True,
+    ) -> None:
+        """``LoggingCallbackManager.remove_callback_from_list_by_object``, on the host's lists."""
+        if not isinstance(callback_list, list):
+            return
+        if require_self:
+            doomed = [c for c in callback_list if getattr(c, "__self__", None) is obj]
+        else:
+            doomed = [c for c in callback_list if c == obj]
+        for callback in doomed:
+            callback_list.remove(callback)
+
+
+class CallbackHost(ScriptedHost):
+    __slots__ = ()
+
+    @property
+    def callbacks(self) -> RemovingHostCallbacks:  # pyright: ignore[reportIncompatibleVariableOverride]
+        return RemovingHostCallbacks(self)
+
+
+def callback_names(callbacks: list[object]) -> list[str]:  # mutable-ok: a fresh list
+    """Each callback by name: a bound method by its function name, anything else by its class name."""
+    names: Final[list[str]] = []
+    for callback in callbacks:
+        function = getattr(callback, "__func__", None)
+        names.append(function.__name__ if function is not None else type(callback).__name__)
+    return names
+
+
+def host_callback_names(host: ScriptedHost) -> dict[str, list[str]]:  # mutable-ok: a fresh dict
+    return {name: callback_names(callbacks) for name, callbacks in host.lists.items()}
+
+
+def windowed(
+    group: str, deployment_id: str, max_input_tokens: int, **params: object
+) -> dict[str, object]:  # mutable-ok: a model_list entry
+    """A deployment declaring ``max_input_tokens`` under a model name no other test uses: a deployment's
+    model_info is written into the process-wide cost map under its model name and id."""
+    model_info: Final = {"max_input_tokens": max_input_tokens, **dict(params.pop("model_info", None) or {})}  # pyright: ignore[reportArgumentType]
+    return deployment(
+        group,
+        deployment_id,
+        model=f"openai/cw-{deployment_id}-{uuid.uuid4().hex[:8]}",
+        model_info=model_info,
+        **params,
+    )
+
+
+def collect_other_routers() -> None:
+    """Drop unreachable routers from earlier tests: retire() and runtime pre-call-check changes look
+    at every live router in the process, not only the ones on this host."""
+    gc.collect()
+
+
+def restoring_model_cost(*names: str) -> Generator[None, None, None]:
+    """Undo what routers built meanwhile wrote into the process cost map: keys they added are removed
+    and the named entries get their old value back."""
+    added_before: Final = frozenset(litellm.model_cost)
+    saved: Final = {name: copy.deepcopy(litellm.model_cost[name]) for name in names if name in litellm.model_cost}
+    yield
+    for key in frozenset(litellm.model_cost) - added_before:
+        litellm.model_cost.pop(key, None)
+    litellm.model_cost.update(saved)
+
+
+__all__ = [
+    "CallbackHost",
+    "RemovingHostCallbacks",
+    "callback_names",
+    "collect_other_routers",
+    "host_callback_names",
+    "restoring_model_cost",
+    "windowed",
+]

tests/unit/test_router/core/endpoints_helpers.py

@@ -0,0 +1,210 @@
+"""
+Helpers for the endpoint characterization tests (``test_endpoints_characterization*.py``).
+
+``EndpointHost`` is a ``ScriptedHost`` that also scripts the host ops ``ScriptedHost`` leaves to
+litellm (assistants, skills, realtime, fine-tuning, batches, ...), so every ``host.invoke`` the core
+makes is answered and recorded.
+
+Some endpoints do not go through ``host.invoke``: the Python core binds litellm's own functions for
+them when it is built (vector stores, Google GenAI, OCR, search, videos, containers, interactions,
+managed agents). ``patch_direct_endpoints`` replaces those litellm functions, before the core is
+built, with recording stand-ins that answer from the same host script, under the same ``__name__``
+(the core routes on the handler's name). A call to one of them is recorded with the function's name
+as its ``op``.
+"""
+
+import importlib
+from collections.abc import Callable, Mapping
+from typing import Final
+
+import pytest
+
+from tests.unit.test_router.core.scripted_host import INFERENCE_OPS, ScriptedHost
+
+EXTRA_HOST_OPS: Final = frozenset(
+    {
+        "aadapter_completion",
+        "acreate_assistants",
+        "adelete_assistant",
+        "aget_assistants",
+        "acreate_thread",
+        "aget_thread",
+        "a_add_message",
+        "aget_messages",
+        "arun_thread",
+        "acompact_responses",
+        "_arealtime",
+        "acreate_realtime_client_secret",
+        "arealtime_calls",
+        "acreate_realtime_transcription_session",
+        "_aresponses_websocket",
+        "acancel_fine_tuning_job",
+        "alist_fine_tuning_jobs",
+        "aretrieve_fine_tuning_job",
+        "afile_list",
+        "aimage_edit",
+        "allm_passthrough_route",
+        "acreate_skill",
+        "alist_skills",
+        "aget_skill",
+        "adelete_skill",
+        "aretrieve_batch",
+        "acancel_batch",
+        "alist_batches",
+    }
+)
+
+
+class EndpointHost(ScriptedHost):
+    """``ScriptedHost`` that scripts every host op the core's endpoints invoke."""
+
+    __slots__ = ()
+
+    def invoke(self, op: str) -> Callable[..., object]:
+        if op in INFERENCE_OPS or op not in EXTRA_HOST_OPS:
+            return super().invoke(op)
+        return recording_endpoint(self, op, is_async=True)
+
+
+def recording_endpoint(host: ScriptedHost, name: str, *, is_async: bool) -> Callable[..., object]:
+    """A stand-in for litellm's ``name`` that records the call on ``host`` and answers from its script."""
+    if is_async:
+
+        async def endpoint(*args: object, **kwargs: object) -> object:  # kwargs-ok: the endpoint's request
+            return await host._answer_async(name, args, kwargs)  # pyright: ignore[reportPrivateUsage]  # the test host
+
+        endpoint.__name__ = name
+        endpoint.__qualname__ = name
+        return endpoint
+
+    def sync_endpoint(*args: object, **kwargs: object) -> object:  # kwargs-ok: the endpoint's request
+        return host._answer_sync(name, args, kwargs)  # pyright: ignore[reportPrivateUsage]  # the test host
+
+    sync_endpoint.__name__ = name
+    sync_endpoint.__qualname__ = name
+    return sync_endpoint
+
+
+# module -> {attribute on the module: the function's own name}
+DIRECT_ENDPOINTS: Final[Mapping[str, Mapping[str, str]]] = {
+    "litellm.vector_stores.main": {
+        name: name
+        for name in (
+            "asearch",
+            "search",
+            "acreate",
+            "create",
+            "aretrieve",
+            "retrieve",
+            "alist",
+            "list",
+            "aupdate",
+            "update",
+            "adelete",
+            "delete",
+        )
+    },
+    "litellm.vector_stores": {"acreate": "acreate"},
+    "litellm.vector_store_files.main": {
+        name: name
+        for name in (
+            "acreate",
+            "create",
+            "alist",
+            "list",
+            "aretrieve",
+            "retrieve",
+            "aretrieve_content",
+            "retrieve_content",
+            "aupdate",
+            "update",
+            "adelete",
+            "delete",
+        )
+    },
+    "litellm.google_genai": {
+        name: name
+        for name in ("agenerate_content", "generate_content", "agenerate_content_stream", "generate_content_stream")
+    },
+    "litellm.ocr": {"aocr": "aocr", "ocr": "ocr"},
+    "litellm.search": {"asearch": "asearch", "search": "search"},
+    "litellm.decisions": {"adecisions": "adecisions", "decisions": "decisions"},
+    "litellm.systemone": {"asystemone": "asystemone", "systemone": "systemone"},
+    "litellm.extraction": {"aextraction": "aextraction", "extraction": "extraction"},
+    "litellm.videos": {
+        f"{prefix}video_{name}": f"{prefix}video_{name}"
+        for prefix in ("a", "")
+        for name in (
+            "generation",
+            "list",
+            "status",
+            "content",
+            "remix",
+            "create_character",
+            "get_character",
+            "edit",
+            "extension",
+        )
+    },
+    "litellm.containers": {
+        f"{prefix}{name}": f"{prefix}{name}"
+        for prefix in ("a", "")
+        for name in ("create_container", "list_containers", "retrieve_container", "delete_container")
+    },
+    "litellm.interactions": {
+        f"{prefix}{name}": f"{prefix}{name}" for prefix in ("a", "") for name in ("create", "get", "delete", "cancel")
+    },
+    "litellm.interactions.agents": {
+        f"{prefix}{name}": f"{prefix}{name}"
+        for prefix in ("a", "")
+        for name in ("create", "get", "delete", "list", "list_versions")
+    },
+}
+
+# Recorded op names for the direct endpoints whose bare names collide across modules.
+RECORDED_NAMES: Final[Mapping[tuple[str, str], str]] = {
+    **{("litellm.vector_stores.main", name): f"vs.{name}" for name in DIRECT_ENDPOINTS["litellm.vector_stores.main"]},
+    ("litellm.vector_stores", "acreate"): "vs.acreate",
+    **{
+        ("litellm.vector_store_files.main", name): f"vsf.{name}"
+        for name in DIRECT_ENDPOINTS["litellm.vector_store_files.main"]
+    },
+    **{("litellm.interactions", name): f"interactions.{name}" for name in DIRECT_ENDPOINTS["litellm.interactions"]},
+    **{
+        ("litellm.interactions.agents", name): f"agents.{name}"
+        for name in DIRECT_ENDPOINTS["litellm.interactions.agents"]
+    },
+}
+
+
+def patch_direct_endpoints(monkeypatch: pytest.MonkeyPatch, host: ScriptedHost) -> None:
+    """Replace litellm's directly bound endpoint functions with recording stand-ins on ``host``.
+
+    The stand-in keeps the function's ``__name__`` (the core routes on it); the call is recorded
+    under ``RECORDED_NAMES`` when the bare name is ambiguous, else under the function's name."""
+    for module_name, attributes in DIRECT_ENDPOINTS.items():
+        module: Final = importlib.import_module(module_name)
+        for attribute, function_name in attributes.items():
+            original: Final[object] = getattr(module, attribute)
+            is_async: Final = function_name.startswith("a")
+            recorded: Final = RECORDED_NAMES.get((module_name, attribute), function_name)
+            stand_in: Final = recording_endpoint(host, recorded, is_async=is_async)
+            stand_in.__name__ = getattr(original, "__name__", function_name)
+            monkeypatch.setattr(module, attribute, stand_in)
+    from litellm.containers import endpoint_factory
+
+    for name in list(endpoint_factory._generated_endpoints):  # pyright: ignore[reportPrivateUsage]  # litellm's registry
+        monkeypatch.setitem(
+            endpoint_factory._generated_endpoints,  # pyright: ignore[reportPrivateUsage]  # litellm's registry
+            name,
+            recording_endpoint(host, name, is_async=name.startswith("a")),
+        )
+
+
+__all__ = [
+    "DIRECT_ENDPOINTS",
+    "EXTRA_HOST_OPS",
+    "EndpointHost",
+    "patch_direct_endpoints",
+    "recording_endpoint",
+]

tests/unit/test_router/core/fallbacks_helpers.py

@@ -0,0 +1,105 @@
+"""Helpers for the fallbacks characterization tests (``test_fallbacks_characterization*.py``)."""
+
+from collections.abc import Callable, Mapping
+from typing import Final
+
+from litellm.router_core import RouterCore
+from litellm.router_python_host import Endpoint
+from tests.unit.test_router.core.scripted_host import Outcome, ScriptedHost, deployment
+
+MakeCore = Callable[..., RouterCore]
+
+MESSAGES: Final = [{"role": "user", "content": "hi"}]
+GROUPS_ABC: Final = [deployment("a", "a"), deployment("b", "b"), deployment("c", "c")]
+GROUP_A_OF_THREE: Final = [deployment("a", "a1"), deployment("a", "a2"), deployment("a", "a3"), deployment("b", "b")]
+TIERED_A: Final = [
+    deployment("a", "a1", order=1),
+    deployment("a", "a2", order=2),
+    deployment("b", "b"),
+    deployment("c", "c"),
+]
+
+
+def build(make_core: MakeCore, **arguments: object) -> RouterCore:  # kwargs-ok: the router's constructor arguments
+    return make_core(**{"model_list": GROUPS_ABC, "num_retries": 0, **arguments})
+
+
+async def complete(core: RouterCore, model: str = "a", **request: object) -> object:  # kwargs-ok: request kwargs
+    return await core.acompletion(model=model, messages=MESSAGES, **request)  # pyright: ignore[reportAttributeAccessIssue]  # endpoint method
+
+
+def complete_sync(core: RouterCore, model: str = "a", **request: object) -> object:  # kwargs-ok: request kwargs
+    return core.completion(model=model, messages=MESSAGES, **request)  # pyright: ignore[reportAttributeAccessIssue]  # endpoint method
+
+
+def hidden(response: object) -> Mapping[str, object]:
+    return getattr(response, "_hidden_params")
+
+
+def headers(response: object) -> Mapping[str, object]:
+    found: Final = hidden(response)["additional_headers"]
+    assert isinstance(found, Mapping)
+    return found
+
+
+def metadata_of(kwargs: Mapping[str, object]) -> Mapping[str, object]:
+    found: Final = kwargs.get("metadata") or kwargs.get("litellm_metadata") or {}
+    assert isinstance(found, Mapping)
+    return found
+
+
+def show_router_debug(host: ScriptedHost) -> None:
+    host.set_dynamic(expose_router_debug_in_errors=True)
+
+
+def fails_once_then_answers(error: BaseException) -> Callable[..., Outcome]:
+    """One scripted outcome shared by several models: the first call to any of them raises
+    ``error``, every later call answers the default response."""
+    calls: Final = [0]
+
+    def outcome(**_request: object) -> Outcome:  # kwargs-ok: the scripted request
+        calls[0] += 1
+        return error if calls[0] == 1 else None
+
+    return outcome
+
+
+class NamedEndpointHost(ScriptedHost):
+    """A ``ScriptedHost`` whose ``anthropic_messages`` endpoint carries that name, as litellm's
+    own function does. The core recognises a /v1/messages call (and its safeguard-refusal gate)
+    by the endpoint's ``__name__``; ``ScriptedHost``'s endpoints are all named ``endpoint``."""
+
+    __slots__ = ()
+
+    def invoke(self, op: str) -> Endpoint:
+        endpoint: Final = super().invoke(op)
+        if op == "anthropic_messages":
+            endpoint.__name__ = op  # pyright: ignore[reportFunctionMemberAccess]  # a plain function
+        return endpoint
+
+
+def anthropic_refusal(*, with_stop_details: bool = True) -> dict[str, object]:  # mutable-ok: a provider response
+    response: Final[dict[str, object]] = {
+        "id": "msg_1",
+        "type": "message",
+        "role": "assistant",
+        "model": "claude",
+        "content": [],
+        "stop_reason": "refusal",
+        "usage": {"input_tokens": 1, "output_tokens": 1},
+    }
+    if with_stop_details:
+        response["stop_details"] = {"type": "refusal", "category": None, "explanation": "no"}
+    return response
+
+
+def anthropic_message(text: str = "fine") -> dict[str, object]:  # mutable-ok: a provider response
+    return {
+        "id": "msg_2",
+        "type": "message",
+        "role": "assistant",
+        "model": "claude",
+        "content": [{"type": "text", "text": text}],
+        "stop_reason": "end_turn",
+        "usage": {"input_tokens": 1, "output_tokens": 1},
+    }

tests/unit/test_router/core/gaps_l11100_helpers.py

@@ -0,0 +1,112 @@
+"""Shared helpers for ``test_gaps_l11100*.py``: strategy-router marker deployments and a host whose
+callback registration can also remove a callback from every list (what the adaptive router's hook
+sync asks of litellm's ``LoggingCallbackManager``)."""
+
+from collections.abc import Callable, Mapping
+from typing import Final
+
+from litellm.router_core import RouterCore
+from tests.unit.test_router.core.scripted_host import HostCallbacks, ScriptedHost
+
+MakeCore = Callable[..., RouterCore]
+
+TIERS: Final = {"SIMPLE": "small", "MEDIUM": "big", "COMPLEX": "big", "REASONING": "big"}
+
+
+class RemovingHostCallbacks(HostCallbacks):
+    """``HostCallbacks`` plus ``remove_callback_from_all_lists``, as litellm's callback manager has it."""
+
+    def __init__(self, host: "RemovingHost") -> None:
+        super().__init__(host)
+        self._owner: Final = host
+
+    def remove_callback_from_all_lists(self, obj: object, require_self: bool = False) -> None:
+        del require_self
+        for name, callbacks in self._owner.lists.items():
+            self._owner.lists[name] = [callback for callback in callbacks if callback is not obj]
+
+
+class RemovingHost(ScriptedHost):
+    """A ``ScriptedHost`` whose callbacks can be removed from every list."""
+
+    __slots__ = ()
+
+    @property
+    def callbacks(self) -> RemovingHostCallbacks:  # pyright: ignore[reportIncompatibleVariableOverride]  # same methods plus one
+        return RemovingHostCallbacks(self)
+
+
+def complexity_marker(
+    name: str = "smart",
+    deployment_id: str = "marker",
+    *,
+    config: Mapping[str, object] | None = None,
+    **params: object,  # kwargs-ok: extra marker litellm_params
+) -> dict[str, object]:  # mutable-ok: a model_list entry
+    """An ``auto_router/complexity_router`` marker with the shipped tiers and default model ``big``."""
+    return {
+        "model_name": name,
+        "litellm_params": {
+            "model": "auto_router/complexity_router",
+            "complexity_router_config": dict(config) if config is not None else {"tiers": dict(TIERS)},
+            "complexity_router_default_model": "big",
+            **params,
+        },
+        "model_info": {"id": deployment_id},
+    }
+
+
+def adaptive_marker(
+    name: str, deployment_id: str, available_models: list[str], **params: object
+) -> dict[str, object]:  # mutable-ok: a model_list entry  # kwargs-ok: extra marker litellm_params
+    """An ``auto_router/adaptive_router`` marker over ``available_models``."""
+    return {
+        "model_name": name,
+        "litellm_params": {
+            "model": "auto_router/adaptive_router",
+            "adaptive_router_config": {"available_models": available_models},
+            **params,
+        },
+        "model_info": {"id": deployment_id},
+    }
+
+
+def marker(
+    name: str, deployment_id: str, model: str, **params: object
+) -> dict[str, object]:  # mutable-ok: a model_list entry  # kwargs-ok: marker litellm_params
+    """Any strategy-router marker: ``model`` is its ``auto_router/...`` litellm model."""
+    return {"model_name": name, "litellm_params": {"model": model, **params}, "model_info": {"id": deployment_id}}
+
+
+def registered_tags(core: RouterCore, kind: str) -> dict[str, list[tuple[str, ...]]]:  # mutable-ok: a fresh dict
+    """``{router name: [tags of each registered strategy]}`` for a pre-routing strategy registry."""
+    registry: Final = core.registry().plugins(kind)  # pyright: ignore[reportCallIssue, reportArgumentType]  # one of the strategy kinds
+    assert isinstance(registry, dict)
+    return {name: [entry.tags for entry in entries] for name, entries in registry.items()}  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # TaggedPreRoutingStrategy lists
+
+
+def model_ids(core: RouterCore) -> list[str]:  # mutable-ok: a fresh list
+    """The ``model_info.id`` of every deployment on the model list, in order."""
+    return [str(entry["model_info"]["id"]) for entry in core.registry().all()]  # pyright: ignore[reportAny]  # model_list rows
+
+
+class ModelsEndpoint:
+    """A stand-in for the ``AsyncHTTPHandler`` that model-info discovery reads ``/v1/models`` with:
+    answers ``body`` (or raises ``error``) and records each request's url and headers."""
+
+    def __init__(self, body: Mapping[str, object] | None = None, error: BaseException | None = None) -> None:
+        self.body: Final = body or {"data": []}
+        self.error: Final = error
+        self.requests: Final[list[tuple[str, dict[str, str]]]] = []
+
+    async def get(
+        self, url: str, headers: dict[str, str] | None = None, **_: object
+    ) -> object:  # kwargs-ok: httpx options
+        import json
+
+        import httpx
+
+        self.requests.append((url, dict(headers or {})))
+        if self.error is not None:
+            raise self.error
+        return httpx.Response(200, content=json.dumps(self.body).encode(), request=httpx.Request("GET", url))

tests/unit/test_router/core/registry_helpers.py

@@ -0,0 +1,47 @@
+"""Shared helpers for the registry characterization tests (``test_registry_characterization*.py``)."""
+
+import uuid
+from collections.abc import Callable, Iterable, Mapping
+
+from litellm.router_core import RouterCore
+from litellm.types.router import Deployment
+from tests.unit.test_router.core.scripted_host import deployment
+
+MakeCore = Callable[..., RouterCore]
+
+MESSAGES = [{"role": "user", "content": "hi"}]
+
+
+def ids(deployments: Iterable[Mapping[str, object]] | None) -> list[str]:  # mutable-ok: a fresh list
+    """The ``model_info.id`` of each deployment dict, in order."""
+    return [str(entry["model_info"]["id"]) for entry in deployments or ()]  # pyright: ignore[reportIndexIssue]  # deployment dicts
+
+
+def names_and_ids(
+    deployments: Iterable[Mapping[str, object]] | None,
+) -> list[tuple[str, str]]:  # mutable-ok: a fresh list
+    """``(model_name, model_info.id)`` of each deployment dict, in order."""
+    return [(str(entry["model_name"]), str(entry["model_info"]["id"])) for entry in deployments or ()]  # pyright: ignore[reportIndexIssue]  # deployment dicts
+
+
+def typed(model_name: str, deployment_id: str, **params: object) -> Deployment:  # kwargs-ok: litellm_params
+    """A ``Deployment`` built from the same entry ``scripted_host.deployment`` returns."""
+    return Deployment(**deployment(model_name, deployment_id, **params))  # pyright: ignore[reportArgumentType]  # a model_list entry
+
+
+def team_deployment(
+    model_name: str, deployment_id: str, team_id: str, public_name: str, **params: object
+) -> dict[str, object]:  # mutable-ok: a model_list entry  # kwargs-ok: litellm_params
+    """A team-owned deployment: ``model_info.team_id`` and ``team_public_model_name`` set."""
+    extra_info: dict[str, object] = dict(params.pop("model_info", None) or {})  # pyright: ignore[reportArgumentType]  # a model_info dict
+    return deployment(
+        model_name,
+        deployment_id,
+        model_info={"team_id": team_id, "team_public_model_name": public_name, **extra_info},
+        **params,
+    )
+
+
+def unique_id(prefix: str = "reg") -> str:
+    """A deployment id no other test uses, for tests that write into the process-wide ``litellm.model_cost``."""
+    return f"{prefix}-{uuid.uuid4().hex}"

tests/unit/test_router/core/retries_health_helpers.py

@@ -0,0 +1,198 @@
+"""
+Helpers for the retries / deployment-health characterization tests (``test_retries_health_characterization*.py``).
+
+Backoff values carry litellm's random jitter (up to ``JITTER`` seconds), so sleeps are asserted inside
+``[base, base + JITTER)``.
+"""
+
+import asyncio
+import copy
+from collections.abc import Callable, Mapping
+from typing import Final
+
+import httpx
+
+import litellm
+from litellm.router_core import RouterCore
+from tests.unit.test_router.core.scripted_host import Call, ScriptedHost
+
+MESSAGES: Final = [{"role": "user", "content": "hi"}]
+
+MakeCore = Callable[..., RouterCore]
+ErrorBuilder = Callable[[], Exception]
+
+_REQUEST: Final = httpx.Request("POST", "https://api.openai.com/v1/chat/completions")
+
+
+def fresh(builder: ErrorBuilder) -> Callable[..., Exception]:
+    """A scripted outcome that raises a new error per call."""
+    return lambda **_: builder()
+
+
+def http_response(status: int, headers: Mapping[str, str] | None = None) -> httpx.Response:
+    return httpx.Response(status, headers=dict(headers or {}), request=_REQUEST)
+
+
+def not_found() -> litellm.NotFoundError:
+    return litellm.NotFoundError(message="no such model", llm_provider="openai", model="gpt-4o")
+
+
+def permission_denied() -> litellm.PermissionDeniedError:
+    return litellm.PermissionDeniedError(
+        message="forbidden", llm_provider="openai", model="gpt-4o", response=http_response(403)
+    )
+
+
+def unprocessable() -> litellm.UnprocessableEntityError:
+    return litellm.UnprocessableEntityError(
+        message="unprocessable", llm_provider="openai", model="gpt-4o", response=http_response(422)
+    )
+
+
+def service_unavailable() -> litellm.ServiceUnavailableError:
+    return litellm.ServiceUnavailableError(message="unavailable", llm_provider="openai", model="gpt-4o")
+
+
+def bad_gateway() -> litellm.BadGatewayError:
+    return litellm.BadGatewayError(message="bad gateway", llm_provider="openai", model="gpt-4o")
+
+
+def api_connection() -> litellm.APIConnectionError:
+    return litellm.APIConnectionError(message="connection reset", llm_provider="openai", model="gpt-4o")
+
+
+def api_status(status: int) -> ErrorBuilder:
+    return lambda: litellm.APIError(
+        status_code=status, message=f"status {status}", llm_provider="openai", model="gpt-4o"
+    )
+
+
+def plain_error() -> ValueError:
+    return ValueError("not a provider error")
+
+
+def rate_limited(retry_after: str | None = None) -> litellm.RateLimitError:
+    headers: Final = {"retry-after": retry_after} if retry_after is not None else {}
+    return litellm.RateLimitError(
+        message="slow down", llm_provider="openai", model="gpt-4o", response=httpx.Response(429, headers=headers)
+    )
+
+
+async def attempt(core: RouterCore, model: str = "g", **kwargs: object) -> object:  # kwargs-ok: request kwargs
+    """The response, or the error the core raised. A request the Rust core does not serve yet propagates, so the
+    test is skipped rather than failed."""
+    from litellm.router_rust_core import RustCoreUnsupported
+
+    try:
+        return await core.acompletion(model=model, messages=MESSAGES, **kwargs)  # pyright: ignore[reportAttributeAccessIssue]  # every core serves acompletion
+    except RustCoreUnsupported:
+        raise
+    except Exception as error:
+        return error
+
+
+def attempt_sync(core: RouterCore, model: str = "g", **kwargs: object) -> object:  # kwargs-ok: request kwargs
+    from litellm.router_rust_core import RustCoreUnsupported
+
+    try:
+        return core.completion(model=model, messages=MESSAGES, **kwargs)  # pyright: ignore[reportAttributeAccessIssue]  # every core serves completion
+    except RustCoreUnsupported:
+        raise
+    except Exception as error:
+        return error
+
+
+def cooled_down(core: RouterCore, ids: list[str]) -> dict[str, Mapping[str, object]]:  # mutable-ok: a fresh dict
+    return dict(core.health().cooldowns.get_active_cooldowns(ids, None))
+
+
+def headers_of(response: object) -> Mapping[str, object]:
+    hidden: Final = getattr(response, "_hidden_params", None) or {}
+    return hidden.get("additional_headers") or {}
+
+
+class MetadataRecorder:
+    """A scripted outcome factory that snapshots the request metadata each call carried, then answers."""
+
+    KEYS: Final = (
+        "attempted_retries",
+        "max_retries",
+        "model_group",
+        "model_group_size",
+        "request_retry_count",
+        "previous_models",
+    )
+
+    def __init__(self) -> None:
+        self.snapshots: Final[list[dict[str, object]]] = []
+
+    def then(self, builder: ErrorBuilder | None) -> Callable[..., object]:
+        def outcome(**kwargs: object) -> object:  # kwargs-ok: the request
+            metadata: Final = kwargs.get("metadata") or kwargs.get("litellm_metadata") or {}
+            assert isinstance(metadata, Mapping)
+            self.snapshots.append({key: copy.deepcopy(metadata.get(key)) for key in self.KEYS})
+            return builder() if builder is not None else None
+
+        return outcome
+
+
+class BackgroundSuccessLoggingHost(ScriptedHost):
+    """``ScriptedHost`` whose async success callbacks run in a background task after the response is returned,
+    as litellm's logging worker runs them (``ScriptedHost`` runs them before returning). Await ``drain()``
+    before reading what they wrote."""
+
+    __slots__ = ("_pending",)
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._pending: list[asyncio.Task[None]] = []
+
+    async def _answer_async(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> object:
+        call, outcome = self._outcome(op, args, kwargs)
+        error, response = self._finish(op, call, outcome)
+        if error is not None:
+            logged_failure: Final = self._logging_kwargs(call, error, response)
+            for callback in list(self.lists.get("failure_callback", ())):
+                if callable(callback):
+                    callback(logged_failure, None, 0.0, 0.0)
+            for callback in list(self.lists.get("_async_failure_callback", ())):
+                if callable(callback):
+                    await callback(logged_failure, None, 0.0, 0.0)
+            raise error
+
+        async def log_success(logged_call: Call) -> None:
+            logged: Final = self._logging_kwargs(logged_call, None, response)
+            for callback in list(self.lists.get("_async_success_callback", ())):
+                if callable(callback):
+                    await callback(logged, response, 0.0, 0.0)
+
+        self._pending.append(asyncio.ensure_future(log_success(call)))
+        return response
+
+    async def drain(self) -> None:
+        while self._pending:
+            await self._pending.pop()
+
+
+__all__ = [
+    "MESSAGES",
+    "BackgroundSuccessLoggingHost",
+    "ErrorBuilder",
+    "MakeCore",
+    "MetadataRecorder",
+    "api_connection",
+    "api_status",
+    "attempt",
+    "attempt_sync",
+    "bad_gateway",
+    "cooled_down",
+    "fresh",
+    "headers_of",
+    "http_response",
+    "not_found",
+    "permission_denied",
+    "plain_error",
+    "rate_limited",
+    "service_unavailable",
+    "unprocessable",
+]

tests/unit/test_router/core/rust_parity_pending.txt

@@ -0,0 +1,4 @@
+# unskipped by p3-selection
+test_construction_precall_characterization.py::test_retiring_one_router_keeps_the_callbacks_of_another_on_the_same_host[rust]
+# unskipped by p6-final
+test_construction_precall_characterization.py::test_alerting_config_registers_slack_alerting_and_its_slow_response_callback[rust]

tests/unit/test_router/core/scripted_host.py

@@ -324,6 +324,9 @@ class ScriptedHost(PythonHost):
 
         return done()
 
+    def sleep_sync(self, seconds: float) -> None:
+        self.sleeps.append(seconds)
+
     def dynamic(self) -> RouterDynamicConfig:
         return self._dynamic
 
@@ -452,25 +455,46 @@ def python_core(host: ScriptedHost, **arguments: object) -> PythonCore:  # kwarg
     return PythonCore(**config.arguments, _host=host)  # pyright: ignore[reportArgumentType]  # the same arguments Router binds
 
 
-CoreBuilder = Callable[..., RouterCore]
-"""``builder(host, **router_arguments)``: a router core on ``host``."""
-
-CORE_BUILDERS: Final[dict[str, CoreBuilder]] = {"python": python_core}
-"""Every core the suite runs on. Another core adds its builder here; a builder that cannot take a
-configuration raises ``CoreDeclined``, and the test is skipped for that core."""
-
-
 class CoreDeclined(Exception):
     """The core does not serve this configuration (the router would stay on the Python core)."""
 
 
+CoreBuilder = Callable[..., RouterCore]
+"""``builder(host, **router_arguments)``: a router core on ``host``."""
+
+
+def rust_core(host: ScriptedHost, **arguments: object) -> RouterCore:  # kwargs-ok: the router's constructor arguments
+    """A ``RustCore`` on ``host``, built the way ``make_core`` builds one when the Rust router is on."""
+    from litellm.router_rust_core import RustCore
+    from litellm.rust_bridge._native import RustBridgeDeclined
+
+    config: Final = build_router_config(arguments)  # an invalid config raises here, as for the Python core
+    try:
+        return RustCore.build(config, host)
+    except RustBridgeDeclined as declined:  # make_core keeps such a router on the Python core
+        raise CoreDeclined(str(declined)) from declined
+
+
+CORE_BUILDERS: Final[dict[str, CoreBuilder]] = {"python": python_core, "rust": rust_core}
+
+
+def records_outcomes_through_callbacks(core: RouterCore) -> bool:
+    """Whether ``core`` learns attempt outcomes from callbacks it registers on its host. The Python
+    core does; the Rust core records every outcome from the attempt itself and registers none."""
+    return isinstance(core, PythonCore)
+
+
+"""Every core the suite runs on. Another core adds its builder here; a builder that cannot take a
+configuration raises ``CoreDeclined``, and the test is skipped for that core."""
+
+
 __all__ = [
-    "Call",
-    "HostCallbacks",
-    "INFERENCE_OPS",
     "CORE_BUILDERS",
+    "INFERENCE_OPS",
+    "Call",
     "CoreBuilder",
     "CoreDeclined",
+    "HostCallbacks",
     "ScriptedHost",
     "auth_error",
     "bad_request",
@@ -482,6 +506,7 @@ __all__ = [
     "embedding_response",
     "python_core",
     "rate_limit",
+    "rust_core",
     "server_error",
     "timeout",
 ]

tests/unit/test_router/core/selection_helpers.py

@@ -0,0 +1,88 @@
+"""
+Helpers for the selection characterization tests.
+
+``TrafficHost`` is a ``ScriptedHost`` whose async inference calls also feed the routing strategy
+loggers the core registered on the host, the way litellm's logging does around a real call: input
+callbacks see the call start (``log_pre_api_call``), and the ``CustomLogger`` instances in the
+``callbacks`` list see it succeed or fail with real start and end times. A call can be held open
+on a gate (an ``asyncio.Event``) or delayed for real, so in-flight counts and latencies are real
+for any core.
+"""
+
+import asyncio
+import datetime
+from collections.abc import Callable, Mapping
+from typing import Final
+
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.router_core import RouterCore
+from tests.unit.test_router.core.scripted_host import Call, ScriptedHost
+
+MakeCore = Callable[..., RouterCore]
+
+MESSAGES: Final = [{"role": "user", "content": "hi"}]
+
+
+class TrafficHost(ScriptedHost):
+    """``ScriptedHost`` plus the strategy-logger traffic litellm's logging produces around a call."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.gates: dict[str, asyncio.Event] = {}
+        self.delays: dict[str, float] = {}
+        self.started: list[str] = []  # mutable-ok: the models whose calls have started, in order
+
+    def _loggers(self, name: str) -> tuple[CustomLogger, ...]:
+        return tuple(callback for callback in self.lists.get(name, ()) if isinstance(callback, CustomLogger))
+
+    async def _answer_async(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> object:
+        model: Final = str(kwargs.get("model") or (args[0] if args else ""))
+        logged: Final = self._logging_kwargs(Call(op=op, model=model, kwargs=dict(kwargs)), None, None)
+        for logger in self._loggers("input_callback"):
+            logger.log_pre_api_call(model=model, messages=kwargs.get("messages"), kwargs=logged)
+        self.started.append(model)
+        start: Final = datetime.datetime.now()
+        gate: Final = self.gates.get(model)
+        if gate is not None:
+            await gate.wait()
+        delay: Final = self.delays.get(model)
+        if delay:
+            await asyncio.sleep(delay)
+        try:
+            response = await super()._answer_async(op, args, kwargs)
+        except Exception as error:
+            failed = {**logged, "exception": error}
+            for logger in self._loggers("callbacks"):
+                await logger.async_log_failure_event(failed, None, start, datetime.datetime.now())
+            raise
+        succeeded: Final = self._logging_kwargs(Call(op=op, model=model, kwargs=dict(kwargs)), None, response)
+        for logger in self._loggers("callbacks"):
+            await logger.async_log_success_event(succeeded, response, start, datetime.datetime.now())
+        return response
+
+
+async def picked_ids(
+    core: RouterCore,
+    model: str,
+    times: int = 20,
+    request_kwargs: Mapping[str, object] | None = None,
+    **select_arguments: object,  # kwargs-ok: select's keyword options
+) -> set[str]:
+    """The deployment ids ``select`` picks over ``times`` picks, each with a fresh copy of ``request_kwargs``."""
+    picked: Final[set[str]] = set()
+    for _ in range(times):
+        chosen = await core.select(model, dict(request_kwargs or {}), **select_arguments)  # pyright: ignore[reportCallIssue]  # PythonCore's select takes keyword options
+        assert isinstance(chosen, Mapping)
+        picked.add(str(chosen["model_info"]["id"]))
+    return picked
+
+
+def ids_of(deployments: object) -> list[str]:  # mutable-ok: a fresh list
+    """The ids of a ``select(kind="healthy")`` result, in order (a single dict gives one id)."""
+    if isinstance(deployments, Mapping):
+        return [str(deployments["model_info"]["id"])]
+    assert isinstance(deployments, list)
+    return [str(deployment["model_info"]["id"]) for deployment in deployments]
+
+
+__all__ = ["MESSAGES", "MakeCore", "TrafficHost", "ids_of", "picked_ids"]

tests/unit/test_router/core/streaming_helpers.py

@@ -0,0 +1,361 @@
+"""
+Scripted streams for the streaming characterization tests.
+
+``ScriptedHost`` outcomes may be callables, so each builder here returns a callable that the host
+calls with the request kwargs and that answers with a fresh stream:
+
+- ``chat_stream``: a ``CustomStreamWrapper`` over chat chunks, optionally raising ``error`` after
+  ``fail_after`` chunks, and ending with a ``finish_reason`` chunk unless ``finish`` is False.
+- ``anthropic_stream``: an async iterator of raw ``/v1/messages`` SSE byte frames, optionally raising.
+- ``responses_stream``: a ``BaseResponsesAPIStreamingIterator`` over Responses API events,
+  optionally raising.
+"""
+
+import datetime
+import json
+import sys
+import types
+from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence
+from typing import Final
+
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
+from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
+from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
+from litellm.types.utils import ModelResponseStream
+
+MESSAGES: Final = [{"role": "user", "content": "hi"}]
+
+
+def install_orjson_stub_if_missing() -> None:
+    """
+    The /v1/messages stream wrapper imports proxy modules that import ``orjson`` and ``soundfile`` at load
+    time. Where either is not installed (some dev containers), a stand-in is registered so the wrapper can
+    load (json-backed for ``orjson``, empty for ``soundfile``, which only audio passthrough logging uses);
+    where they are installed (CI), nothing changes.
+    """
+    try:
+        import orjson  # noqa: F401  # pyright: ignore[reportUnusedImport]  # only probing for the module
+    except ImportError:
+        stub: Final = types.ModuleType("orjson")
+        stub.loads = json.loads  # pyright: ignore[reportAttributeAccessIssue]  # module attribute
+        stub.dumps = lambda value, **_options: json.dumps(value).encode()  # pyright: ignore[reportAttributeAccessIssue]  # module attribute
+        stub.JSONDecodeError = json.JSONDecodeError  # pyright: ignore[reportAttributeAccessIssue]  # module attribute
+        sys.modules["orjson"] = stub
+    try:
+        import soundfile  # noqa: F401  # pyright: ignore[reportUnusedImport, reportMissingImports]  # only probing for the module
+    except (ImportError, OSError):
+        sys.modules["soundfile"] = types.ModuleType("soundfile")
+
+
+def _logging_obj(model: str, call_type: str = "acompletion") -> LiteLLMLogging:
+    logging_obj: Final = LiteLLMLogging(
+        model=model,
+        messages=MESSAGES,
+        stream=True,
+        call_type=call_type,
+        start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
+        litellm_call_id=f"stream-{model}",
+        function_id=f"stream-{model}",
+    )
+    logging_obj.update_environment_variables(model=model, user=None, optional_params={}, litellm_params={})
+    return logging_obj
+
+
+def text_chunk(text: str) -> ModelResponseStream:
+    return ModelResponseStream(choices=[{"index": 0, "delta": {"role": "assistant", "content": text}}])
+
+
+def role_chunk() -> ModelResponseStream:
+    return ModelResponseStream(choices=[{"index": 0, "delta": {"role": "assistant", "content": ""}}])
+
+
+def finish_chunk(reason: str = "stop", usage: Mapping[str, int] | None = None) -> ModelResponseStream:
+    chunk: Final = ModelResponseStream(choices=[{"index": 0, "delta": {}, "finish_reason": reason}])
+    if usage is not None:
+        setattr(chunk, "usage", dict(usage))
+    return chunk
+
+
+def chat_stream(
+    *texts: str,
+    error: BaseException | None = None,
+    fail_after: int | None = None,
+    finish: bool = True,
+    leading_role_chunk: bool = False,
+    model: str = "d1",
+    hidden_params: Mapping[str, object] | None = None,
+    sync: bool = False,
+) -> Callable[..., CustomStreamWrapper]:
+    """
+    A stream of ``texts`` (one chunk each). With ``error``, raises it after ``fail_after`` chunks
+    (default: after every text chunk, before the finish chunk). ``leading_role_chunk`` sends a
+    role-only, empty-content chunk first, which counts as the first chunk but carries no content.
+    """
+
+    def answer(**request: object) -> CustomStreamWrapper:  # kwargs-ok: the host passes the request kwargs
+        produced: list[ModelResponseStream] = []
+        if leading_role_chunk:
+            produced.append(role_chunk())
+        produced.extend(text_chunk(text) for text in texts)
+        cut_at: Final = len(produced) if fail_after is None else fail_after
+
+        def items() -> Iterator[ModelResponseStream]:
+            for index, chunk in enumerate(produced):
+                if error is not None and index == cut_at:
+                    raise error
+                yield chunk
+            if error is not None:
+                raise error
+            if finish:
+                yield finish_chunk()
+
+        async def achunks() -> AsyncIterator[ModelResponseStream]:
+            for chunk in items():
+                yield chunk
+
+        wrapper: Final = CustomStreamWrapper(
+            completion_stream=items() if sync else achunks(),
+            model=model,
+            custom_llm_provider="openai",
+            logging_obj=_logging_obj(model, "completion" if sync else "acompletion"),
+        )
+        if hidden_params is not None:
+            wrapper._hidden_params.update(hidden_params)  # pyright: ignore[reportPrivateUsage]  # test seam
+        return wrapper
+
+    return answer
+
+
+async def collect_texts(stream: object) -> str:
+    """The text content of every chunk of an async chat stream, joined."""
+    texts: Final[list[str]] = []
+    async for chunk in stream:  # pyright: ignore[reportGeneralTypeIssues]  # an async stream
+        choices = getattr(chunk, "choices", None)
+        if choices:
+            content = choices[0].delta.content
+            if content:
+                texts.append(content)
+    return "".join(texts)
+
+
+def collect_texts_sync(stream: object) -> str:
+    texts: Final[list[str]] = []
+    for chunk in stream:  # pyright: ignore[reportGeneralTypeIssues]  # a sync stream
+        choices = getattr(chunk, "choices", None)
+        if choices:
+            content = choices[0].delta.content
+            if content:
+                texts.append(content)
+    return "".join(texts)
+
+
+# --- /v1/messages ------------------------------------------------------------------------------
+
+
+def sse(event: str, data: Mapping[str, object]) -> bytes:
+    return f"event: {event}\ndata: {json.dumps(data)}\n\n".encode()
+
+
+MESSAGE_START: Final = sse(
+    "message_start",
+    {"type": "message_start", "message": {"id": "msg_1", "type": "message", "role": "assistant", "content": []}},
+)
+BLOCK_START: Final = sse(
+    "content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}
+)
+PING: Final = sse("ping", {"type": "ping"})
+MESSAGE_STOP: Final = sse("message_stop", {"type": "message_stop"})
+
+
+def text_delta(text: str) -> bytes:
+    return sse(
+        "content_block_delta",
+        {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}},
+    )
+
+
+def error_frame(error_type: str = "overloaded_error", message: str = "Overloaded") -> bytes:
+    return sse("error", {"type": "error", "error": {"type": error_type, "message": message}})
+
+
+def refusal_delta(stop_details: Mapping[str, object] | None = None) -> bytes:
+    delta: Final[dict[str, object]] = {"stop_reason": "refusal"}
+    if stop_details is not None:
+        delta["stop_details"] = dict(stop_details)
+    return sse("message_delta", {"type": "message_delta", "delta": delta, "usage": {"output_tokens": 1}})
+
+
+class ByteStream:
+    """An async iterator of SSE frames that records whether it was closed."""
+
+    def __init__(
+        self, frames: Sequence[bytes], error: BaseException | None, hidden_params: Mapping[str, object] | None
+    ) -> None:
+        self._frames: Final = list(frames)
+        self._error: Final = error
+        self._index = 0
+        self.closed = False
+        if hidden_params is not None:
+            self._hidden_params = dict(hidden_params)
+
+    def __aiter__(self) -> "ByteStream":
+        return self
+
+    async def __anext__(self) -> bytes:
+        if self._index < len(self._frames):
+            self._index += 1
+            return self._frames[self._index - 1]
+        if self._error is not None:
+            raise self._error
+        raise StopAsyncIteration
+
+    async def aclose(self) -> None:
+        self.closed = True
+
+
+def anthropic_stream(
+    *frames: bytes,
+    error: BaseException | None = None,
+    hidden_params: Mapping[str, object] | None = None,
+    opened: list[ByteStream] | None = None,
+) -> Callable[..., ByteStream]:
+    def answer(**_request: object) -> ByteStream:  # kwargs-ok: the host passes the request kwargs
+        stream: Final = ByteStream(frames, error, hidden_params)
+        if opened is not None:
+            opened.append(stream)
+        return stream
+
+    return answer
+
+
+async def collect_bytes(stream: object) -> list[bytes]:
+    return [frame async for frame in stream]  # pyright: ignore[reportGeneralTypeIssues]  # an async stream
+
+
+# --- Responses API -----------------------------------------------------------------------------
+
+
+class Event:
+    """A Responses API stream event: only ``type`` (and optionally ``response``) is read."""
+
+    def __init__(self, event_type: str, response: object = None, delta: str | None = None) -> None:
+        self.type = event_type
+        self.response = response
+        self.delta = delta
+        self._hidden_params: dict[str, object] = {}
+
+    def __repr__(self) -> str:
+        return f"Event({self.type!r})"
+
+
+class BackgroundResponse:
+    background = True
+    id = "resp_bg"
+
+
+class ScriptedResponsesStream(BaseResponsesAPIStreamingIterator):
+    """A Responses stream over ``events`` that bypasses the provider plumbing of the base class."""
+
+    def __init__(  # pyright: ignore[reportMissingSuperCall]  # the base needs an httpx response
+        self,
+        events: Sequence[object],
+        error: BaseException | None,
+        model: str,
+        custom_llm_provider: str,
+        hidden_params: Mapping[str, object] | None,
+        started_tool_call: bool = False,
+        completed_response: object = None,
+    ) -> None:
+        self._events: Final = list(events)
+        self._started_tool_call = started_tool_call
+        self._error: Final = error
+        self._index = 0
+        self._hidden_params = dict(hidden_params or {})
+        self.model = model
+        self.custom_llm_provider = custom_llm_provider
+        self.logging_obj = _logging_obj(model, "aresponses")
+        self.litellm_metadata = None
+        self.responses_api_provider_config = None
+        self.finished = False
+        self.completed_response = completed_response
+        self.response = None
+        self.start_time = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
+        self.request_data = {}
+        self.call_type = None
+        self.closed = False
+
+    def __aiter__(self) -> "ScriptedResponsesStream":
+        return self
+
+    async def __anext__(self) -> object:
+        if self._index < len(self._events):
+            self._index += 1
+            return self._events[self._index - 1]
+        if self._error is not None:
+            raise self._error
+        raise StopAsyncIteration
+
+    async def aclose(self) -> None:
+        self.closed = True
+
+
+def responses_stream(
+    *events: object,
+    error: BaseException | None = None,
+    model: str = "d1",
+    custom_llm_provider: str = "anthropic",
+    hidden_params: Mapping[str, object] | None = None,
+    opened: list[ScriptedResponsesStream] | None = None,
+    started_tool_call: bool = False,
+    completed_response: object = None,
+) -> Callable[..., ScriptedResponsesStream]:
+    """``completed_response`` is the terminal event the stream latched before failing, if any."""
+
+    def answer(**_request: object) -> ScriptedResponsesStream:  # kwargs-ok: the host passes the request kwargs
+        stream: Final = ScriptedResponsesStream(
+            events, error, model, custom_llm_provider, hidden_params, started_tool_call, completed_response
+        )
+        if opened is not None:
+            opened.append(stream)
+        return stream
+
+    return answer
+
+
+async def drain_into(stream: object, seen: list[object]) -> None:
+    """Append every item of an async stream to ``seen``, letting an error the stream raises propagate."""
+    async for item in stream:  # pyright: ignore[reportGeneralTypeIssues]  # an async stream
+        seen.append(item)
+
+
+async def collect_event_types(stream: object) -> list[str]:
+    return [getattr(event, "type", repr(event)) async for event in stream]  # pyright: ignore[reportGeneralTypeIssues]  # an async stream
+
+
+__all__ = [
+    "BLOCK_START",
+    "MESSAGES",
+    "MESSAGE_START",
+    "MESSAGE_STOP",
+    "PING",
+    "BackgroundResponse",
+    "ByteStream",
+    "Event",
+    "ScriptedResponsesStream",
+    "anthropic_stream",
+    "chat_stream",
+    "collect_bytes",
+    "collect_event_types",
+    "collect_texts",
+    "collect_texts_sync",
+    "drain_into",
+    "error_frame",
+    "install_orjson_stub_if_missing",
+    "finish_chunk",
+    "refusal_delta",
+    "responses_stream",
+    "role_chunk",
+    "sse",
+    "text_chunk",
+    "text_delta",
+]

tests/unit/test_router/core/test_construction_precall_characterization_precall.py

@@ -0,0 +1,663 @@
+"""
+Characterization of the pre-call checks on a router core: with ``enable_pre_call_checks`` the
+context window (token counting over messages, Responses input, instructions, tools and an Anthropic
+system block), the allowed region and unsupported params (``response_format``) narrow a group's
+deployments before one is picked; image requests skip text-only deployments; budget limits drop
+deployments over budget. Covers the errors raised when nothing is left (type and message), which
+deployment ``select`` and the endpoints reach, and that no provider call is made then.
+
+Each test asserts what the Python core does today; oddities are named in the test's docstring.
+"""
+
+import re
+from collections.abc import Callable, Generator, Mapping
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.router_core import RouterCore
+from litellm.types.router import RouterRateLimitError
+from tests.unit.test_router.core.construction_precall_helpers import CallbackHost, restoring_model_cost, windowed
+from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, python_core
+
+MakeCore = Callable[..., RouterCore]
+PICKS: Final = 60  # three-way shuffles miss a deployment in 20 picks about once in a thousand runs
+SHORT: Final = [{"role": "user", "content": "hi"}]
+LONG_TEXT: Final = " ".join(["word"] * 100)
+LONG: Final = [{"role": "user", "content": LONG_TEXT}]
+IMAGE: Final = [
+    {
+        "role": "user",
+        "content": [
+            {"type": "text", "text": "what is this"},
+            {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}},
+        ],
+    }
+]
+UNCOUNTABLE: Final = [
+    {"role": "user", "content": [{"type": "image_url", "image_url": {"detail": "not-a-real-detail-value"}}]}
+]
+JSON_MODE: Final = {"response_format": {"type": "json_object"}}
+NO_JSON_MODE_MODEL: Final = "openai/gpt-3.5-turbo-16k"
+JSON_MODE_MODEL: Final = "openai/gpt-4o"
+CONTEXT_WINDOW_PREFIX: Final = (
+    "litellm.ContextWindowExceededError: litellm.BadRequestError: litellm._pre_call_checks: Context Window exceeded "
+    "for given call. No models have context window large enough for this call.\n"
+)
+UNSUPPORTED_MESSAGE: Final = (
+    "litellm.UnsupportedParamsError: no deployment of model=g supports ['response_format']. To drop these, set "
+    "`litellm.drop_params=True` or for proxy `litellm_settings: drop_params: true`, or send "
+    "allowed_openai_params=['response_format'] to pass them through"
+)
+
+
+@pytest.fixture
+def host() -> ScriptedHost:
+    return CallbackHost()
+
+
+@pytest.fixture(autouse=True)
+def isolated_model_cost() -> Generator[None, None, None]:
+    """Routers write their deployments' model info into the process cost map; undo it after each test."""
+    yield from restoring_model_cost("gpt-3.5-turbo-16k")
+
+
+async def _picked(
+    core: RouterCore, model: str = "g", times: int = PICKS, **select_arguments: object
+) -> set[str]:  # kwargs-ok: select's arguments
+    picked: Final[set[str]] = set()
+    for _ in range(times):
+        chosen = await core.select(model, **select_arguments)  # pyright: ignore[reportCallIssue]
+        assert isinstance(chosen, Mapping)
+        picked.add(str(chosen["model_info"]["id"]))
+    return picked
+
+
+def _small_and_big(
+    make_core: MakeCore, small: int = 5, **arguments: object
+) -> RouterCore:  # kwargs-ok: constructor arguments
+    return make_core(
+        model_list=[windowed("g", "small", small), windowed("g", "big", 10000)],
+        **{"enable_pre_call_checks": True, **arguments},
+    )
+
+
+def _json_mode_group(make_core: MakeCore, **arguments: object) -> RouterCore:  # kwargs-ok: constructor arguments
+    return make_core(
+        model_list=[deployment("g", "old", model=NO_JSON_MODE_MODEL), deployment("g", "new", model=JSON_MODE_MODEL)],
+        **{"enable_pre_call_checks": True, **arguments},
+    )
+
+
+# --- context window ----------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_without_enable_pre_call_checks_a_too_small_context_window_is_still_picked(make_core: MakeCore) -> None:
+    core: Final = _small_and_big(make_core, enable_pre_call_checks=False)
+    assert await _picked(core, messages=LONG) == {"small", "big"}
+
+
+@pytest.mark.asyncio
+async def test_without_enable_pre_call_checks_a_too_long_prompt_reaches_the_provider(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[windowed("g", "small", 5)])
+    await core.acompletion(model="g", messages=LONG)  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.deployments_called() == ["small"]
+
+
+@pytest.mark.asyncio
+async def test_select_skips_a_deployment_whose_context_window_the_prompt_exceeds(make_core: MakeCore) -> None:
+    assert await _picked(_small_and_big(make_core), messages=LONG) == {"big"}
+
+
+@pytest.mark.asyncio
+async def test_a_short_prompt_may_pick_either_deployment(make_core: MakeCore) -> None:
+    assert await _picked(_small_and_big(make_core, small=50), messages=SHORT) == {"small", "big"}
+
+
+@pytest.mark.asyncio
+async def test_async_completion_reaches_only_a_deployment_whose_context_window_fits(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = _small_and_big(make_core)
+    for _ in range(8):
+        await core.acompletion(model="g", messages=LONG)  # pyright: ignore[reportAttributeAccessIssue]
+    assert set(host.deployments_called()) == {"big"}
+
+
+def test_sync_completion_reaches_only_a_deployment_whose_context_window_fits(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = _small_and_big(make_core)
+    for _ in range(8):
+        core.completion(model="g", messages=LONG)  # pyright: ignore[reportAttributeAccessIssue]
+    assert set(host.deployments_called()) == {"big"}
+
+
+@pytest.mark.asyncio
+async def test_pass_through_selection_skips_a_deployment_whose_context_window_the_prompt_exceeds(
+    make_core: MakeCore,
+) -> None:
+    core: Final = make_core(
+        model_list=[
+            windowed("g", "small", 5, use_in_pass_through=True),
+            windowed("g", "big", 10000, use_in_pass_through=True),
+        ],
+        enable_pre_call_checks=True,
+    )
+    assert await _picked(core, kind="pass_through", messages=LONG) == {"big"}
+
+
+@pytest.mark.asyncio
+async def test_select_without_messages_or_input_runs_no_context_window_check(make_core: MakeCore) -> None:
+    assert await _picked(_small_and_big(make_core)) == {"small", "big"}
+
+
+@pytest.mark.asyncio
+async def test_when_no_context_window_fits_select_raises_context_window_exceeded(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[windowed("g", "small", 5)], enable_pre_call_checks=True)
+    with pytest.raises(litellm.ContextWindowExceededError) as error:
+        await core.select("g", messages=LONG)
+    message: Final = str(error.value)
+    assert message.startswith(CONTEXT_WINDOW_PREFIX)
+    assert re.fullmatch(
+        r"Model=openai/cw-small-[0-9a-f]{8}, Max Input Tokens=5, Got=\d+", message.removeprefix(CONTEXT_WINDOW_PREFIX)
+    )
+    assert error.value.model == "g"
+    assert error.value.status_code == 400
+
+
+@pytest.mark.asyncio
+async def test_when_no_context_window_fits_async_completion_raises_without_calling_the_provider(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[windowed("g", "small", 5)], enable_pre_call_checks=True)
+    with pytest.raises(litellm.ContextWindowExceededError, match="Context Window exceeded for given call"):
+        await core.acompletion(model="g", messages=LONG)  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.calls == []
+    assert host.sleeps == []
+
+
+def test_when_no_context_window_fits_sync_completion_raises_without_calling_the_provider(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[windowed("g", "small", 5)], enable_pre_call_checks=True)
+    with pytest.raises(litellm.ContextWindowExceededError):
+        core.completion(model="g", messages=LONG)  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.calls == []
+
+
+@pytest.mark.asyncio
+async def test_the_context_window_error_lists_each_deployment_back_to_back(make_core: MakeCore) -> None:
+    """Oddity: the per-deployment entries are concatenated with no separator between them."""
+    core: Final = make_core(model_list=[windowed("g", "one", 5), windowed("g", "two", 6)], enable_pre_call_checks=True)
+    with pytest.raises(litellm.ContextWindowExceededError) as error:
+        await core.select("g", messages=LONG)
+    assert re.fullmatch(
+        r"Model=openai/cw-one-[0-9a-f]{8}, Max Input Tokens=5, Got=(\d+)Model=openai/cw-two-[0-9a-f]{8}, Max Input Tokens=6, Got=\1",
+        str(error.value).removeprefix(CONTEXT_WINDOW_PREFIX),
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("where", "expected"),
+    [("model_info", "azure/gpt-4-mi-base"), ("litellm_params", "azure/gpt-4-lp-base")],
+)
+async def test_the_context_window_error_names_a_deployment_by_its_base_model(
+    make_core: MakeCore, where: str, expected: str
+) -> None:
+    extra: Final[dict[str, object]] = (
+        {"model_info": {"base_model": expected}} if where == "model_info" else {"base_model": expected}
+    )
+    core: Final = make_core(model_list=[windowed("g", f"base-{where}", 5, **extra)], enable_pre_call_checks=True)
+    with pytest.raises(litellm.ContextWindowExceededError) as error:
+        await core.select("g", messages=LONG)
+    assert f"Model={expected}, Max Input Tokens=5, Got=" in str(error.value)
+
+
+@pytest.mark.asyncio
+async def test_when_no_context_window_fits_the_context_window_fallback_group_answers(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(
+        model_list=[windowed("small-group", "small", 5), deployment("big-group", "big")],
+        enable_pre_call_checks=True,
+        context_window_fallbacks=[{"small-group": ["big-group"]}],
+        num_retries=0,
+    )
+    await core.acompletion(model="small-group", messages=LONG)  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.deployments_called() == ["big"]
+
+
+@pytest.mark.asyncio
+async def test_a_deployment_without_a_known_context_window_is_kept_for_any_prompt(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[windowed("g", "small", 5), deployment("g", "unknown-window-model")], enable_pre_call_checks=True
+    )
+    assert await _picked(core, messages=LONG) == {"unknown-window-model"}
+
+
+@pytest.mark.asyncio
+async def test_the_context_window_comes_from_the_cost_map_when_the_deployment_declares_none(
+    make_core: MakeCore,
+) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "gpt4", model="openai/gpt-4"), deployment("g", "gpt4o", model="openai/gpt-4o")],
+        enable_pre_call_checks=True,
+    )
+    assert await _picked(core, times=10, messages=[{"role": "user", "content": "word " * 10000}]) == {"gpt4o"}
+
+
+@pytest.mark.asyncio
+async def test_an_uncountable_prompt_skips_the_context_window_check(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[windowed("g", "tiny", 1)], enable_pre_call_checks=True)
+    assert await _picked(core, times=2, messages=UNCOUNTABLE) == {"tiny"}
+
+
+@pytest.mark.asyncio
+async def test_responses_input_text_is_counted_against_the_context_window(make_core: MakeCore) -> None:
+    assert await _picked(_small_and_big(make_core), times=10, input=LONG_TEXT) == {"big"}
+
+
+@pytest.mark.asyncio
+async def test_a_short_responses_input_may_pick_either_deployment(make_core: MakeCore) -> None:
+    assert await _picked(_small_and_big(make_core, small=50), input="hi") == {"small", "big"}
+
+
+@pytest.mark.asyncio
+async def test_responses_instructions_are_counted_against_the_context_window(make_core: MakeCore) -> None:
+    core: Final = _small_and_big(make_core, small=50)
+    assert await _picked(core, request_kwargs={"instructions": LONG_TEXT}, input="hi") == {"big"}
+
+
+@pytest.mark.asyncio
+async def test_tool_definitions_are_counted_against_the_context_window(make_core: MakeCore) -> None:
+    tool: Final = {
+        "type": "function",
+        "function": {"name": "f", "description": LONG_TEXT, "parameters": {"type": "object", "properties": {}}},
+    }
+    core: Final = _small_and_big(make_core, small=50)
+    assert await _picked(core, request_kwargs={"tools": [tool]}, messages=SHORT) == {"big"}
+
+
+@pytest.mark.asyncio
+async def test_an_anthropic_system_block_is_counted_against_the_context_window(make_core: MakeCore) -> None:
+    core: Final = _small_and_big(make_core, small=50)
+    assert await _picked(core, request_kwargs={"system": LONG_TEXT}, messages=SHORT) == {"big"}
+
+
+@pytest.mark.asyncio
+async def test_a_deployment_context_window_is_written_into_the_process_cost_map(make_core: MakeCore) -> None:
+    """Oddity: a deployment's declared ``max_input_tokens`` replaces the process-wide cost-map entry of its
+    model, so it outlives the router and narrows another router's deployment of the same model."""
+    first: Final = make_core(
+        model_list=[deployment("g", "declares-five", model=NO_JSON_MODE_MODEL, model_info={"max_input_tokens": 5})]
+    )
+    first.retire()
+    other: Final = python_core(
+        CallbackHost(),
+        model_list=[deployment("g", "declares-nothing", model=NO_JSON_MODE_MODEL)],
+        enable_pre_call_checks=True,
+    )
+    with pytest.raises(litellm.ContextWindowExceededError, match="Max Input Tokens=5"):
+        await other.select("g", messages=SHORT)
+    other.retire()
+
+
+# --- region ------------------------------------------------------------------------------
+
+
+def _regional(make_core: MakeCore, **arguments: object) -> RouterCore:  # kwargs-ok: constructor arguments
+    return make_core(
+        model_list=[
+            deployment("g", "eu", region_name="eu"),
+            deployment("g", "us", region_name="us"),
+            deployment("g", "none"),
+        ],
+        **{"enable_pre_call_checks": True, **arguments},
+    )
+
+
+@pytest.mark.asyncio
+async def test_allowed_model_region_keeps_only_deployments_in_that_region(make_core: MakeCore) -> None:
+    core: Final = _regional(make_core)
+    assert await _picked(core, request_kwargs={"allowed_model_region": "eu"}, messages=SHORT) == {"eu"}
+
+
+@pytest.mark.asyncio
+async def test_allowed_model_region_is_ignored_without_enable_pre_call_checks(make_core: MakeCore) -> None:
+    core: Final = _regional(make_core, enable_pre_call_checks=False)
+    assert await _picked(core, request_kwargs={"allowed_model_region": "eu"}, messages=SHORT) == {"eu", "us", "none"}
+
+
+@pytest.mark.asyncio
+async def test_allowed_model_region_is_ignored_without_messages_or_input(make_core: MakeCore) -> None:
+    core: Final = _regional(make_core)
+    assert await _picked(core, request_kwargs={"allowed_model_region": "eu"}) == {"eu", "us", "none"}
+
+
+@pytest.mark.asyncio
+async def test_no_deployment_in_the_allowed_region_raises_router_rate_limit_error(make_core: MakeCore) -> None:
+    core: Final = _regional(make_core)
+    with pytest.raises(RouterRateLimitError) as error:
+        await core.select("g", {"allowed_model_region": "ap"}, messages=SHORT)
+    assert str(error.value) == (
+        "No deployments available for selected model, Try again in 5 seconds. Passed model=g. "
+        "pre-call-checks=True, cooldown_list=[]"
+    )
+
+
+@pytest.mark.asyncio
+async def test_no_deployment_in_the_allowed_region_never_calls_the_provider(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = _regional(make_core)
+    with pytest.raises(RouterRateLimitError):
+        await core.acompletion(model="g", messages=SHORT, allowed_model_region="ap")  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.calls == []
+
+
+@pytest.mark.asyncio
+async def test_async_completion_with_an_allowed_region_reaches_only_that_region(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = _regional(make_core)
+    for _ in range(6):
+        await core.acompletion(model="g", messages=SHORT, allowed_model_region="us")  # pyright: ignore[reportAttributeAccessIssue]
+    assert set(host.deployments_called()) == {"us"}
+
+
+# --- unsupported params ------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_with_drop_params_off_a_deployment_that_lacks_response_format_is_skipped(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    host.set_dynamic(drop_params=False)
+    assert await _picked(_json_mode_group(make_core), request_kwargs=dict(JSON_MODE), messages=SHORT) == {"new"}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("drop_params", [None, True])
+async def test_unless_drop_params_is_exactly_false_unsupported_params_are_not_checked(
+    make_core: MakeCore, host: ScriptedHost, drop_params: bool | None
+) -> None:
+    """Oddity: the unset process default (``None``) skips the check as ``drop_params=True`` does."""
+    host.set_dynamic(drop_params=drop_params)
+    assert await _picked(_json_mode_group(make_core), request_kwargs=dict(JSON_MODE), messages=SHORT) == {"old", "new"}
+
+
+@pytest.mark.asyncio
+async def test_unsupported_params_are_not_checked_without_enable_pre_call_checks(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    host.set_dynamic(drop_params=False)
+    core: Final = _json_mode_group(make_core, enable_pre_call_checks=False)
+    assert await _picked(core, request_kwargs=dict(JSON_MODE), messages=SHORT) == {"old", "new"}
+
+
+@pytest.mark.asyncio
+async def test_unsupported_params_are_not_checked_without_messages_or_input(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    host.set_dynamic(drop_params=False)
+    assert await _picked(_json_mode_group(make_core), request_kwargs=dict(JSON_MODE)) == {"old", "new"}
+
+
+@pytest.mark.asyncio
+async def test_a_string_response_format_is_not_checked(make_core: MakeCore, host: ScriptedHost) -> None:
+    host.set_dynamic(drop_params=False)
+    core: Final = _json_mode_group(make_core)
+    assert await _picked(core, request_kwargs={"response_format": "b64_json"}, messages=SHORT) == {"old", "new"}
+
+
+@pytest.mark.asyncio
+async def test_when_no_deployment_supports_the_param_select_raises_unsupported_params(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    host.set_dynamic(drop_params=False)
+    core: Final = make_core(model_list=[deployment("g", "old", model=NO_JSON_MODE_MODEL)], enable_pre_call_checks=True)
+    with pytest.raises(litellm.UnsupportedParamsError) as error:
+        await core.select("g", dict(JSON_MODE), messages=SHORT)
+    assert str(error.value) == UNSUPPORTED_MESSAGE
+    assert error.value.status_code == 400
+
+
+@pytest.mark.asyncio
+async def test_when_no_deployment_supports_the_param_async_completion_raises_without_calling_or_retrying(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    host.set_dynamic(drop_params=False)
+    core: Final = make_core(
+        model_list=[deployment("g", "old", model=NO_JSON_MODE_MODEL)], enable_pre_call_checks=True, num_retries=2
+    )
+    with pytest.raises(litellm.UnsupportedParamsError, match="allowed_openai_params"):
+        await core.acompletion(model="g", messages=SHORT, **JSON_MODE)  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.calls == []
+    assert host.sleeps == []
+
+
+def test_when_no_deployment_supports_the_param_sync_completion_raises_without_calling(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    host.set_dynamic(drop_params=False)
+    core: Final = make_core(model_list=[deployment("g", "old", model=NO_JSON_MODE_MODEL)], enable_pre_call_checks=True)
+    with pytest.raises(litellm.UnsupportedParamsError):
+        core.completion(model="g", messages=SHORT, **JSON_MODE)  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.calls == []
+
+
+REQUEST_ESCAPES: Final = [
+    {"allowed_openai_params": ["response_format"]},
+    {"drop_params": True},
+    {"additional_drop_params": ["response_format"]},
+]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("escape", REQUEST_ESCAPES, ids=["allowlist", "drop", "additional-drop"])
+async def test_a_request_that_allows_or_drops_the_param_keeps_the_only_deployment(
+    make_core: MakeCore, host: ScriptedHost, escape: dict[str, object]
+) -> None:
+    host.set_dynamic(drop_params=False)
+    core: Final = make_core(model_list=[deployment("g", "old", model=NO_JSON_MODE_MODEL)], enable_pre_call_checks=True)
+    assert await _picked(core, times=2, request_kwargs={**JSON_MODE, **escape}, messages=SHORT) == {"old"}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("escape", REQUEST_ESCAPES, ids=["allowlist", "drop", "additional-drop"])
+async def test_a_deployment_that_takes_the_param_is_preferred_over_dropping_it(
+    make_core: MakeCore, host: ScriptedHost, escape: dict[str, object]
+) -> None:
+    host.set_dynamic(drop_params=False)
+    core: Final = _json_mode_group(make_core)
+    assert await _picked(core, request_kwargs={**JSON_MODE, **escape}, messages=SHORT) == {"new"}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "deployment_escape",
+    [{"drop_params": True}, {"allowed_openai_params": ["response_format"]}],
+    ids=["drop", "allowlist"],
+)
+async def test_a_deployment_that_allows_or_drops_the_param_is_kept(
+    make_core: MakeCore, host: ScriptedHost, deployment_escape: dict[str, object]
+) -> None:
+    host.set_dynamic(drop_params=False)
+    core: Final = make_core(
+        model_list=[deployment("g", "old", model=NO_JSON_MODE_MODEL, **deployment_escape)], enable_pre_call_checks=True
+    )
+    assert await _picked(core, times=2, request_kwargs=dict(JSON_MODE), messages=SHORT) == {"old"}
+
+
+@pytest.mark.asyncio
+async def test_a_request_drop_params_false_overrides_the_deployment_drop_params(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    host.set_dynamic(drop_params=False)
+    core: Final = make_core(
+        model_list=[deployment("g", "old", model=NO_JSON_MODE_MODEL, drop_params=True)], enable_pre_call_checks=True
+    )
+    with pytest.raises(litellm.UnsupportedParamsError):
+        await core.select("g", {**JSON_MODE, "drop_params": False}, messages=SHORT)
+
+
+@pytest.mark.asyncio
+async def test_a_context_window_error_wins_over_an_unsupported_param(make_core: MakeCore, host: ScriptedHost) -> None:
+    host.set_dynamic(drop_params=False)
+    core: Final = make_core(
+        model_list=[deployment("g", "small-no-json", model=NO_JSON_MODE_MODEL, model_info={"max_input_tokens": 5})],
+        enable_pre_call_checks=True,
+    )
+    with pytest.raises(litellm.ContextWindowExceededError):
+        await core.select("g", dict(JSON_MODE), messages=LONG)
+    core.retire()
+
+
+@pytest.mark.asyncio
+async def test_a_capable_deployment_outside_the_region_gives_no_deployments_rather_than_a_400(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    host.set_dynamic(drop_params=False)
+    core: Final = make_core(
+        model_list=[
+            deployment("g", "old", model=NO_JSON_MODE_MODEL, region_name="eu"),
+            deployment("g", "new", model=JSON_MODE_MODEL, region_name="us"),
+        ],
+        enable_pre_call_checks=True,
+    )
+    with pytest.raises(RouterRateLimitError, match=r"^No deployments available for selected model"):
+        await core.select("g", {**JSON_MODE, "allowed_model_region": "eu"}, messages=SHORT)
+
+
+@pytest.mark.asyncio
+async def test_dropping_the_param_keeps_the_in_region_deployment_that_lacks_it(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    host.set_dynamic(drop_params=False)
+    core: Final = make_core(
+        model_list=[
+            deployment("g", "old", model=NO_JSON_MODE_MODEL, region_name="eu"),
+            deployment("g", "new", model=JSON_MODE_MODEL, region_name="us"),
+        ],
+        enable_pre_call_checks=True,
+    )
+    request: Final = {**JSON_MODE, "allowed_model_region": "eu", "drop_params": True}
+    assert await _picked(core, times=5, request_kwargs=request, messages=SHORT) == {"old"}
+
+
+@pytest.mark.asyncio
+async def test_a_capable_deployment_in_cooldown_gives_no_deployments_rather_than_a_400(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    host.set_dynamic(drop_params=False)
+    core: Final = _json_mode_group(make_core)
+    core.health().cooldowns.add_deployment_to_cooldown(
+        "new", original_exception=Exception("down"), exception_status=429, cooldown_time=60
+    )
+    with pytest.raises(RouterRateLimitError) as error:
+        await core.select("g", dict(JSON_MODE), messages=SHORT)
+    assert str(error.value).startswith(
+        "No deployments available for selected model, Try again in 60 seconds. Passed model=g. pre-call-checks=True"
+    )
+
+
+# --- images --------------------------------------------------------------------------------
+
+
+def _vision_group(make_core: MakeCore) -> RouterCore:
+    return make_core(
+        model_list=[
+            deployment("g", "text", model_info={"supports_vision": False}),
+            deployment("g", "vision", model_info={"supports_vision": True}),
+        ]
+    )
+
+
+@pytest.mark.asyncio
+async def test_an_image_request_skips_a_declared_text_only_deployment_even_without_pre_call_checks(
+    make_core: MakeCore,
+) -> None:
+    assert await _picked(_vision_group(make_core), messages=IMAGE) == {"vision"}
+
+
+@pytest.mark.asyncio
+async def test_an_image_in_the_request_messages_counts_as_an_image_request(make_core: MakeCore) -> None:
+    assert await _picked(_vision_group(make_core), request_kwargs={"messages": IMAGE}) == {"vision"}
+
+
+@pytest.mark.asyncio
+async def test_a_text_request_may_pick_a_text_only_deployment(make_core: MakeCore) -> None:
+    assert await _picked(_vision_group(make_core), messages=SHORT) == {"text", "vision"}
+
+
+@pytest.mark.asyncio
+async def test_an_image_request_keeps_every_deployment_when_all_are_text_only(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            deployment("g", "text", model_info={"supports_vision": False}),
+            deployment("g", "text2", model_info={"supports_vision": False}),
+        ]
+    )
+    assert await _picked(core, messages=IMAGE) == {"text", "text2"}
+
+
+@pytest.mark.asyncio
+async def test_async_completion_with_an_image_reaches_the_vision_deployment(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = _vision_group(make_core)
+    for _ in range(6):
+        await core.acompletion(model="g", messages=IMAGE)  # pyright: ignore[reportAttributeAccessIssue]
+    assert set(host.deployments_called()) == {"vision"}
+
+
+# --- budgets -------------------------------------------------------------------------------
+
+PROVIDER_BUDGET: Final = {"openai": {"budget_limit": 1, "time_period": "1d"}}
+
+
+@pytest.mark.asyncio
+async def test_a_provider_over_budget_leaves_no_deployment_and_select_raises_value_error(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], provider_budget_config=PROVIDER_BUDGET)
+    await core.health().store.async_set_cache("provider_spend:openai:1d", 5.0)
+    with pytest.raises(ValueError, match=r"^No deployments available - crossed budget: ") as error:
+        await core.select("g")
+    assert (
+        str(error.value)
+        == "No deployments available - crossed budget: Exceeded budget for provider openai: 5.0 >= 1.0\n"
+    )
+    core.retire()
+
+
+@pytest.mark.asyncio
+async def test_a_provider_over_budget_raises_from_async_completion_without_calling_the_provider(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], provider_budget_config=PROVIDER_BUDGET)
+    await core.health().store.async_set_cache("provider_spend:openai:1d", 5.0)
+    with pytest.raises(ValueError, match=r"^No deployments available - crossed budget"):
+        await core.acompletion(model="g", messages=SHORT)  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.calls == []
+    core.retire()
+
+
+@pytest.mark.asyncio
+async def test_a_provider_under_budget_is_picked(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], provider_budget_config=PROVIDER_BUDGET)
+    await core.health().store.async_set_cache("provider_spend:openai:1d", 0.5)
+    assert await _picked(core, times=2) == {"a"}
+    core.retire()
+
+
+@pytest.mark.asyncio
+async def test_a_deployment_over_its_own_budget_is_skipped(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a", max_budget=1, budget_duration="1d"), deployment("g", "b")])
+    await core.health().store.async_set_cache("deployment_spend:a:1d", 5.0)
+    assert await _picked(core, times=10) == {"b"}
+    core.retire()

tests/unit/test_router/core/test_deprecated_members.py

@@ -0,0 +1,665 @@
+"""
+The deprecated Router members (``DeprecatedRouterMembers``, litellm/router_deprecated.py) on a Router
+over each core: every old attribute reads what the Python core's ``_core_<name>`` holds, through the
+entry point named in its DeprecationWarning, and the old writes have the Python core's effect.
+"""
+
+from collections.abc import Callable
+from datetime import datetime
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.caching.dual_cache import DualCache
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.router import Router
+from litellm.router_api import SETTING_FIELDS, ChangeSet
+from litellm.router_core import RouterCore
+from litellm.router_utils.cooldown_cache import CooldownCache
+from litellm.router_utils.kubernetes_pod_discovery import KubernetesPodDiscovery
+from litellm.types.router import AllowedFailsPolicy, Deployment, RouterGeneralSettings, RoutingContext
+from tests.unit.test_router.core.registry_helpers import MESSAGES, MakeCore, ids, team_deployment
+from tests.unit.test_router.core.scripted_host import deployment
+
+
+def as_router(core: RouterCore) -> Router:
+    """A ``litellm.Router`` facade over ``core``, as ``Router.from_config`` attaches the core it builds."""
+    router: Final = Router.__new__(Router)
+    router._attach(core)  # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue, reportUnknownMemberType]  # the facade's own attach, as from_config calls it
+    return router
+
+
+def picked_id(picked: object) -> object:
+    """The ``model_info.id`` of a picked deployment dict."""
+    return (as_str_mapping((as_str_mapping(picked) or {}).get("model_info")) or {}).get("id")
+
+
+def read(router: Router, name: str) -> object:
+    with pytest.warns(DeprecationWarning, match=rf"Router\.{name} is deprecated"):
+        return getattr(router, name)  # pyright: ignore[reportAny]  # each old member's own type
+
+
+def write(router: Router, name: str, value: object) -> None:
+    with pytest.warns(DeprecationWarning, match=rf"Router\.{name} is deprecated"):
+        setattr(router, name, value)
+
+
+_CONFIGURED: Final[dict[str, object]] = {  # mutable-ok: constructor arguments, copied per router
+    "model_list": [
+        deployment("gpt", "a"),
+        deployment("gpt", "b"),
+        deployment("claude", "c", model="anthropic/claude-x"),
+        team_deployment("int-t", "t1", "TA", "pub", model="openai/team-model"),
+        deployment("openai/*", "w", model="openai/*"),
+    ],
+    "fallbacks": [{"gpt": ["claude"]}],
+    "context_window_fallbacks": [{"gpt": ["claude"]}],
+    "content_policy_fallbacks": [{"claude": ["gpt"]}],
+    "cooldown_time": 7,
+    "allowed_fails": 2,
+    "allowed_fails_policy": AllowedFailsPolicy(RateLimitErrorAllowedFails=4),
+    "disable_cooldowns": True,
+    "enable_pre_call_checks": True,
+    "enable_tag_filtering": True,
+    "tag_filtering_match_any": False,
+    "tag_routing_prefix": "x-",
+    "default_litellm_params": {"metadata": {"k": "v"}},
+    "routing_strategy": "least-busy",
+    "default_max_parallel_requests": 5,
+    "guardrail_list": [{"guardrail_name": "g", "litellm_params": {"guardrail": "x"}}],
+    "search_tools": [{"search_tool_name": "s", "litellm_params": {"search_provider": "tavily"}}],
+    "health_check_ignore_transient_errors": True,
+    "enable_health_check_routing": True,
+    "background_health_check_model_groups": ["gpt"],
+    "model_group_alias": {"fast": "gpt"},
+    "cache_responses": True,
+}
+
+# The Python core's values for _CONFIGURED, by setting (the objects it builds are compared by type).
+_CONFIGURED_SETTINGS: Final[dict[str, object]] = {  # mutable-ok: expected values
+    "routing_strategy": "least-busy",
+    "fallbacks": [{"gpt": ["claude"]}],
+    "context_window_fallbacks": [{"gpt": ["claude"]}],
+    "content_policy_fallbacks": [{"claude": ["gpt"]}],
+    "default_deployment": None,
+    "default_litellm_params": {"timeout": None, "max_retries": 0, "metadata": {"k": "v", "caching_groups": None}},
+    "default_max_parallel_requests": 5,
+    "cooldown_time": 7,
+    "allowed_fails": 2,
+    "allowed_fails_policy": AllowedFailsPolicy(RateLimitErrorAllowedFails=4),
+    "disable_cooldowns": True,
+    "enable_pre_call_checks": True,
+    "enable_health_check_routing": True,
+    "health_check_ignore_transient_errors": True,
+    "background_health_check_model_groups": frozenset({"gpt"}),
+    "enable_tag_filtering": True,
+    "tag_filtering_match_any": False,
+    "tag_routing_prefix": "x-",
+    "router_general_settings": RouterGeneralSettings(),
+    "provider_budget_config": None,
+    "provider_default_deployment_ids": ["w"],
+    "fallback_access_check": None,
+    "fallback_budget_check": None,
+    "search_tools": [{"search_tool_name": "s", "litellm_params": {"search_provider": "tavily"}}],
+    "guardrail_list": [{"guardrail_name": "g", "litellm_params": {"guardrail": "x"}}],
+    "cache_responses": True,
+}
+
+
+def _configured(make_core: MakeCore) -> Router:
+    return as_router(make_core(**{name: _copy(value) for name, value in _CONFIGURED.items()}))
+
+
+def _copy(value: object) -> object:
+    import copy
+
+    return copy.deepcopy(value)
+
+
+# -- settings ------------------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("name", sorted(set(SETTING_FIELDS) - {"kubernetes_pod_discovery"}))
+def test_a_setting_attribute_reads_the_configured_setting(make_core: MakeCore, name: str) -> None:
+    router: Final = _configured(make_core)
+    value: Final = read(router, name)
+    assert value == _CONFIGURED_SETTINGS[name]
+    assert value is getattr(router.settings(), name)
+
+
+def test_the_pod_discovery_setting_is_the_routers_pod_lookup(make_core: MakeCore) -> None:
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
+    discovery: Final = read(router, "kubernetes_pod_discovery")
+    assert isinstance(discovery, KubernetesPodDiscovery)
+    assert discovery is router.settings().kubernetes_pod_discovery
+
+
+def test_default_settings_read_as_the_python_cores_defaults(make_core: MakeCore) -> None:
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
+    assert read(router, "routing_strategy") == "simple-shuffle"
+    assert read(router, "cooldown_time") == 5
+    assert read(router, "allowed_fails") == 3
+    assert read(router, "fallbacks") is None
+    assert read(router, "default_litellm_params") == {
+        "timeout": None,
+        "max_retries": 0,
+        "metadata": {"caching_groups": None},
+    }
+    assert read(router, "tag_filtering_match_any") is True
+    assert read(router, "tag_routing_prefix") == ""
+    assert read(router, "cache_responses") is False
+    assert read(router, "search_tools") == []
+    assert read(router, "guardrail_list") == []
+
+
+def test_a_setting_attribute_write_is_a_changeset(make_core: MakeCore) -> None:
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
+    write(router, "cooldown_time", 9)
+    write(router, "fallbacks", [{"gpt": ["gpt"]}])
+    assert read(router, "cooldown_time") == 9
+    assert router.settings().cooldown_time == 9
+    assert read(router, "fallbacks") == [{"gpt": ["gpt"]}]
+
+
+def test_a_deleted_setting_attribute_no_longer_reads(make_core: MakeCore) -> None:
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
+    with pytest.warns(DeprecationWarning, match=r"Router\.fallbacks is deprecated"):
+        del router.fallbacks
+    assert not router.settings().is_set("fallbacks")
+    with pytest.raises(AttributeError, match="'fallbacks'"):
+        read(router, "fallbacks")
+
+
+# -- registry state ------------------------------------------------------------------------------
+
+_REGISTRY_READS: Final[dict[str, Callable[[Router], object]]] = {
+    "model_list": lambda r: r.registry().all(),
+    "model_names": lambda r: r.registry().names(kind="group_set"),
+    "deployment_names": lambda r: r.registry().names(kind="deployment_names"),
+    "team_public_model_names": lambda r: r.registry().names(kind="team_public"),
+    "model_name_to_deployment_indices": lambda r: r.registry().index("model_name"),
+    "team_model_to_deployment_indices": lambda r: r.registry().index("team_model"),
+    "cached_deployment_model_info": lambda r: r.registry().deployment_model_info_cache(),
+    "model_group_alias": lambda r: r.registry().aliases(),
+    "pattern_router": lambda r: r.registry().patterns(),
+    "team_pattern_routers": lambda r: r.registry().patterns(team=True),
+    "routing_groups": lambda r: r.registry().routing_groups(),
+    "auto_routers": lambda r: r.registry().plugins("auto"),
+    "complexity_routers": lambda r: r.registry().plugins("complexity"),
+    "quality_routers": lambda r: r.registry().plugins("quality"),
+    "adaptive_routers": lambda r: r.registry().plugins("adaptive"),
+    "routing_plugins": lambda r: r.registry().plugins("routing"),
+    "prompt_caching_pin_cache": lambda r: r.registry().plugins("prompt_caching_pins"),
+    "cache": lambda r: r.health().store,
+    "cooldown_cache": lambda r: r.health().cooldowns,
+    "health_state_cache": lambda r: r.health().health_checks,
+    "deployment_latency_map": lambda r: r.health().latency,
+}
+
+
+@pytest.mark.parametrize("name", sorted(_REGISTRY_READS))
+def test_a_state_attribute_is_the_live_object_its_entry_point_returns(make_core: MakeCore, name: str) -> None:
+    router: Final = _configured(make_core)
+    assert read(router, name) is _REGISTRY_READS[name](router)
+
+
+def test_the_registry_index_attributes_hold_the_python_cores_values(make_core: MakeCore) -> None:
+    router: Final = _configured(make_core)
+    assert read(router, "model_names") == {"gpt", "claude", "int-t", "openai/*"}
+    assert read(router, "deployment_names") == [
+        "openai/a",
+        "openai/b",
+        "anthropic/claude-x",
+        "openai/team-model",
+        "openai/*",
+    ]
+    assert read(router, "team_public_model_names") == frozenset({"pub"})
+    assert read(router, "model_name_to_deployment_indices") == {
+        "gpt": [0, 1],
+        "claude": [2],
+        "int-t": [3],
+        "openai/*": [4],
+    }
+    assert read(router, "team_model_to_deployment_indices") == {("TA", "pub"): [3]}
+    assert read(router, "model_group_alias") == {"fast": "gpt"}
+    assert read(router, "routing_groups") == {}
+    with pytest.warns(DeprecationWarning, match=r"Router\.routing_groups is deprecated"):
+        assert router._routing_groups == {}  # pyright: ignore[reportPrivateUsage]  # the old private alias
+    assert read(router, "routing_plugins") == []
+    assert read(router, "deployment_latency_map") == {
+        "openai/a": 0,
+        "openai/b": 0,
+        "anthropic/claude-x": 0,
+        "openai/team-model": 0,
+        "openai/*": 0,
+    }
+    assert ids(read(router, "model_list")) == ["a", "b", "c", "t1", "w"]  # pyright: ignore[reportArgumentType]  # the live model list
+
+
+def test_the_cached_model_info_attribute_is_the_lru_cache(make_core: MakeCore) -> None:
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
+    with pytest.warns(DeprecationWarning, match=r"Router\.cached_deployment_model_info is deprecated"):
+        cached: Final = router.cached_deployment_model_info
+    first: Final = cached("a", "gpt")
+    assert cached("a", "gpt") is first
+    assert cached.cache_info().hits == 1
+    assert first == router.registry().deployment_model_info("a", "gpt")
+
+
+def test_a_configured_routing_plugin_is_listed(make_core: MakeCore) -> None:
+    class Keep:
+        async def run(self, context: RoutingContext) -> RoutingContext:
+            return context
+
+    plugin: Final = Keep()
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a")], plugins=[plugin]))
+    assert read(router, "routing_plugins") == [plugin]
+
+
+def test_a_routing_plugin_appended_to_the_live_list_runs(make_core: MakeCore) -> None:
+    class Narrow:
+        async def run(self, context: RoutingContext) -> RoutingContext:
+            context.candidate_models = [m for m in context.candidate_models if m == "openai/b"]
+            return context
+
+    class Keep:
+        async def run(self, context: RoutingContext) -> RoutingContext:
+            return context
+
+    router: Final = as_router(make_core(model_list=[deployment("g", "a"), deployment("g", "b")], plugins=[Keep()]))
+    with pytest.warns(DeprecationWarning, match=r"Router\.routing_plugins is deprecated"):
+        plugins: Final = router.routing_plugins
+    plugins.append(Narrow())
+
+    import asyncio
+
+    for _ in range(4):
+        picked = asyncio.run(router.select("g", {}, messages=MESSAGES))  # pyright: ignore[reportCallIssue]
+        assert picked_id(picked) == "b"
+
+
+# -- state writes --------------------------------------------------------------------------------
+
+
+def test_a_registry_state_write_is_what_the_registry_reads(make_core: MakeCore) -> None:
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
+    write(router, "model_names", {"x"})
+    write(router, "deployment_names", ["openai/x"])
+    write(router, "model_name_to_deployment_indices", {"x": [0]})
+    write(router, "auto_routers", {})
+    assert read(router, "model_names") == {"x"}
+    assert router.registry().names(kind="group_set") == {"x"}
+    assert router.registry().names(kind="deployment_names") == ["openai/x"]
+    assert router.registry().index("model_name") == {"x": [0]}
+    assert router.registry().plugins("auto") == {}
+
+
+def test_a_routing_state_write_is_what_health_reads(make_core: MakeCore) -> None:
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
+    store: Final = DualCache()
+    cooldowns: Final = CooldownCache(cache=store, default_cooldown_time=1)
+    latency: Final[dict[str, float]] = {"openai/a": 0.5}
+    write(router, "cache", store)
+    write(router, "cooldown_cache", cooldowns)
+    write(router, "deployment_latency_map", latency)
+    assert read(router, "cache") is store
+    assert router.health().store is store
+    assert router.health().cooldowns is cooldowns
+    assert router.health().latency is latency
+
+
+def test_an_assigned_slack_alerting_logger_is_the_routers_observer(make_core: MakeCore) -> None:
+    from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
+
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
+    with (
+        pytest.raises(AttributeError, match="slack_alerting_logger"),
+        pytest.warns(DeprecationWarning, match=r"Router\.slack_alerting_logger is deprecated"),
+    ):
+        _ = router.slack_alerting_logger
+    logger: Final = SlackAlerting(alerting=["slack"], default_webhook_url="https://example.invalid/hook")
+    write(router, "slack_alerting_logger", logger)
+    assert read(router, "slack_alerting_logger") is logger
+    assert router.observe().slack_alerting_logger is logger
+
+
+@pytest.mark.asyncio
+async def test_an_assigned_async_pick_answers_select(make_core: MakeCore) -> None:
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a"), deployment("gpt", "b")]))
+    calls: Final[list[str]] = []  # mutable-ok: the models the pick was asked for
+
+    async def pick(
+        model: str,
+        request_kwargs: dict[str, object],  # mutable-ok: the old pick's signature
+        messages: object = None,
+        input: object = None,
+        specific_deployment: object = False,
+    ) -> dict[str, object]:  # mutable-ok: a deployment dict
+        calls.append(model)
+        return deployment("gpt", "b")
+
+    write(router, "async_get_available_deployment", pick)
+    assert read(router, "async_get_available_deployment") is pick
+    picked: Final = await router.select("gpt", {})
+    assert picked_id(picked) == "b"
+    assert calls == ["gpt"]
+
+
+@pytest.mark.asyncio
+async def test_the_async_pick_attribute_picks_like_select(make_core: MakeCore) -> None:
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
+    with pytest.warns(DeprecationWarning, match=r"Router\.async_get_available_deployment is deprecated"):
+        pick: Final = router.async_get_available_deployment
+    picked: Final = await pick(model="gpt", request_kwargs={})
+    assert picked_id(picked) == "a"
+
+
+# -- methods -------------------------------------------------------------------------------------
+
+
+def _call(router: Router, name: str, *args: object, **kwargs: object) -> object:  # kwargs-ok: the old member's
+    with pytest.warns(DeprecationWarning, match=rf"Router\.{name} is deprecated"):
+        return getattr(router, name)(*args, **kwargs)  # pyright: ignore[reportAny]  # each old member's own type
+
+
+def test_registry_methods_answer_through_the_registry(make_core: MakeCore) -> None:
+    router: Final = _configured(make_core)
+    registry: Final = router.registry()
+    found: Final = _call(router, "get_deployment", "a")
+    assert isinstance(found, Deployment)
+    assert found.model_info.id == "a"
+    assert _call(router, "has_model_id", "a") is True
+    assert _call(router, "get_model_info", "zzz") is None
+    assert ids(_call(router, "get_model_list", "gpt")) == ["a", "b"]  # pyright: ignore[reportArgumentType]
+    assert ids(_call(router, "get_all_deployments", "gpt")) == ["a", "b"]  # pyright: ignore[reportArgumentType]
+    assert ids(_call(router, "get_model_list_from_model_alias", "fast")) == ["a", "b"]  # pyright: ignore[reportArgumentType]
+    assert _call(router, "get_model_names") == registry.names()
+    assert _call(router, "get_model_ids", "gpt") == ["a", "b"]
+    assert _call(router, "get_model_from_alias", "fast") == "gpt"
+    assert _call(router, "routable_model_group", "fast") == "gpt"
+    assert _call(router, "is_recognized_model", "claude") is True
+    assert _call(router, "get_model_group", "a") == registry.resolve("a", kind="group_of_id")
+    assert _call(router, "map_team_model", "pub", "TA") == "pub"
+    assert _call(router, "get_model_access_groups") == registry.access_groups()
+    assert _call(router, "get_deployment_credentials", "a") == {"api_key": "sk-test"}
+    assert _call(router, "get_router_deployment_budget_limiter") is None
+    assert _call(router, "should_include_deployment", "gpt", deployment("gpt", "a")) is True
+    assert _call(router, "get_router_model_info", None, "gpt", id="a") == registry.router_model_info(
+        None, "gpt", id="a"
+    )
+
+
+def test_settings_methods_answer_through_settings(make_core: MakeCore) -> None:
+    router: Final = _configured(make_core)
+    assert _call(router, "get_settings") == dict(router.settings().as_dict())
+    assert _call(router, "get_available_guardrail", "g") == {
+        "guardrail_name": "g",
+        "litellm_params": {"guardrail": "x"},
+    }
+    with (
+        pytest.raises(ValueError, match="No guardrail found with name: nope"),
+        pytest.warns(DeprecationWarning, match=r"Router\.get_available_guardrail is deprecated"),
+    ):
+        router.get_available_guardrail("nope")
+
+
+@pytest.mark.asyncio
+async def test_health_and_selection_methods_answer_through_their_entry_points(make_core: MakeCore) -> None:
+    router: Final = as_router(
+        make_core(model_list=[deployment("gpt", "a"), deployment("pt", "p", use_in_pass_through=True)])
+    )
+    assert _call(router, "get_fully_blocked_model_names") == set()
+    with pytest.warns(DeprecationWarning, match=r"Router\.async_get_fully_unhealthy_model_names is deprecated"):
+        unhealthy: Final = await router.async_get_fully_unhealthy_model_names()
+    assert unhealthy == set()
+    with pytest.warns(DeprecationWarning, match=r"Router\.get_model_group_usage is deprecated"):
+        usage: Final = await router.get_model_group_usage("gpt")
+    assert usage == (None, None)
+    with pytest.warns(DeprecationWarning, match=r"Router\.get_remaining_model_group_usage is deprecated"):
+        remaining: Final = await router.get_remaining_model_group_usage("gpt")
+    assert remaining == {}
+    with pytest.warns(DeprecationWarning, match=r"Router\.async_get_healthy_deployments is deprecated"):
+        healthy = await router.async_get_healthy_deployments("gpt", {})  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # the old untyped dicts
+    assert isinstance(healthy, list)
+    assert [picked_id(entry) for entry in healthy] == ["a"]  # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType]  # see above
+    assert picked_id(_call(router, "get_available_deployment_for_pass_through", "pt")) == "p"
+
+
+def test_write_methods_go_through_apply(make_core: MakeCore) -> None:
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
+    _call(router, "update_settings", allowed_fails=6)
+    assert router.settings().allowed_fails == 6
+    router.apply(ChangeSet())
+    assert _call(router, "delete_deployment", "a") is not None
+    assert router.registry().all() == []
+
+
+def test_the_priority_split_answers_through_health(make_core: MakeCore) -> None:
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a")], default_priority=2))
+    assert _call(router, "split_request_priority", "gpt", {"priority": 3, "x": 1}) == (3, {"x": 1})
+    assert _call(router, "split_request_priority", "gpt", {"x": 1}) == (2, {"x": 1})
+    assert router.health().priority_header_fields("gpt", {}, "5") == {"priority": 5}
+    assert router.health().priority_header_fields("gpt", {}, None) == {}
+    with pytest.raises(litellm.BadRequestError, match="priority must be an integer, got str 'bad'"):
+        router.health().split_priority("gpt", {"priority": "bad"})
+
+
+# -- plugin context, request logging -------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_the_pre_routing_hook_answers_through_the_routing_context(make_core: MakeCore) -> None:
+    router: Final = as_router(
+        make_core(model_list=[deployment("small", "s"), deployment("big", "b"), _complexity_marker()])
+    )
+    request: Final[dict[str, object]] = {}
+    with pytest.warns(DeprecationWarning, match=r"Router\.async_pre_routing_hook is deprecated"):
+        response = await router.async_pre_routing_hook("smart", request, messages=MESSAGES)  # pyright: ignore[reportUnknownMemberType]  # the old signature's bare ``list``
+    assert response is not None
+    assert response.model == "small"
+    decision: Final = as_str_mapping((as_str_mapping(request.get("metadata")) or {}).get("routing_decision")) or {}
+    assert decision.get("router_model_name") == "smart"
+    assert decision.get("routed_model") == "small"
+    plain: Final[dict[str, object]] = {}
+    with pytest.warns(DeprecationWarning, match="async_pre_routing_hook"):
+        hooked: Final = await router.async_pre_routing_hook("small", plain)  # pyright: ignore[reportUnknownMemberType]  # see above
+    assert hooked is None
+    assert plain == {}
+
+
+def _complexity_marker() -> dict[str, object]:  # mutable-ok: a model_list entry
+    return {
+        "model_name": "smart",
+        "litellm_params": {
+            "model": "auto_router/complexity_router",
+            "complexity_router_config": {"tiers": {"SIMPLE": "small", "MEDIUM": "big", "COMPLEX": "big"}},
+            "complexity_router_default_model": "big",
+        },
+        "model_info": {"id": "marker"},
+    }
+
+
+def test_log_retry_records_the_failure_on_the_request_metadata(make_core: MakeCore) -> None:
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a", base_model="bm")]))
+    kwargs: Final[dict[str, object]] = {
+        "model": "gpt",
+        "metadata": {"model_info": {"id": "a"}, "deployment": "openai/a", "attempted_retries": 1},
+    }
+    assert _call(router, "log_retry", kwargs, ValueError("boom")) is kwargs
+    assert kwargs["metadata"] == {
+        "model_info": {"id": "a"},
+        "deployment": "openai/a",
+        "attempted_retries": 1,
+        "previous_models": (
+            {
+                "model_group": "gpt",
+                "deployment_id": "a",
+                "model": "openai/a",
+                "base_model": "bm",
+                "exception_type": "ValueError",
+                "exception_string": "boom",
+                "attempted_retries": 1,
+            },
+        ),
+        "request_retry_count": 1,
+    }
+    _call(router, "log_retry", kwargs, KeyError("again"))
+    metadata: Final = as_str_mapping(kwargs["metadata"]) or {}
+    assert metadata["request_retry_count"] == 2
+    previous: Final = metadata["previous_models"]
+    assert isinstance(previous, tuple)
+    assert len(previous) == 2  # pyright: ignore[reportUnknownArgumentType]  # an untyped tuple of breadcrumbs
+
+
+def test_arming_the_read_prefetch_without_a_request_batch_does_nothing(make_core: MakeCore) -> None:
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a")], routing_strategy="usage-based-routing-v2"))
+    assert _call(router, "arm_routing_read_prefetch", "gpt", {}) is None
+    assert _call(router, "arm_routing_read_prefetch", "unknown-model") is None
+
+
+def test_binding_to_request_logging_binds_through_observe(make_core: MakeCore) -> None:
+    from litellm.litellm_core_utils.litellm_logging import Logging
+
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
+    logging_obj: Final = Logging(
+        model="gpt",
+        messages=MESSAGES,
+        stream=False,
+        call_type="acompletion",
+        start_time=datetime.now(),
+        litellm_call_id="call-1",
+        function_id="fn-1",
+    )
+    assert _call(router, "bind_to_request_logging", logging_obj) is None
+    with (
+        pytest.raises(AttributeError, match="add_dynamic_function_callbacks"),
+        pytest.warns(DeprecationWarning, match=r"Router\.bind_to_request_logging is deprecated"),
+    ):
+        router.bind_to_request_logging(None)  # pyright: ignore[reportArgumentType]
+
+
+# -- writes with a routing effect ----------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_assigned_routing_groups_route_from_the_next_call(make_core: MakeCore) -> None:
+    from litellm.types.router import RoutingGroup
+
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a"), deployment("claude", "c")]))
+    with pytest.raises(Exception, match="grp"):
+        await router.select("grp", {})
+    write(
+        router,
+        "routing_groups",
+        {"grp": RoutingGroup(group_name="grp", models=["claude"], routing_strategy="simple-shuffle")},
+    )
+    assert picked_id(await router.select("grp", {})) == "c"
+    assert router.registry().routing_groups().keys() == {"grp"}
+
+
+class _NarrowTo:
+    def __init__(self, keep: str) -> None:
+        self.keep: Final = keep
+
+    async def run(self, context: RoutingContext) -> RoutingContext:
+        context.candidate_models = [m for m in context.candidate_models if m == self.keep]
+        return context
+
+
+@pytest.mark.asyncio
+async def test_a_plugin_added_to_an_empty_plugin_list_runs(make_core: MakeCore) -> None:
+    router: Final = as_router(make_core(model_list=[deployment("g", "a"), deployment("g", "b")]))
+    with pytest.warns(DeprecationWarning, match=r"Router\.routing_plugins is deprecated"):
+        plugins: Final = router.routing_plugins
+    plugins.append(_NarrowTo("openai/b"))
+    for _ in range(4):
+        assert picked_id(await router.select("g", {}, messages=MESSAGES)) == "b"  # pyright: ignore[reportCallIssue]
+    plugins.clear()
+    assert {picked_id(await router.select("g", {}, messages=MESSAGES)) for _ in range(40)} == {"a", "b"}  # pyright: ignore[reportCallIssue]
+
+
+@pytest.mark.asyncio
+async def test_an_assigned_plugin_list_runs(make_core: MakeCore) -> None:
+    router: Final = as_router(make_core(model_list=[deployment("g", "a"), deployment("g", "b")]))
+    write(router, "routing_plugins", [_NarrowTo("openai/a")])
+    for _ in range(4):
+        assert picked_id(await router.select("g", {}, messages=MESSAGES)) == "a"  # pyright: ignore[reportCallIssue]
+
+
+def test_an_assigned_prompt_caching_pin_cache_is_the_one_flushed(make_core: MakeCore) -> None:
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
+    pins: Final = DualCache()
+    write(router, "prompt_caching_pin_cache", pins)
+    assert read(router, "prompt_caching_pin_cache") is pins
+    pins.in_memory_cache.set_cache("k", "v")  # pyright: ignore[reportUnknownMemberType]  # the in-memory cache is untyped
+    router.flush_cache()  # pyright: ignore[reportAttributeAccessIssue]
+    assert pins.in_memory_cache.get_cache("k") is None  # pyright: ignore[reportUnknownMemberType]  # see above
+
+
+# -- pod resolution and cooldown_time --------------------------------------------------------------
+
+
+class _TaggingDiscovery:
+    """Stands in for KubernetesPodDiscovery: stamps which discovery resolved the pick."""
+
+    def __init__(self, name: str) -> None:
+        self.name: Final = name
+
+    def resolve_deployment(self, deployment: object, request_kwargs: object) -> object:
+        return {**(as_str_mapping(deployment) or {}), "resolved_by": self.name}
+
+    async def async_resolve_deployment(self, deployment: object, request_kwargs: object) -> object:
+        return self.resolve_deployment(deployment, request_kwargs)
+
+
+@pytest.mark.asyncio
+async def test_a_custom_strategy_resolves_pods_through_the_discovery_bound_when_registered(
+    make_core: MakeCore,
+) -> None:
+    from litellm.types.router import CustomRoutingStrategyBase
+
+    class Pick(CustomRoutingStrategyBase):
+        async def async_get_available_deployment(  # pyright: ignore[reportIncompatibleMethodOverride]
+            self,
+            model: str,
+            messages: object = None,
+            input: object = None,
+            specific_deployment: object = False,
+            request_kwargs: object = None,
+        ) -> object:
+            return deployment("gpt", "b")
+
+        def get_available_deployment(  # pyright: ignore[reportIncompatibleMethodOverride]
+            self,
+            model: str,
+            messages: object = None,
+            input: object = None,
+            specific_deployment: object = False,
+            request_kwargs: object = None,
+        ) -> object:
+            return deployment("gpt", "b")
+
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a"), deployment("gpt", "b")]))
+    router.apply(ChangeSet(assign={"kubernetes_pod_discovery": _TaggingDiscovery("first")}))
+    assert (as_str_mapping(await router.select("gpt", {})) or {}).get("resolved_by") == "first"
+    router.register_plugin(Pick())
+    router.apply(ChangeSet(assign={"kubernetes_pod_discovery": _TaggingDiscovery("second")}))
+    picked: Final = as_str_mapping(await router.select("gpt", {})) or {}
+    assert picked_id(picked) == "b"
+    assert picked.get("resolved_by") == "first"
+
+
+def test_cooldown_time_reads_back_as_given(make_core: MakeCore) -> None:
+    router: Final = as_router(make_core(model_list=[deployment("gpt", "a")], cooldown_time=7))
+    value: Final = read(router, "cooldown_time")
+    assert value == 7 and type(value) is int
+    assert router.settings().as_dict()["cooldown_time"] == 7
+    assert type(router.settings().as_dict()["cooldown_time"]) is int
+    write(router, "cooldown_time", 2.5)
+    assert router.settings().cooldown_time == 2.5
+    _call(router, "update_settings", cooldown_time="3")
+    updated: Final = router.settings().cooldown_time
+    assert updated == 3 and type(updated) is int
+    default: Final = as_router(make_core(model_list=[deployment("gpt", "a")])).settings().cooldown_time
+    assert default == 5 and type(default) is int

tests/unit/test_router/core/test_endpoints.py

@@ -22,6 +22,7 @@ from tests.unit.test_router.core.scripted_host import (
     completion_response,
     deployment,
     embedding_response,
+    records_outcomes_through_callbacks,
 )
 
 MESSAGES: Final = [{"role": "user", "content": "hi"}]
@@ -45,11 +46,16 @@ ENDPOINTS: Final[dict[str, tuple[str, EndpointCall]]] = {
         "anthropic_messages",
         lambda core: core.aanthropic_messages(model="g", messages=MESSAGES, max_tokens=5),
     ),
+    # The proxy's /v1/messages route calls the router by this name (route_request).
+    "anthropic_messages": (
+        "anthropic_messages",
+        lambda core: core.anthropic_messages(model="g", messages=MESSAGES, max_tokens=5),
+    ),
     "call-acompletion": ("acompletion", lambda core: core.call("acompletion", {"model": "g", "messages": MESSAGES})),
     "call-completion": ("completion", lambda core: core.call("completion", {"model": "g", "messages": MESSAGES})),
 }
 SYNC_ENDPOINTS: Final = frozenset({"completion", "embedding", "image_generation"})
-LITELLM_METADATA_ENDPOINTS: Final = frozenset({"aresponses", "aanthropic_messages"})
+LITELLM_METADATA_ENDPOINTS: Final = frozenset({"aresponses", "aanthropic_messages", "anthropic_messages"})
 EMBEDDING_ENDPOINTS: Final = frozenset({"aembedding", "embedding"})
 
 
@@ -276,14 +282,15 @@ async def test_request_level_api_key_replaces_the_deployment_key_on_the_host_cal
 async def test_construction_registers_one_success_and_one_failure_callback_per_mode(
     make_core: MakeCore, host: ScriptedHost
 ):
-    _one_group(make_core)
+    core: Final = _one_group(make_core)
     registered: Final = {name: len(callbacks) for name, callbacks in host.lists.items()}
+    one: Final = 1 if records_outcomes_through_callbacks(core) else 0
     assert registered == {
         "callbacks": 0,
-        "success_callback": 1,
-        "_async_success_callback": 1,
-        "failure_callback": 1,
-        "_async_failure_callback": 1,
+        "success_callback": one,
+        "_async_success_callback": one,
+        "failure_callback": one,
+        "_async_failure_callback": one,
         "input_callback": 0,
     }
     assert all(callable(callbacks[0]) for callbacks in host.lists.values() if callbacks)
@@ -416,7 +423,6 @@ def _scripted_stream(*texts: str) -> Callable[..., CustomStreamWrapper]:
                 yield ModelResponseStream(choices=[{"index": 0, "delta": {"role": "assistant", "content": text}}])
             # A stream that ends without a finish reason fails since #47665.
             yield ModelResponseStream(choices=[{"index": 0, "delta": {}, "finish_reason": "stop"}])
-            yield ModelResponseStream(choices=[{"index": 0, "delta": {}, "finish_reason": "stop"}])
 
         logging_obj: Final = LiteLLMLogging(
             model="d1",

tests/unit/test_router/core/test_endpoints_characterization_batch.py

@@ -0,0 +1,516 @@
+"""
+Characterization of the fan-out endpoints and request shapers: the files API (one file per
+deployment), batches (create, retrieve across deployments, cancel, list), batch completions,
+fastest-response completion, the scheduler (priority) and prompt-management model groups.
+"""
+
+import re
+import datetime
+import io
+import json
+from collections.abc import Callable, Mapping
+from types import SimpleNamespace
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
+from litellm.router_core import RouterCore
+from litellm.types.utils import LiteLLMBatch, ModelResponse
+from tests.unit.test_router.core.endpoints_helpers import EndpointHost
+from tests.unit.test_router.core.scripted_host import ScriptedHost, bad_request, deployment
+
+MESSAGES: Final = [{"role": "user", "content": "hi"}]
+MakeCore = Callable[..., RouterCore]
+TWO_DEPLOYMENTS: Final = [deployment("g", "d1"), deployment("g", "d2")]
+
+
+@pytest.fixture
+def host() -> EndpointHost:
+    return EndpointHost()
+
+
+def _bucket(kwargs: Mapping[str, object], name: str) -> Mapping[str, object]:
+    bucket: Final = kwargs[name]
+    assert isinstance(bucket, Mapping)
+    return bucket
+
+
+def _batch(batch_id: str = "b1") -> LiteLLMBatch:
+    return LiteLLMBatch(
+        id=batch_id,
+        completion_window="24h",
+        created_at=1,
+        endpoint="/v1/chat/completions",
+        input_file_id="f",
+        object="batch",
+        status="completed",
+    )
+
+
+def _jsonl() -> io.BytesIO:
+    return io.BytesIO(b'{"custom_id": "1", "body": {"model": "g", "messages": []}}\n')
+
+
+# --- files ----------------------------------------------------------------------------------
+
+
+async def test_acreate_file_creates_the_file_on_every_deployment_of_the_group(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    await core.acreate_file(model="g", file=_jsonl(), purpose="batch")
+    assert sorted(host.deployments_called("acreate_file")) == ["d1", "d2"]
+
+
+async def test_acreate_file_returns_the_first_response_with_the_deployment_file_id_mapping(
+    make_core: MakeCore, host: EndpointHost
+):
+    first: Final = ModelResponse(id="file-1")
+    second: Final = ModelResponse(id="file-2")
+    host.answer("openai/d1", first)
+    host.answer("openai/d2", second)
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    response: Final = await core.acreate_file(model="g", file=_jsonl(), purpose="batch")
+    assert response in (first, second)
+    assert response._hidden_params["model_file_id_mapping"] == {"d1": "file-1", "d2": "file-2"}
+    assert "model_id" not in response._hidden_params
+
+
+async def test_acreate_file_rewrites_the_jsonl_model_to_each_deployment_model(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    await core.acreate_file(model="g", file=_jsonl(), purpose="batch")
+    models: Final = {}
+    for call in host.calls:
+        sent_file: Final = call.kwargs["file"]
+        assert isinstance(sent_file, io.BytesIO)
+        models[call.deployment_id] = json.loads(sent_file.getvalue().decode().strip())["body"]["model"]
+    assert models == {"d1": "d1", "d2": "d2"}
+
+
+async def test_acreate_file_keeps_a_non_batch_file_as_sent(make_core: MakeCore, host: EndpointHost):
+    upload: Final = _jsonl()
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await core.acreate_file(model="g", file=upload, purpose="assistants")
+    assert host.calls[0].kwargs["file"] is not None
+    sent: Final = host.calls[0].kwargs["file"]
+    assert isinstance(sent, io.BytesIO)
+    assert json.loads(sent.getvalue().decode().strip())["body"]["model"] == "g"
+
+
+async def test_acreate_file_puts_the_model_group_in_metadata_and_the_deployment_in_litellm_metadata(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await core.acreate_file(model="g", file=_jsonl(), purpose="batch")
+    sent: Final = host.calls[0].kwargs
+    assert _bucket(sent, "metadata")["model_group"] == "g"
+    assert "deployment" not in _bucket(sent, "metadata")
+    assert _bucket(sent, "litellm_metadata")["deployment"] == "openai/d1"
+    assert "model_group" not in _bucket(sent, "litellm_metadata")
+    assert sent["custom_llm_provider"] == "openai"
+
+
+async def test_acreate_file_without_a_purpose_raises_before_calling_the_host(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0)
+    with pytest.raises(Exception, match="file and file_purpose are required for create_file"):
+        await core.acreate_file(model="g", file=_jsonl())
+    assert host.calls == []
+
+
+async def test_acreate_file_sends_target_model_names_to_a_litellm_proxy_deployment(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1", model="litellm_proxy/remote-model")])
+    await core.acreate_file(model="g", file=_jsonl(), purpose="batch", extra_body={"keep": 1})
+    assert dict(host.calls[0].kwargs["extra_body"]) == {"keep": 1, "target_model_names": "remote-model"}  # pyright: ignore[reportArgumentType]  # a mapping
+
+
+async def test_acreate_file_sends_the_deployment_gcs_bucket_in_litellm_metadata(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1", gcs_bucket_name="bucket-1")])
+    await core.acreate_file(model="g", file=_jsonl(), purpose="batch")
+    assert _bucket(host.calls[0].kwargs, "litellm_metadata")["gcs_bucket_name"] == "bucket-1"
+
+
+# --- batches --------------------------------------------------------------------------------
+
+
+async def test_acreate_batch_routes_with_litellm_metadata_only(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    response: Final = await core.acreate_batch(
+        model="g", input_file_id="f", endpoint="/v1/chat/completions", completion_window="24h"
+    )
+    call: Final = host.calls[0]
+    assert (call.op, call.model, call.deployment_id, call.kwargs["custom_llm_provider"]) == (
+        "acreate_batch",
+        "openai/d1",
+        "d1",
+        "openai",
+    )
+    assert "metadata" not in call.kwargs
+    assert _bucket(call.kwargs, "litellm_metadata")["model_group"] == "g"
+    assert response._hidden_params["model_id"] == "d1"
+
+
+async def test_acreate_batch_is_retried(make_core: MakeCore, host: EndpointHost):
+    host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"), _batch())
+    core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=1)
+    await core.acreate_batch(model="g", input_file_id="f", endpoint="/v1/chat/completions", completion_window="24h")
+    assert len(host.calls_to("acreate_batch")) == 2
+
+
+async def test_aretrieve_batch_asks_every_deployment_and_returns_the_batch_found(
+    make_core: MakeCore, host: EndpointHost
+):
+    found: Final = _batch()
+    host.answer("openai/d1", bad_request("d1", "no such batch"))
+    host.answer("openai/d2", found)
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    assert await core.aretrieve_batch(model="g", batch_id="b1") is found
+    assert sorted(host.deployments_called("aretrieve_batch")) == ["d1", "d2"]
+    for call in host.calls:
+        assert (call.kwargs["batch_id"], call.kwargs["custom_llm_provider"], call.model_group) == ("b1", "openai", "g")
+        assert "metadata" not in call.kwargs
+
+
+async def test_aretrieve_batch_raises_the_first_error_when_no_deployment_has_the_batch(
+    make_core: MakeCore, host: EndpointHost
+):
+    host.answer("openai/d1", bad_request("d1", "first missing"))
+    host.answer("openai/d2", bad_request("d2", "second missing"))
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    with pytest.raises(litellm.BadRequestError, match="missing"):
+        await core.aretrieve_batch(model="g", batch_id="b1")
+    assert len(host.calls) == 2
+
+
+async def test_aretrieve_batch_ignores_a_non_batch_answer_and_raises_a_generic_error(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    with pytest.raises(Exception, match="Unable to find batch in any model"):
+        await core.aretrieve_batch(model="g", batch_id="b1")
+
+
+async def test_aretrieve_batch_without_a_model_asks_every_deployment_with_its_own_group(
+    make_core: MakeCore, host: EndpointHost
+):
+    host.answer("*", bad_request(message="missing"))
+    core: Final = make_core(model_list=[*TWO_DEPLOYMENTS, deployment("h", "d3")])
+    with pytest.raises(litellm.BadRequestError):
+        await core.aretrieve_batch(batch_id="b1")
+    assert sorted((call.deployment_id, call.model_group) for call in host.calls) == [
+        ("d1", "g"),
+        ("d2", "g"),
+        ("d3", "h"),
+    ]
+
+
+async def test_aretrieve_batch_is_not_retried(make_core: MakeCore, host: EndpointHost):
+    host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
+    core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=3)
+    with pytest.raises(litellm.InternalServerError):
+        await core.aretrieve_batch(model="g", batch_id="b1")
+    assert len(host.calls) == 1
+
+
+async def test_acancel_batch_routes_with_litellm_metadata(make_core: MakeCore, host: EndpointHost):
+    host.answer("openai/d1", _batch())
+    core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")])
+    response: Final = await core.acancel_batch(model="g", batch_id="b1")
+    call: Final = host.calls[0]
+    assert (call.op, call.deployment_id, call.kwargs["batch_id"], call.kwargs["api_base"]) == (
+        "acancel_batch",
+        "d1",
+        "b1",
+        "https://d1.example",
+    )
+    assert "metadata" not in call.kwargs
+    assert response._hidden_params["model_id"] == "d1"
+
+
+async def test_alist_batches_merges_the_lists_of_every_deployment(make_core: MakeCore, host: EndpointHost):
+    host.answer("openai/d1", SimpleNamespace(first_id="a", last_id="b", data=[1, 2], has_more=False))
+    host.answer("openai/d2", SimpleNamespace(first_id="c", last_id="d", data=[3], has_more=True))
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    merged: Final = await core.alist_batches(model="g", limit=2)
+    assert sorted(merged["data"]) == [1, 2, 3]
+    assert (merged["object"], merged["has_more"]) == ("list", True)
+    assert merged["first_id"] in ("a", "c")
+    assert merged["last_id"] in ("b", "d")
+
+
+async def test_alist_batches_sends_the_deployment_params_and_request_without_routing_metadata(
+    make_core: MakeCore, host: EndpointHost
+):
+    host.answer("*", SimpleNamespace(first_id=None, last_id=None, data=[], has_more=False))
+    core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")])
+    await core.alist_batches(model="g", limit=2, api_base="https://request.example")
+    call: Final = host.calls[0]
+    assert (call.model, call.kwargs["limit"], call.kwargs["api_base"], call.deployment_id) == (
+        "openai/d1",
+        2,
+        "https://request.example",
+        None,
+    )
+    assert "metadata" not in call.kwargs and "litellm_metadata" not in call.kwargs
+
+
+async def test_alist_batches_drops_failing_deployments(make_core: MakeCore, host: EndpointHost):
+    host.answer("openai/d1", bad_request("d1"))
+    host.answer("openai/d2", SimpleNamespace(first_id="c", last_id="d", data=[3], has_more=False))
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    merged: Final = await core.alist_batches(model="g")
+    assert merged == {"object": "list", "data": [3], "first_id": "c", "last_id": "d", "has_more": False}
+
+
+async def test_alist_batches_for_an_unknown_group_returns_an_empty_list(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    assert await core.alist_batches(model="nope") == {
+        "object": "list",
+        "data": [],
+        "first_id": None,
+        "last_id": None,
+        "has_more": False,
+    }
+    assert host.calls == []
+
+
+# --- batch completions ------------------------------------------------------------------------
+
+
+async def test_abatch_completion_sends_one_request_to_each_model_and_returns_errors_in_place(
+    make_core: MakeCore, host: EndpointHost
+):
+    error: Final = bad_request("d1", "a failed")
+    host.answer("openai/d1", error)
+    core: Final = make_core(model_list=[deployment("a", "d1"), deployment("b", "d2")], num_retries=0)
+    results: Final = await core.abatch_completion(models=["a", "b"], messages=MESSAGES)
+    assert isinstance(results[0], litellm.BadRequestError)
+    assert isinstance(results[1], ModelResponse) and results[1]._hidden_params["model_id"] == "d2"
+
+
+async def test_abatch_completion_with_message_lists_groups_results_per_request(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("a", "d1"), deployment("b", "d2")])
+    second: Final = [{"role": "user", "content": "second"}]
+    results: Final = await core.abatch_completion(models=["a", "b"], messages=[MESSAGES, second])
+    assert [[response._hidden_params["model_id"] for response in row] for row in results] == [
+        ["d1", "d2"],
+        ["d1", "d2"],
+    ]
+    assert sorted(str(call.kwargs["messages"]) for call in host.calls) == sorted(
+        [str(MESSAGES), str(MESSAGES), str(second), str(second)]
+    )
+
+
+async def test_abatch_completion_forwards_extra_kwargs_to_every_request(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("a", "d1"), deployment("b", "d2")])
+    await core.abatch_completion(models=["a", "b"], messages=MESSAGES, max_tokens=15)
+    assert [call.kwargs["max_tokens"] for call in host.calls] == [15, 15]
+
+
+async def test_abatch_completion_one_model_multiple_requests_returns_one_result_per_request(
+    make_core: MakeCore, host: EndpointHost
+):
+    host.answer("openai/d1", ModelResponse(id="r1"), bad_request("d1"), ModelResponse(id="r3"))
+    core: Final = make_core(model_list=[deployment("a", "d1")], num_retries=0)
+    results: Final = await core.abatch_completion_one_model_multiple_requests(model="a", messages=[MESSAGES] * 3)
+    assert len(results) == 3
+    assert sum(isinstance(result, litellm.BadRequestError) for result in results) == 1
+    assert len(host.calls) == 3
+
+
+async def test_fastest_response_returns_the_first_success_marked_in_hidden_params(
+    make_core: MakeCore, host: EndpointHost
+):
+    host.answer("openai/d1", bad_request("d1"))
+    core: Final = make_core(model_list=[deployment("a", "d1"), deployment("b", "d2")], num_retries=0)
+    response: Final = await core.abatch_completion_fastest_response(model="a, b", messages=MESSAGES)
+    assert response._hidden_params["model_id"] == "d2"
+    assert response._hidden_params["fastest_response_batch_completion"] is True
+
+
+async def test_fastest_response_raises_when_every_model_fails(make_core: MakeCore, host: EndpointHost):
+    host.answer("*", bad_request())
+    core: Final = make_core(model_list=[deployment("a", "d1"), deployment("b", "d2")], num_retries=0)
+    with pytest.raises(Exception, match="All tasks failed"):
+        await core.abatch_completion_fastest_response(model="a,b", messages=MESSAGES)
+
+
+async def test_fastest_response_with_an_unknown_model_counts_it_as_failed(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("a", "d1")], num_retries=0)
+    response: Final = await core.abatch_completion_fastest_response(model="nope, a", messages=MESSAGES)
+    assert response._hidden_params["model_id"] == "d1"
+
+
+# --- scheduler / priority -------------------------------------------------------------------
+
+
+async def test_schedule_acompletion_marks_the_response_as_prioritized_and_does_not_send_priority(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    response: Final = await core.schedule_acompletion(model="g", messages=MESSAGES, priority=0)
+    assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True
+    assert "priority" not in host.calls[0].kwargs
+
+
+async def test_schedule_acompletion_without_a_priority_or_default_raises(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    with pytest.raises(
+        litellm.BadRequestError,
+        match=re.escape("priority is required to queue a request, send one or set router_settings.default_priority"),
+    ):
+        await core.schedule_acompletion(model="g", messages=MESSAGES)
+    assert host.calls == []
+
+
+async def test_schedule_acompletion_without_a_priority_uses_the_default_priority(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")], default_priority=5)
+    response: Final = await core.schedule_acompletion(model="g", messages=MESSAGES)
+    assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True
+
+
+@pytest.mark.parametrize("endpoint", ["schedule_acompletion", "acompletion"])
+async def test_a_non_integer_priority_raises_bad_request(make_core: MakeCore, host: EndpointHost, endpoint: str):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    with pytest.raises(litellm.BadRequestError, match="priority must be an integer, got str 'high'"):
+        await core.call(endpoint, {"model": "g", "messages": MESSAGES, "priority": "high"})
+    assert host.calls == []
+
+
+async def test_a_non_integer_priority_with_drop_params_is_served_unscheduled(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    response: Final = await core.acompletion(model="g", messages=MESSAGES, priority="high", drop_params=True)
+    assert "x-litellm-request-prioritization-used" not in response._hidden_params["additional_headers"]
+    assert "priority" not in host.calls[0].kwargs
+
+
+async def test_a_non_integer_priority_with_drop_params_queues_at_the_default_priority(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")], default_priority=5)
+    response: Final = await core.acompletion(model="g", messages=MESSAGES, priority="high", drop_params=True)
+    assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True
+
+
+@pytest.mark.parametrize(
+    ("endpoint", "request_body"),
+    [("acompletion", {"messages": MESSAGES}), ("atext_completion", {"prompt": "p"})],
+)
+async def test_a_request_priority_schedules_the_call(
+    make_core: MakeCore, host: EndpointHost, endpoint: str, request_body: dict[str, object]
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    response: Final = await core.call(endpoint, {"model": "g", "priority": 3, **request_body})
+    assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True  # pyright: ignore[reportAttributeAccessIssue]  # the scripted response
+    assert "priority" not in host.calls[0].kwargs
+
+
+async def test_default_priority_schedules_every_acompletion(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")], default_priority=5)
+    response: Final = await core.acompletion(model="g", messages=MESSAGES)
+    assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True
+
+
+async def test_a_scheduled_call_error_carries_its_priority(make_core: MakeCore, host: EndpointHost):
+    host.answer("openai/d1", bad_request("d1"))
+    core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0)
+    with pytest.raises(litellm.BadRequestError) as raised:
+        await core.acompletion(model="g", messages=MESSAGES, priority=4)
+    assert raised.value.priority == 4  # pyright: ignore[reportAttributeAccessIssue]  # stamped by the scheduler
+
+
+@pytest.mark.parametrize(("header", "expected"), [("7", {"priority": 7}), (None, {}), ("", {})])
+async def test_priority_header_update_parses_the_priority_header(
+    make_core: MakeCore, host: EndpointHost, header: str | None, expected: dict[str, int]
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    assert dict(core.priority_header_update("g", {}, header)) == expected
+
+
+# --- prompt management models -----------------------------------------------------------------
+
+
+def _logging_obj() -> LiteLLMLogging:
+    return LiteLLMLogging(
+        model="pm",
+        messages=MESSAGES,
+        stream=False,
+        call_type="acompletion",
+        start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
+        litellm_call_id="c1",
+        function_id="f1",
+    )
+
+
+async def test_prompt_management_group_without_a_prompt_id_raises_value_error(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("pm", "p1", model="langfuse/gpt-4o")])
+    with pytest.raises(ValueError, match=re.escape("Prompt ID is not set or not a string. Got=None")):
+        await core.acompletion(model="pm", messages=MESSAGES, litellm_call_id="c1")
+    assert host.calls == []
+
+
+async def test_prompt_management_group_with_non_dict_prompt_variables_raises_value_error(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(
+        model_list=[deployment("pm", "p1", model="langfuse/gpt-4o", prompt_id="p", prompt_variables="x")]
+    )
+    with pytest.raises(ValueError, match=re.escape("Prompt variables is set but not a dictionary. Got=x")):
+        await core.acompletion(model="pm", messages=MESSAGES, litellm_call_id="c1")
+
+
+async def test_prompt_management_group_without_a_call_id_or_logging_object_raises_key_error(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("pm", "p1", model="langfuse/gpt-4o")])
+    with pytest.raises(KeyError, match="litellm_call_id"):
+        await core.acompletion(model="pm", messages=MESSAGES)
+
+
+async def test_litellm_agent_group_without_a_logging_object_raises_key_error(make_core: MakeCore, host: EndpointHost):
+    """function_setup consumes original_function, so the direct-call path cannot pop it."""
+    core: Final = make_core(model_list=[deployment("pm", "p1", model="litellm_agent/openai/gpt-4o")])
+    with pytest.raises(KeyError, match="original_function"):
+        await core.acompletion(model="pm", messages=MESSAGES, litellm_call_id="c1")
+    assert host.calls == []
+
+
+async def test_litellm_agent_group_calls_the_stripped_model_directly_with_the_deployment_params(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(
+        model_list=[deployment("pm", "p1", model="litellm_agent/openai/gpt-4o", timeout=9, temperature=0.2)]
+    )
+    response: Final = await core.acompletion(model="pm", messages=MESSAGES, litellm_logging_obj=_logging_obj())
+    call: Final = host.calls[0]
+    assert (call.op, call.model, call.deployment_id, call.model_group) == ("acompletion", "openai/gpt-4o", "p1", "pm")
+    assert (call.kwargs["timeout"], call.kwargs["temperature"], "original_function" in call.kwargs) == (9.0, 0.2, False)
+    assert response._hidden_params == {}
+
+
+async def test_litellm_agent_group_naming_another_group_routes_into_it_with_the_agent_timeout(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(
+        model_list=[deployment("pm", "p1", model="litellm_agent/g", timeout=9), deployment("g", "d1")]
+    )
+    response: Final = await core.acompletion(model="pm", messages=MESSAGES, litellm_logging_obj=_logging_obj())
+    call: Final = host.calls[0]
+    assert (call.model, call.model_group, call.kwargs["timeout"]) == ("openai/d1", "pm", 9.0)
+    # The agent deployment's api_key travels as a request credential, so the target gets a request-scoped id.
+    assert call.deployment_id != "d1"
+    assert response._hidden_params["additional_headers"]["x-litellm-model-group"] == "g"
+
+
+async def test_a_group_with_two_prompt_management_deployments_routes_normally(make_core: MakeCore, host: ScriptedHost):
+    core: Final = make_core(
+        model_list=[deployment("pm", "p1", model="langfuse/gpt-4o"), deployment("pm", "p2", model="langfuse/gpt-4o")]
+    )
+    await core.acompletion(model="pm", messages=MESSAGES)
+    call: Final = host.calls[0]
+    assert (call.op, call.model, call.model_group) == ("acompletion", "langfuse/gpt-4o", "pm")

tests/unit/test_router/core/test_endpoints_characterization_generic.py

@@ -0,0 +1,780 @@
+"""
+Characterization of the generic endpoints (Responses API and its get/delete/cancel/compact/input
+items, Google GenAI, OCR, search, decisions, video, containers, skills, interactions, managed
+agents, realtime, fine-tuning, files, image edit, LLM pass-through, vector stores, assistants,
+guardrails): which host op each reaches, the kwargs the core builds, and how an id in the request
+pins the deployment.
+
+The Python core binds litellm's own functions for some of these endpoints instead of going through
+``host.invoke``; the ``host`` fixture replaces those functions with recording stand-ins (see
+``endpoints_helpers.py``), so every call shows up in ``host.calls``.
+"""
+
+import re
+import inspect
+from collections.abc import Callable, Mapping
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.responses.utils import ResponsesAPIRequestUtils
+from litellm.router_core import RouterCore
+from litellm.types.videos.utils import encode_video_id_with_provider
+from tests.unit.test_router.core.endpoints_helpers import EndpointHost, patch_direct_endpoints
+from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment
+
+MESSAGES: Final = [{"role": "user", "content": "hi"}]
+MakeCore = Callable[..., RouterCore]
+
+
+@pytest.fixture
+def host(monkeypatch: pytest.MonkeyPatch) -> EndpointHost:
+    endpoint_host: Final = EndpointHost()
+    patch_direct_endpoints(monkeypatch, endpoint_host)
+    return endpoint_host
+
+
+async def _call(core: RouterCore, endpoint: str, request: Mapping[str, object]) -> object:
+    result: Final = getattr(core, endpoint)(**request)
+    return await result if inspect.isawaitable(result) else result
+
+
+def _bucket(kwargs: Mapping[str, object], name: str) -> Mapping[str, object]:
+    bucket: Final = kwargs[name]
+    assert isinstance(bucket, Mapping)
+    return bucket
+
+
+# endpoint -> (recorded op, request without model)
+ROUTED_ASYNC: Final[dict[str, tuple[str, dict[str, object]]]] = {
+    "aresponses": ("aresponses", {"input": "hi"}),
+    "agenerate_content": ("agenerate_content", {"contents": []}),
+    "agenerate_content_stream": ("agenerate_content_stream", {"contents": []}),
+    "afile_delete": ("afile_delete", {"file_id": "f1"}),
+    "afile_content": ("afile_content", {"file_id": "f1"}),
+    "afile_list": ("afile_list", {}),
+    "aget_responses": ("aget_responses", {"response_id": "resp_plain"}),
+    "acancel_responses": ("acancel_responses", {"response_id": "resp_plain"}),
+    "acompact_responses": ("acompact_responses", {"response_id": "resp_plain"}),
+    "adelete_responses": ("adelete_responses", {"response_id": "resp_plain"}),
+    "alist_input_items": ("alist_input_items", {"response_id": "resp_plain"}),
+    "acreate_realtime_client_secret": ("acreate_realtime_client_secret", {}),
+    "arealtime_calls": ("arealtime_calls", {}),
+    "acreate_realtime_transcription_session": ("acreate_realtime_transcription_session", {}),
+    "acreate_fine_tuning_job": ("acreate_fine_tuning_job", {"training_file": "f"}),
+    "acancel_fine_tuning_job": ("acancel_fine_tuning_job", {"fine_tuning_job_id": "j"}),
+    "alist_fine_tuning_jobs": ("alist_fine_tuning_jobs", {}),
+    "aretrieve_fine_tuning_job": ("aretrieve_fine_tuning_job", {"fine_tuning_job_id": "j"}),
+    "aimage_edit": ("aimage_edit", {"image": b"x", "prompt": "p"}),
+    "allm_passthrough_route": ("allm_passthrough_route", {"method": "POST", "endpoint": "/v1/x"}),
+    "aocr": ("aocr", {"document": {}}),
+    "adecisions": ("adecisions", {}),
+    "asystemone": ("asystemone", {}),
+    "aextraction": ("aextraction", {}),
+    "avideo_generation": ("avideo_generation", {"prompt": "p"}),
+    "avideo_list": ("avideo_list", {}),
+    "avideo_status": ("avideo_status", {"video_id": "v"}),
+    "avideo_content": ("avideo_content", {"video_id": "v"}),
+    "avideo_remix": ("avideo_remix", {"video_id": "v", "prompt": "p"}),
+    "avideo_create_character": ("avideo_create_character", {}),
+    "avideo_get_character": ("avideo_get_character", {}),
+    "avideo_edit": ("avideo_edit", {"video_id": "v"}),
+    "avideo_extension": ("avideo_extension", {"video_id": "v"}),
+    "acreate_container": ("acreate_container", {"name": "c"}),
+    "alist_containers": ("alist_containers", {}),
+    "aretrieve_container": ("aretrieve_container", {"container_id": "cntr_native"}),
+    "adelete_container": ("adelete_container", {"container_id": "cntr_native"}),
+    "aupload_container_file": ("aupload_container_file", {"container_id": "cntr_native", "file": b"x"}),
+    "alist_container_files": ("alist_container_files", {"container_id": "cntr_native"}),
+    "aretrieve_container_file": ("aretrieve_container_file", {"container_id": "cntr_native", "file_id": "f"}),
+    "adelete_container_file": ("adelete_container_file", {"container_id": "cntr_native", "file_id": "f"}),
+    "aretrieve_container_file_content": (
+        "aretrieve_container_file_content",
+        {"container_id": "cntr_native", "file_id": "f"},
+    ),
+    "acreate_skill": ("acreate_skill", {}),
+    "alist_skills": ("alist_skills", {}),
+    "aget_skill": ("aget_skill", {"skill_id": "s"}),
+    "adelete_skill": ("adelete_skill", {"skill_id": "s"}),
+    "acreate_interaction": ("interactions.acreate", {"input": "x"}),
+    "avector_store_search": ("vs.asearch", {"vector_store_id": "v", "query": "q"}),
+    "avector_store_retrieve": ("vs.aretrieve", {"vector_store_id": "v"}),
+    "avector_store_list": ("vs.alist", {}),
+    "avector_store_update": ("vs.aupdate", {"vector_store_id": "v"}),
+    "avector_store_delete": ("vs.adelete", {"vector_store_id": "v"}),
+    "avector_store_file_create": ("vsf.acreate", {"vector_store_id": "v", "file_id": "f"}),
+    "avector_store_file_list": ("vsf.alist", {"vector_store_id": "v"}),
+    "avector_store_file_retrieve": ("vsf.aretrieve", {"vector_store_id": "v", "file_id": "f"}),
+    "avector_store_file_content": ("vsf.aretrieve_content", {"vector_store_id": "v", "file_id": "f"}),
+    "avector_store_file_update": ("vsf.aupdate", {"vector_store_id": "v", "file_id": "f"}),
+    "avector_store_file_delete": ("vsf.adelete", {"vector_store_id": "v", "file_id": "f"}),
+}
+
+ROUTED_SYNC: Final[dict[str, tuple[str, dict[str, object]]]] = {
+    "responses": ("responses", {"input": "hi"}),
+    "generate_content": ("generate_content", {"contents": []}),
+    "generate_content_stream": ("generate_content_stream", {"contents": []}),
+    "ocr": ("ocr", {"document": {}}),
+    "decisions": ("decisions", {}),
+    "systemone": ("systemone", {}),
+    "extraction": ("extraction", {}),
+    "video_generation": ("video_generation", {"prompt": "p"}),
+    "video_list": ("video_list", {}),
+    "video_status": ("video_status", {"video_id": "v"}),
+    "video_content": ("video_content", {"video_id": "v"}),
+    "video_remix": ("video_remix", {"video_id": "v", "prompt": "p"}),
+    "create_container": ("create_container", {"name": "c"}),
+    "list_containers": ("list_containers", {}),
+    "retrieve_container": ("retrieve_container", {"container_id": "c"}),
+    "delete_container": ("delete_container", {"container_id": "c"}),
+    "vector_store_search": ("vs.search", {"vector_store_id": "v", "query": "q"}),
+    "vector_store_create": ("vs.create", {"name": "v"}),
+    "vector_store_retrieve": ("vs.retrieve", {"vector_store_id": "v"}),
+    "vector_store_list": ("vs.list", {}),
+    "vector_store_update": ("vs.update", {"vector_store_id": "v"}),
+    "vector_store_delete": ("vs.delete", {"vector_store_id": "v"}),
+}
+
+
+@pytest.mark.parametrize("endpoint", sorted(ROUTED_ASYNC))
+async def test_async_generic_endpoint_routes_to_the_group_deployment_with_litellm_metadata(
+    make_core: MakeCore, host: EndpointHost, endpoint: str
+):
+    op, request = ROUTED_ASYNC[endpoint]
+    core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")])
+    response: Final = await _call(core, endpoint, {"model": "g", **request})
+    assert [call.op for call in host.calls] == [op]
+    call: Final = host.calls[0]
+    assert (call.model, call.deployment_id, call.kwargs["api_base"], call.kwargs["custom_llm_provider"]) == (
+        "openai/d1",
+        "d1",
+        "https://d1.example",
+        "openai",
+    )
+    metadata: Final = _bucket(call.kwargs, "litellm_metadata")
+    assert (metadata["model_group"], metadata["deployment"]) == ("g", "openai/d1")
+    assert "metadata" not in call.kwargs
+    assert response._hidden_params["model_id"] == "d1"  # pyright: ignore[reportAttributeAccessIssue]  # the scripted response
+    assert response._hidden_params["additional_headers"]["x-litellm-model-group"] == "g"  # pyright: ignore[reportAttributeAccessIssue]  # the scripted response
+
+
+@pytest.mark.parametrize("endpoint", sorted(ROUTED_SYNC))
+async def test_sync_generic_endpoint_routes_to_the_group_deployment_and_returns_the_response(
+    make_core: MakeCore, host: EndpointHost, endpoint: str
+):
+    op, request = ROUTED_SYNC[endpoint]
+    core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")])
+    response: Final = getattr(core, endpoint)(model="g", **request)
+    assert not inspect.isawaitable(response)
+    assert [call.op for call in host.calls] == [op]
+    call: Final = host.calls[0]
+    assert (call.model, call.deployment_id, call.kwargs["api_base"], call.kwargs["custom_llm_provider"]) == (
+        "openai/d1",
+        "d1",
+        "https://d1.example",
+        "openai",
+    )
+    assert _bucket(call.kwargs, "litellm_metadata")["model_group"] == "g"
+    assert response is not None
+
+
+@pytest.mark.parametrize("endpoint", ["ocr", "generate_content", "video_generation"])
+async def test_sync_generic_endpoints_set_no_routing_hidden_params(
+    make_core: MakeCore, host: EndpointHost, endpoint: str
+):
+    _, request = ROUTED_SYNC[endpoint]
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    response: Final = getattr(core, endpoint)(model="g", **request)
+    assert response._hidden_params == {}
+
+
+@pytest.mark.parametrize("endpoint", ["ocr", "generate_content"])
+async def test_sync_generic_endpoints_are_not_retried(make_core: MakeCore, host: EndpointHost, endpoint: str):
+    host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
+    _, request = ROUTED_SYNC[endpoint]
+    core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=3)
+    with pytest.raises(litellm.InternalServerError):
+        getattr(core, endpoint)(model="g", **request)
+    assert len(host.calls) == 1
+
+
+async def test_async_generic_endpoint_is_retried_with_the_router_num_retries(make_core: MakeCore, host: EndpointHost):
+    host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
+    core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=2)
+    with pytest.raises(litellm.InternalServerError):
+        await core.aocr(model="g", document={})
+    assert len(host.calls) == 3
+
+
+@pytest.mark.parametrize(
+    "endpoint",
+    [
+        "aadapter_generate_content",
+        "video_create_character",
+        "video_get_character",
+        "video_edit",
+        "video_extension",
+        "upload_container_file",
+        "list_container_files",
+        "retrieve_container_file",
+        "delete_container_file",
+        "retrieve_container_file_content",
+        "get_interaction",
+        "delete_interaction",
+        "cancel_interaction",
+    ],
+)
+async def test_endpoints_without_a_dispatch_branch_resolve_to_none_without_calling_anything(
+    make_core: MakeCore, host: EndpointHost, endpoint: str
+):
+    """These are wrapped by factory_function's async wrapper, which has no branch for them."""
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    result: Final = getattr(core, endpoint)(model="g", container_id="c", video_id="v", interaction_id="i")
+    assert inspect.isawaitable(result)
+    assert await result is None
+    assert host.calls == []
+
+
+async def test_sync_create_interaction_is_awaited_and_fails_with_type_error(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0)
+    with pytest.raises(TypeError, match="can't be used in 'await' expression"):
+        await core.create_interaction(model="g", input="x")
+    assert [call.op for call in host.calls] == ["interactions.create"]
+
+
+# --- request params, provider and timeouts ------------------------------------------------------
+
+
+@pytest.mark.parametrize("endpoint", ["aocr", "ocr"])
+async def test_generic_request_params_win_over_deployment_params(
+    make_core: MakeCore, host: EndpointHost, endpoint: str
+):
+    core: Final = make_core(model_list=[deployment("g", "d1", temperature=0.1)])
+    await _call(core, endpoint, {"model": "g", "document": {}, "temperature": 0.9})
+    assert host.calls[0].kwargs["temperature"] == 0.9
+
+
+async def test_generic_call_always_sends_the_deployment_model(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await core.aocr(model="g", document={})
+    assert host.calls[0].model == "openai/d1"
+
+
+async def test_generic_call_sends_the_declared_custom_llm_provider(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1", model="my-model", custom_llm_provider="hosted_vllm")])
+    await core.aocr(model="g", document={})
+    assert (host.calls[0].model, host.calls[0].kwargs["custom_llm_provider"]) == ("my-model", "hosted_vllm")
+
+
+@pytest.mark.parametrize(
+    ("endpoint", "expected"),
+    [("aocr", 600.0), ("aresponses", 600.0), ("ocr", 6000.0), ("generate_content", 6000.0)],
+)
+async def test_generic_default_timeout_is_the_passthrough_default_for_async_and_the_process_setting_for_sync(
+    make_core: MakeCore, host: EndpointHost, endpoint: str, expected: float
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    request: Final = {**(ROUTED_ASYNC.get(endpoint) or ROUTED_SYNC[endpoint])[1], "model": "g"}
+    await _call(core, endpoint, request)
+    assert host.calls[0].kwargs["timeout"] == expected
+
+
+@pytest.mark.parametrize("endpoint", ["aocr", "ocr"])
+async def test_generic_timeout_takes_the_router_timeout(make_core: MakeCore, host: EndpointHost, endpoint: str):
+    core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42)
+    await _call(core, endpoint, {"model": "g", "document": {}})
+    assert host.calls[0].kwargs["timeout"] == 42
+
+
+@pytest.mark.parametrize(
+    ("endpoint", "request_body"), [("aresponses", {"input": "hi"}), ("agenerate_content", {"contents": []})]
+)
+async def test_streaming_generic_call_sends_the_deployment_stream_timeout_as_timeout(
+    make_core: MakeCore, host: EndpointHost, endpoint: str, request_body: dict[str, object]
+):
+    core: Final = make_core(model_list=[deployment("g", "d1", timeout=11, stream_timeout=3)])
+    await _call(core, endpoint, {"model": "g", "stream": True, **request_body})
+    sent: Final = host.calls[0].kwargs
+    assert (sent["timeout"], sent["stream_timeout"]) == (3.0, 3.0)
+
+
+async def test_non_streaming_aresponses_sends_the_deployment_timeout(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1", timeout=11, stream_timeout=3)])
+    await core.aresponses(model="g", input="hi")
+    assert host.calls[0].kwargs["timeout"] == 11.0
+
+
+async def test_realtime_client_secret_session_model_follows_the_routed_deployment(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await core.acreate_realtime_client_secret(model="g", session={"model": "g", "type": "realtime"})
+    assert host.calls[0].kwargs["session"] == {"model": "openai/d1", "type": "realtime"}
+
+
+async def test_generic_endpoint_with_an_unknown_group_raises_without_calling_the_host(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    with pytest.raises(litellm.BadRequestError, match="model=nope"):
+        await core.agenerate_content(model="nope", contents=[])
+    assert host.calls == []
+
+
+# --- Responses API ids pin the deployment -----------------------------------------------------
+
+TWO_DEPLOYMENTS: Final = [deployment("g", "d1"), deployment("g", "d2")]
+ENCODED_RESPONSE_ID: Final = ResponsesAPIRequestUtils._build_responses_api_response_id("openai", "d2", "resp_abc")  # pyright: ignore[reportPrivateUsage]  # litellm's encoder
+
+
+@pytest.mark.parametrize(
+    "endpoint", ["aget_responses", "acancel_responses", "acompact_responses", "adelete_responses", "alist_input_items"]
+)
+async def test_a_litellm_response_id_pins_the_deployment_it_encodes(
+    make_core: MakeCore, host: EndpointHost, endpoint: str
+):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    for _ in range(5):
+        await _call(core, endpoint, {"response_id": ENCODED_RESPONSE_ID})
+    assert host.deployments_called() == ["d2"] * 5
+
+
+async def test_a_litellm_response_id_overrides_the_request_model(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    for _ in range(5):
+        await core.adelete_responses(response_id=ENCODED_RESPONSE_ID, model="g")
+    assert host.deployments_called() == ["d2"] * 5
+    assert _bucket(host.calls[0].kwargs, "litellm_metadata")["model_group"] == "d2"
+
+
+async def test_the_response_id_reaches_the_host_still_encoded(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    await core.aget_responses(response_id=ENCODED_RESPONSE_ID)
+    assert host.calls[0].kwargs["response_id"] == ENCODED_RESPONSE_ID
+
+
+async def test_a_plain_response_id_without_a_model_fails_with_type_error(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    with pytest.raises(TypeError, match="missing 1 required positional argument: 'model'"):
+        await core.aget_responses(response_id="resp_plain")
+    assert host.calls == []
+
+
+# --- container ids ------------------------------------------------------------------------
+
+ENCODED_CONTAINER_ID: Final = ResponsesAPIRequestUtils.build_container_id("azure", "d2", "cntr_upstream")
+
+
+async def test_a_litellm_container_id_pins_its_deployment_and_is_sent_decoded(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    for _ in range(5):
+        await core.aretrieve_container(container_id=ENCODED_CONTAINER_ID)
+    assert host.deployments_called() == ["d2"] * 5
+    assert {call.kwargs["container_id"] for call in host.calls} == {"cntr_upstream"}
+
+
+async def test_a_routed_container_call_sends_the_deployment_provider_not_the_decoded_one(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    await core.aretrieve_container(container_id=ENCODED_CONTAINER_ID, custom_llm_provider="openai")
+    assert host.calls[0].kwargs["custom_llm_provider"] == "openai"
+
+
+async def test_a_native_container_id_routes_by_the_forwarded_model_id(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    for _ in range(5):
+        await core.aretrieve_container(container_id="cntr_native", model_id=" d1 ")
+    assert host.deployments_called() == ["d1"] * 5
+    assert host.calls[0].kwargs["container_id"] == "cntr_native"
+
+
+async def test_a_native_container_id_without_a_model_calls_the_handler_directly(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    await core.aretrieve_container(container_id="cntr_native", custom_llm_provider="azure")
+    call: Final = host.calls[0]
+    assert (call.deployment_id, call.kwargs["container_id"], call.kwargs["custom_llm_provider"]) == (
+        None,
+        "cntr_native",
+        "azure",
+    )
+    assert "litellm_metadata" not in call.kwargs
+
+
+async def test_create_container_with_an_unknown_model_passes_through_with_that_model(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    await core.acreate_container(name="c", model="not-a-group")
+    call: Final = host.calls[0]
+    assert (call.op, call.model, call.deployment_id) == ("acreate_container", "not-a-group", None)
+
+
+async def test_create_container_without_a_model_calls_the_handler_directly(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    await core.acreate_container(name="c")
+    call: Final = host.calls[0]
+    assert (call.op, call.deployment_id, "model" in call.kwargs) == ("acreate_container", None, False)
+
+
+# --- video ids -----------------------------------------------------------------------------
+
+ENCODED_VIDEO_ID: Final = encode_video_id_with_provider("video_up", "openai", model_id="d2", deployment_id="d2")
+
+
+@pytest.mark.parametrize(
+    "endpoint", ["avideo_status", "avideo_content", "avideo_remix", "avideo_edit", "avideo_extension"]
+)
+async def test_a_litellm_video_id_pins_the_deployment_that_created_it(
+    make_core: MakeCore, host: EndpointHost, endpoint: str
+):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    for _ in range(6):
+        await _call(core, endpoint, {"model": "g", "video_id": ENCODED_VIDEO_ID, "prompt": "p"})
+    assert host.deployments_called() == ["d2"] * 6
+
+
+async def test_a_video_id_on_a_video_generation_call_does_not_pin(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    for _ in range(30):
+        await core.avideo_generation(model="g", prompt="p", video_id=ENCODED_VIDEO_ID)
+    assert set(host.deployments_called()) == {"d1", "d2"}
+
+
+async def test_sync_video_status_pins_the_deployment_of_the_video_id(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    for _ in range(6):
+        core.video_status(model="g", video_id=ENCODED_VIDEO_ID)
+    assert host.deployments_called() == ["d2"] * 6
+
+
+# --- LLM pass-through ----------------------------------------------------------------------
+
+
+async def test_llm_passthrough_with_an_unknown_model_calls_the_handler_with_that_model(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    await core.allm_passthrough_route(model="not-a-group", method="POST", endpoint="/model/not-a-group/invoke")
+    call: Final = host.calls[0]
+    assert (call.model, call.deployment_id, call.kwargs["endpoint"]) == (
+        "not-a-group",
+        None,
+        "/model/not-a-group/invoke",
+    )
+
+
+async def test_llm_passthrough_rewrites_the_group_name_in_the_endpoint_to_the_provider_model(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1", model="bedrock/us.anthropic.claude")])
+    await core.allm_passthrough_route(model="g", method="POST", endpoint="/model/g/invoke")
+    call: Final = host.calls[0]
+    assert (call.model, call.kwargs["endpoint"], call.kwargs["custom_llm_provider"]) == (
+        "bedrock/us.anthropic.claude",
+        "/model/us.anthropic.claude/invoke",
+        "bedrock",
+    )
+
+
+async def test_llm_passthrough_forwards_the_request_client(make_core: MakeCore, host: EndpointHost):
+    client: Final = object()
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await core.allm_passthrough_route(model="g", method="POST", endpoint="/v1/x", client=client)
+    assert host.calls[0].kwargs["client"] is client
+
+
+# --- interactions and managed agents ----------------------------------------------------------
+
+
+async def test_interactions_get_without_a_model_calls_the_handler_with_the_given_provider(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    await core.aget_interaction(interaction_id="i", custom_llm_provider="vertex_ai")
+    call: Final = host.calls[0]
+    assert (call.op, call.deployment_id, call.kwargs["custom_llm_provider"]) == ("interactions.aget", None, "vertex_ai")
+
+
+@pytest.mark.parametrize("endpoint", ["aget_interaction", "adelete_interaction", "acancel_interaction"])
+async def test_interactions_default_to_the_gemini_provider(make_core: MakeCore, host: EndpointHost, endpoint: str):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    await _call(core, endpoint, {"interaction_id": "i"})
+    assert host.calls[0].kwargs["custom_llm_provider"] == "gemini"
+
+
+async def test_interaction_with_an_agent_named_as_the_model_is_not_routed(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("ag", "d1")])
+    await core.acreate_interaction(agent="ag", model="ag", input="x")
+    call: Final = host.calls[0]
+    assert (call.deployment_id, call.kwargs["model"], call.kwargs["custom_llm_provider"]) == (None, None, "gemini")
+
+
+@pytest.mark.parametrize(
+    ("endpoint", "op"),
+    [
+        ("acreate_agent", "agents.acreate"),
+        ("alist_agents", "agents.alist"),
+        ("aget_agent", "agents.aget"),
+        ("adelete_agent", "agents.adelete"),
+        ("alist_agent_versions", "agents.alist_versions"),
+        ("create_agent", "agents.create"),
+        ("list_agents", "agents.list"),
+        ("get_agent", "agents.get"),
+        ("delete_agent", "agents.delete"),
+        ("list_agent_versions", "agents.list_versions"),
+    ],
+)
+async def test_managed_agent_endpoints_call_the_handler_directly_with_gemini_by_default(
+    make_core: MakeCore, host: EndpointHost, endpoint: str, op: str
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await _call(core, endpoint, {"name": "a", "model": "g"})
+    call: Final = host.calls[0]
+    assert (call.op, call.deployment_id, call.kwargs["custom_llm_provider"], call.kwargs["model"]) == (
+        op,
+        None,
+        "gemini",
+        "g",
+    )
+
+
+@pytest.mark.parametrize("endpoint", ["acreate_agent", "create_agent"])
+async def test_managed_agent_endpoints_keep_the_given_provider(make_core: MakeCore, host: EndpointHost, endpoint: str):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await _call(core, endpoint, {"name": "a", "custom_llm_provider": "vertex_ai"})
+    assert host.calls[0].kwargs["custom_llm_provider"] == "vertex_ai"
+
+
+# --- assistants ----------------------------------------------------------------------------
+
+ASSISTANT_ENDPOINTS: Final = (
+    "acreate_assistants",
+    "adelete_assistant",
+    "aget_assistants",
+    "acreate_thread",
+    "aget_thread",
+    "a_add_message",
+    "aget_messages",
+    "arun_thread",
+)
+
+
+@pytest.mark.parametrize("endpoint", ASSISTANT_ENDPOINTS)
+async def test_assistants_endpoints_without_a_provider_or_config_raise(
+    make_core: MakeCore, host: EndpointHost, endpoint: str
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    with pytest.raises(Exception, match=re.escape("'custom_llm_provider' must be set. Either via:")):
+        await getattr(core, endpoint)()
+    assert host.calls == []
+
+
+@pytest.mark.parametrize("endpoint", ASSISTANT_ENDPOINTS)
+async def test_assistants_endpoints_use_the_assistants_config(make_core: MakeCore, host: EndpointHost, endpoint: str):
+    core: Final = make_core(
+        model_list=[deployment("g", "d1")],
+        assistants_config={
+            "custom_llm_provider": "azure",
+            "litellm_params": {"api_key": "k", "api_base": "https://az"},
+        },
+    )
+    await getattr(core, endpoint)(thread_id="t")
+    call: Final = host.calls[0]
+    assert (call.op, call.kwargs["custom_llm_provider"], call.kwargs["api_key"], call.kwargs["api_base"]) == (
+        endpoint,
+        "azure",
+        "k",
+        "https://az",
+    )
+    assert (call.kwargs["client"], call.kwargs["thread_id"]) == (None, "t")
+
+
+async def test_assistants_endpoint_with_an_explicit_provider_ignores_the_config(
+    make_core: MakeCore, host: EndpointHost
+):
+    client: Final = object()
+    core: Final = make_core(
+        model_list=[deployment("g", "d1")],
+        assistants_config={"custom_llm_provider": "azure", "litellm_params": {"api_key": "k"}},
+    )
+    await core.arun_thread(custom_llm_provider="openai", client=client, thread_id="t")
+    call: Final = host.calls[0]
+    assert (call.kwargs["custom_llm_provider"], call.kwargs["client"], "api_key" in call.kwargs) == (
+        "openai",
+        client,
+        False,
+    )
+
+
+# --- vector stores ---------------------------------------------------------------------------
+
+
+async def test_vector_store_search_without_a_model_gets_the_router_and_overrides_a_client_router(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    await core.avector_store_search(vector_store_id="v", query="q", router="client-value")
+    call: Final = host.calls[0]
+    assert (call.op, call.deployment_id, call.kwargs["router"]) == ("vs.asearch", None, core)
+    assert "_direct_vector_store_embedding_executor" in call.kwargs
+
+
+async def test_sync_vector_store_search_without_a_model_gets_the_router(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    core.vector_store_search(vector_store_id="v", query="q")
+    assert host.calls[0].kwargs["router"] is core
+
+
+async def test_routed_vector_store_search_gets_the_embedding_executor_but_not_the_router(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await core.avector_store_search(model="g", vector_store_id="v", query="q")
+    sent: Final = host.calls[0].kwargs
+    assert ("router" in sent, "_direct_vector_store_embedding_executor" in sent) == (False, True)
+
+
+async def test_vector_store_retrieve_without_a_model_calls_the_handler_directly(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    await core.avector_store_retrieve(vector_store_id="v", custom_llm_provider="openai")
+    call: Final = host.calls[0]
+    assert (call.deployment_id, call.kwargs["custom_llm_provider"], "router" in call.kwargs) == (None, "openai", False)
+
+
+async def test_sync_vector_store_create_without_a_model_passes_the_provider(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=TWO_DEPLOYMENTS)
+    core.vector_store_create(name="v", custom_llm_provider="openai")
+    call: Final = host.calls[0]
+    assert (call.op, call.deployment_id, call.kwargs["custom_llm_provider"]) == ("vs.create", None, "openai")
+
+
+@pytest.mark.parametrize(
+    ("endpoint", "op"),
+    [
+        ("vector_store_file_create", "vsf.create"),
+        ("vector_store_file_list", "vsf.list"),
+        ("vector_store_file_retrieve", "vsf.retrieve"),
+        ("vector_store_file_content", "vsf.retrieve_content"),
+        ("vector_store_file_update", "vsf.update"),
+        ("vector_store_file_delete", "vsf.delete"),
+    ],
+)
+async def test_sync_vector_store_file_endpoints_never_route_even_with_a_model(
+    make_core: MakeCore, host: EndpointHost, endpoint: str, op: str
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    getattr(core, endpoint)(model="g", vector_store_id="v", file_id="f")
+    call: Final = host.calls[0]
+    assert (call.op, call.model, call.deployment_id, call.kwargs["custom_llm_provider"], call.kwargs["client"]) == (
+        op,
+        "g",
+        None,
+        None,
+        None,
+    )
+
+
+async def test_avector_store_create_with_a_model_routes_with_metadata_and_no_model_group(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")])
+    await core.avector_store_create(model="g", name="n")
+    call: Final = host.calls[0]
+    assert (call.op, call.model, call.deployment_id, call.kwargs["custom_llm_provider"]) == (
+        "vs.acreate",
+        "openai/d1",
+        "d1",
+        "openai",
+    )
+    metadata: Final = _bucket(call.kwargs, "metadata")
+    assert (metadata["deployment"], "model_group" in metadata) == ("openai/d1", False)
+
+
+async def test_avector_store_create_without_a_model_calls_the_handler_directly(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await core.avector_store_create(model=None, name="n")
+    call: Final = host.calls[0]
+    assert (call.op, call.deployment_id) == ("vs.acreate", None)
+
+
+async def test_avector_store_create_is_not_retried(make_core: MakeCore, host: EndpointHost):
+    host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
+    core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=3)
+    with pytest.raises(litellm.InternalServerError):
+        await core.avector_store_create(model="g", name="n")
+    assert len(host.calls) == 1
+
+
+# --- search tools ---------------------------------------------------------------------------
+
+SEARCH_TOOLS: Final = [{"search_tool_name": "s", "litellm_params": {"search_provider": "tavily", "api_key": "tk"}}]
+
+
+async def test_asearch_calls_the_named_search_tool_with_its_litellm_params(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")], search_tools=SEARCH_TOOLS)
+    response: Final = await core.asearch(model="s", query="q")
+    call: Final = host.calls[0]
+    assert (call.op, call.kwargs["search_provider"], call.kwargs["api_key"], call.kwargs["query"]) == (
+        "asearch",
+        "tavily",
+        "tk",
+        "q",
+    )
+    assert _bucket(call.kwargs, "litellm_metadata")["model_group"] == "s"
+    assert response._hidden_params["additional_headers"]["x-litellm-model-group"] == "s"  # pyright: ignore[reportAttributeAccessIssue]  # the scripted response
+
+
+async def test_asearch_with_an_unknown_tool_raises_value_error(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")], search_tools=SEARCH_TOOLS, num_retries=0)
+    with pytest.raises(ValueError, match=re.escape("Search tool 'g' not found in router.search_tools")):
+        await core.asearch(model="g", query="q")
+    assert host.calls == []
+
+
+# --- guardrails ---------------------------------------------------------------------------
+
+GUARDRAILS: Final = [{"guardrail_name": "gr", "litellm_params": {"guardrail": "presidio", "mode": "pre_call"}}]
+
+
+async def test_aguardrail_runs_the_function_with_the_selected_guardrail(make_core: MakeCore, host: ScriptedHost):
+    seen: Final[list[Mapping[str, object]]] = []
+
+    async def guard(**kwargs: object) -> str:  # kwargs-ok: the guardrail's keyword payload
+        seen.append(kwargs)
+        return "verdict"
+
+    core: Final = make_core(model_list=[deployment("g", "d1")], guardrail_list=GUARDRAILS)
+    assert await core.aguardrail(guardrail_name="gr", original_function=guard, data={"x": 1}) == "verdict"
+    assert sorted(seen[0]) == ["data", "litellm_metadata", "litellm_trace_id", "selected_guardrail"]
+    assert seen[0]["selected_guardrail"] == GUARDRAILS[0]
+    assert _bucket(seen[0], "litellm_metadata")["model_group"] == "gr"
+    assert host.calls == []
+
+
+async def test_aguardrail_with_an_unknown_name_raises_value_error(make_core: MakeCore, host: ScriptedHost):
+    async def guard(**_kwargs: object) -> str:  # kwargs-ok: the guardrail's keyword payload
+        return "verdict"
+
+    core: Final = make_core(model_list=[deployment("g", "d1")], guardrail_list=GUARDRAILS, num_retries=0)
+    with pytest.raises(ValueError, match="No guardrail found with name: nope"):
+        await core.aguardrail(guardrail_name="nope", original_function=guard)
+
+
+async def test_aguardrail_does_not_retry_a_failing_guardrail_function(make_core: MakeCore, host: ScriptedHost):
+    """A guardrail name is no model group (model_group_size 0), so the router num_retries do not apply."""
+    attempts: Final[list[int]] = []
+
+    async def guard(**_kwargs: object) -> str:  # kwargs-ok: the guardrail's keyword payload
+        attempts.append(1)
+        raise litellm.InternalServerError(message="boom", llm_provider="", model="gr")
+
+    core: Final = make_core(model_list=[deployment("g", "d1")], guardrail_list=GUARDRAILS, num_retries=2)
+    with pytest.raises(litellm.InternalServerError, match="boom"):
+        await core.aguardrail(guardrail_name="gr", original_function=guard)
+    assert len(attempts) == 1

tests/unit/test_router/core/test_endpoints_characterization_inference.py

@@ -0,0 +1,648 @@
+"""
+Characterization of the inference endpoints (completion, text completion, embedding, image,
+speech, transcription, rerank, adapter completion, moderation): which host op each invokes and
+which keyword arguments the core builds for it.
+
+Every test builds its core through ``make_core`` on the ``host`` fixture, so it runs once per core.
+"""
+
+import asyncio
+import io
+from collections.abc import Callable, Mapping
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.router_core import RouterCore
+from tests.unit.test_router.core.endpoints_helpers import EndpointHost
+from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, timeout
+
+MESSAGES: Final = [{"role": "user", "content": "hi"}]
+MakeCore = Callable[..., RouterCore]
+
+
+@pytest.fixture
+def host() -> EndpointHost:
+    return EndpointHost()
+
+
+def _sent(host: ScriptedHost, index: int = -1) -> Mapping[str, object]:
+    return host.calls[index].kwargs
+
+
+def _bucket(kwargs: Mapping[str, object], name: str = "metadata") -> Mapping[str, object]:
+    bucket: Final = kwargs[name]
+    assert isinstance(bucket, Mapping)
+    return bucket
+
+
+# --- host op per endpoint -------------------------------------------------------------------
+
+
+async def test_aadapter_completion_invokes_the_adapter_host_op_with_the_adapter_id(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await core.aadapter_completion(adapter_id="my-adapter", model="g", messages=MESSAGES)
+    call: Final = host.calls[0]
+    assert (call.op, call.model, call.kwargs["adapter_id"], call.deployment_id, call.model_group) == (
+        "aadapter_completion",
+        "openai/d1",
+        "my-adapter",
+        "d1",
+        "g",
+    )
+
+
+async def test_amoderation_routes_a_known_group_through_its_deployment(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")])
+    await core.amoderation(model="g", input="x")
+    call: Final = host.calls[0]
+    assert (call.op, call.model, call.kwargs["api_base"], call.deployment_id, call.model_group) == (
+        "amoderation",
+        "openai/d1",
+        "https://d1.example",
+        "d1",
+        "g",
+    )
+    assert "litellm_metadata" not in call.kwargs
+
+
+async def test_amoderation_calls_the_host_directly_when_the_model_is_not_a_group(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await core.amoderation(model="omni-moderation-latest", input="x")
+    call: Final = host.calls[0]
+    assert (call.op, call.model, call.deployment_id) == ("amoderation", "omni-moderation-latest", None)
+    assert _bucket(call.kwargs)["model_group"] == "omni-moderation-latest"
+
+
+async def test_amoderation_deployment_params_overwrite_the_request_params(make_core: MakeCore, host: EndpointHost):
+    """The deployment's litellm_params are applied after the request's (kwargs.update(data))."""
+    core: Final = make_core(model_list=[deployment("g", "d1", temperature=0.5)])
+    await core.amoderation(model="g", input="x", temperature=1.0)
+    assert _sent(host)["temperature"] == 0.5
+
+
+async def test_amoderation_with_a_request_api_key_keeps_the_deployment_key_under_a_request_scoped_id(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await core.amoderation(model="g", input="x", api_key="sk-request")
+    sent: Final = _sent(host)
+    model_info: Final = _bucket(sent, "model_info")
+    assert sent["api_key"] == "sk-test"
+    assert (model_info["original_model_id"], model_info["id"] != "d1") == ("d1", True)
+
+
+async def test_amoderation_is_not_retried(make_core: MakeCore, host: EndpointHost):
+    host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
+    core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=3)
+    with pytest.raises(litellm.InternalServerError):
+        await core.amoderation(model="g", input="x")
+    assert len(host.calls) == 1
+    assert host.sleeps == []
+
+
+async def test_arerank_sends_the_builtin_input_function_as_input(make_core: MakeCore, host: EndpointHost):
+    """arerank sets kwargs["input"] = input, the Python builtin, and it reaches the host."""
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await core.arerank(model="g", query="q", documents=["a"])
+    sent: Final = _sent(host)
+    assert (sent["input"], sent["query"], sent["documents"]) == (input, "q", ["a"])
+
+
+@pytest.mark.parametrize(("request_voice", "expected"), [(None, "deployment-voice"), ("alloy", "alloy")])
+async def test_aspeech_voice_defaults_to_the_deployment_voice(
+    make_core: MakeCore, host: EndpointHost, request_voice: str | None, expected: str
+):
+    core: Final = make_core(model_list=[deployment("g", "d1", voice="deployment-voice")])
+    await core.aspeech(model="g", input="hello", voice=request_voice)
+    assert (_sent(host)["voice"], _sent(host)["input"]) == (expected, "hello")
+
+
+async def test_aspeech_sends_no_caching_flag(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")], cache_responses=True)
+    await core.aspeech(model="g", input="hello", voice="alloy")
+    assert "caching" not in _sent(host)
+
+
+async def test_atranscription_forwards_the_same_file_object(make_core: MakeCore, host: EndpointHost):
+    audio: Final = io.BytesIO(b"audio")
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await core.atranscription(model="g", file=audio)
+    assert _sent(host)["file"] is audio
+
+
+@pytest.mark.parametrize(
+    "endpoint",
+    ["acompletion", "atext_completion", "aembedding", "aimage_generation", "atranscription", "arerank", "completion"],
+)
+async def test_cache_responses_reaches_the_host_as_caching(make_core: MakeCore, host: EndpointHost, endpoint: str):
+    core: Final = make_core(model_list=[deployment("g", "d1")], cache_responses=True)
+    requests: Final[dict[str, dict[str, object]]] = {
+        "acompletion": {"messages": MESSAGES},
+        "completion": {"messages": MESSAGES},
+        "atext_completion": {"prompt": "hi"},
+        "aembedding": {"input": ["a"]},
+        "aimage_generation": {"prompt": "a cat"},
+        "atranscription": {"file": io.BytesIO(b"x")},
+        "arerank": {"query": "q", "documents": ["a"]},
+    }
+    await core.call(endpoint, {"model": "g", **requests[endpoint]})
+    assert _sent(host)["caching"] is True
+
+
+@pytest.mark.parametrize("endpoint", ["acompletion", "completion"])
+async def test_cache_responses_turned_on_after_construction_reaches_the_host(
+    make_core: MakeCore, host: EndpointHost, endpoint: str
+):
+    """The proxy turns ``cache_responses`` on through ``apply`` once its cache exists."""
+    from litellm.router_api import ChangeSet
+
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    _ = core.apply(ChangeSet(assign={"cache_responses": True}))
+    await core.call(endpoint, {"model": "g", "messages": MESSAGES})
+    assert core.settings().cache_responses is True
+    assert _sent(host)["caching"] is True
+
+
+# --- sync text completion and image generation ----------------------------------------------
+
+
+async def test_sync_text_completion_sends_the_router_num_retries_and_no_routing_fields(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=4)
+    core.text_completion(model="g", prompt="p")
+    sent: Final = _sent(host)
+    assert sent["num_retries"] == 4
+    assert "model_info" not in sent
+    assert "litellm_trace_id" not in sent
+    assert sent.get("timeout") is None
+    metadata: Final = _bucket(sent)
+    assert metadata["model_group"] == "g"
+    assert "deployment" not in metadata
+
+
+async def test_sync_text_completion_request_num_retries_wins(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=4)
+    core.text_completion(model="g", prompt="p", num_retries=1)
+    assert _sent(host)["num_retries"] == 1
+
+
+async def test_sync_text_completion_merges_default_metadata_into_request_metadata(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(
+        model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"x": 1}, "user": "u"}
+    )
+    core.text_completion(model="g", prompt="p", metadata={"y": 2})
+    sent: Final = _sent(host)
+    metadata: Final = _bucket(sent)
+    assert (metadata["x"], metadata["y"], metadata["model_group"], sent["user"]) == (1, 2, "g", "u")
+
+
+async def test_sync_text_completion_sends_the_model_group_name_not_the_deployment_model(
+    make_core: MakeCore, host: EndpointHost
+):
+    """The request's model="g" is spread after the deployment params, so it replaces "openai/d1"."""
+    core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")])
+    core.text_completion(model="g", prompt="p")
+    call: Final = host.calls[0]
+    assert (call.model, call.kwargs["api_base"]) == ("g", "https://d1.example")
+
+
+async def test_sync_text_completion_is_not_retried(make_core: MakeCore, host: EndpointHost):
+    host.answer("g", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
+    core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=3)
+    with pytest.raises(litellm.InternalServerError):
+        core.text_completion(model="g", prompt="p")
+    assert len(host.calls) == 1
+    assert host.sleeps == []
+
+
+async def test_sync_image_generation_sends_routing_metadata_without_a_trace_id(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    core.image_generation(model="g", prompt="p")
+    sent: Final = _sent(host)
+    assert "litellm_trace_id" not in sent
+    assert (_bucket(sent)["model_group"], _bucket(sent)["deployment"], sent["prompt"]) == ("g", "openai/d1", "p")
+
+
+@pytest.mark.parametrize("endpoint", ["acompletion", "atext_completion", "aembedding", "aresponses"])
+async def test_async_endpoints_generate_a_trace_id_unless_the_request_sends_one(
+    make_core: MakeCore, host: EndpointHost, endpoint: str
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    request: Final[dict[str, object]] = (
+        {"messages": MESSAGES}
+        if endpoint == "acompletion"
+        else {"prompt": "p"}
+        if endpoint == "atext_completion"
+        else {"input": ["a"]}
+        if endpoint == "aembedding"
+        else {"input": "hi"}
+    )
+    await core.call(endpoint, {"model": "g", **request})
+    await core.call(endpoint, {"model": "g", "litellm_trace_id": "trace-1", **request})
+    generated: Final = _sent(host, 0)["litellm_trace_id"]
+    assert isinstance(generated, str) and len(generated) == 36
+    assert _sent(host, 1)["litellm_trace_id"] == "trace-1"
+
+
+# --- routing metadata ---------------------------------------------------------------------
+
+
+async def test_a_model_group_alias_is_recorded_as_the_model_group_and_its_alias(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")], model_group_alias={"al": "g"})
+    await core.acompletion(model="al", messages=MESSAGES)
+    metadata: Final = _bucket(_sent(host))
+    assert (metadata["model_group"], metadata["model_group_alias"], metadata["deployment_model_name"]) == (
+        "al",
+        "al",
+        "g",
+    )
+
+
+async def test_a_request_for_a_deployment_id_routes_to_that_deployment(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1"), deployment("g", "d2")])
+    for _ in range(4):
+        await core.acompletion(model="d2", messages=MESSAGES)
+    assert host.deployments_called() == ["d2"] * 4
+    assert _bucket(_sent(host))["model_group"] == "d2"
+
+
+async def test_deployment_tags_and_credential_name_join_the_request_tags(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1", tags=["t1"], litellm_credential_name="cred")])
+    await core.acompletion(model="g", messages=MESSAGES, metadata={"tags": ["r1"]})
+    sent: Final = _sent(host)
+    assert _bucket(sent)["tags"] == ["r1", "t1", "Credential: cred"]
+    assert sent["litellm_credential_name"] == "cred"
+
+
+async def test_daily_reset_time_is_not_sent_to_the_host(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await core.acompletion(model="g", messages=MESSAGES, daily_reset_time="00:00")
+    assert "daily_reset_time" not in _sent(host)
+
+
+async def test_deployment_model_id_param_is_sent_as_model_id(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1", model_id="arn:pinned")])
+    await core.acompletion(model="g", messages=MESSAGES)
+    core.text_completion(model="g", prompt="p")
+    assert [call.kwargs["model_id"] for call in host.calls] == ["arn:pinned", "arn:pinned"]
+
+
+async def test_deployment_model_id_reaches_the_host_even_when_clientside_auth_allows_model_id(
+    make_core: MakeCore, host: EndpointHost
+):
+    """The pin is skipped, but the deployment's litellm_params (with model_id) are still spread into the call."""
+    core: Final = make_core(
+        model_list=[deployment("g", "d1", model_id="arn:pinned", configurable_clientside_auth_params=["model_id"])]
+    )
+    await core.acompletion(model="g", messages=MESSAGES)
+    assert _sent(host)["model_id"] == "arn:pinned"
+
+
+async def test_request_model_id_wins_over_a_clientside_configurable_deployment_model_id(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(
+        model_list=[deployment("g", "d1", model_id="arn:pinned", configurable_clientside_auth_params=["model_id"])]
+    )
+    await core.acompletion(model="g", messages=MESSAGES, model_id="arn:request")
+    assert _sent(host)["model_id"] == "arn:request"
+
+
+# --- default litellm params -------------------------------------------------------------------
+
+
+async def test_default_litellm_params_metadata_merges_into_request_metadata(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"team": "t"}})
+    await core.acompletion(model="g", messages=MESSAGES, metadata={"trace": "x"})
+    metadata: Final = _bucket(_sent(host))
+    assert (metadata["team"], metadata["trace"], metadata["model_group"]) == ("t", "x", "g")
+
+
+async def test_default_litellm_params_metadata_overwrites_the_same_request_metadata_key(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"team": "t"}})
+    await core.acompletion(model="g", messages=MESSAGES, metadata={"team": "request"})
+    assert _bucket(_sent(host))["team"] == "t"
+
+
+async def test_default_litellm_params_metadata_goes_to_litellm_metadata_for_generic_endpoints(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"team": "t"}})
+    await core.aresponses(model="g", input="hi")
+    sent: Final = _sent(host)
+    assert _bucket(sent, "litellm_metadata")["team"] == "t"
+    assert "metadata" not in sent
+
+
+async def test_default_litellm_params_with_none_values_are_not_sent(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params={"user": None})
+    await core.acompletion(model="g", messages=MESSAGES)
+    assert "user" not in _sent(host)
+
+
+# --- tools ----------------------------------------------------------------------------------
+
+DEPLOYMENT_TOOL: Final = {"type": "function", "function": {"name": "dep", "parameters": {}}}
+REQUEST_TOOL: Final = {"type": "function", "function": {"name": "req", "parameters": {}}}
+
+
+async def test_deployment_tools_come_before_request_tools(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1", tools=[DEPLOYMENT_TOOL])])
+    await core.acompletion(model="g", messages=MESSAGES, tools=[REQUEST_TOOL])
+    assert _sent(host)["tools"] == [DEPLOYMENT_TOOL, REQUEST_TOOL]
+
+
+async def test_deployment_tools_are_sent_when_the_request_has_none(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1", tools=[DEPLOYMENT_TOOL])])
+    await core.acompletion(model="g", messages=MESSAGES)
+    assert _sent(host)["tools"] == [DEPLOYMENT_TOOL]
+
+
+@pytest.mark.parametrize(("request_choice", "expected"), [(None, "required"), ("none", "none")])
+async def test_tool_choice_comes_from_the_request_else_the_deployment(
+    make_core: MakeCore, host: EndpointHost, request_choice: str | None, expected: str
+):
+    core: Final = make_core(model_list=[deployment("g", "d1", tools=[DEPLOYMENT_TOOL], tool_choice="required")])
+    extra: Final = {} if request_choice is None else {"tool_choice": request_choice}
+    await core.acompletion(model="g", messages=MESSAGES, **extra)
+    assert _sent(host)["tool_choice"] == expected
+
+
+async def test_merging_tools_does_not_change_the_deployment_tools(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1", tools=[DEPLOYMENT_TOOL])])
+    await core.acompletion(model="g", messages=MESSAGES, tools=[REQUEST_TOOL])
+    await core.acompletion(model="g", messages=MESSAGES)
+    assert _sent(host)["tools"] == [DEPLOYMENT_TOOL]
+
+
+# --- reasoning effort -----------------------------------------------------------------------
+
+
+async def test_request_reasoning_effort_drops_the_deployment_thinking_param(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1", thinking={"type": "enabled", "budget_tokens": 100})])
+    await core.acompletion(model="g", messages=MESSAGES, reasoning_effort="low")
+    sent: Final = _sent(host)
+    assert (sent["reasoning_effort"], "thinking" in sent) == ("low", False)
+
+
+async def test_deployment_thinking_is_kept_without_a_request_reasoning_effort(make_core: MakeCore, host: EndpointHost):
+    thinking: Final = {"type": "enabled", "budget_tokens": 100}
+    core: Final = make_core(model_list=[deployment("g", "d1", thinking=thinking)])
+    await core.acompletion(model="g", messages=MESSAGES)
+    assert _sent(host)["thinking"] == thinking
+
+
+async def test_request_reasoning_effort_strips_effort_from_deployment_extra_body(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(
+        model_list=[
+            deployment(
+                "g",
+                "d1",
+                extra_body={"reasoning_effort": "high", "thinking": {"type": "enabled"}, "other": 1},
+                reasoning={"effort": "high", "summary": "auto"},
+            )
+        ]
+    )
+    await core.acompletion(model="g", messages=MESSAGES, reasoning_effort="low")
+    sent: Final = _sent(host)
+    assert sent["extra_body"] == {"other": 1}
+    assert sent["reasoning"] == {"summary": "auto"}
+
+
+async def test_request_reasoning_effort_drops_an_extra_body_left_empty(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1", extra_body={"reasoning_effort": "high"})])
+    await core.acompletion(model="g", messages=MESSAGES, reasoning_effort="low")
+    assert "extra_body" not in _sent(host)
+
+
+async def test_reasoning_override_does_not_change_the_deployment_config(make_core: MakeCore, host: EndpointHost):
+    thinking: Final = {"type": "enabled", "budget_tokens": 100}
+    core: Final = make_core(model_list=[deployment("g", "d1", thinking=thinking)])
+    await core.acompletion(model="g", messages=MESSAGES, reasoning_effort="low")
+    await core.acompletion(model="g", messages=MESSAGES)
+    assert _sent(host)["thinking"] == thinking
+
+
+CLASSIFIER_METADATA: Final = {"internal_call_origin": "autorouter_classifier"}
+
+
+async def test_classifier_call_drops_reasoning_effort_the_deployment_does_not_support(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1", model="openai/gpt-3.5-turbo")])
+    body: Final[dict[str, object]] = {"reasoning_effort": "low", "keep": 1}
+    await core.acompletion(
+        model="g",
+        messages=MESSAGES,
+        reasoning_effort="low",
+        metadata=dict(CLASSIFIER_METADATA),
+        proxy_server_request={"body": body},
+    )
+    sent: Final = _sent(host)
+    assert "reasoning_effort" not in sent
+    assert sent["proxy_server_request"] == {"body": {"keep": 1}}
+
+
+async def test_non_classifier_call_keeps_reasoning_effort_on_a_deployment_without_support(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1", model="openai/gpt-3.5-turbo")])
+    await core.acompletion(model="g", messages=MESSAGES, reasoning_effort="low")
+    assert _sent(host)["reasoning_effort"] == "low"
+
+
+async def test_classifier_call_keeps_reasoning_effort_on_a_deployment_with_support(
+    make_core: MakeCore, host: EndpointHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1", model="openai/o3-mini")])
+    await core.acompletion(
+        model="g", messages=MESSAGES, reasoning_effort="low", litellm_metadata=dict(CLASSIFIER_METADATA)
+    )
+    assert _sent(host)["reasoning_effort"] == "low"
+
+
+# --- silent experiments ---------------------------------------------------------------------
+
+
+async def _settle() -> None:
+    for _ in range(20):
+        await asyncio.sleep(0)
+
+
+async def test_silent_model_mirrors_acompletion_to_the_silent_group(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")])
+    await core.acompletion(model="g", messages=MESSAGES, metadata={"trace": "t"})
+    await _settle()
+    assert sorted(host.deployments_called("acompletion")) == ["d1", "s1"]
+    primary: Final = next(call for call in host.calls if call.deployment_id == "d1")
+    shadow: Final = next(call for call in host.calls if call.deployment_id == "s1")
+    assert "silent_model" not in primary.kwargs
+    assert "is_silent_experiment" not in _bucket(primary.kwargs)
+    shadow_metadata: Final = _bucket(shadow.kwargs)
+    assert (shadow_metadata["is_silent_experiment"], shadow_metadata["model_group"], shadow_metadata["trace"]) == (
+        True,
+        "shadow",
+        "t",
+    )
+
+
+async def test_silent_model_list_mirrors_to_every_listed_group(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(
+        model_list=[
+            deployment("g", "d1", silent_model=["shadow-a", "shadow-b"]),
+            deployment("shadow-a", "sa"),
+            deployment("shadow-b", "sb"),
+        ]
+    )
+    await core.acompletion(model="g", messages=MESSAGES)
+    await _settle()
+    assert sorted(host.deployments_called("acompletion")) == ["d1", "sa", "sb"]
+
+
+async def test_a_failing_silent_experiment_does_not_fail_the_primary_call(make_core: MakeCore, host: EndpointHost):
+    host.answer("openai/s1", litellm.BadRequestError(message="shadow broke", llm_provider="openai", model="s1"))
+    core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")])
+    response: Final = await core.acompletion(model="g", messages=MESSAGES)
+    await _settle()
+    assert response is not None
+    assert "s1" in host.deployments_called("acompletion")
+
+
+async def test_a_malformed_silent_model_is_ignored(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1", silent_model=5)])
+    await core.acompletion(model="g", messages=MESSAGES)
+    await _settle()
+    assert host.deployments_called() == ["d1"]
+    assert "silent_model" not in _sent(host)
+
+
+# --- timeouts -------------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(("stream", "expected"), [(False, (99.0, None)), (True, (99.0, 33.0))])
+async def test_service_tier_timeouts_come_from_the_deployment_tier_maps(
+    make_core: MakeCore, host: EndpointHost, stream: bool, expected: tuple[float, float | None]
+):
+    core: Final = make_core(
+        model_list=[
+            deployment(
+                "g", "d1", timeout=11, timeout_by_service_tier={"flex": 99}, stream_timeout_by_service_tier={"flex": 33}
+            )
+        ]
+    )
+    await core.acompletion(model="g", messages=MESSAGES, service_tier="flex", stream=stream)
+    sent: Final = _sent(host)
+    assert (sent["timeout"], sent.get("stream_timeout")) == expected
+
+
+async def test_router_stream_timeout_applies_to_streaming_acompletion(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1", timeout=11)], stream_timeout=5)
+    await core.acompletion(model="g", messages=MESSAGES, stream=True)
+    sent: Final = _sent(host)
+    assert (sent["timeout"], sent["stream_timeout"]) == (11.0, 5.0)
+
+
+async def test_non_streaming_acompletion_sends_no_stream_timeout(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1", stream_timeout=3)])
+    await core.acompletion(model="g", messages=MESSAGES)
+    assert "stream_timeout" not in _sent(host) or _sent(host)["stream_timeout"] == 3
+
+
+async def test_aembedding_uses_the_request_timeout_over_the_deployment_timeout(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1", timeout=11)])
+    await core.aembedding(model="g", input=["a"], request_timeout=4)
+    assert _sent(host)["timeout"] == 4
+
+
+async def test_timeout_message_carries_deployment_timeouts_when_router_debug_is_exposed(
+    make_core: MakeCore, host: EndpointHost
+):
+    host.set_dynamic(expose_router_debug_in_errors=True)
+    host.answer("openai/d1", timeout("d1"))
+    core: Final = make_core(model_list=[deployment("g", "d1", timeout=11, stream_timeout=3)], num_retries=0)
+    with pytest.raises(litellm.Timeout) as raised:
+        await core.acompletion(model="g", messages=MESSAGES)
+    assert "Deployment Info: request_timeout: None\ntimeout: 11.0\nstream_timeout: 3.0" in str(raised.value)
+
+
+async def test_timeout_message_has_no_deployment_info_by_default(make_core: MakeCore, host: EndpointHost):
+    host.answer("openai/d1", timeout("d1"))
+    core: Final = make_core(model_list=[deployment("g", "d1", timeout=11)], num_retries=0)
+    with pytest.raises(litellm.Timeout) as raised:
+        await core.acompletion(model="g", messages=MESSAGES)
+    assert "Deployment Info" not in str(raised.value)
+
+
+# --- clientside credentials -------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("credential", [{"api_key": "sk-client"}, {"api_base": "https://client.example"}])
+async def test_clientside_credentials_get_a_request_scoped_deployment_id(
+    make_core: MakeCore, host: EndpointHost, credential: dict[str, str]
+):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await core.acompletion(model="g", messages=MESSAGES, **credential)
+    sent: Final = _sent(host)
+    model_info: Final = _bucket(sent, "model_info")
+    assert model_info["original_model_id"] == "d1"
+    assert isinstance(model_info["id"], str) and len(model_info["id"]) == 64
+    assert _bucket(_bucket(sent), "model_info")["id"] == model_info["id"]
+
+
+async def test_clientside_credential_id_is_stable_for_the_same_credentials(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await core.acompletion(model="g", messages=MESSAGES, api_key="sk-a")
+    await core.acompletion(model="g", messages=MESSAGES, api_key="sk-a")
+    await core.acompletion(model="g", messages=MESSAGES, api_key="sk-b")
+    ids: Final = [call.deployment_id for call in host.calls]
+    assert ids[0] == ids[1] != ids[2]
+
+
+async def test_clientside_credentials_do_not_register_a_deployment(make_core: MakeCore, host: EndpointHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await core.acompletion(model="g", messages=MESSAGES, api_key="sk-client")
+    await core.acompletion(model="g", messages=MESSAGES)
+    assert host.deployments_called()[-1] == "d1"
+    assert [d["model_info"]["id"] for d in core.registry().deployments("g")] == ["d1"]  # pyright: ignore[reportIndexIssue, reportUnknownMemberType]  # registry rows are deployment dicts
+
+
+# --- embedding batch size --------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("endpoint", ["aembedding", "embedding"])
+async def test_embedding_over_the_deployment_batch_size_raises_without_calling_the_host(
+    make_core: MakeCore, host: EndpointHost, endpoint: str
+):
+    core: Final = make_core(
+        model_list=[deployment("g", "d1", model_info={"max_embedding_batch_size": 2})], num_retries=0
+    )
+    with pytest.raises(
+        litellm.BadRequestError,
+        match="Embedding request has 3 inputs, which exceeds max_embedding_batch_size=2 for model=g",
+    ):
+        await core.call(endpoint, {"model": "g", "input": ["a", "b", "c"]})
+    assert host.calls == []
+
+
+@pytest.mark.parametrize("endpoint", ["aembedding", "embedding"])
+async def test_embedding_within_the_deployment_batch_size_is_sent(
+    make_core: MakeCore, host: EndpointHost, endpoint: str
+):
+    core: Final = make_core(model_list=[deployment("g", "d1", model_info={"max_embedding_batch_size": 2})])
+    await core.call(endpoint, {"model": "g", "input": ["a", "b"]})
+    assert host.calls[0].kwargs["input"] == ["a", "b"]

tests/unit/test_router/core/test_fallbacks_characterization.py

@@ -0,0 +1,584 @@
+"""
+Characterization of a router core's fallbacks (non-streaming): chain sources and formats,
+client-side fallback lists, per-fallback params, validation, mock-testing flags, depth bounds,
+default fallbacks and wildcard lookup. Complements ``test_fallbacks.py``.
+"""
+
+import pytest
+
+import litellm
+from tests.unit.test_router.core.fallbacks_helpers import (
+    TIERED_A,
+    MakeCore,
+    build,
+    complete,
+    complete_sync,
+    headers,
+    show_router_debug,
+)
+from tests.unit.test_router.core.scripted_host import (
+    ScriptedHost,
+    content_policy,
+    context_window,
+    deployment,
+    rate_limit,
+)
+
+# --- validation of the configured chains ---------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("field", "value", "error", "message"),
+    [
+        ("fallbacks", ["b"], ValueError, "Item 'b' is not a dictionary."),
+        (
+            "fallbacks",
+            [{"a": ["b"], "c": ["b"]}],
+            ValueError,
+            "Dictionary '{'a': ['b'], 'c': ['b']}' must have exactly one key, but has 2 keys.",
+        ),
+        (
+            "fallbacks",
+            [{"a": "b"}],
+            TypeError,
+            "Fallbacks for 'a' must be a list of model names, got 'b'. Use {'a': ['b']}",
+        ),
+        (
+            "content_policy_fallbacks",
+            [{"a": "b"}],
+            TypeError,
+            "Fallbacks for 'a' must be a list of model names, got 'b'. Use {'a': ['b']}",
+        ),
+    ],
+    ids=["plain-string-item", "two-keys", "string-target", "content-policy-string-target"],
+)
+def test_a_malformed_fallbacks_configuration_is_refused_at_construction(
+    make_core: MakeCore, field: str, value: object, error: type[Exception], message: str
+) -> None:
+    with pytest.raises(error) as raised:
+        build(make_core, **{field: value})
+
+    assert str(raised.value) == message
+
+
+@pytest.mark.asyncio
+async def test_context_window_fallbacks_are_not_validated_at_construction(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, context_window_fallbacks=[{"a": "b"}])
+
+    await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a"]
+
+
+@pytest.mark.asyncio
+async def test_a_constructor_chain_of_parameter_dicts_passes_validation_and_applies_to_any_group(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"model": "b"}])
+    host.answer("openai/a", rate_limit())
+
+    await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "b"]
+
+
+@pytest.mark.asyncio
+async def test_an_empty_constructor_chain_defers_to_the_process_setting(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    host.set_dynamic(fallbacks=[{"a": ["c"]}])
+    core = build(make_core, fallbacks=[])
+    host.answer("openai/a", rate_limit())
+
+    await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "c"]
+
+
+# --- client-side fallback lists ------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_a_plain_request_list_is_walked_in_order(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core)
+    host.answer("openai/a", rate_limit())
+    host.answer("openai/b", rate_limit())
+
+    response = await complete(core, fallbacks=["b", "c"])
+
+    assert host.deployments_called("acompletion") == ["a", "b", "c"]
+    assert headers(response)["x-litellm-model-group"] == "c"
+
+
+@pytest.mark.asyncio
+async def test_a_plain_request_list_overrides_the_context_window_chain(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, context_window_fallbacks=[{"a": ["c"]}])
+    host.answer("openai/a", context_window())
+
+    await complete(core, fallbacks=["b"])
+
+    assert host.deployments_called("acompletion") == ["a", "b"]
+
+
+@pytest.mark.asyncio
+async def test_a_dict_target_with_params_overrides_the_request_for_that_attempt(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core)
+    host.answer("openai/a", rate_limit())
+    override = [{"role": "user", "content": "shorter"}]
+
+    await complete(core, fallbacks=[{"model": "b", "temperature": 0.3, "messages": override}])
+
+    primary, fallback = host.calls_to("acompletion")
+    assert (primary.deployment_id, primary.kwargs.get("temperature")) == ("a", None)
+    assert primary.kwargs["messages"] == [{"role": "user", "content": "hi"}]
+    assert (fallback.deployment_id, fallback.kwargs["temperature"], fallback.kwargs["messages"]) == ("b", 0.3, override)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "arguments",
+    [
+        {"request": [{"model": "b", "temperature": 0.3}, {"model": "c"}]},
+        {"constructor": [{"a": [{"model": "b", "temperature": 0.3}, "c"]}]},
+    ],
+    ids=["request-list", "constructor-chain"],
+)
+async def test_a_dict_targets_params_carry_over_to_the_next_target(
+    make_core: MakeCore, host: ScriptedHost, arguments: dict[str, list[object]]
+) -> None:
+    core = build(make_core, **({"fallbacks": arguments["constructor"]} if "constructor" in arguments else {}))
+    host.answer("openai/a", rate_limit())
+    host.answer("openai/b", rate_limit())
+
+    await complete(core, **({"fallbacks": arguments["request"]} if "request" in arguments else {}))
+
+    assert [(c.deployment_id, c.kwargs.get("temperature")) for c in host.calls_to("acompletion")] == [
+        ("a", None),
+        ("b", 0.3),
+        ("c", 0.3),
+    ]
+
+
+@pytest.mark.asyncio
+async def test_a_dict_target_naming_the_failed_group_with_params_retries_that_group(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core)
+    host.answer("openai/a", rate_limit(), None)
+
+    response = await complete(core, fallbacks=[{"model": "a", "temperature": 0}])
+
+    assert [(c.deployment_id, c.kwargs.get("temperature")) for c in host.calls_to("acompletion")] == [
+        ("a", None),
+        ("a", 0),
+    ]
+    assert headers(response)["x-litellm-attempted-fallbacks"] == 1
+
+
+@pytest.mark.asyncio
+async def test_a_dict_target_with_its_own_api_key_runs_on_a_client_credential_deployment(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core)
+    host.answer("openai/a", rate_limit())
+
+    response = await complete(core, fallbacks=[{"model": "b", "api_key": "sk-other"}])
+
+    primary, fallback = host.calls_to("acompletion")
+    assert (fallback.model, fallback.kwargs["api_key"]) == ("openai/b", "sk-other")
+    assert fallback.deployment_id not in {None, "b"}
+    assert headers(response)["x-litellm-model-group"] == "b"
+    assert primary.deployment_id == "a"
+
+
+@pytest.mark.asyncio
+async def test_a_dict_target_carrying_an_oidc_api_key_is_refused_and_the_primary_error_raised(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core)
+    show_router_debug(host)
+    host.answer("openai/a", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError) as raised:
+        await complete(core, fallbacks=[{"model": "b", "api_key": "oidc/google/x"}])
+
+    assert host.deployments_called("acompletion") == ["a"]
+    assert str(raised.value) == (
+        "litellm.RateLimitError: slow down\n\nLiteLLM: model group 'a' failed with the error above. "
+        "Fallback to {'model': 'b', 'api_key': 'oidc*********'} also failed: Rejected Request: api_key holds an "
+        "oidc/ secret reference, which only a deployment or credential a proxy admin configures can carry."
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("fallbacks", "expected"),
+    [(["c", {"a": ["b"]}], ["a", "b", "c"]), (["b", {"x": ["c"]}], ["a", "b"])],
+    ids=["keyed-entry-wins-then-string-catches-the-next-hop", "string-entry-is-a-chain-for-every-group"],
+)
+async def test_a_mixed_list_of_names_and_keyed_entries_is_resolved_as_a_chain_map(
+    make_core: MakeCore, host: ScriptedHost, fallbacks: list[object], expected: list[str]
+) -> None:
+    core = build(make_core)
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError):
+        await complete(core, fallbacks=fallbacks)
+
+    assert host.deployments_called("acompletion") == expected
+
+
+@pytest.mark.asyncio
+async def test_the_request_fallbacks_kwarg_is_not_forwarded_to_the_provider(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core)
+    host.answer("openai/a", rate_limit())
+
+    await complete(core, fallbacks=["b"])
+
+    assert [("fallbacks" in c.kwargs) for c in host.calls_to("acompletion")] == [False, False]
+
+
+# --- disable_fallbacks ------------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_disable_fallbacks_given_as_a_string_does_not_disable_them(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    host.answer("openai/a", rate_limit())
+
+    await complete(core, disable_fallbacks="true")
+
+    assert host.deployments_called("acompletion") == ["a", "b"]
+
+
+@pytest.mark.asyncio
+async def test_disable_fallbacks_also_stops_the_order_tier_walk(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, model_list=TIERED_A)
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError):
+        await complete(core, disable_fallbacks=True)
+
+    assert host.deployments_called("acompletion") == ["a1"]
+
+
+@pytest.mark.asyncio
+async def test_disable_fallbacks_is_not_forwarded_to_the_provider(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core)
+
+    await complete(core, disable_fallbacks=True)
+
+    assert "disable_fallbacks" not in host.calls_to("acompletion")[0].kwargs
+
+
+# --- depth bounds -----------------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_a_request_max_fallbacks_overrides_the_routers(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}, {"b": ["c"]}])
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError):
+        await complete(core, max_fallbacks=1)
+
+    assert host.deployments_called("acompletion") == ["a", "b"]
+
+
+@pytest.mark.asyncio
+async def test_max_fallbacks_zero_also_stops_the_order_tier_walk(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, model_list=TIERED_A, max_fallbacks=0)
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError):
+        await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a1"]
+
+
+@pytest.mark.asyncio
+async def test_max_fallbacks_one_still_tries_every_target_of_the_first_level(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b", "c"]}, {"b": ["c"]}], max_fallbacks=1)
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError):
+        await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "b", "c"]
+
+
+@pytest.mark.asyncio
+async def test_a_fallback_group_spends_its_own_retries(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}], num_retries=1)
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError) as raised:
+        await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "a", "b", "b"]
+    assert str(raised.value) == "litellm.RateLimitError: slow down LiteLLM Retried: 1 times, LiteLLM Max Retries: 1"
+
+
+@pytest.mark.asyncio
+async def test_a_fallback_hop_resumes_the_original_groups_chain(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b", "c"]}], max_fallbacks=5)
+    host.answer("openai/a", rate_limit())
+    host.answer("openai/b", rate_limit())
+
+    response = await complete(core, include_fallback_errors=True)
+
+    assert host.deployments_called("acompletion") == ["a", "b", "c"]
+    # c answered inside b's hop (b resumed a's chain), so b's error is listed before a's.
+    assert headers(response)["x-litellm-fallback-errors"] == (
+        '[{"message": "litellm.RateLimitError: slow down", "type": "throttling_error", "param": null, "code": "429"}, '
+        '{"message": "litellm.RateLimitError: slow down", "type": "throttling_error", "param": null, "code": "429"}]'
+    )
+
+
+# --- order tiers with other chains ---------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "arguments",
+    [{"request": {"fallbacks": ["c"]}}, {"constructor": {"default_fallbacks": ["c"]}}],
+    ids=["plain-request-list", "default-fallbacks"],
+)
+async def test_order_tiers_run_before_a_plain_list_or_the_default_chain(
+    make_core: MakeCore, host: ScriptedHost, arguments: dict[str, dict[str, object]]
+) -> None:
+    core = build(make_core, model_list=TIERED_A, **arguments.get("constructor", {}))
+    host.answer("*", rate_limit())
+    host.answer("openai/c", None)
+
+    response = await complete(core, **arguments.get("request", {}))
+
+    assert host.deployments_called("acompletion") == ["a1", "a2", "c"]
+    assert headers(response)["x-litellm-model-group"] == "c"
+
+
+# --- default fallbacks ---------------------------------------------------------------------------
+
+
+def test_default_fallbacks_are_appended_to_the_process_fallbacks_list_in_place(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    process_fallbacks: list[object] = [{"a": ["b"]}]
+    host.set_dynamic(fallbacks=process_fallbacks)
+
+    core = build(make_core, default_fallbacks=["c"])
+
+    assert core.settings().fallbacks == [{"a": ["b"]}, {"*": ["c"]}]
+    assert process_fallbacks == [{"a": ["b"]}, {"*": ["c"]}]
+
+
+def test_default_fallbacks_leave_the_constructor_fallbacks_list_unchanged(make_core: MakeCore) -> None:
+    fallbacks: list[object] = [{"a": ["b"]}]
+
+    core = build(make_core, fallbacks=fallbacks, default_fallbacks=["c"])
+
+    assert core.settings().fallbacks == [{"a": ["b"]}, {"*": ["c"]}]
+    assert fallbacks == [{"a": ["b"]}]
+
+
+@pytest.mark.asyncio
+async def test_an_unknown_group_is_served_by_the_first_default_fallback_without_a_fallback_hop(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, default_fallbacks=["b", "c"])
+
+    response = await complete(core, model="unknown")
+
+    assert host.deployments_called("acompletion") == ["b"]
+    assert headers(response)["x-litellm-attempted-fallbacks"] == 0
+    assert headers(response)["x-litellm-model-group"] == "unknown"
+    assert host.calls_to("acompletion")[0].model_group == "unknown"
+
+
+@pytest.mark.asyncio
+async def test_an_unknown_group_with_its_own_chain_falls_back_through_it(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"unknown": ["b"]}])
+
+    response = await complete(core, model="unknown")
+
+    assert host.deployments_called("acompletion") == ["b"]
+    assert headers(response)["x-litellm-attempted-fallbacks"] == 1
+    assert headers(response)["x-litellm-model-group"] == "b"
+
+
+@pytest.mark.asyncio
+async def test_default_fallbacks_catch_a_content_policy_error_without_a_content_policy_chain(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, default_fallbacks=["c"])
+    host.answer("openai/a", content_policy())
+
+    await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "c"]
+
+
+@pytest.mark.asyncio
+async def test_a_request_context_window_fallbacks_of_none_falls_through_to_the_general_chain(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, context_window_fallbacks=[{"a": ["c"]}], fallbacks=[{"a": ["b"]}])
+    host.answer("openai/a", context_window())
+
+    await complete(core, context_window_fallbacks=None)
+
+    assert host.deployments_called("acompletion") == ["a", "b"]
+
+
+# --- wildcard groups -----------------------------------------------------------------------------
+
+WILDCARD = [deployment("openai/*", "w", model="openai/*"), deployment("b", "b")]
+
+
+@pytest.mark.asyncio
+async def test_a_wildcard_routed_request_uses_the_chain_keyed_by_the_bare_model_name(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, model_list=WILDCARD, fallbacks=[{"gpt-x": ["b"]}])
+    host.answer("openai/gpt-x", rate_limit())
+
+    response = await complete(core, model="openai/gpt-x")
+
+    assert host.deployments_called("acompletion") == ["w", "b"]
+    assert [c.model for c in host.calls_to("acompletion")] == ["openai/gpt-x", "openai/b"]
+    assert headers(response)["x-litellm-model-group"] == "b"
+
+
+@pytest.mark.asyncio
+async def test_a_chain_keyed_by_the_wildcard_pattern_does_not_match_a_routed_request(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, model_list=WILDCARD, fallbacks=[{"openai/*": ["b"]}])
+    host.answer("openai/gpt-x", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError):
+        await complete(core, model="openai/gpt-x")
+
+    assert host.deployments_called("acompletion") == ["w"]
+
+
+# --- mock testing flags ------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("flag", "field", "expected"),
+    [
+        ("mock_testing_fallbacks", "fallbacks", "b"),
+        ("mock_testing_context_fallbacks", "context_window_fallbacks", "c"),
+        ("mock_testing_content_policy_fallbacks", "content_policy_fallbacks", "c"),
+    ],
+    ids=["general", "context-window", "content-policy"],
+)
+async def test_a_mock_testing_flag_skips_the_primary_and_walks_the_matching_chain(
+    make_core: MakeCore, host: ScriptedHost, flag: str, field: str, expected: str
+) -> None:
+    core = build(make_core, num_retries=2, **{"fallbacks": [{"a": ["b"]}], field: [{"a": [expected]}]})
+
+    response = await complete(core, **{flag: True})
+
+    assert host.deployments_called("acompletion") == [expected]
+    assert headers(response)["x-litellm-attempted-fallbacks"] == 1
+    assert headers(response)["x-litellm-attempted-retries"] == 0
+    assert host.sleeps == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("flag", "error", "message"),
+    [
+        (
+            "mock_testing_fallbacks",
+            litellm.InternalServerError,
+            "litellm.InternalServerError: This is a mock exception for model=a, to trigger a fallback. Fallbacks=None",
+        ),
+        (
+            "mock_testing_context_fallbacks",
+            litellm.ContextWindowExceededError,
+            "litellm.ContextWindowExceededError: litellm.BadRequestError: This is a mock exception for model=a, "
+            "to trigger a fallback.                     Context_Window_Fallbacks=None",
+        ),
+        (
+            "mock_testing_content_policy_fallbacks",
+            litellm.ContentPolicyViolationError,
+            "litellm.BadRequestError: litellm.ContentPolicyViolationError: This is a mock exception for model=a, "
+            "to trigger a fallback.                     Context_Policy_Fallbacks=None",
+        ),
+    ],
+    ids=["general", "context-window", "content-policy"],
+)
+async def test_a_mock_testing_flag_without_a_chain_raises_the_mock_error_without_calling_the_provider(
+    make_core: MakeCore, host: ScriptedHost, flag: str, error: type[Exception], message: str
+) -> None:
+    core = build(make_core)
+
+    with pytest.raises(error) as raised:
+        await complete(core, **{flag: True})
+
+    assert str(raised.value) == message
+    assert host.calls == []
+
+
+@pytest.mark.asyncio
+async def test_the_mock_error_names_the_configured_chain_and_the_fallback_outcome(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    show_router_debug(host)
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.InternalServerError) as raised:
+        await complete(core, mock_testing_fallbacks=True)
+
+    assert str(raised.value) == (
+        "litellm.InternalServerError: This is a mock exception for model=a, to trigger a fallback. "
+        "Fallbacks=[{'a': ['b']}]\n\nLiteLLM: model group 'a' failed with the error above. "
+        "Fallback to b also failed: litellm.RateLimitError: slow down"
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(("value", "expected"), [("true", ["b"]), ("false", ["a"]), (False, ["a"])])
+async def test_a_mock_testing_flag_given_as_a_string_is_parsed_as_a_bool(
+    make_core: MakeCore, host: ScriptedHost, value: object, expected: list[str]
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+
+    await complete(core, mock_testing_fallbacks=value)
+
+    assert host.deployments_called("acompletion") == expected
+
+
+@pytest.mark.asyncio
+async def test_the_mock_testing_flag_is_not_forwarded_to_the_fallback(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+
+    await complete(core, mock_testing_fallbacks=True)
+
+    assert "mock_testing_fallbacks" not in host.calls_to("acompletion")[0].kwargs
+
+
+def test_sync_completion_honours_the_mock_testing_flag(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+
+    response = complete_sync(core, mock_testing_fallbacks=True)
+
+    assert host.deployments_called("completion") == ["b"]
+    assert headers(response)["x-litellm-attempted-fallbacks"] == 1

tests/unit/test_router/core/test_fallbacks_characterization_outcomes.py

@@ -0,0 +1,675 @@
+"""
+Characterization of what a router core's fallbacks leave behind (non-streaming): response headers
+and hidden params, the metadata each attempt carries, the error text when fallbacks give up,
+fallback access and budget checks, fallback events, content-filter responses and weighted failover.
+"""
+
+from collections.abc import Mapping
+
+import pytest
+
+import litellm
+from litellm.integrations.custom_logger import CustomLogger
+from tests.unit.test_router.core.fallbacks_helpers import (
+    GROUP_A_OF_THREE,
+    MakeCore,
+    build,
+    complete,
+    complete_sync,
+    fails_once_then_answers,
+    headers,
+    hidden,
+    metadata_of,
+    show_router_debug,
+)
+from tests.unit.test_router.core.scripted_host import (
+    ScriptedHost,
+    completion_response,
+    context_window,
+    deployment,
+    rate_limit,
+    server_error,
+    timeout,
+)
+
+ERROR_429 = '{"message": "litellm.RateLimitError: slow down", "type": "throttling_error", "param": null, "code": "429"}'
+ERROR_500 = (
+    '{"message": "litellm.InternalServerError: boom", "type": "internal_server_error", "param": null, "code": "500"}'
+)
+
+# --- headers and hidden params ------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_a_two_hop_chain_reports_one_attempted_fallback(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}, {"b": ["c"]}])
+    host.answer("openai/a", rate_limit())
+    host.answer("openai/b", rate_limit())
+
+    response = await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "b", "c"]
+    assert headers(response)["x-litellm-attempted-fallbacks"] == 1
+    assert headers(response)["x-litellm-model-group"] == "c"
+
+
+@pytest.mark.asyncio
+async def test_the_answering_fallback_deployment_is_the_responses_model_id(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    host.answer("openai/a", rate_limit())
+
+    response = await complete(core)
+
+    assert hidden(response)["model_id"] == "b"
+
+
+@pytest.mark.asyncio
+async def test_success_headers_after_a_fallback(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    host.answer("openai/a", rate_limit())
+
+    response = await complete(core)
+
+    assert headers(response) == {
+        "x-litellm-model-group": "b",
+        "x-litellm-attempted-retries": 0,
+        "x-litellm-attempted-fallbacks": 1,
+    }
+
+
+@pytest.mark.asyncio
+async def test_include_fallback_errors_lists_the_primary_error_in_a_header(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    host.answer("openai/a", rate_limit())
+
+    response = await complete(core, include_fallback_errors=True)
+
+    assert headers(response)["x-litellm-fallback-errors"] == f"[{ERROR_429}]"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "fallbacks",
+    [[{"a": ["b", "c"]}], [{"a": ["b"]}, {"b": ["c"]}]],
+    ids=["one-list", "chained"],
+)
+async def test_include_fallback_errors_lists_the_inner_hops_errors_first(
+    make_core: MakeCore, host: ScriptedHost, fallbacks: list[object]
+) -> None:
+    core = build(make_core, fallbacks=fallbacks)
+    host.answer("openai/a", rate_limit())
+    host.answer("openai/b", server_error())
+
+    response = await complete(core, include_fallback_errors=True)
+
+    assert host.deployments_called("acompletion") == ["a", "b", "c"]
+    assert headers(response)["x-litellm-fallback-errors"] == f"[{ERROR_500}, {ERROR_429}]"
+
+
+@pytest.mark.asyncio
+async def test_without_include_fallback_errors_no_errors_header_is_set(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    host.answer("openai/a", rate_limit())
+
+    response = await complete(core)
+
+    assert "x-litellm-fallback-errors" not in headers(response)
+
+
+# --- what each attempt carries to the host ------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_each_attempt_carries_the_fallback_count_and_original_group_in_its_metadata(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}, {"b": ["c"]}])
+    host.answer("openai/a", rate_limit())
+    host.answer("openai/b", rate_limit())
+
+    await complete(core)
+
+    assert [
+        (meta["model_group"], meta["original_model_group"], meta["attempted_fallbacks"])
+        for meta in (metadata_of(call.kwargs) for call in host.calls_to("acompletion"))
+    ] == [("a", "a", 0), ("b", "a", 1), ("c", "a", 2)]
+
+
+@pytest.mark.asyncio
+async def test_a_fallback_attempt_also_receives_the_routers_walk_state_as_kwargs(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    host.answer("openai/a", rate_limit())
+
+    await complete(core)
+
+    primary, fallback = host.calls_to("acompletion")
+    assert {"fallback_depth", "max_fallbacks", "attempted_targets"} & set(primary.kwargs) == set()
+    assert (fallback.kwargs["fallback_depth"], fallback.kwargs["max_fallbacks"]) == (1, 5)
+    assert "attempted_targets" in fallback.kwargs
+
+
+@pytest.mark.asyncio
+async def test_the_fallback_attempt_sees_the_failed_attempt_in_previous_models(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    host.answer("openai/a", rate_limit())
+
+    await complete(core)
+
+    previous = metadata_of(host.calls_to("acompletion")[1].kwargs)["previous_models"]
+    assert [
+        (entry["model_group"], entry["deployment_id"], entry["exception_type"])
+        for entry in previous  # pyright: ignore[reportGeneralTypeIssues]  # a tuple of dicts
+    ] == [("a", "a", "RateLimitError")]
+
+
+# --- the error when fallbacks give up -------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_router_debug_on_a_context_window_error_without_its_chain_names_the_missing_setting(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core)
+    show_router_debug(host)
+    host.answer("openai/a", context_window())
+
+    with pytest.raises(litellm.ContextWindowExceededError) as raised:
+        await complete(core)
+
+    assert str(raised.value) == (
+        "litellm.ContextWindowExceededError: litellm.BadRequestError: too long\n"
+        "model=a. context_window_fallbacks=None. fallbacks=None.\n\n"
+        "Set 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks\n\n"
+        "LiteLLM: model group 'a' failed with the error above. No fallback was attempted."
+    )
+
+
+@pytest.mark.asyncio
+async def test_router_debug_on_a_context_window_error_that_also_fails_its_fallback_nests_the_hint(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    show_router_debug(host)
+    host.answer("*", context_window())
+
+    with pytest.raises(litellm.ContextWindowExceededError) as raised:
+        await complete(core)
+
+    hint = "context_window_fallbacks=None. fallbacks=[{'a': ['b']}].\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks"
+    assert str(raised.value) == (
+        f"litellm.ContextWindowExceededError: litellm.BadRequestError: too long\nmodel=a. {hint}\n\n"
+        "LiteLLM: model group 'a' failed with the error above. Fallback to b also failed: "
+        f"litellm.ContextWindowExceededError: litellm.BadRequestError: too long\nmodel=b. {hint}"
+    )
+
+
+@pytest.mark.asyncio
+async def test_router_debug_names_a_plain_request_list_that_failed(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core)
+    show_router_debug(host)
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError) as raised:
+        await complete(core, fallbacks=["b", "c"])
+
+    assert str(raised.value) == (
+        "litellm.RateLimitError: slow down\n\nLiteLLM: model group 'a' failed with the error above. "
+        "Fallback to b, c also failed: litellm.RateLimitError: slow down"
+    )
+
+
+@pytest.mark.asyncio
+async def test_router_debug_with_max_fallbacks_one_names_only_the_first_level(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}, {"b": ["c"]}], max_fallbacks=1)
+    show_router_debug(host)
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError) as raised:
+        await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "b"]
+    assert str(raised.value).endswith("Fallback to b also failed: litellm.RateLimitError: slow down")
+
+
+@pytest.mark.asyncio
+async def test_router_debug_reports_the_first_fallback_errors_message_not_the_last(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}, {"b": ["c"]}])
+    show_router_debug(host)
+    host.answer("openai/a", rate_limit())
+    host.answer("openai/b", server_error())
+    host.answer("openai/c", timeout())
+
+    with pytest.raises(litellm.RateLimitError) as raised:
+        await complete(core)
+
+    assert str(raised.value) == (
+        "litellm.RateLimitError: slow down\n\nLiteLLM: model group 'a' failed with the error above. "
+        "Fallback to b also failed: litellm.InternalServerError: boom"
+    )
+
+
+@pytest.mark.asyncio
+async def test_router_debug_truncates_a_long_fallback_error(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    show_router_debug(host)
+    host.answer("openai/a", rate_limit())
+    host.answer("openai/b", rate_limit(message="x" * 3000))
+
+    with pytest.raises(litellm.RateLimitError) as raised:
+        await complete(core)
+
+    detail = "litellm.RateLimitError: " + "x" * 3000
+    assert str(raised.value) == (
+        "litellm.RateLimitError: slow down\n\nLiteLLM: model group 'a' failed with the error above. "
+        f"Fallback to b also failed: {detail[:2000]}... [truncated {len(detail) - 2000} characters]"
+    )
+
+
+@pytest.mark.asyncio
+async def test_router_debug_on_an_unmatched_content_policy_chain_says_no_fallback_was_attempted(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    from tests.unit.test_router.core.scripted_host import content_policy
+
+    core = build(make_core, content_policy_fallbacks=[{"other": ["b"]}])
+    show_router_debug(host)
+    host.answer("openai/a", content_policy())
+
+    with pytest.raises(litellm.ContentPolicyViolationError) as raised:
+        await complete(core)
+
+    assert str(raised.value) == (
+        "litellm.BadRequestError: litellm.ContentPolicyViolationError: blocked\n\n"
+        "LiteLLM: model group 'a' failed with the error above. No fallback was attempted."
+    )
+
+
+@pytest.mark.asyncio
+async def test_router_debug_no_chain_message_without_any_configured_keys(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core)
+    show_router_debug(host)
+    host.answer("openai/a", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError) as raised:
+        await complete(core, fallbacks=[{"x": ["b"]}, {"y": ["c"]}])
+
+    assert str(raised.value).endswith(
+        "LiteLLM: model group 'a' failed with the error above and no fallback model group was found for it, "
+        "so the request was not retried on another model. Fallbacks are configured for: x, y. "
+        "Add a fallbacks entry for that model group (Router fallbacks or proxy router_settings.fallbacks) "
+        "to retry on another model."
+    )
+
+
+# --- fallback access and budget checks -------------------------------------------------------------
+
+
+def denying(denied: str, seen: list[str]):  # noqa: ANN201  # a FallbackAccessCheck / FallbackBudgetCheck
+    async def check(*, model: str, request_kwargs: Mapping[str, object], ctx: object) -> bool:
+        seen.append(model)
+        return model != denied
+
+    return check
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("field", ["fallback_access_check", "fallback_budget_check"])
+async def test_a_fallback_target_the_check_rejects_is_skipped(
+    make_core: MakeCore, host: ScriptedHost, field: str
+) -> None:
+    seen: list[str] = []
+    core = build(make_core, fallbacks=[{"a": ["b", "c"]}], **{field: denying("b", seen)})
+    host.answer("openai/a", rate_limit())
+
+    response = await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "c"]
+    assert headers(response)["x-litellm-model-group"] == "c"
+    assert seen == ["b", "c"]
+
+
+def test_sync_completion_runs_the_fallback_access_check(make_core: MakeCore, host: ScriptedHost) -> None:
+    seen: list[str] = []
+    core = build(make_core, fallbacks=[{"a": ["b", "c"]}], fallback_access_check=denying("b", seen))
+    host.answer("openai/a", rate_limit())
+
+    complete_sync(core)
+
+    assert host.deployments_called("completion") == ["a", "c"]
+    assert seen == ["b", "c"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("field", "sentence"),
+    [
+        ("fallback_access_check", "Fallback to b was skipped because the caller is not authorized to call it."),
+        ("fallback_budget_check", "Fallback to b was skipped because the caller is over budget for it."),
+    ],
+)
+async def test_router_debug_explains_a_skipped_fallback_target(
+    make_core: MakeCore, host: ScriptedHost, field: str, sentence: str
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}], **{field: denying("b", [])})
+    show_router_debug(host)
+    host.answer("openai/a", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError) as raised:
+        await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a"]
+    assert str(raised.value) == (
+        f"litellm.RateLimitError: slow down\n\nLiteLLM: model group 'a' failed with the error above. {sentence}"
+    )
+
+
+@pytest.mark.asyncio
+async def test_router_debug_names_both_the_failed_and_the_skipped_targets(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    seen: list[str] = []
+    core = build(
+        make_core,
+        fallbacks=[{"a": ["b", "c"]}],
+        fallback_access_check=denying("b", seen),
+        fallback_budget_check=denying("b", seen),
+    )
+    show_router_debug(host)
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError) as raised:
+        await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "c"]
+    assert str(raised.value) == (
+        "litellm.RateLimitError: slow down\n\nLiteLLM: model group 'a' failed with the error above. "
+        "Fallback to c also failed: litellm.RateLimitError: slow down "
+        "Fallback to b was skipped because the caller is not authorized to call it."
+    )
+    # The access check runs before the budget check; a skipped target is asked again in c's hop.
+    assert seen == ["b", "c", "c", "b"]
+
+
+# --- fallback events ---------------------------------------------------------------------------------
+
+
+class FallbackEvents(CustomLogger):
+    def __init__(self) -> None:
+        super().__init__()
+        self.events: list[tuple[str, str, object, str]] = []
+
+    async def log_success_fallback_event(
+        self, original_model_group: str, kwargs: dict, original_exception: Exception
+    ) -> None:
+        self.events.append(("success", original_model_group, kwargs.get("model"), type(original_exception).__name__))
+
+    async def log_failure_fallback_event(
+        self, original_model_group: str, kwargs: dict, original_exception: Exception
+    ) -> None:
+        self.events.append(("failure", original_model_group, kwargs.get("model"), type(original_exception).__name__))
+
+
+@pytest.mark.asyncio
+async def test_fallback_events_reach_the_process_custom_loggers_innermost_hop_first(
+    make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    events = FallbackEvents()
+    monkeypatch.setattr(litellm, "callbacks", [events])
+    core = build(make_core, fallbacks=[{"a": ["b", "c"]}])
+    host.answer("openai/a", rate_limit())
+    host.answer("openai/b", server_error())
+
+    await complete(core)
+
+    assert events.events == [
+        ("success", "b", "c", "InternalServerError"),
+        ("success", "a", "b", "RateLimitError"),
+    ]
+
+
+@pytest.mark.asyncio
+async def test_a_fallback_that_fails_everywhere_logs_failure_events(
+    make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    events = FallbackEvents()
+    monkeypatch.setattr(litellm, "callbacks", [events])
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError):
+        await complete(core)
+
+    assert events.events == [("failure", "a", "b", "RateLimitError")]
+
+
+# --- content-filter responses ------------------------------------------------------------------------
+
+
+def content_filtered() -> object:
+    response = completion_response()
+    response.choices[0].finish_reason = "content_filter"  # pyright: ignore[reportAttributeAccessIssue]  # Choices
+    return response
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("arguments", "expected"),
+    [
+        ({"content_policy_fallbacks": [{"a": ["b"]}]}, ["a", "b"]),
+        ({"default_fallbacks": ["c"]}, ["a", "c"]),
+        ({"fallbacks": [{"a": ["b"]}]}, ["a"]),
+        ({"content_policy_fallbacks": [{"other": ["b"]}], "default_fallbacks": ["c"]}, ["a"]),
+    ],
+    ids=["content-policy-chain", "default-fallbacks", "general-chain-only", "unmatched-content-policy-chain"],
+)
+async def test_a_content_filtered_response_falls_back_only_when_a_content_policy_fallback_resolves(
+    make_core: MakeCore, host: ScriptedHost, arguments: dict[str, object], expected: list[str]
+) -> None:
+    core = build(make_core, **arguments)
+    host.answer("openai/a", content_filtered())
+
+    response = await complete(core)
+
+    assert host.deployments_called("acompletion") == expected
+    finish_reason = response.choices[0].finish_reason  # pyright: ignore[reportAttributeAccessIssue]  # ModelResponse
+    assert finish_reason == ("content_filter" if expected == ["a"] else "stop")
+
+
+def test_sync_completion_falls_back_on_a_content_filtered_response(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, content_policy_fallbacks=[{"a": ["b"]}])
+    host.answer("openai/a", content_filtered())
+
+    complete_sync(core)
+
+    assert host.deployments_called("completion") == ["a", "b"]
+
+
+@pytest.mark.asyncio
+async def test_a_content_filtered_response_with_fallbacks_disabled_raises_content_policy_violation(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, content_policy_fallbacks=[{"a": ["b"]}])
+    host.answer("openai/a", content_filtered())
+
+    with pytest.raises(litellm.ContentPolicyViolationError) as raised:
+        await complete(core, disable_fallbacks=True)
+
+    assert (
+        str(raised.value)
+        == "litellm.BadRequestError: litellm.ContentPolicyViolationError: Response output was blocked."
+    )
+    assert host.deployments_called("acompletion") == ["a"]
+
+
+@pytest.mark.asyncio
+async def test_the_last_content_filtered_response_is_returned_when_every_fallback_is_filtered(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, content_policy_fallbacks=[{"a": ["b"]}])
+    host.answer("*", content_filtered())
+
+    response = await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "b"]
+    assert response.choices[0].finish_reason == "content_filter"  # pyright: ignore[reportAttributeAccessIssue]  # ModelResponse
+    assert headers(response)["x-litellm-model-group"] == "b"
+
+
+# --- weighted failover ---------------------------------------------------------------------------
+
+
+def a_deployments_fail(host: ScriptedHost, error: object) -> None:
+    for deployment_id in ("a1", "a2", "a3"):
+        host.answer(f"openai/{deployment_id}", error)
+
+
+@pytest.mark.asyncio
+async def test_weighted_failover_tries_every_deployment_of_the_group_before_the_external_chain(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, model_list=GROUP_A_OF_THREE, enable_weighted_failover=True, fallbacks=[{"a": ["b"]}])
+    a_deployments_fail(host, rate_limit())
+
+    response = await complete(core)
+
+    called = host.deployments_called("acompletion")
+    assert sorted(called[:3]) == ["a1", "a2", "a3"]
+    assert called[3:] == ["b"]
+    assert headers(response)["x-litellm-attempted-fallbacks"] == 1
+
+
+@pytest.mark.asyncio
+async def test_without_weighted_failover_one_deployment_fails_over_to_the_external_chain(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, model_list=GROUP_A_OF_THREE, fallbacks=[{"a": ["b"]}])
+    a_deployments_fail(host, rate_limit())
+
+    await complete(core)
+
+    called = host.deployments_called("acompletion")
+    assert len(called) == 2
+    assert called[0] in {"a1", "a2", "a3"}
+    assert called[1] == "b"
+
+
+@pytest.mark.asyncio
+async def test_a_weighted_failover_that_answers_counts_as_a_fallback_inside_the_group(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, model_list=GROUP_A_OF_THREE, enable_weighted_failover=True)
+    host.answer("*", fails_once_then_answers(rate_limit()))
+
+    response = await complete(core)
+
+    calls = host.calls_to("acompletion")
+    assert len(calls) == 2
+    assert calls[0].deployment_id != calls[1].deployment_id
+    assert [call.model_group for call in calls] == ["a", "a"]
+    assert [metadata_of(call.kwargs)["attempted_fallbacks"] for call in calls] == [0, 1]
+    assert headers(response)["x-litellm-model-group"] == "a"
+    assert headers(response)["x-litellm-attempted-fallbacks"] == 1
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(("max_fallbacks", "group_attempts"), [(0, 1), (1, 2), (2, 3)])
+async def test_weighted_failover_is_bounded_by_max_fallbacks(
+    make_core: MakeCore, host: ScriptedHost, max_fallbacks: int, group_attempts: int
+) -> None:
+    core = build(
+        make_core,
+        model_list=GROUP_A_OF_THREE,
+        enable_weighted_failover=True,
+        fallbacks=[{"a": ["b"]}],
+        max_fallbacks=max_fallbacks,
+    )
+    a_deployments_fail(host, rate_limit())
+
+    if max_fallbacks == 0:
+        with pytest.raises(litellm.RateLimitError):
+            await complete(core)
+    else:
+        await complete(core)
+
+    called = host.deployments_called("acompletion")
+    assert len(set(called[:group_attempts])) == group_attempts
+    assert called[group_attempts:] == ([] if max_fallbacks == 0 else ["b"])
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("arguments", "error"),
+    [
+        ({"routing_strategy": "latency-based-routing"}, rate_limit),
+        ({}, context_window),
+    ],
+    ids=["not-simple-shuffle", "context-window-error"],
+)
+async def test_weighted_failover_does_not_apply(
+    make_core: MakeCore, host: ScriptedHost, arguments: dict[str, object], error: object
+) -> None:
+    core = build(
+        make_core, model_list=GROUP_A_OF_THREE, enable_weighted_failover=True, fallbacks=[{"a": ["b"]}], **arguments
+    )
+    a_deployments_fail(host, error())  # pyright: ignore[reportCallIssue]  # an error factory
+
+    await complete(core)
+
+    called = host.deployments_called("acompletion")
+    assert len(called) == 2
+    assert called[1] == "b"
+
+
+@pytest.mark.asyncio
+async def test_weighted_failover_with_fallbacks_disabled_tries_one_deployment(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, model_list=GROUP_A_OF_THREE, enable_weighted_failover=True, fallbacks=[{"a": ["b"]}])
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError):
+        await complete(core, disable_fallbacks=True)
+
+    assert len(host.deployments_called("acompletion")) == 1
+
+
+def test_sync_completion_also_runs_weighted_failover(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, model_list=GROUP_A_OF_THREE, enable_weighted_failover=True, fallbacks=[{"a": ["b"]}])
+    a_deployments_fail(host, rate_limit())
+
+    complete_sync(core)
+
+    called = host.deployments_called("completion")
+    assert sorted(called[:3]) == ["a1", "a2", "a3"]
+    assert called[3:] == ["b"]
+
+
+@pytest.mark.asyncio
+async def test_weighted_failover_on_a_single_deployment_group_goes_straight_to_the_external_chain(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(
+        make_core,
+        model_list=[deployment("a", "a1"), deployment("b", "b")],
+        enable_weighted_failover=True,
+        fallbacks=[{"a": ["b"]}],
+    )
+    host.answer("openai/a1", rate_limit())
+
+    await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a1", "b"]

tests/unit/test_router/core/test_fallbacks_characterization_refusal.py

@@ -0,0 +1,101 @@
+"""
+Characterization of a router core's Anthropic safeguard-refusal fallbacks on /v1/messages
+(non-streaming): a ``stop_reason: "refusal"`` response carrying ``stop_details`` re-enters the
+fallback chain only when a fallback would resolve for it.
+
+The core recognises a /v1/messages call by its endpoint's name, so this module runs on
+``NamedEndpointHost`` (a ``ScriptedHost`` whose ``anthropic_messages`` endpoint is named so).
+"""
+
+import pytest
+
+from litellm.router_core import RouterCore
+from tests.unit.test_router.core.fallbacks_helpers import (
+    MESSAGES,
+    MakeCore,
+    NamedEndpointHost,
+    anthropic_message,
+    anthropic_refusal,
+    build,
+)
+
+
+@pytest.fixture
+def host() -> NamedEndpointHost:
+    return NamedEndpointHost()
+
+
+async def messages(core: RouterCore, **request: object) -> object:  # kwargs-ok: request kwargs
+    return await core.aanthropic_messages(model="a", messages=MESSAGES, max_tokens=10, **request)  # pyright: ignore[reportAttributeAccessIssue]  # endpoint method
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("arguments", "expected"),
+    [
+        ({"content_policy_fallbacks": [{"a": ["b"]}]}, ["a", "b"]),
+        ({"fallbacks": [{"a": ["b"]}]}, ["a", "b"]),
+        ({"default_fallbacks": ["c"]}, ["a", "c"]),
+    ],
+    ids=["content-policy-chain", "general-chain", "default-fallbacks"],
+)
+async def test_a_safeguard_refusal_falls_back_when_a_chain_resolves(
+    make_core: MakeCore, host: NamedEndpointHost, arguments: dict[str, object], expected: list[str]
+) -> None:
+    core = build(make_core, **arguments)
+    host.answer("openai/a", anthropic_refusal())
+    host.answer(f"openai/{expected[-1]}", anthropic_message())
+
+    response = await messages(core)
+
+    assert host.deployments_called("anthropic_messages") == expected
+    assert isinstance(response, dict)
+    assert response["stop_reason"] == "end_turn"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("arguments", "request_kwargs", "refusal"),
+    [
+        ({}, {}, anthropic_refusal()),
+        ({"content_policy_fallbacks": [{"other": ["b"]}], "fallbacks": [{"a": ["b"]}]}, {}, anthropic_refusal()),
+        ({"fallbacks": [{"a": ["b"]}]}, {"disable_fallbacks": True}, anthropic_refusal()),
+        ({"fallbacks": [{"a": ["b"]}]}, {}, anthropic_refusal(with_stop_details=False)),
+    ],
+    ids=[
+        "no-chain",
+        "unmatched-content-policy-chain-is-authoritative",
+        "fallbacks-disabled",
+        "plain-refusal-without-stop-details",
+    ],
+)
+async def test_a_refusal_is_returned_to_the_caller_when_no_fallback_would_resolve(
+    make_core: MakeCore,
+    host: NamedEndpointHost,
+    arguments: dict[str, object],
+    request_kwargs: dict[str, object],
+    refusal: dict[str, object],
+) -> None:
+    core = build(make_core, **arguments)
+    host.answer("openai/a", refusal)
+
+    response = await messages(core, **request_kwargs)
+
+    assert host.deployments_called("anthropic_messages") == ["a"]
+    assert isinstance(response, dict)
+    assert response["stop_reason"] == "refusal"
+    assert response["_hidden_params"]["additional_headers"]["x-litellm-attempted-fallbacks"] == 0
+
+
+@pytest.mark.asyncio
+async def test_the_last_refusal_is_returned_when_every_fallback_refuses(
+    make_core: MakeCore, host: NamedEndpointHost
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    host.answer("*", anthropic_refusal())
+
+    response = await messages(core)
+
+    assert host.deployments_called("anthropic_messages") == ["a", "b"]
+    assert isinstance(response, dict)
+    assert response["stop_reason"] == "refusal"

tests/unit/test_router/core/test_gaps_adapter_timeout.py

@@ -0,0 +1,34 @@
+"""
+Characterization gap: a streaming adapter completion splits its timeouts like a streaming chat
+completion, sending the request timeout as ``timeout`` and the stream timeout beside it, so a
+fake-streaming provider stays bounded by the request timeout rather than the stream timeout.
+
+Every test builds its core through ``make_core`` on the ``host`` fixture, so it runs once per core.
+"""
+
+from collections.abc import Callable
+from typing import Final
+
+import pytest
+
+from litellm.router_core import RouterCore
+from tests.unit.test_router.core.endpoints_helpers import EndpointHost
+from tests.unit.test_router.core.scripted_host import deployment
+
+MESSAGES: Final = [{"role": "user", "content": "hi"}]
+MakeCore = Callable[..., RouterCore]
+
+
+@pytest.fixture
+def host() -> EndpointHost:
+    return EndpointHost()
+
+
+@pytest.mark.parametrize(("stream", "expected"), [(False, (11.0, None)), (True, (11.0, 5.0))])
+async def test_adapter_completion_sends_the_request_timeout_and_the_stream_timeout_beside_it(
+    make_core: MakeCore, host: EndpointHost, stream: bool, expected: tuple[float, float | None]
+):
+    core: Final = make_core(model_list=[deployment("g", "d1", timeout=11)], stream_timeout=5)
+    await core.aadapter_completion(adapter_id="my-adapter", model="g", messages=MESSAGES, stream=stream)
+    sent: Final = host.calls[-1].kwargs
+    assert (sent["timeout"], sent.get("stream_timeout")) == expected

tests/unit/test_router/core/test_gaps_l0001.py

@@ -0,0 +1,445 @@
+"""
+Gap tests for the first part of ``litellm/router_python_core.py`` (module helpers, the entry points,
+``__init__``, callback registration, strategy setup, routing groups and the strategy dispatch):
+behaviors no other characterization test pinned down.
+
+Each test asserts what the Python core does today; oddities are named in the test's docstring.
+"""
+
+import datetime
+import re
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
+from litellm.router_api import ChangeSet
+from litellm.types.router import Deployment, RouterRateLimitError
+from litellm.utils import reapply_runtime_model_cost_registrations
+from tests.unit.test_router.core.construction_precall_helpers import collect_other_routers, host_callback_names
+from tests.unit.test_router.core.registry_helpers import unique_id
+from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment
+from tests.unit.test_router.core.selection_helpers import MESSAGES, MakeCore, picked_ids
+
+NO_DEPLOYMENTS_MESSAGE: Final = (
+    "No deployments available for selected model, Try again in 5 seconds. Passed model={model}. "
+    "pre-call-checks=False, cooldown_list=[]"
+)
+
+
+def _group(
+    name: str, models: list[str], strategy: str, **extra: object
+) -> dict[str, object]:  # mutable-ok: a routing_groups entry  # kwargs-ok: extra RoutingGroup fields
+    return {"group_name": name, "models": models, "routing_strategy": strategy, **extra}
+
+
+def _logging_obj() -> LiteLLMLogging:
+    return LiteLLMLogging(
+        model="g",
+        messages=MESSAGES,
+        stream=False,
+        call_type="acompletion",
+        start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
+        litellm_call_id="c1",
+        function_id="f1",
+    )
+
+
+def _dynamic_callback_names(logging_obj: LiteLLMLogging) -> list[str]:  # mutable-ok: a fresh list
+    return [type(callback).__name__ for callback in logging_obj.dynamic_async_success_callbacks or ()]
+
+
+# --- construction: routing_strategy=None and legacy attributes -------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_routing_strategy_none_is_accepted_but_leaves_every_pick_without_a_deployment(
+    make_core: MakeCore,
+) -> None:
+    """Oddity: ``routing_strategy=None`` passes validation and is stored as None (not the
+    ``simple-shuffle`` default), so no strategy arm matches and every pick rate-limits, while the
+    healthy listing still lists the deployment."""
+    core: Final = make_core(model_list=[deployment("g", "a")], routing_strategy=None)
+    assert core.settings().routing_strategy is None
+    assert core.settings().as_dict()["routing_strategy"] is None
+    with pytest.raises(RouterRateLimitError) as raised:
+        await core.select("g")
+    assert str(raised.value) == NO_DEPLOYMENTS_MESSAGE.format(model="g")
+    healthy: Final = await core.select("g", kind="healthy")
+    assert isinstance(healthy, list)
+    assert [entry["model_info"]["id"] for entry in healthy] == ["a"]
+
+
+LEGACY_ATTRIBUTE_DEFAULTS: Final = {
+    "total_calls": {},
+    "fail_calls": {},
+    "success_calls": {},
+    "deployment_stats": {},
+    "default_cache_time_seconds": 3600,
+    "tenacity": None,
+    "lowesttpm_logger_v2": None,
+    "lowestlatency_logger": None,
+    "lowestcost_logger": None,
+}
+
+
+@pytest.mark.parametrize(("name", "expected"), sorted(LEGACY_ATTRIBUTE_DEFAULTS.items()))
+def test_legacy_attribute_default(make_core: MakeCore, name: str, expected: object) -> None:
+    assert getattr(make_core(model_list=[deployment("g", "a")]), name) == expected
+
+
+def test_healthy_deployments_is_the_live_deployment_list(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")])
+    assert core.healthy_deployments is core.registry().all()  # pyright: ignore[reportAttributeAccessIssue]
+
+
+def test_chat_completions_create_calls_the_router_with_the_default_litellm_params(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], default_litellm_params={"temperature": 0.3})
+    core.chat.completions.create(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert [(call.op, call.deployment_id, call.kwargs.get("temperature")) for call in host.calls] == [
+        ("completion", "a", 0.3)
+    ]
+
+
+@pytest.mark.asyncio
+async def test_a_provider_budget_config_adds_the_budget_limiter_after_the_listed_checks(make_core: MakeCore) -> None:
+    """A provider budget appends ``router_budget_limiting`` to the optional pre-call checks given."""
+    core: Final = make_core(
+        model_list=[deployment("g", "a")],
+        provider_budget_config={"openai": {"budget_limit": 1, "time_period": "1d"}},
+        optional_pre_call_checks=["prompt_caching"],
+    )
+    assert [type(callback).__name__ for callback in core.optional_callbacks] == [  # pyright: ignore[reportAttributeAccessIssue]
+        "PromptCachingDeploymentCheck",
+        "RouterBudgetLimiting",
+    ]
+    core.retire()
+
+
+# --- price data reloads ---------------------------------------------------------------------------
+
+
+def test_a_price_data_reload_restores_a_live_routers_deployment_pricing_until_it_retires(
+    make_core: MakeCore,
+) -> None:
+    model_id: Final = unique_id("l0001")
+    core: Final = make_core(model_list=[deployment("gpt", model_id, model="openai/gpt-4o", input_cost_per_token=1e-3)])
+    try:
+        litellm.model_cost.pop(model_id)
+        reapply_runtime_model_cost_registrations()
+        assert litellm.model_cost[model_id]["input_cost_per_token"] == 1e-3
+        core.retire()
+        litellm.model_cost.pop(model_id)
+        reapply_runtime_model_cost_registrations()
+        assert model_id not in litellm.model_cost
+    finally:
+        litellm.model_cost.pop(model_id, None)
+
+
+# --- runtime routing_strategy_args ----------------------------------------------------------------
+
+
+def test_new_routing_strategy_args_rebuild_the_selector_and_replace_it_on_the_host(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a")], routing_strategy="latency-based-routing", routing_strategy_args={"ttl": 5}
+    )
+    (before,) = host.lists["callbacks"]
+    core.apply(ChangeSet(settings={"routing_strategy_args": {"ttl": 9}}))
+    (after,) = host.lists["callbacks"]
+    assert after is not before
+    assert after is core.lowestlatency_logger  # pyright: ignore[reportAttributeAccessIssue]
+    assert after.routing_args.ttl == 9  # pyright: ignore[reportAttributeAccessIssue]
+    assert core.settings().as_dict()["routing_strategy_args"]["ttl"] == 9
+
+
+def test_invalid_routing_strategy_args_at_runtime_keep_the_previous_selector(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    """Oddity: the invalid args are still stored as ``routing_strategy_args``, while the selector (and
+    so ``as_dict``, which reads the selector) keeps the previous ones."""
+    core: Final = make_core(
+        model_list=[deployment("g", "a")], routing_strategy="latency-based-routing", routing_strategy_args={"ttl": 5}
+    )
+    (before,) = host.lists["callbacks"]
+    core.apply(ChangeSet(settings={"routing_strategy_args": {"ttl": "abc"}}))
+    assert host.lists["callbacks"] == [before]
+    assert core.lowestlatency_logger is before  # pyright: ignore[reportAttributeAccessIssue]
+    assert core.settings().as_dict()["routing_strategy_args"]["ttl"] == 5
+    assert core.routing_strategy_args == {"ttl": "abc"}  # pyright: ignore[reportAttributeAccessIssue]
+
+
+def test_new_least_busy_args_replace_the_selector_in_callbacks_and_input_callbacks(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], routing_strategy="least-busy")
+    before: Final = core.leastbusy_logger  # pyright: ignore[reportAttributeAccessIssue]
+    core.apply(ChangeSet(settings={"routing_strategy_args": {"x": 1}}))
+    after: Final = core.leastbusy_logger  # pyright: ignore[reportAttributeAccessIssue]
+    assert after is not before
+    assert (host.lists["callbacks"], host.lists["input_callback"]) == ([after], [after])
+
+
+def test_new_routing_strategy_args_under_simple_shuffle_register_nothing(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")])
+    core.apply(ChangeSet(settings={"routing_strategy_args": {"ttl": 9}}))
+    assert core.settings().as_dict()["routing_strategy_args"] == {"ttl": 9}
+    assert (host.lists["callbacks"], host.lists["input_callback"]) == ([], [])
+
+
+# --- routing groups: selectors on the host, shadowing ---------------------------------------------
+
+
+def test_routing_group_selectors_are_registered_on_the_host_and_replaced_when_groups_change(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(
+        model_list=[deployment("m1", "a"), deployment("m2", "b")],
+        routing_strategy="least-busy",
+        routing_groups=[_group("g1", ["m1"], "latency-based-routing", routing_strategy_args={"ttl": 7})],
+    )
+    assert host_callback_names(host)["callbacks"] == ["LeastBusyLoggingHandler", "LowestLatencyLoggingHandler"]
+    (latency,) = (
+        callback for callback in host.lists["callbacks"] if type(callback).__name__ == "LowestLatencyLoggingHandler"
+    )
+    assert latency.routing_args.ttl == 7  # pyright: ignore[reportAttributeAccessIssue]
+    core.apply(ChangeSet(settings={"routing_groups": [_group("g2", ["m2"], "usage-based-routing-v2")]}))
+    assert host_callback_names(host)["callbacks"] == ["LeastBusyLoggingHandler", "LowestTPMLoggingHandler_v2"]
+    core.apply(ChangeSet(settings={"routing_groups": []}))
+    assert host_callback_names(host)["callbacks"] == ["LeastBusyLoggingHandler"]
+    assert host_callback_names(host)["input_callback"] == ["LeastBusyLoggingHandler"]
+
+
+@pytest.mark.asyncio
+async def test_a_group_shadowed_by_a_model_name_still_routes_its_members_with_its_strategy(
+    make_core: MakeCore,
+) -> None:
+    """The shadowed group's name is not callable (calling it reaches the model), but its least-busy
+    strategy still serves its member: with no traffic least-busy always picks the first deployment."""
+    core: Final = make_core(
+        model_list=[deployment("m1", "a"), deployment("m2", "b"), deployment("m2", "c")],
+        routing_groups=[_group("m1", ["m2"], "least-busy")],
+    )
+    assert await picked_ids(core, "m2", times=30, messages=MESSAGES) == {"b"}
+    assert core.registry().routing_group("m1") is None
+
+
+@pytest.mark.asyncio
+async def test_a_group_named_like_an_alias_is_shadowed_by_the_alias(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[deployment("m1", "a"), deployment("m2", "b")],
+        model_group_alias={"al": "m1"},
+        routing_groups=[_group("al", ["m2"], "simple-shuffle")],
+    )
+    assert await picked_ids(core, "al") == {"a"}
+    assert core.registry().routing_group("al") is None
+
+
+def test_a_priority_group_named_like_an_alias_is_rejected(make_core: MakeCore) -> None:
+    with pytest.raises(ValueError, match=re.escape("Priority routing group names must not shadow a model or alias")):
+        make_core(
+            model_list=[deployment("m1", "a")],
+            model_group_alias={"al": "m1"},
+            routing_groups=[_group("al", ["m1"], "priority", model_priorities={"m1": 1})],
+        )
+
+
+@pytest.mark.asyncio
+async def test_a_deployment_added_under_a_group_name_shadows_the_group(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[deployment("m1", "a"), deployment("m2", "b")],
+        routing_groups=[_group("grp", ["m1", "m2"], "simple-shuffle")],
+    )
+    core.apply(ChangeSet(upsert=[Deployment(**deployment("grp", "z"))]))  # pyright: ignore[reportArgumentType]
+    assert await picked_ids(core, "grp") == {"z"}
+    assert core.registry().routing_group("grp") is None
+
+
+# --- registry().resolve kinds backed by this range ------------------------------------------------
+
+
+def test_routing_group_alternatives_count_the_deployments_of_every_member(make_core: MakeCore) -> None:
+    """Only a callable routing group (or an alias to one) can have alternatives; a plain model group
+    with two deployments has none by this rule."""
+    core: Final = make_core(
+        model_list=[deployment("m1", "a"), deployment("m1", "a2"), deployment("m2", "b"), deployment("m3", "c")],
+        model_group_alias={"al": "grp"},
+        routing_groups=[
+            _group("grp", ["m2", "m3"], "simple-shuffle"),
+            _group("one", ["m2"], "priority", model_priorities={"m2": 1}),
+        ],
+    )
+    registry: Final = core.registry()
+    answers: Final = {
+        name: registry.resolve(name, kind="routing_group_alternatives")  # pyright: ignore[reportCallIssue, reportArgumentType]
+        for name in (None, "grp", "one", "al", "m1", "zzz")
+    }
+    assert answers == {None: False, "grp": True, "one": False, "al": True, "m1": False, "zzz": False}
+
+
+TEAM_MODELS: Final = [
+    deployment("pub", "t1", model_info={"team_id": "T", "team_public_model_name": "pubm"}),
+    deployment("pub2", "t2", model_info={"team_id": "T", "team_public_model_name": "pubm"}),
+    deployment("solo", "t3", model_info={"team_id": "T", "team_public_model_name": "solo"}),
+    deployment("plain", "p1"),
+    deployment("half", "h1", model_info={"team_id": "T"}),
+]
+
+
+@pytest.mark.parametrize(
+    ("deployment_id", "expected"),
+    [("t1", True), ("t3", False), ("p1", False), ("h1", False), ("zzz", False)],
+    ids=["two-siblings", "only-one", "no-team", "no-public-name", "unknown-id"],
+)
+def test_team_model_alternatives_need_another_routable_deployment_under_the_same_team_name(
+    make_core: MakeCore, deployment_id: str, expected: bool
+) -> None:
+    core: Final = make_core(model_list=list(TEAM_MODELS))
+    assert core.registry().resolve(deployment_id, kind="team_model_alternatives") is expected  # pyright: ignore[reportCallIssue, reportArgumentType]
+
+
+def test_a_blocked_team_sibling_is_no_alternative(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            deployment("pub", "t1", model_info={"team_id": "T", "team_public_model_name": "pubm"}),
+            deployment("pub2", "t2", model_info={"team_id": "T", "team_public_model_name": "pubm", "blocked": True}),
+        ]
+    )
+    assert core.registry().resolve("t1", kind="team_model_alternatives") is False  # pyright: ignore[reportCallIssue, reportArgumentType]
+
+
+def test_search_tool_support_comes_from_model_info_else_an_anthropic_request_shape(make_core: MakeCore) -> None:
+    """Per routable deployment: a configured ``supports_search_tool`` wins, a deployment whose chat
+    config builds an Anthropic request answers True, anything else None; an unknown model has none."""
+    core: Final = make_core(
+        model_list=[
+            deployment("g", "o1"),
+            deployment("g", "an1", model="anthropic/claude-3-5-sonnet-20240620"),
+            deployment(
+                "g", "cfg", model="anthropic/claude-3-5-sonnet-20240620", model_info={"supports_search_tool": False}
+            ),
+        ]
+    )
+    registry: Final = core.registry()
+    assert registry.resolve("g", kind="search_tool_support") == (None, True, False)  # pyright: ignore[reportCallIssue, reportArgumentType]
+    assert registry.resolve("zzz", kind="search_tool_support") == ()  # pyright: ignore[reportCallIssue, reportArgumentType]
+
+
+# --- per-request strategy overrides ---------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_an_override_selector_is_bound_to_the_request_logging_object_and_reused(make_core: MakeCore) -> None:
+    """An override strategy the router does not register globally is bound to the request's logging
+    object; one selector per strategy serves every request. An override matching the router's own
+    strategy reuses the global selector and binds nothing."""
+    core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], routing_strategy="least-busy")
+    own: Final = _logging_obj()
+    await core.select("g", {"routing_strategy": "least-busy", "litellm_logging_obj": own})
+    first: Final = _logging_obj()
+    await core.select("g", {"routing_strategy": "usage-based-routing-v2", "litellm_logging_obj": first})
+    second: Final = _logging_obj()
+    await core.select("g", {"routing_strategy": "usage-based-routing-v2", "litellm_logging_obj": second})
+    assert _dynamic_callback_names(own) == []
+    assert _dynamic_callback_names(first) == ["LowestTPMLoggingHandler_v2"]
+    assert (first.dynamic_async_success_callbacks or [None])[0] is (second.dynamic_async_success_callbacks or [])[0]
+
+
+@pytest.mark.asyncio
+async def test_an_override_matching_a_routing_group_strategy_is_not_bound_to_the_request(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("m", "b")],
+        routing_groups=[_group("gg", ["m"], "usage-based-routing-v2")],
+    )
+    logging_obj: Final = _logging_obj()
+    await core.select("g", {"routing_strategy": "usage-based-routing-v2", "litellm_logging_obj": logging_obj})
+    assert _dynamic_callback_names(logging_obj) == []
+
+
+def test_a_sync_usage_based_v2_override_counts_rpm_before_the_call(make_core: MakeCore, host: ScriptedHost) -> None:
+    """The override selector is on no callback list, so the sync pick runs its pre-call check itself:
+    the first call is counted against ``rpm=1`` and the next pick finds no deployment. Oddity: the sync
+    path raises a bare ``ValueError`` where the async one raises ``RateLimitError``."""
+    core: Final = make_core(model_list=[deployment("g", "a", rpm=1)], num_retries=0)
+    core.completion(model="g", messages=MESSAGES, routing_strategy="usage-based-routing-v2")  # pyright: ignore[reportAttributeAccessIssue]
+    with pytest.raises(ValueError, match=re.escape("No deployments available for selected model. Passed model=g.")):
+        core.completion(model="g", messages=MESSAGES, routing_strategy="usage-based-routing-v2")  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.deployments_called() == ["a"]
+
+
+@pytest.mark.asyncio
+async def test_an_async_usage_based_v2_override_counts_rpm_before_the_call(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a", rpm=1)], num_retries=0)
+    await core.acompletion(model="g", messages=MESSAGES, routing_strategy="usage-based-routing-v2")  # pyright: ignore[reportAttributeAccessIssue]
+    with pytest.raises(litellm.RateLimitError, match=re.escape("No deployments available for selected model.")):
+        await core.acompletion(model="g", messages=MESSAGES, routing_strategy="usage-based-routing-v2")  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.deployments_called() == ["a"]
+
+
+@pytest.mark.asyncio
+async def test_request_scoped_routing_still_runs_the_strategy_rpm_check(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    """After ``observe(request_scoped=True)`` the selector is off the host's lists, but the router's
+    own pre-call checks still reach it, so ``rpm=1`` still holds."""
+    collect_other_routers()
+    core: Final = make_core(
+        model_list=[deployment("g", "a", rpm=1)], routing_strategy="usage-based-routing-v2", num_retries=0
+    )
+    core.observe(request_scoped=True)
+    assert host.lists["callbacks"] == []
+    await core.acompletion(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    with pytest.raises(litellm.RateLimitError):
+        await core.acompletion(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.deployments_called() == ["a"]
+
+
+# --- strategy dispatch arms -----------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_async_pick_under_usage_based_routing_v1_picks_the_first_configured(make_core: MakeCore) -> None:
+    """The async pick calls v1's sync selector inline; with no traffic it picks the first deployment."""
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("g", "b")], routing_strategy="usage-based-routing"
+    )
+    assert await picked_ids(core, "g", times=10, messages=MESSAGES) == {"a"}
+
+
+def test_sync_completion_under_latency_based_routing_reaches_the_group(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("g", "b")], routing_strategy="latency-based-routing"
+    )
+    for _ in range(10):
+        core.completion(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert set(host.deployments_called()) <= {"a", "b"}
+    assert len(host.calls) == 10
+
+
+def test_sync_completion_under_cost_based_routing_finds_no_deployment(make_core: MakeCore, host: ScriptedHost) -> None:
+    """Oddity: the cost-based selector has no sync pick, so every sync pick under cost-based routing
+    rate-limits without calling a provider."""
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("g", "b")], routing_strategy="cost-based-routing", num_retries=0
+    )
+    with pytest.raises(RouterRateLimitError) as raised:
+        core.completion(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert str(raised.value) == NO_DEPLOYMENTS_MESSAGE.format(model="g")
+    assert host.calls == []
+
+
+@pytest.mark.asyncio
+async def test_pass_through_pick_under_cost_based_routing_finds_no_deployment(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a", use_in_pass_through=True)], routing_strategy="cost-based-routing"
+    )
+    with pytest.raises(RouterRateLimitError):
+        await core.select("g", kind="pass_through")

tests/unit/test_router/core/test_gaps_l11100_registry.py

@@ -0,0 +1,657 @@
+"""
+Gap tests for the deployment registry of ``router_python_core.py`` (about lines 11550-14060):
+provider validation in ``_add_deployment``, the os.environ references of Azure ``dataSources``,
+vertex pass-through registration, config-sourced rows, index repair after a delete, cost-map
+registration (``responses/`` keys, strategy-marker pricing, replay), deployment budgets kept in
+step with upserts and deletes, credentials with provider, router / deployment model info, and how
+model group info merges its deployments.
+"""
+
+import os
+import uuid
+from collections.abc import Iterator
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.router_api import ChangeSet
+from litellm.types.router import Deployment
+from tests.unit.test_router.core.gaps_l11100_helpers import MakeCore, ModelsEndpoint, complexity_marker, model_ids
+from tests.unit.test_router.core.registry_helpers import team_deployment
+from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment
+
+
+def _unique(prefix: str = "l11100") -> str:
+    return f"{prefix}-{uuid.uuid4().hex}"
+
+
+@pytest.fixture
+def data_source_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+    monkeypatch.setenv("L11100_ENDPOINT", "https://search.example")
+    monkeypatch.delenv("L11100_MISSING", raising=False)
+    yield
+
+
+# --- provider validation ------------------------------------------------------------------------------
+
+
+def test_a_provider_missing_from_the_hosts_provider_list_is_unsupported(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    host.set_dynamic(provider_list=["anthropic"])
+
+    with pytest.raises(Exception, match=r"^Unsupported provider - openai$"):
+        make_core(model_list=[deployment("gpt", "a")])
+
+
+def test_ignore_invalid_deployments_drops_a_deployment_of_an_unsupported_provider(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    host.set_dynamic(provider_list=["anthropic"])
+
+    core: Final = make_core(
+        model_list=[deployment("gpt", "a"), deployment("claude", "c", model="anthropic/claude-opus-4-6")],
+        ignore_invalid_deployments=True,
+    )
+
+    assert model_ids(core) == ["c"]
+
+
+def test_a_prompt_management_model_skips_provider_validation(make_core: MakeCore, host: ScriptedHost) -> None:
+    """A ``<custom logger>/...`` model (here langfuse) is resolved from the prompt at call time, so it is
+    accepted even when no provider is supported at all."""
+    host.set_dynamic(provider_list=[])
+
+    core: Final = make_core(model_list=[deployment("prompted", "p", model="langfuse/my-prompt")])
+
+    assert model_ids(core) == ["p"]
+    assert core.registry().names(kind="deployment_names") == ["langfuse/my-prompt"]
+
+
+# --- Azure dataSources and vertex pass-through ---------------------------------------------------------
+
+
+@pytest.mark.usefixtures("data_source_env")
+def test_data_source_endpoint_and_key_env_references_are_resolved(make_core: MakeCore) -> None:
+    """Only ``endpoint`` and ``key`` are read from the environment; a missing variable becomes ``""``."""
+    core: Final = make_core(
+        model_list=[
+            deployment(
+                "gpt",
+                "a",
+                dataSources=[
+                    {
+                        "type": "AzureCognitiveSearch",
+                        "parameters": {
+                            "endpoint": "os.environ/L11100_ENDPOINT",
+                            "key": "os.environ/L11100_MISSING",
+                            "indexName": "os.environ/L11100_ENDPOINT",
+                        },
+                    }
+                ],
+            )
+        ]
+    )
+
+    stored: Final = core.registry().all()[0]["litellm_params"]["dataSources"]
+    assert stored == [
+        {
+            "type": "AzureCognitiveSearch",
+            "parameters": {
+                "endpoint": "https://search.example",
+                "key": "",
+                "indexName": "os.environ/L11100_ENDPOINT",
+            },
+        }
+    ]
+
+
+def test_a_vertex_pass_through_deployment_without_project_and_location_is_rejected(make_core: MakeCore) -> None:
+    with pytest.raises(ValueError, match="vertex_project") as raised:
+        make_core(model_list=[deployment("gem", "g", model="vertex_ai/gemini-pro", use_in_pass_through=True)])
+    assert str(raised.value) == (
+        "vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints."
+    )
+
+
+def test_pass_through_needs_no_registration_for_other_providers(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("gpt", "a", use_in_pass_through=True)])
+
+    assert model_ids(core) == ["a"]
+
+
+# --- config rows and index repair -----------------------------------------------------------------------
+
+
+def test_config_deployments_leave_out_rows_that_came_from_the_db(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            deployment("gpt", "a"),
+            deployment("gpt", "b", model_info={"db_model": True}),
+            deployment("other", "c", model_info={"db_model": False}),
+        ]
+    )
+
+    assert [entry["model_info"]["id"] for entry in core.registry().deployments(source="config")] == ["a", "c"]
+
+
+def test_deleting_a_later_deployment_keeps_earlier_team_indices_and_shifts_later_ones(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            team_deployment("internal-1", "t1", "TA", "pub"),
+            deployment("gpt", "b"),
+            team_deployment("internal-2", "t2", "TA", "other"),
+        ]
+    )
+
+    core.apply(ChangeSet(delete=["b"]))
+
+    assert dict(core.registry().index("team_model")) == {("TA", "pub"): [0], ("TA", "other"): [1]}
+
+
+def test_deleting_a_teams_only_wildcard_deployment_drops_its_team_pattern_router(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[team_deployment("internal", "tw", "TA", "openai/*"), deployment("gpt", "b")],
+    )
+    assert list(core.registry().patterns(team=True)) == ["TA"]
+
+    core.apply(ChangeSet(delete=["tw"]))
+
+    assert dict(core.registry().patterns(team=True)) == {}
+
+
+# --- cost-map registration ---------------------------------------------------------------------------------
+
+
+def test_a_responses_model_registers_its_backend_key_with_and_without_the_responses_segment(
+    make_core: MakeCore,
+) -> None:
+    """The shared backend keys carry no custom pricing: that stays under the deployment id."""
+    name: Final = _unique()
+    deployment_id: Final = _unique("l11100-id")
+    make_core(model_list=[deployment("gpt", deployment_id, model=f"openai/responses/{name}", input_cost_per_token=0.1)])
+
+    assert litellm.model_cost[f"openai/responses/{name}"].get("input_cost_per_token") is None
+    assert litellm.model_cost[f"openai/{name}"].get("input_cost_per_token") is None
+    assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.1
+
+
+def test_custom_pricing_on_a_strategy_marker_is_not_registered_as_a_price(make_core: MakeCore) -> None:
+    """A marker is never the deployment that is billed, so even an explicit zero price stays out of its
+    cost-map entry."""
+    marker_id: Final = _unique()
+    make_core(
+        model_list=[
+            deployment("small", "s"),
+            deployment("big", "b"),
+            complexity_marker("smart", marker_id, input_cost_per_token=0.0, output_cost_per_token=0.0),
+        ]
+    )
+
+    entry: Final = litellm.model_cost[marker_id]
+    assert "input_cost_per_token" not in entry
+    assert "output_cost_per_token" not in entry
+
+
+def test_replaying_cost_registrations_restores_the_entries_of_deployments_still_served(make_core: MakeCore) -> None:
+    kept: Final = _unique()
+    deleted: Final = _unique()
+    core: Final = make_core(
+        model_list=[
+            deployment("gpt", kept, input_cost_per_token=0.25),
+            deployment("gpt", deleted, input_cost_per_token=0.5),
+        ]
+    )
+    core.apply(ChangeSet(delete=[deleted]))
+    litellm.model_cost.pop(kept, None)
+    litellm.model_cost.pop(deleted, None)
+
+    core.replay_model_cost_registrations()  # pyright: ignore[reportAttributeAccessIssue]  # public on the core
+
+    assert litellm.model_cost[kept]["input_cost_per_token"] == 0.25
+    assert deleted not in litellm.model_cost
+
+
+# --- deployment budgets --------------------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_deployment_budgets_from_the_model_list_are_registered_at_construction(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            deployment("gpt", "a", max_budget=1.0, budget_duration="1d"),
+            deployment("gpt", "b", max_budget=2.0, budget_duration="1d"),
+            deployment("gpt", "c"),
+        ]
+    )
+
+    limiter: Final = core.registry().plugins("budget_limiter")
+    assert limiter is not None
+    assert limiter.deployment_budget_config is not None
+    assert {key: config.max_budget for key, config in limiter.deployment_budget_config.items()} == {"a": 1.0, "b": 2.0}
+
+
+@pytest.mark.asyncio
+async def test_deleting_and_upserting_deployments_keep_the_budget_limiter_in_step(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            deployment("gpt", "a", max_budget=1.0, budget_duration="1d"),
+            deployment("gpt", "b", max_budget=2.0, budget_duration="1d"),
+        ]
+    )
+    limiter: Final = core.registry().plugins("budget_limiter")
+    assert limiter is not None
+
+    core.apply(ChangeSet(delete=["a"]))
+    assert list(limiter.deployment_budget_config or {}) == ["b"]
+
+    core.apply(ChangeSet(upsert=[Deployment(**deployment("gpt", "b"))]))  # pyright: ignore[reportArgumentType]
+    assert limiter.deployment_budget_config is None
+
+    core.apply(ChangeSet(upsert=[Deployment(**deployment("gpt", "b", max_budget=3.0, budget_duration="1d"))]))  # pyright: ignore[reportArgumentType]
+    assert {key: config.max_budget for key, config in (limiter.deployment_budget_config or {}).items()} == {"b": 3.0}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("params", [{"max_budget": 1.0}, {"budget_duration": "1d"}], ids=["no-duration", "no-budget"])
+async def test_adding_a_deployment_with_half_a_budget_installs_no_budget_limiter(
+    make_core: MakeCore, params: dict[str, object]
+) -> None:
+    core: Final = make_core(model_list=[deployment("gpt", "a")])
+
+    core.add_deployment(Deployment(**deployment("gpt", "half", **params)))  # pyright: ignore[reportAttributeAccessIssue, reportArgumentType]
+
+    assert core.registry().plugins("budget_limiter") is None
+
+
+# --- credentials with provider -------------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("entry", "expected_provider"),
+    [
+        (deployment("gpt", "a", model="gpt-4o"), "openai"),
+        (deployment("gpt", "a", model="gpt-4o", custom_llm_provider="azure", api_base="https://x"), "azure"),
+        (deployment("gpt", "a", model="anthropic/claude-opus-4-6"), "anthropic"),
+    ],
+    ids=["bare-model-defaults-to-openai", "explicit-provider-wins", "provider-prefix"],
+)
+def test_credentials_with_provider_name_the_provider(
+    make_core: MakeCore, entry: dict[str, object], expected_provider: str
+) -> None:
+    found: Final = make_core(model_list=[entry]).registry().credentials("a", form="with_provider")
+
+    assert found is not None
+    assert found["custom_llm_provider"] == expected_provider
+
+
+def test_credentials_with_provider_drop_an_unknown_credential_name(make_core: MakeCore) -> None:
+    found: Final = (
+        make_core(model_list=[deployment("gpt", "a", litellm_credential_name="l11100-no-such-credential")])
+        .registry()
+        .credentials("a", form="with_provider")
+    )
+
+    assert found == {"api_key": "sk-test", "model": "openai/a", "custom_llm_provider": "openai"}
+
+
+# --- router and deployment model info ------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("model", "model_info", "expected_key"),
+    [
+        ("azure/gpt-4o", {}, "azure/gpt-4o"),
+        ("azure/my-deployment", {"base_model": "azure/gpt-4o"}, "azure/gpt-4o"),
+    ],
+    ids=["mapped-deployment-name", "base-model"],
+)
+def test_router_model_info_of_an_azure_deployment(
+    make_core: MakeCore, model: str, model_info: dict[str, object], expected_key: str
+) -> None:
+    core: Final = make_core(
+        model_list=[
+            deployment("az", "az", model=model, api_base="https://x", api_version="2024", model_info=model_info)
+        ]
+    )
+
+    info: Final = core.registry().router_model_info(None, "az", id="az")
+    builtin: Final = litellm.get_model_info("azure/gpt-4o")
+    assert info["key"] == expected_key
+    assert info["max_input_tokens"] == builtin["max_input_tokens"]
+    assert info["input_cost_per_token"] == builtin["input_cost_per_token"]
+
+
+def test_router_model_info_of_an_unmapped_azure_deployment_is_its_zeroed_stub(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[deployment("az", "az", model="azure/l11100-unmapped", api_base="https://x", api_version="2024")]
+    )
+
+    info: Final = core.registry().router_model_info(None, "az", id="az")
+    assert info["key"] == "azure/l11100-unmapped"
+    assert info["max_input_tokens"] is None
+    assert info["input_cost_per_token"] == 0
+
+
+def test_deployment_model_info_takes_defaults_from_a_custom_base_model(make_core: MakeCore) -> None:
+    """The deployment's own pricing wins; the rest comes from its ``base_model``'s built-in entry."""
+    name: Final = _unique()
+    core: Final = make_core(
+        model_list=[
+            deployment(
+                "gpt", name, model=f"openai/{name}", input_cost_per_token=0.5, model_info={"base_model": "gpt-4o"}
+            )
+        ]
+    )
+
+    info: Final = core.registry().deployment_model_info(name, f"openai/{name}")
+    builtin: Final = litellm.get_model_info("gpt-4o")
+    assert info is not None
+    assert info["input_cost_per_token"] == 0.5
+    assert info["output_cost_per_token"] == builtin["output_cost_per_token"]
+    assert info["max_input_tokens"] == builtin["max_input_tokens"]
+
+
+def test_deployment_model_info_of_an_unpriced_unmapped_model_reports_no_prices(make_core: MakeCore) -> None:
+    """``get_model_info`` reads a missing price as 0; the deployment reports ``None`` instead."""
+    name: Final = _unique()
+    core: Final = make_core(model_list=[deployment("gpt", name, model=f"openai/{name}")])
+
+    info: Final = core.registry().deployment_model_info(name, f"openai/{name}")
+    assert info is not None
+    assert (info["input_cost_per_token"], info["output_cost_per_token"]) == (None, None)
+
+
+# --- model group info ------------------------------------------------------------------------------------------
+
+
+def test_group_info_takes_the_widest_limits_highest_prices_and_any_capability(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            deployment("g", "a", model="openai/gpt-3.5-turbo"),
+            deployment("g", "b", model="openai/gpt-4o"),
+            deployment("g", "c", model="openai/o3-mini"),
+            deployment("g", "d", model="anthropic/claude-opus-4-6"),
+        ]
+    )
+
+    info: Final = core.registry().group("g")
+    opus: Final = litellm.get_model_info("anthropic/claude-opus-4-6")
+    assert info is not None
+    assert info.providers == ["openai", "anthropic"]
+    assert (info.max_input_tokens, info.max_output_tokens) == (opus["max_input_tokens"], opus["max_output_tokens"])
+    assert (info.input_cost_per_token, info.output_cost_per_token) == (
+        opus["input_cost_per_token"],
+        opus["output_cost_per_token"],
+    )
+    assert info.supports_vision is True
+    assert info.supports_function_calling is True
+    assert info.supports_parallel_function_calling is True
+    assert info.supports_web_search is True
+    assert info.supports_reasoning is True
+    assert info.supports_url_context is False
+    assert info.supports_fast_mode is False
+
+
+def test_group_info_supports_url_context_when_any_deployment_does(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a", model="openai/gpt-4o"), deployment("g", "b", model="gemini/gemini-2.5-flash")]
+    )
+
+    info: Final = core.registry().group("g")
+    assert info is not None
+    assert info.supports_url_context is True
+
+
+@pytest.mark.parametrize(
+    ("models", "fast"),
+    [
+        (["anthropic/claude-opus-4-8"], True),
+        (["anthropic/claude-opus-4-8", "anthropic/claude-opus-4-6"], False),
+    ],
+    ids=["every-deployment-fast", "one-not-fast"],
+)
+def test_group_info_supports_fast_mode_only_when_every_deployment_does(
+    make_core: MakeCore, models: list[str], fast: bool
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", f"d{i}", model=m) for i, m in enumerate(models)])
+
+    info: Final = core.registry().group("g")
+    assert info is not None
+    assert info.supports_fast_mode is fast
+
+
+@pytest.mark.parametrize(
+    ("models", "efforts"),
+    [
+        (["anthropic/claude-opus-4-8"], ("none", "minimal", "low", "medium", "high", "xhigh", "max")),
+        (
+            ["anthropic/claude-opus-4-8", "anthropic/claude-opus-4-6"],
+            ("none", "minimal", "low", "medium", "high", "max"),
+        ),
+        (["anthropic/claude-sonnet-4-5-20250929", "anthropic/claude-opus-4-6"], None),
+        (["anthropic/claude-opus-4-6", "anthropic/claude-sonnet-4-5-20250929"], None),
+        (["openai/gpt-4o", "openai/gpt-4o"], ()),
+    ],
+    ids=["one", "intersection", "unknown-first-stays-unknown", "unknown-later-clears", "none-supported"],
+)
+def test_group_info_reasoning_efforts_are_the_intersection_or_unknown(
+    make_core: MakeCore, models: list[str], efforts: tuple[str, ...] | None
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", f"d{i}", model=m) for i, m in enumerate(models)])
+
+    info: Final = core.registry().group("g")
+    assert info is not None
+    assert info.supported_reasoning_efforts == efforts
+
+
+def test_group_info_supported_endpoints_are_the_union_in_deployment_order(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            deployment("g", "a", model="openai/gpt-5"),
+            deployment("g", "b", model="openai/gpt-4o", model_info={"supported_endpoints": ["/v1/embeddings"]}),
+        ]
+    )
+
+    info: Final = core.registry().group("g")
+    assert info is not None
+    assert info.supported_endpoints == ("/v1/chat/completions", "/v1/batch", "/v1/responses", "/v1/embeddings")
+
+
+def test_group_info_of_a_prompt_management_model_names_its_logger_as_provider(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("prompted", "p", model="langfuse/my-prompt")])
+
+    info: Final = core.registry().group("prompted")
+    assert info is not None
+    assert info.providers == ["langfuse"]
+
+
+def test_group_info_of_an_alias_to_neither_a_name_nor_a_dict_is_none(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("gpt", "a")])
+    core.apply(ChangeSet(assign={"model_group_alias": {"odd": 5}}))
+
+    assert core.registry().group("odd") is None
+
+
+def test_group_info_of_an_unpriced_unmapped_model_has_no_prices_or_limits(make_core: MakeCore) -> None:
+    name: Final = _unique()
+    core: Final = make_core(model_list=[deployment("g", name, model=f"openai/{name}")])
+
+    info: Final = core.registry().group("g")
+    assert info is not None
+    assert (info.input_cost_per_token, info.output_cost_per_token, info.max_input_tokens) == (None, None, None)
+
+
+# --- lookups ---------------------------------------------------------------------------------------------------
+
+
+def test_from_id_of_a_deployment_id_is_its_group_name(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("gpt", "a")])
+
+    assert core.registry().resolve("a", kind="from_id") == "gpt"
+
+
+def test_from_id_on_an_empty_core_or_of_an_empty_id_is_none(make_core: MakeCore) -> None:
+    assert make_core(model_list=[]).registry().resolve("x", kind="from_id") is None
+    assert make_core(model_list=[deployment("gpt", "a")]).registry().resolve("", kind="from_id") is None
+
+
+def test_route_candidates_of_a_routing_group_are_its_members_deployments(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("h", "b"), deployment("other", "c")],
+        routing_groups=[{"group_name": "rg", "models": ["g", "h"], "routing_strategy": "simple-shuffle"}],
+    )
+
+    assert core.registry().names("rg", kind="route_candidates") == frozenset({"a", "b"})
+
+
+def test_search_tool_support_reads_model_info_then_the_anthropic_route(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            deployment("g", "a", model="anthropic/claude-opus-4-6"),
+            deployment("g", "b", model="openai/gpt-4o", model_info={"supports_search_tool": False}),
+            deployment("g", "c", model="openai/gpt-4o"),
+        ]
+    )
+
+    assert core.registry().resolve("g", kind="search_tool_support") == (True, False, None)
+
+
+# --- discovered model info (aapply refresh_model_info) ----------------------------------------------------
+
+VLLM_CARDS: Final = {"data": [{"id": "my-model", "max_model_len": 8192}, {"id": "other", "max_model_len": 4096}]}
+
+
+def _vllm(
+    deployment_id: str = "v", **params: object
+) -> dict[str, object]:  # mutable-ok: a model_list entry  # kwargs-ok: litellm_params
+    return deployment(
+        "vllm", deployment_id, model="hosted_vllm/my-model", api_base="http://vllm.local", api_key="sk-vllm", **params
+    )
+
+
+@pytest.mark.asyncio
+async def test_a_refresh_reads_the_upstream_models_endpoint_with_the_deployments_key(make_core: MakeCore) -> None:
+    endpoint: Final = ModelsEndpoint(VLLM_CARDS)
+    core: Final = make_core(model_list=[_vllm(extra_headers={"X-Team": "a"})])
+
+    await core.aapply(ChangeSet(refresh_model_info=True, refresh_client=endpoint))  # pyright: ignore[reportAttributeAccessIssue, reportArgumentType]
+
+    assert endpoint.requests == [("http://vllm.local/v1/models", {"authorization": "Bearer sk-vllm", "x-team": "a"})]
+    assert dict(core.registry().discovered_model_info("v")) == {
+        "max_tokens": 8192,
+        "max_input_tokens": 8192,
+        "max_output_tokens": 8192,
+    }
+
+
+@pytest.mark.asyncio
+async def test_discovered_limits_never_override_configured_ones_and_feed_the_group_limits(make_core: MakeCore) -> None:
+    endpoint: Final = ModelsEndpoint(VLLM_CARDS)
+    core: Final = make_core(model_list=[_vllm(model_info={"max_output_tokens": 1000})])
+
+    await core.aapply(ChangeSet(refresh_model_info=True, refresh_client=endpoint))  # pyright: ignore[reportAttributeAccessIssue, reportArgumentType]
+
+    registry: Final = core.registry()
+    assert dict(registry.discovered_model_info("v")) == {"max_tokens": 8192, "max_input_tokens": 8192}
+    assert registry.group("vllm", field="token_limits") == (8192, 1000)
+    listing: Final = registry.group("vllm", field="listing_info")
+    assert listing is not None
+    assert (listing.max_input_tokens, listing.max_output_tokens) == (8192, 1000)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "entry",
+    [
+        deployment("plain", "p", model="anthropic/claude-opus-4-6", api_base="http://vllm.local"),
+        deployment("wild", "w", model="hosted_vllm/*", api_base="http://vllm.local"),
+        deployment("nobase", "n", model="openai/gpt-4o"),
+    ],
+    ids=["provider-without-discovery", "wildcard-model", "no-api-base"],
+)
+async def test_a_refresh_skips_deployments_it_cannot_discover(make_core: MakeCore, entry: dict[str, object]) -> None:
+    endpoint: Final = ModelsEndpoint(VLLM_CARDS)
+    core: Final = make_core(model_list=[entry])
+
+    await core.aapply(ChangeSet(refresh_model_info=True, refresh_client=endpoint))  # pyright: ignore[reportAttributeAccessIssue, reportArgumentType]
+
+    assert endpoint.requests == []
+
+
+@pytest.mark.asyncio
+async def test_an_unreachable_models_endpoint_discovers_nothing(make_core: MakeCore) -> None:
+    endpoint: Final = ModelsEndpoint(error=ConnectionError("down"))
+    core: Final = make_core(model_list=[_vllm()])
+
+    await core.aapply(ChangeSet(refresh_model_info=True, refresh_client=endpoint))  # pyright: ignore[reportAttributeAccessIssue, reportArgumentType]
+
+    assert len(endpoint.requests) == 1
+    assert dict(core.registry().discovered_model_info("v")) == {}
+
+
+@pytest.mark.asyncio
+async def test_discovered_limits_are_forgotten_when_the_deployment_is_replaced(make_core: MakeCore) -> None:
+    endpoint: Final = ModelsEndpoint(VLLM_CARDS)
+    core: Final = make_core(model_list=[_vllm()])
+    await core.aapply(ChangeSet(refresh_model_info=True, refresh_client=endpoint))  # pyright: ignore[reportAttributeAccessIssue, reportArgumentType]
+
+    core.apply(ChangeSet(upsert=[Deployment(**_vllm(rpm=5))]))  # pyright: ignore[reportArgumentType]
+
+    assert dict(core.registry().discovered_model_info("v")) == {}
+
+
+# --- failed upserts and replays under ignore_invalid_deployments ---------------------------------------------
+
+
+def test_a_failing_upsert_of_a_new_deployment_under_ignore_invalid_deployments_adds_nothing(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[deployment("gpt", "a")], ignore_invalid_deployments=True)
+    host.set_dynamic(provider_list=["anthropic"])
+
+    result: Final = core.apply(ChangeSet(upsert=[Deployment(**deployment("gpt", "new"))]))  # pyright: ignore[reportArgumentType]
+
+    assert list(result.upserted) == [None]
+    assert model_ids(core) == ["a"]
+
+
+def test_a_replay_retries_deployments_whose_provider_did_not_resolve_and_keeps_them_out_while_it_still_fails(
+    make_core: MakeCore,
+) -> None:
+    core: Final = make_core(
+        model_list=[deployment("gpt", "a"), deployment("odd", "u", model="l11100-no-provider/x")],
+        ignore_invalid_deployments=True,
+    )
+    assert model_ids(core) == ["a"]
+
+    core.replay_model_cost_registrations()  # pyright: ignore[reportAttributeAccessIssue]  # public on the core
+
+    assert model_ids(core) == ["a"]
+
+
+# --- group info without any model info ------------------------------------------------------------------------
+
+
+def test_group_info_without_a_cost_map_entry_falls_back_to_the_configured_mode_and_costs(make_core: MakeCore) -> None:
+    """With the entries construction registered withdrawn from the cost map (as a price data reload
+    does before the replay) and a model litellm does not map, the group is described from the
+    deployment's own model_info: its mode, its costs (a string cost read as a float, a missing one as
+    None) and the provider's supported params, with no token limits."""
+    name: Final = _unique()
+    core: Final = make_core(
+        model_list=[
+            deployment(
+                "g", name, model=f"openai/{name}", model_info={"mode": "embedding", "input_cost_per_token": "0.5"}
+            )
+        ]
+    )
+    litellm.model_cost.pop(name, None)
+    litellm.model_cost.pop(f"openai/{name}", None)
+
+    info: Final = core.registry().group("g")
+    assert info is not None
+    assert (info.mode, info.input_cost_per_token, info.output_cost_per_token) == ("embedding", 0.5, None)
+    assert (info.max_input_tokens, info.max_output_tokens) == (None, None)
+    assert info.supported_openai_params

tests/unit/test_router/core/test_gaps_l11100_strategy.py

@@ -0,0 +1,438 @@
+"""
+Gap tests for the strategy-router (pre-routing) deployments of ``router_python_core.py`` (about lines
+11114-11550, plus the strategy hooks of set_model_list / upsert / delete): construction-time
+validation of auto, adaptive and quality router markers, the (model_name, tags) slot each marker
+holds and when it is released, the licensed capability ceiling, and the adaptive router's derived
+inputs and post-call hooks.
+"""
+
+from typing import Final
+
+import pytest
+
+from litellm.router_api import ChangeSet
+from litellm.types.router import Deployment
+from tests.unit.test_router.core.gaps_l11100_helpers import (
+    TIERS,
+    MakeCore,
+    RemovingHost,
+    adaptive_marker,
+    complexity_marker,
+    marker,
+    model_ids,
+    registered_tags,
+)
+from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment
+
+
+@pytest.fixture
+def host() -> ScriptedHost:
+    """The adaptive router's hook sync removes callbacks from every list, so the host must support it."""
+    return RemovingHost()
+
+
+PLAIN: Final = [deployment("small", "s"), deployment("big", "b")]
+
+HEURISTIC_V2: Final = {"classifier_type": "heuristic_v2", "tiers": dict(TIERS)}
+
+HEURISTIC_V2_LIMIT_MESSAGE: Final = (
+    "At most 1 auto-router(s) using the heuristic_v2 classifier can be registered but this would make 2. "
+    "Choose the heuristic v1 classifier for this router or remove an existing heuristic_v2 router."
+)
+
+
+# --- marker validation at construction -------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("params", "message"),
+    [
+        (
+            {},
+            "auto_router_config_path or auto_router_config is required for auto-router deployments. "
+            "Please set it in the litellm_params",
+        ),
+        (
+            {"auto_router_config": "{}"},
+            "auto_router_default_model is required for auto-router deployments. Please set it in the litellm_params",
+        ),
+        (
+            {"auto_router_config": "{}", "auto_router_default_model": "big"},
+            "auto_router_embedding_model is required for auto-router deployments. Please set it in the litellm_params",
+        ),
+    ],
+    ids=["no-config", "no-default-model", "no-embedding-model"],
+)
+def test_an_auto_router_marker_missing_a_required_param_is_rejected(
+    make_core: MakeCore, params: dict[str, object], message: str
+) -> None:
+    with pytest.raises(ValueError, match="is required") as raised:
+        make_core(model_list=[*PLAIN, marker("auto", "m", "auto_router/semantic", **params)])
+    assert str(raised.value) == message
+
+
+def test_an_adaptive_router_marker_without_its_config_is_rejected(make_core: MakeCore) -> None:
+    with pytest.raises(ValueError, match="is required") as raised:
+        make_core(model_list=[*PLAIN, marker("ad", "m", "auto_router/adaptive_router")])
+    assert str(raised.value) == "adaptive_router_config is required for adaptive-router deployments."
+
+
+def test_a_quality_router_marker_without_a_default_model_is_rejected(make_core: MakeCore) -> None:
+    with pytest.raises(ValueError, match="is required") as raised:
+        make_core(model_list=[*PLAIN, marker("q", "m", "auto_router/quality_router")])
+    assert str(raised.value) == (
+        "quality_router_default_model is required for quality-router deployments, or set default_model in "
+        "quality_router_config. Please configure it in the litellm_params"
+    )
+
+
+def test_a_quality_router_takes_its_default_model_from_its_config(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            *PLAIN,
+            marker("q", "m", "auto_router/quality_router", quality_router_config={"default_model": "big"}),
+        ]
+    )
+    registry: Final = core.registry().plugins("quality")
+
+    assert registered_tags(core, "quality") == {"q": [()]}
+    assert registry["q"][0].strategy.config.default_model == "big"
+
+
+def test_quality_router_default_model_param_beats_the_one_in_its_config(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            *PLAIN,
+            marker(
+                "q",
+                "m",
+                "auto_router/quality_router",
+                quality_router_default_model="small",
+                quality_router_config={"default_model": "big"},
+            ),
+        ]
+    )
+
+    assert core.registry().plugins("quality")["q"][0].strategy.config.default_model == "small"
+
+
+# --- the (model_name, tags) slot --------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(("tags", "shown"), [(None, "[]"), (["x"], "['x']")], ids=["untagged", "tagged"])
+def test_a_second_marker_with_the_same_name_and_tags_is_rejected(
+    make_core: MakeCore, tags: list[str] | None, shown: str
+) -> None:
+    extra: Final = {} if tags is None else {"tags": tags}
+    with pytest.raises(ValueError, match="already exists") as raised:
+        make_core(
+            model_list=[*PLAIN, complexity_marker("smart", "m1", **extra), complexity_marker("smart", "m2", **extra)]
+        )
+    assert str(raised.value) == (
+        f"Complexity-router deployment smart with tags {shown} already exists. "
+        "Please use a different model name or set different tags."
+    )
+
+
+def test_ignore_invalid_deployments_drops_the_repeated_marker_and_keeps_the_first(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[*PLAIN, complexity_marker("smart", "m1"), complexity_marker("smart", "m2")],
+        ignore_invalid_deployments=True,
+    )
+
+    assert model_ids(core) == ["s", "b", "m1"]
+    assert registered_tags(core, "complexity") == {"smart": [()]}
+
+
+def test_markers_of_one_name_with_different_tags_each_register(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[*PLAIN, complexity_marker("smart", "m1", tags=["a"]), complexity_marker("smart", "m2", tags=["b"])]
+    )
+
+    assert registered_tags(core, "complexity") == {"smart": [("a",), ("b",)]}
+
+
+def test_adding_a_marker_whose_slot_is_taken_raises_and_adds_nothing(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[*PLAIN, complexity_marker("smart", "m1")])
+
+    with pytest.raises(ValueError, match="already exists"):
+        core.add_deployment(Deployment(**complexity_marker("smart", "m2")))  # pyright: ignore[reportAttributeAccessIssue, reportArgumentType]
+    assert model_ids(core) == ["s", "b", "m1"]
+    assert registered_tags(core, "complexity") == {"smart": [()]}
+
+
+def test_deleting_a_marker_frees_its_slot_for_a_new_marker(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[*PLAIN, complexity_marker("smart", "m1")])
+
+    core.apply(ChangeSet(delete=["m1"]))
+    assert registered_tags(core, "complexity") == {}
+    result: Final = core.apply(ChangeSet(upsert=[Deployment(**complexity_marker("smart", "m2"))]))  # pyright: ignore[reportArgumentType]
+
+    assert result.upserted[0] is not None
+    assert registered_tags(core, "complexity") == {"smart": [()]}
+    assert model_ids(core) == ["s", "b", "m2"]
+
+
+def test_deleting_one_tagged_marker_keeps_the_other_tags_of_its_name(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[*PLAIN, complexity_marker("smart", "m1", tags=["a"]), complexity_marker("smart", "m2", tags=["b"])]
+    )
+
+    core.apply(ChangeSet(delete=["m1"]))
+
+    assert registered_tags(core, "complexity") == {"smart": [("b",)]}
+
+
+def test_deleting_a_plain_deployment_sharing_a_markers_name_keeps_the_marker(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[*PLAIN, complexity_marker("smart", "m1"), deployment("smart", "p")])
+
+    core.apply(ChangeSet(delete=["p"]))
+
+    assert registered_tags(core, "complexity") == {"smart": [()]}
+
+
+def test_an_upsert_that_retags_a_marker_moves_it_to_the_new_slot(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[*PLAIN, complexity_marker("smart", "m1")])
+
+    core.apply(ChangeSet(upsert=[Deployment(**complexity_marker("smart", "m1", tags=["t"]))]))  # pyright: ignore[reportArgumentType]
+
+    assert registered_tags(core, "complexity") == {"smart": [("t",)]}
+    assert model_ids(core) == ["s", "b", "m1"]
+
+
+def test_set_model_list_drops_the_strategy_routers_of_the_old_list(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            *PLAIN,
+            complexity_marker("smart", "m1"),
+            marker("q", "m2", "auto_router/quality_router", quality_router_default_model="big"),
+        ]
+    )
+
+    core.set_model_list(list(PLAIN))  # pyright: ignore[reportAttributeAccessIssue]
+
+    assert registered_tags(core, "complexity") == {}
+    assert registered_tags(core, "quality") == {}
+
+
+# --- the licensed capability ceiling -----------------------------------------------------------------
+
+
+def test_a_router_beyond_the_capability_limit_is_rejected(make_core: MakeCore) -> None:
+    with pytest.raises(ValueError, match="At most 1") as raised:
+        make_core(
+            model_list=[
+                *PLAIN,
+                complexity_marker("one", "m1", config=HEURISTIC_V2),
+                complexity_marker("two", "m2", config=HEURISTIC_V2),
+            ],
+            auto_router_capability_limit=lambda: 1,
+        )
+    assert str(raised.value) == HEURISTIC_V2_LIMIT_MESSAGE
+
+
+def test_routers_within_the_capability_limit_or_without_the_capability_register(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            *PLAIN,
+            complexity_marker("one", "m1", config=HEURISTIC_V2),
+            complexity_marker("two", "m2"),
+            complexity_marker("three", "m3"),
+        ],
+        auto_router_capability_limit=lambda: 1,
+    )
+
+    assert registered_tags(core, "complexity") == {"one": [()], "two": [()], "three": [()]}
+
+
+def test_without_a_capability_limit_any_number_of_gated_routers_register(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            *PLAIN,
+            complexity_marker("one", "m1", config=HEURISTIC_V2),
+            complexity_marker("two", "m2", config=HEURISTIC_V2),
+        ]
+    )
+
+    assert sorted(registered_tags(core, "complexity")) == ["one", "two"]
+
+
+def test_the_capability_limit_is_read_again_on_every_registration(make_core: MakeCore) -> None:
+    limit: Final = {"value": 2}
+    core: Final = make_core(
+        model_list=[
+            *PLAIN,
+            complexity_marker("one", "m1", config=HEURISTIC_V2),
+            complexity_marker("two", "m2", config=HEURISTIC_V2),
+        ],
+        auto_router_capability_limit=lambda: limit["value"],
+    )
+    limit["value"] = 1
+
+    with pytest.raises(ValueError, match="At most 1") as raised:
+        core.add_deployment(Deployment(**complexity_marker("three", "m3", config=HEURISTIC_V2)))  # pyright: ignore[reportAttributeAccessIssue, reportArgumentType]
+    assert str(raised.value) == HEURISTIC_V2_LIMIT_MESSAGE.replace("would make 2", "would make 3")
+
+
+def test_a_failed_edit_rolls_back_to_the_serving_router_even_above_a_tightened_limit(make_core: MakeCore) -> None:
+    """The edited router is popped first, so the edit itself counts 1 other + itself = 2 > 1 and fails;
+    the rollback re-adds the previous router with the ceiling switched off."""
+    limit: Final = {"value": 2}
+    core: Final = make_core(
+        model_list=[
+            *PLAIN,
+            complexity_marker("one", "m1", config=HEURISTIC_V2),
+            complexity_marker("two", "m2", config=HEURISTIC_V2),
+        ],
+        auto_router_capability_limit=lambda: limit["value"],
+        ignore_invalid_deployments=True,
+    )
+    limit["value"] = 1
+    edited: Final = complexity_marker("two", "m2", config=HEURISTIC_V2, tags=["edited"])
+
+    result: Final = core.apply(ChangeSet(upsert=[Deployment(**edited)]))  # pyright: ignore[reportArgumentType]
+
+    assert list(result.upserted) == [None]
+    assert model_ids(core) == ["s", "b", "m1", "m2"]
+    assert registered_tags(core, "complexity") == {"one": [()], "two": [()]}
+    restored: Final = core.registry().deployment("m2")
+    assert restored is not None
+    assert restored.litellm_params.tags is None
+
+
+def test_a_failed_edit_without_ignore_invalid_deployments_raises_and_leaves_the_router_removed(
+    make_core: MakeCore,
+) -> None:
+    limit: Final = {"value": 2}
+    core: Final = make_core(
+        model_list=[
+            *PLAIN,
+            complexity_marker("one", "m1", config=HEURISTIC_V2),
+            complexity_marker("two", "m2", config=HEURISTIC_V2),
+        ],
+        auto_router_capability_limit=lambda: limit["value"],
+    )
+    limit["value"] = 1
+
+    with pytest.raises(ValueError, match="At most 1") as raised:
+        core.apply(
+            ChangeSet(upsert=[Deployment(**complexity_marker("two", "m2", config=HEURISTIC_V2, tags=["edited"]))])  # pyright: ignore[reportArgumentType]
+        )
+    assert str(raised.value) == HEURISTIC_V2_LIMIT_MESSAGE
+    assert model_ids(core) == ["s", "b", "m1"]
+    assert registered_tags(core, "complexity") == {"one": [()]}
+
+
+# --- the adaptive router -------------------------------------------------------------------------------
+
+
+def _adaptive_list() -> list[dict[str, object]]:  # mutable-ok: a model_list
+    return [
+        adaptive_marker("ar", "ar1", ["x", "y", "z"]),
+        deployment(
+            "x",
+            "x",
+            input_cost_per_token=0.5,
+            model_info={"input_cost_per_token": 0.9, "adaptive_router_preferences": {"quality_tier": 2}},
+        ),
+        deployment("y", "y", model_info={"input_cost_per_token": 0.25}),
+    ]
+
+
+def _hook_counts(host: ScriptedHost) -> tuple[int, int]:
+    from litellm.router_strategy.adaptive_router.hooks import (
+        AdaptiveRouterPostCallHook,
+        AdaptiveRouterResponseHeaderHook,
+    )
+
+    callbacks: Final = host.lists["callbacks"]
+    return (
+        sum(isinstance(callback, AdaptiveRouterPostCallHook) for callback in callbacks),
+        sum(isinstance(callback, AdaptiveRouterResponseHeaderHook) for callback in callbacks),
+    )
+
+
+def test_an_adaptive_router_is_built_after_the_whole_list_from_the_models_it_names(make_core: MakeCore) -> None:
+    """The marker is listed before x and y, yet sees them: litellm_params pricing wins over model_info
+    pricing, preferences come from model_info, and z (not configured) gets neither."""
+    core: Final = make_core(model_list=_adaptive_list())
+    registry: Final = core.registry().plugins("adaptive")
+
+    assert registered_tags(core, "adaptive") == {"ar": [()]}
+    strategy: Final = registry["ar"][0].strategy
+    assert strategy.model_to_cost == {"x": 0.5, "y": 0.25}
+    assert {name: prefs.quality_tier for name, prefs in strategy.model_to_prefs.items()} == {"x": 2}
+
+
+def test_an_adaptive_router_registers_one_post_call_hook_and_a_header_hook(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=_adaptive_list())
+    if not registered_tags(core, "adaptive"):
+        pytest.fail("no adaptive router was registered")
+
+    post_call, header = _hook_counts(host)
+    assert post_call == 1
+    assert header >= 1
+
+
+def test_deleting_an_adaptive_router_releases_it_and_its_post_call_hook(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=_adaptive_list())
+
+    core.apply(ChangeSet(delete=["ar1"]))
+
+    assert registered_tags(core, "adaptive") == {}
+    assert _hook_counts(host)[0] == 0
+
+
+def test_a_complexity_router_with_adaptive_enabled_also_registers_an_adaptive_router(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[*PLAIN, complexity_marker("smart", "m1", config={"tiers": TIERS, "adaptive": True})]
+    )
+
+    assert registered_tags(core, "complexity") == {"smart": [()]}
+    assert registered_tags(core, "adaptive") == {"smart": [()]}
+
+
+def test_deleting_an_adaptive_complexity_router_releases_both_registrations(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[*PLAIN, complexity_marker("smart", "m1", config={"tiers": TIERS, "adaptive": True})]
+    )
+
+    core.apply(ChangeSet(delete=["m1"]))
+
+    assert registered_tags(core, "complexity") == {}
+    assert registered_tags(core, "adaptive") == {}
+
+
+def test_upserting_an_adaptive_complexity_router_registers_its_adaptive_companion(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=list(PLAIN))
+
+    core.apply(
+        ChangeSet(upsert=[Deployment(**complexity_marker("smart", "m1", config={"tiers": TIERS, "adaptive": True}))])  # pyright: ignore[reportArgumentType]
+    )
+
+    assert registered_tags(core, "adaptive") == {"smart": [()]}
+
+
+def test_an_upsert_that_turns_adaptive_off_releases_the_adaptive_companion(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[*PLAIN, complexity_marker("smart", "m1", config={"tiers": TIERS, "adaptive": True})]
+    )
+
+    core.apply(ChangeSet(upsert=[Deployment(**complexity_marker("smart", "m1"))]))  # pyright: ignore[reportArgumentType]
+
+    assert registered_tags(core, "complexity") == {"smart": [()]}
+    assert registered_tags(core, "adaptive") == {}
+
+
+def test_editing_an_adaptive_router_rebuilds_it(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=_adaptive_list())
+
+    core.apply(ChangeSet(upsert=[Deployment(**adaptive_marker("ar", "ar1", ["y"]))]))  # pyright: ignore[reportArgumentType]
+
+    registry: Final = core.registry().plugins("adaptive")
+    assert registered_tags(core, "adaptive") == {"ar": [()]}
+    assert registry["ar"][0].strategy.model_to_cost == {"y": 0.25}

tests/unit/test_router/core/test_gaps_l14000.py

@@ -0,0 +1,299 @@
+"""
+Gaps in the characterization of ``router_python_core.py`` lines 14000-17669 that the views answer:
+``settings().allowed_fails_for`` under an allowed_fails_policy, ``settings().as_dict()`` for the
+latency strategy and after a ``routing_strategy_args`` update, the ``registry()`` deployment
+sources ``routable`` / ``alias`` / ``routing_groups`` / ``routed_group`` / ``for_request`` and team
+aliases of ``all``, cached clients of a pod-discovery deployment, and PTU-share ceilings.
+"""
+
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling
+from litellm.proxy._types import UserAPIKeyAuth
+from litellm.router_api import ChangeSet
+from litellm.types.router import AllowedFailsPolicy
+from tests.unit.test_router.core.scripted_host import (
+    auth_error,
+    bad_request,
+    content_policy,
+    context_window,
+    deployment,
+    rate_limit,
+    server_error,
+    timeout,
+)
+from tests.unit.test_router.core.selection_helpers import MakeCore, picked_ids
+
+# --- settings().allowed_fails_for ----------------------------------------------------------------
+
+
+def _bad_gateway() -> litellm.BadGatewayError:
+    return litellm.BadGatewayError(message="bad gateway", llm_provider="openai", model="gpt-4o")
+
+
+def _connection_error() -> litellm.APIConnectionError:
+    return litellm.APIConnectionError(message="no route", llm_provider="openai", model="gpt-4o")
+
+
+FULL_POLICY: Final = AllowedFailsPolicy(
+    ContentPolicyViolationErrorAllowedFails=1,
+    BadRequestErrorAllowedFails=2,
+    BadGatewayErrorAllowedFails=3,
+    RateLimitErrorAllowedFails=4,
+)
+
+
+@pytest.mark.parametrize(
+    ("error", "expected"),
+    [
+        (content_policy(), 1),
+        (bad_request(), 2),
+        # A context-window error is a BadRequestError, so it takes the bad-request allowance.
+        (context_window(), 2),
+        (_bad_gateway(), 3),
+        (rate_limit(), 4),
+        # Types the policy names no allowance for answer None, never the router's allowed_fails.
+        (server_error(), None),
+        (timeout(), None),
+        (auth_error(), None),
+        (_connection_error(), None),
+    ],
+    ids=[
+        "content-policy",
+        "bad-request",
+        "context-window",
+        "bad-gateway",
+        "rate-limit",
+        "server-error",
+        "timeout",
+        "auth",
+        "connection",
+    ],
+)
+def test_allowed_fails_for_reads_the_policy_field_of_the_error_type(
+    make_core: MakeCore, error: Exception, expected: int | None
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], allowed_fails_policy=FULL_POLICY)
+    assert core.settings().allowed_fails_for(error) == expected
+
+
+def test_allowed_fails_for_a_content_policy_error_falls_back_to_the_bad_request_allowance(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a")], allowed_fails_policy=AllowedFailsPolicy(BadRequestErrorAllowedFails=2)
+    )
+    assert core.settings().allowed_fails_for(content_policy()) == 2
+
+
+def test_allowed_fails_for_without_a_policy_is_none(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], allowed_fails=7)
+    assert core.settings().allowed_fails_for(rate_limit()) is None
+
+
+# --- settings().as_dict() and routing_strategy_args ----------------------------------------------
+
+LATENCY_ARGS_DEFAULTS: Final = {"lowest_latency_buffer": 0, "max_latency_list_size": 10, "ttft_percentile": None}
+
+
+def test_latency_routing_reports_the_strategys_full_routing_args(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a")], routing_strategy="latency-based-routing", routing_strategy_args={"ttl": 30}
+    )
+    assert core.settings().as_dict()["routing_strategy_args"] == {"ttl": 30.0, **LATENCY_ARGS_DEFAULTS}
+
+
+def test_updating_routing_strategy_args_reaches_the_latency_strategy(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a")], routing_strategy="latency-based-routing", routing_strategy_args={"ttl": 30}
+    )
+    core.apply(ChangeSet(settings={"routing_strategy_args": {"ttl": 99}}))
+    assert core.settings().as_dict()["routing_strategy_args"] == {"ttl": 99.0, **LATENCY_ARGS_DEFAULTS}
+
+
+def test_routing_strategy_args_under_simple_shuffle_are_reported_as_given(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")])
+    assert core.settings().as_dict()["routing_strategy_args"] == {}
+    core.apply(ChangeSet(settings={"routing_strategy_args": {"ttl": 99}}))
+    assert core.settings().as_dict()["routing_strategy_args"] == {"ttl": 99}
+
+
+def test_switching_to_latency_routing_takes_the_args_sent_with_it(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")])
+    core.apply(ChangeSet(settings={"routing_strategy": "latency-based-routing", "routing_strategy_args": {"ttl": 7}}))
+    settings: Final = core.settings().as_dict()
+    assert settings["routing_strategy"] == "latency-based-routing"
+    assert settings["routing_strategy_args"] == {"ttl": 7.0, **LATENCY_ARGS_DEFAULTS}
+
+
+# --- registry() deployment sources ---------------------------------------------------------------
+
+TEAMS_AND_GROUPS: Final = [
+    deployment("internal-x", "tx", model_info={"team_id": "team-x", "team_public_model_name": "pub"}),
+    deployment("internal-y", "ty", model_info={"team_id": "team-y", "team_public_model_name": "pub"}),
+    deployment("g", "a"),
+    deployment("g", "b", model_info={"access_groups": ["ag"]}),
+]
+
+ROUTING_GROUP: Final = {"group_name": "grp", "models": ["g"], "routing_strategy": "simple-shuffle"}
+
+
+def _ids(deployments: object) -> list[str]:  # mutable-ok: a fresh list
+    return [deployment_id for _, deployment_id in _rows(deployments)]
+
+
+def _rows(deployments: object) -> list[tuple[str, str]]:  # mutable-ok: a fresh list
+    assert isinstance(deployments, (list, tuple))
+    return [(str(row["model_name"]), str(row["model_info"]["id"])) for row in deployments]
+
+
+def test_all_deployments_of_a_team_public_name_are_renamed_to_the_alias(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=list(TEAMS_AND_GROUPS))
+    rows: Final = core.registry().deployments("pub", source="all", team_id="team-x", model_alias="nick")
+    assert _rows(rows) == [("nick", "tx")]
+
+
+def test_an_alias_to_a_routing_group_lists_the_members_under_the_alias_without_access_groups(
+    make_core: MakeCore,
+) -> None:
+    core: Final = make_core(
+        model_list=list(TEAMS_AND_GROUPS), routing_groups=[dict(ROUTING_GROUP)], model_group_alias={"al": "grp"}
+    )
+    rows: Final = core.registry().deployments("al")
+    assert _rows(rows) == [("al", "a"), ("al", "b")]
+    assert isinstance(rows, list)
+    assert all("access_groups" not in row["model_info"] for row in rows)
+
+
+def test_the_alias_source_lists_every_alias_row(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=list(TEAMS_AND_GROUPS),
+        routing_groups=[dict(ROUTING_GROUP)],
+        model_group_alias={"al": "grp", "al2": "g"},
+    )
+    rows: Final = core.registry().deployments(source="alias")
+    assert _rows(rows) == [("al", "a"), ("al", "b"), ("al2", "a"), ("al2", "b")]
+    # A plain alias row keeps the member's access groups; only routing-group rows drop them.
+    assert isinstance(rows, list)
+    assert rows[3]["model_info"].get("access_groups") == ["ag"]
+
+
+def test_the_routing_groups_source_lists_members_under_the_group_without_access_groups(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=list(TEAMS_AND_GROUPS), routing_groups=[dict(ROUTING_GROUP)])
+    rows: Final = core.registry().deployments(source="routing_groups")
+    assert _rows(rows) == [("grp", "a"), ("grp", "b")]
+    assert isinstance(rows, tuple)
+    assert all("access_groups" not in row["model_info"] for row in rows)
+
+
+def test_the_routed_group_of_a_team_public_name_is_the_teams_own_deployments(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=list(TEAMS_AND_GROUPS))
+    assert _rows(core.registry().deployments("pub", source="routed_group", team_id="team-y")) == [("internal-y", "ty")]
+    assert core.registry().deployments("pub", source="routed_group") == []
+
+
+def test_the_routable_source_lists_the_deployments_routing_would_consider(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=list(TEAMS_AND_GROUPS))
+    registry: Final = core.registry()
+    assert _ids(registry.deployments("g", source="routable")) == ["a", "b"]
+    assert registry.deployments("nope", source="routable") == ()
+    assert _ids(registry.deployments("a", source="routable")) == ["a"]
+    team_request: Final = {"metadata": {"user_api_key_team_id": "team-x"}}
+    assert _ids(registry.deployments("pub", source="routable", request=team_request)) == ["tx"]
+
+
+def test_the_routable_source_answers_nothing_for_a_name_several_teams_share_for_an_admin(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=list(TEAMS_AND_GROUPS))
+    admin: Final = {"metadata": {"user_api_key_auth": UserAPIKeyAuth(user_role="proxy_admin")}}  # pyright: ignore[reportCallIssue]
+    assert core.registry().deployments("pub", source="routable", request=admin) == ()
+
+
+def test_the_for_request_source_finds_one_teams_public_name_for_a_proxy_admin(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[TEAMS_AND_GROUPS[0], *TEAMS_AND_GROUPS[2:]])
+    admin: Final = {"metadata": {"user_api_key_auth": UserAPIKeyAuth(user_role="proxy_admin")}}  # pyright: ignore[reportCallIssue]
+    assert _ids(core.registry().deployments("pub", source="for_request", request=admin)) == ["tx"]
+    assert core.registry().deployments("pub", source="for_request", request={}) == []
+
+
+def test_the_for_request_source_refuses_a_name_several_teams_share_for_a_proxy_admin(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=list(TEAMS_AND_GROUPS))
+    admin: Final = {"metadata": {"user_api_key_auth": UserAPIKeyAuth(user_role="proxy_admin")}}  # pyright: ignore[reportCallIssue]
+    with pytest.raises(litellm.BadRequestError) as raised:
+        core.registry().deployments("pub", source="for_request", request=admin)
+    assert str(raised.value) == (
+        "litellm.BadRequestError: Model name 'pub' matches deployments from multiple teams. "
+        "Specify the deployment ID directly to disambiguate."
+    )
+
+
+@pytest.mark.parametrize("client_type", ["async", None])
+def test_a_pod_discovery_deployment_has_no_cached_client(make_core: MakeCore, client_type: str | None) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")])
+    pod_discovered: Final = deployment("g", "k", kubernetes_pod_discovery=True)
+    assert core.registry().client(pod_discovered, client_type=client_type) is None
+
+
+# --- PTU shares ----------------------------------------------------------------------------------
+
+
+def _ptu_shared(model_name: str, deployment_id: str) -> dict[str, object]:  # mutable-ok: a model_list entry
+    return deployment(
+        model_name,
+        deployment_id,
+        model=f"azure/{deployment_id}",
+        api_base="https://example.openai.azure.com",
+        model_info={
+            "ptu_shares": {"TA": 50},
+            "ptu_count": 50,
+            "cost_per_ptu_per_hour": 1.0,
+            "ptu_effective_from": "2026-01-01T00:00:00Z",
+            "base_model": "gpt-4o",
+        },
+    )
+
+
+PTU_MODELS: Final = [_ptu_shared("gpt", "shared"), _ptu_shared("solo", "s2"), deployment("solo", "open")]
+
+
+def test_a_share_holding_team_has_a_ptu_ceiling_on_its_group(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=list(PTU_MODELS))
+    expected: Final = PTUTeamCeiling(
+        model_group="gpt", tpm_limit=125000, output_to_input_ratio=4.0, cached_input_ratio=0.0
+    )
+    assert core.health().team_ptu_ceiling("TA", "gpt") == expected
+    assert core.health().team_ptu_ceiling("TA", "solo") == PTUTeamCeiling(
+        model_group="solo", tpm_limit=125000, output_to_input_ratio=4.0, cached_input_ratio=0.0
+    )
+    assert core.health().team_ptu_ceiling("TB", "gpt") is None
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("request_kwargs", "group", "expected"),
+    [
+        ({"metadata": {"user_api_key_team_id": "TA"}}, "gpt", "shared"),
+        ({"metadata": {"user_api_key_team_id": "TA"}}, "solo", "s2"),
+        ({"metadata": {"user_api_key_team_id": "TB"}}, "solo", "open"),
+        ({}, "solo", "open"),
+    ],
+    ids=["holder-shared", "holder-prefers-its-share", "other-team-open", "no-team-open"],
+)
+async def test_ptu_shares_route_holders_to_their_share_and_others_to_open_deployments(
+    make_core: MakeCore, request_kwargs: dict[str, object], group: str, expected: str
+) -> None:
+    core: Final = make_core(model_list=list(PTU_MODELS))
+    assert await picked_ids(core, group, request_kwargs=request_kwargs) == {expected}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("request_kwargs", [{"metadata": {"user_api_key_team_id": "TB"}}, {}], ids=["other", "none"])
+async def test_a_group_only_share_holders_may_use_refuses_everyone_else(
+    make_core: MakeCore, request_kwargs: dict[str, object]
+) -> None:
+    core: Final = make_core(model_list=list(PTU_MODELS))
+    with pytest.raises(litellm.BadRequestError) as raised:
+        await core.select("gpt", dict(request_kwargs))
+    assert str(raised.value) == (
+        "litellm.BadRequestError: Deployment gpt is reserved for the teams holding a PTU share of it"
+    )

tests/unit/test_router/core/test_gaps_l14000_selection.py

@@ -0,0 +1,491 @@
+"""
+Gaps in the characterization of ``router_python_core.py`` lines 14000-17669 on the selection path:
+health-check routing (the cooldown bypass under an allowed_fails_policy, health-check probes, the
+sync filter the pass-through pick uses), pass-through picks (paused deployments by id, priority
+groups, cooling, other strategies, the async entry point), the failure dispatch to a request's
+logging object, ``model_alias_map``, access groups and litellm model names, tier ``litellm_params``
+(dropped params, effort carriers, restored output ceilings), routing-decision redaction and the
+Claude Code session binding's guards.
+"""
+
+import asyncio
+import time
+from collections.abc import Mapping
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.proxy._types import UserAPIKeyAuth
+from litellm.types.router import AllowedFailsPolicy, RouterRateLimitError
+from tests.unit.test_router.core.scripted_host import ScriptedHost, auth_error, deployment, rate_limit
+from tests.unit.test_router.core.selection_helpers import MESSAGES, MakeCore, ids_of, picked_ids
+from tests.unit.test_router.core.test_selection_characterization_prerouting import (
+    HARD_MESSAGES,
+    SESSION_ID,
+    TIERS,
+    _complexity_router,
+    _session_request,
+    _tiered,
+)
+
+SPREAD_PICKS: Final = 40
+
+
+def _unhealthy(*ids: str) -> dict[str, dict[str, object]]:  # mutable-ok: health states
+    return {i: {"is_healthy": False, "timestamp": time.time(), "reason": "probe failed"} for i in ids}
+
+
+def _metadata(request: Mapping[str, object], bucket: str = "metadata") -> Mapping[str, object]:
+    metadata: Final = request.get(bucket)
+    assert isinstance(metadata, Mapping)
+    return metadata
+
+
+# --- health-check routing ------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_health_check_routing_with_an_allowed_fails_policy_ignores_cooldowns_when_all_cool(
+    make_core: MakeCore,
+) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("g", "b")],
+        enable_health_check_routing=True,
+        allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=0),
+        cooldown_time=60,
+    )
+    assert [core.health().record_failure(i, rate_limit(), 429) for i in ("a", "b")] == [True, True]
+    assert await picked_ids(core, "g", times=SPREAD_PICKS) == {"a", "b"}
+
+
+@pytest.mark.asyncio
+async def test_health_check_routing_without_a_policy_keeps_honoring_cooldowns(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("g", "b")], enable_health_check_routing=True, cooldown_time=60
+    )
+    for deployment_id in ("a", "b"):
+        core.health().record_failure(deployment_id, rate_limit(), 429)
+    with pytest.raises(RouterRateLimitError) as raised:
+        await core.select("g")
+    assert str(raised.value) == (
+        "No deployments available for selected model, Try again in 60 seconds. All deployments for selected "
+        "model are in cooldown. Passed model=g. pre-call-checks=False, cooldown_list=['a', 'b']"
+    )
+
+
+@pytest.mark.asyncio
+async def test_a_health_check_probe_gets_no_deployments_when_all_are_unhealthy(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], enable_health_check_routing=True)
+    core.health().health_checks.set_deployment_health_states(_unhealthy("a", "b"))  # pyright: ignore[reportArgumentType]  # health states
+    assert ids_of(await core.select("g", kind="healthy")) == ["a", "b"]
+    with pytest.raises(RouterRateLimitError) as raised:
+        await core.select("g", kind="healthy", health_check_probe=True)  # pyright: ignore[reportCallIssue]
+    assert str(raised.value) == (
+        "No deployments available for selected model, Try again in 5 seconds. Passed model=g. "
+        "pre-call-checks=False, cooldown_list=[]"
+    )
+
+
+@pytest.mark.asyncio
+async def test_a_health_check_probe_still_drops_only_the_unhealthy_deployments(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], enable_health_check_routing=True)
+    core.health().health_checks.set_deployment_health_states(_unhealthy("a"))  # pyright: ignore[reportArgumentType]  # health states
+    assert ids_of(await core.select("g", kind="healthy", health_check_probe=True)) == ["b"]  # pyright: ignore[reportCallIssue]
+
+
+PASS_THROUGH_PAIR: Final = [
+    deployment("g", "pt", use_in_pass_through=True),
+    deployment("g", "pt2", use_in_pass_through=True),
+]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("arguments", "expected"),
+    [
+        ({}, {"pt2"}),
+        ({"allowed_fails_policy": {"RateLimitErrorAllowedFails": 1}}, {"pt", "pt2"}),
+        ({"background_health_check_model_groups": ["other"]}, {"pt", "pt2"}),
+        (
+            {"background_health_check_model_groups": ["g"], "allowed_fails_policy": {"RateLimitErrorAllowedFails": 1}},
+            {"pt2"},
+        ),
+    ],
+    ids=["filtered", "policy-disables", "group-not-listed", "listed-group-with-policy"],
+)
+async def test_pass_through_selection_applies_the_health_check_filter(
+    make_core: MakeCore, arguments: dict[str, object], expected: set[str]
+) -> None:
+    core: Final = make_core(model_list=list(PASS_THROUGH_PAIR), enable_health_check_routing=True, **arguments)
+    core.health().health_checks.set_deployment_health_states(_unhealthy("pt"))  # pyright: ignore[reportArgumentType]  # health states
+    assert await picked_ids(core, "g", times=SPREAD_PICKS, kind="pass_through") == expected
+
+
+@pytest.mark.asyncio
+async def test_pass_through_selection_uses_every_deployment_when_all_are_unhealthy(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=list(PASS_THROUGH_PAIR), enable_health_check_routing=True)
+    core.health().health_checks.set_deployment_health_states(_unhealthy("pt", "pt2"))  # pyright: ignore[reportArgumentType]  # health states
+    assert await picked_ids(core, "g", times=SPREAD_PICKS, kind="pass_through") == {"pt", "pt2"}
+
+
+# --- pass-through selection ----------------------------------------------------------------------
+
+PAUSED_PASS_THROUGH: Final = [
+    deployment("g", "pt", use_in_pass_through=True, model_info={"blocked": True}),
+    deployment("g", "pt2", use_in_pass_through=True),
+]
+
+PAUSED_MESSAGE: Final = (
+    # Oddity: the message names the deployment's litellm model, not the id that was asked for.
+    "litellm.ServiceUnavailableError: Model 'openai/pt' is currently paused and cannot accept requests."
+)
+
+
+@pytest.mark.asyncio
+async def test_pass_through_selection_of_a_paused_deployment_by_id_is_unavailable(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=list(PAUSED_PASS_THROUGH))
+    with pytest.raises(litellm.ServiceUnavailableError) as raised:
+        await core.select("pt", kind="pass_through")
+    assert str(raised.value) == PAUSED_MESSAGE
+
+
+@pytest.mark.asyncio
+async def test_async_pass_through_selection_of_a_paused_deployment_by_id_is_unavailable(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=list(PAUSED_PASS_THROUGH))
+    with pytest.raises(litellm.ServiceUnavailableError) as raised:
+        await core.async_get_available_deployment_for_pass_through("pt", {})  # pyright: ignore[reportAttributeAccessIssue]
+    assert str(raised.value) == PAUSED_MESSAGE
+
+
+@pytest.mark.asyncio
+async def test_async_pass_through_selection_by_id_returns_a_pass_through_deployment(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=list(PAUSED_PASS_THROUGH))
+    chosen: Final = await core.async_get_available_deployment_for_pass_through("pt2", {})  # pyright: ignore[reportAttributeAccessIssue]
+    assert chosen["model_info"]["id"] == "pt2"
+
+
+@pytest.mark.asyncio
+async def test_async_pass_through_selection_by_id_refuses_a_plain_deployment(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("g", "pt", use_in_pass_through=True), deployment("g", "plain")])
+    with pytest.raises(litellm.BadRequestError) as raised:
+        await core.async_get_available_deployment_for_pass_through("plain", {})  # pyright: ignore[reportAttributeAccessIssue]
+    assert str(raised.value) == (
+        "litellm.BadRequestError: Deployment plain does not support pass-through endpoint (use_in_pass_through=False)"
+    )
+
+
+@pytest.mark.asyncio
+async def test_async_pass_through_selection_picks_only_pass_through_deployments(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("g", "pt", use_in_pass_through=True), deployment("g", "plain")])
+    for _ in range(10):
+        chosen = await core.async_get_available_deployment_for_pass_through("g", {})  # pyright: ignore[reportAttributeAccessIssue]
+        assert chosen["model_info"]["id"] == "pt"
+
+
+@pytest.mark.asyncio
+async def test_pass_through_selection_with_a_selector_strategy_picks_through_it(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=list(PASS_THROUGH_PAIR), routing_strategy="least-busy")
+    assert await picked_ids(core, "g", kind="pass_through") == {"pt"}
+    chosen: Final = await core.async_get_available_deployment_for_pass_through("g", {})  # pyright: ignore[reportAttributeAccessIssue]
+    assert chosen["model_info"]["id"] == "pt"
+
+
+PRIORITY_PASS_THROUGH: Final = [
+    deployment("m1", "a", use_in_pass_through=True),
+    deployment("m2", "b", use_in_pass_through=True),
+]
+PRIORITY_GROUP: Final = {
+    "group_name": "pg",
+    "models": ["m1", "m2"],
+    "routing_strategy": "priority",
+    "model_priorities": {"m1": 1, "m2": 2},
+}
+
+
+@pytest.mark.asyncio
+async def test_pass_through_selection_of_a_priority_group_keeps_the_lowest_priority(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=list(PRIORITY_PASS_THROUGH), routing_groups=[dict(PRIORITY_GROUP)])
+    assert await picked_ids(core, "pg", kind="pass_through") == {"a"}
+    assert await picked_ids(core, "pg", request_kwargs={"_target_order": 2}, kind="pass_through") == {"b"}
+
+
+@pytest.mark.asyncio
+async def test_pass_through_selection_of_a_priority_order_nothing_has_rate_limits(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=list(PRIORITY_PASS_THROUGH), routing_groups=[dict(PRIORITY_GROUP)])
+    with pytest.raises(RouterRateLimitError) as raised:
+        await core.select("pg", {"_target_order": 3}, kind="pass_through")
+    assert str(raised.value) == (
+        "No deployments available for selected model, Try again in 5 seconds. Passed model=pg. "
+        "pre-call-checks=False, cooldown_list=[]"
+    )
+
+
+@pytest.mark.asyncio
+async def test_pass_through_selection_with_every_pass_through_deployment_cooling_rate_limits(
+    make_core: MakeCore,
+) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "pt", use_in_pass_through=True), deployment("g", "plain")], cooldown_time=60
+    )
+    assert core.health().record_failure("pt", auth_error(), 401) is True
+    with pytest.raises(RouterRateLimitError) as raised:
+        await core.select("g", kind="pass_through")
+    assert str(raised.value) == (
+        "No deployments available for selected model, Try again in 60 seconds. All deployments for selected "
+        "model are in cooldown. Passed model=g. pre-call-checks=False, cooldown_list=['pt']"
+    )
+
+
+# --- failure dispatch to the request's logging object --------------------------------------------
+
+
+class _RecordingLoggingObject:
+    """Stands in for a request's ``litellm_logging_obj``: records the failure dispatches."""
+
+    def __init__(self) -> None:
+        self.dispatched: list[dict[str, object]] = []  # mutable-ok: the dispatches, in order
+
+    async def dispatch_failure_handlers(self, **kwargs: object) -> None:  # kwargs-ok: the dispatch arguments
+        self.dispatched.append(kwargs)
+
+
+async def _let_tasks_run() -> None:
+    for _ in range(3):
+        await asyncio.sleep(0)
+
+
+@pytest.mark.asyncio
+async def test_a_refused_pick_is_dispatched_to_the_requests_logging_object(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")])
+    logging_obj: Final = _RecordingLoggingObject()
+    with pytest.raises(litellm.BadRequestError) as raised:
+        await core.select("nope", {"litellm_logging_obj": logging_obj})
+    await _let_tasks_run()
+    (dispatched,) = logging_obj.dispatched
+    assert dispatched["exception"] is raised.value
+    assert dispatched["prefer_async_handlers"] is True
+    assert "Traceback" in str(dispatched["traceback_exception"])
+
+
+@pytest.mark.asyncio
+async def test_a_refused_async_pass_through_pick_is_dispatched_to_the_requests_logging_object(
+    make_core: MakeCore,
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")])
+    logging_obj: Final = _RecordingLoggingObject()
+    with pytest.raises(litellm.BadRequestError) as raised:
+        await core.async_get_available_deployment_for_pass_through("g", {"litellm_logging_obj": logging_obj})  # pyright: ignore[reportAttributeAccessIssue]
+    await _let_tasks_run()
+    assert [entry["exception"] for entry in logging_obj.dispatched] == [raised.value]
+
+
+# --- model_alias_map, access groups and litellm model names --------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_the_process_model_alias_map_renames_the_model_in_a_no_deployments_error(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    host.set_dynamic(model_alias_map={"g": "renamed"})
+    core: Final = make_core(model_list=[deployment("g", "a")], cooldown_time=60)
+    assert (await core.select("g"))["model_info"]["id"] == "a"  # pyright: ignore[reportIndexIssue]
+    core.health().record_failure("a", auth_error(), 401)
+    with pytest.raises(RouterRateLimitError) as raised:
+        await core.select("g")
+    assert "Passed model=renamed." in str(raised.value)
+
+
+ACCESS_GROUPED: Final = [
+    deployment("g", "in_a", model_info={"access_groups": ["grpA"]}),
+    deployment("g", "in_b", model_info={"access_groups": ["grpB"]}),
+    deployment("h", "hh"),
+]
+
+
+@pytest.mark.asyncio
+async def test_an_access_group_only_caller_reaches_any_deployment_by_its_litellm_model(make_core: MakeCore) -> None:
+    # Oddity: the access-group narrowing looks up access groups by the requested name, and a
+    # litellm model string names no group, so a grpA-only caller reaches the grpB deployment.
+    core: Final = make_core(model_list=list(ACCESS_GROUPED), default_fallbacks=["h"])
+    auth: Final = {"metadata": {"user_api_key_auth": UserAPIKeyAuth(models=["grpA"])}}  # pyright: ignore[reportCallIssue]
+    assert (await core.select("openai/in_b", dict(auth)))["model_info"]["id"] == "in_b"  # pyright: ignore[reportIndexIssue]
+    assert (await core.select("openai/in_a", dict(auth)))["model_info"]["id"] == "in_a"  # pyright: ignore[reportIndexIssue]
+    assert (await core.select("unknown", dict(auth)))["model_info"]["id"] == "hh"  # pyright: ignore[reportIndexIssue]
+
+
+# --- tier litellm_params -------------------------------------------------------------------------
+
+
+def _pinned(**tier_params: object) -> dict[str, object]:  # mutable-ok: tiers  # kwargs-ok: tier litellm_params
+    return {**TIERS, "SIMPLE": {"model_name": "small", "litellm_params": tier_params}}
+
+
+@pytest.mark.asyncio
+async def test_a_tier_param_no_deployment_supports_is_dropped(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            deployment("small", "s", model="anthropic/claude-3-5-sonnet-20240620"),
+            deployment("big", "b"),
+            _complexity_router(tiers=_pinned(logit_bias={"1": 2}, temperature=0.2)),
+        ]
+    )
+    request: Final[dict[str, object]] = {}
+    chosen: Final = await core.select("smart", request, messages=MESSAGES)  # pyright: ignore[reportCallIssue]
+    assert chosen["model_info"]["id"] == "s"  # pyright: ignore[reportIndexIssue]
+    assert request["temperature"] == 0.2
+    assert "logit_bias" not in request
+
+
+@pytest.mark.asyncio
+async def test_a_tier_param_the_request_allowlists_is_kept(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            deployment("small", "s", model="anthropic/claude-3-5-sonnet-20240620"),
+            deployment("big", "b"),
+            _complexity_router(tiers=_pinned(logit_bias={"1": 2})),
+        ]
+    )
+    request: Final[dict[str, object]] = {"allowed_openai_params": ["logit_bias"]}
+    await core.select("smart", request, messages=MESSAGES)  # pyright: ignore[reportCallIssue]
+    assert request["logit_bias"] == {"1": 2}
+
+
+@pytest.mark.asyncio
+async def test_a_tier_reasoning_effort_removes_the_callers_other_effort_carriers(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=_tiered(tiers=_pinned(reasoning_effort="low")))
+    request: Final[dict[str, object]] = {
+        "thinking": {"type": "enabled"},
+        "output_config": {"effort": "high", "format": "json"},
+        "reasoning": {"effort": "high"},
+        "max_completion_tokens": 100,
+    }
+    await core.select("smart", request, messages=MESSAGES)  # pyright: ignore[reportCallIssue]
+    assert request["reasoning_effort"] == "low"
+    assert "thinking" not in request
+    assert request["output_config"] == {"format": "json"}
+    assert "reasoning" not in request
+    assert request["max_completion_tokens"] == 100
+
+
+@pytest.mark.asyncio
+async def test_a_tier_without_an_output_ceiling_leaves_the_callers_ceiling(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=_tiered(tiers=_pinned(temperature=0.1)))
+    request: Final[dict[str, object]] = {"max_completion_tokens": 100}
+    await core.select("smart", request, messages=MESSAGES)  # pyright: ignore[reportCallIssue]
+    assert request["temperature"] == 0.1
+    assert request["max_completion_tokens"] == 100
+    assert "max_tokens" not in request
+    # The caller's ceiling is stamped on every routed pick, pinned or not.
+    assert _metadata(request)["_client_output_ceiling"] == {"max_completion_tokens": 100}
+
+
+@pytest.mark.asyncio
+async def test_a_later_pick_restores_the_callers_ceiling_a_tier_pin_replaced(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=_tiered(tiers=_pinned(max_tokens=7)))
+    request: Final[dict[str, object]] = {"max_completion_tokens": 100}
+    await core.select("smart", request, messages=MESSAGES)  # pyright: ignore[reportCallIssue]
+    assert request["max_tokens"] == 7
+    assert "max_completion_tokens" not in request
+    assert _metadata(request)["_client_output_ceiling"] == {"max_completion_tokens": 100}
+
+    chosen: Final = await core.select("smart", request, messages=HARD_MESSAGES)  # pyright: ignore[reportCallIssue]
+    assert chosen["model_info"]["id"] == "b"  # pyright: ignore[reportIndexIssue]
+    assert request["max_completion_tokens"] == 100
+    assert "max_tokens" not in request
+
+
+@pytest.mark.asyncio
+async def test_a_second_pick_leaves_the_first_picks_forwarded_marker_params_untracked(make_core: MakeCore) -> None:
+    # Oddity: forwarded marker params already on the request are not "newly forwarded" on the
+    # next attempt, so the tracking key is dropped while the params themselves stay behind.
+    core: Final = make_core(model_list=_tiered(temperature=0.3))
+    request: Final[dict[str, object]] = {}
+    await core.select("smart", request, messages=MESSAGES)  # pyright: ignore[reportCallIssue]
+    forwarded: Final = request["_alias_marker_forwarded_params"]
+    assert isinstance(forwarded, tuple)
+    assert "temperature" in forwarded
+    assert request["temperature"] == 0.3
+
+    await core.select("smart", request, messages=MESSAGES)  # pyright: ignore[reportCallIssue]
+    assert "_alias_marker_forwarded_params" not in request
+    assert request["temperature"] == 0.3
+
+
+# --- routing decision redaction ------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_the_routing_decision_keeps_its_prompt_signals_by_default(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=_tiered())
+    request: Final[dict[str, object]] = {}
+    await core.select("smart", request, messages=MESSAGES)  # pyright: ignore[reportCallIssue]
+    decision: Final = _metadata(request)["routing_decision"]
+    assert isinstance(decision, Mapping)
+    assert "signals" in decision
+
+
+@pytest.mark.asyncio
+async def test_redacted_message_logging_drops_the_prompt_quoting_routing_decision_fields(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=_tiered())
+    request: Final[dict[str, object]] = {"standard_callback_dynamic_params": {"turn_off_message_logging": True}}
+    await core.select("smart", request, messages=MESSAGES)  # pyright: ignore[reportCallIssue]
+    decision: Final = _metadata(request)["routing_decision"]
+    assert isinstance(decision, Mapping)
+    assert "signals" not in decision
+    assert (decision["router_model_name"], decision["tier"], decision["routed_model"]) == ("smart", "SIMPLE", "small")
+
+
+@pytest.mark.asyncio
+async def test_routing_metadata_goes_to_litellm_metadata_when_the_request_has_it(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=_tiered())
+    request: Final[dict[str, object]] = {"litellm_metadata": {}}
+    await core.select("smart", request, messages=MESSAGES)  # pyright: ignore[reportCallIssue]
+    assert "routing_decision" in _metadata(request, "litellm_metadata")
+    assert "metadata" not in request
+
+
+# --- the Claude Code session binding's guards ----------------------------------------------------
+
+
+async def _subagent_pick(core: object) -> str:
+    chosen: Final = await core.select("big", _session_request(agent_id="agent-1"), messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    return str(chosen["model_info"]["id"])
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(("depth", "expected"), [(1, "b"), (0, "s")], ids=["fallback-attempt", "first-attempt"])
+async def test_only_a_first_attempt_binds_its_session(make_core: MakeCore, depth: int, expected: str) -> None:
+    core: Final = make_core(model_list=_tiered())
+    first: Final = _session_request()
+    first["fallback_depth"] = depth
+    await core.select("smart", first, messages=MESSAGES)  # pyright: ignore[reportCallIssue]
+    assert await _subagent_pick(core) == expected
+
+
+@pytest.mark.asyncio
+async def test_a_request_without_a_caller_hash_never_binds(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=_tiered())
+    first: Final = _session_request()
+    first["metadata"] = {}
+    await core.select("smart", first, messages=MESSAGES)  # pyright: ignore[reportCallIssue]
+    assert await _subagent_pick(core) == "b"
+
+
+@pytest.mark.asyncio
+async def test_the_caller_hash_is_read_from_litellm_metadata_when_present(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=_tiered())
+    first: Final = _session_request()
+    first["metadata"] = {}
+    first["litellm_metadata"] = {"user_api_key_hash": "caller-hash"}
+    await core.select("smart", first, messages=MESSAGES)  # pyright: ignore[reportCallIssue]
+    assert await _subagent_pick(core) == "s"
+
+
+@pytest.mark.asyncio
+async def test_the_session_header_name_is_matched_case_insensitively(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=_tiered())
+    first: Final = _session_request()
+    first["proxy_server_request"] = {"headers": {"X-Claude-Code-Session-Id": SESSION_ID}}
+    await core.select("smart", first, messages=MESSAGES)  # pyright: ignore[reportCallIssue]
+    assert await _subagent_pick(core) == "s"

tests/unit/test_router/core/test_gaps_l2800.py

@@ -0,0 +1,431 @@
+"""
+Gap tests for ``router_python_core.py`` lines 2800-5600: optional pre-call check re-registration,
+silent experiments, per-deployment ``num_retries`` stamped on errors, and the request kwargs
+``_update_kwargs_with_deployment`` builds (Kubernetes pod routing, baggage stamps, stream usage
+injection, malformed request tools, per-user GitHub Copilot credentials).
+
+Each test asserts what the Python core does today; oddities are named in the test's docstring.
+"""
+
+import asyncio
+import threading
+import time
+from collections.abc import Callable, Mapping
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.constants import BAGGAGE_MAPPED_TAGS_METADATA_KEY, KUBERNETES_POD_ROUTING_KEY
+from litellm.router_api import ChangeSet
+from litellm.router_core import RouterCore
+from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
+from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
+from litellm.types.router import BaggageMappedTagsStamp
+from tests.unit.test_router.core.construction_precall_helpers import CallbackHost, host_callback_names
+from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, rate_limit
+
+MakeCore = Callable[..., RouterCore]
+MESSAGES: Final = [{"role": "user", "content": "hi"}]
+
+
+@pytest.fixture
+def host() -> ScriptedHost:
+    return CallbackHost()
+
+
+def _metadata(kwargs: Mapping[str, object], name: str = "metadata") -> Mapping[str, object]:
+    bucket: Final = kwargs[name]
+    assert isinstance(bucket, Mapping)
+    return bucket
+
+
+async def _settle() -> None:
+    for _ in range(30):
+        await asyncio.sleep(0)
+
+
+# --- optional pre-call checks added again at runtime (add_optional_pre_call_checks) ------------
+
+
+def test_runtime_affinity_checks_merge_into_the_existing_affinity_callback(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], optional_pre_call_checks=["deployment_affinity"])
+    (existing,) = core.optional_callbacks  # pyright: ignore[reportAttributeAccessIssue]
+    core.apply(ChangeSet(settings={"optional_pre_call_checks": ["deployment_affinity", "session_affinity"]}))
+    (merged,) = core.optional_callbacks  # pyright: ignore[reportAttributeAccessIssue]
+    assert merged is existing
+    assert (
+        merged.enable_user_key_affinity,
+        merged.enable_responses_api_affinity,
+        merged.enable_session_id_affinity,
+    ) == (
+        True,
+        False,
+        True,
+    )
+    assert host_callback_names(host)["callbacks"] == ["DeploymentAffinityCheck"]
+
+
+def test_runtime_affinity_flags_are_only_ever_turned_on(make_core: MakeCore) -> None:
+    """Oddity: a runtime list without ``deployment_affinity`` leaves the flag it set on, since the
+    existing callback's flags are OR-ed with the new list's."""
+    core: Final = make_core(model_list=[deployment("g", "a")], optional_pre_call_checks=["deployment_affinity"])
+    core.apply(ChangeSet(settings={"optional_pre_call_checks": ["responses_api_deployment_check"]}))
+    (merged,) = core.optional_callbacks  # pyright: ignore[reportAttributeAccessIssue]
+    assert (
+        merged.enable_user_key_affinity,
+        merged.enable_responses_api_affinity,
+        merged.enable_session_id_affinity,
+    ) == (
+        True,
+        True,
+        False,
+    )
+
+
+def test_runtime_affinity_merge_takes_the_model_group_affinity_config(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a")],
+        optional_pre_call_checks=["deployment_affinity"],
+        model_group_affinity_config={"g": ["session_affinity"]},
+    )
+    core.apply(ChangeSet(settings={"optional_pre_call_checks": ["session_affinity"]}))
+    (merged,) = [cb for cb in core.optional_callbacks if type(cb).__name__ == "DeploymentAffinityCheck"]  # pyright: ignore[reportAttributeAccessIssue]
+    assert merged.model_group_affinity_config == {"g": ["session_affinity"]}
+    assert merged.enable_session_id_affinity is True
+
+
+def test_runtime_encrypted_content_affinity_reuses_the_existing_callback(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], optional_pre_call_checks=["encrypted_content_affinity"])
+    (existing,) = core.optional_callbacks  # pyright: ignore[reportAttributeAccessIssue]
+    core.apply(ChangeSet(settings={"optional_pre_call_checks": ["encrypted_content_affinity"]}))
+    (kept,) = core.optional_callbacks  # pyright: ignore[reportAttributeAccessIssue]
+    assert kept is existing
+    assert kept.enable_global_affinity is True
+    assert host_callback_names(host)["callbacks"] == ["EncryptedContentAffinityCheck"]
+
+
+def test_runtime_encrypted_content_affinity_turns_global_affinity_on_for_a_group_scoped_callback(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a")], model_group_affinity_config={"g": ["encrypted_content_affinity"]}
+    )
+    (existing,) = [cb for cb in core.optional_callbacks if type(cb).__name__ == "EncryptedContentAffinityCheck"]  # pyright: ignore[reportAttributeAccessIssue]
+    assert existing.enable_global_affinity is False
+    core.add_optional_pre_call_checks(["encrypted_content_affinity"])  # pyright: ignore[reportAttributeAccessIssue]
+    assert existing.enable_global_affinity is True
+    assert host_callback_names(host)["callbacks"] == ["EncryptedContentAffinityCheck", "DeploymentAffinityCheck"]
+
+
+async def test_router_budget_limiting_is_not_added_twice_beside_a_provider_budget_config(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a")],
+        provider_budget_config={"openai": {"budget_limit": 1, "time_period": "1d"}},
+        optional_pre_call_checks=["router_budget_limiting"],
+    )
+    limiters: Final = [cb for cb in core.optional_callbacks if isinstance(cb, RouterBudgetLimiting)]  # pyright: ignore[reportAttributeAccessIssue]
+    assert len(limiters) == 1
+    assert core.registry().plugins("budget_limiter") is limiters[0]
+    core.retire()
+
+
+# --- silent experiments -----------------------------------------------------------------------
+
+
+def _wait_for_calls(host: ScriptedHost, count: int) -> None:
+    deadline: Final = time.monotonic() + 5
+    while len(host.calls) < count and time.monotonic() < deadline:
+        time.sleep(0.01)
+
+
+def test_sync_completion_mirrors_to_the_silent_model_on_a_background_thread(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")])
+    core.completion(model="g", messages=MESSAGES, metadata={"trace": "t"})  # pyright: ignore[reportAttributeAccessIssue]
+    _wait_for_calls(host, 2)
+    # the thread starts before the primary call, so the two calls can land in either order
+    assert sorted((call.op, call.deployment_id) for call in host.calls) == [("acompletion", "s1"), ("completion", "d1")]
+    primary: Final = next(call for call in host.calls if call.deployment_id == "d1")
+    shadow: Final = _metadata(next(call for call in host.calls if call.deployment_id == "s1").kwargs)
+    assert (shadow["is_silent_experiment"], shadow["model_group"], shadow["trace"]) == (True, "shadow", "t")
+    assert "is_silent_experiment" not in _metadata(primary.kwargs)
+
+
+async def test_a_request_already_marked_silent_does_not_start_another_experiment(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")])
+    await core.acompletion(model="g", messages=MESSAGES, metadata={"is_silent_experiment": True})  # pyright: ignore[reportAttributeAccessIssue]
+    await _settle()
+    assert host.deployments_called() == ["d1"]
+
+
+def test_a_sync_request_already_marked_silent_does_not_start_another_experiment(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")])
+    core.completion(model="g", messages=MESSAGES, metadata={"is_silent_experiment": True})  # pyright: ignore[reportAttributeAccessIssue]
+    time.sleep(0.2)
+    assert host.deployments_called() == ["d1"]
+
+
+async def test_silent_experiment_marks_its_own_metadata_when_the_request_metadata_cannot_be_deep_copied(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    lock: Final = threading.Lock()
+    core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")])
+    await core.acompletion(model="g", messages=MESSAGES, metadata={"lock": lock})  # pyright: ignore[reportAttributeAccessIssue]
+    await _settle()
+    primary: Final = next(call for call in host.calls if call.deployment_id == "d1")
+    shadow: Final = next(call for call in host.calls if call.deployment_id == "s1")
+    assert "is_silent_experiment" not in _metadata(primary.kwargs)
+    assert _metadata(primary.kwargs)["model_group"] == "g"
+    assert (_metadata(shadow.kwargs)["is_silent_experiment"], _metadata(shadow.kwargs)["lock"]) == (True, lock)
+
+
+async def test_a_streaming_request_mirrors_a_streaming_silent_experiment(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts
+
+    host.answer("openai/d1", chat_stream("A", model="d1"))
+    host.answer("openai/s1", chat_stream("S", model="s1"))
+    core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")])
+    stream: Final = await core.acompletion(model="g", messages=MESSAGES, stream=True)  # pyright: ignore[reportAttributeAccessIssue]
+    assert await collect_texts(stream) == "A"
+    await _settle()
+    shadow: Final = next(call for call in host.calls if call.deployment_id == "s1")
+    assert shadow.kwargs["stream"] is True
+
+
+# --- per-deployment num_retries stamped on the error ------------------------------------------
+
+
+async def test_a_num_retries_already_on_the_error_wins_over_the_deployment_num_retries(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    error: Final = rate_limit()
+    error.num_retries = 0  # pyright: ignore[reportAttributeAccessIssue]
+    host.answer("openai/d1", error)
+    core: Final = make_core(model_list=[deployment("g", "d1", num_retries=2)])
+    with pytest.raises(litellm.RateLimitError):
+        await core.acompletion(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert len(host.calls) == 1
+
+
+async def test_a_string_deployment_num_retries_is_used_as_an_int(make_core: MakeCore, host: ScriptedHost) -> None:
+    host.answer("openai/d1", rate_limit())
+    core: Final = make_core(model_list=[deployment("g", "d1", num_retries="1")])
+    with pytest.raises(litellm.RateLimitError) as raised:
+        await core.acompletion(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert len(host.calls) == 2
+    assert getattr(raised.value, "num_retries") == 1
+
+
+async def test_a_non_numeric_deployment_num_retries_is_ignored(make_core: MakeCore, host: ScriptedHost) -> None:
+    """The router's own default (2 retries) applies when the deployment value cannot be read as an int."""
+    host.answer("openai/d1", rate_limit())
+    core: Final = make_core(model_list=[deployment("g", "d1", num_retries="abc")])
+    with pytest.raises(litellm.RateLimitError):
+        await core.acompletion(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert len(host.calls) == 3
+
+
+# --- request kwargs built for the deployment --------------------------------------------------
+
+
+async def test_responses_replace_kubernetes_pod_routing_in_the_request_metadata(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    """On the Responses path the routing goes to ``litellm_metadata``; a pod routing key the caller put
+    in ``metadata`` is overwritten with the serving deployment's (none here)."""
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    await core.aresponses(  # pyright: ignore[reportAttributeAccessIssue]
+        model="g", input="hi", metadata={KUBERNETES_POD_ROUTING_KEY: {"pod": "stale"}, "keep": 1}
+    )
+    sent: Final = host.calls[0].kwargs
+    assert _metadata(sent) == {KUBERNETES_POD_ROUTING_KEY: None, "keep": 1}
+    assert _metadata(sent, "litellm_metadata")[KUBERNETES_POD_ROUTING_KEY] is None
+
+
+async def test_a_baggage_stamp_records_the_serving_deployment_tags(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=[deployment("g", "d1", tags=["t1", "t2"])])
+    stamp: Final = BaggageMappedTagsStamp(tags=("b1",))
+    await core.acompletion(  # pyright: ignore[reportAttributeAccessIssue]
+        model="g", messages=MESSAGES, metadata={BAGGAGE_MAPPED_TAGS_METADATA_KEY: stamp, "tags": ["b1"]}
+    )
+    metadata: Final = _metadata(host.calls[0].kwargs)
+    assert metadata[BAGGAGE_MAPPED_TAGS_METADATA_KEY] == BaggageMappedTagsStamp(
+        tags=("b1",), serving_deployment_tags=("t1", "t2")
+    )
+    assert metadata["tags"] == ["t1", "t2", "b1"]
+
+
+@pytest.mark.parametrize(
+    ("request_extra", "injected"),
+    [
+        ({"stream": True}, True),
+        ({"stream": True, "stream_options": {"include_usage": False}}, False),
+        ({}, False),
+    ],
+)
+async def test_a_deployment_stream_options_include_usage_marks_the_streaming_request_as_injected(
+    make_core: MakeCore, host: ScriptedHost, request_extra: dict[str, object], injected: bool
+) -> None:
+    from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts
+
+    host.answer("openai/d1", chat_stream("A"))
+    core: Final = make_core(model_list=[deployment("g", "d1", stream_options={"include_usage": True})])
+    response: Final = await core.acompletion(model="g", messages=MESSAGES, **request_extra)  # pyright: ignore[reportAttributeAccessIssue]
+    if request_extra.get("stream"):
+        await collect_texts(response)
+    assert host.calls[0].kwargs.get(STREAM_USAGE_INJECTED_FIELD, False) is injected
+
+
+async def test_malformed_request_tools_are_dropped_for_the_deployment_tools_when_drop_params_is_set(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    tool: Final = {"type": "function", "function": {"name": "dep_tool", "parameters": {}}}
+    core: Final = make_core(model_list=[deployment("g", "d1", tools=[tool], drop_params=True)])
+    await core.acompletion(model="g", messages=MESSAGES, tools="not-a-list", tool_choice="auto")  # pyright: ignore[reportAttributeAccessIssue]
+    sent: Final = host.calls[0].kwargs
+    assert sent["tools"] == [tool]
+    assert "tool_choice" not in sent
+
+
+async def test_a_per_user_github_copilot_deployment_rejects_another_credential_name(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(
+        model_list=[
+            deployment(
+                "g",
+                "d1",
+                model="github_copilot/gpt-4o",
+                litellm_credential_name="copilot-cred",
+                github_copilot_auth_type="per_user_oauth",
+            )
+        ],
+        num_retries=0,
+    )
+    with pytest.raises(litellm.BadRequestError) as raised:
+        await core.acompletion(model="g", messages=MESSAGES, litellm_credential_name="someone-else")  # pyright: ignore[reportAttributeAccessIssue]
+    assert "litellm_credential_name cannot be overridden on a deployment that uses per-user GitHub OAuth" in str(
+        raised.value
+    )
+    assert host.calls == []
+
+
+# --- timeouts ---------------------------------------------------------------------------------
+
+
+async def test_the_process_request_timeout_outranks_the_router_timeout_once_a_router_timeout_is_set(
+    make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Oddity: this per-attempt timeout is read from ``litellm.request_timeout`` itself (not the host's
+    process settings), and only when the router has a ``timeout`` of its own."""
+    monkeypatch.setattr(litellm, "request_timeout", 33.0)
+    core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42)
+    await core.acompletion(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.calls[0].kwargs["timeout"] == 33.0
+
+
+async def test_the_process_request_timeout_is_the_stream_timeout_without_a_stream_timeout(
+    make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts
+
+    monkeypatch.setattr(litellm, "request_timeout", 33.0)
+    host.answer("openai/d1", chat_stream("A"))
+    core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42)
+    await collect_texts(await core.acompletion(model="g", messages=MESSAGES, stream=True))  # pyright: ignore[reportAttributeAccessIssue]
+    assert (host.calls[0].kwargs["stream_timeout"], host.calls[0].kwargs["timeout"]) == (33.0, 33.0)
+
+
+async def test_default_litellm_params_timeout_applies_when_no_router_or_process_timeout_is_set(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    host.set_dynamic(request_timeout=None)
+    core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params={"timeout": 9})
+    await core.acompletion(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.calls[0].kwargs["timeout"] == 9
+
+
+async def test_default_litellm_params_stream_timeout_is_the_last_stream_timeout_fallback(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts
+
+    host.answer("openai/d1", chat_stream("A"))
+    core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params={"stream_timeout": 4})
+    await collect_texts(await core.acompletion(model="g", messages=MESSAGES, stream=True))  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.calls[0].kwargs["stream_timeout"] == 4
+
+
+# --- sync completion runs the routing pre-call checks for a model group only -------------------
+
+
+def test_sync_completion_runs_routing_pre_call_checks_for_a_group_but_not_a_deployment_id(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    from litellm.integrations.custom_logger import CustomLogger
+
+    checked: Final[list[object]] = []
+
+    class Recorder(CustomLogger):
+        def pre_call_check(self, deployment: dict) -> dict | None:  # pyright: ignore[reportIncompatibleMethodOverride]
+            checked.append(deployment["model_info"]["id"])
+            return None
+
+    host.lists["callbacks"].append(Recorder())
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    core.completion(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    core.completion(model="d1", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert checked == ["d1"]
+    assert host.deployments_called() == ["d1", "d1"]
+
+
+def test_a_sync_routing_pre_call_check_error_stops_the_call(make_core: MakeCore, host: ScriptedHost) -> None:
+    from litellm.integrations.custom_logger import CustomLogger
+
+    class OverLimit(CustomLogger):
+        def pre_call_check(self, deployment: dict) -> dict | None:  # pyright: ignore[reportIncompatibleMethodOverride]
+            raise litellm.RateLimitError(message="over the rpm limit", llm_provider="openai", model="d1")
+
+    host.lists["callbacks"].append(OverLimit())
+    core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0)
+    with pytest.raises(litellm.RateLimitError, match="over the rpm limit"):
+        core.completion(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.calls == []
+
+
+# --- per-model call counters ------------------------------------------------------------------
+
+
+async def test_acompletion_counts_total_success_and_failed_calls_per_deployment_model(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0)
+    await core.acompletion(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    host.answer("openai/d1", rate_limit())
+    with pytest.raises(litellm.RateLimitError):
+        await core.acompletion(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    counts: Final = (
+        core.total_calls["openai/d1"],  # pyright: ignore[reportAttributeAccessIssue]
+        core.success_calls["openai/d1"],  # pyright: ignore[reportAttributeAccessIssue]
+        core.fail_calls["openai/d1"],  # pyright: ignore[reportAttributeAccessIssue]
+    )
+    assert counts == (2, 1, 1)
+
+
+def test_sync_completion_does_not_count_calls(make_core: MakeCore, host: ScriptedHost) -> None:
+    """Oddity: only the async path updates ``total_calls``/``success_calls``/``fail_calls``."""
+    core: Final = make_core(model_list=[deployment("g", "d1")])
+    core.completion(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert core.total_calls["openai/d1"] == 0  # pyright: ignore[reportAttributeAccessIssue]

tests/unit/test_router/core/test_gaps_l2800_streaming.py

@@ -0,0 +1,177 @@
+"""
+Gap tests for the stream wrappers of ``router_python_core.py`` (lines 2800-5600): what a mid-stream
+fallback yields when the fallback answers without a stream, the usage on fallback chunks, and the
+Responses stream's handling of an error that is not a mid-stream fallback error.
+
+Each test asserts what the Python core does today; oddities are named in the test's docstring.
+"""
+
+from collections.abc import AsyncIterator, Callable
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.router_core import RouterCore
+from litellm.types.utils import ModelResponseStream, Usage
+from tests.unit.test_router.core.scripted_host import (
+    ScriptedHost,
+    bad_request,
+    completion_response,
+    deployment,
+    rate_limit,
+)
+from tests.unit.test_router.core.streaming_helpers import (
+    MESSAGES,
+    Event,
+    chat_stream,
+    drain_into,
+    finish_chunk,
+    responses_stream,
+    text_chunk,
+)
+
+MakeCore = Callable[..., RouterCore]
+GROUPS_AB: Final = [deployment("a", "a"), deployment("b", "b")]
+
+
+def build(make_core: MakeCore, **arguments: object) -> RouterCore:  # kwargs-ok: the router's constructor arguments
+    return make_core(**{"model_list": GROUPS_AB, "fallbacks": [{"a": ["b"]}], "num_retries": 0, **arguments})
+
+
+def stream_with_usage(sync: bool = False) -> Callable[..., object]:
+    """A bare chunk iterator (not a ``CustomStreamWrapper``, which would strip the usage) of one text
+    chunk and a finish chunk that carries usage."""
+
+    def answer(**_request: object) -> object:  # kwargs-ok: the host passes the request kwargs
+        chunks: Final = [
+            text_chunk("B"),
+            finish_chunk(usage={"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7}),
+        ]
+
+        async def achunks() -> AsyncIterator[ModelResponseStream]:
+            for chunk in chunks:
+                yield chunk
+
+        return iter(chunks) if sync else achunks()
+
+    return answer
+
+
+# --- a fallback that answers without a stream --------------------------------------------------
+
+
+async def test_a_mid_stream_fallback_that_answers_without_a_stream_yields_one_none_item(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    """Oddity: the fallback's whole response is dropped and the stream yields ``None`` in its place."""
+    host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True))
+    host.answer("openai/b", completion_response("not streamed", model="b"))
+    core: Final = build(make_core)
+    stream: Final = await core.acompletion(model="a", messages=MESSAGES, stream=True)  # pyright: ignore[reportAttributeAccessIssue]
+    seen: list[object] = []
+    await drain_into(stream, seen)
+    assert seen[-1] is None
+    assert host.deployments_called() == ["a", "b"]
+
+
+def test_a_sync_mid_stream_fallback_that_answers_without_a_stream_yields_the_response_fields(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    """Oddity: a ``ModelResponse`` is iterable (pydantic yields ``(field, value)`` pairs), so the sync
+    wrapper streams the fallback response's fields instead of chunks."""
+    host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True, sync=True))
+    host.answer("openai/b", completion_response("not streamed", model="b"))
+    core: Final = build(make_core)
+    stream: Final = core.completion(model="a", messages=MESSAGES, stream=True)  # pyright: ignore[reportAttributeAccessIssue]
+    items: Final = list(stream)  # pyright: ignore[reportArgumentType]
+    fields: Final = [item for item in items if isinstance(item, tuple)]
+    assert ("model", "b") in fields
+    assert {name for name, _ in fields} >= {"id", "choices", "usage"}
+    assert host.deployments_called() == ["a", "b"]
+
+
+# --- usage on fallback chunks ------------------------------------------------------------------
+
+
+async def test_a_mid_stream_fallback_chunk_dict_usage_becomes_an_all_zero_usage(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    """Oddity: a fallback chunk's usage is rebuilt through ``combine_usage_objects``, which reads token
+    counts as attributes, so a usage given as a plain dict comes out as an all-zero ``Usage``."""
+    host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True))
+    host.answer("openai/b", stream_with_usage())
+    core: Final = build(make_core)
+    stream: Final = await core.acompletion(model="a", messages=MESSAGES, stream=True)  # pyright: ignore[reportAttributeAccessIssue]
+    seen: list[object] = []
+    await drain_into(stream, seen)
+    usages: Final = [
+        getattr(item, "usage")
+        for item in seen
+        if isinstance(item, ModelResponseStream) and getattr(item, "usage", None) is not None
+    ]
+    assert usages
+    assert isinstance(usages[-1], Usage)
+    assert (usages[-1].prompt_tokens, usages[-1].completion_tokens, usages[-1].total_tokens) == (0, 0, 0)
+
+
+def test_a_sync_mid_stream_fallback_chunk_dict_usage_becomes_an_all_zero_usage(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    """Oddity: as in the async wrapper, a plain-dict usage comes out as an all-zero ``Usage``."""
+    host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True, sync=True))
+    host.answer("openai/b", stream_with_usage(sync=True))
+    core: Final = build(make_core)
+    stream: Final = core.completion(model="a", messages=MESSAGES, stream=True)  # pyright: ignore[reportAttributeAccessIssue]
+    usages: Final = [
+        getattr(item, "usage")
+        for item in stream  # pyright: ignore[reportGeneralTypeIssues]
+        if isinstance(item, ModelResponseStream) and getattr(item, "usage", None) is not None
+    ]
+    assert usages
+    assert isinstance(usages[-1], Usage)
+    assert (usages[-1].prompt_tokens, usages[-1].completion_tokens, usages[-1].total_tokens) == (0, 0, 0)
+
+
+# --- Responses streams -------------------------------------------------------------------------
+
+
+async def test_a_responses_error_that_is_not_a_mid_stream_error_flushes_held_events_then_raises(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    host.answer(
+        "openai/a",
+        responses_stream(Event("response.created"), Event("response.in_progress"), error=bad_request(message="plain")),
+    )
+    host.answer("openai/b", responses_stream(Event("response.created"), Event("response.completed")))
+    core: Final = build(make_core)
+    stream: Final = await core.aresponses(model="a", input="hi", stream=True)  # pyright: ignore[reportAttributeAccessIssue]
+    seen: list[object] = []
+    with pytest.raises(litellm.BadRequestError, match="plain"):
+        await drain_into(stream, seen)
+    assert [getattr(event, "type") for event in seen] == ["response.created", "response.in_progress"]
+    assert host.deployments_called() == ["a"]
+
+
+async def test_a_responses_stream_fallback_that_answers_without_a_stream_yields_that_response(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    from litellm.exceptions import MidStreamFallbackError
+
+    cut: Final = MidStreamFallbackError(
+        message="stream cut",
+        model="a",
+        llm_provider="anthropic",
+        original_exception=rate_limit(message="stream cut"),
+        generated_content="",
+        is_pre_first_chunk=True,
+    )
+    plain: Final = completion_response("not streamed", model="b")
+    host.answer("openai/a", responses_stream(Event("response.created"), error=cut))
+    host.answer("openai/b", plain)
+    core: Final = build(make_core)
+    stream: Final = await core.aresponses(model="a", input="hi", stream=True)  # pyright: ignore[reportAttributeAccessIssue]
+    seen: list[object] = []
+    await drain_into(stream, seen)
+    assert seen == [plain]
+    assert host.deployments_called() == ["a", "b"]